diff --git a/.changeset/chatty-snakes-hope.md b/.changeset/chatty-snakes-hope.md new file mode 100644 index 0000000000..e0d2083a5d --- /dev/null +++ b/.changeset/chatty-snakes-hope.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +fix: Logging large objects is now much more performant and uses less memory diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 7934bf1a5e..b055724124 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -283,8 +283,8 @@ const EnvironmentSchema = z.object({ PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), - TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"), - TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"), + TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"), + TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"), TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT: z.string().default("10"), diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 803a2ede4e..49e561179b 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -105,8 +105,8 @@ mode: "wide" | `MAXIMUM_DEV_QUEUE_SIZE` | No | — | Max dev queue size. | | `MAXIMUM_DEPLOYED_QUEUE_SIZE` | No | — | Max deployed queue size. | | **OTel limits** | | | | -| `TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT` | No | 256 | OTel span attribute count limit. | -| `TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT` | No | 256 | OTel log attribute count limit. | +| `TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT` | No | 1024 | OTel span attribute count limit. | +| `TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT` | No | 1024 | OTel log attribute count limit. | | `TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT` | No | 131072 | OTel span attribute value length limit. | | `TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT` | No | 131072 | OTel log attribute value length limit. | | `TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT` | No | 10 | OTel span event count limit. | diff --git a/packages/cli-v3/src/entryPoints/dev-run-worker.ts b/packages/cli-v3/src/entryPoints/dev-run-worker.ts index edc1e0ed26..4e220c9a19 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-worker.ts @@ -15,6 +15,7 @@ import { localsAPI, logger, LogLevel, + OTEL_LOG_ATTRIBUTE_COUNT_LIMIT, resourceCatalog, runMetadata, runtime, @@ -191,7 +192,8 @@ async function doBootstrap() { typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true, typeof config.disableConsoleInterceptor === "boolean" ? config.disableConsoleInterceptor - : false + : false, + OTEL_LOG_ATTRIBUTE_COUNT_LIMIT ); const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info"; @@ -200,6 +202,7 @@ async function doBootstrap() { logger: otelLogger, tracer: tracer, level: logLevels.includes(configLogLevel as any) ? (configLogLevel as LogLevel) : "info", + maxAttributeCount: OTEL_LOG_ATTRIBUTE_COUNT_LIMIT, }); logger.setGlobalTaskLogger(otelTaskLogger); diff --git a/packages/cli-v3/src/entryPoints/managed-run-worker.ts b/packages/cli-v3/src/entryPoints/managed-run-worker.ts index 820754c05b..294e2c741a 100644 --- a/packages/cli-v3/src/entryPoints/managed-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/managed-run-worker.ts @@ -14,6 +14,7 @@ import { localsAPI, logger, LogLevel, + OTEL_LOG_ATTRIBUTE_COUNT_LIMIT, resourceCatalog, runMetadata, runtime, @@ -182,7 +183,8 @@ async function doBootstrap() { typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true, typeof config.disableConsoleInterceptor === "boolean" ? config.disableConsoleInterceptor - : false + : false, + OTEL_LOG_ATTRIBUTE_COUNT_LIMIT ); const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info"; @@ -191,6 +193,7 @@ async function doBootstrap() { logger: otelLogger, tracer: tracer, level: logLevels.includes(configLogLevel as any) ? (configLogLevel as LogLevel) : "info", + maxAttributeCount: OTEL_LOG_ATTRIBUTE_COUNT_LIMIT, }); logger.setGlobalTaskLogger(otelTaskLogger); diff --git a/packages/core/src/v3/consoleInterceptor.ts b/packages/core/src/v3/consoleInterceptor.ts index 2d809ffbeb..c24b827e20 100644 --- a/packages/core/src/v3/consoleInterceptor.ts +++ b/packages/core/src/v3/consoleInterceptor.ts @@ -11,7 +11,8 @@ export class ConsoleInterceptor { constructor( private readonly logger: logsAPI.Logger, private readonly sendToStdIO: boolean, - private readonly interceptingDisabled: boolean + private readonly interceptingDisabled: boolean, + private readonly maxAttributeCount?: number ) {} // Intercept the console and send logs to the OpenTelemetry logger @@ -92,7 +93,10 @@ export class ConsoleInterceptor { severityNumber, severityText, body: getLogMessage(parsed.value, severityText), - attributes: { ...this.#getAttributes(severityNumber), ...flattenAttributes(parsed.value) }, + attributes: { + ...this.#getAttributes(severityNumber), + ...flattenAttributes(parsed.value, undefined, this.maxAttributeCount), + }, timestamp, }); diff --git a/packages/core/src/v3/limits.ts b/packages/core/src/v3/limits.ts index 4cead21454..61313c7c58 100644 --- a/packages/core/src/v3/limits.ts +++ b/packages/core/src/v3/limits.ts @@ -13,11 +13,11 @@ function getOtelEnvVarLimit(key: string, defaultValue: number) { export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit( "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", - 256 + 1024 ); export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit( "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT", - 256 + 1024 ); export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit( "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", @@ -51,10 +51,6 @@ export function imposeAttributeLimits(attributes: Attributes): Attributes { continue; } - if (Object.keys(newAttributes).length >= OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) { - break; - } - newAttributes[key] = value; } diff --git a/packages/core/src/v3/logger/taskLogger.ts b/packages/core/src/v3/logger/taskLogger.ts index aed1c1a7fb..59ba2602e7 100644 --- a/packages/core/src/v3/logger/taskLogger.ts +++ b/packages/core/src/v3/logger/taskLogger.ts @@ -16,6 +16,7 @@ export type TaskLoggerConfig = { logger: Logger; tracer: TriggerTracer; level: LogLevel; + maxAttributeCount?: number; }; export type TraceOptions = Prettify< @@ -78,7 +79,12 @@ export class OtelTaskLogger implements TaskLogger { severityNumber: SeverityNumber, properties?: Record ) { - let attributes: Attributes = { ...flattenAttributes(safeJsonProcess(properties)) }; + let attributes: Attributes = {}; + + if (properties) { + // Use flattenAttributes directly - it now handles all non-JSON friendly values efficiently + attributes = flattenAttributes(properties, undefined, this._config.maxAttributeCount); + } const icon = iconStringForSeverity(severityNumber); if (icon !== undefined) { @@ -136,23 +142,3 @@ export class NoopTaskLogger implements TaskLogger { return {} as Span; } } - -function safeJsonProcess(value?: Record): Record | undefined { - try { - return JSON.parse(JSON.stringify(value, jsonErrorReplacer)); - } catch { - return value; - } -} - -function jsonErrorReplacer(key: string, value: unknown) { - if (value instanceof Error) { - return { - name: value.name, - message: value.message, - stack: value.stack, - }; - } - - return value; -} diff --git a/packages/core/src/v3/utils/flattenAttributes.ts b/packages/core/src/v3/utils/flattenAttributes.ts index e2791f21ac..c2deeecca2 100644 --- a/packages/core/src/v3/utils/flattenAttributes.ts +++ b/packages/core/src/v3/utils/flattenAttributes.ts @@ -4,81 +4,230 @@ export const NULL_SENTINEL = "$@null(("; export const CIRCULAR_REFERENCE_SENTINEL = "$@circular(("; export function flattenAttributes( - obj: Record | Array | string | boolean | number | null | undefined, + obj: unknown, prefix?: string, - seen: WeakSet = new WeakSet() + maxAttributeCount?: number ): Attributes { - const result: Attributes = {}; + const flattener = new AttributeFlattener(maxAttributeCount); + flattener.doFlatten(obj, prefix); + return flattener.attributes; +} - // Check if obj is null or undefined - if (obj === undefined) { - return result; - } +class AttributeFlattener { + private seen: WeakSet = new WeakSet(); + private attributeCounter: number = 0; + private result: Attributes = {}; - if (obj === null) { - result[prefix || ""] = NULL_SENTINEL; - return result; - } + constructor(private maxAttributeCount?: number) {} - if (typeof obj === "string") { - result[prefix || ""] = obj; - return result; + get attributes(): Attributes { + return this.result; } - if (typeof obj === "number") { - result[prefix || ""] = obj; - return result; + private canAddMoreAttributes(): boolean { + return this.maxAttributeCount === undefined || this.attributeCounter < this.maxAttributeCount; } - if (typeof obj === "boolean") { - result[prefix || ""] = obj; - return result; + private addAttribute(key: string, value: any): boolean { + if (!this.canAddMoreAttributes()) { + return false; + } + this.result[key] = value; + this.attributeCounter++; + return true; } - if (obj instanceof Date) { - result[prefix || ""] = obj.toISOString(); - return result; - } + doFlatten(obj: unknown, prefix?: string) { + if (!this.canAddMoreAttributes()) { + return; + } - // Check for circular reference - if (obj !== null && typeof obj === "object" && seen.has(obj)) { - result[prefix || ""] = CIRCULAR_REFERENCE_SENTINEL; - return result; - } + // Check if obj is null or undefined + if (obj === undefined) { + return; + } - // Add object to seen set - if (obj !== null && typeof obj === "object") { - seen.add(obj); - } + if (obj === null) { + this.addAttribute(prefix || "", NULL_SENTINEL); + return; + } - for (const [key, value] of Object.entries(obj)) { - const newPrefix = `${prefix ? `${prefix}.` : ""}${Array.isArray(obj) ? `[${key}]` : key}`; - if (Array.isArray(value)) { - for (let i = 0; i < value.length; i++) { - if (typeof value[i] === "object" && value[i] !== null) { - // update null check here as well - Object.assign(result, flattenAttributes(value[i], `${newPrefix}.[${i}]`, seen)); - } else { - if (value[i] === null) { - result[`${newPrefix}.[${i}]`] = NULL_SENTINEL; - } else { - result[`${newPrefix}.[${i}]`] = value[i]; + if (typeof obj === "string") { + this.addAttribute(prefix || "", obj); + return; + } + + if (typeof obj === "number") { + this.addAttribute(prefix || "", obj); + return; + } + + if (typeof obj === "boolean") { + this.addAttribute(prefix || "", obj); + return; + } + + if (obj instanceof Date) { + this.addAttribute(prefix || "", obj.toISOString()); + return; + } + + // Handle Error objects + if (obj instanceof Error) { + this.addAttribute(`${prefix || "error"}.name`, obj.name); + this.addAttribute(`${prefix || "error"}.message`, obj.message); + if (obj.stack) { + this.addAttribute(`${prefix || "error"}.stack`, obj.stack); + } + return; + } + + // Handle functions + if (typeof obj === "function") { + const funcName = obj.name || "anonymous"; + this.addAttribute(prefix || "", `[Function: ${funcName}]`); + return; + } + + // Handle Set objects + if (obj instanceof Set) { + let index = 0; + for (const item of obj) { + if (!this.canAddMoreAttributes()) break; + this.#processValue(item, `${prefix || "set"}.[${index}]`); + index++; + } + return; + } + + // Handle Map objects + if (obj instanceof Map) { + for (const [key, value] of obj) { + if (!this.canAddMoreAttributes()) break; + // Use the key directly if it's a string, otherwise convert it + const keyStr = typeof key === "string" ? key : String(key); + this.#processValue(value, `${prefix || "map"}.${keyStr}`); + } + return; + } + + // Handle File objects + if (typeof File !== "undefined" && obj instanceof File) { + this.addAttribute(`${prefix || "file"}.name`, obj.name); + this.addAttribute(`${prefix || "file"}.size`, obj.size); + this.addAttribute(`${prefix || "file"}.type`, obj.type); + this.addAttribute(`${prefix || "file"}.lastModified`, obj.lastModified); + return; + } + + // Handle ReadableStream objects + if (typeof ReadableStream !== "undefined" && obj instanceof ReadableStream) { + this.addAttribute(`${prefix || "stream"}.type`, "ReadableStream"); + this.addAttribute(`${prefix || "stream"}.locked`, obj.locked); + return; + } + + // Handle WritableStream objects + if (typeof WritableStream !== "undefined" && obj instanceof WritableStream) { + this.addAttribute(`${prefix || "stream"}.type`, "WritableStream"); + this.addAttribute(`${prefix || "stream"}.locked`, obj.locked); + return; + } + + // Handle Promise objects + if (obj instanceof Promise) { + this.addAttribute(prefix || "promise", "[Promise object]"); + // We can't inspect promise state synchronously, so just indicate it's a promise + return; + } + + // Handle RegExp objects + if (obj instanceof RegExp) { + this.addAttribute(`${prefix || "regexp"}.source`, obj.source); + this.addAttribute(`${prefix || "regexp"}.flags`, obj.flags); + return; + } + + // Handle URL objects + if (typeof URL !== "undefined" && obj instanceof URL) { + this.addAttribute(`${prefix || "url"}.href`, obj.href); + this.addAttribute(`${prefix || "url"}.protocol`, obj.protocol); + this.addAttribute(`${prefix || "url"}.host`, obj.host); + this.addAttribute(`${prefix || "url"}.pathname`, obj.pathname); + return; + } + + // Handle ArrayBuffer and TypedArrays + if (obj instanceof ArrayBuffer) { + this.addAttribute(`${prefix || "arraybuffer"}.byteLength`, obj.byteLength); + return; + } + + // Handle TypedArrays (Uint8Array, Int32Array, etc.) + if (ArrayBuffer.isView(obj)) { + const typedArray = obj as any; + this.addAttribute(`${prefix || "typedarray"}.constructor`, typedArray.constructor.name); + this.addAttribute(`${prefix || "typedarray"}.length`, typedArray.length); + this.addAttribute(`${prefix || "typedarray"}.byteLength`, typedArray.byteLength); + this.addAttribute(`${prefix || "typedarray"}.byteOffset`, typedArray.byteOffset); + return; + } + + // Check for circular reference + if (obj !== null && typeof obj === "object" && this.seen.has(obj)) { + this.addAttribute(prefix || "", CIRCULAR_REFERENCE_SENTINEL); + return; + } + + // Add object to seen set + if (obj !== null && typeof obj === "object") { + this.seen.add(obj); + } + + for (const [key, value] of Object.entries(obj)) { + if (!this.canAddMoreAttributes()) { + break; + } + + const newPrefix = `${prefix ? `${prefix}.` : ""}${Array.isArray(obj) ? `[${key}]` : key}`; + + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + if (!this.canAddMoreAttributes()) { + break; } + this.#processValue(value[i], `${newPrefix}.[${i}]`); } - } - } else if (isRecord(value)) { - // update null check here - Object.assign(result, flattenAttributes(value, newPrefix, seen)); - } else { - if (typeof value === "number" || typeof value === "string" || typeof value === "boolean") { - result[newPrefix] = value; - } else if (value === null) { - result[newPrefix] = NULL_SENTINEL; + } else { + this.#processValue(value, newPrefix); } } } - return result; + #processValue(value: unknown, prefix: string) { + if (!this.canAddMoreAttributes()) { + return; + } + + // Handle primitive values directly + if (value === null) { + this.addAttribute(prefix, NULL_SENTINEL); + return; + } + + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + this.addAttribute(prefix, value); + return; + } + + // Handle non-primitive values by recursing + if (typeof value === "object" || typeof value === "function") { + this.doFlatten(value as any, prefix); + } else { + // Convert other types to strings (bigint, symbol, etc.) + this.addAttribute(prefix, String(value)); + } + } } function isRecord(value: unknown): value is Record { diff --git a/packages/core/src/v3/utils/ioSerialization.ts b/packages/core/src/v3/utils/ioSerialization.ts index c3b4adc7f0..10f4edb2c4 100644 --- a/packages/core/src/v3/utils/ioSerialization.ts +++ b/packages/core/src/v3/utils/ioSerialization.ts @@ -1,13 +1,17 @@ import { Attributes, Span } from "@opentelemetry/api"; -import { OFFLOAD_IO_PACKET_LENGTH_LIMIT, imposeAttributeLimits } from "../limits.js"; +import { z } from "zod"; +import { ApiClient } from "../apiClient/index.js"; +import { apiClientManager } from "../apiClientManager-api.js"; +import { + OFFLOAD_IO_PACKET_LENGTH_LIMIT, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + imposeAttributeLimits, +} from "../limits.js"; +import type { RetryOptions } from "../schemas/index.js"; import { SemanticInternalAttributes } from "../semanticInternalAttributes.js"; import { TriggerTracer } from "../tracer.js"; -import { flattenAttributes } from "./flattenAttributes.js"; -import { apiClientManager } from "../apiClientManager-api.js"; import { zodfetch } from "../zodfetch.js"; -import { z } from "zod"; -import type { RetryOptions } from "../schemas/index.js"; -import { ApiClient } from "../apiClient/index.js"; +import { flattenAttributes } from "./flattenAttributes.js"; export type IOPacket = { data?: string | undefined; @@ -347,19 +351,27 @@ export async function createPacketAttributesAsJson( } switch (dataType) { - case "application/json": - return imposeAttributeLimits(flattenAttributes(data, undefined)); - case "application/super+json": + case "application/json": { + return imposeAttributeLimits( + flattenAttributes(data, undefined, OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) + ); + } + case "application/super+json": { const { deserialize } = await loadSuperJSON(); const deserialized = deserialize(data) as any; const jsonify = safeJsonParse(JSON.stringify(deserialized, makeSafeReplacer())); - return imposeAttributeLimits(flattenAttributes(jsonify, undefined)); - case "application/store": + return imposeAttributeLimits( + flattenAttributes(jsonify, undefined, OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) + ); + } + case "application/store": { return data; - default: + } + default: { return {}; + } } } diff --git a/packages/core/src/v3/workers/taskExecutor.ts b/packages/core/src/v3/workers/taskExecutor.ts index 54b68bbb55..a7e7ba9c77 100644 --- a/packages/core/src/v3/workers/taskExecutor.ts +++ b/packages/core/src/v3/workers/taskExecutor.ts @@ -15,6 +15,7 @@ import { attemptKey, flattenAttributes, lifecycleHooks, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, runMetadata, waitUntil, } from "../index.js"; @@ -716,7 +717,9 @@ export class TaskExecutor { const result = await hook.fn({ payload, ctx, signal, task: this.task.id }); if (result && typeof result === "object" && !Array.isArray(result)) { - span.setAttributes(flattenAttributes(result)); + span.setAttributes( + flattenAttributes(result, undefined, OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) + ); return result; } @@ -752,7 +755,9 @@ export class TaskExecutor { const result = await taskInitHook({ payload, ctx, signal, task: this.task.id }); if (result && typeof result === "object" && !Array.isArray(result)) { - span.setAttributes(flattenAttributes(result)); + span.setAttributes( + flattenAttributes(result, undefined, OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) + ); return result; } diff --git a/packages/core/test/flattenAttributes.test.ts b/packages/core/test/flattenAttributes.test.ts index 4b00995163..cdeb6b933b 100644 --- a/packages/core/test/flattenAttributes.test.ts +++ b/packages/core/test/flattenAttributes.test.ts @@ -1,7 +1,7 @@ import { flattenAttributes, unflattenAttributes } from "../src/v3/utils/flattenAttributes.js"; describe("flattenAttributes", () => { - it("handles number keys correctl", () => { + it("handles number keys correctly", () => { expect(flattenAttributes({ bar: { "25": "foo" } })).toEqual({ "bar.25": "foo" }); expect(unflattenAttributes({ "bar.25": "foo" })).toEqual({ bar: { "25": "foo" } }); expect(flattenAttributes({ bar: ["foo", "baz"] })).toEqual({ @@ -157,13 +157,14 @@ describe("flattenAttributes", () => { expect(flattenAttributes(obj, "retry.byStatus")).toEqual(expected); }); - it("handles circular references correctly", () => { + it("handles circular references correctly", () => { const user = { name: "Alice" }; + // @ts-expect-error user["blogPosts"] = [{ title: "Post 1", author: user }]; // Circular reference const result = flattenAttributes(user); expect(result).toEqual({ - "name": "Alice", + name: "Alice", "blogPosts.[0].title": "Post 1", "blogPosts.[0].author": "$@circular((", }); @@ -171,14 +172,329 @@ describe("flattenAttributes", () => { it("handles nested circular references correctly", () => { const user = { name: "Bob" }; + // @ts-expect-error user["friends"] = [user]; // Circular reference const result = flattenAttributes(user); expect(result).toEqual({ - "name": "Bob", + name: "Bob", "friends.[0]": "$@circular((", }); }); + + it("respects maxAttributeCount limit", () => { + const obj = { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + }; + + const result = flattenAttributes(obj, undefined, 3); + expect(Object.keys(result)).toHaveLength(3); + expect(result).toEqual({ + a: 1, + b: 2, + c: 3, + }); + }); + + it("respects maxAttributeCount limit with nested objects", () => { + const obj = { + level1: { + a: 1, + b: 2, + c: 3, + }, + level2: { + d: 4, + e: 5, + }, + }; + + const result = flattenAttributes(obj, undefined, 2); + expect(Object.keys(result)).toHaveLength(2); + expect(result).toEqual({ + "level1.a": 1, + "level1.b": 2, + }); + }); + + it("respects maxAttributeCount limit with arrays", () => { + const obj = { + array: [1, 2, 3, 4, 5], + }; + + const result = flattenAttributes(obj, undefined, 3); + expect(Object.keys(result)).toHaveLength(3); + expect(result).toEqual({ + "array.[0]": 1, + "array.[1]": 2, + "array.[2]": 3, + }); + }); + + it("works normally when maxAttributeCount is undefined", () => { + const obj = { + a: 1, + b: 2, + c: 3, + }; + + const result = flattenAttributes(obj); + expect(Object.keys(result)).toHaveLength(3); + expect(result).toEqual({ + a: 1, + b: 2, + c: 3, + }); + }); + + it("handles maxAttributeCount of 0", () => { + const obj = { + a: 1, + b: 2, + }; + + const result = flattenAttributes(obj, undefined, 0); + expect(Object.keys(result)).toHaveLength(0); + expect(result).toEqual({}); + }); + + it("handles maxAttributeCount with primitive values", () => { + const result1 = flattenAttributes("test", undefined, 1); + expect(result1).toEqual({ "": "test" }); + + const result2 = flattenAttributes("test", undefined, 0); + expect(result2).toEqual({}); + }); + + it("handles Error objects correctly", () => { + const error = new Error("Test error message"); + error.stack = "Error: Test error message\n at test.js:1:1"; + + const result = flattenAttributes({ error }); + expect(result).toEqual({ + "error.name": "Error", + "error.message": "Test error message", + "error.stack": "Error: Test error message\n at test.js:1:1", + }); + }); + + it("handles Error objects as top-level values", () => { + const error = new Error("Top level error"); + const result = flattenAttributes(error); + expect(result["error.name"]).toBe("Error"); + expect(result["error.message"]).toBe("Top level error"); + // Stack trace is also included when present + expect(result["error.stack"]).toBeDefined(); + }); + + it("handles function values correctly", () => { + function namedFunction() {} + const anonymousFunction = function () {}; + const arrowFunction = () => {}; + + const result = flattenAttributes({ + named: namedFunction, + anonymous: anonymousFunction, + arrow: arrowFunction, + }); + + expect(result.named).toBe("[Function: namedFunction]"); + // Note: function expressions with variable names retain their names + expect(result.anonymous).toBe("[Function: anonymousFunction]"); + // Arrow functions also get their variable names in modern JS + expect(result.arrow).toBe("[Function: arrowFunction]"); + }); + + it("handles mixed problematic types", () => { + const complexObj = { + error: new Error("Mixed error"), + func: function testFunc() {}, + date: new Date("2023-01-01"), + normal: "string", + number: 42, + }; + + const result = flattenAttributes(complexObj); + + expect(result["error.name"]).toBe("Error"); + expect(result["error.message"]).toBe("Mixed error"); + expect(result["func"]).toBe("[Function: testFunc]"); + expect(result["date"]).toBe("2023-01-01T00:00:00.000Z"); + expect(result["normal"]).toBe("string"); + expect(result["number"]).toBe(42); + }); + + it("handles bigint and symbol types", () => { + const obj = { + bigNumber: BigInt(123456789), + sym: Symbol("test"), + }; + + const result = flattenAttributes(obj); + expect(result["bigNumber"]).toBe("123456789"); + expect(result["sym"]).toBe("Symbol(test)"); + }); + + it("handles Set objects correctly", () => { + const mySet = new Set([1, "hello", true, { nested: "object" }]); + const result = flattenAttributes({ mySet }); + + expect(result["mySet.[0]"]).toBe(1); + expect(result["mySet.[1]"]).toBe("hello"); + expect(result["mySet.[2]"]).toBe(true); + expect(result["mySet.[3].nested"]).toBe("object"); + }); + + it("handles nested Set objects correctly", () => { + const mySet = new Set([1, 2, 3, { nested: "object" }]); + const result = flattenAttributes({ mySet }); + expect(result["mySet.[0]"]).toBe(1); + expect(result["mySet.[1]"]).toBe(2); + expect(result["mySet.[2]"]).toBe(3); + expect(result["mySet.[3].nested"]).toBe("object"); + }); + + it("handles Map objects correctly", () => { + const myMap = new Map(); + myMap.set("key1", "value1"); + myMap.set("key2", 42); + myMap.set(123, "numeric key"); + + const result = flattenAttributes({ myMap }); + + expect(result["myMap.key1"]).toBe("value1"); + expect(result["myMap.key2"]).toBe(42); + expect(result["myMap.123"]).toBe("numeric key"); + }); + + it("handles nested Map objects correctly", () => { + const myMap = new Map(); + myMap.set("key1", { + key2: "value2", + key3: 42, + }); + const result = flattenAttributes({ myMap }); + expect(result["myMap.key1.key2"]).toBe("value2"); + expect(result["myMap.key1.key3"]).toBe(42); + }); + + it("handles File objects correctly", () => { + if (typeof File !== "undefined") { + const file = new File(["content"], "test.txt", { + type: "text/plain", + lastModified: 1640995200000, + }); + const result = flattenAttributes({ file }); + + expect(result["file.name"]).toBe("test.txt"); + expect(result["file.type"]).toBe("text/plain"); + expect(result["file.size"]).toBe(7); // "content" is 7 bytes + expect(result["file.lastModified"]).toBe(1640995200000); + } + }); + + it("handles ReadableStream objects correctly", () => { + if (typeof ReadableStream !== "undefined") { + const stream = new ReadableStream(); + const result = flattenAttributes({ stream }); + + expect(result["stream.type"]).toBe("ReadableStream"); + expect(result["stream.locked"]).toBe(false); + } + }); + + it("handles Promise objects correctly", () => { + const resolvedPromise = Promise.resolve("value"); + const rejectedPromise = Promise.reject(new Error("failed")); + const pendingPromise = new Promise(() => {}); // Never resolves + + // Catch the rejection to avoid unhandled promise rejection warnings + rejectedPromise.catch(() => {}); + + const result = flattenAttributes({ + resolved: resolvedPromise, + rejected: rejectedPromise, + pending: pendingPromise, + }); + + expect(result["resolved"]).toBe("[Promise object]"); + expect(result["rejected"]).toBe("[Promise object]"); + expect(result["pending"]).toBe("[Promise object]"); + }); + + it("handles RegExp objects correctly", () => { + const regex = /hello.*world/gim; + const result = flattenAttributes({ regex }); + + expect(result["regex.source"]).toBe("hello.*world"); + expect(result["regex.flags"]).toBe("gim"); + }); + + it("handles URL objects correctly", () => { + if (typeof URL !== "undefined") { + const url = new URL("https://example.com:8080/path?query=value#fragment"); + const result = flattenAttributes({ url }); + + expect(result["url.href"]).toBe("https://example.com:8080/path?query=value#fragment"); + expect(result["url.protocol"]).toBe("https:"); + expect(result["url.host"]).toBe("example.com:8080"); + expect(result["url.pathname"]).toBe("/path"); + } + }); + + it("handles ArrayBuffer correctly", () => { + const buffer = new ArrayBuffer(16); + const result = flattenAttributes({ buffer }); + + expect(result["buffer.byteLength"]).toBe(16); + }); + + it("handles TypedArrays correctly", () => { + const uint8Array = new Uint8Array([1, 2, 3, 4]); + const int32Array = new Int32Array([100, 200, 300]); + + const result = flattenAttributes({ + uint8: uint8Array, + int32: int32Array, + }); + + expect(result["uint8.constructor"]).toBe("Uint8Array"); + expect(result["uint8.length"]).toBe(4); + expect(result["uint8.byteLength"]).toBe(4); + expect(result["uint8.byteOffset"]).toBe(0); + + expect(result["int32.constructor"]).toBe("Int32Array"); + expect(result["int32.length"]).toBe(3); + expect(result["int32.byteLength"]).toBe(12); // 3 * 4 bytes + expect(result["int32.byteOffset"]).toBe(0); + }); + + it("handles complex mixed object with all special types", () => { + const complexObj = { + error: new Error("Test error"), + func: function testFunc() {}, + date: new Date("2023-01-01"), + mySet: new Set([1, 2, 3]), + myMap: new Map([["key", "value"]]), + regex: /test/gi, + bigint: BigInt(999), + symbol: Symbol("test"), + }; + + const result = flattenAttributes(complexObj); + + // Verify we get reasonable representations for all types + expect(result["error.name"]).toBe("Error"); + expect(result["func"]).toBe("[Function: testFunc]"); + expect(result["date"]).toBe("2023-01-01T00:00:00.000Z"); + expect(result["regex.source"]).toBe("test"); + expect(result["bigint"]).toBe("999"); + expect(typeof result["symbol"]).toBe("string"); + }); }); describe("unflattenAttributes", () => { @@ -246,10 +562,10 @@ describe("unflattenAttributes", () => { }; expect(unflattenAttributes(flattened)).toEqual(expected); }); - + it("rehydrates circular references correctly", () => { const flattened = { - "name": "Alice", + name: "Alice", "blogPosts.[0].title": "Post 1", "blogPosts.[0].author": "$@circular((", }; diff --git a/references/hello-world/src/trigger/example.ts b/references/hello-world/src/trigger/example.ts index 6c527360e7..8460eda65e 100644 --- a/references/hello-world/src/trigger/example.ts +++ b/references/hello-world/src/trigger/example.ts @@ -328,3 +328,52 @@ export const circularReferenceTask = task({ }; }, }); + +export const largeAttributesTask = task({ + id: "large-attributes", + machine: "large-1x", + run: async ({ length = 100000 }: { length: number }, { signal, ctx }) => { + // Create a large deeply nested object/array of objects that have more than 10k attributes when flattened + const start = performance.now(); + + const largeObject = Array.from({ length }, (_, i) => ({ + a: i, + b: i, + c: i, + })); + + const end = performance.now(); + + console.log(`[${length}] Time taken to create the large object: ${end - start}ms`); + + const start2 = performance.now(); + + logger.info("Hello, world from the large attributes task", { largeObject }); + + const end2 = performance.now(); + + console.log(`[${length}] Time taken to log the large object: ${end2 - start2}ms`); + + class MyClass { + constructor(public name: string) {} + } + + logger.info("Lets log some weird stuff", { + error: new Error("This is an error"), + func: () => { + logger.info("This is a function"); + }, + date: new Date(), + bigInt: BigInt(1000000000000000000), + symbol: Symbol("symbol"), + myClass: new MyClass("MyClass"), + file: new File([], "test.txt"), + stream: new ReadableStream(), + map: new Map([["key", "value"]]), + set: new Set([1, 2, 3]), + promise: Promise.resolve("Hello, world!"), + promiseRejected: Promise.reject(new Error("This is a rejected promise")), + promisePending: Promise.resolve("Hello, world!"), + }); + }, +});