diff --git a/.changeset/otel-structured-output-iteration-span.md b/.changeset/otel-structured-output-iteration-span.md new file mode 100644 index 000000000..3ae307c7b --- /dev/null +++ b/.changeset/otel-structured-output-iteration-span.md @@ -0,0 +1,7 @@ +--- +'@tanstack/ai': patch +--- + +fix(otelMiddleware): open an iteration span for structured-output finalization (#1054) + +No-tools + `outputSchema` calls skip the agent loop and only run the structured-output finalization request. That path previously emitted only a root `chat` span — no generation record, and `captureContent` was a silent no-op. `onConfig` now also opens an iteration span when `phase === 'structuredOutput'`, so each provider model call is observable. Native-combined mode is unaffected (it never fires that phase). diff --git a/docs/advanced/otel.md b/docs/advanced/otel.md index 3bdc1513b..a92cae801 100644 --- a/docs/advanced/otel.md +++ b/docs/advanced/otel.md @@ -14,7 +14,9 @@ keywords: - semantic conventions --- -The `otelMiddleware` factory wires TanStack AI into your existing OpenTelemetry setup. Every `chat()` call produces a root span, one child span per agent-loop iteration, and one grandchild span per tool call — all with [GenAI semantic-convention attributes](https://opentelemetry.io/docs/specs/semconv/gen-ai/). It also records GenAI token and duration histograms when a `Meter` is provided. +The `otelMiddleware` factory wires TanStack AI into your existing OpenTelemetry setup. Every `chat()` call produces a root span, one child span per provider model call (agent-loop turn **or** structured-output finalization), and one grandchild span per tool call — all with [GenAI semantic-convention attributes](https://opentelemetry.io/docs/specs/semconv/gen-ai/). It also records GenAI token and duration histograms when a `Meter` is provided. + +Structured-output calls with no tools skip the agent loop and only run the finalization request. That path still opens an iteration span (via the `structuredOutput` middleware phase) so backends that key off generation spans (e.g. PostHog `$ai_generation`) and `captureContent` both work. Native combined mode (`supportsCombinedToolsAndSchema`) does not fire that phase — the single `beforeModel` span covers the combined call. ## Setup @@ -57,7 +59,7 @@ chat gpt-5.5 (root, kind: INTERNAL) └── chat gpt-5.5 #1 (iteration, kind: CLIENT) ``` -Iteration spans are numbered (`#0`, `#1`, ...) so distinct iterations of the same chat are easy to pick apart in trace viewers. +Iteration spans are numbered (`#0`, `#1`, ...) in the order model calls are observed, so distinct provider round-trips of the same chat are easy to pick apart in trace viewers. ### Attribute reference diff --git a/docs/config.json b/docs/config.json index 67c19a8db..687eae58e 100644 --- a/docs/config.json +++ b/docs/config.json @@ -473,7 +473,7 @@ "label": "OpenTelemetry", "to": "advanced/otel", "addedAt": "2026-05-08", - "updatedAt": "2026-07-31" + "updatedAt": "2026-08-06" } ] }, diff --git a/packages/ai/src/middlewares/otel.ts b/packages/ai/src/middlewares/otel.ts index ee15a266e..e5938bd8c 100644 --- a/packages/ai/src/middlewares/otel.ts +++ b/packages/ai/src/middlewares/otel.ts @@ -33,7 +33,9 @@ import type { * Scope (role) of an OTel span emitted by this middleware. * * - `chat` — the root span for a single `chat()` call - * - `iteration` — one per agent-loop iteration (one model call) + * - `iteration` — one per provider model call (agent-loop `beforeModel` + * turn, or the separate `structuredOutput` finalization when + * `outputSchema` skips the agent loop — see #1054) * - `tool` — one per tool execution inside an iteration * - `generation` — the single span for a media activity call * (`generateImage`, `generateVideo`, `generateSpeech`, …) @@ -401,7 +403,16 @@ export function otelMiddleware( }, onConfig(ctx, config) { - if (ctx.phase !== 'beforeModel') return + // Open an iteration span for every provider model call: + // - `beforeModel`: agent-loop chatStream turns + // - `structuredOutput`: separate structured-output finalization + // (no-tools + outputSchema skips the agent loop, so without this + // phase there is no generation span and captureContent is a silent + // no-op — see #1054). + // Native-combined mode never fires `structuredOutput`, so a run that + // already opened spans via `beforeModel` is not double-counted. + if (ctx.phase !== 'beforeModel' && ctx.phase !== 'structuredOutput') + return safeCall('otel.onConfig', () => { const state = stateByCtx.get(ctx) if (!state) return @@ -411,20 +422,26 @@ export function otelMiddleware( // on it. Close it here, just before opening the next iteration. closeIterationSpan(state, ctx) + // Number spans by the order of model calls this middleware has seen, + // not by `ctx.iteration`. After an agent-loop turn, structured-output + // finalization reuses the engine's last iteration index; using our + // own counter keeps finalization as a distinct leaf (#N+1). + const iteration = state.iterationCount + const info: OtelSpanInfo<'iteration'> = { kind: 'iteration', ctx, - iteration: ctx.iteration, + iteration, } const name = safeCall('otel.spanNameFormatter', () => spanNameFormatter?.(info)) ?? - `chat ${ctx.model} #${ctx.iteration}` + `chat ${ctx.model} #${iteration}` const baseAttrs: Record = { 'gen_ai.system': ctx.provider, 'gen_ai.operation.name': 'chat', 'gen_ai.request.model': ctx.model, - 'tanstack.ai.iteration': ctx.iteration, + 'tanstack.ai.iteration': iteration, } // Sampling options now live in provider-native `modelOptions`, and // providers spell them differently (e.g. `max_output_tokens`, diff --git a/packages/ai/tests/middlewares/otel.test.ts b/packages/ai/tests/middlewares/otel.test.ts index a5d3afa85..1723d464e 100644 --- a/packages/ai/tests/middlewares/otel.test.ts +++ b/packages/ai/tests/middlewares/otel.test.ts @@ -198,6 +198,107 @@ describe('otelMiddleware — iteration span lifecycle', () => { expect(spans[1]!.name).toBe('chat gpt-4o #0') expect(spans[2]!.name).toBe('chat gpt-4o #1') }) + + // #1054 — no-tools + outputSchema skips the agent loop, so the only + // onConfig that fires is phase=structuredOutput. That must open a + // generation (iteration) span or captureContent is a silent no-op and + // backends that key off iteration spans (PostHog $ai_generation) see + // an empty trace. + it('opens an iteration span on onConfig(structuredOutput) — no-tools + outputSchema path', async () => { + const { tracer, spans } = createFakeTracer() + const mw = otelMiddleware({ tracer, captureContent: true }) + const ctx = makeCtx() + + await mw.onStart?.(ctx) + ctx.phase = 'structuredOutput' + await mw.onConfig?.(ctx, { + messages: [{ role: 'user', content: 'Describe this scene' }], + systemPrompts: [], + tools: [], + }) + + expect(spans).toHaveLength(2) + const [rootSpan, iterSpan] = spans + expect(iterSpan!.parent).toBe(rootSpan) + expect(iterSpan!.name).toBe('chat gpt-4o #0') + expect(iterSpan!.kind).toBe(SpanKind.CLIENT) + expect(iterSpan!.attributes['gen_ai.operation.name']).toBe('chat') + expect(iterSpan!.attributes['tanstack.ai.iteration']).toBe(0) + expect(iterSpan!.attributes['gen_ai.input.messages']).toBe( + JSON.stringify([{ role: 'user', content: 'Describe this scene' }]), + ) + + await mw.onChunk?.(ctx, ev.textContent('{"description":"a sunny park"}')) + await mw.onChunk?.(ctx, { + ...ev.runFinished('stop'), + model: 'gpt-4o', + usage: { promptTokens: 12, completionTokens: 8, totalTokens: 20 }, + }) + expect(iterSpan!.attributes['gen_ai.output.messages']).toBe( + JSON.stringify([ + { role: 'assistant', content: '{"description":"a sunny park"}' }, + ]), + ) + expect(iterSpan!.attributes['gen_ai.usage.input_tokens']).toBe(12) + + await mw.onFinish?.(ctx, { + finishReason: 'stop', + duration: 10, + content: '', + }) + expect(iterSpan!.ended).toBe(true) + expect(rootSpan!.ended).toBe(true) + }) + + it('does not open an iteration span for non-model-call phases', async () => { + const { tracer, spans } = createFakeTracer() + const mw = otelMiddleware({ tracer }) + const ctx = makeCtx() + + await mw.onStart?.(ctx) + for (const phase of [ + 'init', + 'modelStream', + 'beforeTools', + 'afterTools', + ] as const) { + ctx.phase = phase + await mw.onConfig?.(ctx, { + messages: [], + systemPrompts: [], + tools: [], + }) + } + + // Root span only — none of those phases are a provider model call. + expect(spans).toHaveLength(1) + }) + + it('numbers structuredOutput finalization after a prior beforeModel span (#N+1)', async () => { + // Tools + outputSchema: agent loop opens #0, then finalization must open + // a distinct #1 rather than reusing ctx.iteration from the last turn. + const { tracer, spans } = createFakeTracer() + const mw = otelMiddleware({ tracer }) + const ctx = makeCtx() + + await runToIterationStart(mw, ctx) + await mw.onChunk?.(ctx, ev.runFinished('tool_calls')) + // Engine leaves ctx.iteration at 0 for finalization; middleware must + // still mint a distinct leaf. + ctx.phase = 'structuredOutput' + ctx.iteration = 0 + await mw.onConfig?.(ctx, { + messages: [{ role: 'user', content: 'hi' }], + systemPrompts: [], + tools: [], + }) + + expect(spans).toHaveLength(3) + expect(spans[1]!.ended).toBe(true) + expect(spans[1]!.name).toBe('chat gpt-4o #0') + expect(spans[2]!.name).toBe('chat gpt-4o #1') + expect(spans[2]!.attributes['tanstack.ai.iteration']).toBe(1) + }) }) describe('otelMiddleware — token histogram', () => { diff --git a/testing/e2e/tests/middleware.spec.ts b/testing/e2e/tests/middleware.spec.ts index d1f146a7f..267ce507e 100644 --- a/testing/e2e/tests/middleware.spec.ts +++ b/testing/e2e/tests/middleware.spec.ts @@ -189,6 +189,63 @@ test.describe('Middleware Lifecycle', () => { ).toBeUndefined() }) + // #1054 — no-tools + outputSchema skips the agent loop. The only model + // call is structured-output finalization (`phase=structuredOutput`). + // otelMiddleware must open an iteration (CLIENT) span for that call so + // backends that key off generation spans (PostHog $ai_generation) and + // captureContent both work. Pin to claude-3-7-sonnet so we take the + // legacy finalization path, not native-combined (#605) which already + // emits a beforeModel iteration span. + test('otel middleware emits iteration span + captureContent for no-tools structured-output finalization', async ({ + page, + testId, + aimockPort, + baseURL, + }) => { + const params = new URLSearchParams() + if (testId) params.set('testId', testId) + if (aimockPort) params.set('aimockPort', String(aimockPort)) + params.set('provider', 'anthropic') + params.set('model', 'claude-3-7-sonnet') + const qs = params.toString() + await page.goto(`/middleware-test?${qs}`) + await page.waitForTimeout(2000) + await page.locator('#mw-scenario-select').selectOption('structured-output') + await page.locator('#mw-mode-select').selectOption('otel') + await page.locator('#mw-run-button').click() + + await page.waitForFunction( + () => + document + .querySelector('#mw-metadata') + ?.getAttribute('data-test-complete') === 'true', + { timeout: 15000 }, + ) + + const capture = await fetchOtelCapture(page, baseURL, testId) + + const chatSpans = capture.spans.filter( + (s: any) => s.kind === SpanKind.INTERNAL, + ) + expect(chatSpans).toHaveLength(1) + + const iterationSpans = capture.spans.filter( + (s: any) => s.kind === SpanKind.CLIENT, + ) + // Exactly one provider call on the skip-agent-loop path → one generation. + expect(iterationSpans).toHaveLength(1) + const iter = iterationSpans[0] + expect(iter.ended).toBe(true) + expect(iter.attributes['gen_ai.operation.name']).toBe('chat') + expect(iter.attributes['tanstack.ai.iteration']).toBe(0) + + // captureContent is enabled on the harness otel middleware — prompt must + // land on the iteration span (the pre-fix silent no-op left this empty). + const inputMessages = iter.attributes['gen_ai.input.messages'] + expect(typeof inputMessages).toBe('string') + expect(inputMessages.length).toBeGreaterThan(0) + }) + test('otel middleware nests tool spans under the iteration span that triggered them', async ({ page, testId,