Skip to content

Commit aea4552

Browse files
isaacsclaude
andcommitted
fix(server-utils): Record Vercel AI experimental_telemetry.metadata on gen_ai spans
v10 got `vercel.ai.telemetry.metadata.<key>` attributes from the AI SDK's own OTel spans. In v11, `ai` 4 to 6 go through the orchestrion subscriber, which never read `metadata`, so v11 dropped it. The orchestrion subscriber now passes the metadata to the channel subscriber, which sets it as span attributes when each span starts. The `invoke_agent`, `generate_content` and `embeddings` spans get the attributes, and `execute_tool` spans don't. This matches the AI SDK, which leaves metadata off its `ai.toolCall` spans. fixes JS-3800 fixes #24720 Re: #24706 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 3844ba0 commit aea4552

4 files changed

Lines changed: 147 additions & 2 deletions

File tree

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import * as Sentry from '@sentry/node';
2+
import { embed, embedMany, generateText, streamText } from 'ai';
3+
import { MockEmbeddingModelV3, MockLanguageModelV3, simulateReadableStream } from 'ai/test';
4+
5+
const usage = {
6+
inputTokens: { total: 10, noCache: 10, cached: 0 },
7+
outputTokens: { total: 20, noCache: 20, cached: 0 },
8+
totalTokens: { total: 30, noCache: 30, cached: 0 },
9+
};
10+
11+
async function run() {
12+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
13+
await generateText({
14+
experimental_telemetry: {
15+
isEnabled: true,
16+
functionId: 'support-chat',
17+
metadata: { requestId: 'req_generate', tenantId: 'acme' },
18+
},
19+
model: new MockLanguageModelV3({
20+
doGenerate: async () => ({
21+
finishReason: { unified: 'stop', raw: 'stop' },
22+
usage,
23+
content: [{ type: 'text', text: 'Hello!' }],
24+
warnings: [],
25+
}),
26+
}),
27+
prompt: 'Hi',
28+
});
29+
30+
const result = streamText({
31+
experimental_telemetry: {
32+
isEnabled: true,
33+
functionId: 'support-stream',
34+
metadata: { requestId: 'req_stream', tenantId: 'acme' },
35+
},
36+
model: new MockLanguageModelV3({
37+
doStream: async () => ({
38+
stream: simulateReadableStream({
39+
chunks: [
40+
{ type: 'stream-start', warnings: [] },
41+
{ type: 'text-start', id: '0' },
42+
{ type: 'text-delta', id: '0', delta: 'Hello!' },
43+
{ type: 'text-end', id: '0' },
44+
{ type: 'finish', finishReason: { unified: 'stop', raw: 'stop' }, usage },
45+
],
46+
}),
47+
}),
48+
}),
49+
prompt: 'Hi',
50+
});
51+
for await (const _part of result.fullStream) {
52+
void _part;
53+
}
54+
55+
await embed({
56+
experimental_telemetry: { isEnabled: true, metadata: { requestId: 'req_embed', tenantId: 'acme' } },
57+
model: new MockEmbeddingModelV3({
58+
doEmbed: async () => ({ embeddings: [[0.1, 0.2, 0.3]], usage: { tokens: 10 } }),
59+
}),
60+
value: 'Embed me',
61+
});
62+
63+
await embedMany({
64+
experimental_telemetry: { isEnabled: true, metadata: { requestId: 'req_embed_many', tenantId: 'acme' } },
65+
model: new MockEmbeddingModelV3({
66+
maxEmbeddingsPerCall: 5,
67+
doEmbed: async () => ({
68+
embeddings: [
69+
[0.1, 0.2, 0.3],
70+
[0.4, 0.5, 0.6],
71+
],
72+
usage: { tokens: 20 },
73+
}),
74+
}),
75+
values: ['First', 'Second'],
76+
});
77+
});
78+
79+
await Sentry.flush(2000);
80+
}
81+
82+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,44 @@ describe('Vercel AI integration (streaming, v6)', () => {
304304
},
305305
);
306306

307+
function spanWithMetadata(name: string, op: string, requestId: string) {
308+
return expect.objectContaining({
309+
name,
310+
attributes: expect.objectContaining({
311+
'vercel.ai.telemetry.metadata.requestId': attr(requestId),
312+
'vercel.ai.telemetry.metadata.tenantId': attr('acme'),
313+
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr(op),
314+
}),
315+
});
316+
}
317+
318+
const EXPECTED_SPANS_TELEMETRY_METADATA = {
319+
items: expect.arrayContaining([
320+
spanWithMetadata('invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate'),
321+
spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate'),
322+
spanWithMetadata('invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'),
323+
spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'),
324+
spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'),
325+
spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'),
326+
]),
327+
};
328+
329+
createEsmAndCjsTests(
330+
__dirname,
331+
'scenario-telemetry-metadata.mjs',
332+
'instrument.mjs',
333+
(createRunner, test) => {
334+
test('records experimental_telemetry.metadata as span attributes', async () => {
335+
await createRunner().expect({ span: EXPECTED_SPANS_TELEMETRY_METADATA }).start().completed();
336+
});
337+
},
338+
{
339+
additionalDependencies: {
340+
ai: '^6.0.0',
341+
},
342+
},
343+
);
344+
307345
createEsmAndCjsTests(
308346
__dirname,
309347
'scenario-error-in-tool.mjs',

‎packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import {
3333
GEN_AI_INVOKE_AGENT,
3434
GEN_AI_RERANK,
3535
} from '@sentry/conventions/op';
36-
import type { Span, SpanAttributes } from '@sentry/core';
36+
import type { Span, SpanAttributes, SpanAttributeValue } from '@sentry/core';
3737
import {
3838
_INTERNAL_skipAiProviderWrapping,
3939
captureException,
@@ -86,6 +86,7 @@ type GenAiOperation = keyof typeof GEN_AI_OPERATION_SPAN_OPS;
8686
const VERCEL_AI_OPERATION_ID_ATTRIBUTE = 'vercel.ai.operationId';
8787
const VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE = 'vercel.ai.model.provider';
8888
const VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE = 'vercel.ai.settings.maxRetries';
89+
const VERCEL_AI_TELEMETRY_METADATA_ATTRIBUTE_PREFIX = 'vercel.ai.telemetry.metadata.';
8990

9091
// Tracks the top-level operationId (and whether it streams) per `callId` so a model-call span can
9192
// name its `doGenerate`/`doStream` operation the same way the OTel integration does. `isStream` is
@@ -417,8 +418,9 @@ export function createSpanFromMessage(
417418
recordToolDescriptions(callId, event.tools);
418419
}
419420

420-
const baseAttributes: Record<string, string | number | boolean> = {
421+
const baseAttributes: SpanAttributes = {
421422
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN,
423+
...telemetryMetadataAttributes(event.telemetryMetadata),
422424
...(provider ? { [GEN_AI_PROVIDER_NAME]: provider, [VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE]: provider } : {}),
423425
...(modelId ? { [GEN_AI_REQUEST_MODEL]: modelId } : {}),
424426
...(maxRetries !== undefined ? { [VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE]: maxRetries } : {}),
@@ -466,6 +468,21 @@ export function createSpanFromMessage(
466468
}
467469
}
468470

471+
/**
472+
* `experimental_telemetry.metadata` (`ai` <= 6) as `vercel.ai.telemetry.metadata.<key>`, the names the OTel
473+
* integration produced from the SDK's `ai.telemetry.metadata.*`. Only the orchestrion adapter sets
474+
* `event.telemetryMetadata`; `ai` 7 has no `telemetry.metadata`.
475+
*/
476+
function telemetryMetadataAttributes(metadata: unknown): SpanAttributes {
477+
const attributes: SpanAttributes = {};
478+
if (isObjectLike(metadata)) {
479+
for (const [key, value] of Object.entries(metadata)) {
480+
attributes[`${VERCEL_AI_TELEMETRY_METADATA_ATTRIBUTE_PREFIX}${key}`] = value as SpanAttributeValue;
481+
}
482+
}
483+
return attributes;
484+
}
485+
469486
/** Start a `gen_ai.<operation>` span named `<operation> <suffix>` (or just `<operation>` when no suffix). */
470487
function startGenAiSpan(operation: GenAiOperation, suffix: string | undefined, attributes: SpanAttributes): Span {
471488
return startInactiveSpan({

‎packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ const callIdBySpan = new WeakMap<Span, string>();
107107
// child `generate_content` span (whose event would fall back to the global default). v7's channel forwards
108108
// these flags on every event, so this keeps v6 identical.
109109
const recordingBySpan = new WeakMap<Span, ReturnType<typeof recording>>();
110+
// The operation's `experimental_telemetry.metadata`, keyed by its span, so the model-call span records it
111+
// too. The OTel integration put it on both, but not on tool-call spans.
112+
const telemetryMetadataBySpan = new WeakMap<Span, unknown>();
110113
interface OperationErrorInfo {
111114
// The span active when the operation was invoked (its call site, e.g. the enclosing request/`main` span).
112115
// When an operation's error bubbles out unhandled it reaches the global handler outside any span, so — as
@@ -163,6 +166,7 @@ export function subscribeVercelAiOrchestrionChannels(
163166
...modelFields(callOptions.model),
164167
maxRetries: callOptions.maxRetries,
165168
value: callOptions.value,
169+
telemetryMetadata: telemetry.metadata,
166170
...recording(telemetry),
167171
},
168172
}),
@@ -180,6 +184,7 @@ export function subscribeVercelAiOrchestrionChannels(
180184
...modelFields(callOptions.model),
181185
maxRetries: callOptions.maxRetries,
182186
values: callOptions.values,
187+
telemetryMetadata: telemetry.metadata,
183188
...recording(telemetry),
184189
},
185190
}),
@@ -263,6 +268,7 @@ function bindOperation(
263268
callIdBySpan.set(span, callId);
264269
}
265270
recordingBySpan.set(span, recording(telemetry));
271+
telemetryMetadataBySpan.set(span, telemetry.metadata);
266272
// v5 has no `executeToolCall` channel, so patch each tool's `execute` to emit the tool-call span.
267273
// Inert on v6 (guarded inside `patchToolExecute` when the parent is `executeToolCall`'s own span).
268274
if (isObjectLike(callOptions.tools)) {
@@ -490,6 +496,7 @@ function patchModelMethod(
490496
// top-level `tools` array. Reading both keeps `tool.definitions` populated on the model-call span.
491497
tools: callArgs.tools ?? (isObjectLike(callArgs.mode) ? callArgs.mode.tools : undefined),
492498
messages: callArgs.prompt,
499+
telemetryMetadata: telemetryMetadataBySpan.get(parent),
493500
// Inherit the enclosing operation's per-call recording flags so inputs/tools/outputs are recorded on
494501
// the model-call span whenever they are on the parent `invoke_agent` span.
495502
...recordingBySpan.get(parent),
@@ -669,6 +676,7 @@ function buildTextMessage(type: 'generateText' | 'streamText' | 'generateObject'
669676
// Normalize to the message-array shape the shared core (and v7's channel) expects: a bare string
670677
// `prompt` becomes a single user message, matching the SDK's own normalization.
671678
messages: normalizePromptMessages(options),
679+
telemetryMetadata: telemetry.metadata,
672680
...recording(telemetry),
673681
},
674682
});

0 commit comments

Comments
 (0)