Skip to content

Commit 131bd44

Browse files
isaacsclaude
andcommitted
fix(server-utils): Record Vercel AI experimental_telemetry.metadata on gen_ai spans
v10 got `vercel.ai.telemetry.metadata.<key>` attributes from the AI SDK's own OTel spans. In v11, `ai` 4 to 6 go through the orchestrion subscriber, which never read `metadata`, so v11 dropped it. The orchestrion subscriber now passes the metadata to the channel subscriber, which sets it as span attributes when each span starts. The `invoke_agent`, `generate_content` and `embeddings` spans get the attributes, and `execute_tool` spans don't. This matches the AI SDK, which leaves metadata off its `ai.toolCall` spans. Values are cast to string, as discussed in the linked sentry-conventions PR. If this causes a problem we can revisit, but all extant examples found in the wild are strings already, so it is likely safe, and extremely cheap. fixes JS-3800 fixes #24720 Re: #24706 Re: getsentry/sentry-conventions#662 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 21ba35d commit 131bd44

5 files changed

Lines changed: 213 additions & 1 deletion

File tree

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import * as Sentry from '@sentry/node';
2+
import { embed, embedMany, generateText, streamText } from 'ai';
3+
import { MockEmbeddingModelV3, MockLanguageModelV3, simulateReadableStream } from 'ai/test';
4+
5+
const usage = {
6+
inputTokens: { total: 10, noCache: 10, cached: 0 },
7+
outputTokens: { total: 20, noCache: 20, cached: 0 },
8+
totalTokens: { total: 30, noCache: 30, cached: 0 },
9+
};
10+
11+
async function run() {
12+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
13+
await generateText({
14+
experimental_telemetry: {
15+
isEnabled: true,
16+
functionId: 'support-chat',
17+
metadata: { requestId: 'req_generate', tenantId: 'acme', attempt: 2 },
18+
},
19+
model: new MockLanguageModelV3({
20+
doGenerate: async () => ({
21+
finishReason: { unified: 'stop', raw: 'stop' },
22+
usage,
23+
content: [{ type: 'text', text: 'Hello!' }],
24+
warnings: [],
25+
}),
26+
}),
27+
prompt: 'Hi',
28+
});
29+
30+
const result = streamText({
31+
experimental_telemetry: {
32+
isEnabled: true,
33+
functionId: 'support-stream',
34+
metadata: { requestId: 'req_stream', tenantId: 'acme' },
35+
},
36+
model: new MockLanguageModelV3({
37+
doStream: async () => ({
38+
stream: simulateReadableStream({
39+
chunks: [
40+
{ type: 'stream-start', warnings: [] },
41+
{ type: 'text-start', id: '0' },
42+
{ type: 'text-delta', id: '0', delta: 'Hello!' },
43+
{ type: 'text-end', id: '0' },
44+
{ type: 'finish', finishReason: { unified: 'stop', raw: 'stop' }, usage },
45+
],
46+
}),
47+
}),
48+
}),
49+
prompt: 'Hi',
50+
});
51+
for await (const _part of result.fullStream) {
52+
void _part;
53+
}
54+
55+
await embed({
56+
experimental_telemetry: { isEnabled: true, metadata: { requestId: 'req_embed', tenantId: 'acme' } },
57+
model: new MockEmbeddingModelV3({
58+
doEmbed: async () => ({ embeddings: [[0.1, 0.2, 0.3]], usage: { tokens: 10 } }),
59+
}),
60+
value: 'Embed me',
61+
});
62+
63+
await embedMany({
64+
experimental_telemetry: { isEnabled: true, metadata: { requestId: 'req_embed_many', tenantId: 'acme' } },
65+
model: new MockEmbeddingModelV3({
66+
maxEmbeddingsPerCall: 5,
67+
doEmbed: async () => ({
68+
embeddings: [
69+
[0.1, 0.2, 0.3],
70+
[0.4, 0.5, 0.6],
71+
],
72+
usage: { tokens: 20 },
73+
}),
74+
}),
75+
values: ['First', 'Second'],
76+
});
77+
});
78+
79+
await Sentry.flush(2000);
80+
}
81+
82+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,49 @@ describe('Vercel AI integration (streaming, v6)', () => {
305305
},
306306
);
307307

308+
function spanWithMetadata(name: string, op: string, requestId: string, extra: Record<string, unknown> = {}) {
309+
return expect.objectContaining({
310+
name,
311+
attributes: expect.objectContaining({
312+
'vercel.ai.telemetry.metadata.requestId': attr(requestId),
313+
'vercel.ai.telemetry.metadata.tenantId': attr('acme'),
314+
...extra,
315+
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr(op),
316+
}),
317+
});
318+
}
319+
320+
const EXPECTED_SPANS_TELEMETRY_METADATA = {
321+
items: expect.arrayContaining([
322+
spanWithMetadata('invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate', {
323+
'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' },
324+
}),
325+
spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate', {
326+
'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' },
327+
}),
328+
spanWithMetadata('invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'),
329+
spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'),
330+
spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'),
331+
spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'),
332+
]),
333+
};
334+
335+
createEsmAndCjsTests(
336+
__dirname,
337+
'scenario-telemetry-metadata.mjs',
338+
'instrument.mjs',
339+
(createRunner, test) => {
340+
test('records experimental_telemetry.metadata as span attributes', async () => {
341+
await createRunner().expect({ span: EXPECTED_SPANS_TELEMETRY_METADATA }).start().completed();
342+
});
343+
},
344+
{
345+
additionalDependencies: {
346+
ai: '^6.0.0',
347+
},
348+
},
349+
);
350+
308351
createEsmAndCjsTests(
309352
__dirname,
310353
'scenario-error-in-tool.mjs',

‎packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ type GenAiOperation = keyof typeof GEN_AI_OPERATION_SPAN_OPS;
8686
const VERCEL_AI_OPERATION_ID_ATTRIBUTE = 'vercel.ai.operationId';
8787
const VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE = 'vercel.ai.model.provider';
8888
const VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE = 'vercel.ai.settings.maxRetries';
89+
const VERCEL_AI_TELEMETRY_METADATA_ATTRIBUTE_PREFIX = 'vercel.ai.telemetry.metadata.';
8990

9091
// Tracks the top-level operationId (and whether it streams) per `callId` so a model-call span can
9192
// name its `doGenerate`/`doStream` operation the same way the OTel integration does. `isStream` is
@@ -417,8 +418,9 @@ export function createSpanFromMessage(
417418
recordToolDescriptions(callId, event.tools);
418419
}
419420

420-
const baseAttributes: Record<string, string | number | boolean> = {
421+
const baseAttributes: SpanAttributes = {
421422
[SENTRY_ORIGIN]: ORIGIN,
423+
...telemetryMetadataAttributes(event.telemetryMetadata),
422424
...(provider ? { [GEN_AI_PROVIDER_NAME]: provider, [VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE]: provider } : {}),
423425
...(modelId ? { [GEN_AI_REQUEST_MODEL]: modelId } : {}),
424426
...(maxRetries !== undefined ? { [VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE]: maxRetries } : {}),
@@ -466,6 +468,22 @@ export function createSpanFromMessage(
466468
}
467469
}
468470

471+
/**
472+
* `experimental_telemetry.metadata` (`ai` <= 6) as `vercel.ai.telemetry.metadata.<key>`, the names the OTel
473+
* integration produced from the SDK's `ai.telemetry.metadata.*`. Only the orchestrion adapter sets
474+
* `event.telemetryMetadata`; `ai` 7 has no `telemetry.metadata`.
475+
*/
476+
function telemetryMetadataAttributes(metadata: unknown): Record<string, string> {
477+
const attributes: Record<string, string> = {};
478+
if (isObjectLike(metadata)) {
479+
for (const [key, value] of Object.entries(metadata)) {
480+
if (value === null || value === undefined) continue;
481+
attributes[`${VERCEL_AI_TELEMETRY_METADATA_ATTRIBUTE_PREFIX}${key}`] = String(value);
482+
}
483+
}
484+
return attributes;
485+
}
486+
469487
/** Start a `gen_ai.<operation>` span named `<operation> <suffix>` (or just `<operation>` when no suffix). */
470488
function startGenAiSpan(operation: GenAiOperation, suffix: string | undefined, attributes: SpanAttributes): Span {
471489
return startInactiveSpan({

‎packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ const callIdBySpan = new WeakMap<Span, string>();
107107
// child `generate_content` span (whose event would fall back to the global default). v7's channel forwards
108108
// these flags on every event, so this keeps v6 identical.
109109
const recordingBySpan = new WeakMap<Span, ReturnType<typeof recording>>();
110+
// The operation's `experimental_telemetry.metadata`, keyed by its span, so the model-call span records it
111+
// too. The OTel integration put it on both, but not on tool-call spans.
112+
const telemetryMetadataBySpan = new WeakMap<Span, unknown>();
110113
interface OperationErrorInfo {
111114
// The span active when the operation was invoked (its call site, e.g. the enclosing request/`main` span).
112115
// When an operation's error bubbles out unhandled it reaches the global handler outside any span, so — as
@@ -163,6 +166,7 @@ export function subscribeVercelAiOrchestrionChannels(
163166
...modelFields(callOptions.model),
164167
maxRetries: callOptions.maxRetries,
165168
value: callOptions.value,
169+
telemetryMetadata: telemetry.metadata,
166170
...recording(telemetry),
167171
},
168172
}),
@@ -180,6 +184,7 @@ export function subscribeVercelAiOrchestrionChannels(
180184
...modelFields(callOptions.model),
181185
maxRetries: callOptions.maxRetries,
182186
values: callOptions.values,
187+
telemetryMetadata: telemetry.metadata,
183188
...recording(telemetry),
184189
},
185190
}),
@@ -263,6 +268,7 @@ function bindOperation(
263268
callIdBySpan.set(span, callId);
264269
}
265270
recordingBySpan.set(span, recording(telemetry));
271+
telemetryMetadataBySpan.set(span, telemetry.metadata);
266272
// v5 has no `executeToolCall` channel, so patch each tool's `execute` to emit the tool-call span.
267273
// Inert on v6 (guarded inside `patchToolExecute` when the parent is `executeToolCall`'s own span).
268274
if (isObjectLike(callOptions.tools)) {
@@ -490,6 +496,7 @@ function patchModelMethod(
490496
// top-level `tools` array. Reading both keeps `tool.definitions` populated on the model-call span.
491497
tools: callArgs.tools ?? (isObjectLike(callArgs.mode) ? callArgs.mode.tools : undefined),
492498
messages: callArgs.prompt,
499+
telemetryMetadata: telemetryMetadataBySpan.get(parent),
493500
// Inherit the enclosing operation's per-call recording flags so inputs/tools/outputs are recorded on
494501
// the model-call span whenever they are on the parent `invoke_agent` span.
495502
...recordingBySpan.get(parent),
@@ -669,6 +676,7 @@ function buildTextMessage(type: 'generateText' | 'streamText' | 'generateObject'
669676
// Normalize to the message-array shape the shared core (and v7's channel) expects: a bare string
670677
// `prompt` becomes a single user message, matching the SDK's own normalization.
671678
messages: normalizePromptMessages(options),
679+
telemetryMetadata: telemetry.metadata,
672680
...recording(telemetry),
673681
},
674682
});
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
2+
import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core';
3+
import type { Span } from '@sentry/core';
4+
import { createSpanFromMessage } from '../../../src/integrations/vercel-ai/vercel-ai-dc-subscriber';
5+
import { getDefaultTestClientOptions, TestClient } from '../../mocks/client';
6+
7+
describe('Vercel AI telemetry metadata', () => {
8+
beforeEach(() => {
9+
getMainCarrier().__SENTRY__ = undefined;
10+
});
11+
12+
afterEach(() => {
13+
getMainCarrier().__SENTRY__ = undefined;
14+
});
15+
16+
function setupClient(): Span[] {
17+
const client = new TestClient(
18+
getDefaultTestClientOptions({
19+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
20+
tracesSampleRate: 1,
21+
}),
22+
);
23+
setCurrentClient(client);
24+
client.init();
25+
26+
const endedSpans: Span[] = [];
27+
client.on('spanEnd', span => endedSpans.push(span));
28+
return endedSpans;
29+
}
30+
31+
it('records metadata values as strings and skips null and undefined', () => {
32+
const endedSpans = setupClient();
33+
const span = createSpanFromMessage(
34+
{
35+
type: 'generateText',
36+
event: {
37+
telemetryMetadata: {
38+
requestId: 'req_1',
39+
attempt: 2,
40+
cached: false,
41+
tags: ['a', 'b'],
42+
empty: null,
43+
missing: undefined,
44+
},
45+
},
46+
} as Parameters<typeof createSpanFromMessage>[0],
47+
{} as Parameters<typeof createSpanFromMessage>[1],
48+
);
49+
span?.end();
50+
51+
const data = spanToStaticSpanJSON(endedSpans[0]!).data;
52+
expect(data).toMatchObject({
53+
'vercel.ai.telemetry.metadata.requestId': 'req_1',
54+
'vercel.ai.telemetry.metadata.attempt': '2',
55+
'vercel.ai.telemetry.metadata.cached': 'false',
56+
'vercel.ai.telemetry.metadata.tags': 'a,b',
57+
});
58+
expect(data).not.toHaveProperty('vercel.ai.telemetry.metadata.empty');
59+
expect(data).not.toHaveProperty('vercel.ai.telemetry.metadata.missing');
60+
});
61+
});

0 commit comments

Comments
 (0)