Add OpenAI Agents auto-instrumentation with real e2e coverage

.changeset/brave-clouds-trace.md

@@ -0,0 +1,5 @@
+---
+"braintrust": patch
+---
+
+feat: Add OpenAI Agents SDK auto-instrumentation

e2e/config/pr-comment-scenarios.json

@@ -21,6 +21,14 @@
       }
     ]
   },
+  {
+    "scenarioDirName": "openai-agents-instrumentation",
+    "label": "OpenAI Agents Instrumentation",
+    "metadataScenario": "openai-agents-instrumentation",
+    "variants": [
+      { "variantKey": "openai-agents-auto-hook", "label": "Auto-hook" }
+    ]
+  },
   {
     "scenarioDirName": "anthropic-instrumentation",
     "label": "Anthropic Instrumentation",

e2e/scenarios/openai-agents-instrumentation/__cassettes__/openai-agents-auto-hook.cassette.json

@@ -0,0 +1,215 @@
+{
+  "version": 1,
+  "meta": {
+    "createdAt": "2026-05-19T00:00:00.000Z",
+    "seinfeldVersion": "0.0.0"
+  },
+  "entries": [
+    {
+      "id": "a1b2c3d4e5f67890",
+      "matchKey": "POST api.openai.com/v1/responses",
+      "callIndex": 0,
+      "recordedAt": "2026-05-19T00:00:00.000Z",
+      "request": {
+        "method": "POST",
+        "url": "https://api.openai.com/v1/responses",
+        "headers": {
+          "accept": "application/json",
+          "authorization": "[REDACTED]",
+          "content-type": "application/json",
+          "user-agent": "Agents/JavaScript 0.0.14",
+          "x-stainless-arch": "arm64",
+          "x-stainless-lang": "js",
+          "x-stainless-os": "MacOS",
+          "x-stainless-package-version": "5.23.2",
+          "x-stainless-retry-count": "0",
+          "x-stainless-runtime": "node",
+          "x-stainless-runtime-version": "v26.0.0"
+        },
+        "body": {
+          "kind": "json",
+          "value": {
+            "instructions": "Use the lookup_weather tool exactly once, then answer only with the forecast.",
+            "model": "gpt-4o-mini",
+            "input": [
+              {
+                "role": "user",
+                "content": "What is the weather in Vienna? Answer only with the forecast."
+              }
+            ],
+            "include": [],
+            "tools": [
+              {
+                "type": "function",
+                "name": "lookup_weather",
+                "description": "Look up the weather forecast for a city.",
+                "parameters": {
+                  "type": "object",
+                  "properties": {
+                    "city": { "type": "string" }
+                  },
+                  "required": ["city"],
+                  "additionalProperties": false
+                },
+                "strict": false
+              }
+            ],
+            "temperature": 0,
+            "tool_choice": "required",
+            "stream": false
+          }
+        }
+      },
+      "response": {
+        "status": 200,
+        "statusText": "OK",
+        "headers": {
+          "content-type": "application/json",
+          "openai-version": "2020-10-01",
+          "x-request-id": "req_a1b2c3d4e5f67890"
+        },
+        "body": {
+          "kind": "json",
+          "value": {
+            "id": "resp_a1b2c3d4e5f67890abcdef01",
+            "object": "response",
+            "created_at": 1777420003,
+            "model": "gpt-4o-mini-2024-07-18",
+            "status": "completed",
+            "completed_at": 1777420004,
+            "output": [
+              {
+                "id": "fc_a1b2c3d4e5f67890",
+                "type": "function_call",
+                "call_id": "call_a1b2c3d4e5f67890abcdef",
+                "name": "lookup_weather",
+                "arguments": "{\"city\":\"Vienna\"}",
+                "status": "completed"
+              }
+            ],
+            "usage": {
+              "input_tokens": 85,
+              "output_tokens": 16,
+              "total_tokens": 101,
+              "input_tokens_details": { "cached_tokens": 0 },
+              "output_tokens_details": { "reasoning_tokens": 0 }
+            },
+            "error": null,
+            "incomplete_details": null
+          }
+        }
+      }
+    },
+    {
+      "id": "b2c3d4e5f6789012",
+      "matchKey": "POST api.openai.com/v1/responses",
+      "callIndex": 1,
+      "recordedAt": "2026-05-19T00:00:01.000Z",
+      "request": {
+        "method": "POST",
+        "url": "https://api.openai.com/v1/responses",
+        "headers": {
+          "accept": "application/json",
+          "authorization": "[REDACTED]",
+          "content-type": "application/json",
+          "user-agent": "Agents/JavaScript 0.0.14",
+          "x-stainless-arch": "arm64",
+          "x-stainless-lang": "js",
+          "x-stainless-os": "MacOS",
+          "x-stainless-package-version": "5.23.2",
+          "x-stainless-retry-count": "0",
+          "x-stainless-runtime": "node",
+          "x-stainless-runtime-version": "v26.0.0"
+        },
+        "body": {
+          "kind": "json",
+          "value": {
+            "instructions": "Use the lookup_weather tool exactly once, then answer only with the forecast.",
+            "model": "gpt-4o-mini",
+            "input": [
+              {
+                "role": "user",
+                "content": "What is the weather in Vienna? Answer only with the forecast."
+              },
+              {
+                "id": "fc_a1b2c3d4e5f67890",
+                "type": "function_call",
+                "call_id": "call_a1b2c3d4e5f67890abcdef",
+                "name": "lookup_weather",
+                "arguments": "{\"city\":\"Vienna\"}",
+                "status": "completed"
+              },
+              {
+                "type": "function_call_output",
+                "call_id": "call_a1b2c3d4e5f67890abcdef",
+                "output": "Sunny in Vienna"
+              }
+            ],
+            "include": [],
+            "tools": [
+              {
+                "type": "function",
+                "name": "lookup_weather",
+                "description": "Look up the weather forecast for a city.",
+                "parameters": {
+                  "type": "object",
+                  "properties": {
+                    "city": { "type": "string" }
+                  },
+                  "required": ["city"],
+                  "additionalProperties": false
+                },
+                "strict": false
+              }
+            ],
+            "temperature": 0,
+            "stream": false
+          }
+        }
+      },
+      "response": {
+        "status": 200,
+        "statusText": "OK",
+        "headers": {
+          "content-type": "application/json",
+          "openai-version": "2020-10-01",
+          "x-request-id": "req_b2c3d4e5f6789012"
+        },
+        "body": {
+          "kind": "json",
+          "value": {
+            "id": "resp_b2c3d4e5f6789012abcdef02",
+            "object": "response",
+            "created_at": 1777420005,
+            "model": "gpt-4o-mini-2024-07-18",
+            "status": "completed",
+            "completed_at": 1777420006,
+            "output": [
+              {
+                "id": "msg_b2c3d4e5f6789012",
+                "type": "message",
+                "role": "assistant",
+                "status": "completed",
+                "content": [
+                  {
+                    "type": "output_text",
+                    "text": "Sunny in Vienna"
+                  }
+                ]
+              }
+            ],
+            "usage": {
+              "input_tokens": 110,
+              "output_tokens": 4,
+              "total_tokens": 114,
+              "input_tokens_details": { "cached_tokens": 0 },
+              "output_tokens_details": { "reasoning_tokens": 0 }
+            },
+            "error": null,
+            "incomplete_details": null
+          }
+        }
+      }
+    }
+  ]
+}

e2e/scenarios/openai-agents-instrumentation/assertions.ts

@@ -0,0 +1,117 @@
+import { beforeAll, describe, expect, test } from "vitest";
+import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server";
+import {
+  withScenarioHarness,
+  type ScenarioRunContext,
+} from "../../helpers/scenario-harness";
+import {
+  findChildSpans,
+  findLatestChildSpan,
+  findLatestSpan,
+} from "../../helpers/trace-selectors";
+import {
+  AGENT_NAME,
+  FINAL_OUTPUT,
+  MODEL_NAME,
+  OPERATION_NAME,
+  ROOT_NAME,
+  SCENARIO_NAME,
+  TOOL_NAME,
+} from "./constants.mjs";
+
+type RunOpenAIAgentsScenario = (harness: {
+  runNodeScenarioDir: (options: {
+    entry: string;
+    env?: Record<string, string>;
+    nodeArgs: string[];
+    runContext?: ScenarioRunContext;
+    scenarioDir: string;
+    timeoutMs: number;
+  }) => Promise<unknown>;
+}) => Promise<void>;
+
+function findModelSpans(
+  events: CapturedLogEvent[],
+  parentId: string | undefined,
+): CapturedLogEvent[] {
+  return [
+    ...findChildSpans(events, "Response", parentId),
+    ...findChildSpans(events, "Generation", parentId),
+  ];
+}
+
+export function defineOpenAIAgentsAutoInstrumentationAssertions(options: {
+  name: string;
+  runScenario: RunOpenAIAgentsScenario;
+  timeoutMs: number;
+}): void {
+  describe(options.name, () => {
+    let events: CapturedLogEvent[] = [];
+
+    beforeAll(async () => {
+      await withScenarioHarness(async (harness) => {
+        await options.runScenario(harness);
+        events = harness.events();
+      });
+    }, options.timeoutMs);
+
+    test(
+      "captures OpenAI Agents spans through the auto-hook setup",
+      { timeout: options.timeoutMs },
+      () => {
+        const root = findLatestSpan(events, ROOT_NAME);
+        const operation = findLatestSpan(events, OPERATION_NAME);
+        const workflow = findLatestChildSpan(
+          events,
+          "Agent workflow",
+          operation?.span.id,
+        );
+        const agent = findLatestChildSpan(
+          events,
+          AGENT_NAME,
+          workflow?.span.id,
+        );
+        const modelSpans = findModelSpans(events, agent?.span.id);
+        const toolSpan = findLatestChildSpan(events, TOOL_NAME, agent?.span.id);
+
+        expect(root).toBeDefined();
+        expect(root?.row.metadata).toMatchObject({
+          scenario: SCENARIO_NAME,
+        });
+        expect(operation).toBeDefined();
+        expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]);
+
+        expect(workflow).toBeDefined();
+        expect(workflow?.span.type).toBe("task");
+        expect(workflow?.span.parentIds).toEqual([operation?.span.id ?? ""]);
+
+        expect(agent).toBeDefined();
+        expect(agent?.span.type).toBe("task");
+        expect(agent?.row.metadata).toMatchObject({
+          tools: [TOOL_NAME],
+          output_type: "text",
+        });
+
+        expect(modelSpans.length).toBeGreaterThanOrEqual(1);
+        for (const modelSpan of modelSpans) {
+          expect(modelSpan.span.type).toBe("llm");
+          expect(String(modelSpan.row.metadata?.model)).toContain(MODEL_NAME);
+          expect(modelSpan.metrics).toMatchObject({
+            completion_tokens: expect.any(Number),
+            prompt_tokens: expect.any(Number),
+            tokens: expect.any(Number),
+          });
+          expect(modelSpan.input).toEqual(
+            expect.arrayContaining([expect.anything()]),
+          );
+          expect(modelSpan.output).toBeDefined();
+        }
+
+        expect(toolSpan).toBeDefined();
+        expect(toolSpan?.span.type).toBe("tool");
+        expect(toolSpan?.input).toBe(JSON.stringify({ city: "Vienna" }));
+        expect(toolSpan?.output).toBe(FINAL_OUTPUT);
+      },
+    );
+  });
+}

e2e/scenarios/openai-agents-instrumentation/cassette-filter.mjs

@@ -0,0 +1,12 @@
+// @ts-check
+/** @type {import("@braintrust/seinfeld").FilterSpec} */
+export const filter = [
+  "default",
+  {
+    ignoreBodyFields: [
+      // Ignore all body fields — deterministic call order makes callIndex
+      // the sole discriminator, which is stable across SDK releases.
+      "**",
+    ],
+  },
+];

e2e/scenarios/openai-agents-instrumentation/constants.mjs

@@ -0,0 +1,7 @@
+export const ROOT_NAME = "openai-agents-auto-instrumentation-root";
+export const SCENARIO_NAME = "openai-agents-instrumentation";
+export const OPERATION_NAME = "openai-agents-run-operation";
+export const AGENT_NAME = "Weather Agent";
+export const MODEL_NAME = "gpt-4o-mini";
+export const TOOL_NAME = "lookup_weather";
+export const FINAL_OUTPUT = "Sunny in Vienna";

e2e/scenarios/openai-agents-instrumentation/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "openai-agents-instrumentation-scenario",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "@openai/agents": "0.0.14",
+    "zod": "3.25.67"
+  },
+  "braintrustScenario": {
+    "canary": {
+      "dependencies": {
+        "@openai/agents": "latest",
+        "zod": "zod@^4.0.0"
+      }
+    }
+  }
+}