From 2cbc13b473436e0b7e60470dce21e28981736193 Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Sat, 10 Oct 2026 12:20:00 -0400 Subject: [PATCH 1/2] feat(eve): label every attachment and inline only verified media Signed-off-by: Andrew Barba --- .../attachments-labels-verified-media.md | 5 + docs/sandbox/index.mdx | 8 +- .../attachment-staging.integration.test.ts | 133 +++++++++++++++--- .../eve/src/harness/attachment-staging.ts | 57 ++++---- packages/eve/src/harness/token-estimate.ts | 14 +- packages/eve/src/harness/tool-loop.test.ts | 22 ++- .../internal/attachments/media-metadata.ts | 47 +++++++ .../src/internal/attachments/sandbox-refs.ts | 25 +++- 8 files changed, 249 insertions(+), 62 deletions(-) create mode 100644 .changeset/attachments-labels-verified-media.md diff --git a/.changeset/attachments-labels-verified-media.md b/.changeset/attachments-labels-verified-media.md new file mode 100644 index 0000000000..11b51d556a --- /dev/null +++ b/.changeset/attachments-labels-verified-media.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Every attachment now reaches the model with a label that names its sandbox path, including images and PDFs that also arrive as bytes, so the agent can pass a visible file to `bash` or open it again. eve checks image and PDF media types against the file's bytes. Only PNG, JPEG, GIF, and WebP images up to 3 MiB and 8000 pixels per side, and PDFs up to 20 MiB, go to the model as bytes, and tool results follow the same rule. A HEIC photo, an oversized screenshot, or a mislabeled file renders as its label instead of failing the model call. Live sessions pay one prompt-cache rewrite after upgrading. diff --git a/docs/sandbox/index.mdx b/docs/sandbox/index.mdx index 0e5471e8da..cbad0511bc 100644 --- a/docs/sandbox/index.mdx +++ b/docs/sandbox/index.mdx @@ -153,10 +153,12 @@ Docker and microsandbox mount the compiled trees read-only under `/eve/resources eve writes byte-backed files that reach the agent into the sandbox under `/workspace/.eve/attachments/` and keeps only a reference in session history. This covers files attached to user messages and files that tools return to the model with `toolOutputPart.file` or as MCP image, audio, and blob content. -Before every model call, eve reads the referenced files back from the sandbox: +Before every model call, eve reads the referenced files back from the sandbox. Every file, whether attached to a message or returned by a tool, reaches the model with a label that names its sandbox path, such as `Attached file /workspace/.eve/attachments//report.csv (text/csv)`. The agent can open that path with `read_file` or `bash`. The file's bytes follow its label only when every major provider reads the format natively: -- An image up to 3 MiB or a PDF up to 20 MiB attached to a message reaches the model as a file. Larger files and other types reach the model as a text reference, such as `Attached file /workspace/.eve/attachments//report.csv (text/csv)`, that the agent can open with `read_file` or `bash`. -- A file returned by a tool always reaches the model as a file. +- A PNG, JPEG, GIF, or WebP image up to 3 MiB and at most 8000 pixels per side. +- A PDF up to 20 MiB. + +Other files, such as CSV, HEIC, audio, and video, reach the model as the label alone. eve checks image and PDF media types against the file's bytes when it stages the file. A file whose bytes don't match its declared image or PDF type is staged as `application/octet-stream`, so a provider never receives bytes it would reject. If a referenced file is missing, or its bytes no longer match what eve staged because something in the sandbox overwrote it, the model receives a `FileNotFound` note in its place. diff --git a/packages/eve/src/harness/attachment-staging.integration.test.ts b/packages/eve/src/harness/attachment-staging.integration.test.ts index 043d1ff65a..a359e03624 100644 --- a/packages/eve/src/harness/attachment-staging.integration.test.ts +++ b/packages/eve/src/harness/attachment-staging.integration.test.ts @@ -1,4 +1,4 @@ -import type { FilePart, ModelMessage, UserContent } from "ai"; +import type { FilePart, ModelMessage, ToolResultPart, UserContent } from "ai"; import { describe, expect, it, vi } from "vitest"; import type { ChannelAdapter, FetchFileResult } from "#channel/adapter.js"; @@ -11,8 +11,10 @@ import { ATTACHMENTS_ROOT, hydrateSandboxAttachments, stageAttachmentsToSandbox, + stageToolResultMedia, } from "#harness/attachment-staging.js"; import { captureLogRecords } from "#internal/testing/log-records.js"; +import { pngBytes } from "#internal/testing/media-fixtures.js"; import { requestPublicUrl } from "#execution/web-fetch/request.js"; vi.mock("#execution/web-fetch/request.js", () => ({ requestPublicUrl: vi.fn() })); @@ -248,7 +250,7 @@ describe("stageAttachmentsToSandbox (integration)", () => { }); it("refines FilePart.mediaType when fetchFile returns a FetchFileResult", async () => { - const resolvedBytes = Buffer.from("PNGDATA", "utf8"); + const resolvedBytes = pngBytes(1, 1); const adapter: ChannelAdapter = { async fetchFile() { const result: FetchFileResult = { @@ -310,7 +312,7 @@ describe("stageAttachmentsToSandbox (integration)", () => { it("downloads a link no channel resolver claims, so the provider never fetches it", async () => { vi.mocked(requestPublicUrl).mockResolvedValueOnce( - new Response("PNG", { headers: { "content-type": "image/png" } }), + new Response(pngBytes(1, 1), { headers: { "content-type": "image/png" } }), ); const adapter: ChannelAdapter = { kind: "custom-channel", state: {} }; const sandbox = mockSandbox({ id: "sbx_public_link" }); @@ -597,13 +599,10 @@ describe("stageAttachmentsToSandbox (integration)", () => { }); describe("hydrateSandboxAttachments (integration)", () => { - // 1 KiB of PNG-like bytes. Small enough to fall under the image - // inline-cap, arbitrary enough that the byte-equality assertion - // catches any corruption on the sandbox round trip. - const smallImageBytes = Buffer.alloc(1024, 0x89); - // 1 KiB of PDF-like bytes. Small enough to fall under the PDF - // inline-cap. - const smallPdfBytes = Buffer.alloc(1024, 0x25); + // A small PNG and PDF under the inline caps; the byte-equality assertions + // catch any corruption on the sandbox round trip. + const smallImageBytes = pngBytes(32, 32, 1024); + const smallPdfBytes = pdfBytes(1024); it("hydrates small images inline as bytes — provider consumes them multimodally", async () => { const sandbox = mockSandbox({ id: "sbx_hydrate_image" }); @@ -641,6 +640,12 @@ describe("hydrateSandboxAttachments (integration)", () => { expect(hydratedFilePart.filename).toMatch( /^\/workspace\/\.eve\/attachments\/[0-9a-f]{16}\/logo\.png$/, ); + // A label naming the sandbox path precedes the bytes. + expect(hydratedContent).toEqual([ + { type: "text", text: "describe the image" }, + { text: `Attached file ${hydratedFilePart.filename} (image/png)`, type: "text" }, + hydratedFilePart, + ]); }); it("hydrates small PDFs inline as bytes — provider handles them natively", async () => { @@ -709,7 +714,7 @@ describe("hydrateSandboxAttachments (integration)", () => { const runtime = await createTestRuntime(); // One byte over the 3 MiB cap so the ref size fails the inline // check without allocating two massive buffers in the test. - const oversizedImage = Buffer.alloc(3 * 1024 * 1024 + 1, 0x89); + const oversizedImage = pngBytes(32, 32, 3 * 1024 * 1024 + 1); const stagedContent = (await runtime.runAsSession({ sandbox }, async () => stageAttachmentsToSandbox([ @@ -735,7 +740,7 @@ describe("hydrateSandboxAttachments (integration)", () => { it("treats oversized PDFs (>20 MiB) as non-inlinable — renders a text reference instead of bytes", async () => { const sandbox = mockSandbox({ id: "sbx_hydrate_big_pdf" }); const runtime = await createTestRuntime(); - const oversizedPdf = Buffer.alloc(20 * 1024 * 1024 + 1, 0x25); + const oversizedPdf = pdfBytes(20 * 1024 * 1024 + 1); const stagedContent = (await runtime.runAsSession({ sandbox }, async () => stageAttachmentsToSandbox([ @@ -794,6 +799,94 @@ describe("hydrateSandboxAttachments (integration)", () => { }); }); + it("inlines only images whose bytes prove a format every provider reads", async () => { + const sandbox = mockSandbox({ id: "sbx_verified_media" }); + const runtime = await createTestRuntime(); + + const staged = (await runtime.runAsSession({ sandbox }, async () => + stageAttachmentsToSandbox([ + { + data: Buffer.from("not a png"), + filename: "fake.png", + mediaType: "image/png", + type: "file", + }, + { + data: Buffer.from("ftypheic"), + filename: "photo.heic", + mediaType: "image/heic", + type: "file", + }, + { data: pngBytes(8001, 10), filename: "wide.png", mediaType: "image/png", type: "file" }, + { data: pngBytes(4, 4), filename: "real.jpg", mediaType: "image/jpeg", type: "file" }, + ] as UserContent), + )) as FilePart[]; + const hydrated = await runtime.runAsSession({ sandbox }, async () => + hydrateSandboxAttachments([{ content: staged, role: "user" }]), + ); + + expect(staged.map((part) => part.mediaType)).toEqual([ + "application/octet-stream", + "image/heic", + "image/png", + "image/png", + ]); + expect(hydrated[0]?.content).toEqual([ + { text: `Attached file ${staged[0]!.filename} (application/octet-stream)`, type: "text" }, + { text: `Attached file ${staged[1]!.filename} (image/heic)`, type: "text" }, + { text: `Attached file ${staged[2]!.filename} (image/png)`, type: "text" }, + { text: `Attached file ${staged[3]!.filename} (image/png)`, type: "text" }, + expect.objectContaining({ mediaType: "image/png", type: "file" }), + ]); + }); + + it("applies the same rule to files a tool returns", async () => { + const sandbox = mockSandbox({ id: "sbx_tool_gate" }); + const runtime = await createTestRuntime(); + const file = (data: Buffer, filename: string) => ({ + data: { data: data.toString("base64"), type: "data" as const }, + filename, + mediaType: "image/png", + type: "file" as const, + }); + const messages: ModelMessage[] = [ + { + content: [ + { + output: { + type: "content", + value: [file(pngBytes(9000, 9000), "huge.png"), file(pngBytes(8, 8), "shot.png")], + }, + toolCallId: "shot-1", + toolName: "screenshot", + type: "tool-result", + }, + ], + role: "tool", + }, + ]; + + const hydrated = await runtime.runAsSession({ sandbox }, async () => + hydrateSandboxAttachments(await stageToolResultMedia(messages)), + ); + + const [result] = hydrated[0]!.content as ToolResultPart[]; + expect(result!.output).toEqual({ + type: "content", + value: [ + { + text: expect.stringMatching(/^Attached file .*\/huge\.png \(image\/png\)$/), + type: "text", + }, + { + text: expect.stringMatching(/^Attached file .*\/shot\.png \(image\/png\)$/), + type: "text", + }, + expect.objectContaining({ filename: "shot.png", type: "file" }), + ], + }); + }); + it("renders a link or eve-url marker left in history by an older release as a note", async () => { const runtime = await createTestRuntime(); const messages: ModelMessage[] = [ @@ -959,18 +1052,16 @@ describe("hydrateSandboxAttachments (integration)", () => { const stagedContent = (await runtime.runAsSession({ sandbox }, async () => stageAttachmentsToSandbox([ - { - data: Buffer.from("original"), - filename: "logo.png", - mediaType: "image/png", - type: "file", - }, + { data: smallImageBytes, filename: "logo.png", mediaType: "image/png", type: "file" }, ] as UserContent), )) as UserContent; const stagedPath = (stagedContent[0] as FilePart).filename as string; // Same size, different bytes: only the content address can tell them apart. - await sandbox.session.writeBinaryFile({ content: Buffer.from("replaced"), path: stagedPath }); + await sandbox.session.writeBinaryFile({ + content: Buffer.alloc(smallImageBytes.byteLength, 0x41), + path: stagedPath, + }); const messages = [{ content: stagedContent, role: "user" as const }]; const hydrated = await runtime.runAsSession({ sandbox }, async () => @@ -1012,3 +1103,7 @@ describe("hydrateSandboxAttachments (integration)", () => { } }); }); + +function pdfBytes(paddingBytes: number): Buffer { + return Buffer.concat([Buffer.from("%PDF-1.4\n"), Buffer.alloc(paddingBytes, 0x25)]); +} diff --git a/packages/eve/src/harness/attachment-staging.ts b/packages/eve/src/harness/attachment-staging.ts index acf8cac8aa..8217fb271f 100644 --- a/packages/eve/src/harness/attachment-staging.ts +++ b/packages/eve/src/harness/attachment-staging.ts @@ -13,7 +13,7 @@ import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js"; import { isUnresolvedFileData, readFileData } from "#internal/attachments/data.js"; import { EveAttachmentError } from "#internal/attachments/errors.js"; import { createLogger } from "#internal/logging.js"; -import { readMediaMetadata } from "#internal/attachments/media-metadata.js"; +import { readMediaMetadata, verifyMediaType } from "#internal/attachments/media-metadata.js"; import { createPublicDownloads, type PublicDownloads } from "#internal/attachments/public-link.js"; import { deserializeUrlFilePart, isSerializedUrlFilePart } from "#internal/attachments/url-refs.js"; import { @@ -165,9 +165,9 @@ export async function stageToolResultMedia( /** * Hydrates `eve-sandbox:` file refs for a single model call. * - * Tool-result refs always hydrate as bytes: the tool chose to show them to - * the model. Inbound attachments inline small images and PDFs; larger or - * unsupported files become text references to their sandbox path. Every + * Every attachment, inbound or returned by a tool, renders as a label naming + * its sandbox path. Small verified images and PDFs follow their label as + * bytes; anything else stays a label the agent can open with tools. Every * decision is pure in the ref, so each message renders identically on every * call and the provider's prompt cache stays valid. The returned messages * must not be written back to session history, which stays ref-only. @@ -332,25 +332,27 @@ async function hydrateMessageContent(content: unknown, sandbox: SandboxSession): if (!Array.isArray(content)) { return content; } - return Promise.all( - content.map(async (part) => { + const hydrated = await Promise.all( + content.map(async (part): Promise => { if (isSandboxRefFilePart(part)) { const ref = decodeSandboxRef(part.data as URL); + const label = renderSandboxRefAsTextPart(ref); if (!inlinesSandboxRefAsBytes(ref)) { - return renderSandboxRefAsTextPart(ref); + return [label]; } const bytes = await readSandboxRefBytes(ref, sandbox); return bytes === null - ? renderMissingSandboxRef(ref) - : { ...part, data: bytes, mediaType: ref.mediaType }; + ? [renderMissingSandboxRef(ref)] + : [label, { ...part, data: bytes, mediaType: ref.mediaType }]; } if ((part as { type?: unknown }).type === "tool-result") { const toolResult = part as ToolResultPart; - return { ...toolResult, output: await hydrateToolOutput(toolResult.output, sandbox) }; + return [{ ...toolResult, output: await hydrateToolOutput(toolResult.output, sandbox) }]; } - return part; + return [part]; }), ); + return hydrated.flat(); } async function hydrateToolOutput( @@ -361,18 +363,20 @@ async function hydrateToolOutput( return output; } const value = await Promise.all( - output.value.map(async (part): Promise => { - if (!isToolOutputRefFile(part) || part.data.type !== "url") return part; + output.value.map(async (part): Promise => { + if (!isToolOutputRefFile(part) || part.data.type !== "url") return [part]; const ref = decodeSandboxRef(part.data.url); + const label = renderSandboxRefAsTextPart(ref); + if (!inlinesSandboxRefAsBytes(ref)) return [label]; const bytes = await readSandboxRefBytes(ref, sandbox); - if (bytes === null) return renderMissingSandboxRef(ref); - return { - ...part, - data: { data: Buffer.from(bytes).toString("base64"), type: "data" }, - }; + if (bytes === null) return [renderMissingSandboxRef(ref)]; + return [ + label, + { ...part, data: { data: Buffer.from(bytes).toString("base64"), type: "data" } }, + ]; }), ); - return { ...output, value }; + return { ...output, value: value.flat() }; } async function readSandboxRefBytes( @@ -459,8 +463,9 @@ function createReturnedFilesMessage(files: readonly FilePart[]): ModelMessage { } /** - * Renders a sandbox-resident attachment as a {@link TextPart} the model - * can use to reach the payload through filesystem tools. + * Labels a sandbox-resident attachment with a {@link TextPart} the model can + * use to reach the payload through filesystem tools, whether or not the + * bytes follow. * * Matches the text shape produced by the compaction summarizer for * `FilePart`s so the model sees one consistent surface for "there is a @@ -530,16 +535,20 @@ async function stageResolvedBytes( resolved.filename ?? part.filename, sandbox, ); - return { ...part, data: encodeSandboxRef(ref), filename: ref.path, mediaType }; + return { ...part, data: encodeSandboxRef(ref), filename: ref.path, mediaType: ref.mediaType }; } -/** Writes content-addressed bytes under {@link ATTACHMENTS_ROOT} and describes them as a ref. */ +/** + * Writes content-addressed bytes under {@link ATTACHMENTS_ROOT} and describes + * them as a ref whose media type the bytes confirm. + */ async function writeSandboxRef( bytes: Buffer, - mediaType: string, + declaredMediaType: string, filename: string | undefined, sandbox: SandboxSession, ): Promise { + const mediaType = verifyMediaType(bytes, declaredMediaType); const sha = sha256Prefix(bytes); const authored = `${ATTACHMENTS_ROOT}/${sha}/${safeFilename(filename, sha, mediaType)}`; await sandbox.writeBinaryFile({ content: bytes, path: authored }); diff --git a/packages/eve/src/harness/token-estimate.ts b/packages/eve/src/harness/token-estimate.ts index 5aa5a5f5c7..c5e8663eb4 100644 --- a/packages/eve/src/harness/token-estimate.ts +++ b/packages/eve/src/harness/token-estimate.ts @@ -65,11 +65,7 @@ function estimateFilePartTokens(value: unknown): number | undefined { } function estimateFileDataTokens(data: unknown, mediaType: string): number | undefined { - if (isSandboxRefUrl(data)) { - // Inbound attachments that hydrate as text references stay text. - const ref = decodeSandboxRef(data); - return inlinesSandboxRefAsBytes(ref) ? estimateMediaTokens(ref) : undefined; - } + if (isSandboxRefUrl(data)) return estimateSandboxRefTokens(data); if (data instanceof URL) return estimateRemoteTokens(mediaType); if (isTaggedFileData(data)) { switch (data.type) { @@ -77,7 +73,7 @@ function estimateFileDataTokens(data: unknown, mediaType: string): number | unde return estimateInlineTokens(data.data, mediaType); case "url": return isSandboxRefUrl(data.url) - ? estimateMediaTokens(decodeSandboxRef(data.url)) + ? estimateSandboxRefTokens(data.url) : estimateRemoteTokens(mediaType); case "reference": return estimateRemoteTokens(mediaType); @@ -88,6 +84,12 @@ function estimateFileDataTokens(data: unknown, mediaType: string): number | unde return estimateInlineTokens(data, mediaType); } +/** Attachments that render as their label alone stay text. */ +function estimateSandboxRefTokens(url: URL): number | undefined { + const ref = decodeSandboxRef(url); + return inlinesSandboxRefAsBytes(ref) ? estimateMediaTokens(ref) : undefined; +} + function isTaggedFileData( value: unknown, ): value is { readonly type: string; readonly data?: unknown; readonly url?: unknown } { diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts index 3c211323ab..8b4999cc02 100644 --- a/packages/eve/src/harness/tool-loop.test.ts +++ b/packages/eve/src/harness/tool-loop.test.ts @@ -10482,9 +10482,9 @@ describe("createToolLoopHarness", () => { toolResults: [], }); - // Small PNG-like payload — under the 3 MiB image inline cap so - // the hydration pass substitutes bytes (not a text reference). - const imageBytes = Buffer.alloc(1024, 0x89); + // A small PNG under the 3 MiB image inline cap, so the hydration pass + // adds its bytes after the label. + const imageBytes = pngBytes(64, 64, 1024); const userContent: UserContent = [ { type: "text", text: "describe the image" }, { data: imageBytes, filename: "logo.png", mediaType: "image/png", type: "file" }, @@ -10618,7 +10618,21 @@ describe("createToolLoopHarness", () => { expect(laterCalls).toHaveLength(2); for (const call of laterCalls) { expect(call.messages.find((message) => message.role === "tool")?.content).toEqual([ - toolResult, + { + ...toolResult, + output: { + type: "content", + value: [ + { + text: expect.stringMatching( + /^Attached file \/workspace\/\.eve\/attachments\/[0-9a-f]{16}\/chart\.png \(image\/png\)$/, + ), + type: "text", + }, + file, + ], + }, + }, ]); } }); diff --git a/packages/eve/src/internal/attachments/media-metadata.ts b/packages/eve/src/internal/attachments/media-metadata.ts index 0a8d5abaef..ae81558ac9 100644 --- a/packages/eve/src/internal/attachments/media-metadata.ts +++ b/packages/eve/src/internal/attachments/media-metadata.ts @@ -55,6 +55,53 @@ export function estimateMediaTokens(metadata: MediaMetadata): number { return Math.ceil(metadata.size / 3); } +const GENERIC_MEDIA_TYPE = "application/octet-stream"; +const PDF_MEDIA_TYPE = "application/pdf"; +// The PDF header may follow up to 1 KiB of leading bytes. +const PDF_HEADER_WINDOW = 1024; + +/** + * The media type a staged file carries: the format its bytes prove for an + * image, PDF, or untyped file, or the declared type otherwise. A declared + * PNG, JPEG, GIF, WebP, or PDF that the bytes don't confirm becomes + * `application/octet-stream`, so no provider receives bytes it would reject. + */ +export function verifyMediaType(bytes: Uint8Array, declared: string): string { + const normalized = declared.toLowerCase(); + const checkable = + normalized.startsWith("image/") || + normalized === PDF_MEDIA_TYPE || + normalized === GENERIC_MEDIA_TYPE; + if (!checkable) return declared; + const detected = + detectImageMediaType(bytes) ?? (hasPdfHeader(bytes) ? PDF_MEDIA_TYPE : undefined); + if (detected !== undefined) return detected; + return VERIFIABLE_MEDIA_TYPES.has(normalized) ? GENERIC_MEDIA_TYPE : declared; +} + +/** The image formats eve can verify from bytes, and that every major provider reads. */ +export const INLINE_IMAGE_MEDIA_TYPES: ReadonlySet = new Set([ + "image/gif", + "image/jpeg", + "image/png", + "image/webp", +]); + +const VERIFIABLE_MEDIA_TYPES: ReadonlySet = new Set([ + ...INLINE_IMAGE_MEDIA_TYPES, + "image/jpg", + PDF_MEDIA_TYPE, +]); + +function hasPdfHeader(bytes: Uint8Array): boolean { + const head = Buffer.from( + bytes.buffer, + bytes.byteOffset, + Math.min(bytes.byteLength, PDF_HEADER_WINDOW), + ); + return head.includes("%PDF-", 0, "latin1"); +} + /** Names the PNG, GIF, JPEG, or WebP format its leading bytes identify. */ export function detectImageMediaType(bytes: Uint8Array): string | undefined { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); diff --git a/packages/eve/src/internal/attachments/sandbox-refs.ts b/packages/eve/src/internal/attachments/sandbox-refs.ts index 2e93b33802..0557a642b4 100644 --- a/packages/eve/src/internal/attachments/sandbox-refs.ts +++ b/packages/eve/src/internal/attachments/sandbox-refs.ts @@ -9,7 +9,10 @@ * ``` */ -import type { MediaMetadata } from "#internal/attachments/media-metadata.js"; +import { + INLINE_IMAGE_MEDIA_TYPES, + type MediaMetadata, +} from "#internal/attachments/media-metadata.js"; /** * Custom URL scheme used by every sandbox-resident attachment ref. The @@ -28,6 +31,9 @@ const METADATA_QUERY_KEYS = ["width", "height", "pages"] as const; */ const INLINE_IMAGE_MAX_BYTES = 3 * 1024 * 1024; +/** Providers reject an image wider or taller than this, failing the whole request. */ +const INLINE_IMAGE_MAX_SIDE_PX = 8000; + /** Upper bound, in bytes, on inbound PDFs that hydrate as inline bytes. */ const INLINE_PDF_MAX_BYTES = 20 * 1024 * 1024; @@ -116,13 +122,20 @@ export function decodeSandboxRef(value: URL | string): SandboxRef { } /** - * Whether an inbound attachment ref reaches the model as bytes rather than a - * text reference. Only shapes every major provider reads natively qualify: - * images up to 3 MiB and PDFs up to 20 MiB. Pure in the ref, so a message - * renders the same way on every model call. + * Whether an attachment ref reaches the model as bytes as well as its label. + * Only shapes every major provider reads natively qualify: PNG, JPEG, GIF, + * and WebP images up to 3 MiB and 8000 px per side, and PDFs up to 20 MiB. + * Staging verifies these media types from the bytes. Pure in the ref, so a + * message renders the same way on every model call. */ export function inlinesSandboxRefAsBytes(ref: SandboxRef): boolean { - if (ref.mediaType.startsWith("image/")) return ref.size <= INLINE_IMAGE_MAX_BYTES; + if (INLINE_IMAGE_MEDIA_TYPES.has(ref.mediaType)) { + return ( + ref.size <= INLINE_IMAGE_MAX_BYTES && + (ref.width ?? 0) <= INLINE_IMAGE_MAX_SIDE_PX && + (ref.height ?? 0) <= INLINE_IMAGE_MAX_SIDE_PX + ); + } if (ref.mediaType === "application/pdf") return ref.size <= INLINE_PDF_MAX_BYTES; return false; } From aa86601690aeee6c3a973480821486494359d90a Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Sat, 10 Oct 2026 12:45:55 -0400 Subject: [PATCH 2/2] fix(eve): send tool-result files under their verified type and keep the compaction eval inlined Signed-off-by: Andrew Barba --- .../agent/agent.ts | 2 +- .../agent/tools/emit-compaction-content.ts | 20 ++++++++++++++----- .../attachment-staging.integration.test.ts | 16 ++++++++++++--- .../eve/src/harness/attachment-staging.ts | 6 +++++- .../internal/attachments/media-metadata.ts | 11 +++------- 5 files changed, 37 insertions(+), 18 deletions(-) diff --git a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts index 9ce1fc68af..2865082298 100644 --- a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts +++ b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts @@ -110,7 +110,7 @@ const taskModel = mockModel({ toolText.includes(CONTENT_OUTPUT_LEAD_MARKER) ? "LEAD_PRESERVED" : "LEAD_LOST", // The stub names the file's sandbox path, which ends in its filename. toolText.includes("Attached file ") && - toolText.includes(`${CONTENT_OUTPUT_FILENAME} (application/octet-stream)`) + toolText.includes(`${CONTENT_OUTPUT_FILENAME} (image/png)`) ? "FILE_STUB_RENDERED" : "FILE_STUB_LOST", request.userMessages.some((text) => text.includes("[case: content-output-file-stub]")) diff --git a/e2e/fixtures/agent-compaction-regressions/agent/tools/emit-compaction-content.ts b/e2e/fixtures/agent-compaction-regressions/agent/tools/emit-compaction-content.ts index 180a058f0f..104919df44 100644 --- a/e2e/fixtures/agent-compaction-regressions/agent/tools/emit-compaction-content.ts +++ b/e2e/fixtures/agent-compaction-regressions/agent/tools/emit-compaction-content.ts @@ -8,10 +8,20 @@ import { CONTENT_OUTPUT_TAIL_MARKER, } from "../../constants"; -// The canary sits within the first 2000 serialized characters, where the old -// prefix-clipping rendering would expose it to the compaction model. The rest -// of the payload pushes the trailing text far past that clip budget. -const INLINE_FILE_BASE64 = `${"A".repeat(400)}${CONTENT_OUTPUT_PAYLOAD_CANARY}${"A".repeat(11_600)}`; +// A PNG signature and 2000x2000 IHDR chunk (33 bytes, so 44 base64 characters +// with no padding) make eve verify the file as an image it inlines, so the +// payload reaches the model and weighs on history like any screenshot. The +// canary sits within the first 2000 serialized characters, where the old +// prefix-clipping rendering would expose it to the compaction model, and on a +// base64 group boundary so it survives eve's decode and re-encode unchanged. +// The rest of the payload pushes the trailing text far past that clip budget. +const PNG_HEADER = Buffer.from( + "89504e470d0a1a0a" + "0000000d49484452" + "000007d0000007d00806000000" + "00000000", + "hex", +); +const INLINE_FILE_BASE64 = + `${PNG_HEADER.toString("base64")}${"A".repeat(400)}` + + `${CONTENT_OUTPUT_PAYLOAD_CANARY}${"A".repeat(11_599)}`; export default defineTool({ description: @@ -28,7 +38,7 @@ export default defineTool({ toolOutputPart.text(`Alice's completed review note: ${CONTENT_OUTPUT_LEAD_MARKER}`), toolOutputPart.file(INLINE_FILE_BASE64, { filename: CONTENT_OUTPUT_FILENAME, - mediaType: "application/octet-stream", + mediaType: "image/png", }), // "The attachment's filename" is deliberately not spelled out: the // checkpoint can only carry it by reading the rendered stub. diff --git a/packages/eve/src/harness/attachment-staging.integration.test.ts b/packages/eve/src/harness/attachment-staging.integration.test.ts index a359e03624..979411340d 100644 --- a/packages/eve/src/harness/attachment-staging.integration.test.ts +++ b/packages/eve/src/harness/attachment-staging.integration.test.ts @@ -843,10 +843,10 @@ describe("hydrateSandboxAttachments (integration)", () => { it("applies the same rule to files a tool returns", async () => { const sandbox = mockSandbox({ id: "sbx_tool_gate" }); const runtime = await createTestRuntime(); - const file = (data: Buffer, filename: string) => ({ + const file = (data: Buffer, filename: string, mediaType = "image/png") => ({ data: { data: data.toString("base64"), type: "data" as const }, filename, - mediaType: "image/png", + mediaType, type: "file" as const, }); const messages: ModelMessage[] = [ @@ -855,7 +855,11 @@ describe("hydrateSandboxAttachments (integration)", () => { { output: { type: "content", - value: [file(pngBytes(9000, 9000), "huge.png"), file(pngBytes(8, 8), "shot.png")], + value: [ + file(pngBytes(9000, 9000), "huge.png"), + file(pngBytes(8, 8), "shot.png"), + file(pngBytes(8, 8), "mislabeled.jpg", "image/jpeg"), + ], }, toolCallId: "shot-1", toolName: "screenshot", @@ -883,6 +887,12 @@ describe("hydrateSandboxAttachments (integration)", () => { type: "text", }, expect.objectContaining({ filename: "shot.png", type: "file" }), + { + text: expect.stringMatching(/^Attached file .*\/mislabeled\.jpg \(image\/png\)$/), + type: "text", + }, + // The bytes go to the provider under the type they prove. + expect.objectContaining({ filename: "mislabeled.jpg", mediaType: "image/png" }), ], }); }); diff --git a/packages/eve/src/harness/attachment-staging.ts b/packages/eve/src/harness/attachment-staging.ts index 8217fb271f..d9d7814e9e 100644 --- a/packages/eve/src/harness/attachment-staging.ts +++ b/packages/eve/src/harness/attachment-staging.ts @@ -304,7 +304,11 @@ async function stageToolOutputFiles( }; } const ref = await writeSandboxRef(data.bytes, part.mediaType, part.filename, sandbox); - return { ...part, data: { type: "url" as const, url: encodeSandboxRef(ref) } }; + return { + ...part, + data: { type: "url" as const, url: encodeSandboxRef(ref) }, + mediaType: ref.mediaType, + }; }), ); return { ...output, value }; diff --git a/packages/eve/src/internal/attachments/media-metadata.ts b/packages/eve/src/internal/attachments/media-metadata.ts index ae81558ac9..94c0604d2f 100644 --- a/packages/eve/src/internal/attachments/media-metadata.ts +++ b/packages/eve/src/internal/attachments/media-metadata.ts @@ -57,8 +57,6 @@ export function estimateMediaTokens(metadata: MediaMetadata): number { const GENERIC_MEDIA_TYPE = "application/octet-stream"; const PDF_MEDIA_TYPE = "application/pdf"; -// The PDF header may follow up to 1 KiB of leading bytes. -const PDF_HEADER_WINDOW = 1024; /** * The media type a staged file carries: the format its bytes prove for an @@ -93,13 +91,10 @@ const VERIFIABLE_MEDIA_TYPES: ReadonlySet = new Set([ PDF_MEDIA_TYPE, ]); +// The format allows leading bytes before the header, but providers expect it +// at the start, and a header deeper in would mark text that mentions it. function hasPdfHeader(bytes: Uint8Array): boolean { - const head = Buffer.from( - bytes.buffer, - bytes.byteOffset, - Math.min(bytes.byteLength, PDF_HEADER_WINDOW), - ); - return head.includes("%PDF-", 0, "latin1"); + return readAscii(bytes, 0, 5) === "%PDF-"; } /** Names the PNG, GIF, JPEG, or WebP format its leading bytes identify. */