diff --git a/.github/workflows/workshop-evals.yml b/.github/workflows/workshop-evals.yml new file mode 100644 index 000000000..063ae2380 --- /dev/null +++ b/.github/workflows/workshop-evals.yml @@ -0,0 +1,95 @@ +name: Workshop evals + +on: + workflow_dispatch: + inputs: + models: + description: Comma-separated Workshop model IDs + default: "@cf/zai-org/glm-5.2,@cf/moonshotai/kimi-k2.7-code" + required: true + trials: + description: Repetitions per task and model + default: "3" + required: true + +permissions: + contents: read + +jobs: + evals: + name: shard ${{ matrix.shard }}/3 + runs-on: ubuntu-latest + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Enable Corepack + run: corepack enable + + - name: Set up Vite+, Node.js and dependencies + uses: voidzero-dev/setup-vp@313600b80b104eadebb9111787d37a2e83e014ca # v1.17.0 + with: + node-version: "24.19.0" + cache: true + run-install: true + + - name: Build + run: pnpm build + + - name: Run eval shard + id: run + continue-on-error: true + working-directory: packages/workshop-evals + env: + CF_AI_GATEWAY: ${{ secrets.CF_AI_GATEWAY_NAME }} + CF_AI_GATEWAY_ACCOUNT_ID: ${{ secrets.CF_AI_GATEWAY_ACCOUNT_ID }} + CF_AI_GATEWAY_API_TOKEN: ${{ secrets.CF_AI_GATEWAY_TOKEN }} + WORKSHOP_EVAL_MODELS: ${{ inputs.models }} + WORKSHOP_EVAL_TRIALS: ${{ inputs.trials }} + run: | + pnpm exec vitest run --config vitest.eval.config.ts \ + --shard=${{ matrix.shard }}/3 \ + --reporter=vitest-evals/reporter \ + --reporter=json \ + --outputFile.json=.wrangler/evals/results-${{ matrix.shard }}.json + + - name: Require eval results + run: test -f packages/workshop-evals/.wrangler/evals/results-${{ matrix.shard }}.json + + - name: Upload eval results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: workshop-evals-${{ matrix.shard }} + path: packages/workshop-evals/.wrangler/evals/results-${{ matrix.shard }}.json + retention-days: 30 + + report: + name: report + if: always() + needs: evals + runs-on: ubuntu-latest + permissions: + contents: read + checks: write + steps: + - name: Download eval results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: workshop-evals-* + path: eval-results + merge-multiple: true + + - name: Publish combined eval report + uses: getsentry/vitest-evals@d37153f06d7d086393927c57a2faaeff464a55e2 # v0 + with: + results: eval-results/*.json + publish-check: true + check-name: Workshop evals + fail-on-failures: true diff --git a/docs/testing-strategy.md b/docs/testing-strategy.md new file mode 100644 index 000000000..a9bdb4e56 --- /dev/null +++ b/docs/testing-strategy.md @@ -0,0 +1,105 @@ +# Agent evaluation + +This document defines how we evaluate the Workshop agent. It covers live agent evals and the +integration-test code that they reuse. + +## Goal + +An eval gives the production agent a natural-language task. The agent runs in a real Workshop and +builds a real Gadget. The verifier then calls that Gadget's RPC and checks the requested behavior. + +The score measures what the agent delivered. Tool calls, time, errors, tokens, and cost explain the +result. They do not replace behavioral verification. + +## One scenario, two targets + +Every portable scenario can run against either target: + +- **Local** starts the Workshop, its Durable Objects, and the Worker Loader under local workerd. +- **Preview** connects through a deployed preview router, Cloudflare Access, and the preview's real + service bindings. + +The task prompt and verifier do not contain target-specific code. Preview runs therefore act as +functional end-to-end tests of a deployed user scenario. Local runs provide faster iteration, more +trials, and controlled failure testing. + +Some low-level tests remain local because the public API cannot abort an arbitrary Durable Object or +seed invalid internal state. They reuse the same integration-test primitives but are not duplicate +agent scenarios. + +## Ownership + +`packages/integration-tests` owns the Cloudflare-specific runtime bridge: + +- start a multi-Worker workerd instance +- connect to the public Cap'n Web API +- authenticate a fresh local user or an Access preview identity +- drive one chat across multiple turns +- read canonical history and workpieces +- connect to generated Gadget RPCs +- accept the current chat's proposed changes + +`vitest-evals` owns generic evaluation infrastructure: + +- normalized transcripts, tools, traces, errors, usage, and timings +- scored judges +- local report UI +- GitHub summaries, annotations, gates, and sharded report reduction + +`packages/workshop-evals` contains only the thin bridge between those layers and the authored Gadget +scenarios. It does not implement another report format, statistics package, trace viewer, or shard +reducer. + +## Tasks and checks + +A task contains one or more prompts and a verifier. Later turns share the same workspace and chat. +They must re-check earlier requirements when a new feature could break them. + +Checks observe outcomes through stable interfaces. Examples include: + +- arithmetic and validation through Gadget RPC +- no overselling under concurrent RPC calls +- stored state surviving a later turn's code changes +- a later turn preserving behavior from an earlier turn +- creation of a real standard Doc rather than a custom imitation + +A check must not require one implementation technique when several correct techniques exist. + +## Scores and diagnostics + +Each trial reports: + +- full task pass or failure +- fraction of behavioral checks passed +- model and target +- total wall time +- time spent in each agent turn and verifier +- model turns +- tool calls and failed tool calls +- agent errors +- provider token and cost metadata when the Workshop exposes them + +A tool error does not fail a task if the agent recovers and delivers the requested behavior. A +platform or provider failure remains visible as an invalid or failed run and must not silently become +a model score. + +Repeated trials are required before comparing models or harnesses. Results must identify the model, +harness commit, target, task version, and trial. A model comparison holds the harness fixed. A +harness comparison holds the model fixed. + +## Running evals + +`pnpm evals` builds the workspace and runs the suite locally. Set `WORKSHOP_EVAL_TARGET` to an Access +preview URL and provide `CF_ACCESS_TOKEN` to run the same scenarios against that deployment. + +The GitHub workflow has only `workflow_dispatch`. It never runs on a pull request, push, merge, or +schedule automatically. The caller selects models and trial count. + +The workflow uses existing AI Gateway secrets. A deployed preview uses its own configured Workers AI +binding and model catalog. + +## Results + +Vitest JSON is the durable interchange format for one run. `vitest-evals` renders it locally and +combines CI shards. A separate experiment service can ingest the normalized rows for long-term model +and harness leaderboards. That service is not part of the Workshop execution bridge. diff --git a/package.json b/package.json index 39bf22ca0..70b5b0bc7 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,8 @@ "packageManager": "pnpm@11.17.0+sha512.cca3cea332ad254bb84145f966d19f4879615210346fc92c79a047f23a0d7b3cca3c3792f0076ba1f1831d277efbcf0a9119b31a9a60eca7fb3d6231f331ef72", "scripts": { "build": "vp run -r --cache build", + "evals": "pnpm build && pnpm --filter @gadgets/workshop-evals evals", + "evals:ui": "pnpm --filter @gadgets/workshop-evals evals:ui", "run-local": "node scripts/run-local.ts", "test": "node --test 'scripts/**/*.test.ts' && vp run --filter '!cloudflare-os' --cache test", "preview:config": "node scripts/preview/preview.ts config", diff --git a/packages/workshop-evals/README.md b/packages/workshop-evals/README.md new file mode 100644 index 000000000..1bc7c399f --- /dev/null +++ b/packages/workshop-evals/README.md @@ -0,0 +1,78 @@ +# Workshop agent evals + +These evals run the production Workshop agent, execute the Gadget it creates, and verify the result +through the Gadget's real RPC. + +The package uses the workerd and Cap'n Web helpers from `@gadgets/integration-tests`. `vitest-evals` +owns transcript normalization, scoring, reports, traces, and CI result reduction. + +Live cases stay under `evals/**/*.eval.ts`. The package's cached `test` task includes only +`src/**/*.test.ts`, so `pnpm test` never runs a model. The root `pnpm evals` script builds first, then +invokes the live package script directly; model credentials do not pass through a cached Vite+ task. + +## Run locally + +Use an existing AI Gateway configuration: + +```sh +export CF_AI_GATEWAY=... +export CF_AI_GATEWAY_ACCOUNT_ID=... +export CF_AI_GATEWAY_API_TOKEN=... +pnpm evals +``` + +A Wrangler OAuth token can call Workers AI directly for local development: + +```sh +export CLOUDFLARE_ACCOUNT_ID=... +export CLOUDFLARE_API_TOKEN=... +pnpm evals +``` + +Optional controls: + +```sh +export WORKSHOP_EVAL_MODELS='@cf/zai-org/glm-5.2,@cf/moonshotai/kimi-k2.7-code' +export WORKSHOP_EVAL_TRIALS=3 +``` + +The defaults are both models and one trial. + +## Run against a preview + +Provide the preview router URL and an Access application JWT for the user that runs the scenarios: + +```sh +export WORKSHOP_EVAL_TARGET='https://pr123-branch-router.example.workers.dev' +export CF_ACCESS_TOKEN=... +pnpm evals +``` + +The preview supplies its own model catalog and Workers AI binding. The runner does not need separate +model credentials in this mode. The same task prompts and verifiers run locally and on the preview. + +## Inspect results + +The run writes `.wrangler/evals/results.json` in this package. Open the report UI with: + +```sh +pnpm evals:ui +``` + +The report includes behavioral scores, transcripts, tool calls and errors, timings, usage, and target +metadata. + +## CI + +The **Workshop evals** workflow is manual-only. Start it with `workflow_dispatch` and choose the model +list and trial count. It uses existing repository AI Gateway credentials, native Vitest sharding, and +the official `getsentry/vitest-evals` action to publish one combined report. + +## Add a scenario + +1. Add `tasks/.task.ts` with a prompt and RPC verifier. +2. Add `evals/.eval.ts` that imports the task and calls `defineTaskEval(task)`. + +A verifier should check user-visible behavior and stable RPC contracts. It should not require a +particular implementation technique. Multi-turn tasks should re-check earlier behavior after each +extension. diff --git a/packages/workshop-evals/evals/appointment-desk.eval.ts b/packages/workshop-evals/evals/appointment-desk.eval.ts new file mode 100644 index 000000000..a8c9f79c3 --- /dev/null +++ b/packages/workshop-evals/evals/appointment-desk.eval.ts @@ -0,0 +1,4 @@ +import { defineTaskEval } from "../src/eval.js"; +import task from "../tasks/appointment-desk.task.js"; + +defineTaskEval(task); diff --git a/packages/workshop-evals/evals/expense-ledger.eval.ts b/packages/workshop-evals/evals/expense-ledger.eval.ts new file mode 100644 index 000000000..67f298433 --- /dev/null +++ b/packages/workshop-evals/evals/expense-ledger.eval.ts @@ -0,0 +1,4 @@ +import { defineTaskEval } from "../src/eval.js"; +import task from "../tasks/expense-ledger.task.js"; + +defineTaskEval(task); diff --git a/packages/workshop-evals/evals/project-doc.eval.ts b/packages/workshop-evals/evals/project-doc.eval.ts new file mode 100644 index 000000000..8d71d5f97 --- /dev/null +++ b/packages/workshop-evals/evals/project-doc.eval.ts @@ -0,0 +1,4 @@ +import { defineTaskEval } from "../src/eval.js"; +import task from "../tasks/project-doc.task.js"; + +defineTaskEval(task); diff --git a/packages/workshop-evals/package.json b/packages/workshop-evals/package.json new file mode 100644 index 000000000..8ae181365 --- /dev/null +++ b/packages/workshop-evals/package.json @@ -0,0 +1,26 @@ +{ + "name": "@gadgets/workshop-evals", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc --noEmit", + "test:run": "vitest run --config vitest.config.ts", + "test:watch": "vitest --config vitest.config.ts", + "evals": "vitest run --config vitest.eval.config.ts --reporter=vitest-evals/reporter --reporter=json --outputFile.json=.wrangler/evals/results.json", + "evals:ui": "vitest-evals serve .wrangler/evals/results.json" + }, + "dependencies": { + "@gadgets/integration-tests": "workspace:*", + "@gadgets/workshop-shared": "workspace:*", + "capnweb": "catalog:", + "vitest-evals": "0.16.1", + "zod": "^4.4.3" + }, + "devDependencies": { + "@cloudflare/workers-types": "^5.20260808.1", + "@types/node": "^26.1.0", + "typescript": "catalog:", + "vitest": "catalog:" + } +} diff --git a/packages/workshop-evals/src/config.test.ts b/packages/workshop-evals/src/config.test.ts new file mode 100644 index 000000000..56bf29682 --- /dev/null +++ b/packages/workshop-evals/src/config.test.ts @@ -0,0 +1,24 @@ +import { expect, it } from "vitest"; +import { EVAL_RUN_BUDGET_MS, EVAL_TEST_TIMEOUT_MS, evalMatrix } from "./config.js"; + +it("reserves cleanup time outside the agent run budget", () => { + expect(EVAL_TEST_TIMEOUT_MS).toBeGreaterThan(EVAL_RUN_BUDGET_MS); +}); + +it("uses both Workers AI models and one trial by default", () => { + expect(evalMatrix({})).toEqual({ + models: ["@cf/zai-org/glm-5.2", "@cf/moonshotai/kimi-k2.7-code"], + trials: 1, + }); +}); + +it("accepts model and trial overrides", () => { + expect(evalMatrix({ + WORKSHOP_EVAL_MODELS: " model-a, model-b ", + WORKSHOP_EVAL_TRIALS: "3", + })).toEqual({ models: ["model-a", "model-b"], trials: 3 }); +}); + +it("rejects an invalid trial count", () => { + expect(() => evalMatrix({ WORKSHOP_EVAL_TRIALS: "0" })).toThrow("positive integer"); +}); diff --git a/packages/workshop-evals/src/config.ts b/packages/workshop-evals/src/config.ts new file mode 100644 index 000000000..d20b2d822 --- /dev/null +++ b/packages/workshop-evals/src/config.ts @@ -0,0 +1,26 @@ +/** Models and repetitions expanded with native Vitest case tables. */ +export type EvalMatrix = { + models: string[]; + trials: number; +}; + +const DEFAULT_MODELS = ["@cf/zai-org/glm-5.2", "@cf/moonshotai/kimi-k2.7-code"]; + +/** Budget shared by all agent turns and verification inside one trial. */ +export const EVAL_RUN_BUDGET_MS = 30 * 60_000; +/** Outer Vitest deadline; cleanup gets two minutes after the run budget expires. */ +export const EVAL_TEST_TIMEOUT_MS = EVAL_RUN_BUDGET_MS + 2 * 60_000; + +/** Parse non-secret eval controls. Model credentials belong to the selected target. */ +export function evalMatrix(environment: NodeJS.ProcessEnv = process.env): EvalMatrix { + const models = (environment.WORKSHOP_EVAL_MODELS ?? "") + .split(",") + .map(model => model.trim()) + .filter(Boolean); + const rawTrials = environment.WORKSHOP_EVAL_TRIALS?.trim(); + const trials = rawTrials === undefined || rawTrials === "" ? 1 : Number(rawTrials); + if (!Number.isInteger(trials) || trials < 1) { + throw new Error("WORKSHOP_EVAL_TRIALS must be a positive integer"); + } + return { models: models.length > 0 ? models : [...DEFAULT_MODELS], trials }; +} diff --git a/packages/workshop-evals/src/eval.ts b/packages/workshop-evals/src/eval.ts new file mode 100644 index 000000000..5f3817670 --- /dev/null +++ b/packages/workshop-evals/src/eval.ts @@ -0,0 +1,44 @@ +import { createJudge, describeEval } from "vitest-evals"; +import { expect } from "vitest"; +import { evalMatrix } from "./config.js"; +import { createWorkshopHarness } from "./harness.js"; +import type { EvalRunInput, EvalRunOutput, EvalTask } from "./task.js"; +import { resolveWorkshopTarget } from "./target.js"; + +const ChecksJudge = createJudge( + "behavioral checks", + ({ output }) => { + const checks = output.turns.flatMap(turn => turn.checks); + const passed = checks.filter(check => check.pass).length; + return { + score: checks.length === 0 ? 0 : passed / checks.length, + metadata: { + passed, + total: checks.length, + failed: checks.filter(check => !check.pass).map(check => check.id), + }, + }; + }, +); + +/** Register one real Gadget task using native Vitest cases and vitest-evals reporting. */ +export function defineTaskEval(task: EvalTask): void { + const target = resolveWorkshopTarget(); + const matrix = evalMatrix(); + const cases = matrix.models.flatMap(model => + Array.from({ length: matrix.trials }, (_unused, trial) => ({ + name: `${model} | trial ${trial + 1}`, + model, + trial, + }))); + const harness = createWorkshopHarness(task, target); + + describeEval(task.id, { harness }, it => { + it.for(cases)("$name", async ({ model, trial }, { run }) => { + const result = await run({ model, trial }); + await expect(result).toSatisfyJudge(ChecksJudge, { + threshold: task.expectation === "required" ? 1 : null, + }); + }); + }); +} diff --git a/packages/workshop-evals/src/harness.ts b/packages/workshop-evals/src/harness.ts new file mode 100644 index 000000000..5b0488a1e --- /dev/null +++ b/packages/workshop-evals/src/harness.ts @@ -0,0 +1,102 @@ +import type { AgentTurnResult } from "@gadgets/integration-tests/agent-session"; +import type { AiChatMessage, WorkpieceSummary } from "@gadgets/workshop-shared/api"; +import { createHarness } from "vitest-evals"; +import { EVAL_RUN_BUDGET_MS } from "./config.js"; +import type { EvalRunInput, EvalRunOutput, EvalTask, EvalTurnResult } from "./task.js"; +import { measureHistory, toTranscriptEvents } from "./transcript.js"; +import { openWorkshopTarget } from "./target.js"; +import type { WorkshopTarget } from "./target.js"; +import { EvalVerifier } from "./verifier.js"; + +const HARNESS_OVERHEAD_MS = 2 * 60_000; + + +/** Adapt one real Workshop task to the generic vitest-evals harness contract. */ +export function createWorkshopHarness(task: EvalTask, target: WorkshopTarget) { + return createHarness({ + name: "workshop-agent", + run: async ({ input, signal, setArtifact }) => { + const startedAt = Date.now(); + const turnTimeoutMs = Math.floor( + (EVAL_RUN_BUDGET_MS - HARNESS_OVERHEAD_MS) / task.turns.length, + ); + setArtifact("scenario", { + id: task.id, + title: task.title, + expectation: task.expectation, + model: input.model, + trial: input.trial, + target: target.kind, + }); + + const opened = await openWorkshopTarget(target, input.model, turnTimeoutMs); + try { + const turns: EvalTurnResult[] = []; + let history: AiChatMessage[] = []; + let workpieces: WorkpieceSummary[] = []; + let chatId: number | undefined; + let usage: AgentTurnResult["usage"] = {}; + + for (const [index, turn] of task.turns.entries()) { + const agentStartedAt = Date.now(); + const result = await opened.session.run(turn.prompt, { signal }); + const agentDurationMs = Date.now() - agentStartedAt; + ({ chatId, history, workpieces } = result); + usage = result.usage; + + const verificationStartedAt = Date.now(); + const verifier = new EvalVerifier(opened.session, result.chatId, result.workpieces); + turns.push({ + index, + checks: await verifier.collect(turn.verify), + agentDurationMs, + verificationDurationMs: Date.now() - verificationStartedAt, + }); + } + if (chatId === undefined) throw new Error(`Eval task ${task.id} has no turns`); + + const checks = turns.flatMap(turn => turn.checks); + const metrics = measureHistory(history); + const output: EvalRunOutput = { + taskId: task.id, + taskTitle: task.title, + expectation: task.expectation, + target: target.kind, + model: input.model, + trial: input.trial, + workspaceId: opened.session.workspaceId, + chatId, + passed: checks.length > 0 && checks.every(check => check.pass), + turns, + workpieces: workpieces.map(({ id, type, title }) => ({ id, type, title })), + metrics, + }; + + return { + output, + events: toTranscriptEvents(history), + usage: { + provider: "cloudflare", + model: input.model, + totalTokens: usage.totalTokens, + toolCalls: metrics.toolCalls, + metadata: { + ...(usage.costUsd === undefined ? {} : { costUsd: usage.costUsd }), + tokenScope: "last-model-step", + }, + }, + timings: { totalMs: Date.now() - startedAt }, + errors: history.flatMap(message => + message.type === "error" ? [{ name: "AgentError", message: message.message }] : []), + metadata: { + taskId: task.id, + expectation: task.expectation, + target: target.kind, + }, + }; + } finally { + await opened.close(); + } + }, + }); +} diff --git a/packages/workshop-evals/src/target.test.ts b/packages/workshop-evals/src/target.test.ts new file mode 100644 index 000000000..c6c2c9469 --- /dev/null +++ b/packages/workshop-evals/src/target.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { resolveWorkshopTarget } from "./target.js"; + +describe("resolveWorkshopTarget", () => { + it("uses an Access-authenticated preview when a target URL is configured", () => { + expect(resolveWorkshopTarget({ + WORKSHOP_EVAL_TARGET: "https://preview.example.com", + CF_ACCESS_TOKEN: "access-token", + })).toEqual({ + kind: "preview", + url: new URL("https://preview.example.com"), + accessToken: "access-token", + }); + }); + + it("requires an Access token for a preview", () => { + expect(() => resolveWorkshopTarget({ + WORKSHOP_EVAL_TARGET: "https://preview.example.com", + })).toThrow("CF_ACCESS_TOKEN"); + }); + + it("uses the existing AI Gateway configuration for local workerd", () => { + expect(resolveWorkshopTarget({ + CF_AI_GATEWAY: "gateway", + CF_AI_GATEWAY_ACCOUNT_ID: "account", + CF_AI_GATEWAY_API_TOKEN: "token", + })).toEqual({ + kind: "local", + modelAccess: { + kind: "gateway", + gateway: "gateway", + accountId: "account", + apiToken: "token", + }, + }); + }); + + it("uses existing Cloudflare credentials for direct local Workers AI", () => { + expect(resolveWorkshopTarget({ + CLOUDFLARE_ACCOUNT_ID: "account", + CLOUDFLARE_API_TOKEN: "token", + })).toEqual({ + kind: "local", + modelAccess: { kind: "direct", accountId: "account", apiToken: "token" }, + }); + }); + + it("fails clearly when local workerd has no model credentials", () => { + expect(() => resolveWorkshopTarget({})).toThrow("model credentials"); + }); +}); diff --git a/packages/workshop-evals/src/target.ts b/packages/workshop-evals/src/target.ts new file mode 100644 index 000000000..ac12f6718 --- /dev/null +++ b/packages/workshop-evals/src/target.ts @@ -0,0 +1,166 @@ +import { AgentSession } from "@gadgets/integration-tests/agent-session"; +import { startHarness, type WorkerConfig } from "@gadgets/integration-tests/harness"; + +/** Existing credentials used to run models from a local workerd Workshop. */ +export type LocalModelAccess = { + kind: "gateway"; + gateway: string; + accountId: string; + apiToken: string; +} | { + kind: "direct"; + accountId: string; + apiToken: string; +}; + +/** Where one eval trial runs. Scenario definitions are target-independent. */ +export type WorkshopTarget = { + kind: "local"; + modelAccess: LocalModelAccess; +} | { + kind: "preview"; + url: URL; + accessToken: string; +}; + +/** One isolated Workshop session plus its target-specific cleanup. */ +export type OpenedWorkshop = { + session: AgentSession; + close(): Promise; +}; + +function value(environment: NodeJS.ProcessEnv, key: string): string | undefined { + const candidate = environment[key]?.trim(); + return candidate ? candidate : undefined; +} + +/** Resolve the local or deployed-preview target from the eval process environment. */ +export function resolveWorkshopTarget( + environment: NodeJS.ProcessEnv = process.env): WorkshopTarget { + const configuredTarget = value(environment, "WORKSHOP_EVAL_TARGET"); + if (configuredTarget !== undefined && configuredTarget !== "local") { + let url: URL; + try { + url = new URL(configuredTarget); + } catch { + throw new Error("WORKSHOP_EVAL_TARGET must be local or an absolute HTTP(S) URL"); + } + if (url.protocol !== "https:" && url.protocol !== "http:") { + throw new Error("WORKSHOP_EVAL_TARGET must be local or an absolute HTTP(S) URL"); + } + const accessToken = value(environment, "CF_ACCESS_TOKEN"); + if (accessToken === undefined) { + throw new Error("CF_ACCESS_TOKEN is required when WORKSHOP_EVAL_TARGET is a preview URL"); + } + return { kind: "preview", url, accessToken }; + } + + const gateway = value(environment, "CF_AI_GATEWAY"); + const gatewayAccountId = value(environment, "CF_AI_GATEWAY_ACCOUNT_ID"); + const gatewayApiToken = value(environment, "CF_AI_GATEWAY_API_TOKEN"); + const gatewayValues = [gateway, gatewayAccountId, gatewayApiToken]; + if (gatewayValues.some(Boolean)) { + if (!gateway || !gatewayAccountId || !gatewayApiToken) { + throw new Error( + "Local AI Gateway evals require CF_AI_GATEWAY, CF_AI_GATEWAY_ACCOUNT_ID, and " + + "CF_AI_GATEWAY_API_TOKEN together", + ); + } + return { + kind: "local", + modelAccess: { + kind: "gateway", + gateway, + accountId: gatewayAccountId, + apiToken: gatewayApiToken, + }, + }; + } + + const accountId = value(environment, "CLOUDFLARE_ACCOUNT_ID"); + const apiToken = value(environment, "CLOUDFLARE_API_TOKEN"); + if (accountId && apiToken) { + return { kind: "local", modelAccess: { kind: "direct", accountId, apiToken } }; + } + throw new Error( + "Local Workshop evals need existing model credentials: configure the CF_AI_GATEWAY trio or " + + "CLOUDFLARE_ACCOUNT_ID with CLOUDFLARE_API_TOKEN", + ); +} + +function configureGateway(config: WorkerConfig, access: Extract) { + config.vars = { + ...config.vars, + CF_AI_GATEWAY: access.gateway, + CF_AI_GATEWAY_ACCOUNT_ID: access.accountId, + CF_AI_GATEWAY_API_TOKEN: access.apiToken, + CF_AI_GATEWAY_PROVIDERS: "cloudflare", + }; +} + +/** Open one isolated trial against local workerd or a deployed Access preview. */ +export async function openWorkshopTarget( + target: WorkshopTarget, model: string, timeoutMs: number): Promise { + if (target.kind === "preview") { + let session: AgentSession | undefined; + try { + session = await AgentSession.create(target.url, { + accessToken: target.accessToken, + modelId: model, + timeoutMs, + }); + await session.waitForOutputFormats(); + const openedSession = session; + return { + session: openedSession, + close: async () => { + try { + await openedSession.deleteWorkspace(); + } finally { + openedSession[Symbol.dispose](); + } + }, + }; + } catch (error) { + session?.[Symbol.dispose](); + throw error; + } + } + + const modelAccess = target.modelAccess; + const harness = await startHarness({ + gatekeepers: [], + enableGadgetExecution: true, + ...(modelAccess.kind === "gateway" + ? { patchWorkshop: config => configureGateway(config, modelAccess) } + : {}), + }); + try { + const session = await AgentSession.create(harness.url, { + modelId: model, + ...(modelAccess.kind === "direct" ? { + userModel: { + profile: { type: "agent", id: model, name: model }, + config: { + provider: "cloudflare", + model, + accountId: modelAccess.accountId, + apiToken: modelAccess.apiToken, + }, + }, + } : {}), + timeoutMs, + }); + await session.waitForOutputFormats(); + return { + session, + close: async () => { + session[Symbol.dispose](); + await harness.server.close(); + }, + }; + } catch (error) { + await harness.server.close(); + throw error; + } +} diff --git a/packages/workshop-evals/src/task.ts b/packages/workshop-evals/src/task.ts new file mode 100644 index 000000000..0a4fdca82 --- /dev/null +++ b/packages/workshop-evals/src/task.ts @@ -0,0 +1,79 @@ +import type { JsonValue } from "vitest-evals"; +import type { EvalVerifier } from "./verifier.js"; + +/** Whether a task should gate a run or only report frontier capability. */ +export type EvalExpectation = "required" | "frontier"; + +/** Result of one deterministic observation of the Gadget the agent built. */ +export type EvalCheckOutcome = boolean | { + pass: boolean; + evidence?: unknown; +}; + +/** JSON-safe observation retained in the vitest-evals output. */ +export type EvalCheck = { + id: string; + pass: boolean; + evidence?: JsonValue; +}; + +/** One prompt and the behavioral verification performed after it settles. */ +export type EvalTurn = { + prompt: string; + verify(verifier: EvalVerifier): Promise; +}; + +/** A real Workshop task run through the production agent. */ +export type EvalTask = { + id: string; + title: string; + expectation: EvalExpectation; + turns: readonly [EvalTurn, ...EvalTurn[]]; +}; + +const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +/** Validate one authored task at module load, before it can spend inference. */ +export function defineEvalTask(task: EvalTask): EvalTask { + if (!ID_PATTERN.test(task.id)) throw new Error(`Invalid eval task ID ${JSON.stringify(task.id)}`); + if (task.title.trim() === "") throw new Error(`Eval task ${task.id} needs a title`); + task.turns.forEach((turn, index) => { + if (turn.prompt.trim() === "") throw new Error(`Eval task ${task.id} turn ${index} is empty`); + }); + return task; +} + +/** One model repetition of an authored task. */ +export type EvalRunInput = { + model: string; + trial: number; +}; + +/** Checks and timings recorded after one agent turn. */ +export type EvalTurnResult = { + index: number; + checks: EvalCheck[]; + agentDurationMs: number; + verificationDurationMs: number; +}; + +/** Cloudflare-specific result returned through the generic vitest-evals harness. */ +export type EvalRunOutput = { + taskId: string; + taskTitle: string; + expectation: EvalExpectation; + target: "local" | "preview"; + model: string; + trial: number; + workspaceId: string; + chatId: number; + passed: boolean; + turns: EvalTurnResult[]; + workpieces: Array<{ id: number; type: string; title: string }>; + metrics: { + modelTurns: number; + toolCalls: number; + toolErrors: number; + agentErrors: number; + }; +}; diff --git a/packages/workshop-evals/src/transcript.test.ts b/packages/workshop-evals/src/transcript.test.ts new file mode 100644 index 000000000..b1cdf6b3f --- /dev/null +++ b/packages/workshop-evals/src/transcript.test.ts @@ -0,0 +1,55 @@ +import type { AiChatMessage } from "@gadgets/workshop-shared/api"; +import { expect, it } from "vitest"; +import { measureHistory, toTranscriptEvents } from "./transcript.js"; + +const user = { type: "user", id: "user", name: "User" } as const; +const agent = { type: "agent", id: "model", name: "Model" } as const; + +it("normalizes Workshop messages and failed tools for vitest-evals", () => { + const history: AiChatMessage[] = [{ + chatId: 1, + sequence: 0, + timestamp: new Date(0), + author: user, + type: "message", + message: "Build it", + }, { + chatId: 1, + sequence: 1, + timestamp: new Date(1), + author: agent, + type: "message", + message: "Trying", + toolCalls: [{ + toolCallId: "call-1", + toolName: "listBlueprints", + input: {}, + error: "catalog unavailable", + }], + }, { + chatId: 1, + sequence: 2, + timestamp: new Date(2), + author: agent, + type: "error", + message: "model stopped", + }]; + + expect(toTranscriptEvents(history)).toMatchObject([ + { type: "message", role: "user", content: "Build it" }, + { type: "message", role: "assistant", content: "Trying" }, + { type: "tool_call", id: "call-1", name: "listBlueprints" }, + { + type: "tool_result", + toolCallId: "call-1", + name: "listBlueprints", + error: { message: "catalog unavailable" }, + }, + ]); + expect(measureHistory(history)).toEqual({ + modelTurns: 1, + toolCalls: 1, + toolErrors: 1, + agentErrors: 1, + }); +}); diff --git a/packages/workshop-evals/src/transcript.ts b/packages/workshop-evals/src/transcript.ts new file mode 100644 index 000000000..9b72fa833 --- /dev/null +++ b/packages/workshop-evals/src/transcript.ts @@ -0,0 +1,70 @@ +import type { AiChatMessage, AiToolCall } from "@gadgets/workshop-shared/api"; +import { toJsonValue, type JsonValue, type TranscriptEvent } from "vitest-evals"; + +function toolArguments(call: AiToolCall): Record | undefined { + const value = toJsonValue(call.input); + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value + : undefined; +} + +/** Convert canonical Workshop history into the transcript format owned by vitest-evals. */ +export function toTranscriptEvents(history: readonly AiChatMessage[]): TranscriptEvent[] { + const events: TranscriptEvent[] = []; + for (const message of history) { + if (message.type !== "message") continue; + const role = message.author.type === "user" ? "user" + : message.author.type === "agent" ? "assistant" : undefined; + if (role === undefined) continue; + const metadata = { + sequence: message.sequence, + timestamp: message.timestamp.toISOString(), + }; + if (message.message !== "") { + events.push({ type: "message", role, content: message.message, metadata }); + } + if (role !== "assistant") continue; + for (const call of message.toolCalls ?? []) { + const argumentsValue = toolArguments(call); + events.push({ + type: "tool_call", + id: call.toolCallId, + name: call.toolName, + ...(argumentsValue === undefined ? {} : { arguments: argumentsValue }), + metadata, + }); + const output = "output" in call ? toJsonValue(call.output) : undefined; + events.push({ + type: "tool_result", + toolCallId: call.toolCallId, + name: call.toolName, + ...(call.error === undefined + ? (output === undefined ? {} : { content: output }) + : { error: { name: "Error", message: call.error } }), + metadata, + }); + } + } + return events; +} + +/** Product-specific counts not already shown as generic trace fields. */ +export function measureHistory(history: readonly AiChatMessage[]) { + let modelTurns = 0; + let toolCalls = 0; + let toolErrors = 0; + let agentErrors = 0; + for (const message of history) { + if (message.type === "error") { + agentErrors++; + continue; + } + if (message.type !== "message" || message.author.type !== "agent") continue; + modelTurns++; + for (const call of message.toolCalls ?? []) { + toolCalls++; + if (call.error !== undefined) toolErrors++; + } + } + return { modelTurns, toolCalls, toolErrors, agentErrors }; +} diff --git a/packages/workshop-evals/src/verifier.test.ts b/packages/workshop-evals/src/verifier.test.ts new file mode 100644 index 000000000..cea039ef4 --- /dev/null +++ b/packages/workshop-evals/src/verifier.test.ts @@ -0,0 +1,102 @@ +import type { WorkpieceSummary } from "@gadgets/workshop-shared/api"; +import { describe, expect, it } from "vitest"; +import { EvalVerifier, resolveGadget, type VerifierSession } from "./verifier.js"; +import type { EvalCheck } from "./task.js"; + +const unusedSession: VerifierSession = { + connectToGadget: () => { throw new Error("connectToGadget is not used by this test"); }, +}; + +function gadget(id: number, title: string): WorkpieceSummary { + return { id, type: "gadget", title }; +} + +function collect( + verify: (verifier: EvalVerifier) => Promise, + workpieces: readonly WorkpieceSummary[] = []): Promise { + return new EvalVerifier(unusedSession, 1, workpieces).collect(verify); +} + +describe("resolveGadget", () => { + it("resolves an exact title", () => { + expect(resolveGadget([gadget(3, "Split"), gadget(4, "Shelf")], "Shelf")).toBe(4); + }); + + it("reports what was built when the title is missing", () => { + expect(() => resolveGadget([gadget(3, "Splitter")], "Split")) + .toThrow('found 0 among ["Splitter"]'); + }); + + it("refuses an ambiguous title", () => { + expect(() => resolveGadget([gadget(3, "Split"), gadget(4, "Split")], "Split")).toThrow("found 2"); + }); +}); + +describe("EvalVerifier", () => { + it("records outcomes with and without evidence", async () => { + expect(await collect(async verifier => { + await verifier.check("bare", async () => true); + await verifier.check("detailed", async () => ({ pass: false, evidence: { seen: 2 } })); + })).toEqual([ + { id: "bare", pass: true }, + { id: "detailed", pass: false, evidence: { seen: 2 } }, + ]); + }); + + it("records a throw inside a check as a failure and keeps going", async () => { + expect(await collect(async verifier => { + await verifier.check("explodes", async () => { throw new Error("RPC refused"); }); + await verifier.check("still-runs", async () => true); + })).toEqual([ + { id: "explodes", pass: false, evidence: "RPC refused" }, + { id: "still-runs", pass: true }, + ]); + }); + + it("keeps the observations made before the verifier body itself threw", async () => { + const checks = await collect(async verifier => { + await verifier.check("recorded", async () => true); + throw new Error("resolveGadget failed outside a check"); + }); + expect(checks.at(0)).toEqual({ id: "recorded", pass: true }); + expect(checks.at(1)).toMatchObject({ id: "verifier.threw", pass: false }); + }); + + it("keeps registration order when checks run concurrently", async () => { + const checks = await collect(async verifier => { + await Promise.all([ + verifier.check("slow", async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return true; + }), + verifier.check("fast", async () => true), + ]); + }); + expect(checks.map(check => check.id)).toEqual(["slow", "fast"]); + }); + + it("settles checks the author started without awaiting", async () => { + expect(await collect(async verifier => { + void verifier.check("forgotten", async () => { + await new Promise(resolve => setTimeout(resolve, 5)); + return true; + }); + })).toEqual([{ id: "forgotten", pass: true }]); + }); + + it("marks evidence it could not represent instead of dropping it", async () => { + const checks = await collect(async verifier => { + await verifier.check("cyclic", async () => ({ pass: false, evidence: () => "not json" })); + }); + expect(checks.at(0)?.evidence).toBe(""); + }); + + it("records a duplicate check ID as a failure rather than losing the turn", async () => { + const checks = await collect(async verifier => { + await verifier.check("same", async () => true); + await verifier.check("same", async () => true); + }); + expect(checks.at(0)).toEqual({ id: "same", pass: true }); + expect(checks.at(1)?.evidence).toContain("Duplicate eval check ID"); + }); +}); diff --git a/packages/workshop-evals/src/verifier.ts b/packages/workshop-evals/src/verifier.ts new file mode 100644 index 000000000..1fdb0fd02 --- /dev/null +++ b/packages/workshop-evals/src/verifier.ts @@ -0,0 +1,125 @@ +import type { RpcCompatible, RpcStub } from "capnweb"; +import type { AgentSession } from "@gadgets/integration-tests/agent-session"; +import type { WorkpieceId, WorkpieceSummary } from "@gadgets/workshop-shared/api"; +import { toJsonValue } from "vitest-evals"; +import type { EvalCheck, EvalCheckOutcome } from "./task.js"; + +const EVIDENCE_LIMIT = 2_000; + +/** Recorded when a check's `id` was never reached because the verifier body itself threw. */ +const VERIFIER_THREW = "verifier.threw"; + +// `toJsonValue` answers undefined for anything it cannot represent, which would otherwise be +// indistinguishable from an author who supplied no evidence at all. +const UNSERIALIZABLE_EVIDENCE = ""; + +function describeError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message.length > EVIDENCE_LIMIT ? `${message.slice(0, EVIDENCE_LIMIT)}...` : message; +} + +/** The session capabilities a verifier needs, which is far less than a whole agent session. */ +export type VerifierSession = Pick; + +// Declared signature plus an untyped implementation, matching `connectTyped` in agent-session.ts: +// capnweb's recursive `RpcStub` generic does not survive being forwarded through another generic. +function connectTyped>( + session: VerifierSession, id: WorkpieceId): Promise>; +function connectTyped(session: VerifierSession, id: WorkpieceId) { + return session.connectToGadget(id); +} + +/** + * Task prompts require an exact Gadget title. A mismatch error includes the titles the agent + * created, which makes naming failures diagnosable from the report. + */ +export function resolveGadget( + workpieces: readonly WorkpieceSummary[], title: string): WorkpieceId { + const matches = workpieces.filter( + workpiece => workpiece.type === "gadget" && workpiece.title === title); + const match = matches.at(0); + if (matches.length !== 1 || match === undefined) { + const built = workpieces.map(workpiece => JSON.stringify(workpiece.title)).join(", "); + throw new Error(`Expected exactly one Gadget titled ${JSON.stringify(title)}, ` + + `found ${matches.length} among [${built}]`); + } + return match.id; +} + +/** + * Verifies the agent's provisional branch before a user accepts it. Checks retain registration + * order, and one failed observation does not hide later evidence from the same trial. + */ +export class EvalVerifier { + /** Chat whose provisional branch is under verification. */ + readonly chatId: number; + /** Workpieces visible after the turn settled. */ + readonly workpieces: readonly WorkpieceSummary[]; + readonly #session: VerifierSession; + readonly #checks: EvalCheck[] = []; + readonly #pending: Promise[] = []; + + constructor(session: VerifierSession, chatId: number, workpieces: readonly WorkpieceSummary[]) { + this.#session = session; + this.chatId = chatId; + this.workpieces = workpieces; + } + + /** + * Isolates RPC and schema failures to one observation. An expensive trial still reports every + * independent check it can run. + */ + async check(id: string, body: () => Promise): Promise { + if (this.#checks.some(check => check.id === id)) { + throw new Error(`Duplicate eval check ID ${JSON.stringify(id)} within one turn`); + } + const index = this.#checks.length; + this.#checks.push({ id, pass: false, evidence: "check did not complete" }); + const settled = this.#run(index, id, body); + this.#pending.push(settled); + await settled; + } + + async #run(index: number, id: string, body: () => Promise): Promise { + try { + const outcome = await body(); + if (typeof outcome === "boolean") { + this.#checks[index] = { id, pass: outcome }; + return; + } + const evidence = outcome.evidence === undefined + ? undefined + : toJsonValue(outcome.evidence) ?? UNSERIALIZABLE_EVIDENCE; + this.#checks[index] = { + id, + pass: outcome.pass, + ...(evidence === undefined ? {} : { evidence }), + }; + } catch (error) { + this.#checks[index] = { id, pass: false, evidence: describeError(error) }; + } + } + + /** + * Connects a typed stub to the provisional server of the Gadget with this exact title. The caller + * owns the stub, so bind it with `using`. Throws when no Gadget matches, or when several do. + */ + connect>(gadgetTitle: string): Promise> { + return connectTyped(this.#session, resolveGadget(this.workpieces, gadgetTitle)); + } + + + /** + * Preserves checks recorded before an error escapes the verifier body. The escaped error becomes + * a final failed check instead of erasing the trial's earlier evidence. + */ + async collect(verify: (verifier: EvalVerifier) => Promise): Promise { + try { + await verify(this); + } catch (error) { + this.#checks.push({ id: VERIFIER_THREW, pass: false, evidence: describeError(error) }); + } + await Promise.all(this.#pending); + return this.#checks; + } +} diff --git a/packages/workshop-evals/tasks/appointment-desk.task.ts b/packages/workshop-evals/tasks/appointment-desk.task.ts new file mode 100644 index 000000000..0199552ec --- /dev/null +++ b/packages/workshop-evals/tasks/appointment-desk.task.ts @@ -0,0 +1,160 @@ +import { z } from "zod"; +import { defineEvalTask } from "../src/task.js"; + +const OkSchema = z.discriminatedUnion("ok", [ + z.object({ ok: z.literal(true) }).loose(), + z.object({ ok: z.literal(false), error: z.string().min(1) }).loose(), +]); + +const StatusSchema = z.object({ + capacity: z.number().int().nonnegative(), + bookedCount: z.number().int().nonnegative(), + personIds: z.array(z.string()), +}).loose(); + +interface DeskApi { + defineSlot(input: { slotId: string; startIso: string; capacity: number }): + Promise>; + book(input: { bookingId: string; slotId: string; personId: string }): + Promise>; + cancel(input: { bookingId: string }): Promise>; + slotStatus(input: { slotId: string }): Promise>; +} + +/** Settles a batch of concurrent results into counts of accepted and each rejection code. */ +function tally(results: readonly z.infer[]) { + const accepted = results.filter(result => result.ok).length; + const errors: Record = {}; + for (const result of results) { + if (result.ok) continue; + errors[result.error] = (errors[result.error] ?? 0) + 1; + } + return { accepted, errors }; +} + +/** + * Durable Object RPC calls can interleave at each `await`. A count check followed by an awaited + * insert can therefore oversell. Both shipped format blueprints guard this race, so the task probes + * a platform-relevant capability rather than prompt recall. + * + * The prompt states the capacity invariant without prescribing synchronization. In a measured GLM + * 5.2 run, synchronous SQL plus a BEFORE INSERT trigger passed. Requiring a mutation queue would + * have rejected that correct implementation. + */ +export default defineEvalTask({ + id: "appointment-desk", + title: "Appointment desk under concurrent booking", + expectation: "required", + turns: [{ + prompt: `Build a Gadget named exactly "Desk" for booking appointment slots at a small clinic. +Reception shares the link, so several people book at the same moment from their own phones. A slot +must never be oversold: if it has three places, exactly three bookings can ever succeed, no matter +how many arrive at once. Getting this wrong is the one failure I cannot live with, so make it +impossible rather than unlikely. Store everything in the Gadget's own storage. + +It needs a stable server RPC taking and returning plain data, so I can verify it: + +- defineSlot({ slotId: string, startIso: string, capacity: integer > 0 }) + -> { ok: true } | { ok: false, error: string } + Rejects "DUPLICATE_SLOT" if that slotId already exists. + +- book({ bookingId: string, slotId: string, personId: string }) + -> { ok: true } | { ok: false, error: string } + Reject these, changing nothing, with exactly these error codes: + "UNKNOWN_SLOT" - no slot with that id + "SLOT_FULL" - the slot already holds its capacity in bookings + "DUPLICATE_BOOKING" - a booking with that bookingId already exists + +- cancel({ bookingId: string }) -> { ok: true } | { ok: false, error: string } + Rejects "UNKNOWN_BOOKING". A cancelled booking frees its place for someone else. + +- slotStatus({ slotId: string }) + -> { capacity: integer, bookedCount: integer, personIds: string[] } + personIds lists the people currently holding a place, and its length always equals bookedCount.`, + verify: async verifier => { + await verifier.check("sequential-booking-respects-capacity", async () => { + using api = await verifier.connect("Desk"); + const defined = OkSchema.parse(await api.defineSlot({ + slotId: "seq", startIso: "2026-07-01T09:00:00Z", capacity: 2, + })); + const results = [ + OkSchema.parse(await api.book({ bookingId: "seq-1", slotId: "seq", personId: "ana" })), + OkSchema.parse(await api.book({ bookingId: "seq-2", slotId: "seq", personId: "ben" })), + OkSchema.parse(await api.book({ bookingId: "seq-3", slotId: "seq", personId: "cy" })), + ]; + const status = StatusSchema.parse(await api.slotStatus({ slotId: "seq" })); + return { + pass: defined.ok && + results[0]?.ok === true && results[1]?.ok === true && + results[2]?.ok === false && results[2].error === "SLOT_FULL" && + status.bookedCount === 2 && status.personIds.length === 2, + evidence: { results, status }, + }; + }); + + await verifier.check("cancelling-frees-a-place", async () => { + using api = await verifier.connect("Desk"); + await api.defineSlot({ slotId: "cancel", startIso: "2026-07-02T09:00:00Z", capacity: 1 }); + await api.book({ bookingId: "cancel-1", slotId: "cancel", personId: "ana" }); + const blocked = OkSchema.parse( + await api.book({ bookingId: "cancel-2", slotId: "cancel", personId: "ben" })); + const cancelled = OkSchema.parse(await api.cancel({ bookingId: "cancel-1" })); + const unknown = OkSchema.parse(await api.cancel({ bookingId: "cancel-nope" })); + const reused = OkSchema.parse( + await api.book({ bookingId: "cancel-3", slotId: "cancel", personId: "ben" })); + const status = StatusSchema.parse(await api.slotStatus({ slotId: "cancel" })); + return { + pass: !blocked.ok && blocked.error === "SLOT_FULL" && cancelled.ok && + !unknown.ok && unknown.error === "UNKNOWN_BOOKING" && reused.ok && + status.bookedCount === 1 && status.personIds.join() === "ben", + evidence: { blocked, cancelled, unknown, reused, status }, + }; + }); + + await verifier.check("rejects-unknown-slot", async () => { + using api = await verifier.connect("Desk"); + const result = OkSchema.parse( + await api.book({ bookingId: "ghost", slotId: "no-such-slot", personId: "ana" })); + return { + pass: !result.ok && result.error === "UNKNOWN_SLOT", + evidence: result, + }; + }); + + // Ten bookings dispatched without awaiting arrive as ten interleaved RPC calls, so a + // check-then-write implementation oversells the slot. + await verifier.check("concurrent-booking-never-oversells", async () => { + using api = await verifier.connect("Desk"); + const capacity = 3; + await api.defineSlot({ slotId: "race", startIso: "2026-07-03T09:00:00Z", capacity }); + const results = (await Promise.all(Array.from({ length: 10 }, (_unused, index) => + api.book({ + bookingId: `race-${index}`, slotId: "race", personId: `person-${index}`, + })))).map(result => OkSchema.parse(result)); + const status = StatusSchema.parse(await api.slotStatus({ slotId: "race" })); + const { accepted, errors } = tally(results); + return { + pass: accepted === capacity && status.bookedCount === capacity && + status.personIds.length === capacity && + new Set(status.personIds).size === capacity && + errors.SLOT_FULL === 10 - capacity, + evidence: { accepted, errors, status }, + }; + }); + + await verifier.check("concurrent-duplicate-booking-ids-collapse-to-one", async () => { + using api = await verifier.connect("Desk"); + await api.defineSlot({ slotId: "dupe", startIso: "2026-07-04T09:00:00Z", capacity: 8 }); + const results = (await Promise.all(Array.from({ length: 6 }, () => + api.book({ bookingId: "dupe-same", slotId: "dupe", personId: "ana" })))) + .map(result => OkSchema.parse(result)); + const status = StatusSchema.parse(await api.slotStatus({ slotId: "dupe" })); + const { accepted, errors } = tally(results); + return { + pass: accepted === 1 && errors.DUPLICATE_BOOKING === 5 && status.bookedCount === 1, + evidence: { accepted, errors, status }, + }; + }); + }, + }], +}); diff --git a/packages/workshop-evals/tasks/expense-ledger.task.ts b/packages/workshop-evals/tasks/expense-ledger.task.ts new file mode 100644 index 000000000..f8e99bf96 --- /dev/null +++ b/packages/workshop-evals/tasks/expense-ledger.task.ts @@ -0,0 +1,373 @@ +import { z } from "zod"; +import { defineEvalTask } from "../src/task.js"; + +const OkSchema = z.discriminatedUnion("ok", [ + z.object({ ok: z.literal(true) }).loose(), + z.object({ ok: z.literal(false), error: z.string().min(1) }).loose(), +]); + +const BalancesSchema = z.object({ + perPerson: z.array(z.object({ personId: z.string(), netCents: z.number().int() }).loose()), +}).loose(); + +const SettlementSchema = z.object({ + transfers: z.array(z.object({ + fromId: z.string(), + toId: z.string(), + amountCents: z.number().int().positive(), + }).loose()), +}).loose(); + +const MonthlySchema = z.object({ + totalCents: z.number().int(), + byPayer: z.array(z.object({ + personId: z.string(), + paidCents: z.number().int(), + }).loose()), +}).loose(); + +const BudgetStatusSchema = z.object({ + byCategory: z.array(z.object({ + category: z.string(), + limitCents: z.number().int().nullable(), + spentCents: z.number().int(), + remainingCents: z.number().int().nullable(), + }).loose()), +}).loose(); + +type Expense = { + id: string; + description: string; + payerId: string; + amountCents: number; + splitBetween: string[]; + dateIso: string; + category?: string; +}; + +interface LedgerApi { + addExpense(input: Expense): Promise>; + balances(): Promise>; + settlement(): Promise>; + monthlyTotals(input: { month: string }): Promise>; +} + +interface BudgetedLedgerApi extends LedgerApi { + setBudget(input: { category: string; month: string; limitCents: number }): Promise<{ ok: boolean }>; + budgetStatus(input: { month: string }): Promise>; +} + +/** Net balance for one person, or null when the ledger does not know them. */ +function net(balances: z.infer, personId: string): number | null { + return balances.perPerson.find(entry => entry.personId === personId)?.netCents ?? null; +} + +const TURN_ONE_MONTH = "2026-03"; +const TURN_ONE_IDS = ["survive-1", "survive-2"]; + +/** + * A shared-expense ledger, which is the archetypal thing people build on a platform like this. + * + * Most checks assert an invariant rather than a hand-computed constant: balances sum to zero, + * transfers clear them, and split shares total the expense exactly. An invariant survives an + * arithmetic slip in the task and still catches the bug it describes, such as per-share rounding that + * loses a cent. + */ +export default defineEvalTask({ + id: "expense-ledger", + title: "Household ledger, then budgets", + expectation: "required", + turns: [{ + prompt: `Build a Gadget named exactly "Ledger" for tracking shared household expenses between +flatmates and working out who owes whom. I want to add an expense, see everyone's running balance, +and get a short list of payments that settles us up. Make it comfortable on a phone, since I'll be +adding expenses while standing in a shop. + +All money is integer cents — never floating point. Store everything in the Gadget's own storage so +nothing is lost when the server restarts. + +It also needs a stable server RPC taking and returning plain data, so I can verify it: + +- addExpense({ id: string, description: string, payerId: string, amountCents: integer > 0, + splitBetween: string[], dateIso: "YYYY-MM-DD" }) -> { ok: true } | { ok: false, error: string } + The payer paid amountCents on behalf of everyone in splitBetween, split as evenly as integer + cents allow. When it does not divide evenly, give the leftover cents to the earliest people in + splitBetween order, so the shares always sum to exactly amountCents. + Reject these, changing nothing, with exactly these error codes: + "DUPLICATE_ID" - an expense with that id already exists + "INVALID_AMOUNT" - amountCents is not a positive integer + "EMPTY_SPLIT" - splitBetween is empty + "BAD_DATE" - dateIso is not a YYYY-MM-DD calendar date + +- balances() -> { perPerson: Array<{ personId: string, netCents: integer }> } + netCents is positive when that person is owed money, negative when they owe it. Include everyone + who appears in any expense as a payer or in a split. + +- settlement() -> { transfers: Array<{ fromId: string, toId: string, amountCents: integer > 0 }> } + Payments that bring every balance to zero. Keep it short: never more transfers than one fewer + than the number of people with a non-zero balance. + +- monthlyTotals({ month: "YYYY-MM" }) -> { totalCents: integer, + byPayer: Array<{ personId: string, paidCents: integer }> } + Counting only expenses whose dateIso falls inside that month.`, + verify: async verifier => { + await verifier.check("splits-leftover-cents-without-losing-any", async () => { + using api = await verifier.connect("Ledger"); + // 100 split three ways cannot divide evenly; flooring each share would lose a cent. + const added = await api.addExpense({ + id: "cents-1", + description: "Milk", + payerId: "cents-ana", + amountCents: 100, + splitBetween: ["cents-ana", "cents-ben", "cents-cy"], + dateIso: "2026-05-04", + }); + const balances = BalancesSchema.parse(await api.balances()); + const trio = ["cents-ana", "cents-ben", "cents-cy"].map(id => net(balances, id)); + const [ana, ben, cy] = trio; + return { + pass: OkSchema.parse(added).ok && + // Ana paid 100 and owes the first, largest share of 34, leaving her owed 66. Flooring + // every share to 33 would lose a cent and show up as a 65/-33/-33 that does not balance. + ana === 66 && ben === -33 && cy === -33 && + trio.reduce((sum, value) => (sum ?? 0) + (value ?? 0), 0) === 0, + evidence: { ana, ben, cy }, + }; + }); + + await verifier.check("balances-always-sum-to-zero", async () => { + using api = await verifier.connect("Ledger"); + const added = [ + await api.addExpense({ + id: "sum-1", + description: "Rent", + payerId: "sum-ana", + amountCents: 133_337, + splitBetween: ["sum-ana", "sum-ben", "sum-cy", "sum-dee"], + dateIso: "2026-05-01", + }), + await api.addExpense({ + id: "sum-2", + description: "Wifi", + payerId: "sum-ben", + amountCents: 4_999, + splitBetween: ["sum-ben", "sum-cy"], + dateIso: "2026-05-02", + }), + await api.addExpense({ + id: "sum-3", + description: "Cleaner", + payerId: "sum-cy", + amountCents: 7, + splitBetween: ["sum-ana", "sum-ben", "sum-cy", "sum-dee"], + dateIso: "2026-05-03", + }), + ]; + const balances = BalancesSchema.parse(await api.balances()); + const total = balances.perPerson.reduce((sum, entry) => sum + entry.netCents, 0); + return { + pass: added.every(result => OkSchema.parse(result).ok) && total === 0 && + balances.perPerson.every(entry => Number.isInteger(entry.netCents)), + evidence: { total, perPerson: balances.perPerson }, + }; + }); + + await verifier.check("settlement-clears-every-balance-and-stays-short", async () => { + using api = await verifier.connect("Ledger"); + const balances = BalancesSchema.parse(await api.balances()); + const settlement = SettlementSchema.parse(await api.settlement()); + const remaining = new Map(balances.perPerson.map(entry => [entry.personId, entry.netCents])); + for (const transfer of settlement.transfers) { + remaining.set(transfer.fromId, (remaining.get(transfer.fromId) ?? 0) + transfer.amountCents); + remaining.set(transfer.toId, (remaining.get(transfer.toId) ?? 0) - transfer.amountCents); + } + const owing = balances.perPerson.filter(entry => entry.netCents !== 0).length; + const unsettled = [...remaining].filter(([, value]) => value !== 0); + return { + pass: unsettled.length === 0 && + settlement.transfers.length <= Math.max(0, owing - 1) && + settlement.transfers.every(transfer => transfer.fromId !== transfer.toId), + evidence: { + transfers: settlement.transfers.length, + owing, + unsettled: Object.fromEntries(unsettled), + }, + }; + }); + + await verifier.check("rejects-bad-input-with-named-codes", async () => { + using api = await verifier.connect("Ledger"); + const valid: Expense = { + id: "reject-1", + description: "Beer", + payerId: "reject-ana", + amountCents: 500, + splitBetween: ["reject-ana", "reject-ben"], + dateIso: "2026-05-05", + }; + const accepted = OkSchema.parse(await api.addExpense(valid)); + const before = BalancesSchema.parse(await api.balances()); + const rejections = { + DUPLICATE_ID: OkSchema.parse(await api.addExpense({ ...valid, amountCents: 999 })), + INVALID_AMOUNT: OkSchema.parse( + await api.addExpense({ ...valid, id: "reject-2", amountCents: 0 })), + EMPTY_SPLIT: OkSchema.parse( + await api.addExpense({ ...valid, id: "reject-3", splitBetween: [] })), + BAD_DATE: OkSchema.parse( + await api.addExpense({ ...valid, id: "reject-4", dateIso: "2026-02-31" })), + }; + const after = BalancesSchema.parse(await api.balances()); + const wrong = Object.entries(rejections) + .filter(([code, result]) => result.ok || result.error !== code) + .map(([code, result]) => ({ expected: code, got: result })); + return { + pass: accepted.ok && wrong.length === 0 && + JSON.stringify(before.perPerson) === JSON.stringify(after.perPerson), + evidence: { wrong, changedState: JSON.stringify(before) !== JSON.stringify(after) }, + }; + }); + + await verifier.check("monthly-totals-count-only-that-month", async () => { + using api = await verifier.connect("Ledger"); + const added = [ + await api.addExpense({ + id: TURN_ONE_IDS[0] ?? "survive-1", + description: "March groceries", + payerId: "month-ana", + amountCents: 2_500, + splitBetween: ["month-ana", "month-ben"], + dateIso: `${TURN_ONE_MONTH}-05`, + }), + await api.addExpense({ + id: TURN_ONE_IDS[1] ?? "survive-2", + description: "March taxi", + payerId: "month-ben", + amountCents: 1_200, + splitBetween: ["month-ana", "month-ben"], + dateIso: `${TURN_ONE_MONTH}-31`, + }), + await api.addExpense({ + id: "month-outside", + description: "April rent", + payerId: "month-ana", + amountCents: 90_000, + splitBetween: ["month-ana", "month-ben"], + dateIso: "2026-04-01", + }), + ]; + const march = MonthlySchema.parse(await api.monthlyTotals({ month: TURN_ONE_MONTH })); + const paid = (personId: string) => + march.byPayer.find(entry => entry.personId === personId)?.paidCents ?? null; + return { + pass: added.every(result => OkSchema.parse(result).ok) && + march.totalCents === 3_700 && paid("month-ana") === 2_500 && paid("month-ben") === 1_200, + evidence: march, + }; + }); + }, + }, { + prompt: `Now I want to see where the money actually goes. Add to the same Gadget: + +- addExpense takes an optional category: string. Expenses recorded before this change, and any + added without one, count as "uncategorized". +- setBudget({ category: string, month: "YYYY-MM", limitCents: integer >= 0 }) -> { ok: boolean }. + Setting the same category and month again replaces the previous limit. +- budgetStatus({ month: "YYYY-MM" }) -> { byCategory: Array<{ category: string, + limitCents: integer | null, spentCents: integer, remainingCents: integer | null }> } + One row per category that either has a budget for that month or had spending in it. Spending + counts the full expense amount against the month its dateIso falls in. limitCents and + remainingCents are null when no budget is set. remainingCents is limitCents - spentCents and is + allowed to go negative. + +Everything that already worked must keep working, including the expenses I have already recorded. +Show the category breakdown in the UI too.`, + verify: async verifier => { + // The agent's first write in this turn restarts the Gadget's server, discarding memory but + // not storage. Anything recorded in turn one that is missing now was never really persisted. + await verifier.check("expenses-recorded-before-the-change-survive", async () => { + using api = await verifier.connect("Ledger"); + const march = MonthlySchema.parse(await api.monthlyTotals({ month: TURN_ONE_MONTH })); + return { + pass: march.totalCents === 3_700, + evidence: march, + }; + }); + + await verifier.check("pre-existing-spending-counts-as-uncategorized", async () => { + using api = await verifier.connect("Ledger"); + const status = BudgetStatusSchema.parse(await api.budgetStatus({ month: TURN_ONE_MONTH })); + const uncategorized = status.byCategory.find(row => row.category === "uncategorized"); + return { + pass: uncategorized?.spentCents === 3_700 && uncategorized.limitCents === null && + uncategorized.remainingCents === null, + evidence: status, + }; + }); + + await verifier.check("budget-remaining-tracks-spending-and-can-go-negative", async () => { + using api = await verifier.connect("Ledger"); + await api.setBudget({ category: "food", month: "2026-06", limitCents: 10_000 }); + await api.setBudget({ category: "food", month: "2026-06", limitCents: 5_000 }); + await api.setBudget({ category: "travel", month: "2026-06", limitCents: 20_000 }); + const added = [ + await api.addExpense({ + id: "budget-1", + description: "Market", + payerId: "budget-ana", + amountCents: 6_500, + splitBetween: ["budget-ana", "budget-ben"], + dateIso: "2026-06-02", + category: "food", + }), + await api.addExpense({ + id: "budget-2", + description: "Train", + payerId: "budget-ben", + amountCents: 4_000, + splitBetween: ["budget-ana", "budget-ben"], + dateIso: "2026-06-03", + category: "travel", + }), + ]; + const status = BudgetStatusSchema.parse(await api.budgetStatus({ month: "2026-06" })); + const row = (category: string) => status.byCategory.find(entry => entry.category === category); + return { + pass: added.every(result => OkSchema.parse(result).ok) && + row("food")?.limitCents === 5_000 && row("food")?.spentCents === 6_500 && + row("food")?.remainingCents === -1_500 && + row("travel")?.limitCents === 20_000 && row("travel")?.spentCents === 4_000 && + row("travel")?.remainingCents === 16_000, + evidence: status, + }; + }); + + await verifier.check("turn-one-contract-still-holds", async () => { + using api = await verifier.connect("Ledger"); + const duplicate = OkSchema.parse(await api.addExpense({ + id: TURN_ONE_IDS[0] ?? "survive-1", + description: "Replay", + payerId: "after-ana", + amountCents: 111, + splitBetween: ["after-ana", "after-ben"], + dateIso: "2026-06-09", + })); + const balances = BalancesSchema.parse(await api.balances()); + const settlement = SettlementSchema.parse(await api.settlement()); + const remaining = new Map(balances.perPerson.map(entry => [entry.personId, entry.netCents])); + for (const transfer of settlement.transfers) { + remaining.set(transfer.fromId, (remaining.get(transfer.fromId) ?? 0) + transfer.amountCents); + remaining.set(transfer.toId, (remaining.get(transfer.toId) ?? 0) - transfer.amountCents); + } + return { + pass: !duplicate.ok && duplicate.error === "DUPLICATE_ID" && + balances.perPerson.reduce((sum, entry) => sum + entry.netCents, 0) === 0 && + [...remaining.values()].every(value => value === 0), + evidence: { + duplicate, + total: balances.perPerson.reduce((sum, entry) => sum + entry.netCents, 0), + }, + }; + }); + }, + }], +}); diff --git a/packages/workshop-evals/tasks/project-doc.task.ts b/packages/workshop-evals/tasks/project-doc.task.ts new file mode 100644 index 000000000..8ec25bdd4 --- /dev/null +++ b/packages/workshop-evals/tasks/project-doc.task.ts @@ -0,0 +1,101 @@ +import { z } from "zod"; +import { defineEvalTask } from "../src/task.js"; + +const DocumentSchema = z.object({ + revision: z.number().int().nonnegative(), + title: z.string(), + // Null means the document was created but never written to, which is distinct from empty. + blocks: z.array(z.object({ + id: z.string(), + html: z.string(), + version: z.number().int().nonnegative(), + }).loose()).nullable(), +}).loose(); + +type Document = z.infer; + +interface DocsApi { + getDocument(): Promise; + setDocument(input: { + blocks: { id: string; html: string; baseVersion?: number }[]; + title: string; + senderId: string; + }): Promise; +} + +const TITLE = "Harbour Refit"; + +function text(document: Document): string { + return (document.blocks ?? []).map(block => block.html).join("\n").toLowerCase(); +} + +/** + * Docs are a standard platform output. The system prompt advertises the shipped blueprint, and a + * real Doc appears on the outputs page and supports the collaborative Docs RPC. A custom notes + * Gadget lacks both properties. + * + * No current-main baseline shows this task passing every check, so it remains frontier. Promote it + * only after a repeated baseline establishes that result. + */ +export default defineEvalTask({ + id: "project-doc", + title: "Project notes as a standard Doc", + expectation: "frontier", + turns: [{ + prompt: `I'm kicking off a project called Harbour Refit. Set me up a doc named exactly +"${TITLE}" for my running notes on it. + +Start it off with a heading for the project and three bullet points, one each for scope, budget and +timeline. Put a sentence of placeholder detail under each — I'll replace them as things firm up.`, + verify: async verifier => { + await verifier.check("is-presented-as-a-document-output", async () => { + const doc = verifier.workpieces.find(workpiece => workpiece.title === TITLE); + return { + pass: doc?.output?.id === "document", + evidence: verifier.workpieces.map(workpiece => + ({ title: workpiece.title, output: workpiece.output ?? null })), + }; + }); + + await verifier.check("answers-the-docs-contract-with-real-content", async () => { + using api = await verifier.connect(TITLE); + const document = DocumentSchema.parse(await api.getDocument()); + const body = text(document); + return { + pass: document.revision >= 1 && document.blocks !== null && + document.blocks.length >= 4 && + document.title.toLowerCase().includes("harbour") && + ["scope", "budget", "timeline"].every(word => body.includes(word)), + evidence: { + revision: document.revision, + title: document.title, + blockCount: document.blocks?.length ?? null, + blocks: (document.blocks ?? []).slice(0, 8).map(block => block.html.slice(0, 120)), + }, + }; + }); + + await verifier.check("accepts-a-full-document-rewrite", async () => { + using api = await verifier.connect(TITLE); + const before = DocumentSchema.parse(await api.getDocument()); + const written = DocumentSchema.parse(await api.setDocument({ + title: "Harbour Refit — revised", + senderId: "eval", + blocks: [ + { id: "h", html: "

Harbour Refit

" }, + { id: "b1", html: "

Scope: replace the decking.

" }, + ], + })); + const reread = DocumentSchema.parse(await api.getDocument()); + return { + pass: written.revision > before.revision && + written.title === "Harbour Refit — revised" && + reread.revision === written.revision && + (reread.blocks ?? []).map(block => block.id).join() === "h,b1" && + (reread.blocks ?? []).every(block => block.version >= 1), + evidence: { beforeRevision: before.revision, written, reread }, + }; + }); + }, + }], +}); diff --git a/packages/workshop-evals/tsconfig.json b/packages/workshop-evals/tsconfig.json new file mode 100644 index 000000000..f4124518e --- /dev/null +++ b/packages/workshop-evals/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "target": "ES2022", + "lib": ["ESNext"], + "module": "ESNext", + "moduleResolution": "bundler", + "types": ["node", "@cloudflare/workers-types"], + "noEmit": true, + "paths": { + "@gadgets/integration-tests/*": ["../integration-tests/src/*"], + "@gadgets/workshop-shared/*": ["../workshop-shared/src/*"] + } + }, + "include": ["src", "evals", "*.ts"], + "exclude": ["node_modules"] +} diff --git a/packages/workshop-evals/vite.config.ts b/packages/workshop-evals/vite.config.ts new file mode 100644 index 000000000..6b837faf2 --- /dev/null +++ b/packages/workshop-evals/vite.config.ts @@ -0,0 +1,3 @@ +import vitestTaskViteConfig from "../../scripts/vitest-task-vite-config.js"; + +export default vitestTaskViteConfig("vitest run --config vitest.config.ts"); diff --git a/packages/workshop-evals/vitest.config.ts b/packages/workshop-evals/vitest.config.ts new file mode 100644 index 000000000..ce36a7426 --- /dev/null +++ b/packages/workshop-evals/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + environment: "node", + }, +}); diff --git a/packages/workshop-evals/vitest.eval.config.ts b/packages/workshop-evals/vitest.eval.config.ts new file mode 100644 index 000000000..0ab43a124 --- /dev/null +++ b/packages/workshop-evals/vitest.eval.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; +import { EVAL_TEST_TIMEOUT_MS } from "./src/config.js"; + +export default defineConfig({ + test: { + include: ["evals/**/*.eval.ts"], + environment: "node", + fileParallelism: false, + testTimeout: EVAL_TEST_TIMEOUT_MS, + hookTimeout: 3 * 60_000, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eaef363fb..a7d2721a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -844,6 +844,37 @@ importers: specifier: 'catalog:' version: 4.120.0(@cloudflare/workers-types@5.20260808.1) + packages/workshop-evals: + dependencies: + '@gadgets/integration-tests': + specifier: workspace:* + version: link:../integration-tests + '@gadgets/workshop-shared': + specifier: workspace:* + version: link:../workshop-shared + capnweb: + specifier: 'catalog:' + version: 0.12.0 + vitest-evals: + specifier: 0.16.1 + version: 0.16.1(tinyrainbow@3.1.1)(vitest@4.1.10)(zod@4.4.3) + zod: + specifier: ^4.4.3 + version: 4.4.3 + devDependencies: + '@cloudflare/workers-types': + specifier: ^5.20260808.1 + version: 5.20260808.1 + '@types/node': + specifier: 26.1.0 + version: 26.1.0 + typescript: + specifier: 'catalog:' + version: 7.0.2 + vitest: + specifier: 'catalog:' + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.0)(@vitest/browser-preview@4.1.10)(jsdom@26.1.0(supports-color@10.2.2))(vite@7.3.6(@types/node@26.1.0)(jiti@2.7.0)(lightningcss@1.33.0)(terser@5.49.2)(yaml@2.9.0)) + packages/workshop-frontend: dependencies: '@cloudflare/kumo': @@ -2791,6 +2822,12 @@ packages: peerDependencies: vite: 7.3.6 + '@vitest-evals/core@0.16.1': + resolution: {integrity: sha512-e4BfGirm4HOP2HfoYvE61iWp7K389tYwUOdPj+CS8lCI1D2efIlLptClU1knhDocLVaU6F/9ty2+p7u1tDkaIg==} + + '@vitest-evals/report-ui@0.16.1': + resolution: {integrity: sha512-3oHbMntbksRWQB1Nu1j8UXIy8+we6iUng24wAKxhSli7vjIgkaX02JjjfXgK4CWG4DhgOQkwJqw+wNaOMduY5A==} + '@vitest/browser-preview@4.1.10': resolution: {integrity: sha512-14MJrL59ZFkqXLjwfSk6RzTDy5Czf9UG4+8q8L6Gxjs2aPjEce/cVNYV14bXAc2BvMjUNu904+ZEZA1Xc1wtvQ==} peerDependencies: @@ -4884,6 +4921,20 @@ packages: yaml: optional: true + vitest-evals@0.16.1: + resolution: {integrity: sha512-MuPVetClAOn55ewgajxGxZcQDFWUftrQ19/rZQn8zXvaxHri+ElgEb5W0OIMRQwSvF5TBYIsE5rWdQB/qrWJkg==} + hasBin: true + peerDependencies: + ai: '>=4 <7' + tinyrainbow: '>=2 <4' + vitest: '>=4 <5' + zod: '>=3 <5' + peerDependenciesMeta: + ai: + optional: true + zod: + optional: true + vitest@4.1.10: resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -6810,6 +6861,14 @@ snapshots: transitivePeerDependencies: - supports-color + '@vitest-evals/core@0.16.1': + dependencies: + zod: 4.4.3 + + '@vitest-evals/report-ui@0.16.1': + dependencies: + '@vitest-evals/core': 0.16.1 + '@vitest/browser-preview@4.1.10(vite@7.3.6(@types/node@26.1.0)(jiti@2.7.0)(lightningcss@1.33.0)(terser@5.49.2)(yaml@2.9.0))(vitest@4.1.10)': dependencies: '@testing-library/dom': 10.4.1 @@ -9131,6 +9190,15 @@ snapshots: terser: 5.49.2 yaml: 2.9.0 + vitest-evals@0.16.1(tinyrainbow@3.1.1)(vitest@4.1.10)(zod@4.4.3): + dependencies: + '@vitest-evals/core': 0.16.1 + '@vitest-evals/report-ui': 0.16.1 + tinyrainbow: 3.1.1 + vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.0)(@vitest/browser-preview@4.1.10)(jsdom@26.1.0(supports-color@10.2.2))(vite@7.3.6(@types/node@26.1.0)(jiti@2.7.0)(lightningcss@1.33.0)(terser@5.49.2)(yaml@2.9.0)) + optionalDependencies: + zod: 4.4.3 + vitest@4.1.10(@opentelemetry/api@1.9.0)(@types/node@26.1.0)(@vitest/browser-preview@4.1.10)(jsdom@26.1.0(supports-color@10.2.2))(vite@7.3.6(@types/node@26.1.0)(jiti@2.7.0)(lightningcss@1.33.0)(terser@5.49.2)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.10