diff --git a/.gitignore b/.gitignore index 37e886d1d..e7a9f30af 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,12 @@ eval/corpus/ # Bench artifacts (events.jsonl, PNG dumps, workspaces) — one dir per run bench/runs/ +# Local runtime outputs +*.log.err +*.log.out +*-debug.err +*-debug.out + # eval run artifacts (outputs, not sources) eval/verbatim-15/out_*/ eval/verbatim-15/retest_rep*/ diff --git a/Dockerfile b/Dockerfile index 3196ccd1b..9cc466d37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,6 @@ EXPOSE 47821 VOLUME ["/data"] HEALTHCHECK --interval=5s --timeout=2s --start-period=2s --retries=3 \ - CMD node -e "const s=require('node:net').connect(process.env.PORT,'127.0.0.1');s.setTimeout(1500);s.on('connect',()=>{s.destroy();process.exit(0)});s.on('timeout',()=>process.exit(1));s.on('error',()=>process.exit(1))" + CMD node -e "fetch('http://127.0.0.1:' + process.env.PORT + '/healthz').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))" CMD ["node", "dist/node.js"] diff --git a/README.md b/README.md index 4653b2816..f0359a5a5 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,33 @@ Same thing without `ANTHROPIC_BASE_URL`, so `/remote-control`, claude.ai connectors, and first-party gates keep working. Full instructions in the dashboard. +### Windows launcher and health check + +This fork includes `pxpipe-run.cmd` for launching pxpipe on Windows. After it +starts, verify the local instance and routing with: + +```powershell +pwsh -File scripts/pxpipe-healthcheck.ps1 +``` + +The endpoint reports `200` when healthy; a `503` includes a routing diagnosis. +For how dashboard savings, cache tiers, and Codex rollout records are measured, +see [the measurement and calibration audit](docs/MEASUREMENT_AUDIT.md). + +### Docker Compose + +`docker compose up -d` publishes pxpipe only on `127.0.0.1` and makes the +dashboard plus `/healthz` available at the usual local port. The compose file +explicitly trusts only its fixed Docker bridge gateway for those routes; do +not copy that setting to a deployment whose published port is reachable from +an untrusted network. Compose also sets +`PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS=1` because `HOST=0.0.0.0` is confined to +the container while the host port is loopback-only. Outside this bundled +topology, pxpipe refuses a non-loopback bind when it would inject a server-owned +OpenAI/Cloudflare credential or gateway header: otherwise +any reachable client could spend that credential through the unauthenticated +proxy. Set the override only when a firewall, private network, or equivalent +trusted access boundary enforces the same restriction. ## Offline export (no proxy) You can render text, files, or diffs to PNG pages without running the proxy or diff --git a/compose.yml b/compose.yml index 4df1d248e..289e4bbb3 100644 --- a/compose.yml +++ b/compose.yml @@ -12,6 +12,13 @@ services: ports: - "127.0.0.1:${PXPIPE_PORT:-47821}:47821" environment: + # Docker bridge requests are not loopback inside the container. The port + # remains published to host loopback only, so dashboard/health stay local. + - PXPIPE_TRUSTED_DASHBOARD_PROXY=172.30.250.1 + # HOST is 0.0.0.0 inside the container, but the only host publication is + # the loopback binding above. Acknowledge that external access boundary + # so server-owned upstream credentials may be injected safely. + - PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS=1 - PXPIPE_MODELS - PXPIPE_DISABLE - PXPIPE_UPSTREAM @@ -30,6 +37,14 @@ services: - PXPIPE_DEBUG_CAPTURE_4XX volumes: - pxpipe-data:/data + networks: + - pxpipe-local volumes: pxpipe-data: + +networks: + pxpipe-local: + ipam: + config: + - subnet: 172.30.250.0/24 diff --git a/docs/MEASUREMENT_AUDIT.md b/docs/MEASUREMENT_AUDIT.md new file mode 100644 index 000000000..2e5ce8a2c --- /dev/null +++ b/docs/MEASUREMENT_AUDIT.md @@ -0,0 +1,74 @@ +# Measurement and calibration audit + +`Input reduction` is a cache-aware counterfactual, not an invoice. A Claude +row enters the measured numerator only when pxpipe has both the upstream +`usage` block and successful pre-transform `count_tokens` probes. A failed +probe receives zero credited saving. Responses/GPT rows use local tokenizer +and vision-token math and are disclosed separately as estimates. + +## Cache tiers + +For actual Anthropic usage pxpipe honours the server's reported cache-create +split: 5-minute writes are `1.25x`, one-hour writes are `2.0x`, and reads are +`0.1x`. Older responses may omit that split. They remain readable and use the +legacy 5-minute assumption, but the dashboard reports those tokens as +`cache_create_tier_unknown_tokens`; they are not evidence of a 5-minute TTL. +It also shows a **TTL sensitivity** number that reprices those unknown actual +creates at the 1-hour rate. That is a downside scenario for one assumption, +not a statistical confidence interval and not a complete lower bound. + +The hypothetical text path's cache state cannot be observed. Its warm/cold +state follows the actual request's reported cache read and is therefore a +modelled counterfactual, even when its text token count is measured. + +## Dollar display + +The dollar tile prices each measured Claude row by model family using the +official first-party list price checked **2026-07-13**: Fable 5 `$10/M`, Opus +4.5–4.8 `$5/M`, Sonnet 4.5/4.6 `$3/M`, Haiku 4.5 `$1/M`; Sonnet 5 is `$2/M` +through **2026-08-31** and is scheduled to become `$3/M` on 2026-09-01. +The lookup applies that UTC date boundary automatically; it does not leave the +introductory rate active indefinitely. An exact model override still takes +precedence. +Configure a private gateway or nonstandard model explicitly before treating it +as money: + +```powershell +$env:PXPIPE_MODEL_INPUT_USD_PER_MTOK = '{"claude-opus-4-8":5,"claude-fable-5":10}' +``` + +The value is an input-side list-price conversion, not a provider billing +export, subscription credit, or tax-inclusive invoice. Unknown models are +excluded from the dollar total and counted in the audit drawer. + +## Manual calibration only + +pxpipe never samples requests or silently routes traffic around compression. +To collect an observed baseline, explicitly use **Disable compression** in the +dashboard and confirm the dialog. Send comparable normal-text requests, then +enable compression and send comparable imaged requests. The toggle records +only the completed usage-bearing requests in those two in-memory phases. + +The first eligible baseline row locks the comparison to one Claude model and +one `first_user_sha8` session. Rows from other models/sessions and OpenAI/Codex +rows are ignored and counted as out of scope. The result remains sequential and +observational: context size, cache warmth and user workload can still differ +between phases. Treat it as a calibration check, not a paired billing +experiment. Restarting pxpipe clears the calibration note. + +## OpenAI cache diagnostics + +When an OpenAI-compatible upstream reports `cache_write_tokens` under +`input_tokens_details` or `prompt_tokens_details`, pxpipe persists it on the +event as a diagnostic subset of `input_tokens`. It is never added to the input +total a second time. Cache hits (`cached_tokens`), writes, output, and reasoning +remain separately auditable. + +## Codex rollout coverage + +Codex provider usage is read incrementally from retained local rollout files +whose `model_provider` is `pxpipe`. The index reads only appended bytes, keeps +partial JSONL rows until completion, and rebuilds a file after truncate or +replacement. Provider-reported token records are ground truth for those +retained records; they are not proof of complete billing history when rollout +files have been deleted or are unavailable. diff --git a/docs/superpowers/plans/2026-07-15-pxpipe-safeguards-phase-a.md b/docs/superpowers/plans/2026-07-15-pxpipe-safeguards-phase-a.md new file mode 100644 index 000000000..3ef944897 --- /dev/null +++ b/docs/superpowers/plans/2026-07-15-pxpipe-safeguards-phase-a.md @@ -0,0 +1,808 @@ +# pxpipe Safeguards — Phase A Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make pxpipe detect and loudly surface the "OpenAI upstream is not chatgpt.com → Codex 404" class of misconfiguration itself (startup banner, `/healthz`, `/api/health.json`), and fail a bad restart legibly — with no UI and no routing mutation. + +**Architecture:** A pure core evaluator (`src/core/health.ts`) turns a state snapshot into findings. A small host-side counter (`src/health-counters.ts`) tracks recent `/backend-api/codex/*` traffic so the codex check is evidence-driven. A host builder (`src/health-state.ts`) assembles the snapshot from the resolved upstreams, model scope, compression state, and counter. `src/node.ts` wires the counter into `onRequest`, serves `/healthz` (200/503) + `/api/health.json`, prints warn/error findings at startup, and handles `EADDRINUSE` with an actionable message + non-zero exit. A standalone `scripts/pxpipe-healthcheck.ps1` lets a launcher verify a running instance. + +**Tech Stack:** TypeScript (ESM, `.js` import specifiers), Node `node:http`, Vitest, built via `node scripts/build.mjs` to `dist/`. + +## Global Constraints + +- **Language:** TypeScript, ESM. Import specifiers use `.js` extensions (e.g. `./core/health.js`), matching the repo. +- **Core purity:** `src/core/*` must not touch the filesystem or network and must not call `Date.now()`/`new Date()` internally — any "now" is passed in as a parameter. (`src/health-*.ts` are host modules and MAY use `Date.now()`.) +- **Fail-open:** nothing added here may throw into the request path or block startup. Wrap host-side health calls in try/catch and treat a throw as "no findings". +- **Build/verify:** after code changes run `npm run typecheck`, `npx vitest run `, then `npm run build`. `bin/cli.js` runs `dist/`, so nothing is live until `npm run build` succeeds. +- **Known-good OpenAI host for Codex:** `chatgpt.com` (hostname `chatgpt.com` or `*.chatgpt.com`). The default/footgun host is `api.openai.com`. +- **Durable-fix hint string (verbatim, reused):** + `Set OPENAI_UPSTREAM=https://chatgpt.com (User env: setx OPENAI_UPSTREAM "https://chatgpt.com" then re-login, or in pxpipe-run.cmd) and restart pxpipe.` +- **Pre-existing test noise:** the full `vitest run` has two environment-dependent failures unrelated to this work (`reflow` corpus timeout, `proxy-usage` flaky under full-suite parallel load; both pass in isolation). Do not treat these as regressions; verify your new files in isolation. + +--- + +### Task 1: Pure health evaluator (`src/core/health.ts`) + +**Files:** +- Create: `src/core/health.ts` +- Test: `tests/health.test.ts` + +**Interfaces:** +- Consumes: nothing (pure; uses global `URL`). +- Produces: + - `type Severity = 'error' | 'warn' | 'info'` + - `interface Remediation { kind: 'set-openai-upstream'; target: 'https://chatgpt.com'; durableHint: string }` + - `interface HealthFinding { id: string; severity: Severity; title: string; detail: string; remediation?: Remediation }` + - `interface HealthRecentTraffic { codexResponses404: number; codexResponsesTotal: number; windowSeconds: number }` + - `interface HealthState { anthropicUpstream: string; openaiUpstream: string; openaiUpstreamOverridden: boolean; modelScope: string[]; compressionEnabled: boolean; recent: HealthRecentTraffic }` + - `function evaluateHealth(state: HealthState): HealthFinding[]` + - `function summarizeHealth(findings: HealthFinding[]): { ok: boolean; httpStatus: 200 | 503 }` + +Note (refines spec §4.1): the codex check emits a **single** finding `codex-upstream-mismatch` whose severity **escalates** — `warn` when the upstream host is wrong but no codex 404 has been observed yet (knowable at cold start, so the launcher/banner can warn proactively), `error` once a real codex 404 confirms it (drives `/healthz` 503). This keeps zero false-positive *errors* while still warning up front. + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/health.test.ts +import { describe, it, expect } from 'vitest'; +import { evaluateHealth, summarizeHealth, type HealthState } from '../src/core/health.js'; + +function state(over: Partial = {}): HealthState { + return { + anthropicUpstream: 'https://api.anthropic.com', + openaiUpstream: 'https://chatgpt.com', + openaiUpstreamOverridden: false, + modelScope: ['claude-fable-5'], + compressionEnabled: true, + recent: { codexResponses404: 0, codexResponsesTotal: 0, windowSeconds: 300 }, + ...over, + }; +} +const ids = (s: HealthState) => evaluateHealth(s).map((f) => f.id); +const find = (s: HealthState, id: string) => evaluateHealth(s).find((f) => f.id === id); + +describe('evaluateHealth — codex upstream', () => { + it('no finding when upstream is chatgpt.com', () => { + expect(ids(state())).not.toContain('codex-upstream-mismatch'); + }); + it('no finding when upstream is a chatgpt.com subdomain', () => { + expect(ids(state({ openaiUpstream: 'https://api.chatgpt.com' }))).not.toContain('codex-upstream-mismatch'); + }); + it('warns (not errors) when host is wrong but no codex 404 seen yet', () => { + const f = find(state({ openaiUpstream: 'https://api.openai.com' }), 'codex-upstream-mismatch'); + expect(f?.severity).toBe('warn'); + expect(f?.remediation?.target).toBe('https://chatgpt.com'); + }); + it('escalates to error once a codex 404 is observed', () => { + const f = find( + state({ openaiUpstream: 'https://api.openai.com', recent: { codexResponses404: 3, codexResponsesTotal: 3, windowSeconds: 300 } }), + 'codex-upstream-mismatch', + ); + expect(f?.severity).toBe('error'); + }); + it('does not error when host is chatgpt.com even with 404s (not our fault)', () => { + const f = find( + state({ recent: { codexResponses404: 5, codexResponsesTotal: 5, windowSeconds: 300 } }), + 'codex-upstream-mismatch', + ); + expect(f).toBeUndefined(); + }); + it('treats an unparseable upstream as wrong host (warns)', () => { + expect(find(state({ openaiUpstream: 'not a url' }), 'codex-upstream-mismatch')?.severity).toBe('warn'); + }); +}); + +describe('evaluateHealth — info findings', () => { + it('flags passthrough when compression is off', () => { + expect(ids(state({ compressionEnabled: false }))).toContain('compression-passthrough'); + }); + it('flags passthrough when model scope is empty', () => { + expect(ids(state({ modelScope: [] }))).toContain('compression-passthrough'); + }); + it('flags an active runtime override', () => { + expect(ids(state({ openaiUpstreamOverridden: true }))).toContain('openai-upstream-overridden'); + }); +}); + +describe('summarizeHealth', () => { + it('ok + 200 when no error findings', () => { + expect(summarizeHealth(evaluateHealth(state()))).toEqual({ ok: true, httpStatus: 200 }); + }); + it('not ok + 503 when an error finding is present', () => { + const findings = evaluateHealth(state({ openaiUpstream: 'https://api.openai.com', recent: { codexResponses404: 1, codexResponsesTotal: 1, windowSeconds: 300 } })); + expect(summarizeHealth(findings)).toEqual({ ok: false, httpStatus: 503 }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run tests/health.test.ts` +Expected: FAIL — cannot resolve `../src/core/health.js` / `evaluateHealth is not a function`. + +- [ ] **Step 3: Write the implementation** + +```ts +// src/core/health.ts +/** Pure self-diagnosis for pxpipe. No fs, no network, no clock — the caller + * passes a state snapshot (including any "recent traffic" aggregate) and gets + * back findings. Runs anywhere core runs. */ + +export type Severity = 'error' | 'warn' | 'info'; + +export interface Remediation { + kind: 'set-openai-upstream'; + target: 'https://chatgpt.com'; + durableHint: string; +} + +export interface HealthFinding { + id: string; + severity: Severity; + title: string; + detail: string; + remediation?: Remediation; +} + +export interface HealthRecentTraffic { + codexResponses404: number; + codexResponsesTotal: number; + windowSeconds: number; +} + +export interface HealthState { + anthropicUpstream: string; + openaiUpstream: string; + openaiUpstreamOverridden: boolean; + modelScope: string[]; + compressionEnabled: boolean; + recent: HealthRecentTraffic; +} + +const CHATGPT_DURABLE_HINT = + 'Set OPENAI_UPSTREAM=https://chatgpt.com (User env: setx OPENAI_UPSTREAM "https://chatgpt.com" then re-login, or in pxpipe-run.cmd) and restart pxpipe.'; + +/** Lower-cased hostname, or null when the URL cannot be parsed. */ +function hostOf(url: string): string | null { + try { + return new URL(url).hostname.toLowerCase(); + } catch { + return null; + } +} + +/** True only for chatgpt.com or a *.chatgpt.com subdomain. An unparseable URL + * is NOT chatgpt.com (so it surfaces as a mismatch rather than passing). */ +function isChatGPTHost(url: string): boolean { + const h = hostOf(url); + return h === 'chatgpt.com' || (h !== null && h.endsWith('.chatgpt.com')); +} + +export function evaluateHealth(state: HealthState): HealthFinding[] { + const findings: HealthFinding[] = []; + + if (!isChatGPTHost(state.openaiUpstream)) { + const confirmed = state.recent.codexResponses404 > 0; + findings.push({ + id: 'codex-upstream-mismatch', + severity: confirmed ? 'error' : 'warn', + title: confirmed + ? 'Codex requests are 404ing: OpenAI upstream is not chatgpt.com' + : 'OpenAI upstream is not chatgpt.com — Codex paths will 404', + detail: confirmed + ? `${state.recent.codexResponses404}/${state.recent.codexResponsesTotal} recent /backend-api/codex/* requests returned 404 while the OpenAI upstream is ${state.openaiUpstream}. That path only exists on chatgpt.com.` + : `The OpenAI upstream is ${state.openaiUpstream}. Codex uses /backend-api/codex/*, which exists only on chatgpt.com and 404s here. (No codex traffic seen yet.)`, + remediation: { + kind: 'set-openai-upstream', + target: 'https://chatgpt.com', + durableHint: CHATGPT_DURABLE_HINT, + }, + }); + } + + if (!state.compressionEnabled || state.modelScope.length === 0) { + findings.push({ + id: 'compression-passthrough', + severity: 'info', + title: 'Compression is not active — traffic passes through untransformed', + detail: !state.compressionEnabled + ? 'The dashboard compression kill-switch is off; every request forwards unchanged.' + : 'The model scope is empty, so no model is eligible for imaging; every request forwards unchanged.', + }); + } + + if (state.openaiUpstreamOverridden) { + findings.push({ + id: 'openai-upstream-overridden', + severity: 'info', + title: 'OpenAI upstream is overridden at runtime', + detail: `A runtime hot-swap is routing OpenAI traffic to ${state.openaiUpstream}. This is in-memory only; set OPENAI_UPSTREAM to make it durable.`, + }); + } + + return findings; +} + +/** ok = no error-severity findings. Maps to the /healthz status code. */ +export function summarizeHealth(findings: HealthFinding[]): { ok: boolean; httpStatus: 200 | 503 } { + const ok = !findings.some((f) => f.severity === 'error'); + return { ok, httpStatus: ok ? 200 : 503 }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run tests/health.test.ts` +Expected: PASS (all cases green). + +- [ ] **Step 5: Typecheck + commit** + +```bash +npm run typecheck +git add src/core/health.ts tests/health.test.ts +git commit -m "feat(health): pure evaluateHealth + summarizeHealth (codex-upstream check)" +``` + +--- + +### Task 2: Recent-traffic counter (`src/health-counters.ts`) + +**Files:** +- Create: `src/health-counters.ts` +- Test: `tests/health-counters.test.ts` + +**Interfaces:** +- Consumes: nothing. +- Produces: + - `class HealthCounters` with: + - `constructor(windowMs?: number)` (default 300000) + - `record(path: string, status: number, nowMs: number): void` + - `snapshot(nowMs: number): { codexResponses404: number; codexResponsesTotal: number; windowSeconds: number }` + - Only `/backend-api/codex/*` paths are counted; the return shape matches `HealthRecentTraffic` from Task 1. + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/health-counters.test.ts +import { describe, it, expect } from 'vitest'; +import { HealthCounters } from '../src/health-counters.js'; + +describe('HealthCounters', () => { + it('ignores non-codex paths', () => { + const c = new HealthCounters(); + c.record('/v1/messages', 200, 1000); + c.record('/v1/messages', 404, 1000); + expect(c.snapshot(1000)).toEqual({ codexResponses404: 0, codexResponsesTotal: 0, windowSeconds: 300 }); + }); + + it('counts codex requests and 404s within the window', () => { + const c = new HealthCounters(); + c.record('/backend-api/codex/responses', 404, 1000); + c.record('/backend-api/codex/responses', 404, 1500); + c.record('/backend-api/codex/models', 200, 2000); + expect(c.snapshot(2000)).toEqual({ codexResponses404: 2, codexResponsesTotal: 3, windowSeconds: 300 }); + }); + + it('drops entries older than the window', () => { + const c = new HealthCounters(10_000); // 10s window + c.record('/backend-api/codex/responses', 404, 1_000); + c.record('/backend-api/codex/responses', 404, 20_000); + // at t=20s, the first entry (t=1s) is >10s old and must be dropped + expect(c.snapshot(20_000)).toEqual({ codexResponses404: 1, codexResponsesTotal: 1, windowSeconds: 10 }); + }); + + it('snapshot alone (no new record) still trims stale entries', () => { + const c = new HealthCounters(10_000); + c.record('/backend-api/codex/responses', 404, 1_000); + expect(c.snapshot(50_000)).toEqual({ codexResponses404: 0, codexResponsesTotal: 0, windowSeconds: 10 }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run tests/health-counters.test.ts` +Expected: FAIL — cannot resolve `../src/health-counters.js`. + +- [ ] **Step 3: Write the implementation** + +```ts +// src/health-counters.ts +/** Host-side rolling counter of recent /backend-api/codex/* traffic. Feeds the + * evidence-driven codex-upstream check. Time is passed in (Date.now() from the + * host) so it is deterministically testable. Not in core: it holds state and + * reads the clock via its caller. */ + +const DEFAULT_WINDOW_MS = 300_000; // 5 minutes + +interface Entry { + ts: number; + is404: boolean; +} + +export class HealthCounters { + private codex: Entry[] = []; + + constructor(private readonly windowMs: number = DEFAULT_WINDOW_MS) {} + + private isCodexPath(path: string): boolean { + return path.startsWith('/backend-api/codex/'); + } + + record(path: string, status: number, nowMs: number): void { + if (!this.isCodexPath(path)) return; + this.codex.push({ ts: nowMs, is404: status === 404 }); + this.trim(nowMs); + } + + /** Entries are appended in time order, so stale ones are a prefix. */ + private trim(nowMs: number): void { + const cutoff = nowMs - this.windowMs; + let i = 0; + while (i < this.codex.length && this.codex[i]!.ts < cutoff) i++; + if (i > 0) this.codex.splice(0, i); + } + + snapshot(nowMs: number): { codexResponses404: number; codexResponsesTotal: number; windowSeconds: number } { + this.trim(nowMs); + let c404 = 0; + for (const e of this.codex) if (e.is404) c404++; + return { + codexResponses404: c404, + codexResponsesTotal: this.codex.length, + windowSeconds: Math.round(this.windowMs / 1000), + }; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run tests/health-counters.test.ts` +Expected: PASS. + +- [ ] **Step 5: Typecheck + commit** + +```bash +npm run typecheck +git add src/health-counters.ts tests/health-counters.test.ts +git commit -m "feat(health): rolling codex-traffic counter" +``` + +--- + +### Task 3: State builder + compression accessor (`src/health-state.ts`, `src/dashboard.ts`) + +**Files:** +- Create: `src/health-state.ts` +- Modify: `src/dashboard.ts` (add a public `isCompressionEnabled()` getter to `DashboardState`) +- Test: `tests/health-state.test.ts` + +**Interfaces:** +- Consumes: `resolveUpstreams` + `type ProxyConfig` from `./core/proxy.js`; `getAllowedModelBases` from `./core/applicability.js`; `type HealthState` from `./core/health.js`; `HealthCounters` from `./health-counters.js`. +- Produces: + - `DashboardState.isCompressionEnabled(): boolean` + - `function buildHealthState(config: ProxyConfig, compression: { isCompressionEnabled(): boolean }, counters: HealthCounters, nowMs: number): HealthState` + - Phase A always sets `openaiUpstreamOverridden: false` (no runtime override exists yet). + +- [ ] **Step 1: Add the `isCompressionEnabled` getter to `DashboardState`** + +In `src/dashboard.ts`, find the `handleCompressionToggle` method (search for `handleCompressionToggle`). Immediately above it, add: + +```ts + /** Whether the compression kill-switch is currently on. Read by the health + * subsystem to report passthrough state. */ + isCompressionEnabled(): boolean { + return this.compressionEnabled; + } +``` + +(`compressionEnabled` is the existing `private compressionEnabled = true;` field.) + +- [ ] **Step 2: Write the failing test** + +```ts +// tests/health-state.test.ts +import { describe, it, expect, beforeEach } from 'vitest'; +import { buildHealthState } from '../src/health-state.js'; +import { HealthCounters } from '../src/health-counters.js'; +import { evaluateHealth } from '../src/core/health.js'; +import { setAllowedModelBases } from '../src/core/applicability.js'; + +const compressionOn = { isCompressionEnabled: () => true }; + +describe('buildHealthState', () => { + beforeEach(() => setAllowedModelBases(null)); // reset runtime override between tests + + it('reads the resolved OpenAI upstream from config', () => { + const s = buildHealthState({ openAIUpstream: 'https://api.openai.com' }, compressionOn, new HealthCounters(), 1000); + expect(s.openaiUpstream).toBe('https://api.openai.com'); + expect(s.openaiUpstreamOverridden).toBe(false); + }); + + it('surfaces a codex 404 recorded in the counter as an error finding', () => { + const counters = new HealthCounters(); + counters.record('/backend-api/codex/responses', 404, 1000); + const s = buildHealthState({ openAIUpstream: 'https://api.openai.com' }, compressionOn, counters, 1000); + const f = evaluateHealth(s).find((x) => x.id === 'codex-upstream-mismatch'); + expect(f?.severity).toBe('error'); + }); + + it('reflects the live model scope and compression state', () => { + setAllowedModelBases(['gpt-5.6-terra']); + const s = buildHealthState({}, { isCompressionEnabled: () => false }, new HealthCounters(), 1000); + expect(s.modelScope).toEqual(['gpt-5.6-terra']); + expect(s.compressionEnabled).toBe(false); + }); +}); +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `npx vitest run tests/health-state.test.ts` +Expected: FAIL — cannot resolve `../src/health-state.js`. + +- [ ] **Step 4: Write the implementation** + +```ts +// src/health-state.ts +/** Host-side assembler: turns pxpipe's live config + runtime state into the + * HealthState snapshot the pure evaluator consumes. Kept out of node.ts so it + * is importable in tests without starting a server. */ + +import { resolveUpstreams, type ProxyConfig } from './core/proxy.js'; +import { getAllowedModelBases } from './core/applicability.js'; +import type { HealthState } from './core/health.js'; +import type { HealthCounters } from './health-counters.js'; + +export function buildHealthState( + config: ProxyConfig, + compression: { isCompressionEnabled(): boolean }, + counters: HealthCounters, + nowMs: number, +): HealthState { + const routes = resolveUpstreams(config); + return { + anthropicUpstream: routes.anthropic, + openaiUpstream: routes.openai, + // Phase A: no runtime upstream override exists yet — always false. + openaiUpstreamOverridden: false, + modelScope: getAllowedModelBases(), + compressionEnabled: compression.isCompressionEnabled(), + recent: counters.snapshot(nowMs), + }; +} +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `npx vitest run tests/health-state.test.ts` +Expected: PASS. + +- [ ] **Step 6: Guard against regressions in the dashboard getter** + +Run: `npx vitest run tests/dashboard-api.test.ts` +Expected: PASS (the new getter is additive; existing dashboard tests stay green). + +- [ ] **Step 7: Typecheck + commit** + +```bash +npm run typecheck +git add src/health-state.ts src/dashboard.ts tests/health-state.test.ts +git commit -m "feat(health): buildHealthState + DashboardState.isCompressionEnabled" +``` + +--- + +### Task 4: Serve `/healthz` + `/api/health.json` and feed the counter (`src/node.ts`) + +**Files:** +- Modify: `src/node.ts` (imports; `HealthCounters` instance; `onRequest` record; two health routes in the server callback) + +**Interfaces:** +- Consumes: `evaluateHealth`, `summarizeHealth` from `./core/health.js`; `HealthCounters` from `./health-counters.js`; `buildHealthState` from `./health-state.js`. +- Produces: HTTP `GET /healthz` (200 when ok / 503 when any error) and `GET /api/health.json` (always 200, `{ ok, findings, state }`). These are host routes handled before the dashboard dispatch. + +This task is integration glue over a live `node:http` server, so it is verified by a documented manual run rather than a unit test (the pure logic it composes is already covered by Tasks 1–3). + +- [ ] **Step 1: Add imports** + +In `src/node.ts`, next to the existing `import { getAllowedModelBases, setAllowedModelBases } from './core/applicability.js';` line, add: + +```ts +import { evaluateHealth, summarizeHealth } from './core/health.js'; +import { HealthCounters } from './health-counters.js'; +import { buildHealthState } from './health-state.js'; +``` + +- [ ] **Step 2: Create the counter instance** + +Find where the dashboard is constructed (search for `const dashboard = new DashboardState(`). Immediately after that statement, add: + +```ts + const healthCounters = new HealthCounters(); +``` + +- [ ] **Step 3: Record every request into the counter** + +In the `onRequest: async (e) => {` handler, as its first statement (before `dashboard.update(e);`), add: + +```ts + // Feed the health counter first — cheap and must never be skipped by an + // early return further down. Best-effort; never throw into onRequest. + try { + healthCounters.record(e.path, e.status, Date.now()); + } catch { + /* ignore */ + } +``` + +- [ ] **Step 4: Serve the health routes** + +Find the server request callback (search for `const route = dashboardPath(url.pathname);`). Immediately **before** that line, insert: + +```ts + // Health endpoints — host-level (compose config + counters + dashboard), + // handled before the dashboard router. Fail-open: any throw → 200 with + // an empty report rather than a 500. + if (url.pathname === '/healthz' || url.pathname === '/api/health.json') { + let ok = true; + let payload = '{"ok":true,"findings":[],"state":null}'; + try { + const state = buildHealthState(config, dashboard, healthCounters, Date.now()); + const findings = evaluateHealth(state); + ok = summarizeHealth(findings).ok; + payload = JSON.stringify({ ok, findings, state }, null, 2); + } catch { + /* fail-open: keep ok=true, empty report */ + } + const status = url.pathname === '/healthz' ? (ok ? 200 : 503) : 200; + res.statusCode = status; + res.setHeader('content-type', 'application/json'); + res.end(payload); + return; + } +``` + +(`config` is the `ProxyConfig` built earlier in this function and is in the closure; `dashboard` satisfies `{ isCompressionEnabled(): boolean }` after Task 3.) + +- [ ] **Step 5: Typecheck + build** + +```bash +npm run typecheck +npm run build +``` +Expected: typecheck clean; build prints `✓ built dist/node.js`. + +- [ ] **Step 6: Manual verification (documented)** + +In one shell, start a throwaway instance on a spare port pointed at the footgun host: +```bash +OPENAI_UPSTREAM=https://api.openai.com PORT=47899 node bin/cli.js +``` +In another shell: +```bash +# no codex traffic yet → warn only → /healthz stays 200 +curl.exe -s -o NUL -w "healthz=%{http_code}\n" http://127.0.0.1:47899/healthz +curl.exe -s http://127.0.0.1:47899/api/health.json +``` +Expected: `healthz=200`; the JSON `findings` contains `codex-upstream-mismatch` with `"severity":"warn"`. +Then simulate a codex hit (it will 404 upstream) and re-check: +```bash +curl.exe -s -o NUL http://127.0.0.1:47899/backend-api/codex/models +curl.exe -s -o NUL -w "healthz=%{http_code}\n" http://127.0.0.1:47899/healthz +``` +Expected: `healthz=503`; the finding severity is now `"error"`. Stop the throwaway instance (Ctrl-C). + +- [ ] **Step 7: Commit** + +```bash +git add src/node.ts +git commit -m "feat(health): serve /healthz (200/503) + /api/health.json; feed counter" +``` + +--- + +### Task 5: Startup banner + actionable `EADDRINUSE` (`src/node.ts`) + +**Files:** +- Modify: `src/node.ts` (server `error` handler; warn/error findings printed on `listen`) + +**Interfaces:** +- Consumes: `buildHealthState`, `evaluateHealth` (already imported in Task 4); `healthCounters`, `config`, `dashboard`, `opts` from the closure. +- Produces: no exports — startup-time console output + a non-zero process exit on bind failure. + +Integration glue; verified by documented manual run. + +- [ ] **Step 1: Add the `EADDRINUSE` handler** + +Find `const server = createServer((req, res) => {`. Immediately **after** the full `createServer(...)` statement (i.e. after the closing `});` of that call), add: + +```ts + server.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'EADDRINUSE') { + console.error(`[pxpipe] ⛔ port ${opts.port} is already in use — another pxpipe (or process) is bound to ${opts.host}:${opts.port}.`); + console.error(`[pxpipe] Find it: Get-NetTCPConnection -LocalPort ${opts.port} -State Listen | Select-Object OwningProcess`); + console.error(`[pxpipe] Stop it: Stop-Process -Id -Force`); + console.error(`[pxpipe] Then re-run the launcher.`); + } else { + console.error(`[pxpipe] server error: ${err.message}`); + } + process.exit(1); + }); +``` + +- [ ] **Step 2: Print warn/error findings at startup** + +Find the `server.listen(opts.port, opts.host, () => {` callback. Inside it, **after** the existing `console.log(\`[pxpipe] dashboard → ...\`)` line, add: + +```ts + try { + const findings = evaluateHealth(buildHealthState(config, dashboard, healthCounters, Date.now())); + for (const f of findings) { + if (f.severity !== 'error' && f.severity !== 'warn') continue; + const mark = f.severity === 'error' ? '⛔' : '⚠️'; + console.warn(`[pxpipe] ${mark} ${f.title}`); + console.warn(`[pxpipe] ${f.detail}`); + if (f.remediation) console.warn(`[pxpipe] fix: ${f.remediation.durableHint}`); + } + } catch { + /* never block startup on a health-print failure */ + } +``` + +- [ ] **Step 3: Typecheck + build** + +```bash +npm run typecheck +npm run build +``` +Expected: clean typecheck; `✓ built dist/node.js`. + +- [ ] **Step 4: Manual verification — startup banner** + +```bash +OPENAI_UPSTREAM=https://api.openai.com PORT=47899 node bin/cli.js +``` +Expected: alongside the usual banner lines, a `⚠️ OpenAI upstream is not chatgpt.com — Codex paths will 404` block with a `fix:` line. Stop it (Ctrl-C). Re-run with `OPENAI_UPSTREAM=https://chatgpt.com PORT=47899` and confirm **no** warning appears. + +- [ ] **Step 5: Manual verification — EADDRINUSE** + +Start one instance on 47899, then start a second on the same port: +```bash +OPENAI_UPSTREAM=https://chatgpt.com PORT=47899 node bin/cli.js # shell A +OPENAI_UPSTREAM=https://chatgpt.com PORT=47899 node bin/cli.js # shell B +``` +Expected in shell B: the `⛔ port 47899 is already in use …` block with the `Get-NetTCPConnection` / `Stop-Process` hints, and the process exits with a non-zero code (check `echo $?` in bash → non-zero). Stop shell A. + +- [ ] **Step 6: Commit** + +```bash +git add src/node.ts +git commit -m "feat(health): loud startup findings banner + actionable EADDRINUSE exit" +``` + +--- + +### Task 6: Launcher health-check helper (`scripts/pxpipe-healthcheck.ps1`) + +**Files:** +- Create: `scripts/pxpipe-healthcheck.ps1` +- Modify: `README.md` (short "Verifying a running instance" note) + +**Interfaces:** +- Consumes: a running pxpipe's `GET /healthz`. +- Produces: a script that exits `0` when healthy, `1` otherwise, printing a green/red line. Callable by the desktop launcher or the user after starting pxpipe. + +Rationale (refines spec §8): `pxpipe-run.cmd` runs `node bin\cli.js` in the **foreground** (it *is* the service window), so an in-script `curl` after it would only run once node exits. A standalone check the launcher/user calls after start is the clean fit; the startup banner (Task 5) already surfaces the same diagnosis in node's own console/log. + +- [ ] **Step 1: Create the script** + +```powershell +# scripts/pxpipe-healthcheck.ps1 +# Verify a running pxpipe instance. Exit 0 = healthy, 1 = unhealthy/unreachable. +param( + [int]$Port = 47821, + [int]$Retries = 20 +) +$ErrorActionPreference = 'SilentlyContinue' +$url = "http://127.0.0.1:$Port/healthz" +for ($i = 0; $i -lt $Retries; $i++) { + try { + $resp = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 2 + if ($resp.StatusCode -eq 200) { + $body = $resp.Content | ConvertFrom-Json + Write-Host "[pxpipe] OK healthz 200 openai upstream -> $($body.state.openaiUpstream)" -ForegroundColor Green + exit 0 + } + } catch { + $r = $_.Exception.Response + if ($r -and [int]$r.StatusCode -eq 503) { + $reader = New-Object System.IO.StreamReader($r.GetResponseStream()) + $body = $reader.ReadToEnd() | ConvertFrom-Json + $err = ($body.findings | Where-Object { $_.severity -eq 'error' } | Select-Object -First 1) + Write-Host "[pxpipe] FAIL healthz 503 $($err.title)" -ForegroundColor Red + if ($err.remediation) { Write-Host "[pxpipe] fix: $($err.remediation.durableHint)" -ForegroundColor Yellow } + exit 1 + } + } + Start-Sleep -Milliseconds 500 +} +Write-Host "[pxpipe] FAIL healthz unreachable on port $Port (is pxpipe running?)" -ForegroundColor Red +exit 1 +``` + +- [ ] **Step 2: Manual verification** + +With a healthy instance running (`OPENAI_UPSTREAM=https://chatgpt.com PORT=47821 node bin/cli.js`): +```powershell +pwsh -File scripts/pxpipe-healthcheck.ps1 -Port 47821 ; echo "exit=$LASTEXITCODE" +``` +Expected: green `OK healthz 200` line, `exit=0`. + +Then with the footgun host **and** after one codex 404 (so the finding is `error`/503): +```powershell +# start: OPENAI_UPSTREAM=https://api.openai.com PORT=47821 node bin/cli.js +# then: curl.exe -s -o NUL http://127.0.0.1:47821/backend-api/codex/models +pwsh -File scripts/pxpipe-healthcheck.ps1 -Port 47821 ; echo "exit=$LASTEXITCODE" +``` +Expected: red `FAIL healthz 503` + yellow `fix:` line, `exit=1`. + +- [ ] **Step 3: Document it in the README** + +Add a short subsection (near the existing run/launch docs) titled **"Verifying a running instance"**: + +```markdown +### Verifying a running instance + +After starting pxpipe, confirm it is healthy and correctly routed: + + pwsh -File scripts/pxpipe-healthcheck.ps1 # exit 0 = healthy, 1 = problem + +It checks `GET /healthz` (200 = ok, 503 = a real problem such as Codex traffic +404ing because the OpenAI upstream is not chatgpt.com) and prints the fix. A +launcher can gate on its exit code. The same diagnosis is printed by pxpipe at +startup and shown at `/api/health.json`. +``` + +- [ ] **Step 4: Commit** + +```bash +git add scripts/pxpipe-healthcheck.ps1 README.md +git commit -m "feat(health): launcher healthcheck script + docs" +``` + +--- + +### Task 7: Full-suite gate + Phase A wrap-up + +**Files:** none (verification only). + +- [ ] **Step 1: Run the new tests together** + +Run: `npx vitest run tests/health.test.ts tests/health-counters.test.ts tests/health-state.test.ts tests/dashboard-api.test.ts` +Expected: all PASS. + +- [ ] **Step 2: Typecheck + build once more** + +```bash +npm run typecheck && npm run build +``` +Expected: clean; `✓ built dist/node.js`. + +- [ ] **Step 3: Full suite (informational)** + +Run: `npx vitest run` +Expected: only the two known pre-existing failures (`reflow` corpus timeout, `proxy-usage` under full-suite load) may fail — everything else green. If any *other* test fails, investigate before declaring Phase A done. + +- [ ] **Step 4: Confirm Phase A scope is complete** + +Checklist (all must be true): `health.ts` codex check (warn→error escalation) ✓; recent-traffic counter ✓; `/healthz` 200/503 + `/api/health.json` ✓; startup warn/error banner ✓; actionable `EADDRINUSE` + non-zero exit ✓; `pxpipe-healthcheck.ps1` + README ✓. No UI, no routing mutation (those are Phase B). + +--- + +## Notes for the implementer + +- **Do not** add the runtime upstream override, the dashboard health panel, or the Fix button — those are **Phase B** and out of scope here. +- The live production process is a separate concern: this plan does not restart or reconfigure the running pxpipe. Landing Phase A changes `dist/` but only takes effect on the next restart, which remains a manual user action (pxpipe is this session's own upstream). +- Keep `src/core/*` free of fs/network/clock. If a check ever needs "now", pass it in — never call `Date.now()` inside core. diff --git a/docs/superpowers/specs/2026-07-15-pxpipe-safeguards-design.md b/docs/superpowers/specs/2026-07-15-pxpipe-safeguards-design.md new file mode 100644 index 000000000..c77797458 --- /dev/null +++ b/docs/superpowers/specs/2026-07-15-pxpipe-safeguards-design.md @@ -0,0 +1,289 @@ +# pxpipe safeguards & self-diagnosis — design + +**Date:** 2026-07-15 +**Status:** Draft for review +**Author:** brainstormed with Claude Code + +## 1. Motivation + +On 2026-07-15 Codex broke with a bare `404 Not Found` on +`POST /backend-api/codex/responses` (cf-ray header from Cloudflare). Root +cause: the running pxpipe process had been started **without** +`OPENAI_UPSTREAM=https://chatgpt.com`, so it resolved the OpenAI upstream to +the default `https://api.openai.com`. The ChatGPT-Codex path +`/backend-api/codex/*` only exists on `chatgpt.com`; on `api.openai.com` it +404s. pxpipe faithfully forwarded the request and returned the upstream's 404 +with no indication that *it* was the misconfigured link in the chain. + +The incident exposed four distinct weaknesses: + +1. **Config footgun** — a single missing env var silently routes a whole + provider to the wrong host. +2. **No self-detection** — the proxy could not tell it was misconfigured; it + forwarded and relayed someone else's 404. +3. **Fragile restart** — a corrective restart failed with `EADDRINUSE` + (the bad instance still held the port); the failure landed only in + `pxpipe-debug.err` and the bad instance kept running. +4. **Single point of failure** — Claude Code itself routes through pxpipe + (`ANTHROPIC_BASE_URL → 127.0.0.1:47821`), so a broken/misconfigured pxpipe + takes the user's agent down with it. + +## 2. Goal + +Build into pxpipe a **self-diagnosis and safe-remediation subsystem** so this +class of failure is **detected by the proxy itself**, **surfaced loudly** +(startup banner, dashboard health panel, `/healthz` for the launcher), and +**fixable in one click** (live in-runtime upstream swap plus a durable-command +hint) — with **no automatic magic** and **no new security hole**. + +### Priorities (chosen by the user) + +- Detection + loud warnings. +- Preventive config validation. +- Resilience as a dependency (pxpipe must not silently take down Claude Code). +- A manual **Fix** button (user-triggered), not silent auto-heal. + +### Non-goals (explicitly out of scope for now) + +- Automatic runtime self-healing (silently rewriting config on error). +- `pxpipe doctor` headless CLI (deferred — Phase C). +- Automatic port takeover of another instance (deferred — Phase C, and only + ever behind an explicit flag). +- Reading `~/.codex` or any external tool's config to infer intent. + +## 3. Design principles + +- **Detection is traffic-driven, not guessed.** The definitive + "you are misconfigured" signal is a real observed `/backend-api/codex/*` + request returning 404 while the OpenAI upstream host is not `chatgpt.com`. + We do **not** raise an error at cold start merely because the default + upstream is `api.openai.com` — many users legitimately use `api.openai.com` + for `/v1/responses` and never touch codex. No false positives. +- **Fixed remediations, not free-form input.** The Fix action performs a + *specific, hard-coded* remediation (set OpenAI upstream to the known-good + `https://chatgpt.com`). The unauthenticated dashboard never accepts an + arbitrary upstream URL — that would turn a local panel into an SSRF / + upstream-substitution vector. +- **Pure core, impure host.** Health evaluation is a pure function over a state + snapshot (no fs, no network), so it is trivially testable and runs on + Workers too. The Node host collects state, prints banners, serves JSON, and + owns any persistence. +- **Best-effort, fail-open.** Nothing in the safeguard path may throw into a + request. A health check that errors is dropped; the proxy keeps forwarding. + +## 4. Architecture & components + +### 4.1 `src/core/health.ts` — pure invariant core + +```ts +type Severity = 'error' | 'warn' | 'info'; + +interface Remediation { + kind: 'set-openai-upstream'; + target: 'https://chatgpt.com'; // fixed known-good host + durableHint: string; // e.g. the OPENAI_UPSTREAM / launcher command +} + +interface HealthFinding { + id: string; // stable slug, e.g. 'codex-upstream-mismatch' + severity: Severity; + title: string; + detail: string; + remediation?: Remediation; +} + +interface HealthState { + anthropicUpstream: string; + openaiUpstream: string; // the EFFECTIVE upstream (override or env) + openaiUpstreamOverridden: boolean; + modelScope: string[]; + compressionEnabled: boolean; + // Recent-traffic aggregate, supplied by the host from the dashboard ring: + recent: { + codexResponses404: number; // count of /backend-api/codex/* → 404 + codexResponsesTotal: number; + windowSeconds: number; + }; +} + +export function evaluateHealth(state: HealthState): HealthFinding[]; +``` + +Initial checks: + +| id | severity | fires when | remediation | +|----|----------|------------|-------------| +| `codex-upstream-mismatch` | error | `recent.codexResponses404 > 0` **and** `host(openaiUpstream) !== 'chatgpt.com'` | `set-openai-upstream` → chatgpt.com | +| `compression-passthrough` | info | `!compressionEnabled` **or** `modelScope` empty | none (informational) | +| `openai-upstream-overridden` | info | `openaiUpstreamOverridden` | none (shows a live hot-swap is active + how to make it durable) | + +The design keeps the check set intentionally small; new invariants are added as +data-driven table rows, not scattered conditionals. + +Note: `openaiUpstreamOverridden` and the `openai-upstream-overridden` info +finding are inert in Phase A (no override mechanism exists yet, so the field is +always `false`); they light up only once Phase B ships the live hot-swap. + +### 4.2 `src/core/upstream-override.ts` — live OpenAI-upstream hot-swap + +Mirrors the model-scope runtime-override pattern already shipped in +`applicability.ts`: + +```ts +const ALLOWED_OPENAI_HOSTS = ['chatgpt.com', 'api.openai.com']; + +let openAIUpstreamOverride: string | null = null; // in-memory only + +/** Reject anything outside the allowlist (defense in depth: the dashboard only + * ever sends the fixed remediation, but the setter must never accept an + * arbitrary host from any caller). Returns whether it was applied. */ +export function setOpenAIUpstreamOverride(url: string | null): boolean; + +/** Effective upstream: override (if set + allowlisted) else the env-resolved + * value. Read live per request. */ +export function effectiveOpenAIUpstream(envResolved: string): string; + +export function isOpenAIUpstreamOverridden(): boolean; +``` + +`createProxy` (proxy.ts) currently captures `openAIUpstream` once in its +closure (line ~710). Change: compute `effectiveOpenAIUpstream(routes.openai)` +**per request** where `upstreamBase` is chosen, so a hot-swap takes effect on +the next request with no restart. + +**Persistence decision — DECIDED: session-only (in-memory), not persisted.** +The hot-swap fixes the *running* process; the *durable* fix is a visible, +standard config source — `OPENAI_UPSTREAM` at the User env level +(`setx OPENAI_UPSTREAM "https://chatgpt.com"`) or in the launcher — surfaced as +the `durableHint` alongside the button. Rationale: a persisted routing override +would be a bespoke, invisible config source that silently outranks the explicit +`OPENAI_UPSTREAM` on the next boot — the *same* invisible-config failure class +that caused the original incident, and it would bite a future intentional +upstream change (e.g. switching to a gateway). Model-scope persistence (a user +*preference*) is a different risk class than upstream routing (ops config), so +the two intentionally differ. `setx` is registry-backed, inspectable +(`reg query "HKCU\Environment"`, the System Properties GUI, any new shell), and +a launcher `set` still overrides it for launcher-started instances — a visible +baseline, not a hidden trap. The launcher `/healthz` check (below) makes a lost +hot-swap after restart loud rather than silent, so nothing fails quietly. + +### 4.3 `src/node.ts` — host wiring + +- **Startup banner:** after constructing the dashboard, build a `HealthState` + and print any `error` findings as a prominent multi-line banner (alongside + the existing `anthropic/openai upstream →` lines), each with its + `durableHint`. `warn`/`info` are dashboard-only to keep the banner signal + high. +- **`GET /api/health.json`**: returns + `{ findings, state, ok: findings.every(f => f.severity !== 'error') }`. +- **`GET /healthz`**: same evaluation but returns **HTTP 200 when `ok`, HTTP 503 + when any `error` finding is present** (with a short text/JSON body). This lets + the launcher check a *status code* (`curl -f`) rather than parse JSON in a + `.cmd` — matching conventional healthz semantics. +- **`EADDRINUSE` handler:** `server.on('error', …)` catches the bind failure, + prints an actionable message — which PID holds `:47821` and the exact stop + command — instead of a raw stack trace to `.err`, and **exits non-zero** so + the launcher can detect the failed start via exit code. +- **Launcher (`pxpipe-run.cmd`) verification:** after starting node, poll + `curl -f http://127.0.0.1:47821/healthz` (short retry for bind) and check + node's exit code; print a red `✗` line with the diagnosis on failure, a green + `✓ upstream ` on success. `/healthz` confirms the *config* of whatever + is live on the port; the exit code catches "my instance never started" + (EADDRINUSE, old bad instance still holding the port). Together they close + failure mode #3. + +### 4.4 Dashboard (Phase B) — `/fragments/health` + Fix button + +- New fragment `/fragments/health`, refreshed `every 2s` like the others, + rendered inside the existing settings shell. +- Shows: effective anthropic/openai upstreams, model scope, compression state, + and findings with a severity light (red/amber/grey). +- A finding with a `remediation` renders a **Fix** button + (`POST /fragments/health/fix`, loopback-guarded, no body input — the action + is fully determined by the finding id: body is `{ id: }` drawn + from the fixed set of remediable findings, mapped server-side to its + hard-coded `Remediation` — never a URL from the client) plus an inline, + copy-friendly rendering of `durableHint` so the user can also make it durable. + +## 5. Data flow + +**Detection (continuous):** +`request → proxy forwards → onRequest records RecentRow (route, status) → +dashboard ring buffer → host aggregates codex-404 counts → evaluateHealth → +findings → banner (startup) / health panel (live) / /healthz (on demand).` + +**Remediation (user-triggered, Phase B):** +`user clicks Fix → POST /fragments/health/fix → host validates loopback → +setOpenAIUpstreamOverride('https://chatgpt.com') → next codex request reads +effectiveOpenAIUpstream() → routes to chatgpt.com → 404s stop → next +evaluateHealth clears the finding → panel goes green.` The panel keeps showing +`durableHint` until the user also sets the env/launcher value. + +## 6. Error handling + +- `evaluateHealth` is pure and total; the host wraps its call in try/catch and + treats a throw as "no findings" (never blocks startup or a request). +- `setOpenAIUpstreamOverride` rejects non-allowlisted hosts (returns `false`), + leaving the current value unchanged; the Fix route reports the outcome. +- The Fix route is idempotent (re-clicking is harmless) and loopback-guarded; + off-loopback it is refused (consistent with the existing kill-switch + exposure warning in node.ts). +- All health/aggregate reads degrade cleanly to zero/empty on missing data. + +## 7. Testing strategy + +- **`health.ts` (unit, table-driven):** state → findings. + - mismatch fires only when `codexResponses404 > 0` **and** host ≠ chatgpt.com; + - mismatch does **not** fire when host is chatgpt.com even with 404s; + - mismatch does **not** fire at cold start (zero codex traffic) — no false + positive on the default `api.openai.com`; + - info findings for passthrough / override. +- **`upstream-override.ts` (unit):** allowlist accepts chatgpt.com / + api.openai.com, rejects others (returns false, no mutation); + `effectiveOpenAIUpstream` returns override when set else env value. +- **proxy integration:** a codex-path request routes to the *effective* + upstream; flipping the override mid-test changes the target on the next + request (proves live read, not closure capture). +- **host:** `/api/health.json` shape + `ok` flag; EADDRINUSE message content + (focused test or documented manual check). +- Full suite (`vitest run`) stays green; two pre-existing environment-dependent + failures (`reflow` corpus timeout, `proxy-usage` flaky under full-suite load) + are unrelated and out of scope. + +## 8. Phasing + +- **Phase A (first standalone release):** `health.ts` + `codex-upstream-mismatch` + (traffic-driven) + startup banner + `/api/health.json` + `/healthz` (200/503) + + actionable `EADDRINUSE` (non-zero exit) + `pxpipe-run.cmd` verification + (`curl -f /healthz` + exit-code check). Pure logic + host wiring; no UI, no + live swap, nothing that mutates routing. Immediately makes the incident + self-evident and gives the launcher a verification hook. +- **Phase B (next):** `upstream-override.ts` (live hot-swap) + `/fragments/health` + panel + **Fix** button + durable-command hint. +- **Phase C (deferred):** `pxpipe doctor` CLI, generalized invariant framework, + explicit-flag port takeover. + +## 9. Resolved decisions + +1. **Upstream-override persistence — DECIDED: session-only** (in-memory), not + persisted. The durable path is `OPENAI_UPSTREAM` at User env level + (`setx`) or the launcher — a visible, standard, inspectable config source — + not a bespoke override file that would silently outrank explicit env. See + §4.2 for the full rationale. `setx` caveats to carry into the plan/docs: + it applies to the auto-launcher only after next logon/reboot, and true + removal is `reg delete "HKCU\Environment" /v OPENAI_UPSTREAM /f` (never + `setx ""`, which leaves an empty value that resolves to a broken upstream). +2. **Launcher integration — DECIDED: yes, in Phase A.** `pxpipe-run.cmd` polls + `curl -f /healthz` (200/503) and checks node's exit code, printing a red + diagnosis on failure / green `✓ upstream ` on success. See §4.3 / §8. + +## 10. Relationship to the live incident + +This design does **not** fix the currently-running misconfigured process +(PID 17216, `openai upstream → https://api.openai.com`). Codex stays 404 until +pxpipe is restarted via `pxpipe-run.cmd` (which sets +`OPENAI_UPSTREAM=https://chatgpt.com` and also loads the already-built +model-scope persistence + Terra/Sol/Lun chips). The restart remains a manual +user action because pxpipe is this session's own upstream. The safeguards above +are what make the *next* occurrence self-evident and one-click, not a +multi-step investigation. diff --git a/eval/gemini-profile/dimension-research.mjs b/eval/gemini-profile/dimension-research.mjs index 22cba43bf..f6773dc97 100644 --- a/eval/gemini-profile/dimension-research.mjs +++ b/eval/gemini-profile/dimension-research.mjs @@ -49,14 +49,14 @@ const VERBATIM_TRIALS = [ async function run() { console.log(`=== Gemini 3.6 Flash Dimension & Geometry Research ===\n`); - + const tokenResults = []; for (const dim of DIMENSION_PROBES) { const png = renderCanvasImage(dim.w, dim.h); let imgTokens = null; let totalTokens = null; let ms = null; - + if (LIVE) { const content = [ { type: 'input_image', image_url: `data:image/png;base64,${png.toString('base64')}` }, @@ -71,7 +71,7 @@ async function run() { console.error(`Error on ${dim.name}:`, e.message); } } - + const aspect = (dim.w / dim.h).toFixed(2); tokenResults.push({ ...dim, aspect, imgTokens, totalTokens, ms }); console.log(`${dim.name.padEnd(22)} ${dim.w}x${dim.h} (aspect ${aspect}) -> image tokens: ${imgTokens ?? 'N/A'}`); @@ -101,7 +101,7 @@ async function run() { const png = imgs[0].png; let ok = false; let got = ''; - + if (LIVE) { const content = [ { type: 'input_image', image_url: `data:image/png;base64,${Buffer.from(png).toString('base64')}` }, diff --git a/eval/gemini-profile/novel-arithmetic.mjs b/eval/gemini-profile/novel-arithmetic.mjs index f2d1a5f96..a1d4b8ded 100644 --- a/eval/gemini-profile/novel-arithmetic.mjs +++ b/eval/gemini-profile/novel-arithmetic.mjs @@ -75,7 +75,7 @@ const rows = await pool(ps, CONCURRENCY, async (p) => { const imgs = await renderTextToPngs(p.question, profile.stripCols, profile.style, profile.maxHeightPx); const urls = imgs.map((im) => ({ type: 'input_image', image_url: `data:image/png;base64,${Buffer.from(im.png).toString('base64')}` })); const imageTokens = imgs.reduce((n, im) => n + visionTokensForModel(MODEL, im.width, im.height), 0); - + let text, pure, prod; try { text = await callGemini({ model: MODEL, content: [{ type: 'input_text', text: `${ask}\n\n${p.question}` }], maxOutputTokens: 256, timeoutMs: TIMEOUT }); } catch (e) { text = { text: '', error: String(e.message || e) }; } try { pure = await callGemini({ model: MODEL, content: [...urls, { type: 'input_text', text: `The problem is in the image. ${ask}` }], maxOutputTokens: 256, timeoutMs: TIMEOUT }); } catch (e) { pure = { text: '', error: String(e.message || e) }; } @@ -83,7 +83,7 @@ const rows = await pool(ps, CONCURRENCY, async (p) => { const fs = factSheetText(p.question, profile.factSheetFormat); prod = await callGemini({ model: MODEL, content: [...urls, ...(fs ? [{ type: 'input_text', text: fs }] : []), { type: 'input_text', text: `The problem is in the image; use the exact-number factsheet if present. ${ask}` }], maxOutputTokens: 256, timeoutMs: TIMEOUT }); } catch (e) { prod = { text: '', error: String(e.message || e) }; } - + const textGot = num(text.text), pureGot = num(pure.text), prodGot = num(prod.text); const row = { ...p, imageTokens, textGot, pureGot, prodGot, textOk: textGot === p.answer, pureOk: pureGot === p.answer, prodOk: prodGot === p.answer, textUsage: text.usage || null, pureUsage: pure.usage || null, prodUsage: prod.usage || null, textError: text.error || null, pureError: pure.error || null, prodError: prod.error || null }; console.log(`q${p.i} text=${row.textOk ? 'Y' : 'N'}(${textGot}) pure=${row.pureOk ? 'Y' : 'N'}(${pureGot}) prod=${row.prodOk ? 'Y' : 'N'}(${prodGot}) gold=${p.answer}`); diff --git a/pxpipe-run.cmd b/pxpipe-run.cmd new file mode 100644 index 000000000..2093bdfb7 --- /dev/null +++ b/pxpipe-run.cmd @@ -0,0 +1,21 @@ +@echo off +rem pxpipe service launcher (clean env, no quoting surprises). +rem Launch: Start-Process cmd.exe -ArgumentList '/c','pxpipe-run.cmd' -WorkingDirectory (Get-Location) +rem Verify: pwsh -File scripts\pxpipe-healthcheck.ps1 (exit 0 = healthy, 1 = problem) + +rem Correct OpenAI upstream for Codex signed in through ChatGPT. +rem /backend-api/codex/* exists ONLY on chatgpt.com; api.openai.com 404s it. +set "OPENAI_UPSTREAM=https://chatgpt.com" + +rem Compression scope fallback (which model families pxpipe images) when no +rem dashboard choice is saved. Dashboard chip changes are saved to +rem %USERPROFILE%\.pxpipe\model-scope.json and OVERRIDE this until you press +rem "Reset to default" (which then falls back to exactly this set). +set "PXPIPE_MODELS=claude-opus-4-8,claude-sonnet-5,claude-fable-5,gpt-5.6-terra,gpt-5.6-sol,gpt-5.6-lun" + +cd /d "%~dp0" + +rem Capture stdout (startup banner + request log) and stderr (warnings, errors, +rem upstream error bodies) to files, overwritten each launch. Check the startup +rem health warning in pxpipe.log.err, or just run scripts\pxpipe-healthcheck.ps1. +node bin\cli.js > pxpipe.log 2> pxpipe.log.err diff --git a/scripts/pxpipe-healthcheck.ps1 b/scripts/pxpipe-healthcheck.ps1 new file mode 100644 index 000000000..239a17267 --- /dev/null +++ b/scripts/pxpipe-healthcheck.ps1 @@ -0,0 +1,35 @@ +# scripts/pxpipe-healthcheck.ps1 +# Verify a running pxpipe instance. Exit 0 = healthy, 1 = unhealthy/unreachable. +# Requires PowerShell 6+ (pwsh) for -SkipHttpErrorCheck. Run: pwsh -File scripts/pxpipe-healthcheck.ps1 +param( + [int]$Port = 47821, + [int]$Retries = 20 +) +$url = "http://127.0.0.1:$Port/healthz" +for ($i = 0; $i -lt $Retries; $i++) { + try { + # -SkipHttpErrorCheck returns the response object on 503 instead of throwing, + # so we read StatusCode/Content uniformly. A throw here means the connection + # was refused (server not up yet) — retry. + $resp = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 2 -SkipHttpErrorCheck + } catch { + Start-Sleep -Milliseconds 500 + continue + } + $code = [int]$resp.StatusCode + if ($code -eq 200) { + $body = $resp.Content | ConvertFrom-Json + Write-Host "[pxpipe] OK healthz 200 openai upstream -> $($body.state.openaiUpstream)" -ForegroundColor Green + exit 0 + } + if ($code -eq 503) { + $body = $resp.Content | ConvertFrom-Json + $err = ($body.findings | Where-Object { $_.severity -eq 'error' } | Select-Object -First 1) + Write-Host "[pxpipe] FAIL healthz 503 $($err.title)" -ForegroundColor Red + if ($err.remediation) { Write-Host "[pxpipe] fix: $($err.remediation.durableHint)" -ForegroundColor Yellow } + exit 1 + } + Start-Sleep -Milliseconds 500 +} +Write-Host "[pxpipe] FAIL healthz unreachable or unexpected status on port $Port (is pxpipe running?)" -ForegroundColor Red +exit 1 diff --git a/src/codex-usage.ts b/src/codex-usage.ts new file mode 100644 index 000000000..e8715222d --- /dev/null +++ b/src/codex-usage.ts @@ -0,0 +1,608 @@ +/** + * Exact Codex usage imported from the official local rollout logs. + * + * Codex persists provider-reported TokenCount snapshots under + * $CODEX_HOME/sessions//rollout-*.jsonl + * Community usage tools use the same source. We only index sessions whose + * session_meta.model_provider is `pxpipe`, keep no prompts or response text, + * and expose aggregate token/rate-limit telemetry to the local dashboard. + */ + +import * as fs from 'node:fs'; +import * as fsp from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import * as readline from 'node:readline'; + +export interface CodexTokenBreakdown { + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + reasoningOutputTokens: number; + totalTokens: number; +} + +export interface CodexRateLimitWindow { + usedPercent: number; + windowMinutes: number; + resetsAt: number; +} + +export interface CodexRateLimits { + limitId: string | null; + limitName: string | null; + primary: CodexRateLimitWindow | null; + secondary: CodexRateLimitWindow | null; + planType: string | null; + observedAt: string | null; +} + +export interface CodexQuotaWindow extends CodexRateLimitWindow { + limitId: string | null; + limitName: string | null; + observedAt: string | null; +} + +/** Quota windows belonging to one provider-reported limit. */ +export interface CodexQuotaGroup { + limitId: string | null; + limitName: string | null; + /** Latest observation for each duration, sorted shortest first. */ + windows: CodexQuotaWindow[]; +} + +export interface CodexUsageSnapshot extends CodexTokenBreakdown { + source: string; + loading: boolean; + error: string | null; + sessionFiles: number; + usageSnapshots: number; + modelContextWindow: number | null; + earliestEventAt: string | null; + latestEventAt: string | null; + rateLimits: CodexRateLimits | null; + /** Latest observation for each reported window duration (e.g. 300m, 10080m). */ + quotaWindows: CodexQuotaWindow[]; +} + +interface RawTokenUsage { + input_tokens?: unknown; + cached_input_tokens?: unknown; + output_tokens?: unknown; + reasoning_output_tokens?: unknown; + total_tokens?: unknown; +} + +interface FileSummary extends CodexTokenBreakdown { + usageSnapshots: number; + modelContextWindow: number | null; + earliestEventAt: string | null; + latestEventAt: string | null; + rateLimits: CodexRateLimits | null; + quotaWindows: CodexQuotaWindow[]; +} + +interface FileState { + provider: string; + dev: number; + ino: number; + size: number; + mtimeMs: number; + /** Byte offset already read from the file. */ + offset: number; + /** Bytes after the most recent newline, retained until the JSONL row is complete. */ + pending: Buffer; + /** A small observed suffix used to distinguish append from in-place replacement. */ + tail: Buffer; + parseState: ParseState | null; + summary: FileSummary | null; +} + +interface ParseState extends FileSummary { + seenTotals: Set; + previousTotal: CodexTokenBreakdown | null; + quotaByMinutes: Map; +} + +const EMPTY_BREAKDOWN: CodexTokenBreakdown = { + inputTokens: 0, + cachedInputTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0, + totalTokens: 0, +}; + +function emptyParseState(): ParseState { + return { + ...EMPTY_BREAKDOWN, + usageSnapshots: 0, + modelContextWindow: null, + earliestEventAt: null, + latestEventAt: null, + rateLimits: null, + quotaWindows: [], + seenTotals: new Set(), + previousTotal: null, + quotaByMinutes: new Map(), + }; +} + +function finiteNumber(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, value) : 0; +} + +function tokenBreakdown(value: unknown): CodexTokenBreakdown | null { + if (!value || typeof value !== 'object') return null; + const u = value as RawTokenUsage; + return { + inputTokens: finiteNumber(u.input_tokens), + cachedInputTokens: finiteNumber(u.cached_input_tokens), + outputTokens: finiteNumber(u.output_tokens), + reasoningOutputTokens: finiteNumber(u.reasoning_output_tokens), + totalTokens: finiteNumber(u.total_tokens), + }; +} + +function subtractBreakdown( + current: CodexTokenBreakdown, + previous: CodexTokenBreakdown | null, +): CodexTokenBreakdown { + return { + inputTokens: Math.max(0, current.inputTokens - (previous?.inputTokens ?? 0)), + cachedInputTokens: Math.max(0, current.cachedInputTokens - (previous?.cachedInputTokens ?? 0)), + outputTokens: Math.max(0, current.outputTokens - (previous?.outputTokens ?? 0)), + reasoningOutputTokens: Math.max(0, current.reasoningOutputTokens - (previous?.reasoningOutputTokens ?? 0)), + totalTokens: Math.max(0, current.totalTokens - (previous?.totalTokens ?? 0)), + }; +} + +function totalKey(u: CodexTokenBreakdown): string { + return [u.inputTokens, u.cachedInputTokens, u.outputTokens, u.reasoningOutputTokens, u.totalTokens].join(':'); +} + +function rateWindow(value: unknown): CodexRateLimitWindow | null { + if (!value || typeof value !== 'object') return null; + const v = value as Record; + if (typeof v.used_percent !== 'number') return null; + return { + usedPercent: finiteNumber(v.used_percent), + windowMinutes: finiteNumber(v.window_minutes), + resetsAt: finiteNumber(v.resets_at), + }; +} + +function parseRateLimits(value: unknown, observedAt: string | null): CodexRateLimits | null { + if (!value || typeof value !== 'object') return null; + const r = value as Record; + const primary = rateWindow(r.primary); + const secondary = rateWindow(r.secondary); + if (!primary && !secondary) return null; + return { + limitId: typeof r.limit_id === 'string' ? r.limit_id : null, + limitName: typeof r.limit_name === 'string' ? r.limit_name : null, + primary, + secondary, + planType: typeof r.plan_type === 'string' ? r.plan_type : null, + observedAt, + }; +} + +function recordQuotaWindows(state: ParseState, limits: CodexRateLimits): void { + for (const window of [limits.primary, limits.secondary]) { + if (!window || window.windowMinutes <= 0) continue; + const candidate: CodexQuotaWindow = { + ...window, + limitId: limits.limitId, + limitName: limits.limitName, + observedAt: limits.observedAt, + }; + const limitKey = limits.limitId !== null + ? `id:${limits.limitId}` + : `name:${limits.limitName ?? ''}`; + const key = `${limitKey}:${window.windowMinutes}`; + const previous = state.quotaByMinutes.get(key); + if (!previous || (candidate.observedAt ?? '') >= (previous.observedAt ?? '')) { + state.quotaByMinutes.set(key, candidate); + } + } + state.quotaWindows = [...state.quotaByMinutes.values()] + .sort((a, b) => a.windowMinutes - b.windowMinutes); +} + +/** + * Group quota windows without allowing equal-duration windows from different + * provider limits to be presented as one quota. Unknown ids are kept apart by + * their provider-reported names when available. + */ +export function groupCodexQuotaWindows(windows: Iterable): CodexQuotaGroup[] { + const groups = new Map; + }>(); + + for (const window of windows) { + const key = window.limitId !== null + ? `id:${window.limitId}` + : `name:${window.limitName ?? ''}`; + let group = groups.get(key); + if (!group) { + group = { + limitId: window.limitId, + limitName: window.limitName, + observedAt: window.observedAt ?? '', + byMinutes: new Map(), + }; + groups.set(key, group); + } else if ((window.observedAt ?? '') >= group.observedAt) { + group.limitName = window.limitName; + group.observedAt = window.observedAt ?? ''; + } + const previous = group.byMinutes.get(window.windowMinutes); + if (!previous || (window.observedAt ?? '') >= (previous.observedAt ?? '')) { + group.byMinutes.set(window.windowMinutes, window); + } + } + + return [...groups.values()] + .map((group) => ({ + limitId: group.limitId, + limitName: group.limitName, + windows: [...group.byMinutes.values()].sort((a, b) => a.windowMinutes - b.windowMinutes), + })) + .sort((a, b) => (a.limitName ?? a.limitId ?? '').localeCompare(b.limitName ?? b.limitId ?? '')); +} + +/** Apply one rollout JSONL line. Exported for focused schema/dedup tests. */ +function applyCodexUsageLine(state: ParseState, line: string): boolean { + // Avoid parsing prompt/response records entirely. TokenCount rows are small, + // and this fast gate keeps indexing both cheaper and less privacy-invasive. + if (!line.includes('"token_count"')) return false; + let row: unknown; + try { + row = JSON.parse(line); + } catch { + return false; + } + // A true return means that a non-newline-terminated tail was nevertheless + // a complete JSON value and can be committed. It does not mean it carried + // a usable TokenCount payload. + if (!row || typeof row !== 'object') return true; + const obj = row as Record; + if (obj.type !== 'event_msg' || !obj.payload || typeof obj.payload !== 'object') return true; + const payload = obj.payload as Record; + if (payload.type !== 'token_count' || !payload.info || typeof payload.info !== 'object') return true; + const info = payload.info as Record; + const total = tokenBreakdown(info.total_token_usage); + if (!total) return true; + + // Status refreshes often repeat the cumulative token total while carrying + // newer quota percentages/reset times. Metadata must advance independently + // of token deduplication or the dashboard can show a stale quota window. + if (typeof info.model_context_window === 'number' && Number.isFinite(info.model_context_window)) { + state.modelContextWindow = info.model_context_window; + } + const timestamp = typeof obj.timestamp === 'string' ? obj.timestamp : null; + if (timestamp && (!state.earliestEventAt || timestamp < state.earliestEventAt)) { + state.earliestEventAt = timestamp; + } + if (timestamp && (!state.latestEventAt || timestamp > state.latestEventAt)) { + state.latestEventAt = timestamp; + } + const limits = parseRateLimits(payload.rate_limits, timestamp); + if (limits) { + state.rateLimits = newerRateLimits(state.rateLimits, limits); + recordQuotaWindows(state, limits); + } + + const key = totalKey(total); + if (state.seenTotals.has(key)) return true; // Codex emits duplicate status snapshots. + state.seenTotals.add(key); + + const last = tokenBreakdown(info.last_token_usage); + const delta = last ?? subtractBreakdown(total, state.previousTotal); + state.previousTotal = total; + state.inputTokens += delta.inputTokens; + state.cachedInputTokens += delta.cachedInputTokens; + state.outputTokens += delta.outputTokens; + state.reasoningOutputTokens += delta.reasoningOutputTokens; + state.totalTokens += delta.totalTokens; + state.usageSnapshots += 1; + return true; +} + +function summaryFromParseState(state: ParseState): FileSummary { + const { seenTotals: _seen, previousTotal: _previous, quotaByMinutes: _quota, ...summary } = state; + return summary; +} + +/** Parse an in-memory fixture; production uses the streaming file variant. */ +export function summarizeCodexRolloutLines(lines: Iterable): FileSummary { + const state = emptyParseState(); + for (const line of lines) applyCodexUsageLine(state, line); + return summaryFromParseState(state); +} + +async function firstSessionProvider(file: string): Promise { + const stream = fs.createReadStream(file, { encoding: 'utf8' }); + const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of rl) { + // session_meta can carry a large base_instructions string. Extract only + // the non-sensitive provider discriminator instead of JSON-parsing it. + const match = /"model_provider"\s*:\s*"([^"\\]{1,80})"/.exec(line); + if (match) return match[1]!; + break; + } + return 'unknown'; + } finally { + rl.close(); + stream.destroy(); + } +} + +const FILE_TAIL_BYTES = 256; + +async function readFileRange(file: string, start: number, endExclusive: number): Promise { + if (endExclusive <= start) return Buffer.alloc(0); + const chunks: Buffer[] = []; + const stream = fs.createReadStream(file, { start, end: endExclusive - 1 }); + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +function applyRolloutBytes(state: ParseState, bytes: Buffer): number { + let lineStart = 0; + for (let i = 0; i < bytes.length; i += 1) { + if (bytes[i] !== 0x0a) continue; + let lineEnd = i; + if (lineEnd > lineStart && bytes[lineEnd - 1] === 0x0d) lineEnd -= 1; + applyCodexUsageLine(state, bytes.toString('utf8', lineStart, lineEnd)); + lineStart = i + 1; + } + + if (lineStart < bytes.length) { + // Codex normally terminates every JSONL row with LF. Still accept a final + // complete token_count JSON value, while retaining an incomplete write for + // the next refresh. Non-token rows stay pending until their newline so we + // never JSON.parse large prompt/response records merely to detect EOF. + const tail = bytes.toString('utf8', lineStart); + if (applyCodexUsageLine(state, tail)) return bytes.length; + } + return lineStart; +} + +function updateObservedTail(state: FileState, observed: Buffer): void { + if (observed.length === 0) return; + const combined = state.tail.length > 0 ? Buffer.concat([state.tail, observed]) : observed; + state.tail = Buffer.from(combined.subarray(Math.max(0, combined.length - FILE_TAIL_BYTES))); +} + +async function observedTailMatches(file: string, state: FileState): Promise { + if (state.tail.length === 0) return true; + if (state.offset < state.tail.length) return false; + const actual = await readFileRange(file, state.offset - state.tail.length, state.offset); + return actual.equals(state.tail); +} + +async function appendRolloutFile(file: string, state: FileState, size: number): Promise { + const parseState = state.parseState ?? emptyParseState(); + state.parseState = parseState; + const bytes = await readFileRange(file, state.offset, size); + const candidate = state.pending.length > 0 ? Buffer.concat([state.pending, bytes]) : bytes; + const consumed = applyRolloutBytes(parseState, candidate); + // Copy the usually tiny partial row so a slice does not retain a large + // candidate allocation after complete rows have already been consumed. + state.pending = Buffer.from(candidate.subarray(consumed)); + updateObservedTail(state, bytes); + state.offset += bytes.length; + state.summary = summaryFromParseState(parseState); +} + +function freshFileState(provider: string, stat: fs.Stats): FileState { + return { + provider, + dev: stat.dev, + ino: stat.ino, + size: 0, + mtimeMs: 0, + offset: 0, + pending: Buffer.alloc(0), + tail: Buffer.alloc(0), + parseState: provider === 'pxpipe' ? emptyParseState() : null, + summary: null, + }; +} + +function fileIdentityChanged(state: FileState, stat: fs.Stats): boolean { + return state.dev !== stat.dev || state.ino !== stat.ino; +} + +async function rolloutFiles(dir: string): Promise { + const out: string[] = []; + const walk = async (current: string): Promise => { + let entries: fs.Dirent[]; + try { + entries = await fsp.readdir(current, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const child = path.join(current, entry.name); + if (entry.isDirectory()) await walk(child); + else if (entry.isFile() && entry.name.endsWith('.jsonl')) out.push(child); + } + }; + await walk(dir); + return out; +} + +function newerRateLimits(a: CodexRateLimits | null, b: CodexRateLimits | null): CodexRateLimits | null { + if (!a) return b; + if (!b) return a; + return (b.observedAt ?? '') > (a.observedAt ?? '') ? b : a; +} + +export function resolveCodexSessionsDir(): string { + const home = process.env.CODEX_HOME || path.join(os.homedir(), '.codex'); + return path.join(home, 'sessions'); +} + +export class CodexUsageIndex { + private readonly states = new Map(); + private refreshing = false; + private timer: NodeJS.Timeout | null = null; + private current: CodexUsageSnapshot; + + constructor(private readonly sessionsDir = resolveCodexSessionsDir()) { + this.current = { + ...EMPTY_BREAKDOWN, + source: 'retained local Codex rollout logs', + loading: true, + error: null, + sessionFiles: 0, + usageSnapshots: 0, + modelContextWindow: null, + earliestEventAt: null, + latestEventAt: null, + rateLimits: null, + quotaWindows: [], + }; + } + + start(intervalMs = 10_000): void { + void this.refresh(); + if (this.timer) return; + this.timer = setInterval(() => void this.refresh(), intervalMs); + this.timer.unref(); + } + + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + snapshot(): CodexUsageSnapshot { + return structuredClone(this.current); + } + + async refresh(): Promise { + if (this.refreshing) return; + this.refreshing = true; + try { + const files = await rolloutFiles(this.sessionsDir); + const found = new Set(files); + for (const known of this.states.keys()) { + if (!found.has(known)) this.states.delete(known); + } + + for (const file of files) { + let stat: fs.Stats; + try { + stat = await fsp.stat(file); + } catch { + continue; + } + let state = this.states.get(file); + if (!state) { + const provider = await firstSessionProvider(file); + // Remember the observed file revision even when session_meta is not + // readable yet. Active rollouts can briefly exist as an empty or + // partial file; an `unknown` provider must be retried after the file + // grows instead of being excluded for the lifetime of the process. + state = freshFileState(provider, stat); + this.states.set(file, state); + } else { + const identityChanged = fileIdentityChanged(state, stat); + const shrank = stat.size < state.size; + const sameSizeRewrite = stat.size === state.size && stat.mtimeMs !== state.mtimeMs; + let prefixChanged = false; + if (!identityChanged && !shrank && stat.size > state.size && state.provider === 'pxpipe') { + prefixChanged = !(await observedTailMatches(file, state)); + } + if (identityChanged || shrank || sameSizeRewrite || prefixChanged) { + // Truncate, atomic replace, and in-place rewrites invalidate both + // cumulative-token deduplication and metadata. Rebuild only this + // file from byte zero and detect its provider again. + state = freshFileState(await firstSessionProvider(file), stat); + this.states.set(file, state); + } else if ( + state.provider === 'unknown' + && (state.size !== stat.size || state.mtimeMs !== stat.mtimeMs) + ) { + state.provider = await firstSessionProvider(file); + if (state.provider === 'pxpipe') state.parseState = emptyParseState(); + } + } + + if (state.provider === 'pxpipe') { + if (state.size !== stat.size || state.mtimeMs !== stat.mtimeMs || !state.summary) { + await appendRolloutFile(file, state, stat.size); + } + } + state.size = stat.size; + state.mtimeMs = stat.mtimeMs; + } + + const next: CodexUsageSnapshot = { + ...EMPTY_BREAKDOWN, + source: 'retained local Codex rollout logs', + loading: false, + error: null, + sessionFiles: 0, + usageSnapshots: 0, + modelContextWindow: null, + earliestEventAt: null, + latestEventAt: null, + rateLimits: null, + quotaWindows: [], + }; + const quotaByMinutes = new Map(); + for (const state of this.states.values()) { + if (state.provider !== 'pxpipe' || !state.summary) continue; + const s = state.summary; + next.sessionFiles += 1; + next.usageSnapshots += s.usageSnapshots; + next.inputTokens += s.inputTokens; + next.cachedInputTokens += s.cachedInputTokens; + next.outputTokens += s.outputTokens; + next.reasoningOutputTokens += s.reasoningOutputTokens; + next.totalTokens += s.totalTokens; + if (s.modelContextWindow !== null) next.modelContextWindow = s.modelContextWindow; + if (s.earliestEventAt && (!next.earliestEventAt || s.earliestEventAt < next.earliestEventAt)) { + next.earliestEventAt = s.earliestEventAt; + } + if (s.latestEventAt && (!next.latestEventAt || s.latestEventAt > next.latestEventAt)) { + next.latestEventAt = s.latestEventAt; + } + next.rateLimits = newerRateLimits(next.rateLimits, s.rateLimits); + for (const window of s.quotaWindows) { + const limitKey = window.limitId !== null + ? `id:${window.limitId}` + : `name:${window.limitName ?? ''}`; + const key = `${limitKey}:${window.windowMinutes}`; + const previous = quotaByMinutes.get(key); + if (!previous || (window.observedAt ?? '') >= (previous.observedAt ?? '')) { + quotaByMinutes.set(key, window); + } + } + } + next.quotaWindows = [...quotaByMinutes.values()] + .sort((a, b) => a.windowMinutes - b.windowMinutes); + this.current = next; + } catch (err) { + this.current = { + ...this.current, + loading: false, + // /proxy-stats may be exposed via HOST; do not leak absolute local + // paths embedded in filesystem errors into the dashboard payload. + error: err instanceof Error ? err.name : 'Codex rollout scan failed', + }; + } finally { + this.refreshing = false; + } + } +} diff --git a/src/core/applicability.ts b/src/core/applicability.ts index 48f16a3f1..28eb2b409 100644 --- a/src/core/applicability.ts +++ b/src/core/applicability.ts @@ -26,7 +26,7 @@ function baseModelId(model: string): string { /** Dashboard runtime override; null = fall back to PXPIPE_MODELS env / built-in default. In-memory only. */ let runtimeModelBases: readonly string[] | null = null; -/** Built-in default scope when PXPIPE_MODELS is unset: Fable 5 and Gemini 3.6 Flash. +/** Built-in default scope when PXPIPE_MODELS is unset: Fable 5, Opus 5, and Gemini 3.6 Flash. * Everything else is opt-in via dashboard chips or PXPIPE_MODELS: * - Opus 4.7/4.8 — worse at reading imaged content (FINDINGS.md 2026-06-16: * Opus 4.8 ~2pp arithmetic, 6/15 dense-hex vs Fable 100/100). @@ -44,7 +44,7 @@ function falsey(v: string): boolean { /** PXPIPE_MODELS env / built-in default, ignoring the runtime override. One CSV * controls every family (Claude + GPT). Resolution (read per-call so scope flips LIVE): - * - unset or empty → built-in default (Fable 5 only) + * - unset or empty → built-in default (Fable 5, Opus 5, Gemini 3.6 Flash) * - `off`/`0`/`false`/... → compress nothing * - CSV of model bases → exactly those families (e.g. `claude-fable-5,gpt-5.6-sol`) */ function envOrDefaultBases(): string[] { @@ -62,6 +62,19 @@ function allowedModelBases(): string[] { return envOrDefaultBases(); } +/** Whether a concrete model id is enabled by a scope list. Kept separate from + * the route eligibility helpers so the dashboard can display the same prefix + * semantics that the proxy applies (for example `gpt-5.6` → `gpt-5.6-sol`). */ +export function isModelScopeEnabled(model: string, bases: readonly string[] = allowedModelBases()): boolean { + const base = baseModelId(model).toLowerCase(); + const unqualified = unqualifiedModelId(base); + return bases.some((candidate) => { + const target = candidate.toLowerCase(); + const hit = (id: string): boolean => id === target || id.startsWith(`${target}-`); + return hit(base) || (unqualified !== null && hit(unqualified)); + }); +} + /** Current effective allowed-model scope (Claude + GPT). */ export function getAllowedModelBases(): string[] { return allowedModelBases(); @@ -100,12 +113,7 @@ function isAllowed(model: string | null | undefined): boolean { // (e.g. an unmeasured Gemini sibling falling through to the OpenAI fallback). // Which ids those are is profile-table knowledge, not a rule maintained here. if (isMisresolvedModelId(base)) return false; - const unqualified = unqualifiedModelId(base); - return allowedModelBases().some((b) => { - const target = b.toLowerCase(); - const hit = (id: string): boolean => id === target || id.startsWith(`${target}-`); - return hit(base) || (unqualified !== null && hit(unqualified)); - }); + return isModelScopeEnabled(base); } /** True when pxpipe may transform this Anthropic model. */ diff --git a/src/core/baseline.ts b/src/core/baseline.ts index c85b9e4b8..436984c98 100644 --- a/src/core/baseline.ts +++ b/src/core/baseline.ts @@ -4,11 +4,44 @@ * See docs/CACHING_AND_SAVINGS.md for the full derivation and audit history. */ -/** Documented Anthropic price ratios: cc_5m = 1.25×, cc_1h = 2×, cr = 0.1× base input. */ -export const CACHE_CREATE_RATE = 1.25; -const CACHE_CREATE_1H_RATE = 2.0; +/** Documented Anthropic cache price ratios to ordinary input. */ +export const CACHE_CREATE_5M_RATE = 1.25; +export const CACHE_CREATE_1H_RATE = 2.0; +/** @deprecated Prefer the explicit 5m/1h constants. Kept for source compatibility. */ +export const CACHE_CREATE_RATE = CACHE_CREATE_5M_RATE; export const CACHE_READ_RATE = 0.1; +/** + * Server-reported cache-create tier split. Anthropic omits it on some older + * responses; that remainder is deliberately treated as a 5-minute estimate, + * not as proof that the request used that tier. Callers can surface coverage. + */ +export interface CacheCreateBreakdown { + fiveMinuteTokens?: number; + oneHourTokens?: number; +} + +export function cacheCreateEffectiveTokens( + totalTokens: number, + breakdown?: CacheCreateBreakdown, +): number { + const total = Math.max(0, totalTokens || 0); + const five = Math.min(total, Math.max(0, breakdown?.fiveMinuteTokens ?? 0)); + const one = Math.min(total - five, Math.max(0, breakdown?.oneHourTokens ?? 0)); + return five * CACHE_CREATE_5M_RATE + one * CACHE_CREATE_1H_RATE + + (total - five - one) * CACHE_CREATE_5M_RATE; +} + +export function cacheCreateUnknownTokens( + totalTokens: number, + breakdown?: CacheCreateBreakdown, +): number { + const total = Math.max(0, totalTokens || 0); + const five = Math.min(total, Math.max(0, breakdown?.fiveMinuteTokens ?? 0)); + const one = Math.min(total - five, Math.max(0, breakdown?.oneHourTokens ?? 0)); + return total - five - one; +} + /** Effective cache-write rate for this request. Older usage payloads do not * expose the tier split; preserve the historical/conservative 5-minute rate * in that case. The text counterfactual uses the same observed tier mix as @@ -59,18 +92,6 @@ export interface BaselineWarmthPrev { * When cr proves warmth, a completed same-prefix prior is used only to estimate * how much of the text prefix was reused vs grown. If none is available, assume * full reuse of this turn's cacheable prefix; this is conservative for savings. - * - * @param prev this session's previous usage-bearing turn, or undefined. - * @param nowSec request-start wall-clock seconds, used only to reject prior - * rows that had not completed before this request was sent. - * @param cacheable this turn's cacheable-prefix tokens (the full-reuse credit - * when warm only via cr, since cr proves a read but not the split). - * @param cr observed cache-read tokens this turn; the only warm/cold signal. - * @param ttlSec legacy parameter; no longer decides warm/cold. It only - * bounds whether a prior prefix size is used for reused/grown - * splitting after cr > 0 has already proved warmth. - * @param prefixSha stable-prefix fingerprint for the text counterfactual. A - * prior prefix size is reused only when this matches. */ export function deriveBaselineWarmth( prev: BaselineWarmthPrev | undefined, @@ -95,21 +116,6 @@ export function deriveBaselineWarmth( /** * Weighted input cost for the unproxied TEXT counterfactual (see docs/CACHING_AND_SAVINGS.md). - * - * Warmth matters: a TEXT prefix is only a cheap cache-read when a warm cache - * actually existed this turn. The previous warmth-FREE version always priced - * the cacheable prefix at CACHE_READ_RATE, which fabricated a "free read" on - * cold/TTL-expiry turns where text would in fact have paid a 1.25× create — - * that produced a phantom loss vs the imaged path (which DOES pay the create). - * - * cold turn (first turn / >5min since this session's last turn): - * text has no warm cache either ⇒ cacheable×CACHE_CREATE_RATE + coldTail×1.0 - * warm turn (a prior turn cached the prefix within TTL): - * text append-caches ⇒ reused×CACHE_READ_RATE + grown×CACHE_CREATE_RATE + coldTail×1.0 - * where reused = min(prevCacheable, cacheable), grown = cacheable − reused. - * This is what TEXT pays regardless of whether pxpipe's image busted its - * own cache on a growth turn — so the real growth loss is preserved. - * * Saving = baseline_eff − actual_eff; can be negative (honestly reported, not floored). * * @param baselineCacheable tokens up to the last cache_control marker. ≤0 ⇒ credit nothing. @@ -154,16 +160,14 @@ export function computeBaselineInputEffWithCacheTier( const createRate = cacheCreateRate(cc, cacheCreate5mTokens, cacheCreate1hTokens); if (warm) { // Text reads the prefix it already had cached (0.10×) and creates only the - // growth since last turn (at the observed write-tier rate). Independent of - // the image path's cache. + // growth since last turn (at the observed write-tier rate). const reused = Math.min(Math.max(prevCacheable, 0), cacheable); const grown = cacheable - reused; - return reused * CACHE_READ_RATE + grown * createRate + coldTail * 1.0; + return reused * CACHE_READ_RATE + grown * createRate + coldTail; } - // Cold (first turn / TTL expiry): no warm cache for text either, so it - // re-creates the whole cacheable prefix at the create rate — same event the - // imaged path pays. Removes the phantom "free read" that fabricated a loss. - return cacheable * createRate + coldTail * 1.0; + // Cold: no warm cache for text either, so it re-creates the whole cacheable + // prefix at the observed create rate — the same event the imaged path pays. + return cacheable * createRate + coldTail; } /** Weighted input cost pxpipe actually paid this turn. */ @@ -171,8 +175,9 @@ export function computeActualInputEff( inputTokens: number, cc: number, cr: number, + cacheCreate?: CacheCreateBreakdown, ): number { - return computeActualInputEffWithCacheTier(inputTokens, cc, cr, 0); + return inputTokens + cacheCreateEffectiveTokens(cc, cacheCreate) + cr * CACHE_READ_RATE; } /** Tier-aware variant for internal telemetry accounting. */ @@ -186,4 +191,4 @@ export function computeActualInputEffWithCacheTier( return inputTokens + cc * cacheCreateRate(cc, cacheCreate5mTokens, cacheCreate1hTokens) + cr * CACHE_READ_RATE; -} +} \ No newline at end of file diff --git a/src/core/health.ts b/src/core/health.ts new file mode 100644 index 000000000..1ff988e51 --- /dev/null +++ b/src/core/health.ts @@ -0,0 +1,104 @@ +/** Pure self-diagnosis for pxpipe. No fs, no network, no clock — the caller + * passes a state snapshot (including any "recent traffic" aggregate) and gets + * back findings. Runs anywhere core runs. */ + +export type Severity = 'error' | 'warn' | 'info'; + +export interface Remediation { + kind: 'set-openai-upstream'; + target: 'https://chatgpt.com'; + durableHint: string; +} + +export interface HealthFinding { + id: string; + severity: Severity; + title: string; + detail: string; + remediation?: Remediation; +} + +export interface HealthRecentTraffic { + codexResponses404: number; + codexResponsesTotal: number; + windowSeconds: number; +} + +export interface HealthState { + anthropicUpstream: string; + openaiUpstream: string; + openaiUpstreamOverridden: boolean; + modelScope: string[]; + compressionEnabled: boolean; + recent: HealthRecentTraffic; +} + +const CHATGPT_DURABLE_HINT = + 'Set OPENAI_UPSTREAM=https://chatgpt.com (User env: setx OPENAI_UPSTREAM "https://chatgpt.com" then re-login, or in pxpipe-run.cmd) and restart pxpipe.'; + +/** Lower-cased hostname, or null when the URL cannot be parsed. */ +function hostOf(url: string): string | null { + try { + return new URL(url).hostname.toLowerCase(); + } catch { + return null; + } +} + +/** True only for chatgpt.com or a *.chatgpt.com subdomain. An unparseable URL + * is NOT chatgpt.com (so it surfaces as a mismatch rather than passing). */ +function isChatGPTHost(url: string): boolean { + const h = hostOf(url); + return h === 'chatgpt.com' || (h !== null && h.endsWith('.chatgpt.com')); +} + +export function evaluateHealth(state: HealthState): HealthFinding[] { + const findings: HealthFinding[] = []; + + if (!isChatGPTHost(state.openaiUpstream)) { + const confirmed = state.recent.codexResponses404 > 0; + findings.push({ + id: 'codex-upstream-mismatch', + severity: confirmed ? 'error' : 'warn', + title: confirmed + ? 'Codex requests are 404ing: OpenAI upstream is not chatgpt.com' + : 'OpenAI upstream is not chatgpt.com — Codex paths will 404', + detail: confirmed + ? `${state.recent.codexResponses404}/${state.recent.codexResponsesTotal} recent /backend-api/codex/* requests returned 404 while the OpenAI upstream is ${state.openaiUpstream}. That path only exists on chatgpt.com.` + : `The OpenAI upstream is ${state.openaiUpstream}. Codex uses /backend-api/codex/*, which exists only on chatgpt.com and 404s here. (No codex traffic seen yet.)`, + remediation: { + kind: 'set-openai-upstream', + target: 'https://chatgpt.com', + durableHint: CHATGPT_DURABLE_HINT, + }, + }); + } + + if (!state.compressionEnabled || state.modelScope.length === 0) { + findings.push({ + id: 'compression-passthrough', + severity: 'info', + title: 'Compression is not active — traffic passes through untransformed', + detail: !state.compressionEnabled + ? 'The dashboard compression kill-switch is off; every request forwards unchanged.' + : 'The model scope is empty, so no model is eligible for imaging; every request forwards unchanged.', + }); + } + + if (state.openaiUpstreamOverridden) { + findings.push({ + id: 'openai-upstream-overridden', + severity: 'info', + title: 'OpenAI upstream is overridden at runtime', + detail: `A runtime hot-swap is routing OpenAI traffic to ${state.openaiUpstream}. This is in-memory only; set OPENAI_UPSTREAM to make it durable.`, + }); + } + + return findings; +} + +/** ok = no error-severity findings. Maps to the /healthz status code. */ +export function summarizeHealth(findings: HealthFinding[]): { ok: boolean; httpStatus: 200 | 503 } { + const ok = !findings.some((f) => f.severity === 'error'); + return { ok, httpStatus: ok ? 200 : 503 }; +} diff --git a/src/core/proxy.ts b/src/core/proxy.ts index a657dead4..72e12c974 100644 --- a/src/core/proxy.ts +++ b/src/core/proxy.ts @@ -103,6 +103,9 @@ export interface ProxyEvent { /** Ground-truth char counts from the response stream, independent of usage.output_tokens. * Absent when the body couldn't be scanned (5xx, unknown content-type). See OutputMeasurement. */ measurement?: OutputMeasurement; + /** Upstream response media/encoding metadata for scanner diagnostics. */ + responseContentType?: string; + responseContentEncoding?: string; } /** Max chars of 4xx error body captured on ProxyEvent — enough for Anthropic's full error JSON. */ @@ -227,6 +230,45 @@ function withClientDisconnect( * past it stream through unbuffered. */ const MODEL_SNIFF_MAX_BYTES = 1 << 20; +/** Hard safety ceiling for request shapes pxpipe must buffer before transforming. + * This is deliberately separate from provider/model request limits: it bounds the + * proxy's own heap exposure, including when a chunked request has no Content-Length. + * The largest successful provider request seen in local telemetry is 11.2 MiB, so + * 16 MiB preserves measured traffic while preventing an unbounded allocation. */ +const TRANSFORM_BODY_MAX_BYTES = 16 * 1024 * 1024; + +async function readTransformBody(req: Request): Promise { + const reader = req.body?.getReader(); + if (!reader) return new Uint8Array(); + const chunks: Uint8Array[] = []; + let total = 0; + let tooLarge = false; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (tooLarge) continue; // drain without retaining bytes + total += value.byteLength; + if (total > TRANSFORM_BODY_MAX_BYTES) { + tooLarge = true; + chunks.length = 0; + continue; + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + if (tooLarge) return null; + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return body; +} + /** Read the actual top-level `model` field. The body is already buffered for * transformation, so parsing it is both safer and simpler than a prefix regex * (which could mistake `metadata.model` for the routing model). */ @@ -239,6 +281,53 @@ function readModelField(body: Uint8Array): string | null { } } +/** Responses defaults to non-streaming when `stream` is absent. */ +function readStreamField(body: Uint8Array): boolean { + try { + const parsed = JSON.parse(new TextDecoder().decode(body)) as { stream?: unknown }; + return parsed.stream === true; + } catch { + return false; + } +} + +/** Read only a bounded clone of a request while leaving its original body + * untouched for streaming passthrough. */ +async function readBoundedClone(req: Request): Promise { + const reader = req.clone().body?.getReader(); + if (!reader) return new Uint8Array(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + while (total <= MODEL_SNIFF_MAX_BYTES) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MODEL_SNIFF_MAX_BYTES) return null; + chunks.push(value); + } + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return body; + } catch { + return null; + } finally { + void reader.cancel().catch(() => undefined); + } +} + +/** Read only a bounded clone of a bypassed JSON request. Bypass must keep the + * original body byte-for-byte, but Responses telemetry still needs to know + * whether a headerless Codex reply is SSE or JSON. */ +async function readStreamFieldFromClone(req: Request): Promise { + const body = await readBoundedClone(req); + return body !== null && readStreamField(body); +} + // Claude Code only admits gateway-discovered ids beginning with "claude" or // "anthropic". Prefix provider ids for discovery, then remove that compatibility // prefix before PXPIPE_MODELS matching and upstream routing. @@ -316,22 +405,63 @@ function processSseEvent( return; } const obj = j as Record; + // The Responses wire format identifies events in the JSON `type` field. + // Some providers also emit an SSE `event:` line, but ChatGPT's + // /backend-api/codex/responses commonly does not. Prefer the official JSON + // discriminator when present and use the SSE name as a compatibility fallback. + const eventType = typeof obj.type === 'string' ? obj.type : event; // OpenAI chunks have no `event:` line; usage only present when stream_options.include_usage is set. const openAIUsage = normalizeUsage((obj as { usage?: unknown }).usage); if (openAIUsage) state.usage = openAIUsage; // OpenAI Responses API streams usage nested under `response` on the terminal - // `response.completed` (or `.incomplete`) event — not at the top level. - if (event === 'response.completed' || event === 'response.incomplete') { + // terminal Responses event — not at the top level. + if ( + eventType === 'response.completed' + || eventType === 'response.incomplete' + || eventType === 'response.failed' + ) { const resp = obj.response as - | { usage?: unknown; incomplete_details?: { reason?: unknown } } + | { + usage?: unknown; + incomplete_details?: { reason?: unknown }; + error?: { code?: unknown; message?: unknown }; + } | undefined; const respUsage = normalizeUsage(resp?.usage); if (respUsage) state.usage = respUsage; - // Responses API has no stop_reason; normalize the terminal status/reason instead. + // Responses API has no stop_reason; normalize terminal status/reason. + // Preserve a refusal observed in an earlier delta rather than overwriting it. const reason = resp?.incomplete_details?.reason; + const failureCode = resp?.error?.code; state.stopReason = typeof reason === 'string' ? reason - : event === 'response.incomplete' ? 'incomplete' : 'stop'; + : eventType === 'response.incomplete' ? 'incomplete' + : eventType === 'response.failed' + ? (typeof failureCode === 'string' ? failureCode : 'failed') + : state.stopReason ?? 'stop'; + } + // Responses streaming deltas are not Chat Completions `choices[]` chunks. + // Count their visible/reasoning/tool payloads directly from the JSON event. + if (eventType === 'response.refusal.delta' || eventType === 'response.refusal.done') { + // `.done` repeats the completed refusal text; only deltas are additive. + if (eventType === 'response.refusal.delta' && typeof obj.delta === 'string') { + m.textChars += obj.delta.length; + } + state.stopReason = 'refusal'; + } else if (eventType === 'response.output_text.delta' && typeof obj.delta === 'string') { + m.textChars += obj.delta.length; + } else if ( + (eventType === 'response.reasoning_text.delta' + || eventType === 'response.reasoning_summary_text.delta') + && typeof obj.delta === 'string' + ) { + m.thinkingChars += obj.delta.length; + } else if ( + (eventType === 'response.function_call_arguments.delta' + || eventType === 'response.custom_tool_call_input.delta') + && typeof obj.delta === 'string' + ) { + m.toolUseChars += obj.delta.length; } // Google AI Studio streaming chunks: usageMetadata object. if (obj.usageMetadata && typeof obj.usageMetadata === 'object') { @@ -350,14 +480,14 @@ function processSseEvent( } } - if (event === 'message_start') { + if (eventType === 'message_start') { const msg = obj.message as { usage?: Usage } | undefined; const usage = normalizeUsage(msg?.usage); if (usage) state.usage = usage; - } else if (event === 'content_block_start') { + } else if (eventType === 'content_block_start') { const cb = obj.content_block as { type?: string } | undefined; if (cb?.type === 'redacted_thinking') m.redactedBlockCount += 1; - } else if (event === 'content_block_delta') { + } else if (eventType === 'content_block_delta') { const d = obj.delta as | { type?: string; text?: string; thinking?: string; partial_json?: string } | undefined; @@ -368,7 +498,7 @@ function processSseEvent( } else if (d?.type === 'input_json_delta' && typeof d.partial_json === 'string') { m.toolUseChars += d.partial_json.length; } - } else if (event === 'message_delta') { + } else if (eventType === 'message_delta') { // Anthropic ships the final stop_reason here ("end_turn", "refusal", …). const d = obj.delta as { stop_reason?: unknown } | undefined; if (typeof d?.stop_reason === 'string') state.stopReason = d.stop_reason; @@ -458,6 +588,18 @@ function normalizeUsage(raw: unknown): Usage | undefined { if (details && typeof details.cached_tokens === 'number') { out.cached_tokens = details.cached_tokens; } + // Some OpenAI-compatible providers additionally expose prompt-cache writes. + // Like cached_tokens, this is a diagnostic subset of input_tokens: consumers + // must not add it to the input total a second time. + if (details && typeof details.cache_write_tokens === 'number') { + out.cache_write_tokens = details.cache_write_tokens; + } + const outputDetails = + (u.output_tokens_details as Record | undefined) ?? + (u.completion_tokens_details as Record | undefined); + if (outputDetails && typeof outputDetails.reasoning_tokens === 'number') { + out.reasoning_output_tokens = outputDetails.reasoning_tokens; + } return Object.keys(out).length > 0 ? out : undefined; } @@ -600,7 +742,11 @@ function readStopReasonFromJson(j: unknown): string | undefined { * Streams are scanned to EOF (final output_tokens is in message_delta; redacted_thinking * blocks can appear anywhere). 4xx bodies are capped at ERROR_BODY_MAX. 5xx is skipped. */ -function teeForUsage(res: Response): { +function teeForUsage( + res: Response, + assumeResponsesSse = false, + assumeResponsesJson = false, +): { response: Response; usagePromise: Promise; errorBodyPromise: Promise; @@ -677,7 +823,10 @@ function teeForUsage(res: Response): { let buf = ''; try { - if (ct.includes('text/event-stream')) { + // ChatGPT's /backend-api/codex/responses currently omits Content-Type. + // The request's top-level `stream` field selects SSE vs JSON for that + // route; explicit unknown media types still fail closed. + if (ct.includes('text/event-stream') || (ct === '' && assumeResponsesSse)) { // Walk every SSE event to EOF — message_delta (final output_tokens) is last. const m: OutputMeasurement = { textChars: 0, @@ -693,20 +842,30 @@ function teeForUsage(res: Response): { const { done, value } = await reader.read(); if (done) break; buf += decoder.decode(value, { stream: true }); - // SSE events are terminated by a blank line; support LF and CRLF. + // SSE events are terminated by a blank line; support LF, CRLF, and CR. let boundary: RegExpExecArray | null; while ((boundary = /\r\n\r\n|\n\n|\r\r/.exec(buf)) !== null) { const block = buf.slice(0, boundary.index); buf = buf.slice(boundary.index + boundary[0].length); processSseEvent(block, m, state); } + // A malformed/headerless non-SSE response must not make the audit + // branch buffer without bound. Ordinary Responses events are far + // smaller; fail closed and drain if no delimiter arrives within 4 MiB. + if (buf.length > 4 * 1024 * 1024) { + while (true) { + const { done: drained } = await reader.read(); + if (drained) break; + } + return { usage: undefined, measurement: undefined, stopReason: undefined }; + } } buf += decoder.decode(); if (buf.trim().length > 0) processSseEvent(buf, m, state); // trailing partial event return { usage: state.usage, measurement: m, stopReason: state.stopReason }; } - if (ct.includes('application/json')) { + if (ct.includes('application/json') || (ct === '' && assumeResponsesJson)) { // Buffer fully, capped at 4 MiB. const MAX = 4 * 1024 * 1024; while (buf.length < MAX) { @@ -830,7 +989,18 @@ function isOpenAIChatPath(pathname: string): boolean { } function isOpenAIResponsesPath(pathname: string): boolean { - return OPENAI_RESPONSES_PATH.test(pathname); + return OPENAI_RESPONSES_PATH.test(pathname) + || pathname === '/backend-api/codex/responses'; +} + +/** ChatGPT endpoints used by Codex when `requires_openai_auth = true` is set + * on a custom provider. They must go to OPENAI_UPSTREAM, never the Anthropic + * default. Only `responses` is transformed; the models catalogue is forwarded + * byte-for-byte so Codex can refresh its model list without noisy 404s. */ +function isChatGPTCodexPath(pathname: string): boolean { + return pathname === '/backend-api/codex/responses' + || pathname === '/backend-api/codex/models' + || pathname.startsWith('/backend-api/codex/models/'); } function isCanonicalOpenAIPath(pathname: string, headers: Headers, hasOpenAIKey: boolean): boolean { @@ -846,6 +1016,7 @@ function isCanonicalOpenAIPath(pathname: string, headers: Headers, hasOpenAIKey: return pathname === '/v1/chat/completions' || pathname === '/v1/responses' || pathname.startsWith('/v1/responses/') + || isChatGPTCodexPath(pathname) || (isModelsPath && looksOpenAIAuth); } @@ -900,14 +1071,27 @@ async function countGoogleTokensUpstream( } } -/** Resolve upstream URLs from config. Pure — unit-testable. */ +/** + * Resolve upstream URLs from config. Pure — unit-testable. + * + * Every env-derived input is trimmed of leading/trailing whitespace before + * URL construction, and trailing slashes are stripped so `base + path` joins + * cleanly. The trim is defensive: a stray space in OPENAI_UPSTREAM / + * ANTHROPIC_UPSTREAM / PXPIPE_GATEWAY_BASE_URL (commonly introduced by + * cmd.exe `set VAR=...`, a copy-paste with a trailing space, or a shell + * quoting bug in a launcher script) would otherwise build URLs like + * "https://api.openai.com /v1/..." — fetch() then throws + * "Failed to parse URL" with no actionable log line, and the operator is + * left guessing. Trimming at the URL boundary keeps the failure mode loud + * (the proxy still returns a real 401/502 from upstream) instead of silent. + */ export function resolveUpstreams(config: ProxyConfig): { anthropic: string; openai: string; stripOpenAIV1: boolean; } { if (config.provider === 'cloudflare-ai-gateway') { - const base = (config.gatewayBaseUrl ?? '').replace(/\/+$/, ''); + const base = (config.gatewayBaseUrl ?? '').trim().replace(/\/+$/, ''); if (!base) { throw new Error( "provider 'cloudflare-ai-gateway' requires gatewayBaseUrl (PXPIPE_GATEWAY_BASE_URL)", @@ -916,8 +1100,8 @@ export function resolveUpstreams(config: ProxyConfig): { return { anthropic: `${base}/anthropic`, openai: `${base}/openai`, stripOpenAIV1: true }; } return { - anthropic: (config.upstream ?? DEFAULT_UPSTREAM).replace(/\/+$/, ''), - openai: (config.openAIUpstream ?? DEFAULT_OPENAI_UPSTREAM).replace(/\/+$/, ''), + anthropic: (config.upstream ?? DEFAULT_UPSTREAM).trim().replace(/\/+$/, ''), + openai: (config.openAIUpstream ?? DEFAULT_OPENAI_UPSTREAM).trim().replace(/\/+$/, ''), stripOpenAIV1: false, }; } @@ -969,8 +1153,11 @@ export function createProxy(config: ProxyConfig = {}) { const routes = resolveUpstreams(config); const upstream = routes.anthropic; const openAIUpstream = routes.openai; + // Same trim policy as resolveUpstreams: keep provider-prefixed passthroughs + // (e.g. /openai/*, /google-ai-studio/*) safe from env whitespace — see the + // JSDoc on resolveUpstreams for the full rationale. const passthroughUpstream = config.provider === 'cloudflare-ai-gateway' - ? (config.gatewayBaseUrl ?? '').replace(/\/+$/, '') + ? (config.gatewayBaseUrl ?? '').trim().replace(/\/+$/, '') : upstream; const gatewayHeaders = config.gatewayHeaders ?? {}; const applyGatewayHeaders = (h: Headers): Headers => { @@ -1006,6 +1193,8 @@ export function createProxy(config: ProxyConfig = {}) { // reqBodyBytes: kept for lazy gzip on 4xx. reqBodySha8: computed eagerly for correlation. let reqBodyBytes: Uint8Array | undefined; let reqBodySha8: string | undefined; +let responseContentType: string | undefined; + let responseContentEncoding: string | undefined; let reqBodySha256: string | undefined; const fire = ( @@ -1082,7 +1271,7 @@ export function createProxy(config: ProxyConfig = {}) { // GPT image/baseline telemetry and renders As text / Saved as dashes. accountingProvider: isGoogleRoute ? 'google' - : isOpenAIChat || isOpenAIResponses || bridgedGptMessages || bridgedChatMessages + : isOpenAIChatWire || isOpenAIResponsesWire || bridgedGptMessages || bridgedChatMessages ? 'openai' : 'anthropic', status, @@ -1096,6 +1285,8 @@ export function createProxy(config: ProxyConfig = {}) { reqBodyGz, measurement, stopReason, + responseContentType, + responseContentEncoding, }); }; void finalize(); @@ -1110,9 +1301,15 @@ export function createProxy(config: ProxyConfig = {}) { const bypassHeader = req.headers.get('x-pxpipe-bypass'); const bypass = bypassHeader !== null && !/^(?:0|false|off|no)$/i.test(bypassHeader.trim()); const providerPrefixed = isProviderPrefixedPath(url.pathname); - const isMessages = !bypass && req.method === 'POST' && isAnthropicMessagesPath(url.pathname); - const isOpenAIChat = !bypass && req.method === 'POST' && isOpenAIChatPath(url.pathname); - const isOpenAIResponses = !bypass && req.method === 'POST' && isOpenAIResponsesPath(url.pathname); + // Wire-shape detection stays independent from transform eligibility. A + // bypassed Codex request is still an OpenAI Responses request for routing, + // accounting, and response scanning; only its body transformation is off. + const isMessagesWire = req.method === 'POST' && isAnthropicMessagesPath(url.pathname); + const isOpenAIChatWire = req.method === 'POST' && isOpenAIChatPath(url.pathname); + const isOpenAIResponsesWire = req.method === 'POST' && isOpenAIResponsesPath(url.pathname); + const isMessages = !bypass && isMessagesWire; + const isOpenAIChat = !bypass && isOpenAIChatWire; + const isOpenAIResponses = !bypass && isOpenAIResponsesWire; const googleModel = req.method === 'POST' ? parseGoogleModelFromPath(url.pathname) : null; @@ -1141,14 +1338,32 @@ export function createProxy(config: ProxyConfig = {}) { let baselinePromise: Promise | undefined; let baselineCacheablePromise: Promise | undefined; let baselineStatusApplies = false; + let responsesStreaming = false; + + if (bypass && isOpenAIResponsesWire + && (req.headers.get('content-type') ?? '').toLowerCase().includes('json')) { + responsesStreaming = await readStreamFieldFromClone(req); + } if (isMessages || isOpenAIChat || isOpenAIResponses || isGoogle) { - const bodyIn = new Uint8Array(await req.arrayBuffer()); + const bodyIn = await readTransformBody(req); + if (bodyIn === null) { + const message = `pxpipe request body exceeds safety limit (${TRANSFORM_BODY_MAX_BYTES} bytes)`; + fire(413, undefined, message); + const error = isMessages + ? { type: 'error', error: { type: 'request_too_large', message } } + : { error: { type: 'request_too_large', message } }; + return new Response(JSON.stringify(error), { + status: 413, + headers: { 'content-type': 'application/json' }, + }); + } try { const transformOpts = typeof config.transform === 'function' ? config.transform() : config.transform; // Fail-closed: unreadable model → no compression, not a risky guess. - const model = googleModel ?? readModelField(bodyIn); +const model = googleModel ?? readModelField(bodyIn); + if (isOpenAIResponses) responsesStreaming = readStreamField(bodyIn); requestModel = model ?? undefined; // A turn whose only content is `@pxpipe pin` / `@pxpipe unpin` is // configuration, not a question. Answer it here: forwarding it would bill @@ -1354,12 +1569,12 @@ export function createProxy(config: ProxyConfig = {}) { declaredType.toLowerCase().includes('json') && !(Number.isFinite(declaredLength) && declaredLength > MODEL_SNIFF_MAX_BYTES); if (worthBuffering) { - const bodyIn = new Uint8Array(await req.arrayBuffer()); - requestModel ??= readModelField(bodyIn) ?? undefined; - bodyOut = bodyIn; - } else { - bodyOut = req.body; // pass through unchanged, model stays unknown + const sniffed = await readBoundedClone(req); + if (sniffed !== null) requestModel ??= readModelField(sniffed) ?? undefined; } + // Label-only inspection must never consume/buffer the actual request. A + // large or chunked JSON body remains byte-for-byte streaming passthrough. + bodyOut = req.body; } else { bodyOut = req.body; // pass through unchanged } @@ -1379,7 +1594,17 @@ export function createProxy(config: ProxyConfig = {}) { const bridgeKey = bridgedChatMessages ? config.cloudflareApiKey : config.openAIApiKey; - if (bridgeKey) outHeaders.set('authorization', `Bearer ${bridgeKey}`); + // Codex talks to chatgpt.com with the client's ChatGPT OAuth bearer. + // OPENAI_API_KEY is for canonical /v1 OpenAI endpoints only; replacing + // Codex's bearer here makes an otherwise valid signed-in session 401. + const isCodexPath = isChatGPTCodexPath(url.pathname); + const inboundBearerIsAnthropic = /^Bearer\s+sk-ant-/i.test(outHeaders.get('authorization') ?? ''); + if (isCodexPath && inboundBearerIsAnthropic) outHeaders.delete('authorization'); + if (bridgeKey && ( + !isCodexPath || inboundBearerIsAnthropic || !outHeaders.has('authorization') + )) { + outHeaders.set('authorization', `Bearer ${bridgeKey}`); + } } else if (config.apiKey && (!providerPrefixed || url.pathname.startsWith('/anthropic/'))) { outHeaders.set('x-api-key', config.apiKey); } @@ -1521,16 +1746,22 @@ export function createProxy(config: ProxyConfig = {}) { } const firstByteMs = Date.now() - t0; + responseContentType = upstreamRes.headers.get('content-type') ?? undefined; + responseContentEncoding = upstreamRes.headers.get('content-encoding') ?? undefined; // Tee: client gets one side; scanner reads the other for usage/measurement/error body. - let teed: Response; +let teed: Response; let usagePromise: Promise; let errorBodyPromise: Promise; let measurementPromise: Promise; let stopReasonPromise: Promise; try { ({ response: teed, usagePromise, errorBodyPromise, measurementPromise, stopReasonPromise } = - teeForUsage(upstreamRes)); + teeForUsage( + upstreamRes, + isOpenAIResponsesWire && responsesStreaming, + isOpenAIResponsesWire && !responsesStreaming, + )); } catch (e) { releaseInFlight(); throw e; diff --git a/src/core/tracker.ts b/src/core/tracker.ts index 4c59b7754..81fd760fa 100644 --- a/src/core/tracker.ts +++ b/src/core/tracker.ts @@ -118,6 +118,10 @@ export interface TrackEvent { cache_read_tokens?: number; /** OpenAI prompt-cache hits (subset of input_tokens), from input/prompt_tokens_details.cached_tokens. */ cached_tokens?: number; + /** OpenAI prompt-cache writes (diagnostic subset of input_tokens; not additive). */ + cache_write_tokens?: number; + /** OpenAI reasoning tokens, a subset of output_tokens (not additive). */ + reasoning_output_tokens?: number; /** Cache_create split by tier — 1.25x (5-min) and 2x (1-hour) input rates. * Their sum equals `cache_create_tokens` when both fields are present. */ cache_create_5m_tokens?: number; @@ -161,6 +165,8 @@ export interface TrackEvent { req_body_sample_b64?: string; /** Node host only: path to gzipped sidecar when inline cap exceeded. Workers drop oversized samples. */ req_body_sample_path?: string; + response_content_type?: string; + response_content_encoding?: string; } /** Max inline base64 body per JSONL row (32 KiB). Larger goes to sidecar (Node) or is dropped (Workers). */ @@ -191,6 +197,8 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { if (ev.error) out.error = ev.error; if (ev.errorBody) out.error_body = ev.errorBody; if (ev.reqBodySha8) out.req_body_sha8 = ev.reqBodySha8; + if (ev.responseContentType) out.response_content_type = ev.responseContentType; + if (ev.responseContentEncoding) out.response_content_encoding = ev.responseContentEncoding; // Body sample: sidecar path (Node) > inline base64 if it fits > drop (Workers, oversized). if (ev.reqBodySamplePath) { out.req_body_sample_path = ev.reqBodySamplePath; @@ -314,6 +322,10 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { out.cache_read_tokens = u.cache_read_input_tokens; if (u.cached_tokens !== undefined) out.cached_tokens = u.cached_tokens; + if (u.cache_write_tokens !== undefined) + out.cache_write_tokens = u.cache_write_tokens; + if (u.reasoning_output_tokens !== undefined) + out.reasoning_output_tokens = u.reasoning_output_tokens; // cache_creation splits cache_creation_input_tokens across 5-min (1.25x) and 1-hour (2x) tiers. if (u.cache_creation) { if (u.cache_creation.ephemeral_5m_input_tokens !== undefined) diff --git a/src/core/transform.ts b/src/core/transform.ts index 5ec69c95f..988640bc1 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -1760,17 +1760,12 @@ export async function transformRequest( if (o.compressTools && Array.isArray(req.tools) && req.tools.length > 0) { const docs: string[] = []; toolsRewritten = req.tools.map((t) => { - // #43: native server-side tools (versioned `type`, e.g. advisor_20260301, - // web_search_20250305) have a fixed API schema that rejects a `description` - // field on the entry ("Extra inputs are not permitted" → 400). Only - // client-defined tools — no `type`, or the explicit type:"custom" shape — - // accept description/input_schema, so everything else passes through - // byte-identical and stays out of the imaged reference (its docs live - // server-side; there is nothing to compress). Mirrors the OpenAI-path - // guard (openai.ts isFunctionTool). +// Native server-side tools (versioned `type`, e.g. advisor_20260301, + // web_search_20250305) reject client-defined descriptions. The explicit + // type:"custom" shape is safe to transform; every other typed tool passes + // through byte-identical. Deferred tools also stay out of context until + // selected by Anthropic tool search, so imaging their docs would defeat it. if (t.type !== undefined && t.type !== 'custom') return t; - // Anthropic excludes deferred tool definitions from model context until - // selected by tool search; imaging their docs would defeat that behavior. if ((t as ToolDef & { defer_loading?: boolean }).defer_loading === true) return t; docs.push(renderToolDoc(t)); // tools[] keeps the annotation-STRIPPED schema: structure (type/properties/ diff --git a/src/core/types.ts b/src/core/types.ts index 990a03ed8..0b842ec47 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -80,6 +80,12 @@ export interface Usage { * (billed at ~0.1× for the gpt-5 family). Mapped from `input_tokens_details` * / `prompt_tokens_details.cached_tokens`. Anthropic uses cache_read instead. */ cached_tokens?: number; + /** OpenAI prompt-cache writes reported by compatible providers. This is a + * diagnostic subset of `input_tokens`, never an additional token charge. */ + cache_write_tokens?: number; + /** OpenAI reasoning tokens — a subset of output_tokens, exposed separately + * for observability and never added to output_tokens a second time. */ + reasoning_output_tokens?: number; } export interface MessagesRequest { diff --git a/src/dashboard.ts b/src/dashboard.ts index f6a21e7c5..dbbe9e905 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -30,9 +30,12 @@ import * as fs from 'node:fs'; import * as readline from 'node:readline'; +import * as crypto from 'node:crypto'; import type { ProxyEvent } from './core/proxy.js'; import type { TrackEvent } from './core/tracker.js'; +import type { CodexUsageSnapshot } from './codex-usage.js'; import { + cacheCreateUnknownTokens, computeActualInputEffWithCacheTier, computeBaselineInputEffWithCacheTier, deriveBaselineWarmth, @@ -59,6 +62,7 @@ import { renderPage, renderToggleFragment, renderModelsFragment, + KNOWN_MODEL_SCOPE_IDS, renderContextMapFragment, renderSessionSummaryFragment, renderHeaderFragment, @@ -67,11 +71,12 @@ import { renderSessionsFragment, renderStatsTableFragment, type ContextMapData, + type ManualCalibrationStatus, } from './dashboard/fragments.js'; import { getAllowedModelBases, getConfiguredModelBases, - isPxpipeSupportedModel, + isModelScopeEnabled, setAllowedModelBases, } from './core/applicability.js'; import type { @@ -269,8 +274,12 @@ interface Totals { * numerator. Included in the denominator so the headline drops honestly * toward zero on output-heavy workloads. */ allOutputWeighted: number; - /** Count of requests that contributed to allActualInputWeighted (had a - * usage block). Lets the UI annotate "N of M paid requests". */ + /** Count of completed responses with non-zero provider usage. These are the + * responses that contributed to the actual/counterfactual paid-traffic + * totals; the explicit name avoids implying that every proxy request was + * billable or carried usage. */ + usageBearingResponses: number; + /** Upstream model-scope aggregate name; equal to usageBearingResponses. */ allUsageRequests: number; /** Direct compressed-vs-passthrough actual-cost split. No counterfactuals, * no probe gating — just sum what each path actually billed. @@ -288,6 +297,31 @@ interface Totals { passthroughPaidRequests: number; passthroughActualInputWeighted: number; passthroughOutputWeighted: number; + /** Audit coverage for the cache-create price tier reported by Anthropic. */ + cacheCreate5mTokens: number; + cacheCreate1hTokens: number; + cacheCreateTierUnknownTokens: number; + measuredCacheCreateTierUnknownTokens: number; + /** Rows actually eligible for the measured Claude counterfactual numerator. */ + measuredSavingsRequests: number; + /** Claude savings backed by count_tokens + provider usage, in input-token + * equivalents. Kept separate from the locally modeled OpenAI estimate. */ + measuredClaudeSavedInputEquivalents: number; + /** GPT/OpenAI rows use local tokenizer/vision math, never the Claude headline. */ + estimatedOpenAISavingsRequests: number; + /** OpenAI/Responses savings modeled from local tokenizer/vision accounting, + * in input-token equivalents. Never presented as measured Claude savings. */ + modeledOpenAISavedInputEquivalents: number; + /** Gemini/Google rows have provider usage, but their text-versus-image + * baseline is either optional-provider-probe or local transform accounting. + * Keep them visible as their own unpriced evidence bucket. */ + estimatedGoogleSavingsRequests: number; + modeledGoogleSavedInputEquivalents: number; + /** Paid compressed rows deliberately excluded because their baseline probe was unavailable. */ + baselineProbeExcludedRequests: number; + pricedMeasuredSavingsRequests: number; + unpricedMeasuredSavingsRequests: number; + pricedMeasuredSavingsUsd: number; /** Sum of ground-truth output character counts from the SSE/JSON scanner * (see `OutputMeasurement` in proxy.ts). These three accumulators are * independent of Anthropic's `usage.output_tokens` — they let the operator @@ -320,12 +354,27 @@ function emptyTotals(startedAt = Date.now() / 1000): Totals { allActualInputWeighted: 0, allOutputWeighted: 0, allUsageRequests: 0, + usageBearingResponses: 0, compressedPaidRequests: 0, compressedActualInputWeighted: 0, compressedOutputWeighted: 0, passthroughPaidRequests: 0, passthroughActualInputWeighted: 0, passthroughOutputWeighted: 0, + cacheCreate5mTokens: 0, + cacheCreate1hTokens: 0, + cacheCreateTierUnknownTokens: 0, + measuredCacheCreateTierUnknownTokens: 0, + measuredSavingsRequests: 0, + measuredClaudeSavedInputEquivalents: 0, + estimatedOpenAISavingsRequests: 0, + modeledOpenAISavedInputEquivalents: 0, + estimatedGoogleSavingsRequests: 0, + modeledGoogleSavedInputEquivalents: 0, + baselineProbeExcludedRequests: 0, + pricedMeasuredSavingsRequests: 0, + unpricedMeasuredSavingsRequests: 0, + pricedMeasuredSavingsUsd: 0, textCharsMeasured: 0, thinkingCharsMeasured: 0, toolUseCharsMeasured: 0, @@ -380,6 +429,50 @@ const OUTPUT_TOKEN_RATE = 5.0; // understates the real bill - treat it as "input-side $ saved". export const ASSUMED_INPUT_USD_PER_MTOK = 10.0; +/** Per-model list-price conversion for the dollar display. A private gateway + * can override an exact model with PXPIPE_MODEL_INPUT_USD_PER_MTOK JSON. */ +const MODEL_INPUT_USD_PER_MTOK: ReadonlyArray<[prefix: string, usd: number]> = [ + ['claude-fable-5', 10], + ['claude-opus-', 5], + ['claude-sonnet-', 3], + ['claude-haiku-', 1], +]; +/** Sonnet 5 introductory price is valid through 2026-08-31 inclusive. */ +const SONNET_5_INTRO_END_MS = Date.UTC(2026, 8, 1); + +/** Pure official-list-price lookup, exported so the scheduled transition is testable. */ +export function officialInputUsdPerMtokForModel( + model: string | undefined, + atMs = Date.now(), +): number | undefined { + const m = (model ?? '').toLowerCase(); + if (m.startsWith('claude-sonnet-5')) { + return atMs < SONNET_5_INTRO_END_MS ? 2 : 3; + } + return MODEL_INPUT_USD_PER_MTOK.find(([prefix]) => m.startsWith(prefix))?.[1]; +} + +let modelPriceOverrides: Record | undefined; +function inputUsdPerMtokForModel( + model: string | undefined, + atMs = Date.now(), +): number | undefined { + if (modelPriceOverrides === undefined) { + try { + const raw = process.env.PXPIPE_MODEL_INPUT_USD_PER_MTOK; + const parsed = raw ? JSON.parse(raw) : {}; + modelPriceOverrides = Object.fromEntries( + Object.entries(parsed as Record) + .filter(([, v]) => typeof v === 'number' && Number.isFinite(v) && v >= 0), + ) as Record; + } catch { modelPriceOverrides = {}; } + } + const m = (model ?? '').toLowerCase(); + const override = modelPriceOverrides[m]; + if (override !== undefined) return override; + return officialInputUsdPerMtokForModel(m, atMs); +} + /** Route per-event accounting by upstream. OpenAI paths use the GPT cost * model (vision-token imaging, automatic 0.1× prefix cache, no count_tokens * probe, 8× output); everything else uses the Anthropic cache-aware baseline. @@ -495,12 +588,16 @@ function gptEff(args: { export class DashboardState { private recent: RecentRow[] = []; - /** Per-session dollar-weighted totals, keyed by `info.firstUserSha8`. The + /** Per-session dollar-weighted totals, keyed by a first-user/project cohort. The * dashboard surfaces ONLY the most-recently-active session via the * `serveCurrentSessionJson` endpoint — older sessions linger in the Map * so a tab refresh during a brief lull still finds the previous session, * but get evicted at `SESSION_CAP` to bound memory in long-running hosts. */ private sessions: Map = new Map(); + /** First known project for each first-user fingerprint. A fingerprint is not + * globally unique: a second project must not share live totals or cache warmth + * with the first one. Mirrors the disk-backed session aggregation policy. */ + private readonly primaryProjectByFirstUser = new Map(); /** sha8 of the most-recently-active session id. null when no events have * ever carried a `firstUserSha8` (e.g. a cold start with only passthrough * hits that the upstream probe never tagged). */ @@ -518,9 +615,9 @@ export class DashboardState { * comfortably under a MB even with fat bucket/passthrough histograms. */ private static readonly SESSION_CAP = 50; private readonly startedAt = Date.now() / 1000; - /** Lifetime accounting partitioned by exact model id. The dashboard sums - * only currently enabled model families, so toggling a model updates the - * overall numbers without discarding its history. */ + /** Lifetime accounting partitioned by exact model id. Overview combines + * every since-restart row: scope controls future transforms, not whether a + * later settings click erases prior traffic from the displayed totals. */ private readonly totalsByModel = new Map(); /** Bounded ring of the most recently rendered images (last IMAGE_RING_CAP). * Each request that rendered an image pushes one entry; the matching @@ -544,15 +641,45 @@ export class DashboardState { * matches Fable. Verbatim recall is still lossy; the dashboard toggle * remains the kill switch. See FINDINGS.md. */ private compressionEnabled = true; + /** + * Operator-started comparison note. There is deliberately no sampler: a + * baseline begins only when the user presses the existing kill switch. + * It is in-memory because historic passthrough traffic has no provenance. + */ + private manualCalibration: ManualCalibrationStatus = { + active: false, + phase: null, + baselineRequests: 0, + imagedRequests: 0, + scopeModel: null, + scopeSession: null, + skippedMismatches: 0, + }; /** Recent requests' transform breakdowns, for the Context Map panel + its * history selector. In-memory ring, newest last. */ private contextHistory: ContextMapData[] = []; setCompressionEnabled(on: boolean): void { + if (!on && this.compressionEnabled) { + this.manualCalibration = { + active: true, + phase: 'baseline', + baselineRequests: 0, + imagedRequests: 0, + scopeModel: null, + scopeSession: null, + skippedMismatches: 0, + }; + } else if (on && !this.compressionEnabled && this.manualCalibration.active) { + this.manualCalibration.phase = 'imaged'; + } this.compressionEnabled = on; } getCompressionEnabled(): boolean { return this.compressionEnabled; } + getManualCalibration(): ManualCalibrationStatus { + return { ...this.manualCalibration }; + } /** Resolved disk paths for the events.jsonl + 4xx-bodies sidecar dir. The * new sessions / cleanup endpoints need this; legacy callers that don't * pass `paths` opt out of those endpoints by returning 503. */ @@ -563,21 +690,33 @@ export class DashboardState { * path. Lets unit tests run in tens of ms instead of scanning hundreds of * the developer's actual Claude Code session files. */ private readonly ccMapFn: () => Promise>; + /** Cached official Codex rollout usage supplied by the Node host. Tests and + * non-Codex callers use an empty snapshot and never touch ~/.codex. */ + private readonly codexUsageFn: () => CodexUsageSnapshot; /** Host-provided persistence hook for the runtime model scope. The core * override stays in-memory (Edge-safe); a Node host passes a saver that * writes the `models` key of the config file so chip toggles survive a * restart. Best-effort: failures are the hook's problem, never the API's. */ - private readonly persistModelBases: ((bases: readonly string[]) => void) | undefined; + /** `null` means Reset: remove the host's persisted override and fall back to + * its configured/default scope. */ + private readonly persistModelBases: ((bases: readonly string[] | null) => void) | undefined; constructor( paths?: SessionsPaths, ccMapFn?: () => Promise>, - persistModelBases?: (bases: readonly string[]) => void, + persistModelBases?: (bases: readonly string[] | null) => void, + codexUsageFn?: () => CodexUsageSnapshot, ) { this.paths = paths; this.ccMapFn = ccMapFn ?? (() => claudeCodeMap()); this.persistModelBases = persistModelBases; + this.codexUsageFn = codexUsageFn ?? (() => ({ + source: '', loading: false, error: null, sessionFiles: 0, usageSnapshots: 0, + inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, + reasoningOutputTokens: 0, totalTokens: 0, modelContextWindow: null, + earliestEventAt: null, latestEventAt: null, rateLimits: null, quotaWindows: [], + })); } private totalsForModel(model: string | undefined): Totals { @@ -590,10 +729,25 @@ export class DashboardState { return totals; } - private enabledTotals(): Totals { + /** Derive the same best-effort cohort key used by the disk sessions view. */ + private sessionCohortId(info: ProxyEvent['info']): string | undefined { + const firstUser = info?.firstUserSha8; + if (!firstUser) return undefined; + const project = info.env?.cwd; + if (!this.primaryProjectByFirstUser.has(firstUser)) { + this.primaryProjectByFirstUser.set(firstUser, project); + } else if (!this.primaryProjectByFirstUser.get(firstUser) && project) { + this.primaryProjectByFirstUser.set(firstUser, project); + } + const primaryProject = this.primaryProjectByFirstUser.get(firstUser); + if (!project || !primaryProject || project === primaryProject) return firstUser; + const suffix = crypto.createHash('sha256').update(project, 'utf8').digest('hex').slice(0, 8); + return `${firstUser}@${suffix}`; + } + + private lifetimeTotals(): Totals { const combined = emptyTotals(this.startedAt); - for (const [model, totals] of this.totalsByModel) { - if (!isPxpipeSupportedModel(model)) continue; + for (const totals of this.totalsByModel.values()) { for (const key of Object.keys(combined) as Array) { if (key !== 'startedAt') combined[key] += totals[key]; } @@ -655,6 +809,7 @@ export class DashboardState { const u = ev.usage; const info = ev.info; + const liveSessionId = this.sessionCohortId(info); const compressed = info?.compressed === true; const totals = this.totalsForModel(ev.model); @@ -664,9 +819,46 @@ export class DashboardState { const cc5m = u?.cache_creation?.ephemeral_5m_input_tokens; const cc1h = u?.cache_creation?.ephemeral_1h_input_tokens ?? 0; const cr = u?.cache_read_input_tokens ?? 0; + const cacheCreate = u?.cache_creation + ? { + fiveMinuteTokens: u.cache_creation.ephemeral_5m_input_tokens, + oneHourTokens: u.cache_creation.ephemeral_1h_input_tokens, + } + : undefined; const gpt = isOpenAIEvent(ev.path, ev.accountingProvider); const google = ev.accountingProvider === 'google' - || ev.path.includes('google-ai-studio') || ev.path.includes('generateContent'); + || (ev.path ?? '').includes('google-ai-studio') || (ev.path ?? '').includes('generateContent'); + + // Count only completed, usage-bearing requests after an operator explicitly + // started the manual baseline/image phases. This never changes routing and + // intentionally does not retrofit ordinary historical passthrough events. + const hasAnyUsage = u !== undefined && + ((u.input_tokens ?? 0) > 0 || (u.output_tokens ?? 0) > 0 || + (u.cache_creation_input_tokens ?? 0) > 0 || (u.cache_read_input_tokens ?? 0) > 0 || + (u.cached_tokens ?? 0) > 0); + // The manual comparison is currently a Claude/count_tokens cohort. Do not + // mix OpenAI/Codex usage into counters that the UI presents as a text-vs- + // image Claude calibration; those providers use a different baseline. + if (!gpt && this.manualCalibration.active && hasAnyUsage) { + const calibrationModel = (ev.model ?? '').toLowerCase(); + const calibrationSession = info?.firstUserSha8 ?? ''; + const c = this.manualCalibration; + if (!calibrationModel || !calibrationSession) { + c.skippedMismatches += 1; + } else { + // The first eligible baseline request locks a comparable cohort. The + // image phase then accepts only the same Claude model and session. + if (c.scopeModel === null && c.phase === 'baseline' && !compressed) { + c.scopeModel = calibrationModel; + c.scopeSession = calibrationSession; + } + const inScope = c.scopeModel === calibrationModel + && c.scopeSession === calibrationSession; + if (!inScope) c.skippedMismatches += 1; + else if (c.phase === 'baseline' && !compressed) c.baselineRequests += 1; + else if (c.phase === 'imaged' && compressed) c.imagedRequests += 1; + } + } // Unified per-row accounting, filled by the provider branch below. The // downstream totals / per-session / recent-row code reads only these — @@ -775,7 +967,7 @@ export class DashboardState { // reused/grown split for the text baseline. Use request start for that // lookup; an overlapping request that had not completed could not provide a // prior prefix size for this in-flight request. - const sidNow = info?.firstUserSha8; + const sidNow = liveSessionId; const prefixShaNow = info?.systemSha8; const completionSec = Date.now() / 1000; const requestStartSec = completionSec - Math.max(0, ev.durationMs || 0) / 1000; @@ -880,16 +1072,48 @@ export class DashboardState { // baseline. An // uncompressed row contributes zero saved (baseline === actual), so // including it here would only dilute the "saved on rows we moved" %. - const dollarEligible = !google; + // Only Claude rows with a known model-specific price may enter dollar + // totals. OpenAI/Responses and Google rows still enter token, activity, + // and session accounting, but their savings are deliberately disclosed as + // modeled/unpriced rather than silently repriced at a Claude fallback. + const inputUsdPerMtok = !gpt && !google ? inputUsdPerMtokForModel(ev.model) : undefined; + const pricedEligible = inputUsdPerMtok !== undefined; if (creditSaving) { + const savedInputEquivalents = baselineInputEff - actualInputEff; totals.baselineInputWeighted += baselineInputEff; totals.actualInputWeighted += actualInputEff; totals.outputWeighted += outputEquiv; - if (dollarEligible) { + if (pricedEligible) { totals.pricedBaselineInputWeighted += baselineInputEff; totals.pricedActualInputWeighted += actualInputEff; totals.pricedOutputWeighted += outputEquiv; } + if (gpt) { + // Keep locally modeled OpenAI/Responses estimates separate from the + // count_tokens-measured Anthropic numerator. + totals.estimatedOpenAISavingsRequests += 1; + totals.modeledOpenAISavedInputEquivalents += savedInputEquivalents; + } else if (google) { + // Google reports actual provider usage, but the comparable text side + // is an optional probe or transform-side estimate. Show it separately + // rather than mislabelling it as Claude measurement or losing it from + // the evidence breakdown altogether. + totals.estimatedGoogleSavingsRequests += 1; + totals.modeledGoogleSavedInputEquivalents += savedInputEquivalents; + } else if (!google) { + totals.measuredSavingsRequests += 1; + totals.measuredClaudeSavedInputEquivalents += savedInputEquivalents; + totals.measuredCacheCreateTierUnknownTokens += cacheCreateUnknownTokens(cc, cacheCreate); + if (inputUsdPerMtok === undefined) totals.unpricedMeasuredSavingsRequests += 1; + else { + totals.pricedMeasuredSavingsRequests += 1; + totals.pricedMeasuredSavingsUsd += savedInputEquivalents * inputUsdPerMtok / 1e6; + } + } + } else if (!gpt && !google && compressed && haveUsage) { + // A successful Anthropic request without both count_tokens probes is + // deliberately excluded from the measured numerator rather than guessed. + totals.baselineProbeExcludedRequests += 1; } // All-rows COUNTERFACTUAL spend, ungated on the probe — the honest // denominator for "did pxpipe move my real bill". Measured rows @@ -899,13 +1123,19 @@ export class DashboardState { // can't measure the counterfactual, so actual ≈ baseline). This // keeps the ratio bounded at 100% — you can't save more than you // would have paid. - if (haveUsage && dollarEligible) { + if (haveUsage) { + if (!gpt && !google) { + totals.cacheCreate5mTokens += cacheCreate?.fiveMinuteTokens ?? 0; + totals.cacheCreate1hTokens += cacheCreate?.oneHourTokens ?? 0; + totals.cacheCreateTierUnknownTokens += cacheCreateUnknownTokens(cc, cacheCreate); + } // baselineInputEff already folds the uncompressed/probe-failed fallback // to actualInputEff, so passthrough rows contribute zero saved here. totals.allBaselineEquivalentWeighted += baselineInputEff; totals.allActualInputWeighted += actualInputEff; totals.allOutputWeighted += outputEquiv; totals.allUsageRequests += 1; + totals.usageBearingResponses += 1; // Direct observed compressed-vs-passthrough split. No counterfactual, // no probe gating — just partition the paid-rows set by which path // actually ran this turn. Headline answers "is the compressed path @@ -924,12 +1154,13 @@ export class DashboardState { } // Per-session aggregation. Uses the SAME baseline/actual/output math as - // the global accumulators above, partitioned by `info.firstUserSha8` + // the global accumulators above, partitioned by the same first-user/project + // cohort used by disk session aggregation // so the dashboard's "current session" panel can show what's happening // RIGHT NOW instead of stale lifetime numbers. Untagged events (no // firstUserSha8 — cold start, passthrough probe failures) are skipped // rather than bucketed into a synthetic "unknown" session. - const sid = info?.firstUserSha8; + const sid = liveSessionId; if (typeof sid === 'string' && sid.length > 0) { this.currentSessionId = sid; let s = this.sessions.get(sid); @@ -961,7 +1192,7 @@ export class DashboardState { // update() so the lifetime totals block (above) and the per-session // block (here) read the same values. Re-deriving them here would // duplicate the cache-aware-baseline math and invite drift. - if (creditSaving && dollarEligible) { + if (creditSaving) { s.baselineInputWeighted += baselineInputEff; s.actualInputWeighted += actualInputEff; s.baselineMeasuredCount += 1; @@ -974,7 +1205,7 @@ export class DashboardState { // above (allActualInputWeighted / allOutputWeighted). Used as the // honest denominator for the session's saved-% so caching wins on // unmeasured requests still count toward "what you actually paid". - if (haveUsage && dollarEligible) { + if (haveUsage) { s.allActualInputWeighted += actualInputEff; s.allOutputWeighted += outputEquiv; } @@ -1134,7 +1365,10 @@ export class DashboardState { // Warm/cold is reconstructed from server-observed cr only. Persisted // completion ts + duration_ms are used only to find a prior prefix size // for the reused/grown split after cr>0 has proved warmth. - const sidR = (t as { first_user_sha8?: string }).first_user_sha8; + const sidR = this.sessionCohortId({ + firstUserSha8: (t as { first_user_sha8?: string }).first_user_sha8, + env: { cwd: (t as { cwd?: string }).cwd }, + } as ProxyEvent['info']); const prefixShaR = (t as { system_sha8?: string }).system_sha8; const completionSecR = Date.parse(t.ts) / 1000; const requestStartSecR = completionSecR - Math.max(0, t.duration_ms || 0) / 1000; @@ -1255,14 +1489,21 @@ export class DashboardState { * panel rather than zeroes when a session goes idle (NO `lastSeen > * threshold` check here; see comment in `update()` for the rationale). */ - serveCurrentSessionJson(): Response { - if (!this.currentSessionId) { + serveCurrentSessionJson(requestedSessionId?: string | null): Response { + // A dashboard can receive traffic from several agents at once. Honor a + // client-selected session whenever it is still retained; otherwise fall + // back to the newest event. This prevents the visible panel from jumping + // between concurrent agents on every two-second poll. + const sessionId = requestedSessionId && this.sessions.has(requestedSessionId) + ? requestedSessionId + : this.currentSessionId; + if (!sessionId) { return jsonResponse({ sessionId: null, message: 'no active session yet', }); } - const s = this.sessions.get(this.currentSessionId); + const s = this.sessions.get(sessionId); if (!s) { return jsonResponse({ sessionId: null, message: 'no active session yet' }); } @@ -1306,11 +1547,16 @@ export class DashboardState { // What Anthropic's weekly limit actually meters — input × 1.0 + // output × 5.0 (the same ratio as the per-MTok price card). This is // the number that moves your "%% used this week" indicator. - const totals = this.enabledTotals(); + const totals = this.lifetimeTotals(); const baseline = totals.baselineInputWeighted; const actual = totals.actualInputWeighted; const output = totals.outputWeighted; // already × OUTPUT_TOKEN_RATE const saved = baseline - actual; + // Sensitivity, not a confidence interval: legacy rows lack the server's + // create-tier split. Repricing just that unknown actual side at 1h shows + // the downside of the historical 5m estimate. + const savedIfUnknownCreatesWere1h = + saved - totals.measuredCacheCreateTierUnknownTokens * (2.0 - 1.25); const pricedBaseline = totals.pricedBaselineInputWeighted; const pricedActual = totals.pricedActualInputWeighted; const pricedOutput = totals.pricedOutputWeighted; @@ -1381,6 +1627,7 @@ export class DashboardState { baseline_input_weighted: Math.round(baseline), actual_input_weighted: Math.round(actual), saved_input_tokens: Math.round(saved), + saved_if_unknown_cache_create_1h: Math.round(savedIfUnknownCreatesWere1h), // saved_pct kept for back-compat with existing dashboard HTML; it is // the input-only number. New code should read saved_pct_input_only. saved_pct: round1(pctInput), @@ -1395,7 +1642,10 @@ export class DashboardState { all_baseline_equivalent_weighted: Math.round(allBaselineEquiv), all_actual_input_weighted: Math.round(allActual), all_output_weighted: Math.round(allOutput), + // Back-compatible alias plus an explicit name for the count of responses + // that carried provider usage and entered paid-traffic totals. all_usage_requests: totals.allUsageRequests, + usage_bearing_responses: totals.usageBearingResponses, // Direct observed split — replaces "share of spend saved" as the // headline. Total actual $ and average $/req per path, plus a delta // gated on `split_sufficient_sample`. No counterfactual: each @@ -1409,7 +1659,25 @@ export class DashboardState { compressed_minus_passthrough_avg_usd: round4(splitDeltaUsd), split_sufficient_sample: splitSufficient, split_min_sample_per_bucket: SUFFICIENT, - saved_usd: round4((pricedSaved * ASSUMED_INPUT_USD_PER_MTOK) / 1e6), + // This is already accumulated per row with the exact model rate (and + // any explicit override). Repricing the combined token total at the + // historical Fable fallback would overstate Opus/Sonnet savings. + saved_usd: round4(totals.pricedMeasuredSavingsUsd), + measured_anthropic_savings_requests: totals.measuredSavingsRequests, + measured_claude_saved_input_equivalents: + Math.round(totals.measuredClaudeSavedInputEquivalents), + estimated_openai_savings_requests: totals.estimatedOpenAISavingsRequests, + modeled_openai_saved_input_equivalents: + Math.round(totals.modeledOpenAISavedInputEquivalents), + estimated_google_savings_requests: totals.estimatedGoogleSavingsRequests, + modeled_google_saved_input_equivalents: + Math.round(totals.modeledGoogleSavedInputEquivalents), + baseline_probe_excluded_requests: totals.baselineProbeExcludedRequests, + cache_create_5m_tokens: Math.round(totals.cacheCreate5mTokens), + cache_create_1h_tokens: Math.round(totals.cacheCreate1hTokens), + cache_create_tier_unknown_tokens: Math.round(totals.cacheCreateTierUnknownTokens), + priced_measured_savings_requests: totals.pricedMeasuredSavingsRequests, + unpriced_measured_savings_requests: totals.unpricedMeasuredSavingsRequests, output_weighted: Math.round(output), baseline_token_equivalent: Math.round(baselineTotal), actual_token_equivalent: Math.round(actualTotal), @@ -1433,6 +1701,7 @@ export class DashboardState { measured_tool_use_chars: totals.toolUseCharsMeasured, measured_redacted_block_count: totals.redactedBlockCountMeasured, events_with_measurement: totals.eventsWithMeasurement, + codex_actual_usage: this.codexUsageFn(), uptime_sec: uptimeSec, compression_enabled: this.compressionEnabled, }; @@ -1506,7 +1775,7 @@ export class DashboardState { async serveFragment(name: string, url: URL, port: number): Promise { switch (name) { case 'toggle': - return htmlResponse(renderToggleFragment(this.compressionEnabled)); + return htmlResponse(renderToggleFragment(this.compressionEnabled, this.getManualCalibration())); case 'models': return htmlResponse( renderModelsFragment( @@ -1530,9 +1799,11 @@ export class DashboardState { ); } case 'session-summary': { - // Lifetime hero — same cumulative payload as the header strip so the - // headline and the "$ saved" tiles never disagree and it stops jumping. - const s = (await this.serveStats().json()) as StatsPayload; + // Explicitly scoped to the most recently active session. The Overview + // above it remains the since-restart aggregate. A browser may pin one + // retained session while other agents continue to send traffic. + const requestedSessionId = url.searchParams.get('session'); + const s = (await this.serveCurrentSessionJson(requestedSessionId).json()) as CurrentSessionPayload; return htmlResponse(renderSessionSummaryFragment(s)); } case 'header': { @@ -1621,19 +1892,48 @@ export class DashboardState { * restart resets to the default (on). */ handleCompressionToggle(body: { enabled?: unknown }): Response { const on = body.enabled === true; - this.compressionEnabled = on; + this.setCompressionEnabled(on); return jsonResponse({ compression_enabled: on }); } /** POST /fragments/models — add/remove ONE model (Claude or GPT) from the * runtime compress scope. The model checks read this live. Persisted via * the host's `persistModelBases` hook when provided (Node writes the - * config file); otherwise in-memory only and restart resets to the - * PXPIPE_MODELS env / built-in default. */ + * sidecar); otherwise in-memory only and restart resets to the configured + * PXPIPE_CONFIG / PXPIPE_MODELS / built-in default. */ handleModelsToggle(model: string, on: boolean): void { const next = new Set(getAllowedModelBases()); if (on) next.add(model); - else next.delete(model); + else { + // A broad base (for example gpt-5.6) enables every matching child. + // Expand it to its known concrete siblings before removing the child: + // deleting the broad base outright would unexpectedly disable Terra and + // Lun when the user only turned Sol off. + const known = new Set([ + ...KNOWN_MODEL_SCOPE_IDS, + ...getConfiguredModelBases(), + ...next, + ]); + for (const base of [...next]) { + if (!isModelScopeEnabled(model, [base])) continue; + next.delete(base); + // Turning off the broad shortcut itself must turn its whole group off. + // Only a *properly broader* entry is expanded to retain siblings when + // the operator turns off one concrete child. + if (isModelScopeEnabled(base, [model])) continue; + for (const candidate of known) { + // A candidate that also enables `base` is that broad base (or an + // even broader alias), not a concrete sibling to preserve. + if ( + candidate !== model + && isModelScopeEnabled(candidate, [base]) + && !isModelScopeEnabled(base, [candidate]) + ) { + next.add(candidate); + } + } + } + } this.applyModelBases([...next]); } @@ -1649,6 +1949,17 @@ export class DashboardState { this.applyModelBases(bases); } + /** Reset removes the runtime and persisted override. It is intentionally + * distinct from an empty scope, which means "compress nothing". */ + handleModelsReset(): void { + setAllowedModelBases(null); + try { + this.persistModelBases?.(null); + } catch { + // The live reset still took effect even when persistence cleanup fails. + } + } + private applyModelBases(bases: string[]): void { setAllowedModelBases(bases); try { diff --git a/src/dashboard/fragments.ts b/src/dashboard/fragments.ts index a0f5cb8df..dac61ac1c 100644 --- a/src/dashboard/fragments.ts +++ b/src/dashboard/fragments.ts @@ -3,6 +3,8 @@ import { HTMX_JS, ALPINE_JS } from './vendor.js'; import { CACHE_CREATE_RATE, CACHE_READ_RATE } from '../core/baseline.js'; +import { isModelScopeEnabled } from '../core/applicability.js'; +import { groupCodexQuotaWindows } from '../codex-usage.js'; import type { StatsPayload, RecentPayload, @@ -44,32 +46,109 @@ function formatDuration(s: number): string { return (h ? h + 'h ' : '') + (m || h ? m + 'm ' : '') + sec + 's'; } +function formatReset(epochSeconds: number | null | undefined): string { + if (!epochSeconds || !Number.isFinite(epochSeconds)) return 'reset unknown'; + return `resets ${new Date(epochSeconds * 1000).toLocaleString('en-GB', { + month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', + })}`; +} + +function formatObservedAt(iso: string | null | undefined): string { + if (!iso) return 'time unknown'; + const d = new Date(iso); + if (!Number.isFinite(d.getTime())) return 'time unknown'; + return d.toLocaleString('en-GB', { + month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', + }); +} + function shortPath(p: string | null | undefined): string { if (!p) return '-'; const parts = String(p).split('/'); return parts[parts.length - 1] || p; } -// ---- compression toggle (kill switch) ------------------------------------ +/** Accessible, touch-friendly contextual help. Native
keeps the + * explanation usable without hover and GLUE_JS preserves open state on HTMX refresh. */ +function helpTip(id: string, title: string, what: string, why: string, read: string): string { + return ( + `
` + + `?` + + `
` + + `${escapeHtml(title)}` + + `

What it is. ${escapeHtml(what)}

` + + `

Why it matters. ${escapeHtml(why)}

` + + `

How to read it. ${escapeHtml(read)}

` + + `
` + ); +} -export function renderToggleFragment(enabled: boolean): string { +// ---- compression toggle (kill switch + manual calibration) ---------------- + +/** + * An operator-controlled A/B note. This deliberately is not a sampler: the + * proxy never changes a request's path for calibration. It only records the + * phase entered after the operator explicitly uses the kill switch. + */ +export interface ManualCalibrationStatus { + active: boolean; + /** `baseline` means the operator manually disabled compression. */ + phase: 'baseline' | 'imaged' | null; + baselineRequests: number; + imagedRequests: number; + /** Locked by the first eligible baseline row; prevents mixed-model/session cohorts. */ + scopeModel: string | null; + scopeSession: string | null; + skippedMismatches: number; +} + +export function renderToggleFragment( + enabled: boolean, + calibration: ManualCalibrationStatus = { + active: false, + phase: null, + baselineRequests: 0, + imagedRequests: 0, + scopeModel: null, + scopeSession: null, + skippedMismatches: 0, + }, +): string { + const toggleHelp = helpTip( + 'compression-switch', 'Compression switch', + 'The runtime kill switch that decides whether eligible requests are imaged or sent upstream unchanged.', + 'It gives you immediate control and is also the only way to start a manual calibration baseline. PXPIPE never disables compression automatically for sampling.', + 'Compression on applies the selected model scope. Compression off means passthrough for every request. The switch resets to on after a proxy restart.', + ); // NOTE: "PASSTHROUGH MODE", "Disable compression", "Enable compression" are asserted by tests. const banner = enabled ? '' - : ``; + : ``; // Button POSTs the OPPOSITE of current state; 2s poll keeps it fresh. const confirm = enabled - ? ` hx-confirm="Turn compression off?\n\nRequests will pass straight through to Claude, unchanged. Restarting the proxy turns it back on."` + ? ` hx-confirm="Start a MANUAL baseline phase?\n\nCompression will be turned off only because you explicitly approve it. Send comparable requests as normal text, then enable compression to collect the image phase. pxpipe never samples or switches requests automatically. Restarting the proxy turns compression back on and clears this in-memory calibration note."` + : ''; + const scope = calibration.scopeModel && calibration.scopeSession + ? `${escapeHtml(calibration.scopeModel)} · session ${escapeHtml(calibration.scopeSession.slice(0, 8))}` + : 'waiting for the first Claude request to lock model + session'; + const skipped = calibration.skippedMismatches > 0 + ? ` · ${numFmt(calibration.skippedMismatches)} out-of-scope rows ignored` : ''; + const calibrationNote = !calibration.active + ? `manual calibration is idle · disable compression to explicitly start a normal-text baseline · no automatic passthrough sampling` + : calibration.phase === 'baseline' + ? `Manual comparison: baseline phase · ${numFmt(calibration.baselineRequests)} in-scope Claude request${calibration.baselineRequests === 1 ? '' : 's'} · ${scope}${skipped} · re-enable compression when ready` + : `Manual comparison: image phase · ${numFmt(calibration.baselineRequests)} normal-text vs ${numFmt(calibration.imagedRequests)} imaged · ${scope}${skipped} · same model/session, but still sequential and observational`; return ( banner + `
` + - `${enabled ? 'Compression on' : 'Compression off'}` + + `${enabled ? 'Compression on' : 'Compression off'}${toggleHelp}` + `` + `kill switch · resets to on when you restart` + `
` + + `
${calibrationNote}
` ); } @@ -79,10 +158,21 @@ export function renderToggleFragment(enabled: boolean): string { const MODEL_CATALOG: ReadonlyArray<{ id: string; label: string }> = [ { id: 'claude-fable-5', label: 'Fable 5' }, { id: 'claude-opus-5', label: 'Opus 5' }, + // Retained as explicit opt-in chips so their benchmarked below-bar status + // remains visible rather than silently becoming an unmeasured unknown. + { id: 'claude-opus-4-8', label: 'Opus 4.8' }, + { id: 'claude-opus-4-7', label: 'Opus 4.7' }, ]; const GPT_MODEL_CATALOG: ReadonlyArray<{ id: string; label: string }> = [ + // Terra/Sol/Lun are the GPT 5.6 sibling variants (Terra is the local Codex + // provider's model). Each is an explicit chip so it toggles independently, + // rather than only being reachable via the broad `gpt-5.6` base. The broad + // chip stays as a one-click "all 5.6 siblings" shortcut. + { id: 'gpt-5.6', label: 'GPT 5.6' }, + { id: 'gpt-5.6-terra', label: 'GPT 5.6 Terra' }, { id: 'gpt-5.6-sol', label: 'GPT 5.6 Sol' }, + { id: 'gpt-5.6-lun', label: 'GPT 5.6 Lun' }, { id: 'gpt-5.5', label: 'GPT 5.5' }, ]; @@ -94,12 +184,80 @@ const GEMINI_MODEL_CATALOG: ReadonlyArray<{ id: string; label: string }> = [ { id: 'gemini-3.6-flash', label: 'Gemini 3.6 Flash' }, ]; +/** Known concrete model ids used when a broad scope must be expanded while + * preserving its sibling chips. The broad shortcut itself is intentionally + * omitted from the expansion by the caller. */ +export const KNOWN_MODEL_SCOPE_IDS: readonly string[] = [ + ...MODEL_CATALOG.map((model) => model.id), + ...GPT_MODEL_CATALOG.map((model) => model.id), + ...GROK_MODEL_CATALOG.map((model) => model.id), + ...GEMINI_MODEL_CATALOG.map((model) => model.id), +]; + +/** Per-model readiness for pxpipe imaging, keyed to committed eval receipts — + * NOT opinion (see FINDINGS.md / applicability.ts). The dashboard derives its + * warning from `status`: + * - validated → reads dense imaged content at/above the Fable bar + * (≈13/15 verbatim + graceful failure). No warning. + * - below-bar → benchmarked and measured under the bar. Blocking confirm on + * enable + ⚠ marker (proven risk). + * - unmeasured → no committed benchmark. Non-blocking ⚠ marker + tooltip + * (unknown, not proven-bad). This is also the default for any + * model absent from the table. + * Promote to `validated` only by committing receipts that clear the bar. */ +type Readiness = 'validated' | 'below-bar' | 'unmeasured'; + +const MODEL_READINESS: Readonly> = { + // Validated — the only reader proven at the production density. + 'claude-fable-5': { status: 'validated' }, + // Below-bar — benchmarked, measured under the Fable bar. + 'claude-opus-4-8': { + status: 'below-bar', + evidence: 'Opus 4.8: 0/15 verbatim dense-hex, ~7% arithmetic read-tax, silent confabulation (fixable with abstention prompting).', + }, + 'claude-opus-4-7': { + status: 'below-bar', + evidence: 'Opus 4.7: below the Fable bar on imaged verbatim recall (0/15 dense-hex), silent confabulation.', + }, + 'gpt-5.5': { + status: 'below-bar', + evidence: 'GPT-5.5: degrades on imaged history/context; recall of older turns can suffer.', + }, + 'gpt-5.6-sol': { + status: 'below-bar', + evidence: 'GPT-5.6 Sol: 0/15 verbatim dense-hex, gist 79/93, 98/100 arithmetic — below the Fable bar.', + }, + 'grok-4.5': { + status: 'below-bar', + evidence: 'Grok 4.5: 82/100 arithmetic, 83/98 gist, 13/18 state tracking on imaged content.', + }, + // Broad gpt-5.6 chip enables every 5.6 sibling incl. Sol → carry Sol's risk. + 'gpt-5.6': { + status: 'below-bar', + evidence: 'Enabling GPT-5.6 turns on all 5.6 siblings including Sol (0/15 verbatim dense-hex, below the Fable bar).', + }, + // Unmeasured — no committed benchmark (explicit; unlisted ids default here too). + 'claude-sonnet-5': { status: 'unmeasured' }, + 'claude-sonnet-4-6': { status: 'unmeasured' }, + 'gpt-5.6-terra': { status: 'unmeasured' }, + 'gpt-5.6-lun': { status: 'unmeasured' }, +}; + +/** Shared risk statement for any non-validated model. */ +const IMAGED_RISK = + 'pxpipe images older history at a density where exact recall of IDs/hashes/exact numbers is unsafe and can fail via silent confabulation.'; +const UNMEASURED_NOTE = `No committed pxpipe reading benchmark for this model — its accuracy on imaged context is unmeasured. ${IMAGED_RISK}`; + +function readinessOf(id: string): { status: Readiness; evidence?: string } { + return MODEL_READINESS[id] ?? { status: 'unmeasured' }; +} + + export function renderModelsFragment( active: string[], configured: string[], enabled: boolean, ): string { - const on = new Set(active); const labelOf = new Map( [...MODEL_CATALOG, ...GPT_MODEL_CATALOG, ...GROK_MODEL_CATALOG, ...GEMINI_MODEL_CATALOG].map((m) => [m.id, m.label]), ); @@ -122,12 +280,25 @@ export function renderModelsFragment( } } const chipFor = (id: string): string => { - const lit = on.has(id); + const lit = isModelScopeEnabled(id, active); const label = labelOf.get(id) ?? id; + const r = readinessOf(id); + const tip = + r.status === 'below-bar' ? (r.evidence ?? `${label} reads pxpipe-imaged context below the Fable bar. ${IMAGED_RISK}`) + : r.status === 'unmeasured' ? UNMEASURED_NOTE + : ''; + // Persistent, hover-readable explanation for any flagged chip. + const titleAttr = tip ? ` title="${escapeHtml(tip)}"` : ''; + // below-bar = proven risk → block on ENABLE (off → on). unmeasured never + // blocks (unknown, not proven-bad); validated never flags. Disable never prompts. + const confirmAttr = r.status === 'below-bar' && !lit ? ` hx-confirm="${escapeHtml(`${tip} Enable anyway?`)}"` : ''; + // ⚠ marker on any lit non-validated chip so the risk stays visible after enabling. + const warnMark = r.status !== 'validated' && lit ? ' ⚠' : ''; return ( `` + `hx-vals='${escapeHtml(JSON.stringify({ model: id, on: !lit }))}'>${escapeHtml(label)}${lit ? ' ✓' : ''}${warnMark}` ); }; const claudeChips = ids.filter((id) => id.startsWith('claude')).map(chipFor).join(''); @@ -138,34 +309,55 @@ export function renderModelsFragment( .filter((id) => !id.startsWith('claude') && !id.startsWith('gpt') && !id.startsWith('grok') && !id.includes('gemini')) .map(chipFor) .join(''); - const moot = enabled - ? '' - : `
compression is off — these settings have no effect right now
`; + + const moot = enabled ? '' : ` compression is off, so this has no effect right now`; + const claudeHelp = helpTip( + 'claude-model-scope', 'Image Claude models', + 'The Claude model families currently eligible for context imaging.', + 'Model scope prevents experimental or unsupported models from being transformed unintentionally.', + 'A highlighted chip with a check mark is enabled. Your choice is saved and survives restart until you press Reset (which falls back to PXPIPE_CONFIG, PXPIPE_MODELS, or the built-in default).', + ); + const grokHelp = helpTip( + 'grok-model-scope', 'Image Grok models', + 'Opt-in model bases routed through the OpenAI Responses-compatible imaging path.', + 'These models do not use Anthropic count_tokens or cache_control accounting.', + 'Enable only models you intentionally want PXPIPE to image. Savings for this path are locally modeled, not provider-measured.', + ); + const gptHelp = helpTip( + 'gpt-model-scope', 'Image OpenAI Responses models', + 'GPT model bases currently eligible for context imaging on the Responses path.', + 'It controls transformation only; it does not change the configured upstream model or provider.', + 'Checked chips are enabled. Their reduction is locally modeled and kept separate from Claude provider-measured savings.', + ); + return ( moot + `
` + - `Image Claude models` + + `Image Claude models${claudeHelp}` + claudeChips + - `unlisted models get plain text` + + + `everything else is sent as normal text · your choice is saved until Reset${moot}` + `
` + `
` + - `Image Gemini models` + - geminiChips + - `enabled by default · 100/100 vision reader` + + `Image Grok models${grokHelp}` + + grokChips + + otherChips + + `opt-in only · OpenAI Responses path · your choice is saved until Reset${moot}` + `
` + `
` + - `Image OpenAI Responses models` + + `Image OpenAI Responses models${gptHelp}` + gptChips + - grokChips + - otherChips + - `opt-in · no Anthropic cache_control` + + `imaging only, no Anthropic cache_control · one scope for all families · your choice is saved until Reset${moot}` + + `
` + + `
` + + `` + + `clears your saved choice · falls back to configured defaults` + `
` + `
` + `PXPIPE_MODELS` + - `` + - `CSV of bases, or off · applies on enter/blur · export to persist` + + `` + + `CSV of bases, or off · applies on enter/blur` + `
` ); } @@ -176,47 +368,71 @@ export function renderModelsFragment( const INPUT_USD_PER_MTOK = 10.0; void INPUT_USD_PER_MTOK; // suppress unused-var; renderHeaderFragment uses the server's pricing block. -// Lifetime hero. Reads the SAME cumulative weighted totals as the header strip -// (serveStats), so the headline and the "$ saved" tiles can never disagree, and -// the number stops swinging on tiny per-session samples. Cache-weighted on -// purpose ("lifeweight"): it answers "did pxpipe move my real, cache-discounted -// bill since this proxy started", not a raw token count. -export function renderSessionSummaryFragment(s: StatsPayload): string { - const measured = s.compressed_requests ?? 0; - if (measured <= 0) { +// Compact, explicitly scoped summary for the most recently active session. +// The Overview above it owns the since-restart aggregate. +export function renderSessionSummaryFragment(s: CurrentSessionPayload): string { + const help = helpTip( + 'current-session', + 'Current session', + 'A cache-aware comparison for the most recently active PXPIPE session only.', + 'It separates the work happening now from totals accumulated since the proxy restarted.', + 'Positive percentages mean less estimated input than the same context as text. Negative percentages mean imaging cost more. Output is excluded because PXPIPE does not change it.', + ); + const percentHelp = helpTip( + 'current-session-percent', 'Current-session percentage', + 'The cache-aware input difference for the selected session’s comparable responses.', + 'It shows the direction and relative size of PXPIPE’s attributed input effect while you work.', + 'It changes only when a new comparable response enters this selected session. It is not a provider bill, and output is intentionally excluded.', + ); + const effectiveInputHelp = helpTip( + 'current-session-effective-input', 'Effective input comparison', + 'Actual input sent through PXPIPE compared with the same context estimated as plain text.', + 'Both values apply the same cache-create and cache-read weights, so they are comparable.', + 'The first number is actual imaged-path input; the second is the text counterfactual. Smaller actual input produces a positive percentage.', + ); + if (!s.sessionId) { return ( - `
` + - `
Since start
` + - `
Warming up…
` + - `
Point Claude Code at this proxy with ANTHROPIC_BASE_URL, or launch it with pxpipe warp -- claude to keep /remote-control and claude.ai connectors working. Send a message and your running savings show up right here.
` + + + `
` + + `
Current session
${help}
` + + `
Waiting for session traffic…
` + + `
Send a request through PXPIPE. This panel will then show only the most recently active session.
` + + `
` + ); + } + const baselineW = s.baselineInputWeighted ?? 0; + const actualW = s.actualInputWeighted ?? 0; + const measured = s.baselineMeasuredCount ?? 0; + if (measured <= 0 || baselineW <= 0) { + return ( + `
` + + `
Current session · ${escapeHtml(s.sessionId.slice(0, 8))}
${help}
` + + `
Waiting for a comparable response…
` + + `
Traffic exists, but this session does not yet have both an actual input count and a text counterfactual.
` + + `
Watching this session · no input-change claim yet
` + + `
` ); } - // Cache-aware reduction — same basis as the Details panel + Saved column. - // Raw count_tokens would over-claim: most of the text baseline would have been - // cheap cache-reads (~0.1×), not full-price tokens. Weighting both sides at their - // real cache rate is the only comparison that can't contradict the Saved column. - // Input-only: pxpipe never touches output, so lumping it in just dampened the %. - const baselineW = s.baseline_input_weighted ?? 0; // same context as text, cache-aware - const actualW = s.actual_input_weighted ?? 0; // what we actually sent, cache-aware - const outMult = s.pricing_assumptions?.output_multiplier || 5; - const rawOutput = (s.output_weighted ?? 0) / outMult; // reply — never compressed const inputPct = baselineW > 0 ? (1 - actualW / baselineW) * 100 : 0; const positive = inputPct >= 0; const bigNum = `${Math.abs(inputPct).toFixed(0)}%`; - const word = positive ? 'fewer tokens' : 'more tokens'; + const word = positive ? 'less estimated input' : 'more estimated input'; + const rawOutput = s.rawOutputTokens ?? 0; return ( - `
` + - `
Since start · ${numFmt(measured)} request${measured === 1 ? '' : 's'} imaged
` + - `
${bigNum} ${word}
` + + + `
` + + `
Current session · ${escapeHtml(s.sessionId.slice(0, 8))} · ${numFmt(measured)} comparable response${measured === 1 ? '' : 's'}
${help}
` + + `
${bigNum}${percentHelp} ${word} after caching
` + `
` + - `${kFmt(actualW)} provider-accounted input tokens vs ${kFmt(baselineW)} if this same context ` + - `stayed plain text. Your latest messages and model output are never compressed.` + + `${kFmt(actualW)} effective tokens vs ${kFmt(baselineW)} if this same context ` + + `stayed plain text — both counted after normal cache discounts in this session. ${effectiveInputHelp} This is a counterfactual estimate, not an invoice.` + `
` + `
` + - `Provider-token basis; cache discounts applied where measurable · ` + - `output untouched (${kFmt(rawOutput)}) · no $ assumptions` + + `Watching this session · output untouched (${kFmt(rawOutput)}) · no dollar assumptions` + + `` + + `
` + `
` ); @@ -229,139 +445,301 @@ function mathRow(key: string, val: number | string | undefined, note = ''): stri return `
${key}: ${escapeHtml(v)} ${note}
`; } -function mathBlock(title: string, body: string): string { - return `

${title}

${body}
`; -} - -/** Stat tile; `tip` adds a hover "?" explainer. */ -function statTile( - label: string, - value: string, - sub: string, - cls = '', - tip = '', -): string { - const q = tip - ? `?` - : ''; - return ( - `
` + - `
${label}${q}
` + - `
${value}
` + - `
${sub}
` + - `
` - ); +function mathBlock(title: string, body: string, help: string): string { + return `

${title}

${help}
${body}
`; } export function renderHeaderFragment(s: StatsPayload, port: number): string { const pa = s.pricing_assumptions; - const unpricedImaged = Math.max( - 0, - (s.compressed_requests ?? 0) - (s.compressed_paid_requests ?? 0), + + const pricedRows = s.priced_measured_savings_requests ?? 0; + const unpricedRows = s.unpriced_measured_savings_requests ?? 0; + const measuredClaudeRows = s.measured_anthropic_savings_requests ?? 0; + const estimatedResponsesRows = s.estimated_openai_savings_requests ?? 0; + const estimatedGeminiRows = s.estimated_google_savings_requests ?? 0; + const excludedProbeRows = s.baseline_probe_excluded_requests ?? 0; + const cacheCreate5m = s.cache_create_5m_tokens ?? 0; + const cacheCreate1h = s.cache_create_1h_tokens ?? 0; + const cacheCreateUnknown = s.cache_create_tier_unknown_tokens ?? 0; + const priceCoverageTotal = pricedRows + unpricedRows; + const priceCoverage = priceCoverageTotal > 0 + ? `${pricedRows}/${priceCoverageTotal} rows` + : 'No priced rows yet'; + const codex = s.codex_actual_usage ?? { + source: '', loading: false, error: null, sessionFiles: 0, usageSnapshots: 0, + inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, + reasoningOutputTokens: 0, totalTokens: 0, modelContextWindow: null, + earliestEventAt: null, latestEventAt: null, rateLimits: null, quotaWindows: [], + }; + const codexCachePct = codex.inputTokens > 0 + ? ((codex.cachedInputTokens / codex.inputTokens) * 100).toFixed(1) + : '0.0'; + const measuredClaudeSaved = s.measured_claude_saved_input_equivalents ?? 0; + const modeledResponsesSaved = s.modeled_openai_saved_input_equivalents ?? 0; + const modeledGeminiSaved = s.modeled_google_saved_input_equivalents ?? 0; + const usageBearingResponses = s.usage_bearing_responses ?? s.all_usage_requests ?? 0; + const paidCompressed = s.compressed_paid_requests ?? 0; + const paidPassthrough = s.passthrough_paid_requests ?? 0; + const modeledRows = estimatedResponsesRows + estimatedGeminiRows; + const evidenceLabel = measuredClaudeRows > 0 && modeledRows > 0 + ? 'Mixed evidence' + : measuredClaudeRows > 0 + ? 'Provider-measured' + : modeledRows > 0 + ? 'Modeled only' + : 'Collecting data'; + const evidenceClass = measuredClaudeRows > 0 && modeledRows === 0 && excludedProbeRows === 0 + ? 'good' + : measuredClaudeRows > 0 || modeledRows > 0 + ? 'mixed' + : 'waiting'; + const outcomeClass = (s.saved_input_tokens ?? 0) < 0 ? ' negative' : ''; + const overviewHelp = helpTip( + 'overview', 'Overview', + 'A since-restart summary of PXPIPE effect, paid response activity, and evidence quality.', + 'These values have different meanings. Keeping effect, activity, and confidence separate prevents token savings from being confused with provider usage.', + 'Start with input change, then check whether it is Claude measured or Responses/Gemini modeled. Use Paid LLM responses for sample size and Reliability for limitations.', + ); + const changeHelp = helpTip( + 'input-change', 'Estimated input change', + 'The cache-aware difference between the input PXPIPE sent and a counterfactual where the same imageable context stayed as text.', + 'It estimates the part PXPIPE can influence. It is not total provider usage and not an invoice.', + 'Positive is an estimated reduction; negative means imaging used more effective input. Claude rows use provider counts. Responses rows use a local tokenizer and vision model.', + ); + const paidHelp = helpTip( + 'paid-responses', 'Paid LLM responses', + 'Responses with non-zero upstream usage that entered paid-traffic accounting since restart.', + 'Health checks, errors without usage, and other proxy traffic should not inflate the sample size used to judge savings.', + 'Imaged and passthrough are the two paths. “Uncredited” is a subset of imaged rows whose baseline could not be verified, so zero savings were assigned.', + ); + const reliabilityHelp = helpTip( + 'reliability', 'Estimate reliability', + 'A summary of where the estimate came from and how much of the Claude value has an exact configured model price.', + 'Provider-measured Claude rows are stronger evidence than locally modeled Responses or Gemini rows. Missing probes and missing prices reduce what can be claimed.', + 'Provider-measured is strongest. Mixed evidence combines measured and modeled rows. Modeled-only provider groups should be treated as directional. Open Audit for formulas and exclusions.', + ); + const claudeMeasuredHelp = helpTip( + 'claude-measured', 'Claude provider-measured reduction', + 'Claude input change calculated from an Anthropic text count for the baseline and upstream usage for the actual request.', + 'Both sides are provider measurements, making this the strongest savings evidence PXPIPE exposes.', + 'The token-equivalent value includes cache weights. Dollar value covers only rows whose exact model price is configured; the priced-row fraction is shown inline.', + ); + const responsesModeledHelp = helpTip( + 'responses-modeled', 'Responses locally modeled reduction', + 'OpenAI Responses input change estimated locally with the matching text tokenizer and the vision-token model used for imaged content.', + 'Responses does not provide the same count endpoint as Claude, so PXPIPE cannot claim this part as provider-measured.', + 'Use it as a directional token-equivalent estimate. It is disclosed separately and deliberately excluded from dollar savings.', + ); + const geminiModeledHelp = helpTip( + 'gemini-modeled', 'Gemini provider-usage reduction', + 'Gemini actual input comes from Google provider usage. The text counterfactual comes from an optional baseline probe when available, otherwise PXPIPE\'s transform-side token estimate.', + 'Google does not share the Claude count_tokens evidence path, so this value is shown separately and is never assigned Claude dollar pricing.', + 'Use it as a provider-usage-backed but unpriced input-reduction estimate.', + ); + const imagedHelp = helpTip( + 'paid-imaged', 'Imaged paid responses', + 'Paid responses whose eligible context PXPIPE rendered into images before sending upstream.', + 'They are the rows where PXPIPE can potentially change input usage.', + 'This is a path count, not a savings count: an imaged row can still be uncredited if no trustworthy baseline was available.', + ); + const passthroughHelp = helpTip( + 'paid-passthrough', 'Passthrough paid responses', + 'Paid responses sent upstream without imaging.', + 'They show the real amount of traffic for which PXPIPE did not transform the context.', + 'They remain in paid-traffic totals but normally contribute zero attributed input change.', + ); + const uncreditedHelp = helpTip( + 'paid-uncredited', 'Uncredited imaged rows', + 'Imaged paid responses that lacked a successful comparable text baseline.', + 'PXPIPE assigns them zero savings instead of guessing, which keeps the estimate conservative.', + 'This count is a subset of imaged responses, not a third traffic path to add to the total.', + ); + const priceCoverageHelp = helpTip( + 'price-coverage', 'Exact model-price coverage', + 'The fraction of provider-measured Claude rows with a configured official input price.', + 'Only those rows can contribute to the model-priced dollar value without using a generic tariff.', + 'A lower fraction means the displayed dollar amount is deliberately partial; token-equivalent reduction can still include all measured rows.', ); - const onlyUnpriced = unpricedImaged > 0 && (s.compressed_paid_requests ?? 0) === 0; - - // Compare the same imaged requests on both sides. Passthrough requests are - // generally smaller because the profitability gate selected them, so their - // average is not a valid "without pxpipe" counterfactual. - const cAvg = s.compressed_avg_usd_per_request ?? 0; - const paidImaged = s.compressed_paid_requests ?? 0; - const withoutAvg = paidImaged > 0 ? cAvg + (s.saved_usd ?? 0) / paidImaged : 0; - const costTile = paidImaged > 0 - ? statTile( - 'Cost per request', - `$${cAvg.toFixed(4)}`, - `vs $${withoutAvg.toFixed(4)} without pxpipe`, - cAvg <= withoutAvg ? 'pos' : 'neg', - 'Average cost of paid imaged requests versus the cache-aware text counterfactual for those same requests. Unmeasured requests are assigned zero savings.', - ) - : onlyUnpriced - ? statTile( - 'Cost per request', - '—', - 'provider pricing not configured', - 'muted-val', - 'Token savings are available, but this provider is excluded from Claude-priced dollar estimates.', - ) - : statTile( - 'Cost per request', - 'collecting…', - 'waiting for a paid imaged request', - 'muted-val', - 'The comparison appears after an imaged request returns provider usage.', - ); - const savedUsdTile = onlyUnpriced - ? statTile( - 'Estimated saved', - '—', - 'provider pricing not configured', - 'muted-val', - 'Token savings are shown separately. Dollar estimates require provider-specific pricing and are not inferred from Claude rates.', - ) - : statTile( - 'Estimated saved', - `$${(s.saved_usd ?? 0).toFixed(2)}`, - unpricedImaged > 0 - ? `${numFmt(unpricedImaged)} provider-priced request${unpricedImaged === 1 ? '' : 's'} excluded` - : `at $${pa.input_per_mtok}/M base input price`, - '', - 'Cache-aware estimate for requests covered by the configured pricing assumptions. Other providers remain excluded.', - ); + const overview = + `
` + + `Overview · since restartWhat PXPIPE changedlive · ${formatDuration(s.uptime_sec)}HideShow` + + `
` + + `
` + + `

Effect, activity, and evidence are separated so unlike numbers are not compared accidentally.

` + + overviewHelp + + `
` + + `
` + + `
` + + `
Estimated input change
${changeHelp}
` + + `
${numFmt(s.saved_input_tokens)}
` + + `
cache-aware counterfactual · not a provider bill
` + + `
` + + `
Claudeprovider-measured · ${numFmt(measuredClaudeRows)} rows${claudeMeasuredHelp}${numFmt(measuredClaudeSaved)}${pricedRows > 0 ? `Model-priced input value$${(s.saved_usd ?? 0).toFixed(2)}${pricedRows}/${priceCoverageTotal} priced Claude rows` : 'unpriced'}
` + + `
Responseslocally modeled · ${numFmt(estimatedResponsesRows)} rows${responsesModeledHelp}${numFmt(modeledResponsesSaved)} not priced
` + + `
Geminiprovider usage + modeled baseline · ${numFmt(estimatedGeminiRows)} rows${geminiModeledHelp}${numFmt(modeledGeminiSaved)} not priced
` + + `
` + + `
` + + `
Paid LLM responses
${paidHelp}
` + + `
${numFmt(usageBearingResponses)}
` + + `
` + + `${numFmt(paidCompressed)} imaged${imagedHelp}` + + `${numFmt(paidPassthrough)} passthrough${passthroughHelp}` + + `${numFmt(excludedProbeRows)} of imaged rows uncredited${uncreditedHelp}` + + `
${numFmt(s.requests)} total proxy responses observed` + + `
` + + `
` + + `
How reliable is the estimate?
${evidenceLabel}${reliabilityHelp}
` + + `
` + + `${numFmt(measuredClaudeRows)} Claude rows measured by provider` + + `${numFmt(estimatedResponsesRows)} Responses rows modeled locally` + + `${numFmt(estimatedGeminiRows)} Gemini rows with provider usage + modeled baseline` + + `${priceCoverage} exact model-price coverage${priceCoverageHelp}` + + `
See methodology and assumptions ↓` + + `
` + + `
` + + `
` + + `Reduction = estimated counterfactual` + + `$ value = measured Claude only` + + `Codex usage = consumption, not savings` + + `
`; + + const quotaLabel = (minutes: number): string => { + if (minutes === 300) return '5-hour'; + if (minutes === 10_080) return 'Weekly'; + if (minutes > 0 && minutes % 1_440 === 0) return `${minutes / 1_440}-day`; + if (minutes > 0 && minutes % 60 === 0) return `${minutes / 60}-hour`; + return `${numFmt(minutes)} min`; + }; + const quotaRows = groupCodexQuotaWindows(codex.quotaWindows ?? []).map((group) => { + const limitId = group.limitId ?? 'codex'; + const displayName = group.limitName || (limitId === 'codex' ? 'Codex' : limitId); + return `
${escapeHtml(displayName)}${escapeHtml(limitId)}` + + group.windows.map((window) => + `${quotaLabel(window.windowMinutes)}${window.usedPercent.toFixed(1)}%${formatReset(window.resetsAt)}`, + ).join('') + `
`; + }).join(''); + const codexStatus = codex.loading + ? 'Scanning official rollouts…' + : codex.error + ? 'Rollout scan unavailable' + : 'Retained rollout coverage'; + const codexHelp = helpTip( + 'codex-usage', 'Codex provider-reported usage', + 'Exact token and rate-limit observations read from retained local Codex rollout files whose provider is PXPIPE.', + 'This is the best available view of actual Codex consumption, but it covers retained files rather than every request ever made.', + 'Input includes cached input. Output includes reasoning. Do not add those subcomponents twice, and do not compare this usage total directly with estimated savings.', + ); + const codexInputHelp = helpTip( + 'codex-input', 'Actual input', + 'Provider-reported Codex input tokens found in retained PXPIPE rollouts.', + 'It shows consumption, not what PXPIPE saved.', + 'Treat it as a cumulative retained-rollout total. Cached input is already included in this number.', + ); + const codexCacheHelp = helpTip( + 'codex-cache', 'Cached input', + 'The portion of actual Codex input served from provider cache.', + 'Cached tokens are usually billed or limited differently, but they are still part of input usage.', + 'Read the token count and percentage as a subset of Actual input. Never add it to Actual input again.', + ); + const codexOutputHelp = helpTip( + 'codex-output', 'Actual output', + 'Provider-reported Codex output tokens, including reasoning output.', + 'PXPIPE compresses input context only, so output is usage context rather than a savings claim.', + 'The smaller reasoning value is already included in total output and should not be added twice.', + ); + const codexCoverageHelp = helpTip( + 'codex-coverage', 'PXPIPE coverage', + 'The number of usable token snapshots and retained Codex session files scanned locally.', + 'It defines the boundary of the Codex totals shown here.', + 'More records improve retained-history coverage, but deleted, moved, direct-provider, or unreadable rollouts remain outside the total.', + ); + const quotaHelp = helpTip( + 'codex-quotas', 'Provider quota windows', + 'The freshest provider-reported percentage and reset time for every quota window, grouped by limit identifier.', + 'Different Codex products can report separate limits with the same duration; grouping prevents them from being merged accidentally.', + 'Each percentage is used quota for that named limit and window. Reset times come from the provider and are shown in local time.', + ); + const codexPanel = + `
` + + `Usage & limits · retained rolloutsCodex provider-reported usage${codexStatus}HideShow` + + `
` + + `
` + + `

Consumption found in retained local PXPIPE rollouts. This is usage, not evidence of token savings.

${codexHelp}
` + + `
` + + `
` + + `
Actual input${codexInputHelp}
${numFmt(codex.inputTokens)}provider-reported input tokens
` + + `
Cached input${codexCacheHelp}
${numFmt(codex.cachedInputTokens)} · ${codexCachePct}%included in actual input, not added twice
` + + `
Actual output${codexOutputHelp}
${numFmt(codex.outputTokens)}${numFmt(codex.reasoningOutputTokens)} reasoning tokens included
` + + `
PXPIPE coverage${codexCoverageHelp}
${numFmt(codex.usageSnapshots)} usage records${numFmt(codex.sessionFiles)} retained Codex session files
` + + `
` + + `
Observed ${formatObservedAt(codex.earliestEventAt)} → ${formatObservedAt(codex.latestEventAt)} · ${numFmt(codex.sessionFiles)} retained files
` + + `
Provider quota windows${quotaHelp}
` + + `
${quotaRows || '
No quota windows reported yet
'}
` + + `

Usage, not savings. Cached input is included in input; reasoning is included in output. Deleted or unavailable rollouts are outside this coverage.

` + + `
`; - const strip = - `
` + - statTile('Requests', numFmt(s.requests), `${numFmt(s.compressed_requests)} turned into images`) + - statTile( - 'Input tokens saved', - numFmt(s.saved_input_tokens), - 'vs sending the same context as text', - 'pos', - 'Bulky context sent as compact images instead of text. Uses provider-reported input tokens and a measured or model-profile text counterfactual; recent turns and output stay text.', - ) + - savedUsdTile + - costTile + - `
`; // math drawer const savedMath = `
formula: saved = baseline − actual
` + - `
weights: input×1.0, cache_write_5m×1.25, cache_write_1h×2.0, cache_read×0.10
` + + + `
weights: input×1.0, cache_create_5m×1.25, cache_create_1h×2.0, cache_read×0.10
` + `
` + mathRow('baseline', s.baseline_input_weighted, '(cache-aware: cacheable×weight + cold_tail)') + - mathRow('actual', s.actual_input_weighted, '(input + cc_5m×1.25 + cc_1h×2.0 + cr×0.10)') + + mathRow('actual', s.actual_input_weighted, '(input + server-reported cache tier + cache-read from usage)') + + mathRow('saved', s.saved_input_tokens, `= baseline − actual`) + - `output excluded — identical with/without compression`; + mathRow('Claude measured reduction', measuredClaudeSaved, 'provider count_tokens baseline + upstream usage') + + mathRow('Responses modeled reduction', modeledResponsesSaved, 'local tokenizer/vision counterfactual; not dollar-priced') + + mathRow('Gemini reduction', modeledGeminiSaved, 'provider usage + optional probe or transform-side baseline; not dollar-priced') + + mathRow('TTL sensitivity (unknown creates→1h)', s.saved_if_unknown_cache_create_1h, 'downside scenario for rows whose server response omitted the 5m/1h split; not a confidence interval') + + `
` + + mathRow('measured Claude rows', s.measured_anthropic_savings_requests, 'count_tokens + upstream usage; high-confidence part of the headline') + + mathRow('estimated Responses rows', s.estimated_openai_savings_requests, 'local tokenizer/vision model; disclosed separately in this mixed legacy aggregate') + + mathRow('estimated Gemini rows', s.estimated_google_savings_requests, 'provider usage + optional probe or transform-side baseline; disclosed separately') + + mathRow('probe-excluded rows', s.baseline_probe_excluded_requests, 'compressed paid requests with no successful baseline; zero saving credited') + + mathRow('cache-create 5m / 1h / unknown', `${numFmt(s.cache_create_5m_tokens)} / ${numFmt(s.cache_create_1h_tokens)} / ${numFmt(s.cache_create_tier_unknown_tokens)}`, 'unknown tier retains the legacy 5m assumption; not proof of its TTL') + + `output excluded — identical with/without compression; baseline cache TTL remains a modeled counterfactual`; + + + const usdMath = + `
formula: $ saved = Σ(row_saved × that model’s input rate)
` + - const usdMath = onlyUnpriced - ? `
Unavailable for this provider.
` + - `Token savings are still reported; Claude pricing is not applied across providers.` - : - `
formula: $ saved = saved_tokens × $${pa.input_per_mtok}/Mtok
` + `
` + - mathRow('saved_tokens', s.saved_input_tokens, '(cache-aware, input-side)') + - mathRow('saved_usd', `$${(s.saved_usd || 0).toFixed(4)} `, `= saved_tokens × input_rate / 1e6`) + - `source: ${escapeHtml(pa.source || 'docs.anthropic.com pricing')}`; + mathRow('priced Claude rows', pricedRows, `${numFmt(unpricedRows)} measured Claude rows excluded until configured`) + + mathRow('saved_usd', `$${(s.saved_usd || 0).toFixed(4)} `, `= sum of model-priced rows`) + + `source: ${escapeHtml(pa.source || 'docs.anthropic.com pricing')} · override: PXPIPE_MODEL_INPUT_USD_PER_MTOK JSON`; - const costPerRequestMath = - `
formula: without_pxpipe = actual_imaged + measured_savings
` + - `
why: both averages cover the same paid imaged requests. Passthrough requests are not used because the profitability gate selects a different, generally smaller population.
` + + + const usdToTokenEquiv = (usd: number | undefined): number => + pa.input_per_mtok > 0 ? ((usd ?? 0) * 1e6) / pa.input_per_mtok : 0; + const splitMath = + `
formula: cost_index = weighted actual input + output × ${pa.output_multiplier}
` + + `
why: partitions paid responses by the path that ran. It is a normalized token-equivalent index across mixed providers, not model-priced dollars. Selection bias still applies — read it with the sample counts.
` + `
` + - mathRow(`actual imaged (n=${paidImaged})`, `$${(s.compressed_actual_usd || 0).toFixed(4)}`, `total · avg $${cAvg.toFixed(4)}/req`) + - mathRow('measured savings', `$${(s.saved_usd || 0).toFixed(4)}`, 'cache-aware input-side total') + - mathRow('without pxpipe', `$${withoutAvg.toFixed(4)}/req`, '= (actual imaged + measured savings) / n') + - `unmeasured imaged rows remain in n and actual cost, with zero assumed savings`; + mathRow(`imaged (n=${s.compressed_paid_requests})`, usdToTokenEquiv(s.compressed_actual_usd), `total equivalents · avg ${numFmt(usdToTokenEquiv(s.compressed_avg_usd_per_request))}/response`) + + mathRow(`passthrough (n=${s.passthrough_paid_requests})`, usdToTokenEquiv(s.passthrough_actual_usd), `total equivalents · avg ${numFmt(usdToTokenEquiv(s.passthrough_avg_usd_per_request))}/response`) + + mathRow( + 'imaged − passthrough', + `${numFmt(usdToTokenEquiv(s.compressed_minus_passthrough_avg_usd))}/response`, + s.split_sufficient_sample + ? `(both buckets ≥ ${s.split_min_sample_per_bucket} — delta is meaningful)` + : `(small sample: need ≥ ${s.split_min_sample_per_bucket} per bucket; treat as noisy)`, + ) + + `observed normalized cost index; mixed-provider and not a currency value`; + const pctMath = - `
formula: share_of_spend = saved / (all_baseline_equivalent + all_output × ${pa.output_multiplier})
` + - `
diagnostic, not the headline: this is a counterfactual ("what you WOULD have paid"). It leans on the count_tokens probe, the cache-aware split, and an input-rate assumption. Useful as a sanity check; the real-traffic answer is the compressed-vs-passthrough split above.
` + + `
formula: reduction_share = combined_reduction / (all_baseline_equivalent + all_output × ${pa.output_multiplier})
` + + `
diagnostic, not spend: this mixed-provider counterfactual combines provider-measured Claude and locally modeled Responses reductions in normalized token equivalents. It is not a share of an invoice.
` + `
` + - mathRow('saved', s.saved_input_tokens, '(measured-rows numerator; cache-aware)') + + mathRow('combined_reduction', s.saved_input_tokens, '(Claude measured + Responses modeled; cache-aware)') + mathRow('all_baseline_equivalent', s.all_baseline_equivalent_weighted, '(every paid request; baseline on measured + actual on the rest)') + mathRow(`all_output × ${pa.output_multiplier}`, s.all_output_weighted, '(every paid request)') + - mathRow('share_of_spend', (s.saved_pct_of_all_spend || 0).toFixed(1) + '%', `= saved / counterfactual_total × 100`) + + mathRow('reduction_share', (s.saved_pct_of_all_spend || 0).toFixed(1) + '%', `= combined reduction / counterfactual total × 100`) + mathRow('all_usage_requests', s.all_usage_requests, '(denominator request count — compressed + passthrough + probe-failed)') + - `measured numerator, all-rows counterfactual denominator — bounded at 100%`; + `mixed evidence numerator, all-rows normalized denominator — bounded at 100%`; const tokeqMath = `
formula: token_equivalent = input + output × ${pa.output_multiplier}
` + @@ -378,21 +756,48 @@ export function renderHeaderFragment(s: StatsPayload, port: number): string { `measured — no estimation`; const drawer = - `
` + - `Show the math & honesty receipts` + - `
Every number above, derived from the same per-event log. The proxy only moves input tokens; output is shown on both sides so percentages stay honest.
` + + `
` + + `Show the math & honesty receipts ` + + `
Audit note: the savings headline is a counterfactual (the same request as plain text), not an invoice. Claude credit requires upstream usage and a successful text probe; OpenAI/Responses use a local text-versus-image estimate; Gemini uses provider usage plus an optional probe or transform-side baseline. The proxy only moves input tokens; output is shown on both sides so percentages stay honest.
` + `
` + - mathBlock('Input tokens saved', savedMath) + - mathBlock('Dollars saved', usdMath) + - mathBlock('Cost per imaged request', costPerRequestMath) + - mathBlock('Share of total spend (diagnostic)', pctMath) + - mathBlock('Token-equivalent (what the weekly cap counts)', tokeqMath) + + + mathBlock('Input tokens saved', savedMath, helpTip( + 'audit-input', 'Input tokens saved math', + 'The exact cache-aware counterfactual formula behind the input-change headline.', + 'It exposes the weights, exclusions, and measured-versus-modeled split instead of hiding assumptions in one total.', + 'Compare baseline with actual, then inspect Claude measured and Responses modeled subtotals. Probe-excluded rows receive zero credit.', + )) + + mathBlock('Dollars saved', usdMath, helpTip( + 'audit-dollars', 'Dollar value math', + 'The sum of Claude measured reductions multiplied by each row’s configured model input price.', + 'A single generic price would be misleading when models have different tariffs.', + 'Only priced Claude rows contribute. Responses modeled savings and unknown Claude prices are deliberately excluded.', + )) + + mathBlock('Observed path cost index (diagnostic)', splitMath, helpTip( + 'audit-path-index', 'Observed path cost index', + 'A normalized input-equivalent comparison of actual imaged and passthrough responses.', + 'It is an observational sanity check that does not require a counterfactual baseline.', + 'Use averages only when both samples are large enough. It is not currency, and path selection differences can bias the comparison.', + )) + + mathBlock('Estimated reduction share (mixed diagnostic)', pctMath, helpTip( + 'audit-share', 'Estimated reduction share', + 'Combined estimated reduction divided by a normalized all-response counterfactual total.', + 'It answers how large the attributed change is relative to all paid traffic, including passthrough and uncredited rows.', + 'Treat it as a mixed-evidence diagnostic, not a percentage of an invoice or provider quota.', + )) + + mathBlock('Token-equivalent (what the weekly cap counts)', tokeqMath, helpTip( + 'audit-equivalent', 'Token-equivalent', + 'Input plus output multiplied by the configured output-to-input price ratio.', + 'It places input and output on one normalized scale for limit and cost diagnostics.', + 'Output appears on both actual and baseline sides because PXPIPE does not compress it. Encrypted blocks can be billed even when their characters are not measurable.', + )) + + `
`; // NOTE: tests assert the header fragment contains the port number. const updated = `
live · port ${port} · uptime ${formatDuration(s.uptime_sec)}
`; - return strip + drawer + updated; + return overview + codexPanel + drawer + updated; } // ---- request x-ray (image vs text breakdown) ----------------------------- @@ -468,6 +873,24 @@ export function renderContextMapFragment( const pct = showCompare ? Math.round((1 - real / base) * 100) : 0; const rawShrink = c.baselineTokens > 0 ? Math.round((1 - c.realInput / c.baselineTokens) * 100) : 0; const totalImagedChars = CTXMAP_BUCKETS.reduce((a, [key]) => a + (c.buckets[key] ?? 0), 0); + const billingBasisHelp = helpTip( + 'context-billing-basis', 'Billing-equivalent comparison', + 'The request-level comparison of imaged input with the same context estimated as text, using cache-aware weights.', + 'Raw character counts and raw token counts can disagree with billed input when a cache is warm or newly written.', + 'Use this headline and the Saved/lost table column for the comparable basis. Treat raw content shrinkage as a supporting diagnostic only.', + ); + const imageColumnHelp = helpTip( + 'context-images', 'Compressed into images', + 'Context buckets that PXPIPE rendered into PNG pages for the upstream model.', + 'These are the parts whose token representation can be reduced, at the trade-off of visual rather than byte-exact interpretation.', + 'Character counts describe source size. Inspect pages when fidelity matters, especially for numbers, identifiers, and code-like content.', + ); + const textColumnHelp = helpTip( + 'context-text', 'Kept as plain text', + 'Context PXPIPE deliberately leaves native, including the newest user messages and model output.', + 'Keeping high-precision or recent content as text protects exactness and conversational continuity.', + 'These rows do not become images. “Verbatim” means they are forwarded as text; model output is shown only as usage context.', + ); const imgRows = CTXMAP_BUCKETS.map(([key, label]) => [label, c.buckets[key] ?? 0] as const) .filter(([, ch]) => ch > 0) @@ -557,17 +980,17 @@ export function renderContextMapFragment( return ( `
` + - `
${title} ${headline}
` + + `
${title} ${headline} ${billingBasisHelp}
` + `
${subnote}
` + `
Became an imageStayed as text
` + `
` + `
` + - `
Compressed into images ${kFmt(totalImagedChars)} chars · ${c.imageCount} page${c.imageCount === 1 ? '' : 's'}
` + + `
Compressed into images${imageColumnHelp} ${kFmt(totalImagedChars)} chars · ${c.imageCount} page${c.imageCount === 1 ? '' : 's'}
` + (imgRows || `
nothing imaged this request
`) + `
pxpipe can misread exact values inside images — treat these as gist, not byte-exact.
` + `
` + `
` + - `
Kept as plain text byte-exact
` + + `
Kept as plain text${textColumnHelp} byte-exact
` + `
Your latest messagesverbatim
` + `
Model reply (output)${kFmt(c.output)} tok
` + `
never imaged — safe for IDs, hashes and exact numbers.
` + @@ -589,6 +1012,36 @@ function statusCls(status: number): string { export function renderRecentFragment(p: RecentPayload): string { const rows = (p.recent ?? []).slice().reverse(); + const sentAsHelp = helpTip( + 'recent-sent-as', 'Sent as', + 'Whether the eligible context was sent upstream as images or normal text for this response.', + 'It identifies the actual path, not whether a saving was proven.', + 'Image means PXPIPE transformed eligible context. Text means passthrough. Use Saved/lost to see the attributed input difference when a baseline exists.', + ); + const cacheHitsHelp = helpTip( + 'recent-cache-hits', 'Cache hits', + 'Provider input tokens served from a warm cache on this request.', + 'Warm cache reads have a lower billing weight, so they materially affect fair text-versus-image comparisons.', + 'This is a subset of input usage. A high value can make raw text reduction look larger than billing-equivalent savings.', + ); + const asTextHelp = helpTip( + 'recent-as-text', 'As text', + 'The cache-aware input counterfactual if the same request had stayed as plain text.', + 'It is the baseline used to attribute PXPIPE input change.', + 'Compare it with Sent only on rows where both values exist. It is not a separately billed request.', + ); + const sentHelp = helpTip( + 'recent-sent', 'Sent', + 'Actual cache-aware input usage of the request that was sent upstream.', + 'This is the observed side of the text-versus-image comparison.', + 'Compare it with As text. It includes the appropriate cache create/read weighting, not raw input tokens alone.', + ); + const savedHelp = helpTip( + 'recent-saved-lost', 'Saved/lost', + 'As text minus Sent, expressed as cache-aware input equivalents.', + 'It shows the attributed effect for this one request using the same basis as the dashboard overview.', + 'Positive means estimated reduction; negative means the imaged path cost more. A create badge marks a one-time cache-write premium that later reads may recoup.', + ); const body = rows.length === 0 ? `No requests yet — they stream in here live.` @@ -597,7 +1050,7 @@ export function renderRecentFragment(p: RecentPayload): string { const viewId = (e.img_ids ?? (e.img_id != null ? [e.img_id] : []))[0]; const viewLink = viewId != null - ? `Details →` + ? `` : `—`; const saved = e.session_saved_so_far_delta; // A loss that disappears when the newly written prefix is repriced at @@ -645,11 +1098,13 @@ export function renderRecentFragment(p: RecentPayload): string { `Result` + `Endpoint` + `Model` + - `Sent as` + - `Cache hits` + - `As text` + - `Sent` + - `Saved/lost` + + + `Sent as${sentAsHelp}` + + `Cache hits${cacheHitsHelp}` + + `As text${asTextHelp}` + + `Sent${sentHelp}` + + `Saved/lost${savedHelp}` + + `` + `${body}` ); @@ -857,13 +1312,14 @@ const CSS = ` .muted { color: var(--muted); } /* topbar */ - .topbar { display: flex; align-items: flex-start; justify-content: space-between; - gap: 16px; flex-wrap: wrap; margin-bottom: 18px; } + .topbar { position: sticky; top: 0; z-index: 200; display: flex; align-items: flex-start; justify-content: space-between; + gap: 16px; flex-wrap: wrap; margin: -22px -26px 18px; padding: 14px 26px 12px; background: color-mix(in srgb, var(--bg) 94%, transparent); + border-bottom: 1px solid var(--border); box-shadow: 0 5px 18px rgba(45,28,16,.06); backdrop-filter: blur(12px); } .brand { display: flex; align-items: center; gap: 12px; } .flame-dot { width: 14px; height: 14px; border-radius: 50%; background: radial-gradient(circle at 35% 30%, #ffd0a8, var(--flame) 55%, var(--flame-strong)); box-shadow: 0 0 0 4px var(--flame-tint); flex: none; } - .wordmark { font-size: 22px; font-weight: 800; color: var(--ink); letter-spacing: -0.02em; } + .wordmark { margin: 0; font-size: 22px; font-weight: 800; color: var(--ink); letter-spacing: -0.02em; } .tagline { font-size: 12.5px; color: var(--muted); margin-top: 1px; max-width: 460px; } .controls { display: flex; flex-direction: column; align-items: flex-end; gap: 6px; } @@ -887,6 +1343,61 @@ const CSS = ` box-shadow: var(--shadow); display: inline-flex; align-items: center; gap: 6px; line-height: 1; } .theme-btn:hover { border-color: var(--flame); color: var(--flame-ink); } + .page-nav { display: flex; align-items: center; gap: 5px; margin: -8px 0 14px; overflow-x: auto; } + .page-nav a { color: var(--muted); text-decoration: none; font-size: 11.5px; font-weight: 650; + white-space: nowrap; padding: 5px 9px; border-radius: 7px; } + .page-nav a:hover { color: var(--flame-ink); background: var(--flame-tint); } + .settings-shell { position: relative; } + .settings-shell > .help-tip { position: absolute; z-index: 5; top: 8px; right: 12px; } + .settings-panel { margin: 0 0 14px; background: var(--surface); border: 1px solid var(--border); + border-radius: 10px; box-shadow: var(--shadow); } + .settings-panel > summary { cursor: pointer; list-style: none; padding: 9px 13px; color: var(--ink-2); + padding-right: 46px; font-size: 11.5px; font-weight: 650; } + .settings-panel > summary::-webkit-details-marker { display: none; } + .settings-panel > summary::before { content: '⚙'; margin-right: 7px; color: var(--muted); } + .settings-panel[open] > summary { border-bottom: 1px solid var(--border); } + .settings-panel > summary:focus-visible, .page-nav a:focus-visible, .text-link:focus-visible, + .drawer > summary:focus-visible { outline: 2px solid var(--flame); outline-offset: 2px; } + .settings-panel #frag-models { padding: 12px 13px 2px; } + .settings-panel .models { margin-bottom: 10px; } + + /* Contextual help: native details works with mouse, keyboard, and touch. */ + .help-tip { position: relative; display: inline-flex; flex: none; font-size: 12px; } + .help-tip[open] { z-index: 100; } + .help-tip > summary { display: inline-flex; align-items: center; justify-content: center; width: 19px; height: 19px; + list-style: none; cursor: help; user-select: none; color: var(--muted); background: var(--surface); + border: 1px solid var(--border-strong); border-radius: 50%; font: 750 11px/1 var(--mono); } + .help-tip > summary::-webkit-details-marker { display: none; } + .help-tip > summary:hover, .help-tip > summary:focus-visible, .help-tip[open] > summary { + color: var(--flame-ink); border-color: var(--flame); background: var(--flame-tint); outline: none; } + .help-tip > summary:focus-visible { box-shadow: 0 0 0 2px var(--surface), 0 0 0 4px var(--flame); } + .help-popover { position: absolute; z-index: 110; top: calc(100% + 7px); left: 0; width: min(360px, calc(100vw - 44px)); + padding: 12px 13px; color: var(--ink-2); background: var(--surface); border: 1px solid var(--border-strong); + border-radius: 10px; box-shadow: 0 12px 36px rgba(45, 28, 16, .20); font-size: 11.5px; line-height: 1.45; + text-align: left; text-transform: none; letter-spacing: normal; font-weight: 400; } + :root[data-theme="dark"] .help-popover { box-shadow: 0 14px 40px rgba(0,0,0,.6); } + .help-popover > strong { display: block; margin-bottom: 7px; color: var(--ink); font-size: 12.5px; } + .help-popover p { margin: 6px 0 0; } + .help-popover p b { color: var(--ink); } + .field-label, .th-help, .hero-number-group { display: inline-flex; align-items: center; gap: 6px; } + .th-help { justify-content: flex-end; white-space: nowrap; } + .rtable .help-tip { vertical-align: middle; } + .rtable .help-popover { position: fixed; top: 92px; right: 22px; left: auto; } + .block-label-row, .title-help-row, .card-head-row, .section-title-row, .quota-head, .evidence-actions { + display: flex; align-items: flex-start; justify-content: space-between; gap: 9px; } + .block-label-row > .help-tip .help-popover, .title-help-row > .help-tip .help-popover, + .card-head-row > .help-tip .help-popover, .section-title-row > .help-tip .help-popover, + .quota-head > .help-tip .help-popover, .evidence-actions > .help-tip .help-popover, + .settings-shell > .help-tip .help-popover { left: auto; right: 0; } + .title-help-row { align-items: center; } + .block-label-row .hero-eyebrow, .block-label-row .quality-label { margin-bottom: 0; } + .hero > .block-label-row { margin-bottom: 8px; } + .quota-head { align-items: center; margin-top: 12px; color: var(--ink-2); font-size: 11px; } + @media (max-width: 520px) { + .help-popover { position: fixed; top: 76px; left: 22px !important; right: 22px !important; width: auto; + max-height: calc(100vh - 98px); overflow: auto; } + } + /* model chips */ .models { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; margin: 0 0 18px; } .models-label { color: var(--ink-2); font-size: 12px; font-weight: 600; } @@ -926,61 +1437,167 @@ const CSS = ` margin: 0 0 12px; } /* session hero */ + #current-session { scroll-margin-top: 118px; } #frag-session { display: block; margin-bottom: 16px; } .hero { background: linear-gradient(135deg, var(--flame-tint), var(--surface) 60%); border: 1px solid var(--border); - border-left: 4px solid var(--flame); border-radius: var(--radius); padding: 20px 24px; box-shadow: var(--shadow); } + border-left: 4px solid var(--flame); border-radius: var(--radius); padding: 17px 20px; box-shadow: var(--shadow); } .hero-neg { border-left-color: var(--bad); } .hero-eyebrow { font-size: 11.5px; font-weight: 700; letter-spacing: 0.08em; text-transform: uppercase; color: var(--muted); margin-bottom: 8px; } - .hero-headline { font-size: 28px; font-weight: 700; color: var(--ink); letter-spacing: -0.02em; line-height: 1.1; } - .hero-num { font-size: 56px; font-weight: 800; line-height: 1; margin-right: 8px; + .hero-headline { font-size: 22px; font-weight: 700; color: var(--ink); letter-spacing: -0.02em; line-height: 1.15; } + .hero-num { font-size: 40px; font-weight: 800; line-height: 1; margin-right: 7px; background: linear-gradient(135deg, #ff9a4d, var(--flame) 55%, var(--flame-strong)); -webkit-background-clip: text; background-clip: text; color: transparent; font-variant-numeric: tabular-nums; } .hero-neg .hero-num { background: linear-gradient(135deg, #f0857a, var(--bad)); -webkit-background-clip: text; background-clip: text; color: transparent; } .hero-sub { font-size: 14.5px; color: var(--ink-2); margin-top: 12px; max-width: 720px; } - .hero-meta { font-size: 12px; color: var(--muted); margin-top: 10px; padding-top: 10px; - border-top: 1px dashed var(--border-strong); } + .hero-meta { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; + font-size: 12px; color: var(--muted); margin-top: 10px; padding-top: 10px; border-top: 1px dashed var(--border-strong); } + .session-follow { flex: none; padding: 4px 8px; color: var(--flame-ink); background: var(--surface); border: 1px solid var(--border-strong); + border-radius: 6px; cursor: pointer; font: 650 10.5px/1.2 inherit; } + .session-follow:hover, .session-follow:focus-visible { color: var(--flame-ink); border-color: var(--flame); outline: none; } .hero-empty .hero-headline { color: var(--muted); font-size: 24px; } - /* stat strip */ - .strip { display: grid; grid-template-columns: repeat(4, 1fr); gap: 14px; margin-bottom: 14px; } - @media (max-width: 1000px) { .strip { grid-template-columns: repeat(2, 1fr); } } - @media (max-width: 560px) { .strip { grid-template-columns: 1fr; } } - .tile { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); - padding: 14px 16px; box-shadow: var(--shadow); } - .tile-label { font-size: 11.5px; font-weight: 600; color: var(--ink-2); margin-bottom: 8px; - display: flex; align-items: center; gap: 5px; } - .tile-value { font-size: 26px; font-weight: 800; color: var(--ink); font-variant-numeric: tabular-nums; - letter-spacing: -0.01em; line-height: 1.1; } - .tile-value.pos { color: var(--good); } .tile-value.neg { color: var(--bad); } - .tile-value.muted-val { color: var(--muted); font-size: 18px; font-weight: 600; } - .tile-sub { font-size: 11.5px; color: var(--muted); margin-top: 6px; } - .q { display: inline-flex; align-items: center; justify-content: center; width: 14px; height: 14px; - border-radius: 50%; background: var(--surface-2); border: 1px solid var(--border-strong); - color: var(--muted); font-size: 9px; font-weight: 700; cursor: help; position: relative; outline: none; } - .q:hover, .q:focus-visible { color: var(--flame-ink); border-color: var(--flame); } - .q::after { content: attr(data-tip); position: absolute; z-index: 50; left: 50%; bottom: calc(100% + 8px); - width: min(280px, 75vw); transform: translate(-50%, 4px); padding: 8px 10px; border-radius: 7px; - background: var(--ink); color: var(--surface); box-shadow: var(--shadow); font-size: 11px; font-weight: 500; - line-height: 1.4; text-align: left; pointer-events: none; opacity: 0; visibility: hidden; - transition: opacity .12s, transform .12s, visibility .12s; } - .q::before { content: ''; position: absolute; z-index: 51; left: 50%; bottom: calc(100% + 3px); - transform: translateX(-50%); border: 5px solid transparent; border-top-color: var(--ink); - pointer-events: none; opacity: 0; visibility: hidden; transition: opacity .12s, visibility .12s; } - .q:hover::after, .q:focus-visible::after { opacity: 1; visibility: visible; transform: translate(-50%, 0); } - .q:hover::before, .q:focus-visible::before { opacity: 1; visibility: visible; } + /* Overview — one reading order: effect, activity, evidence. */ + .overview-panel { margin: 0 0 16px; padding: 20px; background: var(--surface); border: 1px solid var(--border); + border-radius: var(--radius); box-shadow: var(--shadow); scroll-margin-top: 118px; } + .overview-head { display: flex; align-items: flex-start; justify-content: space-between; gap: 18px; margin-bottom: 15px; } + .overview-head h2 { margin: 2px 0 3px; color: var(--ink); font-size: 20px; line-height: 1.2; letter-spacing: -.015em; } + .overview-head p { margin: 0; color: var(--muted); font-size: 12px; } + .collapsible-panel > summary { display: flex; align-items: center; justify-content: space-between; gap: 14px; + list-style: none; cursor: pointer; user-select: none; } + .collapsible-panel > summary::-webkit-details-marker { display: none; } + .collapsible-panel > summary:focus-visible { outline: 2px solid var(--flame); outline-offset: 4px; } + .collapsible-title { display: block; margin-top: 2px; color: var(--ink); font-size: 18px; font-weight: 750; line-height: 1.2; } + .collapsible-actions { display: inline-flex; flex: none; align-items: center; gap: 8px; } + .collapse-control { display: inline-flex; align-items: center; min-height: 25px; padding: 4px 9px; color: var(--muted); + border: 1px solid var(--border-strong); border-radius: 999px; font-size: 10.5px; font-weight: 700; } + .collapse-control::before { content: '▾'; margin-right: 5px; color: var(--flame); } + .collapsible-panel:not([open]) .collapse-control::before { content: '▸'; } + .when-closed { display: none; } + .collapsible-panel:not([open]) .when-open { display: none; } + .collapsible-panel:not([open]) .when-closed { display: inline; } + .collapsible-content { padding-top: 15px; } + .codex-panel > summary .scope-label { color: var(--txt); } + @media (max-width: 660px) { + .collapsible-panel > summary { align-items: flex-start; } + .collapsible-actions { flex-direction: column; align-items: flex-end; gap: 5px; } + .collapsible-actions .scope-chip, .collapsible-actions .codex-badge { font-size: 9.5px; } + } + .scope-label { color: var(--flame-ink); font-size: 10px; font-weight: 800; letter-spacing: .09em; text-transform: uppercase; } + .scope-chip { flex: none; display: inline-flex; align-items: center; gap: 7px; padding: 5px 9px; + border: 1px solid var(--border); border-radius: 999px; color: var(--muted); font-size: 10.5px; white-space: nowrap; } + .overview-grid { display: grid; grid-template-columns: minmax(360px, 1.7fr) repeat(2, minmax(210px, 1fr)); gap: 11px; } + .outcome-card, .overview-card { min-width: 0; padding: 15px; background: var(--surface-2); + border: 1px solid var(--border); border-radius: 11px; } + .outcome-card { background: linear-gradient(135deg, var(--good-tint), var(--surface) 76%); border-left: 3px solid var(--good); } + .outcome-card.negative { background: linear-gradient(135deg, var(--bad-tint), var(--surface) 76%); border-left-color: var(--bad); } + .card-eyebrow { color: var(--muted); font-size: 10.5px; font-weight: 750; letter-spacing: .05em; text-transform: uppercase; } + .outcome-value, .overview-value { margin-top: 4px; color: var(--ink); font-size: 31px; font-weight: 820; + line-height: 1.05; letter-spacing: -.025em; font-variant-numeric: tabular-nums; } + .outcome-value { color: var(--good); } + .outcome-card.negative .outcome-value { color: var(--bad); } + .outcome-note { margin-top: 4px; color: var(--muted); font-size: 10.5px; } + .effect-breakdown { display: grid; gap: 7px; margin-top: 13px; padding-top: 11px; border-top: 1px dashed var(--border-strong); } + .effect-row { display: flex; align-items: center; justify-content: space-between; gap: 12px; } + .effect-row > span, .effect-row > strong { min-width: 0; } + .effect-name { display: inline-flex; align-items: center; gap: 7px; } + .effect-result { display: grid; justify-items: end; gap: 4px; text-align: right; } + .effect-row b { display: block; font-size: 12px; } + .effect-row small { display: block; color: var(--muted); font-size: 10px; font-weight: 400; } + .effect-row strong { color: var(--ink); text-align: right; font-size: 13px; font-variant-numeric: tabular-nums; } + .effect-row em { display: block; color: var(--muted); font-size: 10.5px; font-style: normal; font-weight: 550; } + .effect-row.measured em { color: var(--good); } + .price-callout { display: inline-grid; justify-items: end; gap: 1px; padding: 5px 7px; background: var(--good-tint); + border: 1px solid color-mix(in srgb, var(--good) 55%, var(--border)); border-radius: 7px; white-space: nowrap; } + .price-callout small { color: var(--ink-2); font-size: 9.5px; font-weight: 650; } + .price-value { color: var(--good); font-size: 20px; line-height: 1; font-weight: 850; } + .price-callout em { color: var(--muted); font-size: 9.5px; font-weight: 600; } + .overview-lines { display: grid; gap: 5px; margin: 12px 0 9px; font-size: 11.5px; } + .overview-lines span { display: flex; justify-content: space-between; gap: 8px; color: var(--ink-2); } + .overview-lines.compact span { display: block; } + .overview-card > small { color: var(--muted); font-size: 10px; } + .evidence-title { display: flex; align-items: flex-start; justify-content: space-between; gap: 8px; } + .text-link { color: var(--flame-ink); font-size: 10.5px; font-weight: 650; text-decoration: none; } + .text-link:hover { text-decoration: underline; } + .reading-legend { display: flex; flex-wrap: wrap; gap: 7px 18px; margin-top: 12px; padding: 9px 11px; + background: var(--surface-2); border-radius: 8px; color: var(--muted); font-size: 10.5px; } + .reading-legend b { color: var(--ink-2); } + @media (max-width: 1050px) { .overview-grid { grid-template-columns: 1fr 1fr; } .outcome-card { grid-column: 1 / -1; } } + @media (max-width: 660px) { + .overview-panel { padding: 15px; } .overview-head { flex-direction: column; } + .overview-grid { grid-template-columns: 1fr; } .outcome-card { grid-column: auto; } + .effect-row { align-items: flex-start; } + } + + /* estimate quality — visible trust summary, not hidden in the math drawer */ + .quality-panel { margin: 0 0 14px; padding: 18px; background: linear-gradient(135deg, var(--good-tint), var(--surface) 68%); + border: 1px solid var(--border); border-left: 4px solid var(--good); border-radius: var(--radius); box-shadow: var(--shadow); } + .quality-head { display: flex; align-items: flex-start; justify-content: space-between; gap: 20px; } + .quality-eyebrow { margin-bottom: 5px; color: var(--good); font-size: 10.5px; font-weight: 800; + letter-spacing: .09em; text-transform: uppercase; } + .quality-title { margin: 0; color: var(--ink); font-size: 18px; line-height: 1.25; letter-spacing: -.01em; } + .quality-lead { max-width: 860px; margin: 7px 0 0; color: var(--ink-2); font-size: 12.5px; line-height: 1.5; } + .quality-badge { flex: none; display: inline-flex; align-items: center; gap: 6px; padding: 5px 10px; + border-radius: 999px; font-size: 11px; font-weight: 700; white-space: nowrap; } + .quality-badge::before { content: ''; width: 7px; height: 7px; border-radius: 50%; background: currentColor; } + .quality-badge.good { color: var(--good); background: var(--good-tint); border: 1px solid currentColor; } + .quality-badge.mixed { color: var(--warn); background: var(--warn-tint); border: 1px solid currentColor; } + .quality-badge.waiting { color: var(--muted); background: var(--surface-2); border: 1px solid var(--border-strong); } + .quality-grid { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; margin-top: 15px; } + .quality-metric { min-width: 0; padding: 11px 12px; background: color-mix(in srgb, var(--surface) 82%, transparent); + border: 1px solid var(--border); border-radius: 9px; } + .quality-label { display: block; margin-bottom: 5px; color: var(--muted); font-size: 10.5px; font-weight: 700; + letter-spacing: .03em; text-transform: uppercase; } + .quality-metric strong { display: block; color: var(--ink); font-size: 16px; line-height: 1.25; font-variant-numeric: tabular-nums; } + .quality-metric small { display: block; margin-top: 5px; color: var(--muted); font-size: 10.5px; line-height: 1.35; } + .quality-foot { display: flex; align-items: center; flex-wrap: wrap; gap: 7px 13px; margin-top: 13px; } + .quality-check { color: var(--good); font-size: 11px; font-weight: 600; } + .quality-excluded { margin-left: auto; color: var(--muted); font-size: 11px; } + .quality-caveat { margin: 12px 0 0; padding-top: 10px; border-top: 1px dashed var(--border-strong); + color: var(--muted); font-size: 11px; line-height: 1.45; } + .quality-caveat strong { color: var(--ink-2); } + @media (max-width: 1000px) { .quality-grid { grid-template-columns: repeat(2, 1fr); } } + @media (max-width: 680px) { + .quality-head { flex-direction: column; gap: 10px; } + .quality-grid { grid-template-columns: 1fr; } + .quality-excluded { width: 100%; margin-left: 0; } + } + + /* exact Codex rollout usage — visually distinct from estimated savings */ + .codex-panel { margin: 0 0 14px; padding: 18px; background: linear-gradient(135deg, var(--txt-tint), var(--surface) 68%); + border: 1px solid var(--border); border-left: 4px solid var(--txt); border-radius: var(--radius); box-shadow: var(--shadow); + scroll-margin-top: 118px; } + .codex-eyebrow { margin-bottom: 5px; color: var(--txt); font-size: 10.5px; font-weight: 800; + letter-spacing: .09em; text-transform: uppercase; } + .codex-badge { flex: none; display: inline-flex; align-items: center; gap: 6px; padding: 5px 10px; + color: var(--txt-ink); background: var(--txt-tint); border: 1px solid var(--txt); border-radius: 999px; + font-size: 11px; font-weight: 700; white-space: nowrap; } + .codex-badge::before { content: ''; width: 7px; height: 7px; border-radius: 50%; background: var(--txt); } + .usage-scope { margin-top: 11px; color: var(--muted); font-size: 10.5px; } + .quota-list { display: grid; gap: 7px; margin-top: 11px; } + .quota-row { display: grid; grid-template-columns: minmax(180px, 1.2fr) repeat(auto-fit, minmax(140px, 1fr)); gap: 10px; + padding: 9px 11px; background: var(--surface); border: 1px solid var(--border); border-radius: 9px; } + .quota-row > span { min-width: 0; } + .quota-row small, .quota-row em { display: block; color: var(--muted); font-size: 9.5px; font-style: normal; } + .quota-row b { display: block; color: var(--ink); font-size: 12px; font-variant-numeric: tabular-nums; } + .quota-name small { font-family: var(--mono); } + .quota-empty { color: var(--muted); font-size: 11px; padding: 9px; } + @media (max-width: 680px) { .quota-row { grid-template-columns: 1fr 1fr; } .quota-name { grid-column: 1 / -1; } } /* drawer */ .drawer { margin: 0 0 14px; background: var(--surface); border: 1px solid var(--border); - border-radius: var(--radius); box-shadow: var(--shadow); overflow: hidden; } + border-radius: var(--radius); box-shadow: var(--shadow); overflow: visible; scroll-margin-top: 118px; } .drawer > summary { cursor: pointer; user-select: none; list-style: none; padding: 12px 16px; font-size: 13px; font-weight: 600; color: var(--flame-ink); display: flex; align-items: center; gap: 8px; } .drawer > summary::-webkit-details-marker { display: none; } .drawer > summary::before { content: '▸'; color: var(--flame); font-size: 11px; } .drawer[open] > summary::before { content: '▾'; } .drawer > summary:hover { background: var(--surface-2); } + .summary-q { display: inline-flex; align-items: center; justify-content: center; width: 18px; height: 18px; + margin-left: auto; border: 1px solid var(--border-strong); border-radius: 50%; color: var(--muted); + font: 750 10px/1 var(--mono); } .drawer-intro { padding: 0 16px 10px; font-size: 12px; color: var(--ink-2); } .drawer-intro em { color: var(--flame-ink); font-style: normal; font-weight: 600; } .math-grid { display: grid; grid-template-columns: repeat(2, 1fr); gap: 12px; padding: 0 16px 16px; } @@ -1006,7 +1623,11 @@ const CSS = ` padding: 16px 18px; box-shadow: var(--shadow); min-width: 0; } .card-head { font-size: 12px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); margin: 0 0 12px; } - .card-head.spaced { margin-top: 22px; padding-top: 16px; border-top: 1px solid var(--border); } + .card-head-row > .card-head { margin-bottom: 12px; } + .card-head-row.spaced { margin-top: 22px; padding-top: 16px; border-top: 1px solid var(--border); } + .card-head-row.spaced > .card-head { margin-bottom: 12px; } + .section-title-row { align-items: baseline; } + .section-title-row > .section-head { flex: 1; } /* x-ray */ .xray { display: grid; grid-template-columns: 1.15fr 1fr; gap: 16px; align-items: start; } @@ -1052,7 +1673,7 @@ const CSS = ` color: var(--muted); font-size: 10px; cursor: default; } /* recent requests */ - .row-view { color: var(--flame-ink); font-weight: 600; text-decoration: none; cursor: pointer; white-space: nowrap; } + .row-view { padding: 0; color: var(--flame-ink); background: transparent; border: 0; font: inherit; font-weight: 600; text-decoration: none; cursor: pointer; white-space: nowrap; } .row-view:hover { text-decoration: underline; } table.rtable, table.dtable { width: 100%; border-collapse: collapse; font-size: 12px; } .rtable th, .dtable th { text-align: left; color: var(--muted); font-weight: 600; padding: 7px 8px; @@ -1136,7 +1757,7 @@ const CSS = ` // Client glue: window.pp (pin+source state) → hx-vals; preserves
open state across swaps; routes htmx errors to toast tray. const GLUE_JS = ` - window.pp = { pin: null, src: false }; + window.pp = { pin: null, src: false, session: null, hashRevealPending: !!location.hash }; function ppPin(id) { window.pp.pin = id; htmx.trigger('#frag-latest', 'pp-refresh'); @@ -1145,17 +1766,57 @@ const GLUE_JS = ` window.pp.src = on; htmx.trigger('#frag-latest', 'pp-refresh'); } + function ppWatchLatest() { + window.pp.session = null; + htmx.trigger('#frag-session', 'pp-refresh'); + } document.body.addEventListener('htmx:beforeSwap', function (ev) { - const open = []; - ev.detail.target.querySelectorAll('details[open][id]').forEach(function (d) { open.push(d.id); }); - ev.detail.target.__ppOpen = open; + const states = []; + ev.detail.target.querySelectorAll('details[id]').forEach(function (d) { + states.push({ id: d.id, open: d.open }); + }); + ev.detail.target.__ppDetails = states; }); document.body.addEventListener('htmx:afterSwap', function (ev) { - (ev.detail.target.__ppOpen || []).forEach(function (id) { - const d = document.getElementById(id); - if (d) d.setAttribute('open', ''); + (ev.detail.target.__ppDetails || []).forEach(function (state) { + const d = document.getElementById(state.id); + if (d) d.toggleAttribute('open', state.open); + }); + if (window.pp.hashRevealPending) ppRevealHash(); + if (ev.detail.target && ev.detail.target.id === 'frag-session') { + var sessionNode = ev.detail.target.querySelector('[data-session-id]'); + var sessionId = sessionNode && sessionNode.getAttribute('data-session-id'); + if (sessionId && sessionId !== window.pp.session) window.pp.session = sessionId; + } + }); + function ppRevealHash() { + if (!location.hash || location.hash.length < 2) { window.pp.hashRevealPending = false; return; } + var target = document.getElementById(location.hash.slice(1)); + if (!target) return false; + if (target.tagName === 'DETAILS') target.setAttribute('open', ''); + requestAnimationFrame(function () { target.scrollIntoView({ block: 'start' }); }); + window.pp.hashRevealPending = false; + return true; + } + document.body.addEventListener('click', function (ev) { + var link = ev.target.closest && ev.target.closest('a[href^="#"]'); + if (link) { window.pp.hashRevealPending = true; setTimeout(ppRevealHash, 0); } + if (!(ev.target.closest && ev.target.closest('.help-tip'))) { + document.querySelectorAll('details.help-tip[open]').forEach(function (d) { d.removeAttribute('open'); }); + } + }); + document.addEventListener('toggle', function (ev) { + var opened = ev.target; + if (!opened.matches || !opened.matches('details.help-tip[open]')) return; + document.querySelectorAll('details.help-tip[open]').forEach(function (d) { + if (d !== opened) d.removeAttribute('open'); }); + }, true); + document.addEventListener('keydown', function (ev) { + if (ev.key !== 'Escape') return; + document.querySelectorAll('details.help-tip[open]').forEach(function (d) { d.removeAttribute('open'); }); }); + window.addEventListener('hashchange', function () { window.pp.hashRevealPending = true; ppRevealHash(); }); document.body.addEventListener('htmx:responseError', function (ev) { window.dispatchEvent(new CustomEvent('pp-toast', { detail: { text: ev.detail.xhr.status + ' ' + ev.detail.requestConfig.path } @@ -1215,8 +1876,8 @@ export function renderPage(port: number): string {
-
pxpipe
-
See exactly what got turned into images to shrink your Claude Code bill.
+

pxpipe

+
Live proxy effect, provider usage, and an auditable explanation of every estimate.
@@ -1225,80 +1886,96 @@ export function renderPage(port: number): string {
+
Connect an agent warp launches any CLI through this proxy · pin keeps instructions last in the request -

Warp starts the agent with the proxy already wired, no env or config edits:

+

Warp starts the agent with the proxy already wired:

pxpipe warp -- claude
 pxpipe warp -- codex
 pxpipe warp -- cursor-agent
-

Aliases work too (pxpipe warp -- pp), and --route pattern=http://host:port adds routes beyond api.anthropic.com. Without warp, point the agent at ANTHROPIC_BASE_URL=http://127.0.0.1:${port} yourself.

-

Pin instructions from inside the session — they get moved to the end of every request, where the model actually reads them:

- 
@pxpipe pin be concise, no walls of text
-@pxpipe unpin 2
-@pxpipe unpin all
-

@pxpipe pin with no text lists what is pinned.

-

A @pxpipe pin … line in your global or project CLAUDE.md (AGENTS.md under Codex / OpenCode) is relocated the same way, on every session, with no typing. Those are file-backed, so unpin and unpin all never touch them — edit the file to remove one.

+

Pin instructions from inside a session with @pxpipe pin …; pinned text is relocated to the end of every request.

-
- Image model scope Fable 5 and Gemini 3.6 Flash by default · expand to experiment with other families -
⚠ Image compression is validated for Fable 5 and Gemini 3.6 Flash — other families can use more tokens, not less. Opt in only for deliberate experiments.
-
-
imaging scope ≠ provider routing — non-Anthropic IDs also need routing env on the proxy
-
+ + + +
+
+ Model scope & routing settings +
+
+ ${helpTip( + 'model-settings', 'Model scope and routing settings', + 'Runtime controls for choosing which model bases PXPIPE may transform.', + 'They let you opt models in or out without changing upstream routing or restarting the proxy.', + 'Checked chips are active now. Your choice is saved automatically and survives restart, overriding PXPIPE_MODELS until you press Reset to default.', + )} +
- -

Routing Claude Code to OpenAI / Cloudflare models

-

Claude models use Anthropic by default. Two optional routes can run together — set on the pxpipe process (keep provider credentials out of Claude Code):

-
    -
  • OPENAI_MODELS — exact model IDs routed to OpenAI Responses (OPENAI_UPSTREAM + OPENAI_API_KEY)
    • -
  • CLOUDFLARE_MODELS — exact model IDs routed to Cloudflare's OpenAI-compatible endpoint (CLOUDFLARE_ACCOUNT_ID + CLOUDFLARE_API_TOKEN)
    • -
-

If a model appears in both lists: CLOUDFLARE_MODELS > OPENAI_MODELS > default routing.

- 
OPENAI_UPSTREAM=https://api.openai.com \\
-OPENAI_API_KEY=your-openai-key \\
-OPENAI_MODELS=gpt-5.6-sol \\
-CLOUDFLARE_ACCOUNT_ID=your-account-id \\
-CLOUDFLARE_API_TOKEN=your-cloudflare-token \\
-CLOUDFLARE_MODELS=moonshotai/kimi-k3 \\
-npx pxpipe-proxy
-

Non-Anthropic IDs are advertised with a claude- prefix because Claude Code needs a Claude-shaped ID; pxpipe strips it before forwarding. Switch to one inside Claude Code with /model claude-<model> — e.g. /model claude-moonshotai/kimi-k3 — or launch with claude --model claude-moonshotai/kimi-k3. Verify discovery with curl …/v1/models.

-

PXPIPE_MODELS above is separate: it controls image compression, not routing. Kimi K3 on Cloudflare is the only non-Anthropic model tested end to end — see docs/CLAUDE_CODE_PROVIDER_ROUTING.md.

- - - -
+
Connecting…
-
+
-
+

What happened to your context click a request to see image vs text

-

Recent requests

+

Recent requests

${helpTip( + 'recent-requests', 'Recent requests', + 'The latest proxy responses with model, status, path, usage, and attributed input change.', + 'It connects summary totals to individual requests so unusual rows can be inspected.', + 'Use status and path first, then compare actual and baseline values. Open Details to inspect the exact image-versus-text composition.', + )}
-

Image vs text breakdown

+

Image vs text breakdown

${helpTip( + 'context-breakdown', 'Image versus text breakdown', + 'A request-level map of which context buckets stayed native text and which were rendered into images.', + 'It explains where the estimated input change came from instead of showing only a final number.', + 'Compare the image and text columns. Cache-aware effective input drives savings; raw token counts are supporting diagnostics.', + )}
-

Image ↔ source inspector

+

Image ↔ source inspector

${helpTip( + 'source-inspector', 'Image and source inspector', + 'A visual preview of the PNG sent to the model, optionally paired with the source text used to create it.', + 'It lets you verify fidelity, layout, and source-to-image pairing for a concrete request.', + 'Select a recent request, inspect each page, and enable source view when you need to compare content. Previews can disappear after ring-buffer eviction or restart.', + )}
-
-

Top sessions by tokens saved

+
+

Top sessions by tokens saved

${helpTip( + 'top-sessions', 'Top sessions', + 'A ranking of retained PXPIPE sessions by their accumulated estimated input change.', + 'It reveals which conversations contribute most to the total and where investigation is most useful.', + 'Longer bars mean larger positive estimated reduction. Negative values indicate sessions where imaging cost more effective input than the text counterfactual.', + )}
-
-

Full history every event on disk

+
+

Full history every event on disk

${helpTip( + 'full-history', 'Full history', + 'The event log PXPIPE retained on disk, including successful, passthrough, and error events.', + 'It is the durable audit trail behind session and request diagnostics.', + 'Use it for completeness and troubleshooting rather than headline savings. Rows without upstream usage may be operational events and are not paid LLM responses.', + )}
diff --git a/src/dashboard/types.ts b/src/dashboard/types.ts index f2090c0d6..1eb54e711 100644 --- a/src/dashboard/types.ts +++ b/src/dashboard/types.ts @@ -1,5 +1,7 @@ // JSON payload shapes for the dashboard. Single source of truth — update here when src/dashboard.ts changes. +import type { CodexUsageSnapshot } from '../codex-usage.js'; + /** /proxy-stats payload. */ export interface StatsPayload { port: number; @@ -10,6 +12,8 @@ export interface StatsPayload { baseline_input_weighted: number; actual_input_weighted: number; saved_input_tokens: number; + /** Sensitivity only: reprices unreported cache-create tiers at 1h on the actual side. */ + saved_if_unknown_cache_create_1h: number; /** Back-compat duplicate of `saved_pct_input_only`. */ saved_pct: number; saved_pct_input_only: number; @@ -20,7 +24,10 @@ export interface StatsPayload { all_baseline_equivalent_weighted: number; all_actual_input_weighted: number; all_output_weighted: number; + /** Back-compatible alias for `usage_bearing_responses`. */ all_usage_requests: number; + /** Responses with non-zero provider usage that entered the paid-traffic totals. */ + usage_bearing_responses: number; /** Observed cost split: compressed vs passthrough paths on real traffic. `split_sufficient_sample` gates the per-request delta (UI shows caveat below threshold). */ compressed_paid_requests: number; passthrough_paid_requests: number; @@ -33,6 +40,26 @@ export interface StatsPayload { split_min_sample_per_bucket: number; /** Cache-tier-aware input-side dollar savings. */ saved_usd: number; + /** Audit coverage: rows in the Claude count_tokens + upstream-usage numerator. */ + measured_anthropic_savings_requests: number; + /** Count_tokens-measured Claude savings, in input-token equivalents. */ + measured_claude_saved_input_equivalents: number; + /** GPT/OpenAI savings use local tokenizer/vision estimates and are not in the Claude headline. */ + estimated_openai_savings_requests: number; + /** Locally modeled OpenAI/Responses savings, in input-token equivalents. */ + modeled_openai_saved_input_equivalents: number; + /** Gemini/Google rows with provider usage and a probe- or transform-derived baseline. */ + estimated_google_savings_requests: number; + /** Gemini/Google input reduction estimate, kept separate from Claude and Responses evidence. */ + modeled_google_saved_input_equivalents: number; + /** Compressed paid Anthropic rows withheld from the numerator when a probe failed. */ + baseline_probe_excluded_requests: number; + /** Server-reported cache-create tier coverage for Anthropic actual usage. */ + cache_create_5m_tokens: number; + cache_create_1h_tokens: number; + cache_create_tier_unknown_tokens: number; + priced_measured_savings_requests: number; + unpriced_measured_savings_requests: number; output_weighted: number; baseline_token_equivalent: number; actual_token_equivalent: number; @@ -42,6 +69,8 @@ export interface StatsPayload { measured_tool_use_chars: number; measured_redacted_block_count: number; events_with_measurement: number; + /** Exact provider-reported Codex usage imported from official rollout logs. */ + codex_actual_usage: CodexUsageSnapshot; uptime_sec_unused?: never; // future-proof compression_enabled: boolean; } diff --git a/src/events-lock.ts b/src/events-lock.ts new file mode 100644 index 000000000..fe610b540 --- /dev/null +++ b/src/events-lock.ts @@ -0,0 +1,115 @@ +/** Cross-process exclusion for the append-only events log and destructive + * rewrites. A writer holds the lock for its lifetime; prune holds the same + * lock from its first scan through the final rename. */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export type EventsLockRole = 'writer' | 'prune'; + +interface LockRecord { + pid: number; + role: EventsLockRole; + token: string; +} + +export interface EventsFileLock { + readonly path: string; + readonly role: EventsLockRole; + release(): void; +} + +function processIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code !== 'ESRCH'; + } +} + +function readRecord(lockPath: string): LockRecord | undefined { + let raw: string; + try { + raw = fs.readFileSync(lockPath, 'utf8'); + } catch { + return undefined; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return undefined; + } + + // Backward compatibility with the old PID-only writer marker. A plain PID + // is valid JSON, so it must be handled after parsing rather than in catch. + if (typeof parsed === 'number') { + if (Number.isSafeInteger(parsed) && parsed > 0) { + return { pid: parsed, role: 'writer', token: `legacy:${parsed}` }; + } + return undefined; + } + if (!parsed || typeof parsed !== 'object') return undefined; + const r = parsed as Partial; + if (!Number.isSafeInteger(r.pid) || (r.pid ?? 0) <= 0) return undefined; + if (r.role !== 'writer' && r.role !== 'prune') return undefined; + if (typeof r.token !== 'string' || r.token.length === 0) return undefined; + return r as LockRecord; +} + +export function acquireEventsFileLock( + eventsFile: string, + role: EventsLockRole, +): EventsFileLock { + const lockPath = eventsFile + '.writer.lock'; + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + const token = `${process.pid}:${role}:${Date.now()}:${Math.random().toString(16).slice(2)}`; + const record: LockRecord = { pid: process.pid, role, token }; + + for (let attempt = 0; attempt < 4; attempt++) { + let fd: number; + try { + fd = fs.openSync(lockPath, 'wx', 0o600); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + const owner = readRecord(lockPath); + if (!owner || processIsAlive(owner.pid)) { + const description = owner ? `${owner.role} pid ${owner.pid}` : 'an unreadable lock'; + throw new Error(`events log is already owned by ${description}`); + } + // Stale owner. Removal and the next exclusive create are race-safe: if + // another contender wins, our next open sees its live record and fails. + try { + fs.unlinkSync(lockPath); + } catch (unlinkErr) { + if ((unlinkErr as NodeJS.ErrnoException).code !== 'ENOENT') throw unlinkErr; + } + continue; + } + try { + fs.writeSync(fd, JSON.stringify(record) + '\n'); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + let released = false; + return { + path: lockPath, + role, + release(): void { + if (released) return; + released = true; + const current = readRecord(lockPath); + if (current?.token !== token) return; + try { + fs.unlinkSync(lockPath); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err; + } + }, + }; + } + throw new Error('could not acquire events log lock after stale-owner races'); +} diff --git a/src/health-counters.ts b/src/health-counters.ts new file mode 100644 index 000000000..183c25738 --- /dev/null +++ b/src/health-counters.ts @@ -0,0 +1,46 @@ +/** Host-side rolling counter of recent /backend-api/codex/* traffic. Feeds the + * evidence-driven codex-upstream check. Time is passed in (Date.now() from the + * host) so it is deterministically testable. Not in core: it holds state and + * reads the clock via its caller. */ + +const DEFAULT_WINDOW_MS = 300_000; // 5 minutes + +interface Entry { + ts: number; + is404: boolean; +} + +export class HealthCounters { + private codex: Entry[] = []; + + constructor(private readonly windowMs: number = DEFAULT_WINDOW_MS) {} + + private isCodexPath(path: string): boolean { + return path.startsWith('/backend-api/codex/'); + } + + record(path: string, status: number, nowMs: number): void { + if (!this.isCodexPath(path)) return; + this.codex.push({ ts: nowMs, is404: status === 404 }); + this.trim(nowMs); + } + + /** Entries are appended in time order, so stale ones are a prefix. */ + private trim(nowMs: number): void { + const cutoff = nowMs - this.windowMs; + let i = 0; + while (i < this.codex.length && this.codex[i]!.ts < cutoff) i++; + if (i > 0) this.codex.splice(0, i); + } + + snapshot(nowMs: number): { codexResponses404: number; codexResponsesTotal: number; windowSeconds: number } { + this.trim(nowMs); + let c404 = 0; + for (const e of this.codex) if (e.is404) c404++; + return { + codexResponses404: c404, + codexResponsesTotal: this.codex.length, + windowSeconds: Math.round(this.windowMs / 1000), + }; + } +} diff --git a/src/health-state.ts b/src/health-state.ts new file mode 100644 index 000000000..634b74a3f --- /dev/null +++ b/src/health-state.ts @@ -0,0 +1,63 @@ +/** Host-side assembler: turns pxpipe's live config + runtime state into the + * HealthState snapshot the pure evaluator consumes. Kept out of node.ts so it + * is importable in tests without starting a server. */ + +import { resolveUpstreams, type ProxyConfig } from './core/proxy.js'; +import { getAllowedModelBases } from './core/applicability.js'; +import { + evaluateHealth, + summarizeHealth, + type HealthFinding, + type HealthState, +} from './core/health.js'; +import type { HealthCounters } from './health-counters.js'; + +export function buildHealthState( + config: ProxyConfig, + compression: { getCompressionEnabled(): boolean }, + counters: HealthCounters, + nowMs: number, +): HealthState { + const routes = resolveUpstreams(config); + return { + anthropicUpstream: routes.anthropic, + openaiUpstream: routes.openai, + // Phase A: no runtime upstream override exists yet — always false. + openaiUpstreamOverridden: false, + modelScope: getAllowedModelBases(), + compressionEnabled: compression.getCompressionEnabled(), + recent: counters.snapshot(nowMs), + }; +} + +export interface HealthReport { + ok: boolean; + findings: HealthFinding[]; + state: HealthState | null; + httpStatus: 200 | 503; +} + +/** Build the public health report without ever mistaking a diagnostic failure + * for a healthy process. The Node handler can still return the JSON report on + * /api/health.json, while /healthz uses httpStatus for its probe semantics. */ +export function buildHealthReport( + config: ProxyConfig, + compression: { getCompressionEnabled(): boolean }, + counters: HealthCounters, + nowMs: number, +): HealthReport { + try { + const state = buildHealthState(config, compression, counters, nowMs); + const findings = evaluateHealth(state); + const summary = summarizeHealth(findings); + return { ok: summary.ok, findings, state, httpStatus: summary.httpStatus }; + } catch { + const findings: HealthFinding[] = [{ + id: 'health-diagnostics-failed', + severity: 'error', + title: 'Health diagnostics failed', + detail: 'pxpipe could not assemble its health state; treat this instance as unhealthy.', + }]; + return { ok: false, findings, state: null, httpStatus: 503 }; + } +} diff --git a/src/model-scope-store.ts b/src/model-scope-store.ts new file mode 100644 index 000000000..4cef46f6c --- /dev/null +++ b/src/model-scope-store.ts @@ -0,0 +1,76 @@ +/** + * Node-host persistence for the dashboard's model-scope choice. + * + * The dashboard chips (renderModelsFragment) drive an in-memory runtime + * override in core (setAllowedModelBases). Core stays filesystem-free so it + * runs on Workers; this module is the Node-only sidecar that makes a chip + * choice survive a restart. + * + * Variant A precedence: the persisted choice is seeded into the runtime + * override at startup, so it OVERRIDES PXPIPE_MODELS until the user hits + * Reset (which clears the file and the override). PXPIPE_MODELS is only the + * default when no persist file exists. + * + * The file lives next to the events log, matching the `4xx-bodies` sidecar + * convention (a single `rm -rf ~/.pxpipe` cleans everything up). + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +/** Path of the persisted model-scope file (sibling of the events log). */ +export function modelScopeFile(eventsFile: string): string { + return path.join(path.dirname(eventsFile), 'model-scope.json'); +} + +/** + * Load the persisted scope. + * - absent OR unreadable OR malformed → `null` (fall back to PXPIPE_MODELS + * env / built-in default; a corrupt file must never block startup). + * - present + valid → the stored array VERBATIM, including `[]`, which is a + * real choice meaning "every model off / compress nothing". + * Best-effort: never throws. + */ +export function loadPersistedModelScope(file: string): string[] | null { + try { + const raw = fs.readFileSync(file, 'utf8'); + const parsed = JSON.parse(raw) as { modelBases?: unknown }; + const bases = parsed?.modelBases; + if (!Array.isArray(bases)) return null; + const normalized = bases + .filter((b): b is string => typeof b === 'string') + .map((b) => b.trim()) + .filter(Boolean); + // [] is an intentional "disable every model" choice. A non-empty array + // that normalizes to nothing, however, is corrupt and must not silently + // turn compression off for every model. + return bases.length > 0 && normalized.length === 0 ? null : normalized; + } catch { + return null; + } +} + +/** + * Persist the current scope. Best-effort; a failed write never breaks a chip + * toggle. An empty list is persisted as-is (distinct from Reset/clear). + */ +export function savePersistedModelScope(file: string, bases: readonly string[]): void { + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify({ modelBases: [...bases] }, null, 2)}\n`); + } catch { + /* best-effort: persistence is a convenience, not a correctness guarantee */ + } +} + +/** + * Clear the persisted scope (Reset → fall back to PXPIPE_MODELS env / default). + * Best-effort; `force` makes an absent file a no-op rather than an error. + */ +export function clearPersistedModelScope(file: string): void { + try { + fs.rmSync(file, { force: true }); + } catch { + /* best-effort */ + } +} diff --git a/src/node.ts b/src/node.ts index 76f774445..2a3a6b5c1 100644 --- a/src/node.ts +++ b/src/node.ts @@ -11,6 +11,7 @@ import { createWarpRuntime } from './warp/index.js'; import { once } from 'node:events'; import * as fs from 'node:fs'; import * as path from 'node:path'; +import * as zlib from 'node:zlib'; import * as os from 'node:os'; import { isIP } from 'node:net'; import { spawnSync } from 'node:child_process'; @@ -36,6 +37,18 @@ import { dashboardPath, type DashboardRoute, } from './dashboard.js'; +import { evaluateHealth } from './core/health.js'; +import { HealthCounters } from './health-counters.js'; +import { buildHealthReport, buildHealthState } from './health-state.js'; +import { CodexUsageIndex } from './codex-usage.js'; +import { acquireEventsFileLock, type EventsFileLock } from './events-lock.js'; +import { + modelScopeFile, + loadPersistedModelScope, + savePersistedModelScope, + clearPersistedModelScope, +} from './model-scope-store.js'; +import { setAllowedModelBases } from './core/applicability.js'; /** Runtime config. The core transform tuning comes from DEFAULTS in * transform.ts; startup knobs cover deployment plus emergency GPT scope @@ -60,6 +73,15 @@ interface RuntimeConfig { /** Persist 4xx request and upstream error bodies for debugging. Off unless * PXPIPE_DEBUG_CAPTURE_4XX=1. */ captureErrorReqBody: boolean; + /** Exact trusted proxy address for loopback-published dashboard traffic. */ + trustedDashboardProxy?: string; + /** Public origins accepted for dashboard mutations behind a reverse proxy + * (PXPIPE_DASHBOARD_ORIGINS, comma-separated). */ + dashboardOrigins?: readonly string[]; + /** Explicit acknowledgement that an external boundary (for example the + * bundled Compose loopback port publication) protects a non-loopback bind + * that injects server-owned upstream credentials. */ + allowNonLoopbackCredentials: boolean; } const DEFAULT_CONFIG_FILE = path.join(os.homedir(), '.config', 'pxpipe', 'config.json'); @@ -94,48 +116,6 @@ function applyConfigFileDefaults(): void { } } -/** Dashboard persistence hook: write the runtime model scope back to the - * config file's `models` key so chip toggles survive a restart. Other keys - * are preserved; an invalid existing file is left untouched. - * NOTE: on the next start an explicit PXPIPE_MODELS env still wins over the - * persisted value (same precedence as every other config-file default). */ -function persistModelBasesToConfig(bases: readonly string[]): void { - const file = process.env.PXPIPE_CONFIG ?? DEFAULT_CONFIG_FILE; - let cfg: Record = {}; - try { - const parsed = JSON.parse(fs.readFileSync(file, 'utf8')) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - console.warn(`[pxpipe] could not persist model scope: invalid config object ${file}`); - return; - } - cfg = parsed as Record; - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== 'ENOENT') { - console.warn(`[pxpipe] could not persist model scope: invalid config ${file}: ${(e as Error).message}`); - return; - } - } - // Empty array round-trips as 'off' via normalizeModelsConfig on load. - cfg.models = [...bases]; - const tmp = `${file}.tmp-${process.pid}`; - try { - const parentExists = fs.existsSync(path.dirname(file)); - fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); - if (!parentExists) fs.chmodSync(path.dirname(file), 0o700); - // Write-then-rename so a crash mid-write can't corrupt the config. - fs.writeFileSync(tmp, `${JSON.stringify(cfg, null, 2)}\n`, { mode: 0o600 }); - fs.renameSync(tmp, file); - fs.chmodSync(file, 0o600); - } catch (e) { - try { - fs.unlinkSync(tmp); - } catch { - // The write may have failed before the temporary file was created. - } - console.warn(`[pxpipe] could not persist model scope to ${file}: ${(e as Error).message}`); - } -} - function parseCli(argv: string[]): RuntimeConfig { // Only flags accepted are --help and --version. Anything else is an // error — there is exactly ONE way to run pxpipe and the dashboard @@ -170,15 +150,26 @@ function parseCli(argv: string[]): RuntimeConfig { port: Number(process.env.PORT ?? 47821), // Loopback by default; opt into all-interfaces exposure explicitly via HOST. host: process.env.HOST?.trim() || '127.0.0.1', - upstream: process.env.ANTHROPIC_UPSTREAM ?? sharedUpstream ?? 'https://api.anthropic.com', - openAIUpstream: process.env.OPENAI_UPSTREAM ?? sharedUpstream ?? 'https://api.openai.com', + // Env-driven upstream URLs are .trim()-ed here (and again in + // resolveUpstreams) to defend against whitespace sneaking in from the + // surrounding shell. A stray space in OPENAI_UPSTREAM / + // ANTHROPIC_UPSTREAM / PXPIPE_GATEWAY_BASE_URL — typically from a + // cmd.exe `set VAR=...` line, a copy-paste with a trailing space, or a + // shell-quoting bug in a launcher script — would otherwise build URLs + // like "https://api.openai.com /v1/...". fetch() then throws + // "Failed to parse URL" with no actionable log line and the operator + // is left guessing. Trimming at the env boundary keeps the failure + // mode loud (the proxy still returns 401/502 from upstream) instead + // of silent. + upstream: (process.env.ANTHROPIC_UPSTREAM ?? sharedUpstream ?? 'https://api.anthropic.com').trim(), + openAIUpstream: (process.env.OPENAI_UPSTREAM ?? sharedUpstream ?? 'https://api.openai.com').trim(), openAIApiKey: process.env.OPENAI_API_KEY, cloudflareUpstream, cloudflareApiKey: cfToken, openAIModels: parseModels(process.env.OPENAI_MODELS), cloudflareModels: parseModels(process.env.CLOUDFLARE_MODELS), provider: parseProvider(process.env.PXPIPE_PROVIDER), - gatewayBaseUrl: process.env.PXPIPE_GATEWAY_BASE_URL, + gatewayBaseUrl: process.env.PXPIPE_GATEWAY_BASE_URL?.trim(), gatewayHeaders: parseGatewayHeaders(process.env.PXPIPE_GATEWAY_HEADERS), eventsFile: process.env.PXPIPE_LOG ?? @@ -186,6 +177,11 @@ function parseCli(argv: string[]): RuntimeConfig { // Off by default: either side of a 4xx may hold prompts or secrets. // Opt in for debugging only. (issue #69) captureErrorReqBody: process.env.PXPIPE_DEBUG_CAPTURE_4XX === '1', + trustedDashboardProxy: process.env.PXPIPE_TRUSTED_DASHBOARD_PROXY?.trim() || undefined, + dashboardOrigins: process.env.PXPIPE_DASHBOARD_ORIGINS + ?.split(',').map((s) => s.trim()).filter(Boolean), + allowNonLoopbackCredentials: + process.env.PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS === '1', }; } @@ -243,11 +239,33 @@ Environment: PXPIPE_CONFIG JSON config path (default ~/.config/pxpipe/config.json) supports {"models": [...]} or {"models": "off"} PXPIPE_LOG JSONL events path (default ~/.pxpipe/events.jsonl) + PXPIPE_LOG_MAX_MB rotate the events file once it exceeds this size + (default 100) + PXPIPE_LOG_KEEP number of rotated generations to retain on disk, + .1 .. .N (default 1) + PXPIPE_LOG_COMPRESS set to 1 to gzip each rotated generation as + .1.gz, .2.gz, ... + PXPIPE_LOG_FSYNC_MS periodic fsync interval in ms; 0 = fsync only on + shutdown (default 0) PXPIPE_DUMP_DIR debug: write every rendered PNG here (what the model sees); off unless set. Compress arm only. PXPIPE_DEBUG_CAPTURE_4XX debug: set to 1 to persist full 4xx request and upstream error bodies (prompts + any secrets in context) to disk. Off by default. + PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS + set to 1 only when an external access boundary protects + a non-loopback HOST (the bundled loopback-only Compose + publication sets this explicitly) + PXPIPE_TRUSTED_DASHBOARD_PROXY + exact source address of a trusted reverse proxy in + front of the dashboard; such a request is allowed + without a loopback Host. Host validation stays + mandatory for every other source. + PXPIPE_DASHBOARD_ORIGINS + comma-separated public origins accepted for dashboard + mutations behind that proxy (e.g. + https://pxpipe.example.com). Cross-site writes are + still rejected via Sec-Fetch-Site. Use with Claude Code: ANTHROPIC_BASE_URL=http://127.0.0.1:47821 claude @@ -430,21 +448,71 @@ function isLoopbackHostname(hostname: string): boolean { || (isIP(host) === 4 && host.split('.')[0] === '127'); } +function isAllowedDashboardClient( + address: string | undefined, + hostname: string, + trustedDashboardProxy: string | undefined, +): boolean { + // A request from the explicitly configured trusted proxy has already passed + // the operator's access boundary; requiring a loopback Host on top of that + // only forces the proxy to forge one. Host validation stays mandatory for + // every other source. + if (trustedDashboardProxy && address === trustedDashboardProxy) return true; + return isLoopbackHostname(hostname) && isLoopbackAddress(address); +} + +function injectedCredentialSources(opts: RuntimeConfig): string[] { + const sources: string[] = []; + if (opts.openAIApiKey?.trim()) sources.push('OPENAI_API_KEY'); + if (opts.cloudflareApiKey?.trim()) sources.push('CLOUDFLARE_API_TOKEN'); + // Gateway headers are operator-controlled and their names are arbitrary; + // treating only familiar auth names as sensitive would let a custom header + // such as `x-upstream-credential` (or a neutral-looking quota key) bypass + // the non-loopback guard. + if (Object.keys(opts.gatewayHeaders ?? {}).length > 0) { + sources.push('PXPIPE_GATEWAY_HEADERS'); + } + return sources; +} + +function assertCredentialInjectionIsLocal(opts: RuntimeConfig): void { + // Bind hosts may be DNS names. Do not treat an arbitrary hostname beginning + // with "127." as loopback: only validated 127/8 IP literals, ::1, and the + // exact localhost name are local. + const loopbackBind = isLoopbackHostname(opts.host); + const sources = injectedCredentialSources(opts); + if (loopbackBind || sources.length === 0 || opts.allowNonLoopbackCredentials) return; + throw new Error( + `refusing non-loopback HOST=${opts.host} with server-owned credentials from ` + + `${sources.join(', ')}; bind HOST to loopback, or set ` + + 'PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS=1 only when a trusted external access boundary is in place', + ); +} + function isDashboardMutation(route: DashboardRoute, method: string): boolean { return method === 'POST' && ( route.kind === 'api-compression' - || (route.kind === 'fragment' && (route.name === 'toggle' || route.name === 'models')) + || (route.kind === 'fragment' && ( + route.name === 'toggle' || route.name === 'models' || route.name === 'models/reset' + )) ); } /** Reject browser cross-site writes to the unauthenticated loopback dashboard. */ -function isSameOriginDashboardRequest(req: IncomingMessage, url: URL): boolean { +function isSameOriginDashboardRequest( + req: IncomingMessage, + url: URL, + allowedOrigins: readonly string[] = [], +): boolean { const fetchSite = req.headers['sec-fetch-site']; if (fetchSite === 'cross-site') return false; const origin = req.headers.origin; if (origin === undefined) return true; // local CLI/curl clients do not send Origin try { - return new URL(origin).origin === url.origin; + const reqOrigin = new URL(origin).origin; + // Same origin, or an operator-declared public origin for a reverse-proxied + // dashboard (PXPIPE_DASHBOARD_ORIGINS). + return reqOrigin === url.origin || allowedOrigins.includes(reqOrigin); } catch { return false; } @@ -555,6 +623,10 @@ async function dispatchDashboard( else if (model) dashboard.handleModelsToggle(model, on); return dashboard.serveFragment('models', url, port); } + if (route.name === 'models/reset' && method === 'POST') { + dashboard.handleModelsReset(); + return dashboard.serveFragment('models', url, port); + } if (method !== 'GET') return undefined; return dashboard.serveFragment(route.name, url, port); } @@ -588,21 +660,74 @@ async function dispatchDashboard( * Node-only — uses node:fs. The Worker host uses tracker.JsonLogTracker with * console.log instead (Cloudflare ingests that as Workers Logs). * - * Rotation: when the current file exceeds MAX_FILE_BYTES (100 MB by default), - * it's renamed to `.1` (overwriting any previous .1) and a fresh file - * is opened. Keeps one generation of history; for longer retention pipe - * the file off-host yourself. + * Rotation: when the current file exceeds the configured max (default 100 MB + * via PXPIPE_LOG_MAX_MB), the file is rotated out. By default a single .1 + * generation is kept (PXPIPE_LOG_KEEP=1); raise it to retain a chain .1, .2, + * ... up to N. Set PXPIPE_LOG_COMPRESS=1 to gzip each rotated generation + * (.1.gz, .2.gz, ...) so long-term retention costs less. + * + * Durability: by default fsync runs only on shutdown. Set + * PXPIPE_LOG_FSYNC_MS to a positive interval (e.g. 500) to fsync on a + * unref'd timer so a hard kill (SIGKILL / Windows TerminateProcess / power + * loss) loses at most that many ms of writes. Cost is one fsync per tick. * * Failures here NEVER propagate — the proxy must keep serving requests even * if the disk is full or the path is unwritable. */ class FileTracker implements Tracker { + // Defaults preserve the original hardcoded behavior. Override via + // FileTracker.configure() from main() with env-driven values. Static so + // existing tests that construct FileTracker directly keep working. + private static config: { + maxBytes: number; + keep: number; + compress: boolean; + fsyncMs: number; + } = { + maxBytes: 100 * 1024 * 1024, + keep: 1, + compress: false, + fsyncMs: 0, + }; + + /** Apply env-driven defaults from main(). Safe to call once at startup. */ + static configure(env: { + maxMb?: number; + keep?: number; + compress?: boolean; + fsyncMs?: number; + }): void { + if (typeof env.maxMb === 'number' && env.maxMb > 0) { + FileTracker.config.maxBytes = Math.floor(env.maxMb * 1024 * 1024); + } + if (typeof env.keep === 'number' && env.keep >= 1) { + FileTracker.config.keep = Math.floor(env.keep); + } + if (typeof env.compress === 'boolean') { + FileTracker.config.compress = env.compress; + } + if (typeof env.fsyncMs === 'number' && env.fsyncMs >= 0) { + FileTracker.config.fsyncMs = Math.floor(env.fsyncMs); + } + } + private fd: number | null = null; private bytesWritten = 0; private brokenLogged = false; - private static readonly MAX_FILE_BYTES = 100 * 1024 * 1024; - - constructor(private readonly filePath: string) {} + private flushTimer: NodeJS.Timeout | null = null; + private readonly writerLock: EventsFileLock; + + constructor(private readonly filePath: string) { + // Fail startup explicitly rather than allow two append fds whose rotations + // and prune coordination could silently split or lose the log. + this.writerLock = acquireEventsFileLock(filePath, 'writer'); + if (FileTracker.config.fsyncMs > 0) { + // unref() so the timer never blocks process exit; flush() on a null fd + // is a no-op so the tick is safe even after close(). + this.flushTimer = setInterval(() => this.flush(), FileTracker.config.fsyncMs); + this.flushTimer.unref(); + } + } private ensureOpen(): boolean { if (this.fd != null) return true; @@ -644,12 +769,55 @@ class FileTracker implements Tracker { } this.fd = null; } - try { - fs.renameSync(this.filePath, this.filePath + '.1'); - } catch { - /* if rename fails (e.g. .1 locked) we'll just keep growing — better - than dropping events */ + + const { keep, compress } = FileTracker.config; + const ext = (n: number) => `.${n}${compress ? '.gz' : ''}`; + + // Cascade: shift .N -> .(N+1) until the cap. The oldest is dropped to + // make room. With keep=1 this matches the original single-rename behavior. + for (let n = keep; n >= 1; n--) { + const oldName = this.filePath + ext(n); + if (n === keep) { + try { + fs.unlinkSync(oldName); + } catch { + /* may not exist on first rotation */ + } + continue; + } + const newName = this.filePath + ext(n + 1); + try { + fs.renameSync(oldName, newName); + } catch { + /* oldName may not exist; ignore */ + } + } + + // Move current -> .1 (or .1.gz). On a compression failure fall back to a + // plain rename so we never drop events on disk. + const target = this.filePath + ext(1); + if (compress) { + try { + const data = fs.readFileSync(this.filePath); + const gz = zlib.gzipSync(data, { level: zlib.constants.Z_BEST_SPEED }); + fs.writeFileSync(target, gz, { mode: 0o600 }); + fs.unlinkSync(this.filePath); + } catch { + try { + fs.renameSync(this.filePath, this.filePath + '.1'); + } catch { + /* keep growing */ + } + } + } else { + try { + fs.renameSync(this.filePath, this.filePath + '.1'); + } catch { + /* if rename fails (e.g. .1 locked) we'll just keep growing — better + than dropping events */ + } } + this.bytesWritten = 0; } @@ -660,7 +828,7 @@ class FileTracker implements Tracker { const buf = Buffer.from(line, 'utf8'); fs.writeSync(this.fd!, buf); this.bytesWritten += buf.length; - if (this.bytesWritten > FileTracker.MAX_FILE_BYTES) this.rotate(); + if (this.bytesWritten > FileTracker.config.maxBytes) this.rotate(); } catch (err) { if (!this.brokenLogged) { console.error( @@ -682,6 +850,10 @@ class FileTracker implements Tracker { } close(): void { + if (this.flushTimer != null) { + clearInterval(this.flushTimer); + this.flushTimer = null; + } if (this.fd != null) { try { fs.fsyncSync(this.fd); @@ -695,6 +867,7 @@ class FileTracker implements Tracker { } this.fd = null; } + this.writerLock.release(); } } @@ -1049,6 +1222,11 @@ async function main(): Promise { createWarpRuntime({ port: opts.port }).launch(warpCommand); return; } + // A non-loopback unauthenticated proxy plus a server-owned upstream key lets + // any reachable client spend that credential. Docker binds 0.0.0.0 only + // inside its network namespace and explicitly acknowledges the host-side + // loopback publication in compose.yml; other deployments must fail closed. + assertCredentialInjectionIsLocal(opts); // A/B harness passthrough switch (see the `transform` callback below). const forcePassthrough = /^(1|true|yes|on)$/i.test(process.env.PXPIPE_DISABLE ?? ''); if (forcePassthrough) { @@ -1084,6 +1262,15 @@ async function main(): Promise { // rows) showed 5 mode flips ever and losses at 0.8% of wins — all // one-time cache-create amortization — so closing the loop would not // change decisions. Re-run that reconciliation before wiring one in. + // FileTracker knobs read straight from env. Defaults inside the class + // match the previous hardcoded behavior (100 MB, keep 1, no compression, + // fsync only on shutdown) so this is a no-op when the env vars are unset. + FileTracker.configure({ + maxMb: Number(process.env.PXPIPE_LOG_MAX_MB) || undefined, + keep: Number(process.env.PXPIPE_LOG_KEEP) || undefined, + compress: process.env.PXPIPE_LOG_COMPRESS === '1', + fsyncMs: Number(process.env.PXPIPE_LOG_FSYNC_MS) || undefined, + }); const tracker: Tracker = new FileTracker(opts.eventsFile); // Sidecar dir for oversized 4xx request-body samples. Lives next to the @@ -1095,14 +1282,29 @@ async function main(): Promise { // served via the route interception in front of the proxy handler. The // SessionsPaths handle lets the dashboard surface session/disk/stats data // without reaching back into module-scope globals. + const codexUsage = new CodexUsageIndex(); + codexUsage.start(); + // A dashboard choice is an explicit operator override: it takes precedence + // over PXPIPE_MODELS and survives launcher restarts. Reset deletes only this + // sidecar, preserving any deliberately configured PXPIPE_CONFIG defaults. + const scopeFile = modelScopeFile(opts.eventsFile); + const persistedScope = loadPersistedModelScope(scopeFile); + if (persistedScope !== null) setAllowedModelBases(persistedScope); const dashboard = new DashboardState( { eventsFile: opts.eventsFile, sidecarDir: bodySidecarDir, }, undefined, - persistModelBasesToConfig, + (bases) => { + if (bases === null) clearPersistedModelScope(scopeFile); + else savePersistedModelScope(scopeFile, bases); + }, + () => codexUsage.snapshot(), ); + // Rolling counter of recent /backend-api/codex/* traffic, feeding the + // evidence-driven Codex-upstream health check (see /healthz below). + const healthCounters = new HealthCounters(); // Seed the "recent requests" table from the JSONL log so a process restart // doesn't reset what you can see in the UI. Best-effort; ignored on error. await dashboard.replay(opts.eventsFile).catch(() => {}); @@ -1135,6 +1337,13 @@ async function main(): Promise { return {}; }, onRequest: async (e) => { + // Feed the health counter first — cheap and must never be skipped by an + // early return further down. Best-effort; never throw into onRequest. + try { + healthCounters.record(e.path, e.status, Date.now()); + } catch { + /* ignore */ + } // Feed the dashboard BEFORE tracker.emit — toTrackEvent strips // info.firstImagePng, so capturing has to happen on the raw event. dashboard.update(e); @@ -1224,14 +1433,34 @@ async function main(): Promise { // Local dashboard routes — handled BEFORE the proxy so they never hit // api.anthropic.com (which would 404 them). const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`); + // Health endpoints — host-level (compose config + counters + dashboard), + // handled before the dashboard router. A failure in the diagnostics + // themselves is unhealthy: probes must not silently fail open. + if (url.pathname === '/healthz' || url.pathname === '/api/health.json') { + if (!isAllowedDashboardClient(req.socket.remoteAddress, url.hostname, opts.trustedDashboardProxy)) { + await writeWebResponse(new Response('health endpoint is loopback-only', { status: 403 }), res); + return; + } + const report = buildHealthReport(config, dashboard, healthCounters, Date.now()); + const payload = JSON.stringify({ + ok: report.ok, + findings: report.findings, + state: report.state, + }, null, 2); + const status = url.pathname === '/healthz' ? report.httpStatus : 200; + res.statusCode = status; + res.setHeader('content-type', 'application/json'); + res.end(payload); + return; + } const route = dashboardPath(url.pathname); if (route) { - if (!isLoopbackAddress(req.socket.remoteAddress) || !isLoopbackHostname(url.hostname)) { + if (!isAllowedDashboardClient(req.socket.remoteAddress, url.hostname, opts.trustedDashboardProxy)) { await writeWebResponse(new Response('dashboard is loopback-only', { status: 403 }), res); return; } if (isDashboardMutation(route, req.method ?? 'GET') - && !isSameOriginDashboardRequest(req, url)) { + && !isSameOriginDashboardRequest(req, url, opts.dashboardOrigins)) { await writeWebResponse(new Response('cross-origin dashboard mutation denied', { status: 403 }), res); return; } @@ -1253,6 +1482,18 @@ async function main(): Promise { }); }); + server.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'EADDRINUSE') { + console.error(`[pxpipe] ⛔ port ${opts.port} is already in use — another pxpipe (or process) is bound to ${opts.host}:${opts.port}.`); + console.error(`[pxpipe] Find it: Get-NetTCPConnection -LocalPort ${opts.port} -State Listen | Select-Object OwningProcess`); + console.error(`[pxpipe] Stop it: Stop-Process -Id -Force`); + console.error(`[pxpipe] Then re-run the launcher.`); + } else { + console.error(`[pxpipe] server error: ${err.message}`); + } + process.exit(1); + }); + // IPv6 literals need bracket notation to form a valid URL (http://[::1]:47821). const displayHost = opts.host.includes(':') ? `[${opts.host}]` : opts.host; const isLoopbackHost = @@ -1282,11 +1523,23 @@ async function main(): Promise { if (!isLoopbackHost) { console.warn( `[pxpipe] bound to ${opts.host}; proxy API is reachable off-host, ` + - `but dashboard routes remain loopback-only.`, + `but dashboard and health routes remain loopback-only.`, ); } announce(); console.log(`[pxpipe] dashboard → http://127.0.0.1:${opts.port}/`); + try { + const findings = evaluateHealth(buildHealthState(config, dashboard, healthCounters, Date.now())); + for (const f of findings) { + if (f.severity !== 'error' && f.severity !== 'warn') continue; + const mark = f.severity === 'error' ? '⛔' : '⚠️'; + console.warn(`[pxpipe] ${mark} ${f.title}`); + console.warn(`[pxpipe] ${f.detail}`); + if (f.remediation) console.warn(`[pxpipe] fix: ${f.remediation.durableHint}`); + } + } catch { + /* never block startup on a health-print failure */ + } }); // server.close() only stops accepting new connections and waits for open @@ -1303,6 +1556,7 @@ async function main(): Promise { } shuttingDown = true; console.log(`[pxpipe] ${sig} — shutting down`); + codexUsage.stop(); // Flush+close the tracker so we don't drop the last few events on exit. if (tracker instanceof FileTracker) tracker.close(); server.close(() => process.exit(0)); diff --git a/src/sessions.ts b/src/sessions.ts index 9e798ccc6..b9a7d7be3 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -2,14 +2,14 @@ * Shared data layer for the dashboard's session views. Node-only (filesystem * I/O) — never imported from `src/core/`. * - * ## Why we group by first_user_sha8 (Path B) + * ## How synthetic session cohorts are formed * * Every TrackEvent carries `first_user_sha8` (see src/core/tracker.ts), an - * sha256 prefix of the conversation's first user message. Within a single - * Claude Code session that hash is stable across every turn; across two - * different sessions it is virtually never the same. That makes it a - * better-than-good-enough session key without coupling pxpipe to Claude - * Code's internal file layout for *correctness*. + * sha256 prefix of the conversation's first user message. It is stable across + * turns, but it is not a real conversation ID: separate projects can begin + * with the same template. Collisions across known working directories are + * qualified with a project hash; identical prompts in the same project remain + * inherently ambiguous until a client supplies a genuine session identifier. * * We *do* read `~/.claude/projects/` opportunistically (see `claudeCodeMap`) * to enrich the dashboard with real Claude Code session IDs + project @@ -34,11 +34,17 @@ import { computeBaselineInputEffWithCacheTier, deriveBaselineWarmth, } from './core/baseline.js'; +import { + computeOpenAIActualInputEff, + computeOpenAIBaselineInputEff, +} from './core/openai-savings.js'; +import { acquireEventsFileLock } from './events-lock.js'; // ---- Types ----------------------------------------------------------------- export interface SessionSummary { - /** The synthetic session ID = first_user_sha8 (or '' if missing). */ + /** Synthetic first-user fingerprint, project-qualified on a known collision + * (or '' if missing). It is not a provider conversation ID. */ id: string; /** Working directory of the first event in the session, if any. */ project: string | undefined; @@ -52,11 +58,11 @@ export interface SessionSummary { * useful only as a rough "we shaved X kB off the wire" callout. Not * load-bearing math; the real number is `tokensSavedEst`. */ charsSaved: number; - /** Real input-side tokens saved: sum of `baseline_tokens − (input + - * cache_create×1.25 + cache_read×0.10)` across events that carry both - * a /v1/messages/count_tokens probe and an upstream usage block. - * Events missing either side contribute to requestCount but not here. - * No estimation — can go negative when a compression net-lost. */ + /** Provider-specific input-side tokens saved. Anthropic uses the measured + * cache-aware count_tokens counterfactual; OpenAI uses its persisted + * tokenizer/vision delta and automatic-cache rate; Google uses its provider + * countTokens result (or the same persisted local fallback as live stats). + * Events without a usable provider baseline contribute only to requestCount. */ tokensSavedEst: number; /** Sum of cache_read_input_tokens — actual prompt-cache hits. */ cacheReadTokens: number; @@ -104,29 +110,92 @@ export async function* readEvents( const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); for await (const line of rl) { if (!line.trim()) continue; - let ev: TrackEvent; + let parsed: unknown; try { - ev = JSON.parse(line) as TrackEvent; + parsed = JSON.parse(line); } catch { continue; } + if (!isTrackEvent(parsed)) continue; // +1 for the newline FileTracker writes after each row. - yield { ev, rawBytes: Buffer.byteLength(line, 'utf8') + 1 }; + yield { ev: parsed, rawBytes: Buffer.byteLength(line, 'utf8') + 1 }; } } +function isTrackEvent(value: unknown): value is TrackEvent { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const ev = value as Record; + if (typeof ev.ts !== 'string' || !Number.isFinite(Date.parse(ev.ts))) return false; + if (typeof ev.method !== 'string' || typeof ev.path !== 'string') return false; + if (typeof ev.status !== 'number' || !Number.isFinite(ev.status)) return false; + if (typeof ev.duration_ms !== 'number' || !Number.isFinite(ev.duration_ms) || ev.duration_ms < 0) { + return false; + } + const optionalStrings = [ + 'model', 'cwd', 'first_user_sha8', 'system_sha8', 'req_body_sample_path', + ]; + for (const key of optionalStrings) { + if (ev[key] !== undefined && typeof ev[key] !== 'string') return false; + } + if ( + ev.accounting_provider !== undefined + && ev.accounting_provider !== 'anthropic' + && ev.accounting_provider !== 'openai' + && ev.accounting_provider !== 'google' + ) return false; + if (ev.compressed !== undefined && typeof ev.compressed !== 'boolean') return false; + if ( + ev.baseline_probe_status !== undefined + && ev.baseline_probe_status !== 'ok' + && ev.baseline_probe_status !== 'partial' + && ev.baseline_probe_status !== 'failed' + ) return false; + const optionalNonNegativeNumbers = [ + 'input_tokens', 'output_tokens', 'cache_create_tokens', 'cache_read_tokens', + 'cached_tokens', 'cache_create_5m_tokens', 'cache_create_1h_tokens', + 'baseline_tokens', 'baseline_cacheable_tokens', 'image_tokens', + 'baseline_imaged_tokens', 'native_injected_tokens', + ]; + for (const key of optionalNonNegativeNumbers) { + const n = ev[key]; + if (n !== undefined && (typeof n !== 'number' || !Number.isFinite(n) || n < 0)) return false; + } + return true; +} + // ---- Aggregation ----------------------------------------------------------- export const UNKNOWN_SESSION = ''; -function sessionIdOf(ev: TrackEvent): string { +function baseSessionIdOf(ev: TrackEvent): string { return ev.first_user_sha8 ?? UNKNOWN_SESSION; } +/** A first-message fingerprint is not a conversation id: two projects can + * legitimately start with the same prompt. Keep the historical bare id for + * the first project (back-compatible API), and qualify colliding projects. */ +function projectQualifiedSessionId(base: string, project: string): string { + const suffix = crypto.createHash('sha256').update(project, 'utf8').digest('hex').slice(0, 8); + return `${base}@${suffix}`; +} + +function sessionIdOf( + ev: TrackEvent, + primaryProjectByBase: ReadonlyMap, +): string { + const base = baseSessionIdOf(ev); + const primaryProject = primaryProjectByBase.get(base); + if (!ev.cwd || !primaryProject || ev.cwd === primaryProject) return base; + return projectQualifiedSessionId(base, ev.cwd); +} + export interface AggregateResult { sessions: Map; /** sessionId -> set of absolute sidecar paths referenced by its events. */ sidecarsBySession: Map>; + /** Bare first-message id -> project assigned to its back-compatible primary + * session. Used by prune to resolve rows exactly as aggregation did. */ + primaryProjectByBase: Map; } /** Build a map of sessionId -> SessionSummary by scanning every event. Also @@ -136,6 +205,7 @@ export async function aggregateSessions( ): Promise { const sessions = new Map(); const sidecarsBySession = new Map>(); + const primaryProjectByBase = new Map(); // Per-session prior prefix sizes for the cache-aware text counterfactual. // Warm/cold comes only from server-observed cache_read; this map only refines // reused/grown splitting after cr>0 has proved warmth. Kept out of @@ -148,7 +218,10 @@ export async function aggregateSessions( const sidecarSizes = sidecarFileSizes(paths.sidecarDir); for await (const { ev, rawBytes } of readEvents(paths.eventsFile)) { - const id = sessionIdOf(ev); + const baseId = baseSessionIdOf(ev); + if (!primaryProjectByBase.has(baseId)) primaryProjectByBase.set(baseId, ev.cwd); + else if (!primaryProjectByBase.get(baseId) && ev.cwd) primaryProjectByBase.set(baseId, ev.cwd); + const id = sessionIdOf(ev, primaryProjectByBase); let s = sessions.get(id); if (!s) { s = { @@ -178,13 +251,53 @@ export async function aggregateSessions( // the actual request: cr>0 means warm for both, cr===0 means cold for both. // Events missing either probe stay out of the rollup — no estimation. const inp = ev.input_tokens ?? 0; + const out = ev.output_tokens ?? 0; const cc = ev.cache_create_tokens ?? 0; const cc5m = ev.cache_create_5m_tokens; const cc1h = ev.cache_create_1h_tokens ?? 0; const cr = ev.cache_read_tokens ?? 0; - const haveUsage = inp > 0 || cc > 0 || cr > 0; + const provider = ev.accounting_provider + ?? (ev.path.includes('google-ai-studio') || ev.path.includes('generateContent') + ? 'google' + : ev.path.includes('responses') || ev.path.includes('chat/completions') + ? 'openai' + : 'anthropic'); + const haveUsage = inp > 0 || out > 0 || cc > 0 || cr > 0; const baseline = ev.baseline_tokens; - if ( + if (provider === 'openai' && haveUsage && ev.compressed === true) { + const baselineImaged = ev.baseline_imaged_tokens ?? 0; + if (baselineImaged > 0) { + const actualEff = computeOpenAIActualInputEff(inp, ev.cached_tokens ?? 0, ev.model); + const baselineEff = computeOpenAIBaselineInputEff( + inp, + ev.cached_tokens ?? 0, + ev.image_tokens ?? 0, + baselineImaged, + ev.model, + ev.native_injected_tokens ?? 0, + ); + const tokensSaved = baselineEff - actualEff; + s.tokensSavedEst += Math.round(tokensSaved); + s.charsSaved += Math.round(tokensSaved * 4); + } + } else if (provider === 'google' && haveUsage && ev.compressed === true) { + const measuredBaseline = ev.baseline_probe_status === 'ok' && (baseline ?? 0) > 0 + ? baseline! + : 0; + const estimatedBaseline = (ev.image_tokens ?? 0) > 0 && (ev.baseline_imaged_tokens ?? 0) > 0 + ? Math.max( + 0, + inp - (ev.image_tokens ?? 0) - (ev.native_injected_tokens ?? 0) + + (ev.baseline_imaged_tokens ?? 0), + ) + : 0; + const googleBaseline = measuredBaseline || estimatedBaseline; + if (googleBaseline > 0) { + const tokensSaved = googleBaseline - inp; + s.tokensSavedEst += Math.round(tokensSaved); + s.charsSaved += Math.round(tokensSaved * 4); + } + } else if (provider === 'anthropic' && typeof baseline === 'number' && baseline > 0 && haveUsage && @@ -224,7 +337,7 @@ export async function aggregateSessions( } // Record this completed row's prefix size for future cr>0 split estimates. // Carry prior cacheable when this row had no probe. - if (haveUsage) { + if (provider === 'anthropic' && haveUsage) { const completionSec = Date.parse(ev.ts) / 1000; const cacheable = ev.baseline_cacheable_tokens ?? 0; const prefixSha = ev.system_sha8; @@ -235,22 +348,26 @@ export async function aggregateSessions( prefixSha: prefixSha ?? prev?.prefixSha, }); } - if (typeof ev.cache_read_tokens === 'number') { - s.cacheReadTokens += ev.cache_read_tokens; + const cacheRead = provider === 'anthropic' ? ev.cache_read_tokens : ev.cached_tokens; + if (typeof cacheRead === 'number') { + s.cacheReadTokens += cacheRead; } if (ev.req_body_sample_path) { + const sidecarPath = confinedSidecarPath(paths.sidecarDir, ev.req_body_sample_path); + if (!sidecarPath) continue; let set = sidecarsBySession.get(id); if (!set) { set = new Set(); sidecarsBySession.set(id, set); } - set.add(ev.req_body_sample_path); - const size = sidecarSizes.get(ev.req_body_sample_path); - if (typeof size === 'number') s.sidecarBytes += size; + const firstReference = !set.has(sidecarPath); + set.add(sidecarPath); + const size = sidecarSizes.get(sidecarPath); + if (firstReference && typeof size === 'number') s.sidecarBytes += size; } } - return { sessions, sidecarsBySession }; + return { sessions, sidecarsBySession, primaryProjectByBase }; } // ---- list / filter -------------------------------------------------------- @@ -296,7 +413,7 @@ function sidecarFileSizes(dir: string): Map { return out; } for (const name of entries) { - const full = path.join(dir, name); + const full = path.resolve(dir, name); try { const st = fs.statSync(full); if (st.isFile()) out.set(full, st.size); @@ -307,6 +424,16 @@ function sidecarFileSizes(dir: string): Map { return out; } +/** FileTracker writes sidecars directly inside this directory. Reject nested, + * sibling and absolute-escape paths before they can reach stat/unlink. */ +function confinedSidecarPath(dir: string, candidate: string): string | undefined { + const root = path.resolve(dir); + const resolved = path.resolve(candidate); + if (path.dirname(resolved) !== root) return undefined; + if (path.basename(resolved) === '') return undefined; + return resolved; +} + // ---- prune ----------------------------------------------------------------- export interface PruneOptions { @@ -381,17 +508,18 @@ export function selectSessionsToRemove( * delete the matching 4xx-body sidecars. Atomic: writes to a sibling `.tmp` * file with fsync, then renames over the original. * - * Concurrency note: if the live proxy appends during prune, those new lines - * will be lost (the proxy holds an fd to the pre-rename inode and keeps - * writing to it). For a single-user dev tool that's an acceptable tradeoff; - * the dashboard's confirm dialog warns the user before the destructive op. + * Concurrency note: prune refuses while FileTracker's writer lock is live. + * Rewriting an inode that an append fd still owns would otherwise lose every + * line appended after the rename. */ export async function prune( paths: SessionsPaths, opts: PruneOptions, now: Date = new Date(), ): Promise { - const { sessions, sidecarsBySession } = await aggregateSessions(paths); + const pruneLock = acquireEventsFileLock(paths.eventsFile, 'prune'); + try { + const { sessions, sidecarsBySession, primaryProjectByBase } = await aggregateSessions(paths); const toRemove = selectSessionsToRemove(sessions, opts, now); let jsonlBytesFreed = 0; @@ -408,13 +536,21 @@ export async function prune( // Collect sidecar paths up for deletion (and their on-disk sizes). const sidecarsToDelete: { path: string; size: number }[] = []; + const sidecarsStillReferenced = new Set(); + for (const [id, refs] of sidecarsBySession) { + if (toRemove.has(id)) continue; + for (const p of refs) sidecarsStillReferenced.add(p); + } for (const id of toRemove) { const set = sidecarsBySession.get(id); if (!set) continue; for (const p of set) { + const safePath = confinedSidecarPath(paths.sidecarDir, p); + if (!safePath || sidecarsStillReferenced.has(safePath)) continue; try { - const st = fs.statSync(p); - sidecarsToDelete.push({ path: p, size: st.size }); + const st = fs.statSync(safePath); + if (!st.isFile()) continue; + sidecarsToDelete.push({ path: safePath, size: st.size }); } catch { /* already gone — fine */ } @@ -440,7 +576,7 @@ export async function prune( // Atomic rewrite: stream the original through a filter into events.jsonl.tmp, // fsync, then rename. Any partial state on crash leaves the original intact. - await rewriteEventsFile(paths.eventsFile, toRemove); + await rewriteEventsFile(paths.eventsFile, toRemove, primaryProjectByBase); for (const { path: p } of sidecarsToDelete) { try { @@ -452,11 +588,15 @@ export async function prune( report.applied = true; return report; + } finally { + pruneLock.release(); + } } async function rewriteEventsFile( eventsFile: string, toRemove: Set, + primaryProjectByBase: ReadonlyMap, ): Promise { if (!fs.existsSync(eventsFile)) return; const tmp = eventsFile + '.tmp'; @@ -475,7 +615,7 @@ async function rewriteEventsFile( fs.writeSync(outFd, line + '\n'); continue; } - if (toRemove.has(sessionIdOf(ev))) continue; + if (toRemove.has(sessionIdOf(ev, primaryProjectByBase))) continue; fs.writeSync(outFd, line + '\n'); } fs.fsyncSync(outFd); diff --git a/tests/baseline.test.ts b/tests/baseline.test.ts index fce6481a2..51edd2e36 100644 --- a/tests/baseline.test.ts +++ b/tests/baseline.test.ts @@ -6,6 +6,9 @@ import { computeActualInputEffWithCacheTier, deriveBaselineWarmth, CACHE_CREATE_RATE, + CACHE_CREATE_1H_RATE, + CACHE_CREATE_5M_RATE, + cacheCreateUnknownTokens, CACHE_READ_RATE, CACHE_TTL_SEC, } from '../src/core/baseline.js'; @@ -100,6 +103,21 @@ describe('computeBaselineInputEff (warmth-aware)', () => { }); }); +describe('computeActualInputEff (server cache tiers)', () => { + it('uses the server-reported 5m/1h split instead of flattening all creates to 1.25x', () => { + expect(computeActualInputEff(100, 1_000, 200, { + fiveMinuteTokens: 900, + oneHourTokens: 100, + })).toBe(100 + 900 * CACHE_CREATE_5M_RATE + 100 * CACHE_CREATE_1H_RATE + 200 * CACHE_READ_RATE); + }); + + it('keeps legacy rows readable but reports their unverified cache tier', () => { + expect(computeActualInputEff(0, 100, 0)).toBe(100 * CACHE_CREATE_5M_RATE); + expect(cacheCreateUnknownTokens(100)).toBe(100); + expect(cacheCreateUnknownTokens(100, { fiveMinuteTokens: 100 })).toBe(0); + }); +}); + /** * deriveBaselineWarmth decides WHEN the text counterfactual was warm. The rule * is server-observed: text is warm iff the actual request reported cr>0. A prior diff --git a/tests/codex-usage.test.ts b/tests/codex-usage.test.ts new file mode 100644 index 000000000..acbf8d79d --- /dev/null +++ b/tests/codex-usage.test.ts @@ -0,0 +1,304 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + CodexUsageIndex, + groupCodexQuotaWindows, + summarizeCodexRolloutLines, +} from '../src/codex-usage.js'; + +const tempDirs: string[] = []; +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +function tokenLine( + total: Record, + last: Record | null, + timestamp = '2026-07-14T00:00:00.000Z', +): string { + return JSON.stringify({ + timestamp, + type: 'event_msg', + payload: { + type: 'token_count', + info: { + total_token_usage: total, + last_token_usage: last, + model_context_window: 200_000, + }, + rate_limits: { + limit_id: 'codex', + limit_name: 'Codex', + primary: { used_percent: 12.5, window_minutes: 300, resets_at: 1_789_000_000 }, + secondary: { used_percent: 33, window_minutes: 10_080, resets_at: 1_789_500_000 }, + }, + }, + }); +} + +describe('Codex rollout usage parser', () => { + it('uses last_token_usage and ignores duplicate cumulative snapshots', () => { + const total = { + input_tokens: 1000, cached_input_tokens: 800, output_tokens: 100, + reasoning_output_tokens: 60, total_tokens: 1100, + }; + const last = { + input_tokens: 400, cached_input_tokens: 300, output_tokens: 40, + reasoning_output_tokens: 20, total_tokens: 440, + }; + const first = tokenLine(total, last); + const refreshed = JSON.parse(tokenLine(total, last, '2026-07-14T00:05:00.000Z')); + refreshed.payload.rate_limits.primary.used_percent = 18.75; + refreshed.payload.rate_limits.primary.resets_at = 1_789_000_300; + const summary = summarizeCodexRolloutLines([first, JSON.stringify(refreshed)]); + expect(summary.usageSnapshots).toBe(1); + expect(summary.inputTokens).toBe(400); + expect(summary.cachedInputTokens).toBe(300); + expect(summary.outputTokens).toBe(40); + expect(summary.reasoningOutputTokens).toBe(20); + expect(summary.rateLimits?.primary?.usedPercent).toBe(18.75); + expect(summary.rateLimits?.primary?.resetsAt).toBe(1_789_000_300); + expect(summary.earliestEventAt).toBe('2026-07-14T00:00:00.000Z'); + expect(summary.latestEventAt).toBe('2026-07-14T00:05:00.000Z'); + }); + + it('groups equal-duration quota windows by provider limit id', () => { + const groups = groupCodexQuotaWindows([ + { limitId: 'codex', limitName: 'Codex', usedPercent: 12, windowMinutes: 300, resetsAt: 10, observedAt: '2026-07-14T00:00:00Z' }, + { limitId: 'other', limitName: 'Other', usedPercent: 72, windowMinutes: 300, resetsAt: 20, observedAt: '2026-07-14T00:01:00Z' }, + { limitId: 'codex', limitName: 'Codex', usedPercent: 34, windowMinutes: 10_080, resetsAt: 30, observedAt: '2026-07-14T00:02:00Z' }, + { limitId: 'codex', limitName: 'Codex', usedPercent: 18, windowMinutes: 300, resetsAt: 40, observedAt: '2026-07-14T00:03:00Z' }, + ]); + + expect(groups).toHaveLength(2); + expect(groups.find((group) => group.limitId === 'codex')?.windows).toEqual([ + expect.objectContaining({ windowMinutes: 300, usedPercent: 18 }), + expect.objectContaining({ windowMinutes: 10_080, usedPercent: 34 }), + ]); + expect(groups.find((group) => group.limitId === 'other')?.windows).toEqual([ + expect.objectContaining({ windowMinutes: 300, usedPercent: 72 }), + ]); + }); + + it('keeps unnamed provider limits separate by their reported names', () => { + const total = { input_tokens: 100, total_tokens: 100 }; + const first = JSON.parse(tokenLine(total, total, '2026-07-14T00:00:00.000Z')); + delete first.payload.rate_limits.limit_id; + first.payload.rate_limits.limit_name = 'Limit A'; + const second = JSON.parse(tokenLine(total, total, '2026-07-14T00:01:00.000Z')); + delete second.payload.rate_limits.limit_id; + second.payload.rate_limits.limit_name = 'Limit B'; + + const summary = summarizeCodexRolloutLines([JSON.stringify(first), JSON.stringify(second)]); + const groups = groupCodexQuotaWindows(summary.quotaWindows); + expect(groups.map((group) => group.limitName)).toEqual(['Limit A', 'Limit B']); + expect(groups.every((group) => group.windows.length === 2)).toBe(true); + }); + + it('falls back to cumulative deltas when last_token_usage is absent', () => { + const first = { input_tokens: 100, cached_input_tokens: 50, output_tokens: 10, reasoning_output_tokens: 4, total_tokens: 110 }; + const second = { input_tokens: 250, cached_input_tokens: 180, output_tokens: 30, reasoning_output_tokens: 9, total_tokens: 280 }; + const summary = summarizeCodexRolloutLines([tokenLine(first, null), tokenLine(second, null)]); + expect(summary.usageSnapshots).toBe(2); + expect(summary.inputTokens).toBe(250); + expect(summary.cachedInputTokens).toBe(180); + expect(summary.outputTokens).toBe(30); + expect(summary.totalTokens).toBe(280); + }); + + it('indexes only sessions whose official model_provider is pxpipe', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pxpipe-codex-usage-')); + tempDirs.push(root); + const nested = path.join(root, '2026', '07', '14'); + await fs.mkdir(nested, { recursive: true }); + const usage = { + input_tokens: 500, cached_input_tokens: 450, output_tokens: 25, + reasoning_output_tokens: 10, total_tokens: 525, + }; + await fs.writeFile(path.join(nested, 'rollout-pxpipe.jsonl'), [ + JSON.stringify({ type: 'session_meta', payload: { model_provider: 'pxpipe', session_id: 'one' } }), + tokenLine(usage, usage), + ].join('\n')); + await fs.writeFile(path.join(nested, 'rollout-direct.jsonl'), [ + JSON.stringify({ type: 'session_meta', payload: { model_provider: 'openai', session_id: 'two' } }), + tokenLine({ ...usage, input_tokens: 9999 }, { ...usage, input_tokens: 9999 }), + ].join('\n')); + + const index = new CodexUsageIndex(root); + await index.refresh(); + const snapshot = index.snapshot(); + expect(snapshot.loading).toBe(false); + expect(snapshot.sessionFiles).toBe(1); + expect(snapshot.usageSnapshots).toBe(1); + expect(snapshot.inputTokens).toBe(500); + expect(snapshot.cachedInputTokens).toBe(450); + expect(snapshot.rateLimits?.secondary?.windowMinutes).toBe(10_080); + expect(snapshot.earliestEventAt).toBe('2026-07-14T00:00:00.000Z'); + }); + + it('aggregates the earliest and latest token observations across sessions', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pxpipe-codex-usage-')); + tempDirs.push(root); + const usage = { input_tokens: 10, total_tokens: 10 }; + const document = (timestamp: string): string => [ + JSON.stringify({ type: 'session_meta', payload: { model_provider: 'pxpipe' } }), + tokenLine(usage, usage, timestamp), + '', + ].join('\n'); + await fs.writeFile(path.join(root, 'rollout-later.jsonl'), document('2026-07-14T05:00:00.000Z')); + await fs.writeFile(path.join(root, 'rollout-earlier.jsonl'), document('2026-07-13T22:00:00.000Z')); + + const index = new CodexUsageIndex(root); + await index.refresh(); + + expect(index.snapshot().earliestEventAt).toBe('2026-07-13T22:00:00.000Z'); + expect(index.snapshot().latestEventAt).toBe('2026-07-14T05:00:00.000Z'); + }); + + it('retries provider detection when an active rollout first appears empty', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pxpipe-codex-usage-')); + tempDirs.push(root); + const file = path.join(root, 'rollout-active.jsonl'); + await fs.writeFile(file, ''); + + const index = new CodexUsageIndex(root); + await index.refresh(); + expect(index.snapshot().sessionFiles).toBe(0); + + const usage = { + input_tokens: 700, cached_input_tokens: 600, output_tokens: 30, + reasoning_output_tokens: 12, total_tokens: 730, + }; + await fs.writeFile(file, [ + JSON.stringify({ type: 'session_meta', payload: { model_provider: 'pxpipe' } }), + tokenLine(usage, usage), + ].join('\n')); + await index.refresh(); + + expect(index.snapshot().sessionFiles).toBe(1); + expect(index.snapshot().inputTokens).toBe(700); + }); + + it('increments a growing rollout without recounting prior snapshots', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pxpipe-codex-usage-')); + tempDirs.push(root); + const file = path.join(root, 'rollout-growing.jsonl'); + const first = { + input_tokens: 100, cached_input_tokens: 80, output_tokens: 10, + reasoning_output_tokens: 4, total_tokens: 110, + }; + const secondTotal = { + input_tokens: 250, cached_input_tokens: 200, output_tokens: 25, + reasoning_output_tokens: 9, total_tokens: 275, + }; + const secondLast = { + input_tokens: 150, cached_input_tokens: 120, output_tokens: 15, + reasoning_output_tokens: 5, total_tokens: 165, + }; + await fs.writeFile(file, [ + JSON.stringify({ type: 'session_meta', payload: { model_provider: 'pxpipe' } }), + tokenLine(first, first), + '', + ].join('\n')); + + const index = new CodexUsageIndex(root); + await index.refresh(); + expect(index.snapshot().inputTokens).toBe(100); + + await fs.appendFile(file, `${tokenLine(secondTotal, secondLast, '2026-07-14T00:01:00.000Z')}\n`); + await index.refresh(); + expect(index.snapshot().usageSnapshots).toBe(2); + expect(index.snapshot().inputTokens).toBe(250); + expect(index.snapshot().outputTokens).toBe(25); + + await index.refresh(); + expect(index.snapshot().usageSnapshots).toBe(2); + expect(index.snapshot().inputTokens).toBe(250); + }); + + it('defers an incomplete final JSONL row until the append completes it', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pxpipe-codex-usage-')); + tempDirs.push(root); + const file = path.join(root, 'rollout-partial.jsonl'); + const usage = { + input_tokens: 345, cached_input_tokens: 300, output_tokens: 20, + reasoning_output_tokens: 7, total_tokens: 365, + }; + const row = tokenLine(usage, usage); + const split = Math.floor(row.length / 2); + await fs.writeFile(file, `${JSON.stringify({ type: 'session_meta', payload: { model_provider: 'pxpipe' } })}\n${row.slice(0, split)}`); + + const index = new CodexUsageIndex(root); + await index.refresh(); + expect(index.snapshot().sessionFiles).toBe(1); + expect(index.snapshot().usageSnapshots).toBe(0); + + await fs.appendFile(file, `${row.slice(split)}\n`); + await index.refresh(); + expect(index.snapshot().usageSnapshots).toBe(1); + expect(index.snapshot().inputTokens).toBe(345); + }); + + it('rebuilds per-file state after truncate or path replacement', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pxpipe-codex-usage-')); + tempDirs.push(root); + const file = path.join(root, 'rollout-reset.jsonl'); + const document = (inputTokens: number): string => [ + JSON.stringify({ type: 'session_meta', payload: { model_provider: 'pxpipe' } }), + tokenLine({ input_tokens: inputTokens, total_tokens: inputTokens }, { input_tokens: inputTokens, total_tokens: inputTokens }), + '', + ].join('\n'); + await fs.writeFile(file, document(9000)); + + const index = new CodexUsageIndex(root); + await index.refresh(); + expect(index.snapshot().inputTokens).toBe(9000); + + await fs.writeFile(file, document(40)); + await index.refresh(); + expect(index.snapshot().usageSnapshots).toBe(1); + expect(index.snapshot().inputTokens).toBe(40); + + const replacement = path.join(root, 'replacement.jsonl'); + await fs.writeFile(replacement, document(777)); + await fs.rm(file); + await fs.rename(replacement, file); + await index.refresh(); + expect(index.snapshot().usageSnapshots).toBe(1); + expect(index.snapshot().inputTokens).toBe(777); + }); + + it('preserves dedup and advances quota metadata across incremental appends', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pxpipe-codex-usage-')); + tempDirs.push(root); + const file = path.join(root, 'rollout-metadata.jsonl'); + const usage = { + input_tokens: 120, cached_input_tokens: 100, output_tokens: 8, + reasoning_output_tokens: 3, total_tokens: 128, + }; + const first = tokenLine(usage, usage); + await fs.writeFile(file, [ + JSON.stringify({ type: 'session_meta', payload: { model_provider: 'pxpipe' } }), + first, + '', + ].join('\n')); + + const index = new CodexUsageIndex(root); + await index.refresh(); + + const duplicate = JSON.parse(tokenLine(usage, usage, '2026-07-14T00:10:00.000Z')); + duplicate.payload.rate_limits.primary.used_percent = 91; + duplicate.payload.rate_limits.primary.resets_at = 1_789_999_999; + await fs.appendFile(file, `${JSON.stringify(duplicate)}\n`); + await index.refresh(); + + const snapshot = index.snapshot(); + expect(snapshot.usageSnapshots).toBe(1); + expect(snapshot.inputTokens).toBe(120); + expect(snapshot.rateLimits?.primary?.usedPercent).toBe(91); + expect(snapshot.rateLimits?.primary?.resetsAt).toBe(1_789_999_999); + expect(snapshot.latestEventAt).toBe('2026-07-14T00:10:00.000Z'); + }); +}); diff --git a/tests/context-map.test.ts b/tests/context-map.test.ts index fa8cf7422..fb4081348 100644 --- a/tests/context-map.test.ts +++ b/tests/context-map.test.ts @@ -204,7 +204,25 @@ describe('renderRecentFragment — billed delta presentation', () => { } satisfies RecentPayload); expect(html).toContain('Saved/lost'); + expect(html).toContain('id="help-recent-sent-as"'); + expect(html).toContain('id="help-recent-cache-hits"'); + expect(html).toContain('id="help-recent-as-text"'); + expect(html).toContain('id="help-recent-sent"'); + expect(html).toContain('id="help-recent-saved-lost"'); expect(html).toContain('class="num neg">-61,908'); expect(html).not.toContain('class="num pos">—'); }); + + it('uses a button for Details so inspecting a request cannot follow the current hash anchor', () => { + const html = renderRecentFragment({ + recent: [{ + ts: 0, method: 'POST', path: '/v1/messages', status: 200, compressed: true, img_id: 42, + }], + has_preview: false, + preview_meta: '', + } satisfies RecentPayload); + + expect(html).toContain(''); + expect(off).toContain('GPT 5.6 Terra'); + expect(off).toContain('GPT 5.6 Sol'); expect(off).toContain('GPT 5.5'); - // Sol remains available and ordered before GPT 5.5. + expect(off.indexOf('GPT 5.6 Terra')).toBeLessThan(off.indexOf('GPT 5.6 Sol')); expect(off.indexOf('GPT 5.6 Sol')).toBeLessThan(off.indexOf('GPT 5.5')); expect(getAllowedModelBases()).toContain('claude-fable-5'); expect(getAllowedModelBases()).not.toContain('grok-4.5'); expect(getAllowedModelBases()).not.toContain('gpt-5.6-sol'); expect(getAllowedModelBases()).not.toContain('gpt-5.5'); + dash.handleModelsToggle('gpt-5.6-terra', true); dash.handleModelsToggle('gpt-5.6-sol', true); dash.handleModelsToggle('gpt-5.5', true); const onBoth = await (await dash.serveFragment('models', url, 1234)).text(); expect(onBoth).toContain('GPT 5.5 ✓'); + expect(onBoth).toContain('GPT 5.6 Terra ✓'); expect(onBoth).toContain('GPT 5.6 Sol ✓'); expect(getAllowedModelBases()).toContain('gpt-5.5'); + expect(getAllowedModelBases()).toContain('gpt-5.6-terra'); expect(getAllowedModelBases()).toContain('gpt-5.6-sol'); // Chip flips are reflected back into the textbox CSV. - expect(onBoth).toContain('value="claude-fable-5,claude-opus-5,gemini-3.6-flash,gpt-5.6-sol,gpt-5.5"'); + expect(onBoth).toContain('value="claude-fable-5,claude-opus-5,gemini-3.6-flash,gpt-5.6-terra,gpt-5.6-sol,gpt-5.5"'); } finally { setAllowedModelBases(null); if (prev === undefined) delete process.env.PXPIPE_MODELS; @@ -235,14 +335,53 @@ describe('serveFragment', () => { } }); + it('disables one child of a broad model scope without disabling its known siblings', async () => { + const prev = process.env.PXPIPE_MODELS; + try { + process.env.PXPIPE_MODELS = 'gpt-5.6'; + setAllowedModelBases(null); + expect(isPxpipeSupportedGptModel('gpt-5.6-sol')).toBe(true); + const html = await (await dash.serveFragment('models', url, 1234)).text(); + expect(html).toContain('GPT 5.6 Sol ✓'); + + dash.handleModelsToggle('gpt-5.6-sol', false); + expect(isPxpipeSupportedGptModel('gpt-5.6-sol')).toBe(false); + expect(getAllowedModelBases()).not.toContain('gpt-5.6'); + expect(isPxpipeSupportedGptModel('gpt-5.6-terra')).toBe(true); + expect(isPxpipeSupportedGptModel('gpt-5.6-lun')).toBe(true); + expect(getAllowedModelBases()).toEqual(['gpt-5.6-terra', 'gpt-5.6-lun']); + } finally { + setAllowedModelBases(null); + if (prev === undefined) delete process.env.PXPIPE_MODELS; + else process.env.PXPIPE_MODELS = prev; + } + }); + + it('disables the whole group when the broad model shortcut is turned off', () => { + const prev = process.env.PXPIPE_MODELS; + try { + process.env.PXPIPE_MODELS = 'gpt-5.6'; + setAllowedModelBases(null); + dash.handleModelsToggle('gpt-5.6', false); + expect(getAllowedModelBases()).toEqual([]); + expect(isPxpipeSupportedGptModel('gpt-5.6-terra')).toBe(false); + expect(isPxpipeSupportedGptModel('gpt-5.6-sol')).toBe(false); + expect(isPxpipeSupportedGptModel('gpt-5.6-lun')).toBe(false); + } finally { + setAllowedModelBases(null); + if (prev === undefined) delete process.env.PXPIPE_MODELS; + else process.env.PXPIPE_MODELS = prev; + } + }); + it('invokes the host persistence hook on scope mutations', () => { const prev = process.env.PXPIPE_MODELS; try { delete process.env.PXPIPE_MODELS; setAllowedModelBases(null); - const saved: string[][] = []; + const saved: Array = []; const persisting = new DashboardState(tmp, async () => new Map(), (bases) => { - saved.push([...bases]); + saved.push(bases === null ? null : [...bases]); }); persisting.handleModelsToggle('gpt-5.6-sol', true); @@ -252,7 +391,10 @@ describe('serveFragment', () => { // Empty scope persists too (round-trips as 'off' on load). persisting.handleModelsSet('off'); expect(saved.at(-1)).toEqual([]); - expect(saved).toHaveLength(3); + persisting.handleModelsReset(); + expect(getAllowedModelBases()).toContain('claude-fable-5'); + expect(saved.at(-1)).toBeNull(); + expect(saved).toHaveLength(4); // A throwing hook must not break the live flip or the endpoint. const throwing = new DashboardState(tmp, async () => new Map(), () => { @@ -322,11 +464,11 @@ describe('serveFragment', () => { expect(html).toContain('never calls Anthropic /count_tokens'); }); - it('renders keyboard-accessible hover help for stat question marks', async () => { + it('renders a keyboard-reachable methodology link and audit drawer', async () => { const header = await (await dash.serveFragment('header', url, 4711)).text(); - expect(header).toContain('class="q" tabindex="0"'); - expect(header).toContain('data-tip='); - expect(header).toContain('aria-label='); + expect(header).toContain('href="#audit-drawer"'); + expect(header).toContain('
'); + expect(header.indexOf('href="#audit-drawer"')).toBeLessThan(header.indexOf('id="audit-drawer"')); }); it('uses source text parallel to each captured PNG', async () => { @@ -364,10 +506,32 @@ describe('serveFragment', () => { }); describe('dashboard page help UI', () => { - it('ships visible hover/focus tooltip CSS for question-mark controls', () => { + it('ships progressive-disclosure navigation and visible keyboard focus', () => { const html = renderPage(47821); - expect(html).toContain('.q:hover::after, .q:focus-visible::after'); - expect(html).toContain('content: attr(data-tip)'); + expect(html).toContain('aria-label="Dashboard sections"'); + expect(html).toContain('href="#overview"'); + expect(html).toContain('href="#audit-drawer"'); + expect(html).toContain('
'); + expect(html).toContain('Model scope & routing settings'); + expect(html).toContain('.page-nav a:focus-visible'); + expect(html).toContain('function ppRevealHash()'); + expect(html).toContain("target.tagName === 'DETAILS'"); + expect(html).toContain('id="help-model-settings"'); + expect(html).toContain('id="help-recent-requests"'); + expect(html).toContain('id="help-context-breakdown"'); + expect(html).toContain('id="help-source-inspector"'); + expect(html).toContain('id="help-top-sessions"'); + expect(html).toContain('id="help-full-history"'); + expect(html).toContain('summary aria-label="Help:'); + expect(html).toContain('.help-tip > summary:focus-visible'); + expect(html).toContain("opened.matches('details.help-tip[open]')"); + expect(html).toContain("ev.key !== 'Escape'"); + expect(html).toContain('hashRevealPending: !!location.hash'); + expect(html).toContain('hx-vals=\'js:{session: window.pp.session || ""}\''); + expect(html).toContain('function ppWatchLatest()'); + expect(html).toContain("details[id]').forEach"); + expect(html).toContain('d.toggleAttribute(\'open\', state.open)'); + expect(html).toContain('position: sticky; top: 0; z-index: 200'); }); it('compares imaged requests with their own without-pxpipe counterfactual', () => { @@ -382,10 +546,9 @@ describe('dashboard page help UI', () => { pricing_assumptions: { input_per_mtok: 10, output_multiplier: 5 }, } as StatsPayload, 47821); - expect(html).toContain('$0.1607'); - expect(html).toContain('vs $0.4854 without pxpipe'); - expect(html).not.toContain('vs $0.0722 without pxpipe'); - expect(html).toContain('same paid imaged requests'); + expect(html).toContain('Observed path cost index'); + expect(html).toContain('16,070'); + expect(html).not.toContain('without pxpipe'); }); }); @@ -427,30 +590,44 @@ describe('GPT savings split', () => { expect(stats.actual_input_weighted).toBe(8200); expect(stats.baseline_input_weighted).toBe(12400); expect(stats.saved_input_tokens).toBe(4200); + expect(stats.measured_claude_saved_input_equivalents).toBe(0); + expect(stats.modeled_openai_saved_input_equivalents).toBe(4200); + expect(stats.usage_bearing_responses).toBe(1); expect(stats.saved_pct_input_only).toBeGreaterThan(0); }); - it('includes only currently enabled models in overall stats', async () => { - dash.update(structuredClone(gptUpdate) as never); - dash.update({ - ...structuredClone(gptUpdate), - model: 'gpt-5.6-sol', - info: { ...structuredClone(gptUpdate.info), firstUserSha8: 'gptsol' }, - } as never); - let stats = (await dash.serveStats().json()) as StatsPayload; - expect(stats.requests).toBe(1); - expect(stats.saved_input_tokens).toBe(4200); + it('separates measured Claude from modeled OpenAI savings and counts usage-bearing responses', async () => { + setAllowedModelBases(['gpt-5.5', 'claude-fable-5']); + dash.update(structuredClone(gptUpdate) as never); // modeled OpenAI: 4,200 + dash.update({ + method: 'POST', + path: '/v1/messages', + model: 'claude-fable-5', + status: 200, + durationMs: 100, + usage: { input_tokens: 400, output_tokens: 10 }, + info: { + compressed: true, + baselineTokens: 1000, + baselineCacheableTokens: 800, + baselineProbeStatus: 'ok', + firstUserSha8: 'claudesess1', + }, + } as never); // measured Claude: 800×1.25 + 200 - 400 = 800 + dash.update({ + method: 'GET', path: '/health', status: 200, durationMs: 1, + } as never); // no usage: must not enter the paid-response count - dash.handleModelsToggle('gpt-5.6-sol', true); - stats = (await dash.serveStats().json()) as StatsPayload; - expect(stats.requests).toBe(2); - expect(stats.saved_input_tokens).toBe(8400); + const stats = (await dash.serveStats().json()) as StatsPayload; + expect(stats.saved_input_tokens).toBe(5000); // legacy combined total + expect(stats.measured_claude_saved_input_equivalents).toBe(800); + expect(stats.modeled_openai_saved_input_equivalents).toBe(4200); + expect(stats.measured_anthropic_savings_requests).toBe(1); + expect(stats.estimated_openai_savings_requests).toBe(1); + expect(stats.usage_bearing_responses).toBe(2); + expect(stats.all_usage_requests).toBe(stats.usage_bearing_responses); - dash.handleModelsToggle('gpt-5.5', false); - stats = (await dash.serveStats().json()) as StatsPayload; - expect(stats.requests).toBe(1); - expect(stats.saved_input_tokens).toBe(4200); }); it('populates As-text / Sent / Cache-hits / Saved recent columns for GPT', async () => { @@ -656,6 +833,11 @@ describe('Gemini savings split', () => { const stats = (await dash.serveStats().json()) as StatsPayload; expect(stats.saved_input_tokens).toBe(280); expect(stats.saved_usd).toBe(0); + expect(stats.estimated_google_savings_requests).toBe(1); + expect(stats.modeled_google_saved_input_equivalents).toBe(280); + expect(stats.usage_bearing_responses).toBe(1); + expect(stats.all_usage_requests).toBe(1); + expect(stats.all_baseline_equivalent_weighted).toBe(400); const recent = (await dash.serveRecent().json()) as RecentPayload; expect(recent.recent.at(-1)?.baseline_input).toBe(400); expect(recent.recent.at(-1)?.actual_input).toBe(120); @@ -683,6 +865,62 @@ describe('Gemini savings split', () => { expect(stats.saved_usd).toBe(0); }); + it('keeps modeled Responses savings out of Claude dollar totals', async () => { + setAllowedModelBases(['claude-fable-5', 'gpt-5.6-sol']); + dash.update({ + method: 'POST', path: '/v1/messages', model: 'claude-fable-5', status: 200, durationMs: 1, + usage: { input_tokens: 100, output_tokens: 0 }, + info: { compressed: true, baselineTokens: 300, baselineCacheableTokens: 300, baselineProbeStatus: 'ok' }, + } as never); + const claudeOnly = (await dash.serveStats().json()) as StatsPayload; + expect(claudeOnly.saved_usd).toBeGreaterThan(0); + dash.update({ + method: 'POST', path: '/v1/responses', accountingProvider: 'openai', model: 'gpt-5.6-sol', status: 200, durationMs: 1, + usage: { input_tokens: 100, output_tokens: 0 }, + info: { compressed: true, imageTokens: 100, baselineImagedTokens: 500 }, + } as never); + + const stats = (await dash.serveStats().json()) as StatsPayload; + // Only the Claude row is priced; adding a modeled Responses row must not + // change its model-priced dollar estimate. + expect(stats.saved_usd).toBe(claudeOnly.saved_usd); + expect(stats.estimated_openai_savings_requests).toBe(1); + expect(stats.usage_bearing_responses).toBe(2); + }); + + it('uses each Claude model\'s configured dollar rate for savings', async () => { + setAllowedModelBases(['claude-opus-5']); + dash.update({ + method: 'POST', path: '/v1/messages', model: 'claude-opus-5', status: 200, durationMs: 1, + usage: { input_tokens: 100, output_tokens: 0 }, + info: { + compressed: true, + baselineTokens: 1_000_000, + baselineCacheableTokens: 1, + baselineProbeStatus: 'ok', + }, + } as never); + + const stats = (await dash.serveStats().json()) as StatsPayload; + // 999,900 effective input tokens saved × Opus $5/M, not the legacy $10/M. + expect(stats.saved_usd).toBeCloseTo(4.9995, 4); + }); + + it('keeps since-restart totals after a model is later disabled', async () => { + setAllowedModelBases(['claude-fable-5']); + dash.update({ + method: 'POST', path: '/v1/messages', model: 'claude-fable-5', status: 200, durationMs: 1, + usage: { input_tokens: 100, output_tokens: 0 }, + info: { compressed: true, baselineTokens: 300, baselineCacheableTokens: 300, baselineProbeStatus: 'ok' }, + } as never); + setAllowedModelBases([]); + + const stats = (await dash.serveStats().json()) as StatsPayload; + expect(stats.requests).toBe(1); + expect(stats.usage_bearing_responses).toBe(1); + expect(stats.saved_input_tokens).toBeGreaterThan(0); + }); + it('shows estimated savings when optional Gemini measurement fails', async () => { dash.update({ method: 'POST', @@ -714,8 +952,12 @@ describe('Gemini savings split', () => { const stats = (await dash.serveStats().json()) as StatsPayload; expect(stats.saved_input_tokens).toBe(2900); expect(stats.saved_usd).toBe(0); + expect(stats.estimated_google_savings_requests).toBe(1); + expect(stats.modeled_google_saved_input_equivalents).toBe(2900); const header = await (await dash.serveFragment('header', new URL('http://localhost/fragments/header'), 1)).text(); expect(header).not.toContain('$0.03'); + expect(header).toContain('Gemini'); + expect(header).toContain('provider usage + modeled baseline · 1 rows'); const html = await (await dash.serveFragment('recent', new URL('http://localhost/fragments/recent'), 1)).text(); expect(html).toContain('Details →'); const details = await (await dash.serveFragment( diff --git a/tests/gateway.test.ts b/tests/gateway.test.ts index 3963fbf6c..ee6fb723d 100644 --- a/tests/gateway.test.ts +++ b/tests/gateway.test.ts @@ -128,6 +128,43 @@ describe('gateway end-to-end routing (stubbed fetch)', () => { expect(cap.url).toBe(`${FAKE_BASE}/openai/responses`); }); + it('routes ChatGPT-authenticated Codex Responses traffic to the OpenAI upstream and transforms it', async () => { + const cap: { url?: string; headers?: Headers } = {}; + stubFetch(cap); + await createProxy({ + upstream: 'https://api.anthropic.example.test', + openAIUpstream: 'https://chatgpt.example.test', + transform: { charsPerToken: 1, minCompressChars: 1 }, + })( + new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: 'Bearer chatgpt-token' }, + body: JSON.stringify({ + model: 'gpt-5.6-terra', + instructions: 'System instructions. '.repeat(700), + input: 'hi', + }), + }), + ); + expect(cap.url).toBe('https://chatgpt.example.test/backend-api/codex/responses'); + expect(cap.headers?.get('authorization')).toBe('Bearer chatgpt-token'); + }); + + it('forwards the ChatGPT Codex model catalogue to the OpenAI upstream', async () => { + const cap: { url?: string; headers?: Headers } = {}; + stubFetch(cap); + await createProxy({ + upstream: 'https://api.anthropic.example.test', + openAIUpstream: 'https://chatgpt.example.test', + })( + new Request('http://localhost/backend-api/codex/models?client_version=0.144.0', { + headers: { authorization: 'Bearer chatgpt-token' }, + }), + ); + expect(cap.url).toBe('https://chatgpt.example.test/backend-api/codex/models?client_version=0.144.0'); + expect(cap.headers?.get('authorization')).toBe('Bearer chatgpt-token'); + }); + it('passes unrecognized Anthropic-family paths through untouched', async () => { const cap: { url?: string; headers?: Headers } = {}; stubFetch(cap); diff --git a/tests/health-counters.test.ts b/tests/health-counters.test.ts new file mode 100644 index 000000000..500e61c56 --- /dev/null +++ b/tests/health-counters.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect } from 'vitest'; +import { HealthCounters } from '../src/health-counters.js'; + +describe('HealthCounters', () => { + it('ignores non-codex paths', () => { + const c = new HealthCounters(); + c.record('/v1/messages', 200, 1000); + c.record('/v1/messages', 404, 1000); + expect(c.snapshot(1000)).toEqual({ codexResponses404: 0, codexResponsesTotal: 0, windowSeconds: 300 }); + }); + + it('counts codex requests and 404s within the window', () => { + const c = new HealthCounters(); + c.record('/backend-api/codex/responses', 404, 1000); + c.record('/backend-api/codex/responses', 404, 1500); + c.record('/backend-api/codex/models', 200, 2000); + expect(c.snapshot(2000)).toEqual({ codexResponses404: 2, codexResponsesTotal: 3, windowSeconds: 300 }); + }); + + it('drops entries older than the window', () => { + const c = new HealthCounters(10_000); // 10s window + c.record('/backend-api/codex/responses', 404, 1_000); + c.record('/backend-api/codex/responses', 404, 20_000); + // at t=20s, the first entry (t=1s) is >10s old and must be dropped + expect(c.snapshot(20_000)).toEqual({ codexResponses404: 1, codexResponsesTotal: 1, windowSeconds: 10 }); + }); + + it('snapshot alone (no new record) still trims stale entries', () => { + const c = new HealthCounters(10_000); + c.record('/backend-api/codex/responses', 404, 1_000); + expect(c.snapshot(50_000)).toEqual({ codexResponses404: 0, codexResponsesTotal: 0, windowSeconds: 10 }); + }); +}); diff --git a/tests/health-state.test.ts b/tests/health-state.test.ts new file mode 100644 index 000000000..5ed781612 --- /dev/null +++ b/tests/health-state.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { buildHealthReport, buildHealthState } from '../src/health-state.js'; +import { HealthCounters } from '../src/health-counters.js'; +import { evaluateHealth } from '../src/core/health.js'; +import { setAllowedModelBases } from '../src/core/applicability.js'; + +const compressionOn = { getCompressionEnabled: () => true }; + +describe('buildHealthState', () => { + beforeEach(() => setAllowedModelBases(null)); // reset runtime override between tests + + it('reads the resolved OpenAI upstream from config', () => { + const s = buildHealthState({ openAIUpstream: 'https://api.openai.com' }, compressionOn, new HealthCounters(), 1000); + expect(s.openaiUpstream).toBe('https://api.openai.com'); + expect(s.openaiUpstreamOverridden).toBe(false); + }); + + it('surfaces a codex 404 recorded in the counter as an error finding', () => { + const counters = new HealthCounters(); + counters.record('/backend-api/codex/responses', 404, 1000); + const s = buildHealthState({ openAIUpstream: 'https://api.openai.com' }, compressionOn, counters, 1000); + const f = evaluateHealth(s).find((x) => x.id === 'codex-upstream-mismatch'); + expect(f?.severity).toBe('error'); + }); + + it('reflects the live model scope and compression state', () => { + setAllowedModelBases(['gpt-5.6-terra']); + const s = buildHealthState({}, { getCompressionEnabled: () => false }, new HealthCounters(), 1000); + expect(s.modelScope).toEqual(['gpt-5.6-terra']); + expect(s.compressionEnabled).toBe(false); + }); +}); + +describe('buildHealthReport', () => { + it('fails closed when health-state assembly throws', () => { + const counters = new HealthCounters(); + const report = buildHealthReport( + { provider: 'cloudflare-ai-gateway' }, + { getCompressionEnabled: () => true }, + counters, + 1_000, + ); + + expect(report.ok).toBe(false); + expect(report.httpStatus).toBe(503); + expect(report.state).toBeNull(); + expect(report.findings).toEqual([ + expect.objectContaining({ + id: 'health-diagnostics-failed', + severity: 'error', + }), + ]); + }); +}); diff --git a/tests/health.test.ts b/tests/health.test.ts new file mode 100644 index 000000000..5420b0ac4 --- /dev/null +++ b/tests/health.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from 'vitest'; +import { evaluateHealth, summarizeHealth, type HealthState } from '../src/core/health.js'; + +function state(over: Partial = {}): HealthState { + return { + anthropicUpstream: 'https://api.anthropic.com', + openaiUpstream: 'https://chatgpt.com', + openaiUpstreamOverridden: false, + modelScope: ['claude-fable-5'], + compressionEnabled: true, + recent: { codexResponses404: 0, codexResponsesTotal: 0, windowSeconds: 300 }, + ...over, + }; +} +const ids = (s: HealthState) => evaluateHealth(s).map((f) => f.id); +const find = (s: HealthState, id: string) => evaluateHealth(s).find((f) => f.id === id); + +describe('evaluateHealth — codex upstream', () => { + it('no finding when upstream is chatgpt.com', () => { + expect(ids(state())).not.toContain('codex-upstream-mismatch'); + }); + it('no finding when upstream is a chatgpt.com subdomain', () => { + expect(ids(state({ openaiUpstream: 'https://api.chatgpt.com' }))).not.toContain('codex-upstream-mismatch'); + }); + it('warns (not errors) when host is wrong but no codex 404 seen yet', () => { + const f = find(state({ openaiUpstream: 'https://api.openai.com' }), 'codex-upstream-mismatch'); + expect(f?.severity).toBe('warn'); + expect(f?.remediation?.target).toBe('https://chatgpt.com'); + }); + it('escalates to error once a codex 404 is observed', () => { + const f = find( + state({ openaiUpstream: 'https://api.openai.com', recent: { codexResponses404: 3, codexResponsesTotal: 3, windowSeconds: 300 } }), + 'codex-upstream-mismatch', + ); + expect(f?.severity).toBe('error'); + }); + it('does not error when host is chatgpt.com even with 404s (not our fault)', () => { + const f = find( + state({ recent: { codexResponses404: 5, codexResponsesTotal: 5, windowSeconds: 300 } }), + 'codex-upstream-mismatch', + ); + expect(f).toBeUndefined(); + }); + it('treats an unparseable upstream as wrong host (warns)', () => { + expect(find(state({ openaiUpstream: 'not a url' }), 'codex-upstream-mismatch')?.severity).toBe('warn'); + }); +}); + +describe('evaluateHealth — info findings', () => { + it('flags passthrough when compression is off', () => { + expect(ids(state({ compressionEnabled: false }))).toContain('compression-passthrough'); + }); + it('flags passthrough when model scope is empty', () => { + expect(ids(state({ modelScope: [] }))).toContain('compression-passthrough'); + }); + it('flags an active runtime override', () => { + expect(ids(state({ openaiUpstreamOverridden: true }))).toContain('openai-upstream-overridden'); + }); +}); + +describe('summarizeHealth', () => { + it('ok + 200 when no error findings', () => { + expect(summarizeHealth(evaluateHealth(state()))).toEqual({ ok: true, httpStatus: 200 }); + }); + it('not ok + 503 when an error finding is present', () => { + const findings = evaluateHealth(state({ openaiUpstream: 'https://api.openai.com', recent: { codexResponses404: 1, codexResponsesTotal: 1, windowSeconds: 300 } })); + expect(summarizeHealth(findings)).toEqual({ ok: false, httpStatus: 503 }); + }); +}); diff --git a/tests/hero.test.ts b/tests/hero.test.ts index 8e78d99ce..a699a1160 100644 --- a/tests/hero.test.ts +++ b/tests/hero.test.ts @@ -1,60 +1,140 @@ import { describe, it, expect } from 'vitest'; -import { renderSessionSummaryFragment } from '../src/dashboard/fragments.js'; -import type { StatsPayload } from '../src/dashboard/types.js'; +import { renderHeaderFragment, renderSessionSummaryFragment } from '../src/dashboard/fragments.js'; +import type { CurrentSessionPayload, StatsPayload } from '../src/dashboard/types.js'; -/** - * The hero reads the SAME cache-weighted lifetime pair as the header strip - * (serveStats), not the per-session payload — so it can't disagree with the - * "$ saved" tiles and it stops swinging on tiny samples. The old bug divided - * raw count_tokens (cache-blind) by sent tokens and could claim a big "fewer - * tokens" win on a session the Saved column showed as a net loss. These pin - * direction to `baseline_input_weighted` vs `actual_input_weighted`. - */ -function payload(p: Partial): StatsPayload { +function payload(p: Partial): CurrentSessionPayload { return { - compressed_requests: 1, - output_weighted: 695, // 139 raw × 5 output_multiplier - pricing_assumptions: { output_multiplier: 5 }, + sessionId: 'abcdef1234567890', + baselineMeasuredCount: 1, + rawOutputTokens: 139, ...p, - } as StatsPayload; + }; } describe('renderSessionSummaryFragment hero', () => { - it('shows "fewer tokens" when the weighted image beat weighted text', () => { + it('shows the cache-aware effect for the current session', () => { const html = renderSessionSummaryFragment( - payload({ baseline_input_weighted: 7000, actual_input_weighted: 1800 }), + payload({ baselineInputWeighted: 7000, actualInputWeighted: 1800 }), ); - expect(html).toContain('fewer tokens'); - expect(html).not.toContain('more tokens'); + expect(html).toContain('Current session · abcdef12'); + expect(html).toContain('class="hero-num">74%'); + expect(html).toContain('less estimated input after caching'); + expect(html).toContain('id="help-current-session-percent"'); + expect(html).toContain('id="help-current-session-effective-input"'); expect(html).toContain('74%'); // 1 - 1800/7000 }); - it('flips to "more tokens" on a warm net-loss session (matches Saved "-")', () => { - // The exact trap: raw text (e.g. 7.2k) would look like a huge win, but the - // cache-weighted text baseline (1,546) is below what imaging actually sent (1,863). + it('flips direction on a warm net-loss session', () => { const html = renderSessionSummaryFragment( - payload({ baseline_input_weighted: 1546, actual_input_weighted: 1863 }), + payload({ baselineInputWeighted: 1546, actualInputWeighted: 1863 }), ); - expect(html).toContain('more tokens'); - expect(html).not.toContain('fewer tokens'); + expect(html).toContain('class="hero-num">21%'); + expect(html).toContain('more estimated input after caching'); expect(html).toContain('hero-neg'); // red styling on a loss }); it('never lumps output into the headline ratio', () => { - // Same input pair, wildly different output — headline % must not move. const a = renderSessionSummaryFragment( - payload({ baseline_input_weighted: 2000, actual_input_weighted: 1000, output_weighted: 50 }), + payload({ baselineInputWeighted: 2000, actualInputWeighted: 1000, rawOutputTokens: 10 }), ); const b = renderSessionSummaryFragment( - payload({ baseline_input_weighted: 2000, actual_input_weighted: 1000, output_weighted: 45000 }), + payload({ baselineInputWeighted: 2000, actualInputWeighted: 1000, rawOutputTokens: 9000 }), ); expect(a).toContain('50%'); expect(b).toContain('50%'); }); - it('renders the warming-up state with no measured requests', () => { - const html = renderSessionSummaryFragment(payload({ compressed_requests: 0 })); - expect(html).toContain('Warming up'); - expect(html).not.toContain('fewer tokens'); + it('renders explicit waiting states for no session and an unmeasured session', () => { + expect(renderSessionSummaryFragment({ sessionId: null })).toContain('Waiting for session traffic'); + expect(renderSessionSummaryFragment(payload({ baselineMeasuredCount: 0 }))).toContain('Waiting for a comparable response'); + }); +}); + +describe('renderHeaderFragment estimate quality', () => { + it('shows measured, estimated, excluded, TTL, and pricing coverage without opening the drawer', () => { + const html = renderHeaderFragment({ + requests: 9, + compressed_requests: 8, + saved_input_tokens: 12_345, + measured_claude_saved_input_equivalents: 8_500, + modeled_openai_saved_input_equivalents: 3_845, + usage_bearing_responses: 4, + measured_anthropic_savings_requests: 3, + estimated_openai_savings_requests: 2, + baseline_probe_excluded_requests: 1, + cache_create_5m_tokens: 750, + cache_create_1h_tokens: 250, + cache_create_tier_unknown_tokens: 1_000, + priced_measured_savings_requests: 2, + unpriced_measured_savings_requests: 1, + saved_usd: 0.42, + split_sufficient_sample: false, + compressed_paid_requests: 3, + passthrough_paid_requests: 1, + split_min_sample_per_bucket: 10, + uptime_sec: 60, + pricing_assumptions: { + input_per_mtok: 3, + output_multiplier: 5, + source: 'official pricing', + }, + codex_actual_usage: { + source: 'test sessions', loading: false, error: null, + sessionFiles: 4, usageSnapshots: 12, + inputTokens: 10_000, cachedInputTokens: 9_000, + outputTokens: 600, reasoningOutputTokens: 250, totalTokens: 10_600, + modelContextWindow: 200_000, + earliestEventAt: '2026-07-13T22:00:00Z', latestEventAt: '2026-07-14T00:00:00Z', + quotaWindows: [ + { usedPercent: 25, windowMinutes: 300, resetsAt: 1_789_000_000, limitId: 'codex', limitName: 'Codex', observedAt: '2026-07-14T00:00:00Z' }, + { usedPercent: 40, windowMinutes: 10_080, resetsAt: 1_789_500_000, limitId: 'codex', limitName: 'Codex', observedAt: '2026-07-14T00:00:00Z' }, + { usedPercent: 12, windowMinutes: 300, resetsAt: 1_789_100_000, limitId: 'codex_bengalfox', limitName: 'GPT-5.3-Codex-Spark', observedAt: '2026-07-14T00:00:00Z' }, + { usedPercent: 18, windowMinutes: 10_080, resetsAt: 1_789_600_000, limitId: 'codex_bengalfox', limitName: 'GPT-5.3-Codex-Spark', observedAt: '2026-07-14T00:00:00Z' }, + { usedPercent: 7, windowMinutes: 1_440, resetsAt: 1_789_200_000, limitId: 'codex_bengalfox', limitName: 'GPT-5.3-Codex-Spark', observedAt: '2026-07-14T00:00:00Z' }, + ], + rateLimits: { + limitId: 'codex', limitName: 'Codex', planType: null, + observedAt: '2026-07-14T00:00:00Z', + primary: { usedPercent: 25, windowMinutes: 300, resetsAt: 1_789_000_000 }, + secondary: { usedPercent: 40, windowMinutes: 10_080, resetsAt: 1_789_500_000 }, + }, + }, + } as StatsPayload, 47821); + + expect(html).toContain('Overview · since restart'); + expect(html).toContain('What PXPIPE changed'); + expect(html).toContain('Estimated input change'); + expect(html).toContain('8,500Model-priced input value$0.422/3 priced Claude rows'); + expect(html).toContain('3,845 not priced'); + expect(html).toContain('Paid LLM responses'); + expect(html).toContain('Mixed evidence'); + expect(html).toContain('2/3 rows'); + expect(html).toContain('1 of imaged rows uncredited'); + expect(html).toContain('Usage & limits · retained rollouts'); + expect(html).toContain('Codex provider-reported usage'); + expect(html).toContain('9,000 · 90.0%'); + expect(html).toContain('12 usage records'); + expect(html).toContain('codex_bengalfox'); + expect(html).toContain('GPT-5.3-Codex-Spark'); + expect(html).toContain('1-day'); + expect(html).toContain('Usage, not savings.'); + expect(html).toContain('750 / 250 / 1,000'); + expect(html).toContain('id="audit-drawer"'); + expect(html).toContain('
Hide'); + expect(html).toContain('id="help-overview"'); + expect(html).toContain('id="help-input-change"'); + expect(html).toContain('id="help-paid-responses"'); + expect(html).toContain('id="help-paid-imaged"'); + expect(html).toContain('id="help-paid-passthrough"'); + expect(html).toContain('id="help-paid-uncredited"'); + expect(html).toContain('id="help-price-coverage"'); + expect(html).toContain('id="help-reliability"'); + expect(html).toContain('id="help-codex-quotas"'); + expect(html).toContain('What it is.'); + expect(html).toContain('Why it matters.'); + expect(html).toContain('How to read it.'); + expect(html.indexOf('Overview · since restart')).toBeLessThan(html.indexOf('Show the math')); }); }); diff --git a/tests/model-scope-store.test.ts b/tests/model-scope-store.test.ts new file mode 100644 index 000000000..6542d6ba7 --- /dev/null +++ b/tests/model-scope-store.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + modelScopeFile, + loadPersistedModelScope, + savePersistedModelScope, + clearPersistedModelScope, +} from '../src/model-scope-store.js'; + +describe('model-scope-store (Variant A persistence)', () => { + let dir: string; + let eventsFile: string; + let file: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-scope-')); + eventsFile = path.join(dir, 'events.jsonl'); + file = modelScopeFile(eventsFile); + }); + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('places the file next to the events log', () => { + expect(file).toBe(path.join(dir, 'model-scope.json')); + }); + + it('absent file → null (fall back to PXPIPE_MODELS / default)', () => { + expect(loadPersistedModelScope(file)).toBeNull(); + }); + + it('round-trips a saved scope verbatim', () => { + savePersistedModelScope(file, ['claude-fable-5', 'gpt-5.6-terra']); + expect(loadPersistedModelScope(file)).toEqual(['claude-fable-5', 'gpt-5.6-terra']); + }); + + it('persists an empty list as [] — a real "compress nothing" choice, NOT null', () => { + savePersistedModelScope(file, []); + // The load-bearing distinction of Variant A: [] is a persisted choice, so + // it must NOT fall through to env/default the way an absent file does. + expect(loadPersistedModelScope(file)).toEqual([]); + }); + + it('clear removes the file → load returns null again', () => { + savePersistedModelScope(file, ['claude-sonnet-5']); + clearPersistedModelScope(file); + expect(loadPersistedModelScope(file)).toBeNull(); + }); + + it('clear on an absent file is a no-op (does not throw)', () => { + expect(() => clearPersistedModelScope(file)).not.toThrow(); + expect(loadPersistedModelScope(file)).toBeNull(); + }); + + it('malformed JSON → null (a corrupt file must never block startup)', () => { + fs.writeFileSync(file, '{ not valid json'); + expect(loadPersistedModelScope(file)).toBeNull(); + }); + + it('valid JSON without a modelBases array → null', () => { + fs.writeFileSync(file, JSON.stringify({ modelBases: 'claude-fable-5' })); + expect(loadPersistedModelScope(file)).toBeNull(); + }); + + it('trims and drops blank/non-string entries on load', () => { + fs.writeFileSync( + file, + JSON.stringify({ modelBases: [' claude-fable-5 ', '', 42, 'gpt-5.6-sol'] }), + ); + expect(loadPersistedModelScope(file)).toEqual(['claude-fable-5', 'gpt-5.6-sol']); + }); + + it('all-invalid non-empty arrays → null instead of disabling every model', () => { + fs.writeFileSync(file, JSON.stringify({ modelBases: [42, {}, ' '] })); + expect(loadPersistedModelScope(file)).toBeNull(); + }); +}); diff --git a/tests/models-fragment-warn.test.ts b/tests/models-fragment-warn.test.ts new file mode 100644 index 000000000..de1e4fd24 --- /dev/null +++ b/tests/models-fragment-warn.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest'; +import { renderModelsFragment } from '../src/dashboard/fragments.js'; + +// Readiness tiers (derived from committed eval receipts, see fragments.ts): +// - below-bar → blocking confirm on enable + ⚠ when lit +// - unmeasured → NON-blocking ⚠ + tooltip (default for unlisted ids) +// - validated (Fable) → nothing +const BELOW_BAR = ['claude-opus-4-8', 'claude-opus-4-7', 'gpt-5.5', 'gpt-5.6-sol', 'grok-4.5', 'gpt-5.6']; + +/** Extract the `))?.[0] ?? ''; +} + +describe('renderModelsFragment — readiness-tiered warnings', () => { + it('blocking confirm on exactly the below-bar chips when OFF', () => { + const html = renderModelsFragment([], [], true); + expect((html.match(/hx-confirm=/g) ?? []).length).toBe(BELOW_BAR.length); + // below-bar confirm carries the concrete measured number. + expect(html).toContain('0/15 verbatim dense-hex'); + }); + + it('disabling never prompts: all below-bar ON → no confirm, ⚠ present', () => { + const html = renderModelsFragment(BELOW_BAR, [], true); + expect(html).not.toContain('hx-confirm='); + expect(html).toContain('⚠'); + }); + + it('unmeasured (Terra) never blocks: no confirm off or on; ⚠ + tooltip when on', () => { + const off = chip(renderModelsFragment([], [], true), 'gpt-5.6-terra'); + const on = chip(renderModelsFragment(['gpt-5.6-terra'], [], true), 'gpt-5.6-terra'); + expect(off).not.toContain('hx-confirm='); + expect(on).not.toContain('hx-confirm='); + expect(on).toContain('⚠'); + expect(on).toContain('title='); + expect(on).toContain('unmeasured'); + }); + + it('validated (Fable) is never flagged: no confirm, ⚠, or title', () => { + const on = chip(renderModelsFragment(['claude-fable-5'], [], true), 'claude-fable-5'); + expect(on).not.toContain('hx-confirm='); + expect(on).not.toContain('⚠'); + expect(on).not.toContain('title='); + expect(on).toContain('✓'); + }); + + it('unknown/unlisted model defaults to unmeasured (⚠ + tooltip, no confirm)', () => { + const c = chip(renderModelsFragment(['gpt-9.9-foo'], [], true), 'gpt-9.9-foo'); + expect(c).not.toContain('hx-confirm='); + expect(c).toContain('⚠'); + expect(c).toContain('title='); + }); +}); diff --git a/tests/node-security.test.ts b/tests/node-security.test.ts index 1741a9806..108c6ece3 100644 --- a/tests/node-security.test.ts +++ b/tests/node-security.test.ts @@ -9,6 +9,15 @@ import { fileURLToPath } from 'node:url'; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const tsxCli = path.join(repoRoot, 'node_modules', 'tsx', 'dist', 'cli.mjs'); +const supportsPosixModes = process.platform !== 'win32'; + +function expectPrivateFile(filePath: string): void { + if (supportsPosixModes) expect(fs.statSync(filePath).mode & 0o777).toBe(0o600); +} + +function expectPrivateDirectory(dirPath: string): void { + if (supportsPosixModes) expect(fs.statSync(dirPath).mode & 0o777).toBe(0o700); +} let child: ChildProcess | undefined; let upstream: Server | undefined; @@ -100,6 +109,68 @@ async function startNode(extraEnv: Record = {}): Promise<{ } describe('Node dashboard security', () => { + it('refuses server-owned credential injection on a non-loopback bind', async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-node-security-')); + const port = await freePort(); + const output: string[] = []; + child = spawn(process.execPath, [tsxCli, 'src/node.ts'], { + cwd: repoRoot, + env: { + ...process.env, + PORT: String(port), + HOST: '0.0.0.0', + PXPIPE_LOG: path.join(dir, 'events.jsonl'), + PXPIPE_CONFIG: path.join(dir, 'config.json'), + OPENAI_API_KEY: 'server-owned-test-key', + PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS: '', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout?.on('data', (b) => output.push(String(b))); + child.stderr?.on('data', (b) => output.push(String(b))); + const exitCode = await new Promise((resolve) => child!.once('exit', resolve)); + + expect(exitCode).toBe(1); + expect(output.join('')).toContain('refusing non-loopback HOST=0.0.0.0'); + expect(output.join('')).toContain('OPENAI_API_KEY'); + }); + + it('does not mistake a 127-prefixed DNS hostname for a loopback IP literal', async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-node-security-')); + const port = await freePort(); + const output: string[] = []; + child = spawn(process.execPath, [tsxCli, 'src/node.ts'], { + cwd: repoRoot, + env: { + ...process.env, + PORT: String(port), + HOST: '127.example.com', + PXPIPE_LOG: path.join(dir, 'events.jsonl'), + PXPIPE_CONFIG: path.join(dir, 'config.json'), + OPENAI_API_KEY: 'server-owned-test-key', + PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS: '', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout?.on('data', (b) => output.push(String(b))); + child.stderr?.on('data', (b) => output.push(String(b))); + const exitCode = await new Promise((resolve) => child!.once('exit', resolve)); + + expect(exitCode).toBe(1); + expect(output.join('')).toContain('refusing non-loopback HOST=127.example.com'); + expect(output.join('')).toContain('OPENAI_API_KEY'); + }); + + it('allows the explicit loopback-published Docker credential boundary', async () => { + const { base } = await startNode({ + HOST: '0.0.0.0', + OPENAI_API_KEY: 'server-owned-test-key', + PXPIPE_ALLOW_NON_LOOPBACK_CREDENTIALS: '1', + }); + const response = await fetch(`${base}/healthz`); + expect(response.status).toBe(200); + }); + it('rejects cross-origin mutations and accepts same-origin mutations', async () => { const { base, configFile } = await startNode(); const denied = await fetch(`${base}/fragments/models`, { @@ -124,8 +195,8 @@ describe('Node dashboard security', () => { body: 'list=claude-fable-5', }); expect(allowed.status).toBe(200); - expect(fs.statSync(configFile).mode & 0o777).toBe(0o600); - expect(fs.statSync(path.dirname(configFile)).mode & 0o777).toBe(0o700); + expectPrivateFile(configFile); + expectPrivateDirectory(path.dirname(configFile)); }); it('rejects dashboard requests with a non-loopback Host header', async () => { @@ -143,6 +214,25 @@ describe('Node dashboard security', () => { expect(response.status).toBe(403); }); + it('resets the persisted dashboard model override locally', async () => { + const { base, eventsFile } = await startNode(); + const turnOff = await fetch(`${base}/fragments/models`, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: 'list=off', + }); + expect(turnOff.status).toBe(200); + const scopeFile = path.join(path.dirname(eventsFile), 'model-scope.json'); + expect(fs.existsSync(scopeFile)).toBe(true); + const reset = await fetch(`${base}/fragments/models/reset`, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + }); + expect(reset.status).toBe(200); + expect(await reset.text()).toContain('Fable 5 ✓'); + expect(fs.existsSync(scopeFile)).toBe(false); + }); + it('creates the event log and containing directory with private permissions', async () => { const { base, eventsFile } = await startNode(); await fetch(`${base}/v1/messages`, { @@ -157,8 +247,8 @@ describe('Node dashboard security', () => { for (let i = 0; i < 100 && !fs.existsSync(eventsFile); i++) { await new Promise((resolve) => setTimeout(resolve, 10)); } - expect(fs.statSync(eventsFile).mode & 0o777).toBe(0o600); - expect(fs.statSync(path.dirname(eventsFile)).mode & 0o777).toBe(0o700); + expectPrivateFile(eventsFile); + expectPrivateDirectory(path.dirname(eventsFile)); }); it('creates rendered PNG dumps with private permissions', async () => { @@ -179,9 +269,9 @@ describe('Node dashboard security', () => { files = fs.readdirSync(dumpDir); if (files.length === 0) await new Promise((resolve) => setTimeout(resolve, 10)); } - expect(fs.statSync(dumpDir).mode & 0o777).toBe(0o700); + expectPrivateDirectory(dumpDir); expect(files.length).toBeGreaterThan(0); - expect(fs.statSync(path.join(dumpDir, files[0]!)).mode & 0o777).toBe(0o600); + expectPrivateFile(path.join(dumpDir, files[0]!)); fs.rmSync(dumpDir, { recursive: true, force: true }); }); }); diff --git a/tests/proxy-usage.test.ts b/tests/proxy-usage.test.ts index 29cff284d..347bf64a0 100644 --- a/tests/proxy-usage.test.ts +++ b/tests/proxy-usage.test.ts @@ -1326,6 +1326,402 @@ describe('proxy usage extraction', () => { expect(captured!.usage?.cache_creation_input_tokens).toBe(5000); }); + it('extracts Codex Responses usage when SSE omits the event: line', async () => { + // ChatGPT's /backend-api/codex/responses identifies events with the JSON + // `type` discriminator. Codex itself reads this shape; requiring a separate + // SSE `event:` line made every real Codex row lose its provider usage. + const sseBody = + `data: ${JSON.stringify({ type: 'response.output_text.delta', delta: 'hello' })}\n\n` + + `data: ${JSON.stringify({ type: 'response.reasoning_summary_text.delta', delta: 'think' })}\n\n` + + `data: ${JSON.stringify({ type: 'response.function_call_arguments.delta', delta: '{"path":"x"}' })}\n\n` + + `data: ${JSON.stringify({ + type: 'response.completed', + response: { + id: 'resp_codex_1', + usage: { + input_tokens: 1200, + input_tokens_details: { cached_tokens: 900, cache_write_tokens: 125 }, + output_tokens: 80, + output_tokens_details: { reasoning_tokens: 50 }, + total_tokens: 1280, + }, + }, + })}\n\n`; + + const restore = mockUpstream(() => { + const response = new Response(sseBody, { status: 200 }); + // Response(string) adds text/plain automatically; remove it to mirror + // ChatGPT Codex, whose SSE response currently has no Content-Type. + response.headers.delete('content-type'); + return response; + }); + + let captured: ProxyEvent | undefined; + const proxy = createProxy({ + openAIUpstream: 'https://chatgpt.test', + transform: { minCompressChars: 1_000_000 }, + onRequest: (e) => { captured = e; }, + }); + const body = JSON.stringify({ + model: 'gpt-5.6-sol', + input: [{ role: 'user', content: 'hi' }], + stream: true, + }); + const res = await proxy(new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body, + })); + await res.text(); + await new Promise((r) => setTimeout(r, 20)); + restore(); + + expect(captured?.usage).toEqual({ + input_tokens: 1200, + output_tokens: 80, + cached_tokens: 900, + cache_write_tokens: 125, + reasoning_output_tokens: 50, + }); + expect(captured?.measurement).toEqual({ + textChars: 5, + thinkingChars: 5, + toolUseChars: 12, + redactedBlockCount: 0, + }); + expect(captured?.stopReason).toBe('stop'); + }); + + it('preserves Codex ChatGPT OAuth when OPENAI_API_KEY is configured', async () => { + let upstreamAuth: string | null = null; + const restore = mockUpstream((req) => { + upstreamAuth = req.headers.get('authorization'); + return new Response(JSON.stringify({ id: 'resp_codex_auth', output: [] }), { + headers: { 'content-type': 'application/json' }, + }); + }); + const proxy = createProxy({ + openAIUpstream: 'https://chatgpt.test', + openAIApiKey: 'sk-openai-server-key', + transform: { compress: false }, + }); + + const res = await proxy(new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', + headers: { + authorization: 'Bearer chatgpt-oauth-token', + 'content-type': 'application/json', + }, + body: JSON.stringify({ model: 'gpt-5.6-sol', input: 'hi' }), + })); + await res.text(); + restore(); + + expect(upstreamAuth).toBe('Bearer chatgpt-oauth-token'); + }); + + it('uses the configured OpenAI key on Codex paths when inbound auth is absent', async () => { + let upstreamAuth: string | null = null; + const restore = mockUpstream((req) => { + upstreamAuth = req.headers.get('authorization'); + return new Response(JSON.stringify({ id: 'resp_codex_key', output: [] }), { + headers: { 'content-type': 'application/json' }, + }); + }); + const proxy = createProxy({ + openAIUpstream: 'https://chatgpt.test', + openAIApiKey: 'sk-openai-server-key', + transform: { compress: false }, + }); + + const res = await proxy(new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-5.6-sol', input: 'hi' }), + })); + await res.text(); + restore(); + + expect(upstreamAuth).toBe('Bearer sk-openai-server-key'); + }); + + it('does not forward Anthropic OAuth to the Codex OpenAI upstream', async () => { + let upstreamAuth: string | null = null; + const restore = mockUpstream((req) => { + upstreamAuth = req.headers.get('authorization'); + return new Response(JSON.stringify({ id: 'resp_codex_auth', output: [] }), { + headers: { 'content-type': 'application/json' }, + }); + }); + const proxy = createProxy({ openAIUpstream: 'https://chatgpt.test', openAIApiKey: 'sk-server' }); + await (await proxy(new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', + headers: { authorization: 'Bearer sk-ant-oat01-secret', 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-5.6-sol', input: 'hi' }), + }))).text(); + restore(); + expect(upstreamAuth).toBe('Bearer sk-server'); + }); + + it('rejects an oversized chunked transform request before calling upstream', async () => { + let upstreamCalls = 0; + const restore = mockUpstream(() => { + upstreamCalls++; + return new Response('{}', { headers: { 'content-type': 'application/json' } }); + }); + let captured: ProxyEvent | undefined; + const proxy = createProxy({ + openAIUpstream: 'https://openai.test', + onRequest: (event) => { captured = event; }, + }); + const eightMiB = new Uint8Array(8 * 1024 * 1024); + const body = new ReadableStream({ + start(controller) { + // Reuse the allocation: the stream contains 16 MiB + 1 byte but does not + // need a second 8 MiB test fixture in memory. + controller.enqueue(eightMiB); + controller.enqueue(eightMiB); + controller.enqueue(new Uint8Array(1)); + controller.close(); + }, + }); + const request = new Request('http://localhost/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body, + duplex: 'half', + } as RequestInit & { duplex: 'half' }); + expect(request.headers.has('content-length')).toBe(false); + + const res = await proxy(request); + const payload = await res.json() as { error?: { type?: string; message?: string } }; + await new Promise((resolve) => setTimeout(resolve, 20)); + restore(); + + expect(res.status).toBe(413); + expect(payload.error?.type).toBe('request_too_large'); + expect(payload.error?.message).toContain('16777216 bytes'); + expect(upstreamCalls).toBe(0); + expect(captured?.status).toBe(413); + }); + + it('streams an oversized chunked bypass body without buffering it for model sniffing', async () => { + let releaseTail!: () => void; + const tailReady = new Promise((resolve) => { releaseTail = resolve; }); + let markUpstreamStarted!: () => void; + const upstreamStarted = new Promise((resolve) => { markUpstreamStarted = resolve; }); + let upstreamBytes = 0; + const restore = mockUpstream(async (req) => { + markUpstreamStarted(); + upstreamBytes = (await req.arrayBuffer()).byteLength; + return new Response('{}', { headers: { 'content-type': 'application/json' } }); + }); + const proxy = createProxy({ openAIUpstream: 'https://openai.test' }); + const prefix = new Uint8Array(600 * 1024); + const tail = new Uint8Array(5 * 1024 * 1024); + let step = 0; + const body = new ReadableStream({ + async pull(controller) { + if (step < 2) { + step++; + controller.enqueue(prefix); + return; + } + if (step === 2) await tailReady; + if (step < 5) { + step++; + controller.enqueue(tail); + return; + } + controller.close(); + }, + }); + const request = new Request('http://localhost/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-pxpipe-bypass': '1' }, + body, + duplex: 'half', + } as RequestInit & { duplex: 'half' }); + expect(request.headers.has('content-length')).toBe(false); + + const responsePromise = proxy(request); + const startedBeforeTail = await Promise.race([ + upstreamStarted.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 500)), + ]); + releaseTail(); + const response = await responsePromise; + await response.text(); + restore(); + + expect(startedBeforeTail).toBe(true); + expect(response.status).toBe(200); + expect(upstreamBytes).toBe((2 * prefix.byteLength) + (3 * tail.byteLength)); + expect(upstreamBytes).toBeGreaterThan(16 * 1024 * 1024); + }); + + it('keeps bypassed Codex Responses on OpenAI accounting and scans headerless SSE', async () => { + const sse = `data: ${JSON.stringify({ + type: 'response.completed', response: { usage: { input_tokens: 21, output_tokens: 3 } }, + })}\n\n`; + const restore = mockUpstream(() => { + const response = new Response(sse, { status: 200 }); + response.headers.delete('content-type'); + return response; + }); + let captured: ProxyEvent | undefined; + const proxy = createProxy({ + openAIUpstream: 'https://chatgpt.test', + onRequest: (event) => { captured = event; }, + }); + const response = await proxy(new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-pxpipe-bypass': '1' }, + body: JSON.stringify({ model: 'gpt-5.6-sol', input: 'hi', stream: true }), + })); + await response.text(); + await new Promise((resolve) => setTimeout(resolve, 20)); + restore(); + + expect(captured?.accountingProvider).toBe('openai'); + expect(captured?.usage).toMatchObject({ input_tokens: 21, output_tokens: 3 }); + }); + + it('scans a bypassed headerless non-stream Codex JSON response', async () => { + const restore = mockUpstream(() => { + const response = new Response(JSON.stringify({ + id: 'resp_bypass_json', status: 'completed', usage: { input_tokens: 34, output_tokens: 5 }, output: [], + }), { status: 200 }); + response.headers.delete('content-type'); + return response; + }); + let captured: ProxyEvent | undefined; + const proxy = createProxy({ openAIUpstream: 'https://chatgpt.test', onRequest: (event) => { captured = event; } }); + await (await proxy(new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-pxpipe-bypass': '1' }, + body: JSON.stringify({ model: 'gpt-5.6-sol', input: 'hi', stream: false }), + }))).text(); + await new Promise((resolve) => setTimeout(resolve, 20)); + restore(); + + expect(captured?.usage).toMatchObject({ input_tokens: 34, output_tokens: 5 }); + }); + + it('marks a Responses refusal instead of overwriting it with a successful stop', async () => { + const sseBody = + `data: ${JSON.stringify({ type: 'response.refusal.delta', delta: 'cannot comply' })}\n\n` + + `data: ${JSON.stringify({ + type: 'response.completed', + response: { usage: { input_tokens: 50, output_tokens: 3 } }, + })}\n\n`; + const restore = mockUpstream(() => { + const response = new Response(sseBody, { status: 200 }); + response.headers.delete('content-type'); + return response; + }); + let captured: ProxyEvent | undefined; + const proxy = createProxy({ + openAIUpstream: 'https://chatgpt.test', + transform: { minCompressChars: 1_000_000 }, + onRequest: (e) => { captured = e; }, + }); + const res = await proxy(new Request('http://localhost/backend-api/codex/responses', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-5.6-sol', input: 'hi', stream: true }), + })); + await res.text(); + await new Promise((r) => setTimeout(r, 20)); + restore(); + expect(captured?.stopReason).toBe('refusal'); + expect(captured?.measurement?.textChars).toBe(13); + }); + + it('keeps usage from a response.failed terminal event', async () => { + const sseBody = `data: ${JSON.stringify({ + type: 'response.failed', + response: { + error: { code: 'server_error' }, + usage: { input_tokens: 75, output_tokens: 4 }, + }, + })}\n\n`; + const restore = mockUpstream(() => new Response(sseBody, { + status: 200, headers: { 'content-type': 'text/event-stream' }, + })); + let captured: ProxyEvent | undefined; + const proxy = createProxy({ + openAIUpstream: 'https://chatgpt.test', + transform: { minCompressChars: 1_000_000 }, + onRequest: (e) => { captured = e; }, + }); + const res = await proxy(new Request('http://localhost/v1/responses', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-5.6-sol', input: 'hi', stream: true }), + })); + await res.text(); + await new Promise((r) => setTimeout(r, 20)); + restore(); + expect(captured?.usage?.input_tokens).toBe(75); + expect(captured?.stopReason).toBe('server_error'); + }); + + it('extracts cache and reasoning details from Chat Completions usage aliases', async () => { + const restore = mockUpstream(() => new Response(JSON.stringify({ + choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 90, + completion_tokens: 10, + prompt_tokens_details: { cached_tokens: 60, cache_write_tokens: 15 }, + completion_tokens_details: { reasoning_tokens: 7 }, + }, + }), { status: 200, headers: { 'content-type': 'application/json' } })); + let captured: ProxyEvent | undefined; + const proxy = createProxy({ + openAIUpstream: 'https://openai.test', + transform: { minCompressChars: 1_000_000 }, + onRequest: (e) => { captured = e; }, + }); + const res = await proxy(new Request('http://localhost/v1/chat/completions', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-5.6', messages: [{ role: 'user', content: 'hi' }] }), + })); + await res.text(); + await new Promise((r) => setTimeout(r, 20)); + restore(); + expect(captured?.usage).toEqual({ + input_tokens: 90, + output_tokens: 10, + cached_tokens: 60, + cache_write_tokens: 15, + reasoning_output_tokens: 7, + }); + }); + + it('parses headerless non-streaming Responses JSON as JSON, not SSE', async () => { + const restore = mockUpstream(() => { + const response = new Response(JSON.stringify({ + id: 'resp_json', status: 'completed', output: [], + usage: { input_tokens: 42, output_tokens: 8 }, + }), { status: 200 }); + response.headers.delete('content-type'); + return response; + }); + let captured: ProxyEvent | undefined; + const proxy = createProxy({ + openAIUpstream: 'https://openai.test', + transform: { minCompressChars: 1_000_000 }, + onRequest: (e) => { captured = e; }, + }); + const res = await proxy(new Request('http://localhost/v1/responses', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-5.6', input: 'hi', stream: false }), + })); + await res.text(); + await new Promise((r) => setTimeout(r, 20)); + restore(); + expect(captured?.usage).toMatchObject({ input_tokens: 42, output_tokens: 8 }); + }); + it('fires the event with undefined usage when the response is an error', async () => { const restore = mockUpstream( () => diff --git a/tests/reflow.test.ts b/tests/reflow.test.ts index c98f1b1a9..a1cb36f13 100644 --- a/tests/reflow.test.ts +++ b/tests/reflow.test.ts @@ -643,10 +643,18 @@ describe('reflow L0 contract – real corpus', () => { let textsChecked = 0; let violations: string[] = []; + // Local error captures can be arbitrarily large. This is a sampled + // regression check, so avoid making a developer's retained diagnostics + // determine the test runtime. + const MAX_FILE_BYTES = 1 * 1024 * 1024; + const MAX_TEXT_CHARS = 200_000; + for (const fname of files.slice(0, 50)) { let raw: string; try { - raw = readFileSync(join(dir4xx, fname), 'utf-8'); + const filePath = join(dir4xx, fname); + if (statSync(filePath).size > MAX_FILE_BYTES) continue; + raw = readFileSync(filePath, 'utf-8'); } catch { continue; } @@ -670,6 +678,7 @@ describe('reflow L0 contract – real corpus', () => { for (const parsed of candidates) { for (const text of extractTexts(parsed)) { if (textsChecked >= 500) break; + if (text.length > MAX_TEXT_CHARS) continue; const result = reflow(text); if (result === null) { if (text.indexOf(NL_SENTINEL) < 0) { diff --git a/tests/render.test.ts b/tests/render.test.ts index ba4cd5c0b..2c757ef5d 100644 --- a/tests/render.test.ts +++ b/tests/render.test.ts @@ -661,6 +661,35 @@ describe('transform', () => { expect(refHeader).toContain("this user's local proxy"); }); + it('preserves provider-typed tools that reject custom description fields', async () => { + const advisor = { + type: 'advisor_20260301', + name: 'advisor', + container: { id: 'preserve-this-wire-shape' }, + }; + const req = JSON.stringify({ + model: 'claude-fable-5', + messages: [{ role: 'user', content: 'hi' }], + system: 'x'.repeat(30000), + tools: [ + advisor, + { + name: 'CustomTool', + description: 'A custom tool description. '.repeat(500), + input_schema: { type: 'object', properties: { x: { type: 'string' } } }, + }, + ], + }); + + const { body, info } = await transformRequest(new TextEncoder().encode(req)); + const out = JSON.parse(new TextDecoder().decode(body)); + + expect(out.tools[0]).toEqual(advisor); + expect(out.tools[1].description).toContain('Tool Reference'); + expect(info.imageSourceText).not.toContain('## Tool: advisor'); + expect(info.imageSourceText).toContain('## Tool: CustomTool'); + }); + it('ships annotation-stripped schemas in tools[], full schema in the imaged reference', async () => { // History: a bare `{type:'object'}` stub caused validator 400s; a text // reference paid the annotations at text rates. Current contract: tools[] diff --git a/tests/sessions.test.ts b/tests/sessions.test.ts index 97efebc36..f620e77b4 100644 --- a/tests/sessions.test.ts +++ b/tests/sessions.test.ts @@ -5,9 +5,11 @@ import * as os from 'node:os'; import { aggregateSessions, filterSessions, + prune, type SessionsPaths, } from '../src/sessions.js'; import type { TrackEvent } from '../src/core/tracker.js'; +import { acquireEventsFileLock } from '../src/events-lock.js'; // ---- Test scaffolding ------------------------------------------------------ @@ -76,6 +78,69 @@ describe('aggregateSessions', () => { expect(sessions.get('bbbbbbbb')?.requestCount).toBe(1); }); + it('does not merge the same first-user fingerprint across different projects', async () => { + writeEvents(tmp, [ + ev({ first_user_sha8: 'samehash', cwd: '/project/a' }), + ev({ first_user_sha8: 'samehash', cwd: '/project/b' }), + ]); + const { sessions } = await aggregateSessions(tmp); + expect(sessions.size).toBe(2); + expect(sessions.get('samehash')?.project).toBe('/project/a'); + const second = [...sessions.values()].find((s) => s.project === '/project/b'); + expect(second?.id).toMatch(/^samehash@[0-9a-f]{8}$/); + expect(second?.requestCount).toBe(1); + }); + + it('uses provider-specific GPT and Google savings math in disk session rollups', async () => { + writeEvents(tmp, [ + ev({ + path: '/v1/responses', accounting_provider: 'openai', model: 'gpt-5.6-sol', + first_user_sha8: 'gpt', compressed: true, input_tokens: 100, + output_tokens: 1, cached_tokens: 0, image_tokens: 20, + baseline_imaged_tokens: 200, + }), + ev({ + path: '/google-ai-studio/v1beta/models/gemini-3.6-flash:generateContent', + accounting_provider: 'google', first_user_sha8: 'google', compressed: true, + input_tokens: 120, output_tokens: 10, baseline_tokens: 400, + baseline_probe_status: 'ok', cached_tokens: 30, + }), + ]); + const { sessions } = await aggregateSessions(tmp); + expect(sessions.get('gpt')?.tokensSavedEst).toBe(180); + expect(sessions.get('google')?.tokensSavedEst).toBe(280); + expect(sessions.get('google')?.cacheReadTokens).toBe(30); + }); + + it('recognizes a live legacy PID writer marker before pruning', async () => { + writeEvents(tmp, [ev({ first_user_sha8: 'keepme' })]); + fs.writeFileSync(tmp.eventsFile + '.writer.lock', `${process.pid}\n`); + await expect(prune(tmp, { sessionId: 'keepme', force: true })).rejects.toThrow( + `owned by writer pid ${process.pid}`, + ); + expect(fs.readFileSync(tmp.eventsFile, 'utf8')).toContain('keepme'); + }); + + it('never deletes a sidecar path outside the configured sidecar directory', async () => { + const outside = path.join(path.dirname(tmp.eventsFile), 'must-survive.json.gz'); + fs.writeFileSync(outside, 'important'); + writeEvents(tmp, [ev({ first_user_sha8: 'victim', req_body_sample_path: outside })]); + await prune(tmp, { sessionId: 'victim', force: true }); + expect(fs.existsSync(outside)).toBe(true); + }); + + it('serializes writers and prune with one exclusive lock', () => { + const writer = acquireEventsFileLock(tmp.eventsFile, 'writer'); + expect(() => acquireEventsFileLock(tmp.eventsFile, 'writer')).toThrow('owned'); + expect(() => acquireEventsFileLock(tmp.eventsFile, 'prune')).toThrow('owned'); + writer.release(); + const pruneOwner = acquireEventsFileLock(tmp.eventsFile, 'prune'); + expect(() => acquireEventsFileLock(tmp.eventsFile, 'writer')).toThrow('owned'); + pruneOwner.release(); + const nextWriter = acquireEventsFileLock(tmp.eventsFile, 'writer'); + nextWriter.release(); + }); + it('uses earliest ts for firstSeen and latest for lastSeen even when input is unordered', async () => { writeEvents(tmp, [ ev({ ts: '2026-05-18T00:00:05Z', first_user_sha8: 'aaaaaaaa' }), @@ -106,6 +171,19 @@ describe('aggregateSessions', () => { expect(sidecarsBySession.get('aaaaaaaa')?.has(sidecar)).toBe(true); }); + it('counts a repeated sidecar once and preserves it while a kept session references it', async () => { + const sidecar = writeSidecar(tmp, 'shared.json.gz', 512); + writeEvents(tmp, [ + ev({ first_user_sha8: 'remove', req_body_sample_path: sidecar }), + ev({ first_user_sha8: 'remove', req_body_sample_path: sidecar }), + ev({ first_user_sha8: 'keep', req_body_sample_path: sidecar }), + ]); + const { sessions } = await aggregateSessions(tmp); + expect(sessions.get('remove')?.sidecarBytes).toBe(512); + await prune(tmp, { sessionId: 'remove', force: true }); + expect(fs.existsSync(sidecar)).toBe(true); + }); + it('returns empty when events.jsonl is missing', async () => { const missing: SessionsPaths = { eventsFile: path.join(path.dirname(tmp.eventsFile), 'nope.jsonl'), @@ -129,6 +207,15 @@ describe('aggregateSessions', () => { expect(sessions.get('aaaaaaaa')?.requestCount).toBe(2); }); + it('drops schema-invalid JSON values without aborting aggregation', async () => { + fs.writeFileSync( + tmp.eventsFile, + '{}\n' + JSON.stringify(ev({ first_user_sha8: 'validrow' })) + '\n', + ); + const { sessions } = await aggregateSessions(tmp); + expect([...sessions.keys()]).toEqual(['validrow']); + }); + it('credits the real prefix compression (image prefix fewer tokens than text prefix)', async () => { writeEvents(tmp, [ // First TRACKED turn, but cr=100 > 0 ⇒ the cache was OBSERVABLY warm (pxpipe diff --git a/tests/tracker.test.ts b/tests/tracker.test.ts index f533fc2f2..aedb44184 100644 --- a/tests/tracker.test.ts +++ b/tests/tracker.test.ts @@ -49,6 +49,8 @@ describe('toTrackEvent', () => { output_tokens: 7, cache_creation_input_tokens: 0, cache_read_input_tokens: 100, + cached_tokens: 30, + cache_write_tokens: 12, }, }; const out = toTrackEvent(ev); @@ -74,6 +76,8 @@ describe('toTrackEvent', () => { expect(out.input_tokens).toBe(42); expect(out.cache_read_tokens).toBe(100); expect(out.cache_create_tokens).toBe(0); + expect(out.cached_tokens).toBe(30); + expect(out.cache_write_tokens).toBe(12); // ts is ISO8601 expect(out.ts).toMatch(/^\d{4}-\d{2}-\d{2}T/); });