diff --git a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts index aa483fc3..2564d33e 100644 --- a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts +++ b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts @@ -38,6 +38,8 @@ const getCachedDerivedAgenticMetrics = cachedQuery( * - p75/p90_e2e_norm_intvty: slow-tail OSL / E2E-latency in tok/s/user, * computed as 1 / pXX(per-request E2EL/OSL) across every turn in every * session — plain interactivity charged for the prefill wait. + * - request_length_moments: exact joint (ISL, OSL) moment sums over profiling + * requests, used client-side for closed-form attention-FLOP pricing. * * Ids without a trace_replay blob or with unparseable records are omitted. */ diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index e293ca1c..796c45aa 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -18,7 +18,10 @@ import { resolveComparisonEntries, } from '@/components/inference/utils/comparisonEntry'; import { useBenchmarks, benchmarkQueryOptions } from '@/hooks/api/use-benchmarks'; +import { useDerivedAgenticMetrics } from '@/hooks/api/use-derived-agentic-metrics'; import type { BenchmarkRow } from '@/lib/api'; +import type { RequestLengthMoments } from '@/lib/attention-flops'; +import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { GPU_ALIAS_TO_CANONICAL, getModelSortIndex, @@ -384,6 +387,31 @@ export function useChartData( overviewHistoryPair?.baselineConfigKey, ]); + // Request-length moment sums for agentic points — the attention-FLOPs input + // of the TFLOP/s-per-chip y-metric. Shares the 'derived-agentic-metrics' + // react-query cache with ChartDisplay's derived-x fetch, so switching between + // the two costs one request. The chart renders without them first; the + // TFLOP/s metric fills in when they arrive. + const momentTargetIds = useMemo(() => { + const ids = new Set(); + for (const r of rows) { + const numericId = typeof r.id === 'number' ? r.id : Number(r.id); + if (r.benchmark_type === 'agentic_traces' && isPersistedBenchmarkId(numericId)) { + ids.add(numericId); + } + } + return [...ids].sort((a, b) => a - b); + }, [rows]); + const derivedAgenticQuery = useDerivedAgenticMetrics(momentTargetIds, momentTargetIds.length > 0); + const requestLengthMomentsById = useMemo(() => { + if (!derivedAgenticQuery.data) return undefined; + const map: Record = {}; + for (const metric of Object.values(derivedAgenticQuery.data)) { + map[metric.id] = metric.request_length_moments; + } + return map; + }, [derivedAgenticQuery.data]); + // Transform filtered rows into chart data const { chartData, hardwareConfig: rawHardwareConfig } = useMemo(() => { if (rows.length === 0) @@ -391,8 +419,8 @@ export function useChartData( chartData: [] as InferenceData[][], hardwareConfig: {} as HardwareConfig, }; - return transformBenchmarkRows(rows, selectedPercentile); - }, [rows, selectedPercentile]); + return transformBenchmarkRows(rows, selectedPercentile, requestLengthMomentsById); + }, [rows, selectedPercentile, requestLengthMomentsById]); // Sort hardware config — stabilize reference when keys haven't changed. // Different sequences for the same model often have the same GPU configs, diff --git a/packages/app/src/components/inference/metric-registry.ts b/packages/app/src/components/inference/metric-registry.ts index 046eb0d3..6701385c 100644 --- a/packages/app/src/components/inference/metric-registry.ts +++ b/packages/app/src/components/inference/metric-registry.ts @@ -43,6 +43,48 @@ export const METRIC_REGISTRY = { titleZh: '每芯片输出 token 吞吐量', polarity: 'higher', }, + // New-input-suffix throughput pair: total / input throughput minus the + // infinite-cache theoretical prefix share (trace-derived, agentic only). + // Deliberately based on the theoretical rate rather than server-observed + // cache hits so systems with good cache storage aren't penalized — like + // MFU vs HFU, actual (uncached + output) throughput is always at least the + // theoretical (uncachable suffix + output) throughput. + newInputSuffixOutputTputPerGpu: { + field: 'newInputSuffixOutputTputPerGpu.y', + label: 'New Input Suffix + Output Token Throughput per Chip (tok/s/chip)', + labelZh: '每芯片新输入 suffix + 输出 token 吞吐量(tok/s/chip)', + title: 'New Input Suffix + Output Token Throughput per Chip', + titleZh: '每芯片新输入 suffix + 输出 token 吞吐量', + polarity: 'higher', + }, + newInputSuffixTputPerGpu: { + field: 'newInputSuffixTputPerGpu.y', + label: 'New Input Suffix Token Throughput per Chip (tok/s/chip)', + labelZh: '每芯片新输入 suffix token 吞吐量(tok/s/chip)', + title: 'New Input Suffix Token Throughput per Chip', + titleZh: '每芯片新输入 suffix token 吞吐量', + polarity: 'higher', + x: 'p90_ttft', + xLabel: 'P90 Time To First Token (s)', + heading: 'vs. P90 Time To First Token', + }, + // Achieved model TFLOP/s on the theoretically necessary tokens only: + // (2 × active params + per-model attention FLOPs per computed token) times + // the new-input-suffix + output token throughput above. The attention term + // integrates each architecture's mechanism (MLA, CSA/HCA + indexers, MSA, + // KDA + gated MLA, sliding/full GQA — specs in model-architectures.ts, + // math in attention-flops.ts) over the run's exact per-request (ISL, OSL) + // sums at the theoretical cache hit rate, so different request-length + // distributions are priced individually and summed. Points without stored + // request-length moments or an attention spec omit the metric. + newInputSuffixOutputTflopsPerGpu: { + field: 'newInputSuffixOutputTflopsPerGpu.y', + label: 'New Input Suffix + Output TFLOP/s per Chip (TFLOP/s/chip)', + labelZh: '每芯片新输入 suffix + 输出 TFLOP/s(TFLOP/s/chip)', + title: 'New Input Suffix + Output TFLOP/s per Chip', + titleZh: '每芯片新输入 suffix + 输出 TFLOP/s', + polarity: 'higher', + }, tpPerMw: { field: 'tpPerMw.y', label: 'Token Throughput per All in Utility MW (tok/s/MW)', @@ -473,6 +515,9 @@ export const METRIC_CONTROL_GROUPS: readonly MetricControlGroup[] = [ 'y_tpPerGpu', 'y_inputTputPerGpu', 'y_outputTputPerGpu', + 'y_newInputSuffixOutputTputPerGpu', + 'y_newInputSuffixTputPerGpu', + 'y_newInputSuffixOutputTflopsPerGpu', 'y_tpPerMw', 'y_inputTputPerMw', 'y_outputTputPerMw', diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 124483cc..b8d6000f 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -1,6 +1,7 @@ import type React from 'react'; import type { WorkerPower } from '@semianalysisai/inferencex-db/queries/benchmarks'; +import type { RequestLengthMoments } from '@/lib/attention-flops'; import type { HardwareEntry } from '@/lib/constants'; import type { Model, Sequence } from '@/lib/data-mappings'; import type { MetricKey } from './metric-registry'; @@ -219,6 +220,14 @@ export interface AggDataEntry { total_prompt_tokens?: number; /** Total generated (output) tokens. */ total_generation_tokens?: number; + /** + * Exact joint (ISL, OSL) sums over the run's request population, fetched + * from the derived-agentic-metrics endpoint and merged in by + * `transformBenchmarkRows`. Feeds the attention-FLOPs term of the + * TFLOP/s-per-chip y-metric; absent on fixed-seq points, unofficial-run + * overlays, and rows whose stats haven't been fetched yet. + */ + request_length_moments?: RequestLengthMoments | null; } /** @@ -274,6 +283,12 @@ export interface InferenceData extends Partial { it('computes the true normalized-interactivity frontier', () => { const points = [point(1, 100), point(2, 150), point(3, 120)]; const metrics = { - 1: { id: 1, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10 }, - 2: { id: 2, p75_e2e_norm_intvty: 20, p90_e2e_norm_intvty: 20 }, - 3: { id: 3, p75_e2e_norm_intvty: 30, p90_e2e_norm_intvty: 30 }, + 1: { id: 1, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10, request_length_moments: null }, + 2: { id: 2, p75_e2e_norm_intvty: 20, p90_e2e_norm_intvty: 20, request_length_moments: null }, + 3: { id: 3, p75_e2e_norm_intvty: 30, p90_e2e_norm_intvty: 30, request_length_moments: null }, }; expect( @@ -38,8 +38,8 @@ describe('canonicalNormalizedFrontierIds', () => { it('keeps frontiers independent across dates', () => { const points = [point(1, 500, { date: '2026-08-01' }), point(2, 100, { date: '2026-08-02' })]; const metrics = { - 1: { id: 1, p75_e2e_norm_intvty: 50, p90_e2e_norm_intvty: 50 }, - 2: { id: 2, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10 }, + 1: { id: 1, p75_e2e_norm_intvty: 50, p90_e2e_norm_intvty: 50, request_length_moments: null }, + 2: { id: 2, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10, request_length_moments: null }, }; expect( [...canonicalNormalizedFrontierIds(points, metrics, 'p90', 'upper_left')!].toSorted(), diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index cd469a0c..47a982b2 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -6,6 +6,8 @@ import { generateTooltipContent, generateOverlayTooltipContent, generateGPUGraphTooltipContent, + theoreticalPrefixTokens, + uncachedInputTokens, type TooltipConfig, type OverlayTooltipConfig, } from '@/components/inference/utils/tooltipUtils'; @@ -332,6 +334,98 @@ describe('generateTooltipContent', () => { expect(html).toContain('FP8'); }); + it('shows theoretical prefix and uncached input token rows for agentic points', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: pt({ + benchmark_type: 'agentic_traces', + total_prompt_tokens: 1_000_000, + theoretical_cache_hit_rate: 0.92, + }), + }), + ); + expect(html).toContain('Theoretical Prefix Tokens: 920,000'); + expect(html).toContain('Input Tokens w/o Prefix Caching: 80,000'); + }); + + it('omits the prefix token rows when the theoretical hit rate is missing', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: pt({ + benchmark_type: 'agentic_traces', + total_prompt_tokens: 1_000_000, + theoretical_cache_hit_rate: undefined, + }), + }), + ); + expect(html).toContain('Prompt Tokens: 1,000,000'); + expect(html).not.toContain('Theoretical Prefix Tokens'); + expect(html).not.toContain('Input Tokens w/o Prefix Caching'); + }); + + it('never shows prefix token rows on fixed-sequence points', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: pt({ + benchmark_type: 'single_turn', + total_prompt_tokens: 1_000_000, + theoretical_cache_hit_rate: 0.92, + }), + }), + ); + expect(html).not.toContain('Theoretical Prefix Tokens'); + expect(html).not.toContain('Input Tokens w/o Prefix Caching'); + }); + + it('uses Chinese labels for the prefix token rows on /zh surfaces', () => { + const html = generateTooltipContent( + tooltipConfig({ + locale: 'zh', + data: pt({ + benchmark_type: 'agentic_traces', + total_prompt_tokens: 500_000, + theoretical_cache_hit_rate: 0.9, + }), + }), + ); + expect(html).toContain('理论 prefix token 数: 450,000'); + expect(html).toContain('无 prefix cache 的输入 token 数: 50,000'); + }); +}); + +describe('theoreticalPrefixTokens / uncachedInputTokens', () => { + it('recovers the prefix token sum from the trace hit rate', () => { + const d = pt({ total_prompt_tokens: 1_000_000, theoretical_cache_hit_rate: 0.925 }); + expect(theoreticalPrefixTokens(d)).toBe(925_000); + expect(uncachedInputTokens(d)).toBe(75_000); + }); + + it('returns undefined when either input is missing', () => { + expect(theoreticalPrefixTokens(pt({ total_prompt_tokens: 100 }))).toBeUndefined(); + expect(theoreticalPrefixTokens(pt({ theoretical_cache_hit_rate: 0.5 }))).toBeUndefined(); + expect(uncachedInputTokens(pt({ total_prompt_tokens: 100 }))).toBeUndefined(); + }); + + it('rejects out-of-range or NaN hit rates', () => { + expect( + theoreticalPrefixTokens(pt({ total_prompt_tokens: 100, theoretical_cache_hit_rate: 1.2 })), + ).toBeUndefined(); + expect( + theoreticalPrefixTokens(pt({ total_prompt_tokens: 100, theoretical_cache_hit_rate: -0.1 })), + ).toBeUndefined(); + expect( + theoreticalPrefixTokens(pt({ total_prompt_tokens: 100, theoretical_cache_hit_rate: NaN })), + ).toBeUndefined(); + }); + + it('clamps rounding so uncached input never goes negative', () => { + const d = pt({ total_prompt_tokens: 3, theoretical_cache_hit_rate: 1 }); + expect(theoreticalPrefixTokens(d)).toBe(3); + expect(uncachedInputTokens(d)).toBe(0); + }); +}); + +describe('generateTooltipContent cache metadata', () => { it('shows offload type, backend, and version instead of the binary offload mode', () => { const html = generateTooltipContent( tooltipConfig({ diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index ee740c4f..6e76cd6c 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -220,10 +220,59 @@ const generateCacheMetadataHTML = (d: InferenceData, locale: Locale): string => * separately because fixed-sequence rows can carry it too. */ const AGENTIC_STRINGS = { - en: { speculativeDecoding: 'Speculative Decoding', off: 'Off' }, - zh: { speculativeDecoding: '投机解码', off: '关闭' }, + en: { + speculativeDecoding: 'Speculative Decoding', + off: 'Off', + theoreticalPrefixTokens: 'Theoretical Prefix Tokens', + uncachedInputTokens: 'Input Tokens w/o Prefix Caching', + }, + zh: { + speculativeDecoding: '投机解码', + off: '关闭', + theoreticalPrefixTokens: '理论 prefix token 数', + uncachedInputTokens: '无 prefix cache 的输入 token 数', + }, } as const; +/** + * Theoretical prefix tokens for a point: the sum of every prompt prefix the + * workload has already seen, under an infinite cache. The harness reports that + * sum as a rate over served prompt tokens (`theoretical_cache_hit_rate`, the + * infinite-cache hit rate computed from the trace), so multiplying it back + * with `total_prompt_tokens` recovers the token sum. This is deliberately the + * THEORETICAL prefix — what could have been cached given the trace — not the + * server-observed cache hits, so systems with better real caching are not + * penalized on derived uncached-input metrics. + */ +export const theoreticalPrefixTokens = (d: InferenceData): number | undefined => { + const rate = d.theoretical_cache_hit_rate; + const prompt = d.total_prompt_tokens; + if ( + prompt === undefined || + prompt === null || + rate === undefined || + rate === null || + Number.isNaN(rate) || + rate < 0 || + rate > 1 + ) { + return undefined; + } + return Math.round(prompt * rate); +}; + +/** + * Input tokens without prefix caching: served prompt tokens minus the + * theoretical prefix ({@link theoreticalPrefixTokens}). Approximates the + * prompt tokens that must be prefilled even by a system with an infinite + * prefix cache. + */ +export const uncachedInputTokens = (d: InferenceData): number | undefined => { + const prefix = theoreticalPrefixTokens(d); + if (prefix === undefined || d.total_prompt_tokens === undefined) return undefined; + return Math.max(0, d.total_prompt_tokens - prefix); +}; + const generateAgenticHTML = (d: InferenceData, locale: Locale): string => { if (d.benchmark_type !== 'agentic_traces') return ''; @@ -255,6 +304,16 @@ const generateAgenticHTML = (d: InferenceData, locale: Locale): string => { if (d.total_prompt_tokens !== undefined) { parts.push(tooltipLine('Prompt Tokens', formatNumber(d.total_prompt_tokens))); } + + const theoreticalPrefix = theoreticalPrefixTokens(d); + if (theoreticalPrefix !== undefined) { + parts.push(tooltipLine(t.theoreticalPrefixTokens, formatNumber(theoreticalPrefix))); + } + const uncachedInput = uncachedInputTokens(d); + if (uncachedInput !== undefined) { + parts.push(tooltipLine(t.uncachedInputTokens, formatNumber(uncachedInput))); + } + if (d.total_generation_tokens !== undefined) { parts.push(tooltipLine('Generated Tokens', formatNumber(d.total_generation_tokens))); } diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts index aa2c659c..82831ee7 100644 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts +++ b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts @@ -1,3 +1,5 @@ +import type { RequestLengthMoments } from '@/lib/attention-flops'; + import { bulkIdsFetcher, useBulkIdsQuery } from './benchmark-id-query'; export interface DerivedAgenticMetric { @@ -7,6 +9,8 @@ export interface DerivedAgenticMetric { p75_e2e_norm_intvty: number | null; /** Slow-tail P90 E2E Normalized Interactivity in tok/s/user — 1 / p90(per-request E2EL/OSL). */ p90_e2e_norm_intvty: number | null; + /** Exact joint (ISL, OSL) sums over the run's requests — attention-FLOPs input. */ + request_length_moments: RequestLengthMoments | null; } export type DerivedAgenticMetricMap = Record; diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 4a22d16c..5081bcd8 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -1849,8 +1849,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/derived-agentic-metrics', summary: text('Read derived agentic metrics', '读取派生智能体指标'), description: text( - 'Returns normalized interactivity percentiles keyed by benchmark result ID. IDs are deduplicated and at most 200 are accepted.', - '按基准结果 ID 返回归一化交互性百分位。ID 会去重,最多接受 200 个。', + 'Returns normalized interactivity percentiles and joint request-length moment sums keyed by benchmark result ID. IDs are deduplicated and at most 200 are accepted.', + '按基准结果 ID 返回归一化交互性百分位及请求长度联合矩和。ID 会去重,最多接受 200 个。', ), audience: 'public', stability: 'beta', @@ -1868,16 +1868,43 @@ export const apiOperations: readonly ApiOperation[] = [ ], responses: [ success( - 'Result IDs mapped to p75 and p90 normalized interactivity.', - '结果 ID 映射到 p75 和 p90 归一化交互性。', + 'Result IDs mapped to p75/p90 normalized interactivity and exact joint (ISL, OSL) request-length moment sums (null when the profile blob is missing).', + '结果 ID 映射到 p75/p90 归一化交互性及精确的 (ISL, OSL) 请求长度联合矩和(缺少 profile blob 时为 null)。', mapSchema( objectSchema({ id: integerSchema, p75_e2e_norm_intvty: nullableNumberSchema, p90_e2e_norm_intvty: nullableNumberSchema, + request_length_moments: { + type: ['object', 'null'], + properties: { + n: integerSchema, + sumIsl: { type: 'number' }, + sumIslSq: { type: 'number' }, + sumOsl: { type: 'number' }, + sumOslSq: { type: 'number' }, + sumIslOsl: { type: 'number' }, + }, + required: ['n', 'sumIsl', 'sumIslSq', 'sumOsl', 'sumOslSq', 'sumIslOsl'], + additionalProperties: false, + }, }), ), - { '421': { id: 421, p75_e2e_norm_intvty: 31.2, p90_e2e_norm_intvty: 24.8 } }, + { + '421': { + id: 421, + p75_e2e_norm_intvty: 31.2, + p90_e2e_norm_intvty: 24.8, + request_length_moments: { + n: 2, + sumIsl: 300, + sumIslSq: 50_000, + sumOsl: 150, + sumOslSq: 12_500, + sumIslOsl: 25_000, + }, + }, + }, ), errorResponse( '400', diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index bc49b9bb..b3549e5c 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -191,7 +191,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'get-derived-agentic-metrics', - sourceSha256: '68013b8e1a0354e677c075b5ab31df4be5889fdc0dc2eb82dbe52a108f4d1db2', + sourceSha256: '13c6ba2b0b4d827745336b016728970651580a79bec59bf4fae9743d35d8eb19', }, { source: 'src/app/api/v1/eval-samples-live/route.ts', @@ -711,7 +711,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/agentic-aggregates.ts', - sourceSha256: 'fae8d19971730132cb30cd781f677562bfc6328b1f4e35a8268a8391ad187c18', + sourceSha256: 'd2ce8c8769dd38012ba22054017f600b8e6e978a54a9248dd6085d5fc6b4b642', reviewArea: { en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.', zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。', diff --git a/packages/app/src/lib/attention-flops.test.ts b/packages/app/src/lib/attention-flops.test.ts new file mode 100644 index 00000000..43c33b82 --- /dev/null +++ b/packages/app/src/lib/attention-flops.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; + +import { + attentionFlopsPerComputedToken, + sumComputedTokens, + sumContexts, + type AttentionCostSpec, + type RequestLengthMoments, +} from './attention-flops'; + +/** Build exact moments from raw (ISL, OSL) pairs. */ +function momentsOf(pairs: [number, number][]): RequestLengthMoments { + return { + n: pairs.length, + sumIsl: pairs.reduce((s, [p]) => s + p, 0), + sumIslSq: pairs.reduce((s, [p]) => s + p * p, 0), + sumOsl: pairs.reduce((s, [, o]) => s + o, 0), + sumOslSq: pairs.reduce((s, [, o]) => s + o * o, 0), + sumIslOsl: pairs.reduce((s, [p, o]) => s + p * o, 0), + }; +} + +/** + * Brute-force reference: enumerate every computed token's context. A request + * (P, O) at rate r computes its suffix at contexts r·P+1 … P, then decodes at + * P+1 … P+O. Use rates where r·P is an integer so enumeration is exact. + */ +function bruteForceContexts(pairs: [number, number][], rate: number): number[] { + const contexts: number[] = []; + for (const [p, o] of pairs) { + const cached = rate * p; + for (let t = cached + 1; t <= p + o; t++) contexts.push(t); + } + return contexts; +} + +describe('sumContexts / sumComputedTokens', () => { + it('matches brute-force enumeration at rate 0', () => { + const pairs: [number, number][] = [ + [4, 2], + [10, 3], + [1, 1], + ]; + const contexts = bruteForceContexts(pairs, 0); + const m = momentsOf(pairs); + expect(sumContexts(m, 0)).toBe(contexts.reduce((a, b) => a + b, 0)); + expect(sumComputedTokens(m, 0)).toBe(contexts.length); + }); + + it('matches brute-force enumeration at rate 0.5 (integer cached prefixes)', () => { + const pairs: [number, number][] = [ + [4, 2], + [10, 3], + [100, 7], + ]; + const contexts = bruteForceContexts(pairs, 0.5); + const m = momentsOf(pairs); + expect(sumContexts(m, 0.5)).toBe(contexts.reduce((a, b) => a + b, 0)); + expect(sumComputedTokens(m, 0.5)).toBe(contexts.length); + }); + + it('reduces to decode-only sums at rate 1', () => { + const pairs: [number, number][] = [ + [4, 2], + [10, 3], + ]; + const contexts = bruteForceContexts(pairs, 1); + const m = momentsOf(pairs); + expect(sumContexts(m, 1)).toBe(contexts.reduce((a, b) => a + b, 0)); + expect(sumComputedTokens(m, 1)).toBe(contexts.length); + }); +}); + +describe('attentionFlopsPerComputedToken', () => { + const pairs: [number, number][] = [ + [4, 2], + [10, 3], + ]; + const m = momentsOf(pairs); + + it('prices a pure linear spec exactly (per-context coefficient)', () => { + const spec: AttentionCostSpec = { + groups: [{ label: 'dense', layers: 5, linPerCtx: 1000 }], + }; + const contexts = bruteForceContexts(pairs, 0.5); + const expected = (5 * 1000 * contexts.reduce((a, b) => a + b, 0)) / contexts.length; + expect(attentionFlopsPerComputedToken(spec, m, 0.5)).toBeCloseTo(expected, 8); + }); + + it('prices a constant spec independently of context', () => { + const spec: AttentionCostSpec = { + groups: [{ label: 'linear-attn', layers: 3, constPerToken: 7 }], + }; + expect(attentionFlopsPerComputedToken(spec, m, 0)).toBeCloseTo(21, 10); + expect(attentionFlopsPerComputedToken(spec, m, 0.5)).toBeCloseTo(21, 10); + }); + + it('saturates capped terms at cap × tokens when every context exceeds the cap', () => { + // Single request with a long prompt: all computed contexts are > 2. + const big = momentsOf([[100, 10]]); + const spec: AttentionCostSpec = { + groups: [{ label: 'window', layers: 2, capped: { coeff: 10, cap: 2 } }], + }; + const nTokens = sumComputedTokens(big, 0); + expect(attentionFlopsPerComputedToken(spec, big, 0)).toBeCloseTo( + (2 * 10 * 2 * nTokens) / nTokens, + 10, + ); + }); + + it('falls back to Σctx for capped terms when every context is under the cap', () => { + const spec: AttentionCostSpec = { + groups: [{ label: 'budget', layers: 1, capped: { coeff: 4, cap: 1_000_000 } }], + }; + const contexts = bruteForceContexts(pairs, 0); + const expected = (4 * contexts.reduce((a, b) => a + b, 0)) / contexts.length; + expect(attentionFlopsPerComputedToken(spec, m, 0)).toBeCloseTo(expected, 8); + }); + + it('combines linear + capped + constant groups additively', () => { + const spec: AttentionCostSpec = { + groups: [ + { label: 'a', layers: 2, linPerCtx: 3 }, + { label: 'b', layers: 1, capped: { coeff: 5, cap: 4 } }, + { label: 'c', layers: 4, constPerToken: 11 }, + ], + }; + const nTokens = sumComputedTokens(m, 0); + const ctx = sumContexts(m, 0); + const expected = (2 * 3 * ctx + 5 * Math.min(4 * nTokens, ctx) + 4 * 11 * nTokens) / nTokens; + expect(attentionFlopsPerComputedToken(spec, m, 0)).toBeCloseTo(expected, 8); + }); + + it('returns null for invalid rates, empty moments, and zero computed tokens', () => { + const spec: AttentionCostSpec = { groups: [{ label: 'x', layers: 1, linPerCtx: 1 }] }; + expect(attentionFlopsPerComputedToken(spec, m, -0.1)).toBeNull(); + expect(attentionFlopsPerComputedToken(spec, m, 1.1)).toBeNull(); + expect(attentionFlopsPerComputedToken(spec, m, Number.NaN)).toBeNull(); + expect(attentionFlopsPerComputedToken(spec, momentsOf([]), 0.5)).toBeNull(); + // rate 1 with zero output tokens → nothing is computed. + expect(attentionFlopsPerComputedToken(spec, momentsOf([[10, 0]]), 1)).toBeNull(); + }); +}); diff --git a/packages/app/src/lib/attention-flops.ts b/packages/app/src/lib/attention-flops.ts new file mode 100644 index 00000000..b41fa7d8 --- /dev/null +++ b/packages/app/src/lib/attention-flops.ts @@ -0,0 +1,142 @@ +/** + * Model-specific attention-FLOPs accounting for the + * "New Input Suffix + Output TFLOP/s per Chip" y-metric. + * + * ## What is counted + * + * Activation–activation attention compute only: QK^T score matmuls, + * attention-weighted value aggregation (AV), sparse-attention indexer + * scoring, and linear-attention (KDA) state update/readout. Weight matmuls + * (Q/K/V/O projections, indexer weight projections, MLA up/down projections) + * are EXCLUDED — they are covered by the separate `2 × N_active_params` + * GEMM term (Kaplan et al. / PaLM appendix convention, + * https://arxiv.org/pdf/2204.02311). One multiply-accumulate = 2 FLOPs. + * + * ## Per-layer-group cost model + * + * Every supported architecture's per-computed-token attention cost at + * context length L reduces to the affine-plus-capped form + * + * F_group(L) = linPerCtx · L + capped.coeff · min(L, capped.cap) + constPerToken + * + * summed over layer groups (× layer count). Examples: + * - Dense GQA/MHA/absorbed-MLA: pure `linPerCtx` (4·H·d·L or 2·H·(d_s+d_v)·L). + * - Sliding-window layers (gpt-oss): capped term with cap = window. + * - Top-k sparse attention (DSA/MSA): dense indexer scoring is `linPerCtx`, + * the main attention over the selected set is a capped term with + * cap = token budget (e.g. DeepSeek-V4-Pro CSA's + * 262,144·min(1024, L/4) rewrites to 65,536·min(L, 4096)). + * - Linear attention (Kimi K3 KDA): `constPerToken`, independent of L. + * + * ## Integration over the request population + * + * The DB layer stores exact joint sums over each run's per-request + * (ISL, OSL) pairs (`RequestLengthMoments`). With the theoretical cache hit + * rate r (infinite-cache prefix share of the prompt, so a request with + * prompt P computes its suffix of (1−r)·P tokens at contexts r·P+1 … P, + * then decodes O tokens at contexts P+1 … P+O — cached-prefix tokens are + * NOT recomputed but ARE attended over by every computed token): + * + * Σ contexts = (1−r²)/2·ΣP² + (1−r)/2·ΣP + Σ(P·O) + (ΣO² + ΣO)/2 + * Σ tokens = (1−r)·ΣP + ΣO + * + * both exact given the stored moments. Capped terms use + * Σ min(L, cap) ≈ min(cap · Σtokens, Σcontexts) — an upper bound (min is + * concave) that is exact when every context is on the same side of the cap; + * agentic traces overwhelmingly sit far above the small caps (128-token + * windows) and the error is bounded by the capped term itself for the + * large caps (2048/4096-token budgets). + * + * Per-model formulas and dimensions were verified against HF configs and + * tech reports; sources are cited on each spec in model-architectures.ts. + */ + +/** + * Exact joint (ISL, OSL) sums over a run's request population. Mirrors + * `RequestLengthMoments` in packages/db/src/queries/agentic-shared.ts + * (served through the derived-agentic-metrics endpoint). + */ +export interface RequestLengthMoments { + /** Number of requests with both ISL and OSL present. */ + n: number; + /** Σ ISL_i */ + sumIsl: number; + /** Σ ISL_i² */ + sumIslSq: number; + /** Σ OSL_i */ + sumOsl: number; + /** Σ OSL_i² */ + sumOslSq: number; + /** Σ ISL_i·OSL_i */ + sumIslOsl: number; +} + +/** + * One group of identical attention layers. Per computed token at context L + * the group costs + * layers × (linPerCtx·L + capped.coeff·min(L, capped.cap) + constPerToken) + * FLOPs (MAC = 2 FLOPs; omitted fields default to 0). + */ +export interface AttentionLayerGroup { + /** Human-readable mechanism label (documentation only). */ + label: string; + /** Number of layers of this type. */ + layers: number; + /** FLOPs per computed token per unit of context (the a·L term). */ + linPerCtx?: number; + /** Capped term b·min(L, cap): sliding windows and top-k token budgets. */ + capped?: { coeff: number; cap: number }; + /** Context-independent FLOPs per computed token (linear attention). */ + constPerToken?: number; +} + +/** Full attention-cost specification for one model. */ +export interface AttentionCostSpec { + groups: AttentionLayerGroup[]; +} + +/** Closed-form Σ over all computed tokens of their context lengths. */ +export function sumContexts(moments: RequestLengthMoments, rate: number): number { + const oneMinusR = 1 - rate; + return ( + ((1 - rate * rate) / 2) * moments.sumIslSq + + (oneMinusR / 2) * moments.sumIsl + + moments.sumIslOsl + + (moments.sumOslSq + moments.sumOsl) / 2 + ); +} + +/** Total computed (non-cached) tokens: suffix prefill + decode. */ +export function sumComputedTokens(moments: RequestLengthMoments, rate: number): number { + return (1 - rate) * moments.sumIsl + moments.sumOsl; +} + +/** + * Average attention FLOPs per computed token for a run, integrating the + * model's per-layer-group cost over the request-length distribution at the + * given theoretical cache hit rate. + * + * Returns null when the moments are unusable (no paired requests, or no + * computed tokens — e.g. rate = 1 with zero output tokens). + */ +export function attentionFlopsPerComputedToken( + spec: AttentionCostSpec, + moments: RequestLengthMoments, + rate: number, +): number | null { + if (!Number.isFinite(rate) || rate < 0 || rate > 1) return null; + if (!moments || moments.n <= 0) return null; + const nTokens = sumComputedTokens(moments, rate); + if (!Number.isFinite(nTokens) || nTokens <= 0) return null; + const sumCtx = sumContexts(moments, rate); + if (!Number.isFinite(sumCtx) || sumCtx < 0) return null; + + let totalFlops = 0; + for (const g of spec.groups) { + const lin = (g.linPerCtx ?? 0) * sumCtx; + const capped = g.capped ? g.capped.coeff * Math.min(g.capped.cap * nTokens, sumCtx) : 0; + const constant = (g.constPerToken ?? 0) * nTokens; + totalFlops += g.layers * (lin + capped + constant); + } + return totalFlops / nTokens; +} diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index 4036af80..f1827035 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -19,6 +19,7 @@ import { } from '@/lib/chart-utils'; import { getHardwareConfig } from '@/lib/constants'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; +import type { RequestLengthMoments } from '@/lib/attention-flops'; import type { BenchmarkRow } from '@/lib/api'; /** @@ -362,10 +363,16 @@ export function mergeRunScopedRows( * (default 'median'). Swaps `median_intvty`/`median_e2el` in the chart * definition for the chosen percentile — only agentic rows carry the * full set (median/p90/p99/p99.9) so this mainly affects that scenario. + * @param requestLengthMomentsById Optional per-persisted-id (ISL, OSL) moment + * sums from the derived-agentic-metrics endpoint. Merged into each entry + * before derived fields are built so the TFLOP/s-per-chip metric can price + * attention FLOPs; callers without them (unofficial-run overlays, AI chart) + * simply omit that metric. */ export function transformBenchmarkRows( rows: BenchmarkRow[], percentile = 'median', + requestLengthMomentsById?: Record, ): { chartData: InferenceData[][]; hardwareConfig: HardwareConfig; @@ -378,6 +385,9 @@ export function transformBenchmarkRows( for (let i = 0; i < rows.length; i++) { const row = rows[i]; const entry = rowToAggDataEntry(row); + if (requestLengthMomentsById && entry.id !== undefined) { + entry.request_length_moments = requestLengthMomentsById[entry.id] ?? null; + } const hwKey = getHardwareKey(entry); entry.hwKey = hwKey; diff --git a/packages/app/src/lib/chart-utils.test.ts b/packages/app/src/lib/chart-utils.test.ts index 6134ae8f..3d74836c 100644 --- a/packages/app/src/lib/chart-utils.test.ts +++ b/packages/app/src/lib/chart-utils.test.ts @@ -694,6 +694,116 @@ describe('createChartDataPoint', () => { expect(point.inputTputPerGpu).toEqual({ y: 300, roof: false }); }); + it('computes new-input-suffix throughput fields when the rate is valid', () => { + const e = entry({ + tput_per_gpu: 900, + output_tput_per_gpu: 600, + input_tput_per_gpu: 300, + theoretical_cache_hit_rate: 0.8, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + // input suffix = 300 x (1 - 0.8); suffix + output = 900 - 300 x 0.8 + expect(point.newInputSuffixTputPerGpu?.y).toBeCloseTo(60); + expect(point.newInputSuffixOutputTputPerGpu?.y).toBeCloseTo(660); + }); + + it('omits new-input-suffix throughput fields when the rate is missing', () => { + const e = entry({ tput_per_gpu: 900, input_tput_per_gpu: 300 }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.newInputSuffixTputPerGpu).toBeUndefined(); + expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined(); + }); + + it('derives suffix+output TFLOP/s from GEMM + attention FLOPs for known models', () => { + const e = entry({ + model: 'DeepSeek-R1-0528', // 37B active params, 61 absorbed-MLA layers + tput_per_gpu: 900, + input_tput_per_gpu: 600, + output_tput_per_gpu: 300, + theoretical_cache_hit_rate: 0.8, + // One request: ISL 1000, OSL 100. + request_length_moments: { + n: 1, + sumIsl: 1000, + sumIslSq: 1_000_000, + sumOsl: 100, + sumOslSq: 10_000, + sumIslOsl: 100_000, + }, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + // suffix+output = 900 - 600 × 0.8 = 420 tok/s. + // Σctx = (1-0.64)/2·1e6 + 0.1·1000 + 1e5 + (1e4+100)/2 = 285,150; + // Σtokens = 0.2·1000 + 100 = 300; + // attention = 61·278,528·285,150/300 = 16,149,192,704 FLOPs/token; + // (2·37e9 + 16,149,192,704) × 420 / 1e12 = 37.86266… TFLOP/s. + expect(point.newInputSuffixOutputTflopsPerGpu?.y).toBeCloseTo(37.86266093568, 5); + }); + + it('omits suffix+output TFLOP/s for models without architecture data', () => { + const e = entry({ + model: 'not-a-real-model', + tput_per_gpu: 900, + input_tput_per_gpu: 600, + theoretical_cache_hit_rate: 0.8, + request_length_moments: { + n: 1, + sumIsl: 1000, + sumIslSq: 1_000_000, + sumOsl: 100, + sumOslSq: 10_000, + sumIslOsl: 100_000, + }, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.newInputSuffixOutputTflopsPerGpu).toBeUndefined(); + }); + + it('omits suffix+output TFLOP/s when request-length moments are missing', () => { + const e = entry({ + model: 'DeepSeek-R1-0528', + tput_per_gpu: 900, + input_tput_per_gpu: 600, + theoretical_cache_hit_rate: 0.8, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.newInputSuffixOutputTflopsPerGpu).toBeUndefined(); + }); + + it('omits new-input-suffix throughput fields when input throughput is missing', () => { + const e = entry({ + tput_per_gpu: 900, + input_tput_per_gpu: 0, + theoretical_cache_hit_rate: 0.8, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.newInputSuffixTputPerGpu).toBeUndefined(); + expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined(); + }); + + it('omits new-input-suffix throughput fields when the rate is out of range', () => { + const e = entry({ + tput_per_gpu: 900, + input_tput_per_gpu: 300, + theoretical_cache_hit_rate: 1.2, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.newInputSuffixTputPerGpu).toBeUndefined(); + expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined(); + }); + + it('reports zero input suffix and output-only suffix+output at a full theoretical hit rate', () => { + const e = entry({ + tput_per_gpu: 900, + output_tput_per_gpu: 600, + input_tput_per_gpu: 300, + theoretical_cache_hit_rate: 1, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.newInputSuffixTputPerGpu?.y).toBe(0); + expect(point.newInputSuffixOutputTputPerGpu?.y).toBeCloseTo(600); + }); + it('computes tpPerMw from throughput and hardware power', () => { // tpPerMw = (tput_per_gpu * 1000) / power = (1000 * 1000) / 700 const e = entry({ tput_per_gpu: 1000 }); diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts index 5f49e6b9..e39fa2f5 100644 --- a/packages/app/src/lib/chart-utils.ts +++ b/packages/app/src/lib/chart-utils.ts @@ -18,6 +18,9 @@ import { type BenchmarkMetricKey, } from '@/components/inference/metric-registry'; import { getGpuSpecs, isKnownGpu } from '@/lib/constants'; +import type { Model } from '@/lib/data-mappings'; +import { attentionFlopsPerComputedToken } from '@/lib/attention-flops'; +import { getModelArchitecture } from '@/lib/model-architectures'; import { getVendor, type Vendor } from '@/lib/dynamic-colors'; import type { Locale } from '@/lib/i18n'; @@ -327,6 +330,63 @@ export function buildDerivedChartFields( if (wants('inputTputPerGpu') && inputTputPerGpu) { fields.inputTputPerGpu = chartMetric(inputTputPerGpu); } + // New-input-suffix throughput: subtract the infinite-cache theoretical prefix + // share of input throughput (rate derived from the trace by the harness). + // Uses the theoretical rate rather than server-observed cache hits so + // systems with good cache storage aren't penalized — only agentic trace + // points carry the rate, so fixed-sequence points omit these fields. + const theoreticalHitRate = entry.theoretical_cache_hit_rate; + const hasTheoreticalHitRate = + typeof theoreticalHitRate === 'number' && + Number.isFinite(theoreticalHitRate) && + theoreticalHitRate >= 0 && + theoreticalHitRate <= 1; + // Both metrics require input throughput: without it the prefix share can't + // be subtracted and suffix+output would silently equal total throughput. + if ( + wants('newInputSuffixOutputTputPerGpu') && + hasTheoreticalHitRate && + tputPerGpu && + inputTputPerGpu + ) { + fields.newInputSuffixOutputTputPerGpu = chartMetric( + Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate), + ); + } + // Achieved model TFLOP/s per chip on the theoretically necessary tokens: + // FLOPs/token = 2 × active params (Kaplan/PaLM GEMM convention) + the + // model-specific attention FLOPs per computed token, integrated over the + // run's exact per-request (ISL, OSL) sums at the theoretical hit rate + // (see attention-flops.ts). Requires the moments and an attention spec — + // points without them (fixed-seq, unofficial overlays, models outside the + // architecture registry) omit the metric rather than showing a lower bound + // inconsistent with other points. + if ( + wants('newInputSuffixOutputTflopsPerGpu') && + hasTheoreticalHitRate && + tputPerGpu && + inputTputPerGpu + ) { + const arch = getModelArchitecture(entry.model as Model); + const moments = entry.request_length_moments; + if (arch?.activeParams && arch.attention && moments) { + const attnFlopsPerToken = attentionFlopsPerComputedToken( + arch.attention, + moments, + theoreticalHitRate, + ); + if (attnFlopsPerToken !== null) { + const suffixOutputTput = Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate); + const flopsPerToken = 2 * arch.activeParams * 1e9 + attnFlopsPerToken; + fields.newInputSuffixOutputTflopsPerGpu = chartMetric( + (suffixOutputTput * flopsPerToken) / 1e12, + ); + } + } + } + if (wants('newInputSuffixTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) { + fields.newInputSuffixTputPerGpu = chartMetric(inputTputPerGpu * (1 - theoreticalHitRate)); + } if (wants('tpPerMw')) fields.tpPerMw = chartMetric((tputPerGpu * 1000) / hardwarePower); if (wants('inputTputPerMw') && inputTputPerGpu) { fields.inputTputPerMw = chartMetric( diff --git a/packages/app/src/lib/model-architectures.ts b/packages/app/src/lib/model-architectures.ts index f30393bb..daef3b00 100644 --- a/packages/app/src/lib/model-architectures.ts +++ b/packages/app/src/lib/model-architectures.ts @@ -1,4 +1,5 @@ import { Model } from '@/lib/data-mappings'; +import type { AttentionCostSpec } from '@/lib/attention-flops'; /** * Model architecture types @@ -134,6 +135,13 @@ export interface ModelArchitecture { ffnVariant?: string; /** Elementwise activation applied to the gate projection. Defaults to `SiLU`. */ ffnGateActivation?: string; + /** + * Attention-FLOPs cost model for the TFLOP/s-per-chip y-metric — see + * attention-flops.ts for the accounting conventions (activation–activation + * ops only, MAC = 2 FLOPs, absorbed MLA form). Models without a spec are + * simply omitted from that metric. + */ + attention?: AttentionCostSpec; } /** @@ -177,6 +185,15 @@ export const MODEL_ARCHITECTURES: Partial> = { ], developer: 'DeepSeek', sourceUrl: 'https://huggingface.co/deepseek-ai/DeepSeek-R1-0528', + // Absorbed (MQA-mode) MLA — the form that executes at decode: all 128 + // query heads score against the shared 576-dim cached latent (512 + // kv_lora_rank + 64 rope) and aggregate 512-dim latent values. + // 2·128·(576+512)·L = 278,528·L per layer. Dims: + // https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/config.json; + // MQA-mode description: https://arxiv.org/html/2512.02556v1. + attention: { + groups: [{ label: 'MLA (absorbed)', layers: 61, linPerCtx: 278528 }], + }, }, [Model.DeepSeek_V4_Pro]: { model: Model.DeepSeek_V4_Pro, @@ -242,6 +259,36 @@ export const MODEL_ARCHITECTURES: Partial> = { ], developer: 'DeepSeek', sourceUrl: 'https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro', + // Shared single-latent MQA (d_score = d_v = 512, RoPE in-place in the last + // 64 dims). Config `compress_ratios` decodes the full stack as 31 HCA + + // 30 CSA layers (the 3 hash-routed MoE layers are HCA, HCA, CSA). + // HCA: dense over L/128 pooled entries + 128-token window, no indexer → + // 2·128·(512+512)·(L/128 + 128) = 2,048·L + 33,554,432. + // CSA: FP4 lightning indexer (64 heads × dim 128) over L/4 pooled keys + + // core attention over top-1024 pooled entries + 128-token window → + // 4,096·L + 262,144·(min(1024, L/4) + 128), and + // 262,144·min(1024, L/4) ≡ 65,536·min(L, 4096). + // Attention sink = one learnable per-head logit (0 KV entries, ~0 FLOPs). + // Sources: https://arxiv.org/html/2606.19348v1; + // https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/config.json; + // https://huggingface.co/docs/transformers/en/model_doc/deepseek_v4. + attention: { + groups: [ + { + label: 'HCA (128:1 pooled, dense)', + layers: 31, + linPerCtx: 2048, + constPerToken: 33554432, + }, + { + label: 'CSA (4:1 pooled, indexer top-1024)', + layers: 30, + linPerCtx: 4096, + capped: { coeff: 65536, cap: 4096 }, + constPerToken: 33554432, + }, + ], + }, }, [Model.Llama3_3_70B]: { model: Model.Llama3_3_70B, @@ -259,6 +306,12 @@ export const MODEL_ARCHITECTURES: Partial> = { features: ['Grouped Query Attention', 'RoPE'], developer: 'Meta', sourceUrl: 'https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct', + // Dense causal GQA: 4·H·d·L = 4·64·128·L per layer (GQA KV sharing saves + // cache, not score/AV compute). Dims: + // https://huggingface.co/unsloth/Llama-3.3-70B-Instruct/raw/main/config.json. + attention: { + groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 32768 }], + }, }, [Model.Llama3_1_70B]: { model: Model.Llama3_1_70B, @@ -276,6 +329,11 @@ export const MODEL_ARCHITECTURES: Partial> = { features: ['Grouped Query Attention', 'RoPE'], developer: 'Meta', sourceUrl: 'https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct', + // Identical attention geometry to Llama 3.3 70B (80 × 64 heads × dim 128): + // https://huggingface.co/unsloth/Meta-Llama-3.1-70B-Instruct/raw/main/config.json. + attention: { + groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 32768 }], + }, }, [Model.GptOss]: { model: Model.GptOss, @@ -318,6 +376,17 @@ export const MODEL_ARCHITECTURES: Partial> = { ], developer: 'OpenAI', sourceUrl: 'https://huggingface.co/openai/gpt-oss-120b', + // 18 full + 18 sliding-128 GQA layers, 64 heads × dim 64 → 4·64·64 = + // 16,384 FLOPs/token per ctx unit; sliding layers cap L at 128. The + // learnable sink is one extra softmax logit per head (~0 FLOPs). + // https://huggingface.co/openai/gpt-oss-120b/raw/main/config.json; + // https://arxiv.org/abs/2508.10925. + attention: { + groups: [ + { label: 'Full GQA', layers: 18, linPerCtx: 16384 }, + { label: 'Sliding-128 GQA', layers: 18, capped: { coeff: 16384, cap: 128 } }, + ], + }, }, [Model.Kimi_K2_5]: { model: Model.Kimi_K2_5, @@ -339,6 +408,13 @@ export const MODEL_ARCHITECTURES: Partial> = { features: ['Multi-head Latent Attention', 'DeepSeek-style MoE', 'YaRN RoPE'], developer: 'Moonshot AI', sourceUrl: 'https://huggingface.co/moonshotai/Kimi-K2.5', + // DeepSeek-V3-style absorbed MLA on all 61 layers, 64 heads: score dim + // 576 (512 latent + 64 rope), value dim 512 → 2·64·(576+512)·L = + // 139,264·L per layer. + // https://huggingface.co/moonshotai/Kimi-K2.5/raw/main/config.json. + attention: { + groups: [{ label: 'MLA (absorbed)', layers: 61, linPerCtx: 139264 }], + }, }, [Model.Kimi_K3]: { model: Model.Kimi_K3, @@ -407,6 +483,20 @@ export const MODEL_ARCHITECTURES: Partial> = { ], developer: 'Moonshot AI', sourceUrl: 'https://huggingface.co/moonshotai/Kimi-K3', + // 69 KDA linear-attention layers: gated delta-rule update + readout on a + // 128×128 state per head × 96 heads ≈ 7·H·d² = 11,010,048 FLOPs/token, + // independent of L (recurrent form; the chunked prefill kernel is ~11% + // higher — within noise of this metric). 24 gated-MLA layers are NoPE + // (`mla_use_nope`), so score dim = value dim = 512 → 2·96·(512+512)·L = + // 196,608·L per layer. + // https://huggingface.co/moonshotai/Kimi-K3/raw/main/config.json; + // https://arxiv.org/abs/2510.26692. + attention: { + groups: [ + { label: 'KDA (linear)', layers: 69, constPerToken: 11010048 }, + { label: 'Gated MLA (NoPE, absorbed)', layers: 24, linPerCtx: 196608 }, + ], + }, }, [Model.MiniMax_M2_5]: { model: Model.MiniMax_M2_5, @@ -434,6 +524,12 @@ export const MODEL_ARCHITECTURES: Partial> = { ], developer: 'MiniMax', sourceUrl: 'https://huggingface.co/MiniMaxAI/MiniMax-M2', + // Dense causal GQA on all 62 layers (config `attn_type_list` is all 1s): + // 4·48·128·L = 24,576·L per layer. + // https://huggingface.co/MiniMaxAI/MiniMax-M2.5/raw/main/config.json. + attention: { + groups: [{ label: 'Full GQA', layers: 62, linPerCtx: 24576 }], + }, }, [Model.MiniMax_M3]: { model: Model.MiniMax_M3, @@ -467,6 +563,24 @@ export const MODEL_ARCHITECTURES: Partial> = { ], developer: 'MiniMax', sourceUrl: 'https://huggingface.co/MiniMaxAI/MiniMax-M3', + // Layers 0–2 dense GQA (4·64·128·L = 32,768·L); layers 3–59 MSA: per-token + // index scoring (4 group heads × dim 128, no value head) over all L + // positions = 2·4·128·L = 1,024·L, then exact GQA over the top-16 + // 128-token blocks (2048-token budget) = 32,768·min(L, 2048). Reproduces + // the paper's 28.4× attention-compute reduction at 1M. + // https://huggingface.co/MiniMaxAI/MiniMax-M3/raw/main/config.json; + // https://arxiv.org/html/2606.13392v2. + attention: { + groups: [ + { label: 'Dense GQA (layers 0-2)', layers: 3, linPerCtx: 32768 }, + { + label: 'MSA (top-16 blocks of 128)', + layers: 57, + linPerCtx: 1024, + capped: { coeff: 32768, cap: 2048 }, + }, + ], + }, }, }; diff --git a/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl b/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl index 8b9e22d1..ce9920f0 100644 --- a/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl +++ b/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl @@ -16,3 +16,4 @@ {"rule": "duplicated-technical-loanword", "kind": "exemption", "text": "'这里所说的预热(warmup)发生在正式测量之前。'", "note": "Chinese-first parenthetical explanation is intentional for a broader audience"} {"rule": "duplicated-technical-loanword", "kind": "exemption", "text": "'随机种子(seed)用于固定会话采样。'", "note": "Chinese-first parenthetical explanation remains allowed"} {"rule": "duplicated-technical-loanword", "kind": "exemption", "text": "'将 KV cache 卸载(offload)到 DRAM。'", "note": "Chinese-first parenthetical explanation remains allowed"} +{"rule": "chip-untranslated", "kind": "exemption", "text": "labelZh: '每芯片新输入 suffix + 输出 TFLOP/s(TFLOP/s/chip)',", "note": "TFLOP/s/chip is a unit and stays English like tok/s/chip"} diff --git a/packages/app/src/lib/zh-copy.test.ts b/packages/app/src/lib/zh-copy.test.ts index 9f47e170..27c6b39a 100644 --- a/packages/app/src/lib/zh-copy.test.ts +++ b/packages/app/src/lib/zh-copy.test.ts @@ -84,7 +84,8 @@ function segment(raw: string, isProse: boolean): string[] { // "Chip" is an ordinary English noun whose Chinese equivalent (芯片) is what // readers actually use, so it cannot inherit GPU's exemption. Units keep the // English form per AGENTS.md rule 6 — that is the one recorded exception. -const CHIP_UNITS = /(?:tok|tokens?)\/s\/chip|\$\/chip[/-](?:hr|hour)|[A-Za-z]Chip\b|\bChip[A-Z]/giu; +const CHIP_UNITS = + /(?:tok|tokens?|[KMGT]?FLOPs?)\/s\/chip|\$\/chip[/-](?:hr|hour)|[A-Za-z]Chip\b|\bChip[A-Z]/giu; const RULES: Rule[] = [ { diff --git a/packages/db/src/etl/compute-aggregate-stats.test.ts b/packages/db/src/etl/compute-aggregate-stats.test.ts index d2119c7b..e9b2d40d 100644 --- a/packages/db/src/etl/compute-aggregate-stats.test.ts +++ b/packages/db/src/etl/compute-aggregate-stats.test.ts @@ -97,6 +97,16 @@ describe('computeAggregateStats', () => { expect(stats.e2elPerOsl?.p50).toBeCloseTo(2 / 75, 6); // p90 of 3 values (linear interpolation): pos=1.8 → 0.02667 + 0.8×(0.03-0.02667) expect(stats.e2elPerOsl?.p90).toBeCloseTo(2 / 75 + 0.8 * (0.03 - 2 / 75), 6); + + // Exact joint (ISL, OSL) moments for closed-form attention-FLOP pricing. + expect(stats.requestLengthMoments).toEqual({ + n: 3, + sumIsl: 600, + sumIslSq: 100 * 100 + 200 * 200 + 300 * 300, + sumOsl: 225, + sumOslSq: 50 * 50 + 75 * 75 + 100 * 100, + sumIslOsl: 100 * 50 + 200 * 75 + 300 * 100, + }); }); it('excludes cancelled requests from every streamed profile distribution', async () => { @@ -114,6 +124,8 @@ describe('computeAggregateStats', () => { expect(stats.e2elPerOsl?.n).toBe(1); expect(stats.sequenceLengths.isl?.n).toBe(1); expect(stats.sequenceLengths.osl?.n).toBe(1); + expect(stats.requestLengthMoments?.n).toBe(1); + expect(stats.requestLengthMoments?.sumIslSq).toBe(100 * 100); }); it('accepts a gzip profile split across arbitrarily small compressed chunks', async () => { diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index 07729b59..0c32b753 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -9,10 +9,6 @@ * computation changes so the backfill script knows which rows to recompute. */ -import { createInterface } from 'node:readline'; -import { Readable } from 'node:stream'; -import { createGunzip } from 'node:zlib'; - import { gunzipJsonWithinLimit, streamCollectKeys } from './gzip-json-stream'; import { STATS_VERSION, @@ -22,6 +18,11 @@ import { type MetricPercentiles, type SequenceLengthSketches, } from '../queries/agentic-aggregates'; +import { + extractProfileSamples, + requestLengthMomentsOf, + type RequestLengthMoments, +} from '../queries/agentic-shared'; export { STATS_VERSION }; @@ -39,85 +40,12 @@ export interface AggregateStats { e2elPerOsl: MetricPercentiles | null; /** Bounded mergeable distributions used by the chart-level subtitle. */ sequenceLengths: SequenceLengthSketches; -} - -interface ProfileMetricEnvelope { - value?: number; -} - -interface ProfileRecord { - metadata?: { - benchmark_phase?: string; - was_cancelled?: boolean; - }; - metrics?: { - input_sequence_length?: ProfileMetricEnvelope | number; - output_sequence_length?: ProfileMetricEnvelope | number; - request_latency?: ProfileMetricEnvelope | number; - time_to_first_token?: ProfileMetricEnvelope | number; - }; -} - -function profileMetricValue(value: ProfileMetricEnvelope | number | undefined): number | undefined { - const number = typeof value === 'number' ? value : value?.value; - return typeof number === 'number' && Number.isFinite(number) ? number : undefined; -} - -/** - * Stream a profile export line by line so exceptionally large traces never - * materialize their multi-gigabyte decompressed JSONL as one string. The - * numeric sample arrays are tiny relative to the source and are needed for - * exact percentile calculation. - */ -async function extractProfileSamples( - compressedChunks: Iterable | AsyncIterable, -): Promise<{ - isl: number[]; - osl: number[]; - e2elPerOsl: number[]; -}> { - const input = Readable.from(compressedChunks).pipe(createGunzip()); - const lines = createInterface({ input, crlfDelay: Infinity }); - const isl: number[] = []; - const osl: number[] = []; - const e2elPerOsl: number[] = []; - - for await (const line of lines) { - if (!line) continue; - let record: ProfileRecord; - try { - record = JSON.parse(line) as ProfileRecord; - } catch { - continue; - } - if (record.metadata?.benchmark_phase && record.metadata.benchmark_phase !== 'profiling') { - continue; - } - if (record.metadata?.was_cancelled === true) continue; - - const metrics = record.metrics ?? {}; - const inputLength = profileMetricValue(metrics.input_sequence_length); - const outputLength = profileMetricValue(metrics.output_sequence_length); - if (inputLength !== undefined) isl.push(inputLength); - if (outputLength !== undefined) osl.push(outputLength); - - const requestLatencyMs = profileMetricValue(metrics.request_latency); - const ttftMs = profileMetricValue(metrics.time_to_first_token); - if ( - requestLatencyMs !== undefined && - ttftMs !== undefined && - inputLength !== undefined && - outputLength !== undefined && - requestLatencyMs > 0 && - ttftMs > 0 && - inputLength > 0 && - outputLength > 0 - ) { - e2elPerOsl.push(requestLatencyMs / 1000 / outputLength); - } - } - - return { isl, osl, e2elPerOsl }; + /** + * Exact joint (ISL, OSL) sums over the request population — the sufficient + * statistics the frontend integrates model-specific attention-FLOPs + * formulas over (see agentic-shared.ts). + */ + requestLengthMoments: RequestLengthMoments | null; } /** @@ -132,13 +60,15 @@ export async function computeProfileAggregateStatsFromCompressedChunks( let oslPct: MetricPercentiles | null = null; let e2elPerOsl: MetricPercentiles | null = null; let sequenceLengths: SequenceLengthSketches = { isl: null, osl: null }; + let requestLengthMoments: RequestLengthMoments | null = null; try { - const { isl, osl, e2elPerOsl: ratios } = await extractProfileSamples(compressedChunks); + const { isl, osl, e2elPerOsl: ratios, pairs } = await extractProfileSamples(compressedChunks); islPct = percentilesOf(isl); oslPct = percentilesOf(osl); sequenceLengths = sequenceLengthSketches(isl, osl); e2elPerOsl = percentilesOf(ratios); + requestLengthMoments = requestLengthMomentsOf(pairs); } catch { // Ignore malformed blobs and leave the profile-derived fields null. } @@ -151,6 +81,7 @@ export async function computeProfileAggregateStatsFromCompressedChunks( prefixCacheHitRate: null, e2elPerOsl, sequenceLengths, + requestLengthMoments, }; } diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 89ec4b91..ba3ccf67 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -211,10 +211,19 @@ describe('getAgenticAggregates write-back', () => { const { sql, calls } = mockSql([ // fetchAggregateStatsRows [{ benchmark_result_id: 7, stats: staleStats }], - // Pass 1: profile blob (+ trace_replay_id for write-back) - [{ benchmark_result_id: 7, trace_replay_id: 870, profile_blob: profileBlob }], - // Pass 2: server blob - [{ benchmark_result_id: 7, server_blob: serverBlob }], + // metadata query: ids → trace_replay rows + blob presence (no blob inline) + [ + { + benchmark_result_id: 7, + trace_replay_id: 870, + has_profile_blob: true, + has_server_blob: true, + }, + ], + // Pass 1: profile blob substring chunk (short → stream terminates) + [{ chunk: profileBlob }], + // Pass 2: server blob substring chunk + [{ chunk: serverBlob }], ]); const result = await getAgenticAggregates(sql, [7]); @@ -223,14 +232,17 @@ describe('getAgenticAggregates write-back', () => { expect(result[7]?.isl?.n).toBe(2); expect(result[7]?.kvCacheUtil?.mean).toBeCloseTo(0.25, 6); - // 4 calls: stats read, profile read, server read, write-back UPDATE. - expect(calls).toHaveLength(4); - expect(calls[3]!.text).toContain('update agentic_trace_replay set aggregate_stats'); - expect(calls[3]!.text).toContain('::jsonb where id'); + // 5 calls: stats read, metadata read, profile chunk, server chunk, write-back UPDATE. + expect(calls).toHaveLength(5); + // Blobs are streamed via bounded substring chunks — never selected whole. + expect(calls[2]!.text).toContain('substring(profile_export_jsonl_gz from'); + expect(calls[3]!.text).toContain('substring(server_metrics_json_gz from'); + expect(calls[4]!.text).toContain('update agentic_trace_replay set aggregate_stats'); + expect(calls[4]!.text).toContain('::jsonb where id'); // The payload OBJECT is bound directly (not stringified — that would // double-encode into a JSONB string). - const [written, traceReplayId] = calls[3]!.values as [WrittenStats, number]; + const [written, traceReplayId] = calls[4]!.values as [WrittenStats, number]; expect(traceReplayId).toBe(870); expect(written.version).toBe(STATS_VERSION); // Server field FRESHLY recomputed (0.25), not the stale 0.9 carried forward. @@ -258,16 +270,60 @@ describe('getAgenticAggregates write-back', () => { }; const { sql, calls } = mockSql([ [{ benchmark_result_id: 7, stats: staleStats }], - // Pass 1: no profile blob → nothing to recompute, nothing to heal. - [{ benchmark_result_id: 7, trace_replay_id: 870, profile_blob: null }], - // Pass 2: no server blob either. - [{ benchmark_result_id: 7, server_blob: null }], + // Metadata: both blobs missing → nothing to recompute, nothing to heal, + // and no substring reads are even issued. + [ + { + benchmark_result_id: 7, + trace_replay_id: 870, + has_profile_blob: false, + has_server_blob: false, + }, + ], ]); await getAgenticAggregates(sql, [7]); - // stats read + 2 blob reads only — no write-back (profile parse never succeeded). - expect(calls).toHaveLength(3); + // stats read + metadata read only — no write-back (profile parse never ran). + expect(calls).toHaveLength(2); + expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false); + }); + + it('does not stamp a bundle with null server fields when the server blob fails to parse', async () => { + const profileBlob = gzipSync( + Buffer.from(profileRec({ cid: 's1', isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 })), + ); + const staleStats = { + version: STATS_VERSION - 1, + isl: null, + osl: null, + kvCacheUtil: null, + prefixCacheHitRate: null, + }; + const { sql, calls } = mockSql([ + [{ benchmark_result_id: 7, stats: staleStats }], + [ + { + benchmark_result_id: 7, + trace_replay_id: 870, + has_profile_blob: true, + has_server_blob: true, + }, + ], + // Pass 1: profile blob parses fine. + [{ chunk: profileBlob }], + // Pass 2: server blob chunk is corrupt — gunzip fails. + [{ chunk: Buffer.from('not gzip at all') }], + ]); + + const result = await getAgenticAggregates(sql, [7]); + + // Profile-derived fields are still served for this request… + expect(result[7]?.isl?.n).toBe(1); + expect(result[7]?.kvCacheUtil).toBeNull(); + // …but nothing is written back: stamping a current-version bundle with + // null server fields would permanently cache the miss. + expect(calls).toHaveLength(4); expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false); }); }); diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 9511d248..343b1f82 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -14,7 +14,7 @@ */ import { Readable } from 'node:stream'; -import { createGunzip, gunzipSync } from 'node:zlib'; +import { createGunzip } from 'node:zlib'; import { chain } from 'stream-chain'; @@ -24,15 +24,18 @@ import { streamObject } from 'stream-json/streamers/stream-object.js'; import { gunzipJsonWithinLimit } from '../etl/gzip-json-stream'; import type { DbClient } from '../connection.js'; -import { computeDerivedFromBlob } from './derived-agentic-metrics'; import { - extractIslOsl, + extractProfileSamples, fetchAggregateStatsRows, percentilesOf, + readTraceReplayBlob, + requestLengthMomentsOf, sequenceLengthSketches, STATS_VERSION, + streamTraceReplayBlob, writeBackTraceReplayJsonb, type MetricPercentiles, + type RequestLengthMoments, type SequenceLengthSketches, } from './agentic-shared'; @@ -59,17 +62,6 @@ export interface AgenticAggregate { export type AgenticAggregateMap = Record; -/** - * `profile_export_jsonl_gz` is small (~1-3 MB) so we can batch many per - * round-trip. `server_metrics_json_gz` is much bigger (~17 MB compressed - * for high-conc TP+EP runs; Neon encodes bytea over HTTP at ~1.6× wire - * size, so two of those = ~50 MB and three already trips the 64 MB cap). - * We fetch the two blob types in separate queries with different chunk - * sizes. - */ -const PROFILE_CHUNK_SIZE = 8; -const SERVER_CHUNK_SIZE = 1; - interface TimeSlice { start_ns?: number; end_ns?: number; @@ -259,7 +251,7 @@ export async function getAgenticAggregates( const statsRows = await fetchAggregateStatsRows(sql, benchmarkResultIds); const idsNeedingProfile: number[] = []; - const idsNeedingServer: number[] = []; + for (const row of statsRows) { const id = Number(row.benchmark_result_id); const agg = blankAggregate(id); @@ -272,7 +264,6 @@ export async function getAgenticAggregates( // No stats (or stale version) — schedule the blob-parse fallback below // so the response still surfaces data. Backfill should drain these. idsNeedingProfile.push(id); - idsNeedingServer.push(id); } result[id] = agg; } @@ -282,7 +273,7 @@ export async function getAgenticAggregates( if (!(id in result)) result[id] = blankAggregate(id); } - if (idsNeedingProfile.length === 0 && idsNeedingServer.length === 0) { + if (idsNeedingProfile.length === 0) { return result; } @@ -293,94 +284,102 @@ export async function getAgenticAggregates( // good stored data. const pendingById = new Map(); - // ── Fallback Pass 1: profile_export blobs (cheap; large batches). ────── - for (let i = 0; i < idsNeedingProfile.length; i += PROFILE_CHUNK_SIZE) { - const chunk = idsNeedingProfile.slice(i, i + PROFILE_CHUNK_SIZE); - const rows = (await sql` - select - br.id as benchmark_result_id, - atr.id as trace_replay_id, - atr.profile_export_jsonl_gz as profile_blob - from benchmark_results br - join agentic_trace_replay atr on atr.id = br.trace_replay_id - where br.id = any(${chunk}::bigint[]) - `) as { - benchmark_result_id: number; - trace_replay_id: number; - profile_blob: Buffer | null; - }[]; - for (const row of rows) { - const id = Number(row.benchmark_result_id); - result[id] ??= blankAggregate(id); - if (row.profile_blob) { - try { - const jsonl = gunzipSync(row.profile_blob).toString('utf8'); - const { isl, osl } = extractIslOsl(jsonl); - const islPct = percentilesOf(isl); - const oslPct = percentilesOf(osl); - result[id].isl = islPct; - result[id].osl = oslPct; - // Recompute every profile-derived field from this same JSONL so the - // self-healed bundle is complete at the new version. Server-derived - // fields are filled in Pass 2 (or stay null without a server blob). - const derived = computeDerivedFromBlob(jsonl); - pendingById.set(id, { - traceReplayId: Number(row.trace_replay_id), - stats: { - version: STATS_VERSION, - isl: islPct, - osl: oslPct, - kvCacheUtil: null, - prefixCacheHitRate: null, - e2elPerOsl: derived.e2el_per_osl, - sequenceLengths: sequenceLengthSketches(isl, osl), - }, - }); - } catch { - // ignore malformed blob - } - } + // Both passes stream blobs through bounded `substring` chunks instead of + // selecting whole bytea columns: production profile blobs reach >240 MB + // compressed while Neon's serverless HTTP driver caps a single response at + // 64 MB (HTTP 507 above that), so an inline blob select can fail the whole + // query — the failure mode that blanked derived metrics for stale-version + // rows. One cheap metadata query maps ids to trace_replay rows first. + const metaRows = (await sql` + select + br.id as benchmark_result_id, + atr.id as trace_replay_id, + (atr.profile_export_jsonl_gz is not null) as has_profile_blob, + (atr.server_metrics_json_gz is not null) as has_server_blob + from benchmark_results br + join agentic_trace_replay atr on atr.id = br.trace_replay_id + where br.id = any(${idsNeedingProfile}::bigint[]) + `) as { + benchmark_result_id: number; + trace_replay_id: number; + has_profile_blob: boolean; + has_server_blob: boolean; + }[]; + + // ── Fallback Pass 1: profile_export blobs (streamed line parse). ────── + for (const row of metaRows) { + const id = Number(row.benchmark_result_id); + result[id] ??= blankAggregate(id); + if (!row.has_profile_blob) continue; + try { + // One pass yields every profile-derived field so the self-healed + // bundle is complete at the new version. Server-derived fields are + // filled in Pass 2 (or stay null without a server blob). + const { isl, osl, e2elPerOsl, pairs } = await extractProfileSamples( + streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', Number(row.trace_replay_id)), + ); + const islPct = percentilesOf(isl); + const oslPct = percentilesOf(osl); + result[id].isl = islPct; + result[id].osl = oslPct; + pendingById.set(id, { + traceReplayId: Number(row.trace_replay_id), + stats: { + version: STATS_VERSION, + isl: islPct, + osl: oslPct, + kvCacheUtil: null, + prefixCacheHitRate: null, + e2elPerOsl: percentilesOf(e2elPerOsl), + sequenceLengths: sequenceLengthSketches(isl, osl), + requestLengthMoments: requestLengthMomentsOf(pairs), + }, + }); + } catch { + // ignore malformed/unreadable blob — never fail the whole response } } // ── Fallback Pass 2: server_metrics blobs (huge; one at a time). ─────── // Serial to avoid OOM on the decompressed JSON of a high-conc TP+EP row // (>500 MB raw). The aggregator is fronted by a blob cache, so the slow // path runs at most once per sibling set. - for (let i = 0; i < idsNeedingServer.length; i += SERVER_CHUNK_SIZE) { - const chunk = idsNeedingServer.slice(i, i + SERVER_CHUNK_SIZE); - const rows = (await sql` - select - br.id as benchmark_result_id, - atr.server_metrics_json_gz as server_blob - from benchmark_results br - join agentic_trace_replay atr on atr.id = br.trace_replay_id - where br.id = any(${chunk}::bigint[]) - `) as { benchmark_result_id: number; server_blob: Buffer | null }[]; - for (const row of rows) { - const id = Number(row.benchmark_result_id); - result[id] ??= blankAggregate(id); - if (!row.server_blob) continue; - let parsed: { kvCacheUtil: number[]; prefixCacheHitRate: number[] } | null = null; - try { - const json = gunzipJsonWithinLimit(row.server_blob); + for (const row of metaRows) { + const id = Number(row.benchmark_result_id); + result[id] ??= blankAggregate(id); + if (!row.has_server_blob) continue; + let parsed: { kvCacheUtil: number[]; prefixCacheHitRate: number[] } | null = null; + try { + const serverBlob = await readTraceReplayBlob( + sql, + 'server_metrics_json_gz', + Number(row.trace_replay_id), + ); + if (serverBlob) { + const json = gunzipJsonWithinLimit(serverBlob); parsed = json === null - ? await streamExtractServerMetricSamples(row.server_blob) + ? await streamExtractServerMetricSamples(serverBlob) : extractServerMetricSamples(json); - } catch { - // malformed blob or failed stream fallback — leave nulls } - if (parsed) { - const kvPct = percentilesOf(parsed.kvCacheUtil); - const prefixPct = percentilesOf(parsed.prefixCacheHitRate); - result[id].kvCacheUtil = kvPct; - result[id].prefixCacheHitRate = prefixPct; - const pending = pendingById.get(id); - if (pending) { - pending.stats.kvCacheUtil = kvPct; - pending.stats.prefixCacheHitRate = prefixPct; - } + } catch { + // malformed blob or failed stream fallback — leave nulls + } + if (parsed) { + const kvPct = percentilesOf(parsed.kvCacheUtil); + const prefixPct = percentilesOf(parsed.prefixCacheHitRate); + result[id].kvCacheUtil = kvPct; + result[id].prefixCacheHitRate = prefixPct; + const pending = pendingById.get(id); + if (pending) { + pending.stats.kvCacheUtil = kvPct; + pending.stats.prefixCacheHitRate = prefixPct; } + } else { + // A server blob exists but couldn't be read or parsed this time + // (transient stream error, oversized JSON, ...). Don't self-heal a + // version-stamped bundle with null server fields — that would + // permanently cache the miss and the fast path would never retry. + pendingById.delete(id); } } @@ -420,6 +419,7 @@ interface FullAggregateStats { prefixCacheHitRate: MetricPercentiles | null; e2elPerOsl: MetricPercentiles | null; sequenceLengths: SequenceLengthSketches; + requestLengthMoments: RequestLengthMoments | null; } function blankAggregate(id: number): AgenticAggregate { diff --git a/packages/db/src/queries/agentic-shared.test.ts b/packages/db/src/queries/agentic-shared.test.ts index 35a25d97..44b4c16e 100644 --- a/packages/db/src/queries/agentic-shared.test.ts +++ b/packages/db/src/queries/agentic-shared.test.ts @@ -1,8 +1,17 @@ +import { randomBytes } from 'node:crypto'; +import { gzipSync } from 'node:zlib'; + import { afterEach, describe, expect, it, vi } from 'vitest'; import type { DbClient } from '../connection.js'; -import { _resetWriteBackWarned, writeBackTraceReplayJsonb } from './agentic-shared'; +import { + _resetWriteBackWarned, + BLOB_CHUNK_BYTES, + extractProfileSamples, + streamTraceReplayBlob, + writeBackTraceReplayJsonb, +} from './agentic-shared'; /** * Capture every SQL call: the joined template text plus the bound values, so we @@ -77,3 +86,85 @@ describe('writeBackTraceReplayJsonb', () => { expect(warn.mock.calls[0]![0]).toContain('could not persist chart_series'); }); }); + +/** Serve `substring(col from ? for ?)` reads out of an in-memory buffer. */ +function blobSql(blob: Buffer): { sql: DbClient; chunkReads: number[] } { + const chunkReads: number[] = []; + const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { + const text = strings.join('?'); + expect(text).toContain('substring(profile_export_jsonl_gz from'); + const [offset, length] = values as [number, number, number]; + chunkReads.push(offset); + // SQL substring is 1-based; slice is 0-based. + const chunk = blob.subarray(offset - 1, offset - 1 + length); + return Promise.resolve([{ chunk }]); + }) as unknown as DbClient; + return { sql, chunkReads }; +} + +/** One profiling record padded with poorly compressible bytes. */ +function paddedRec(isl: number, osl: number): string { + return JSON.stringify({ + metadata: { benchmark_phase: 'profiling' }, + metrics: { + request_latency: { value: 1000, unit: 'ms' }, + time_to_first_token: { value: 100, unit: 'ms' }, + input_sequence_length: { value: isl, unit: 'tokens' }, + output_sequence_length: { value: osl, unit: 'tokens' }, + padding: randomBytes(4 * 1024 * 1024).toString('base64'), + }, + }); +} + +describe('streamTraceReplayBlob + extractProfileSamples', () => { + it('reassembles a blob larger than one chunk across multiple substring reads', async () => { + // Poorly compressible padding pushes the gzip past BLOB_CHUNK_BYTES so the + // reader must issue several bounded reads and the gunzip stream must + // reassemble records that straddle chunk boundaries. + const jsonl = [paddedRec(100, 50), paddedRec(200, 100), paddedRec(300, 25)].join('\n'); + const blob = gzipSync(Buffer.from(jsonl)); + expect(blob.length).toBeGreaterThan(BLOB_CHUNK_BYTES); + + const { sql, chunkReads } = blobSql(blob); + const samples = await extractProfileSamples( + streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', 42), + ); + + expect(chunkReads.length).toBeGreaterThan(1); + expect(chunkReads[0]).toBe(1); + expect(chunkReads[1]).toBe(1 + BLOB_CHUNK_BYTES); + expect(samples.isl).toEqual([100, 200, 300]); + expect(samples.osl).toEqual([50, 100, 25]); + expect(samples.e2elPerOsl).toEqual([1 / 50, 1 / 100, 1 / 25]); + expect(samples.pairs).toHaveLength(3); + }); + + it('terminates without reads beyond a short final chunk and skips cancelled records', async () => { + const lines = [ + JSON.stringify({ + metadata: { benchmark_phase: 'profiling' }, + metrics: { + input_sequence_length: { value: 10, unit: 'tokens' }, + output_sequence_length: { value: 5, unit: 'tokens' }, + }, + }), + JSON.stringify({ + metadata: { benchmark_phase: 'profiling', was_cancelled: true }, + metrics: { + input_sequence_length: { value: 7777, unit: 'tokens' }, + output_sequence_length: { value: 7777, unit: 'tokens' }, + }, + }), + ]; + const blob = gzipSync(Buffer.from(lines.join('\n'))); + const { sql, chunkReads } = blobSql(blob); + const samples = await extractProfileSamples( + streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', 42), + ); + // Single short chunk — the reader must not issue a second round-trip. + expect(chunkReads).toEqual([1]); + expect(samples.pairs).toEqual([{ isl: 10, osl: 5 }]); + // No latency fields → no ratio samples; cancelled record fully ignored. + expect(samples.e2elPerOsl).toEqual([]); + }); +}); diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts index c242e4b8..073c4dcb 100644 --- a/packages/db/src/queries/agentic-shared.ts +++ b/packages/db/src/queries/agentic-shared.ts @@ -13,6 +13,10 @@ * write-back. (agentic-aggregates re-exports both for existing importers.) */ +import { createInterface } from 'node:readline'; +import { Readable } from 'node:stream'; +import { createGunzip } from 'node:zlib'; + import type { DbClient } from '../connection.js'; import { @@ -48,38 +52,214 @@ import { * v8: add p95 and bounded mergeable ISL/OSL sketches. The dashboard merges * the sketches for all resident chart points instead of loading request-level * timelines or attempting to combine per-point percentiles. + * + * v9: add `requestLengthMoments` — exact joint moments of the per-request + * (ISL, OSL) pairs (n, ΣISL, ΣISL², ΣOSL, ΣOSL², ΣISL·OSL). The frontend + * integrates model-specific attention-FLOPs formulas over the true request + * population from these sums (prefill attention is quadratic in context, so + * E[ISL²] ≠ E[ISL]² matters); marginal percentiles/sketches can't provide the + * joint ISL·OSL term the decode integral needs. + */ +export const STATS_VERSION = 9; + +/** + * Exact sums over the per-request (ISL, OSL) pairs of one benchmark point. + * Only records carrying BOTH sequence lengths contribute, so every sum is + * over the same request population and cross-terms stay consistent. + * + * These six sums are sufficient statistics for any attention-cost integral + * that is polynomial (≤ quadratic) in per-request context length: e.g. + * Σᵢ suffix-prefill context = (1−r²)/2 · ΣISL² and Σᵢ decode context = + * ΣISL·OSL + (ΣOSL² + ΣOSL)/2, with r the point's theoretical cache hit rate. */ -export const STATS_VERSION = 8; +export interface RequestLengthMoments { + /** Number of requests with both ISL and OSL present. */ + n: number; + sumIsl: number; + sumIslSq: number; + sumOsl: number; + sumOslSq: number; + sumIslOsl: number; +} + +/** Accumulate the joint moments for paired per-request (ISL, OSL) samples. */ +export function requestLengthMomentsOf( + pairs: readonly { isl: number; osl: number }[], +): RequestLengthMoments | null { + if (pairs.length === 0) return null; + const m: RequestLengthMoments = { + n: 0, + sumIsl: 0, + sumIslSq: 0, + sumOsl: 0, + sumOslSq: 0, + sumIslOsl: 0, + }; + for (const { isl, osl } of pairs) { + if (!Number.isFinite(isl) || !Number.isFinite(osl) || isl < 0 || osl < 0) continue; + m.n += 1; + m.sumIsl += isl; + m.sumIslSq += isl * isl; + m.sumOsl += osl; + m.sumOslSq += osl * osl; + m.sumIslOsl += isl * osl; + } + return m.n > 0 ? m : null; +} interface ProfileRecord { metadata?: { benchmark_phase?: string; was_cancelled?: boolean }; metrics?: { + request_latency?: { value?: number; unit?: string } | number; + time_to_first_token?: { value?: number; unit?: string } | number; input_sequence_length?: { value?: number } | number; output_sequence_length?: { value?: number } | number; }; } -/** Parse the profile_export.jsonl → per-request ISL + OSL arrays. */ -export function extractIslOsl(jsonl: string): { isl: number[]; osl: number[] } { - const isl: number[] = []; - const osl: number[] = []; - for (const line of jsonl.split('\n')) { - if (!line) continue; - let rec: ProfileRecord; - try { - rec = JSON.parse(line) as ProfileRecord; - } catch { - continue; - } - if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue; - if (rec.metadata?.was_cancelled === true) continue; - const m = rec.metrics ?? {}; - const i = readNum(m.input_sequence_length); - const o = readNum(m.output_sequence_length); - if (typeof i === 'number') isl.push(i); - if (typeof o === 'number') osl.push(o); +/** + * Per-request samples pulled from a profile_export.jsonl blob in one pass — + * the raw material every profile-derived aggregate is computed from. Both + * query fallbacks and the ingest/backfill path share this single extractor so + * the fast (stored bundle) and slow (blob recompute) paths can never drift. + */ +export interface ProfileSamples { + isl: number[]; + osl: number[]; + /** Per-request E2E latency / OSL ratios (seconds per output token). */ + e2elPerOsl: number[]; + /** (ISL, OSL) pairs over records carrying both lengths. */ + pairs: { isl: number; osl: number }[]; +} + +function addProfileSampleLine(acc: ProfileSamples, line: string): void { + if (!line) return; + let rec: ProfileRecord; + try { + rec = JSON.parse(line) as ProfileRecord; + } catch { + return; + } + if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') return; + if (rec.metadata?.was_cancelled === true) return; + const m = rec.metrics ?? {}; + const isl = readNum(m.input_sequence_length); + const osl = readNum(m.output_sequence_length); + if (isl !== undefined) acc.isl.push(isl); + if (osl !== undefined) acc.osl.push(osl); + if (isl !== undefined && osl !== undefined) acc.pairs.push({ isl, osl }); + const rl = readNum(m.request_latency); + const tt = readNum(m.time_to_first_token); + if ( + rl !== undefined && + tt !== undefined && + isl !== undefined && + osl !== undefined && + rl > 0 && + tt > 0 && + isl > 0 && + osl > 0 + ) { + acc.e2elPerOsl.push(rl / 1000 / osl); } - return { isl, osl }; +} + +/** Collect profile samples from an already-decompressed JSONL string. */ +export function collectProfileSamplesFromJsonl(jsonl: string): ProfileSamples { + const acc: ProfileSamples = { isl: [], osl: [], e2elPerOsl: [], pairs: [] }; + for (const line of jsonl.split('\n')) addProfileSampleLine(acc, line); + return acc; +} + +/** + * Stream a gzipped profile export line by line so exceptionally large traces + * never materialize their multi-gigabyte decompressed JSONL as one string. + * The numeric sample arrays are tiny relative to the source and are needed + * for exact percentile calculation. + */ +export async function extractProfileSamples( + compressedChunks: Iterable | AsyncIterable, +): Promise { + const input = Readable.from(compressedChunks).pipe(createGunzip()); + const lines = createInterface({ input, crlfDelay: Infinity }); + const acc: ProfileSamples = { isl: [], osl: [], e2elPerOsl: [], pairs: [] }; + for await (const line of lines) addProfileSampleLine(acc, line); + return acc; +} + +/** + * Parse the profile_export.jsonl → per-request ISL + OSL arrays, plus the + * joint (ISL, OSL) moments over records carrying both lengths. + */ +export function extractIslOsl(jsonl: string): { + isl: number[]; + osl: number[]; + requestLengthMoments: RequestLengthMoments | null; +} { + const { isl, osl, pairs } = collectProfileSamplesFromJsonl(jsonl); + return { isl, osl, requestLengthMoments: requestLengthMomentsOf(pairs) }; +} + +/** + * 8 MiB of bytea per `substring` read — the hex wire encoding doubles it, so + * each response stays far under Neon's serverless-HTTP 64 MB response cap. + * Production profile blobs reach >240 MB compressed (server blobs are of the + * same order), so selecting a whole blob column inline is NEVER safe: the + * driver rejects the response (HTTP 507) and the whole query fails. + */ +export const BLOB_CHUNK_BYTES = 8 * 1024 * 1024; + +export type TraceReplayBlobColumn = 'profile_export_jsonl_gz' | 'server_metrics_json_gz'; + +/** Normalize a driver-returned bytea value (Buffer, Uint8Array, or hex text). */ +function asBuffer(v: unknown): Buffer | null { + if (v === null || v === undefined) return null; + if (Buffer.isBuffer(v)) return v; + if (v instanceof Uint8Array) return Buffer.from(v); + if (typeof v === 'string' && v.startsWith(String.raw`\x`)) return Buffer.from(v.slice(2), 'hex'); + return null; +} + +/** + * Stream a gzipped blob column in bounded `substring` chunks. Self-terminating + * on the first short/empty chunk, so no size pre-query is needed and a + * `pg_column_size` vs `octet_length` mismatch can never truncate the stream. + */ +export async function* streamTraceReplayBlob( + sql: DbClient, + column: TraceReplayBlobColumn, + traceReplayId: number, +): AsyncGenerator { + for (let offset = 1; ; offset += BLOB_CHUNK_BYTES) { + // Static SQL per column (no dynamic identifiers) — `column` is a + // closed union, not caller-supplied text. + const rows = (await (column === 'profile_export_jsonl_gz' + ? sql` + select substring(profile_export_jsonl_gz from ${offset} for ${BLOB_CHUNK_BYTES}) as chunk + from agentic_trace_replay + where id = ${traceReplayId} + ` + : sql` + select substring(server_metrics_json_gz from ${offset} for ${BLOB_CHUNK_BYTES}) as chunk + from agentic_trace_replay + where id = ${traceReplayId} + `)) as { chunk: unknown }[]; + const chunk = asBuffer(rows[0]?.chunk); + if (!chunk || chunk.length === 0) break; + yield chunk; + if (chunk.length < BLOB_CHUNK_BYTES) break; + } +} + +/** Read a whole blob column via bounded chunks; null when absent/empty. */ +export async function readTraceReplayBlob( + sql: DbClient, + column: TraceReplayBlobColumn, + traceReplayId: number, +): Promise { + const chunks: Buffer[] = []; + for await (const chunk of streamTraceReplayBlob(sql, column, traceReplayId)) chunks.push(chunk); + return chunks.length > 0 ? Buffer.concat(chunks) : null; } export interface MetricPercentiles { diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts index 091fb81e..d5eaa16a 100644 --- a/packages/db/src/queries/derived-agentic-metrics.test.ts +++ b/packages/db/src/queries/derived-agentic-metrics.test.ts @@ -28,6 +28,41 @@ describe('computeDerivedFromBlob', () => { it('returns null when no usable records', () => { const out = computeDerivedFromBlob(''); expect(out.e2el_per_osl).toBeNull(); + expect(out.request_length_moments).toBeNull(); + }); + + it('accumulates exact joint (ISL, OSL) moments over profiling records', () => { + const lines = [ + rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), + rec('s2', 0, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 4000 }), + // Missing TTFT breaks the latency ratio but NOT the length moments. + JSON.stringify({ + metadata: { conversation_id: 's3', turn_index: 0, benchmark_phase: 'profiling' }, + metrics: { + request_latency: { value: 1000, unit: 'ms' }, + input_sequence_length: { value: 10, unit: 'tokens' }, + output_sequence_length: { value: 20, unit: 'tokens' }, + }, + }), + // Warmup phase is excluded from both. + JSON.stringify({ + metadata: { conversation_id: 's4', turn_index: 0, benchmark_phase: 'warmup' }, + metrics: { + input_sequence_length: { value: 9999, unit: 'tokens' }, + output_sequence_length: { value: 9999, unit: 'tokens' }, + }, + }), + ]; + const out = computeDerivedFromBlob(lines.join('\n')); + expect(out.request_length_moments).toEqual({ + n: 3, + sumIsl: 310, + sumIslSq: 100 * 100 + 200 * 200 + 10 * 10, + sumOsl: 170, + sumOslSq: 50 * 50 + 100 * 100 + 20 * 20, + sumIslOsl: 100 * 50 + 200 * 100 + 10 * 20, + }); + expect(out.e2el_per_osl?.n).toBe(2); }); it('computes per-request E2EL/OSL ratios pooled across sessions', () => { @@ -156,8 +191,11 @@ describe('getDerivedAgenticMetrics write-back', () => { const { sql, calls } = mockSql([ // fetchAggregateStatsRows [{ benchmark_result_id: 7, stats: staleStats }], - // fallback profile-blob query - [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], + // fallback metadata query (ids → trace_replay rows; no blob inline) + [{ benchmark_result_id: 7, trace_replay_id: 870 }], + // first substring chunk — shorter than BLOB_CHUNK_BYTES, so the + // streaming reader terminates without another round-trip + [{ chunk: blob }], ]); const result = await getDerivedAgenticMetrics(sql, [7]); @@ -166,10 +204,12 @@ describe('getDerivedAgenticMetrics write-back', () => { expect(result[7]?.p90_e2e_norm_intvty).toBeCloseTo(1 / 0.038, 6); expect(result[7]?.p75_e2e_norm_intvty).toBeCloseTo(1 / 0.035, 6); - // 3 calls: stats read, blob read, write-back UPDATE. - expect(calls).toHaveLength(3); - expect(calls[2]!.text).toContain('update agentic_trace_replay set aggregate_stats'); - expect(calls[2]!.text).toContain('::jsonb where id'); + // 4 calls: stats read, metadata read, blob chunk read, write-back UPDATE. + expect(calls).toHaveLength(4); + // The blob is streamed via bounded substring chunks — never selected whole. + expect(calls[2]!.text).toContain('substring(profile_export_jsonl_gz from'); + expect(calls[3]!.text).toContain('update agentic_trace_replay set aggregate_stats'); + expect(calls[3]!.text).toContain('::jsonb where id'); // The write-back binds a COMPLETE, version-stamped bundle at the new version, // recomputing profile fields and carrying server fields forward untouched. @@ -182,7 +222,7 @@ describe('getDerivedAgenticMetrics write-back', () => { kvCacheUtil: unknown; e2elPerOsl: { p75: number; p90: number; n: number } | null; } - const [written, traceReplayId] = calls[2]!.values as [WrittenStats, number]; + const [written, traceReplayId] = calls[3]!.values as [WrittenStats, number]; expect(traceReplayId).toBe(870); expect(written.version).toBe(STATS_VERSION); expect(written.e2elPerOsl?.n).toBe(2); @@ -210,16 +250,18 @@ describe('getDerivedAgenticMetrics write-back', () => { const { sql, calls } = mockSql([ // fetchAggregateStatsRows — no stored bundle at all [{ benchmark_result_id: 7, stats: null }], - // fallback profile-blob query - [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], + // fallback metadata query + [{ benchmark_result_id: 7, trace_replay_id: 870 }], + // single substring chunk + [{ chunk: blob }], ]); const result = await getDerivedAgenticMetrics(sql, [7]); // Caller still gets the freshly computed metric (1 / 0.02 s-per-token). expect(result[7]?.p90_e2e_norm_intvty).toBeCloseTo(50, 6); - // Stats read + blob read only — no write-back UPDATE. - expect(calls).toHaveLength(2); + // Stats read + metadata read + chunk read only — no write-back UPDATE. + expect(calls).toHaveLength(3); expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false); }); @@ -258,6 +300,11 @@ describe('getDerivedAgenticMetrics write-back', () => { ]); const result = await getDerivedAgenticMetrics(sql, [7]); - expect(result[7]).toEqual({ id: 7, p75_e2e_norm_intvty: null, p90_e2e_norm_intvty: null }); + expect(result[7]).toEqual({ + id: 7, + p75_e2e_norm_intvty: null, + p90_e2e_norm_intvty: null, + request_length_moments: null, + }); }); }); diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts index 5685ef2e..a0a8464c 100644 --- a/packages/db/src/queries/derived-agentic-metrics.ts +++ b/packages/db/src/queries/derived-agentic-metrics.ts @@ -21,18 +21,19 @@ * WORST request's effective token rate. */ -import { gunzipSync } from 'node:zlib'; - import type { DbClient } from '../connection.js'; import { - extractIslOsl, + collectProfileSamplesFromJsonl, + extractProfileSamples, fetchAggregateStatsRows, percentilesOf, - readNum, + requestLengthMomentsOf, sequenceLengthSketches, STATS_VERSION, + streamTraceReplayBlob, writeBackTraceReplayJsonb, type MetricPercentiles, + type RequestLengthMoments, type SequenceLengthSketches, } from './agentic-shared'; @@ -43,6 +44,13 @@ export interface DerivedAgenticMetric { p75_e2e_norm_intvty: number | null; /** Slow-tail P90 E2E Normalized Interactivity in tok/s/user — 1 / p90(per-request E2EL/OSL). */ p90_e2e_norm_intvty: number | null; + /** + * Exact joint (ISL, OSL) sums over the request population. The frontend + * integrates per-model attention-FLOPs formulas over these for the + * TFLOP/s-per-chip y-metric. Null when the blob had no usable records or + * the stored bundle predates v9 and hasn't self-healed yet. + */ + request_length_moments: RequestLengthMoments | null; } export type DerivedAgenticMetricMap = Record; @@ -65,48 +73,7 @@ interface StoredAggregateStats { prefixCacheHitRate: MetricPercentiles | null; e2elPerOsl: MetricPercentiles | null; sequenceLengths: SequenceLengthSketches; -} - -/** - * JSONL blobs can be ~1-2 MB compressed (~5-10 MB raw) and Neon's serverless - * HTTP driver caps responses at 64 MB — chunk to stay well under. - */ -const QUERY_CHUNK_SIZE = 6; - -interface RecordMetrics { - request_latency?: { value?: number; unit?: string } | number; - time_to_first_token?: { value?: number; unit?: string } | number; - input_sequence_length?: { value?: number } | number; - output_sequence_length?: { value?: number } | number; -} - -interface RecordMetadata { - conversation_id?: string; - turn_index?: number; - benchmark_phase?: string; -} - -interface ProfileRecord { - metadata?: RecordMetadata; - metrics?: RecordMetrics; -} - -interface TurnFields { - request_latency_ms: number; - ttft_ms: number; - isl: number; - osl: number; -} - -function extractTurn(rec: ProfileRecord): TurnFields | null { - const m = rec.metrics ?? {}; - const rl = readNum(m.request_latency); - const tt = readNum(m.time_to_first_token); - const isl = readNum(m.input_sequence_length); - const osl = readNum(m.output_sequence_length); - if (rl === undefined || tt === undefined || isl === undefined || osl === undefined) return null; - if (rl <= 0 || tt <= 0 || isl <= 0 || osl <= 0) return null; - return { request_latency_ms: rl, ttft_ms: tt, isl, osl }; + requestLengthMoments?: RequestLengthMoments | null; } /** 1/x for a positive stored ratio; null when the bundle/percentile is absent. */ @@ -123,22 +90,16 @@ function invertRatio(v: number | null | undefined): number | null { */ export function computeDerivedFromBlob(jsonl: string): { e2el_per_osl: MetricPercentiles | null; + request_length_moments: RequestLengthMoments | null; } { - const ratios: number[] = []; - for (const line of jsonl.split('\n')) { - if (!line) continue; - let rec: ProfileRecord; - try { - rec = JSON.parse(line) as ProfileRecord; - } catch { - continue; - } - if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue; - const turn = extractTurn(rec); - if (!turn) continue; - ratios.push(turn.request_latency_ms / 1000 / turn.osl); - } - return { e2el_per_osl: percentilesOf(ratios) }; + // Moments only need the sequence-length pair — the shared collector keeps + // them even when the latency fields the ratio requires are missing or + // non-positive. + const { e2elPerOsl, pairs } = collectProfileSamplesFromJsonl(jsonl); + return { + e2el_per_osl: percentilesOf(e2elPerOsl), + request_length_moments: requestLengthMomentsOf(pairs), + }; } export async function getDerivedAgenticMetrics( @@ -167,6 +128,7 @@ export async function getDerivedAgenticMetrics( id, p75_e2e_norm_intvty: invertRatio(row.stats.e2elPerOsl?.p75), p90_e2e_norm_intvty: invertRatio(row.stats.e2elPerOsl?.p90), + request_length_moments: row.stats.requestLengthMoments ?? null, }; } else { idsNeedingBlob.push(id); @@ -176,40 +138,42 @@ export async function getDerivedAgenticMetrics( if (idsNeedingBlob.length === 0) return result; - // Fallback: parse the profile blob directly. Used for rows whose + // Fallback: recompute from the profile blob. Used for rows whose // `aggregate_stats` is null or computed by an older STATS_VERSION; the // backfill script drains the population so this path should be rare. - // `trace_replay_id` + the (small) stale `aggregate_stats` come along on the - // same join — no extra round-trip — so we can self-heal after recompute. - const rows: { - benchmark_result_id: number; - trace_replay_id: number; - blob: Buffer; - }[] = []; - for (let i = 0; i < idsNeedingBlob.length; i += QUERY_CHUNK_SIZE) { - const chunk = idsNeedingBlob.slice(i, i + QUERY_CHUNK_SIZE); - const chunkRows = (await sql` - select - br.id as benchmark_result_id, - atr.id as trace_replay_id, - atr.profile_export_jsonl_gz as blob - from benchmark_results br - join agentic_trace_replay atr on atr.id = br.trace_replay_id - where br.id = any(${chunk}::bigint[]) - and atr.profile_export_jsonl_gz is not null - `) as { benchmark_result_id: number; trace_replay_id: number; blob: Buffer }[]; - rows.push(...chunkRows); - } - - for (const row of rows) { + // + // The blob is NEVER selected whole: production profile exports reach + // >240 MB compressed while Neon's serverless HTTP driver caps a response at + // 64 MB (HTTP 507 above that — the failure that blanked the TFLOP/s + // metric for every pre-v9 row). Instead a cheap metadata query maps ids to + // trace_replay rows, then each blob streams through bounded `substring` + // chunks into a streaming gunzip line parser — the same pattern + // backfill-aggregate-stats.ts uses for oversized TOAST values. + const metaRows = (await sql` + select + br.id as benchmark_result_id, + atr.id as trace_replay_id + from benchmark_results br + join agentic_trace_replay atr on atr.id = br.trace_replay_id + where br.id = any(${idsNeedingBlob}::bigint[]) + and atr.profile_export_jsonl_gz is not null + `) as { benchmark_result_id: number; trace_replay_id: number }[]; + + // Serial on purpose: each blob already parallelizes nothing and bounding + // concurrency keeps peak memory at one decompression stream. + for (const row of metaRows) { const id = Number(row.benchmark_result_id); try { - const jsonl = gunzipSync(row.blob).toString('utf8'); - const { e2el_per_osl } = computeDerivedFromBlob(jsonl); + const { isl, osl, e2elPerOsl, pairs } = await extractProfileSamples( + streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', Number(row.trace_replay_id)), + ); + const e2el_per_osl = percentilesOf(e2elPerOsl); + const request_length_moments = requestLengthMomentsOf(pairs); result[id] = { id, p75_e2e_norm_intvty: invertRatio(e2el_per_osl?.p75), p90_e2e_norm_intvty: invertRatio(e2el_per_osl?.p90), + request_length_moments, }; // Self-heal the shared `aggregate_stats` bundle. We only have the profile @@ -228,7 +192,6 @@ export async function getDerivedAgenticMetrics( const prior = staleStatsById.get(id) ?? null; const canPreserveServerFields = Boolean(prior?.kvCacheUtil || prior?.prefixCacheHitRate); if (canPreserveServerFields) { - const { isl, osl } = extractIslOsl(jsonl); const merged: StoredAggregateStats = { version: STATS_VERSION, isl: percentilesOf(isl), @@ -237,11 +200,13 @@ export async function getDerivedAgenticMetrics( prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, e2elPerOsl: e2el_per_osl, sequenceLengths: sequenceLengthSketches(isl, osl), + requestLengthMoments: request_length_moments, }; writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); } } catch { - // Skip malformed blobs silently — frontend treats missing ids as "no data". + // One malformed/unreadable blob must never take down the whole + // response — the frontend treats missing ids as "no data". } } return result;