From f92918e92db3bb1ce2cceefde4e883af2a5d8822 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 20:54:41 +0000
Subject: [PATCH 01/13] feat(inference): add theoretical prefix and uncached
input token rows to point overlay
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
For agentic points, derive the theoretical prefix tokens for each point as
the sum of every prompt prefix the trace has already seen: the harness
reports that sum as the infinite-cache theoretical_cache_hit_rate over
served prompt tokens, so multiplying it back with total_prompt_tokens
recovers the token sum. Add two new point-overlay rows:
- Theoretical Prefix Tokens
- Input Tokens w/o Prefix Caching (prompt total minus theoretical prefix)
This is deliberately the theoretical prefix (what could be cached given
the trace), not server-observed cache hits, so systems with better real
caching are not penalized on the derived uncached-input view.
中文：为 agentic 数据点的 point overlay 新增两行指标：理论 prefix token 数
（按该点在 trace 中已出现的全部 prefix 之和计算，即无限 cache 理论命中率
乘以 prompt token 总数）以及无 prefix cache 的输入 token 数（prompt 总数
减去理论 prefix）。刻意采用理论 prefix 而非服务端实际命中，避免 cache
能力更强的系统在该派生指标上被低估。
---
.../inference/utils/tooltip-utils.test.ts | 94 +++++++++++++++++++
.../inference/utils/tooltipUtils.ts | 63 ++++++++++++-
2 files changed, 155 insertions(+), 2 deletions(-)
diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts
index cd469a0c..47a982b2 100644
--- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts
+++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts
@@ -6,6 +6,8 @@ import {
generateTooltipContent,
generateOverlayTooltipContent,
generateGPUGraphTooltipContent,
+ theoreticalPrefixTokens,
+ uncachedInputTokens,
type TooltipConfig,
type OverlayTooltipConfig,
} from '@/components/inference/utils/tooltipUtils';
@@ -332,6 +334,98 @@ describe('generateTooltipContent', () => {
expect(html).toContain('FP8');
});
+ it('shows theoretical prefix and uncached input token rows for agentic points', () => {
+ const html = generateTooltipContent(
+ tooltipConfig({
+ data: pt({
+ benchmark_type: 'agentic_traces',
+ total_prompt_tokens: 1_000_000,
+ theoretical_cache_hit_rate: 0.92,
+ }),
+ }),
+ );
+ expect(html).toContain('Theoretical Prefix Tokens: 920,000');
+ expect(html).toContain('Input Tokens w/o Prefix Caching: 80,000');
+ });
+
+ it('omits the prefix token rows when the theoretical hit rate is missing', () => {
+ const html = generateTooltipContent(
+ tooltipConfig({
+ data: pt({
+ benchmark_type: 'agentic_traces',
+ total_prompt_tokens: 1_000_000,
+ theoretical_cache_hit_rate: undefined,
+ }),
+ }),
+ );
+ expect(html).toContain('Prompt Tokens: 1,000,000');
+ expect(html).not.toContain('Theoretical Prefix Tokens');
+ expect(html).not.toContain('Input Tokens w/o Prefix Caching');
+ });
+
+ it('never shows prefix token rows on fixed-sequence points', () => {
+ const html = generateTooltipContent(
+ tooltipConfig({
+ data: pt({
+ benchmark_type: 'single_turn',
+ total_prompt_tokens: 1_000_000,
+ theoretical_cache_hit_rate: 0.92,
+ }),
+ }),
+ );
+ expect(html).not.toContain('Theoretical Prefix Tokens');
+ expect(html).not.toContain('Input Tokens w/o Prefix Caching');
+ });
+
+ it('uses Chinese labels for the prefix token rows on /zh surfaces', () => {
+ const html = generateTooltipContent(
+ tooltipConfig({
+ locale: 'zh',
+ data: pt({
+ benchmark_type: 'agentic_traces',
+ total_prompt_tokens: 500_000,
+ theoretical_cache_hit_rate: 0.9,
+ }),
+ }),
+ );
+ expect(html).toContain('理论 prefix token 数： 450,000');
+ expect(html).toContain('无 prefix cache 的输入 token 数： 50,000');
+ });
+});
+
+describe('theoreticalPrefixTokens / uncachedInputTokens', () => {
+ it('recovers the prefix token sum from the trace hit rate', () => {
+ const d = pt({ total_prompt_tokens: 1_000_000, theoretical_cache_hit_rate: 0.925 });
+ expect(theoreticalPrefixTokens(d)).toBe(925_000);
+ expect(uncachedInputTokens(d)).toBe(75_000);
+ });
+
+ it('returns undefined when either input is missing', () => {
+ expect(theoreticalPrefixTokens(pt({ total_prompt_tokens: 100 }))).toBeUndefined();
+ expect(theoreticalPrefixTokens(pt({ theoretical_cache_hit_rate: 0.5 }))).toBeUndefined();
+ expect(uncachedInputTokens(pt({ total_prompt_tokens: 100 }))).toBeUndefined();
+ });
+
+ it('rejects out-of-range or NaN hit rates', () => {
+ expect(
+ theoreticalPrefixTokens(pt({ total_prompt_tokens: 100, theoretical_cache_hit_rate: 1.2 })),
+ ).toBeUndefined();
+ expect(
+ theoreticalPrefixTokens(pt({ total_prompt_tokens: 100, theoretical_cache_hit_rate: -0.1 })),
+ ).toBeUndefined();
+ expect(
+ theoreticalPrefixTokens(pt({ total_prompt_tokens: 100, theoretical_cache_hit_rate: NaN })),
+ ).toBeUndefined();
+ });
+
+ it('clamps rounding so uncached input never goes negative', () => {
+ const d = pt({ total_prompt_tokens: 3, theoretical_cache_hit_rate: 1 });
+ expect(theoreticalPrefixTokens(d)).toBe(3);
+ expect(uncachedInputTokens(d)).toBe(0);
+ });
+});
+
+describe('generateTooltipContent cache metadata', () => {
it('shows offload type, backend, and version instead of the binary offload mode', () => {
const html = generateTooltipContent(
tooltipConfig({
diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts
index ee740c4f..6e76cd6c 100644
--- a/packages/app/src/components/inference/utils/tooltipUtils.ts
+++ b/packages/app/src/components/inference/utils/tooltipUtils.ts
@@ -220,10 +220,59 @@ const generateCacheMetadataHTML = (d: InferenceData, locale: Locale): string =>
* separately because fixed-sequence rows can carry it too.
*/
const AGENTIC_STRINGS = {
- en: { speculativeDecoding: 'Speculative Decoding', off: 'Off' },
- zh: { speculativeDecoding: '投机解码', off: '关闭' },
+ en: {
+ speculativeDecoding: 'Speculative Decoding',
+ off: 'Off',
+ theoreticalPrefixTokens: 'Theoretical Prefix Tokens',
+ uncachedInputTokens: 'Input Tokens w/o Prefix Caching',
+ },
+ zh: {
+ speculativeDecoding: '投机解码',
+ off: '关闭',
+ theoreticalPrefixTokens: '理论 prefix token 数',
+ uncachedInputTokens: '无 prefix cache 的输入 token 数',
+ },
} as const;
+/**
+ * Theoretical prefix tokens for a point: the sum of every prompt prefix the
+ * workload has already seen, under an infinite cache. The harness reports that
+ * sum as a rate over served prompt tokens (`theoretical_cache_hit_rate`, the
+ * infinite-cache hit rate computed from the trace), so multiplying it back
+ * with `total_prompt_tokens` recovers the token sum. This is deliberately the
+ * THEORETICAL prefix — what could have been cached given the trace — not the
+ * server-observed cache hits, so systems with better real caching are not
+ * penalized on derived uncached-input metrics.
+ */
+export const theoreticalPrefixTokens = (d: InferenceData): number | undefined => {
+ const rate = d.theoretical_cache_hit_rate;
+ const prompt = d.total_prompt_tokens;
+ if (
+ prompt === undefined ||
+ prompt === null ||
+ rate === undefined ||
+ rate === null ||
+ Number.isNaN(rate) ||
+ rate < 0 ||
+ rate > 1
+ ) {
+ return undefined;
+ }
+ return Math.round(prompt * rate);
+};
+
+/**
+ * Input tokens without prefix caching: served prompt tokens minus the
+ * theoretical prefix ({@link theoreticalPrefixTokens}). Approximates the
+ * prompt tokens that must be prefilled even by a system with an infinite
+ * prefix cache.
+ */
+export const uncachedInputTokens = (d: InferenceData): number | undefined => {
+ const prefix = theoreticalPrefixTokens(d);
+ if (prefix === undefined || d.total_prompt_tokens === undefined) return undefined;
+ return Math.max(0, d.total_prompt_tokens - prefix);
+};
+
const generateAgenticHTML = (d: InferenceData, locale: Locale): string => {
if (d.benchmark_type !== 'agentic_traces') return '';
@@ -255,6 +304,16 @@ const generateAgenticHTML = (d: InferenceData, locale: Locale): string => {
if (d.total_prompt_tokens !== undefined) {
parts.push(tooltipLine('Prompt Tokens', formatNumber(d.total_prompt_tokens)));
}
+
+ const theoreticalPrefix = theoreticalPrefixTokens(d);
+ if (theoreticalPrefix !== undefined) {
+ parts.push(tooltipLine(t.theoreticalPrefixTokens, formatNumber(theoreticalPrefix)));
+ }
+ const uncachedInput = uncachedInputTokens(d);
+ if (uncachedInput !== undefined) {
+ parts.push(tooltipLine(t.uncachedInputTokens, formatNumber(uncachedInput)));
+ }
+
if (d.total_generation_tokens !== undefined) {
parts.push(tooltipLine('Generated Tokens', formatNumber(d.total_generation_tokens)));
}
From 1276c731ffaf72188bb7c36adb45879c3bbbbf3f Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 21:05:23 +0000
Subject: [PATCH 02/13] feat(inference): add theoretical uncached throughput
y-axis metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Add two Throughput y-axis options derived from the trace-level theoretical
(infinite-cache) prefix cache hit rate:
- Uncached Token Throughput per Chip: total throughput minus the theoretical
prefix share of input throughput
- Uncached Input Token Throughput per Chip: input throughput x (1 - rate)
Both deliberately use the theoretical rate rather than server-observed cache
hits so systems with good cache storage aren't penalized. Only agentic trace
points carry the rate; fixed-sequence points omit the fields and drop off the
chart for these metrics.
中文：新增两个基于 trace 级理论（无限 cache）prefix cache 命中率的吞吐量 Y 轴指标：
- 每芯片无 prefix cache token 吞吐量：总吞吐量减去输入吞吐量中的理论 prefix 部分
- 每芯片无 prefix cache 输入 token 吞吐量：输入吞吐量 x（1 - 命中率）
两者刻意采用理论命中率而非服务端实际 cache 命中，避免惩罚 cache 存储能力强的系统。
只有 agentic trace 点携带该命中率；固定序列点省略这些字段，在这两个指标下不显示。
---
.../components/inference/metric-registry.ts | 25 +++++++++++
.../app/src/components/inference/types.ts | 4 ++
packages/app/src/lib/chart-utils.test.ts | 43 +++++++++++++++++++
packages/app/src/lib/chart-utils.ts | 19 ++++++++
4 files changed, 91 insertions(+)
diff --git a/packages/app/src/components/inference/metric-registry.ts b/packages/app/src/components/inference/metric-registry.ts
index 046eb0d3..4a859e37 100644
--- a/packages/app/src/components/inference/metric-registry.ts
+++ b/packages/app/src/components/inference/metric-registry.ts
@@ -43,6 +43,29 @@ export const METRIC_REGISTRY = {
titleZh: '每芯片输出 token 吞吐量',
polarity: 'higher',
},
+ // Theoretical uncached throughput pair: total / input throughput minus the
+ // infinite-cache theoretical prefix share (trace-derived, agentic only).
+ // Deliberately based on the theoretical rate rather than server-observed
+ // cache hits so systems with good cache storage aren't penalized.
+ uncachedTputPerGpu: {
+ field: 'uncachedTputPerGpu.y',
+ label: 'Uncached Token Throughput per Chip (tok/s/chip)',
+ labelZh: '每芯片无 prefix cache token 吞吐量（tok/s/chip）',
+ title: 'Uncached Token Throughput per Chip',
+ titleZh: '每芯片无 prefix cache token 吞吐量',
+ polarity: 'higher',
+ },
+ uncachedInputTputPerGpu: {
+ field: 'uncachedInputTputPerGpu.y',
+ label: 'Uncached Input Token Throughput per Chip (tok/s/chip)',
+ labelZh: '每芯片无 prefix cache 输入 token 吞吐量（tok/s/chip）',
+ title: 'Uncached Input Token Throughput per Chip',
+ titleZh: '每芯片无 prefix cache 输入 token 吞吐量',
+ polarity: 'higher',
+ x: 'p90_ttft',
+ xLabel: 'P90 Time To First Token (s)',
+ heading: 'vs. P90 Time To First Token',
+ },
tpPerMw: {
field: 'tpPerMw.y',
label: 'Token Throughput per All in Utility MW (tok/s/MW)',
@@ -473,6 +496,8 @@ export const METRIC_CONTROL_GROUPS: readonly MetricControlGroup[] = [
'y_tpPerGpu',
'y_inputTputPerGpu',
'y_outputTputPerGpu',
+ 'y_uncachedTputPerGpu',
+ 'y_uncachedInputTputPerGpu',
'y_tpPerMw',
'y_inputTputPerMw',
'y_outputTputPerMw',
diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts
index 124483cc..88ed17b8 100644
--- a/packages/app/src/components/inference/types.ts
+++ b/packages/app/src/components/inference/types.ts
@@ -274,6 +274,10 @@ export interface InferenceData extends Partial {
expect(point.inputTputPerGpu).toEqual({ y: 300, roof: false });
});
+ it('computes theoretical uncached throughput fields when the rate is valid', () => {
+ const e = entry({
+ tput_per_gpu: 900,
+ output_tput_per_gpu: 600,
+ input_tput_per_gpu: 300,
+ theoretical_cache_hit_rate: 0.8,
+ });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ // uncached input = 300 x (1 - 0.8); uncached total = 900 - 300 x 0.8
+ expect(point.uncachedInputTputPerGpu?.y).toBeCloseTo(60);
+ expect(point.uncachedTputPerGpu?.y).toBeCloseTo(660);
+ });
+
+ it('omits theoretical uncached throughput fields when the rate is missing', () => {
+ const e = entry({ tput_per_gpu: 900, input_tput_per_gpu: 300 });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ expect(point.uncachedInputTputPerGpu).toBeUndefined();
+ expect(point.uncachedTputPerGpu).toBeUndefined();
+ });
+
+ it('omits theoretical uncached throughput fields when the rate is out of range', () => {
+ const e = entry({
+ tput_per_gpu: 900,
+ input_tput_per_gpu: 300,
+ theoretical_cache_hit_rate: 1.2,
+ });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ expect(point.uncachedInputTputPerGpu).toBeUndefined();
+ expect(point.uncachedTputPerGpu).toBeUndefined();
+ });
+
+ it('reports zero uncached input and output-only total at a full theoretical hit rate', () => {
+ const e = entry({
+ tput_per_gpu: 900,
+ output_tput_per_gpu: 600,
+ input_tput_per_gpu: 300,
+ theoretical_cache_hit_rate: 1,
+ });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ expect(point.uncachedInputTputPerGpu?.y).toBe(0);
+ expect(point.uncachedTputPerGpu?.y).toBeCloseTo(600);
+ });
+
it('computes tpPerMw from throughput and hardware power', () => {
// tpPerMw = (tput_per_gpu * 1000) / power = (1000 * 1000) / 700
const e = entry({ tput_per_gpu: 1000 });
diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts
index 5f49e6b9..fef68ed4 100644
--- a/packages/app/src/lib/chart-utils.ts
+++ b/packages/app/src/lib/chart-utils.ts
@@ -327,6 +327,25 @@ export function buildDerivedChartFields(
if (wants('inputTputPerGpu') && inputTputPerGpu) {
fields.inputTputPerGpu = chartMetric(inputTputPerGpu);
}
+ // Theoretical uncached throughput: subtract the infinite-cache theoretical
+ // prefix share of input throughput (rate derived from the trace by the
+ // harness). Uses the theoretical rate rather than server-observed cache
+ // hits so systems with good cache storage aren't penalized — only agentic
+ // trace points carry the rate, so fixed-sequence points omit these fields.
+ const theoreticalHitRate = entry.theoretical_cache_hit_rate;
+ const hasTheoreticalHitRate =
+ typeof theoreticalHitRate === 'number' &&
+ Number.isFinite(theoreticalHitRate) &&
+ theoreticalHitRate >= 0 &&
+ theoreticalHitRate <= 1;
+ if (wants('uncachedTputPerGpu') && hasTheoreticalHitRate && tputPerGpu) {
+ fields.uncachedTputPerGpu = chartMetric(
+ Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate),
+ );
+ }
+ if (wants('uncachedInputTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) {
+ fields.uncachedInputTputPerGpu = chartMetric(inputTputPerGpu * (1 - theoreticalHitRate));
+ }
if (wants('tpPerMw')) fields.tpPerMw = chartMetric((tputPerGpu * 1000) / hardwarePower);
if (wants('inputTputPerMw') && inputTputPerGpu) {
fields.inputTputPerMw = chartMetric(
From 5f37ca41d82ebd8723443799634f3cbcedb41f9a Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 21:11:45 +0000
Subject: [PATCH 03/13] test(e2e): expect three y-axis options for
input-token-throughput search
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The new Uncached Input Token Throughput per Chip metric also matches the
"input token throughput" search, so the filtered option count is now three.
中文：新增的每芯片无 prefix cache 输入 token 吞吐量指标同样匹配
"input token throughput" 搜索，过滤后的选项数量变为三个。
---
packages/app/cypress/e2e/speed-overlay.cy.ts | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/packages/app/cypress/e2e/speed-overlay.cy.ts b/packages/app/cypress/e2e/speed-overlay.cy.ts
index 5102a6a1..e0c7299b 100644
--- a/packages/app/cypress/e2e/speed-overlay.cy.ts
+++ b/packages/app/cypress/e2e/speed-overlay.cy.ts
@@ -188,7 +188,9 @@ describe('Y-Axis Metric Search', () => {
cy.get('[data-slot="select-content"]')
.find('input[placeholder="Search..."]')
.type('input token throughput');
- cy.get('[data-slot="select-content"]').find('[role="option"]').should('have.length', 2);
+ // Matches Input Token Throughput per Chip, Uncached Input Token Throughput
+ // per Chip, and Input Token Throughput per All in Utility MW.
+ cy.get('[data-slot="select-content"]').find('[role="option"]').should('have.length', 3);
cy.get('[data-slot="select-content"]')
.find('[role="option"]')
.first()
From 92488c5aab84d6e70233c273bef0115afa127125 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 21:18:34 +0000
Subject: [PATCH 04/13] feat(inference): rename uncached throughput metrics to
prompt suffix naming
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Per thread discussion, name the metric after what it measures: the prompt
suffix (theoretically uncachable input) plus output tokens. Like MFU vs HFU,
actual (uncached + output) throughput is always at least the theoretical
(uncachable suffix + output) throughput.
- Uncached Token Throughput per Chip -> Prompt Suffix + Output Token
Throughput per Chip (key y_promptSuffixOutputTputPerGpu)
- Uncached Input Token Throughput per Chip -> Prompt Suffix Token Throughput
per Chip (key y_promptSuffixTputPerGpu)
中文：按讨论将指标以其实际度量对象命名：prompt suffix（理论上无法缓存的输入）
加输出 token。类似 MFU 与 HFU 的关系，实际（未缓存 + 输出）吞吐量总是不低于
理论（不可缓存 suffix + 输出）吞吐量。
- 每芯片无 prefix cache token 吞吐量 -> 每芯片 prompt suffix + 输出 token 吞吐量
- 每芯片无 prefix cache 输入 token 吞吐量 -> 每芯片 prompt suffix token 吞吐量
---
packages/app/cypress/e2e/speed-overlay.cy.ts | 4 +--
.../components/inference/metric-registry.ts | 34 ++++++++++---------
.../app/src/components/inference/types.ts | 6 ++--
packages/app/src/lib/chart-utils.test.ts | 26 +++++++-------
packages/app/src/lib/chart-utils.ts | 18 +++++-----
5 files changed, 44 insertions(+), 44 deletions(-)
diff --git a/packages/app/cypress/e2e/speed-overlay.cy.ts b/packages/app/cypress/e2e/speed-overlay.cy.ts
index e0c7299b..5102a6a1 100644
--- a/packages/app/cypress/e2e/speed-overlay.cy.ts
+++ b/packages/app/cypress/e2e/speed-overlay.cy.ts
@@ -188,9 +188,7 @@ describe('Y-Axis Metric Search', () => {
cy.get('[data-slot="select-content"]')
.find('input[placeholder="Search..."]')
.type('input token throughput');
- // Matches Input Token Throughput per Chip, Uncached Input Token Throughput
- // per Chip, and Input Token Throughput per All in Utility MW.
- cy.get('[data-slot="select-content"]').find('[role="option"]').should('have.length', 3);
+ cy.get('[data-slot="select-content"]').find('[role="option"]').should('have.length', 2);
cy.get('[data-slot="select-content"]')
.find('[role="option"]')
.first()
diff --git a/packages/app/src/components/inference/metric-registry.ts b/packages/app/src/components/inference/metric-registry.ts
index 4a859e37..706130f7 100644
--- a/packages/app/src/components/inference/metric-registry.ts
+++ b/packages/app/src/components/inference/metric-registry.ts
@@ -43,24 +43,26 @@ export const METRIC_REGISTRY = {
titleZh: '每芯片输出 token 吞吐量',
polarity: 'higher',
},
- // Theoretical uncached throughput pair: total / input throughput minus the
+ // Prompt-suffix throughput pair: total / input throughput minus the
// infinite-cache theoretical prefix share (trace-derived, agentic only).
// Deliberately based on the theoretical rate rather than server-observed
- // cache hits so systems with good cache storage aren't penalized.
- uncachedTputPerGpu: {
- field: 'uncachedTputPerGpu.y',
- label: 'Uncached Token Throughput per Chip (tok/s/chip)',
- labelZh: '每芯片无 prefix cache token 吞吐量（tok/s/chip）',
- title: 'Uncached Token Throughput per Chip',
- titleZh: '每芯片无 prefix cache token 吞吐量',
+ // cache hits so systems with good cache storage aren't penalized — like
+ // MFU vs HFU, actual (uncached + output) throughput is always at least the
+ // theoretical (uncachable suffix + output) throughput.
+ promptSuffixOutputTputPerGpu: {
+ field: 'promptSuffixOutputTputPerGpu.y',
+ label: 'Prompt Suffix + Output Token Throughput per Chip (tok/s/chip)',
+ labelZh: '每芯片 prompt suffix + 输出 token 吞吐量（tok/s/chip）',
+ title: 'Prompt Suffix + Output Token Throughput per Chip',
+ titleZh: '每芯片 prompt suffix + 输出 token 吞吐量',
polarity: 'higher',
},
- uncachedInputTputPerGpu: {
- field: 'uncachedInputTputPerGpu.y',
- label: 'Uncached Input Token Throughput per Chip (tok/s/chip)',
- labelZh: '每芯片无 prefix cache 输入 token 吞吐量（tok/s/chip）',
- title: 'Uncached Input Token Throughput per Chip',
- titleZh: '每芯片无 prefix cache 输入 token 吞吐量',
+ promptSuffixTputPerGpu: {
+ field: 'promptSuffixTputPerGpu.y',
+ label: 'Prompt Suffix Token Throughput per Chip (tok/s/chip)',
+ labelZh: '每芯片 prompt suffix token 吞吐量（tok/s/chip）',
+ title: 'Prompt Suffix Token Throughput per Chip',
+ titleZh: '每芯片 prompt suffix token 吞吐量',
polarity: 'higher',
x: 'p90_ttft',
xLabel: 'P90 Time To First Token (s)',
@@ -496,8 +498,8 @@ export const METRIC_CONTROL_GROUPS: readonly MetricControlGroup[] = [
'y_tpPerGpu',
'y_inputTputPerGpu',
'y_outputTputPerGpu',
- 'y_uncachedTputPerGpu',
- 'y_uncachedInputTputPerGpu',
+ 'y_promptSuffixOutputTputPerGpu',
+ 'y_promptSuffixTputPerGpu',
'y_tpPerMw',
'y_inputTputPerMw',
'y_outputTputPerMw',
diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts
index 88ed17b8..43c68fe9 100644
--- a/packages/app/src/components/inference/types.ts
+++ b/packages/app/src/components/inference/types.ts
@@ -274,10 +274,10 @@ export interface InferenceData extends Partial {
expect(point.inputTputPerGpu).toEqual({ y: 300, roof: false });
});
- it('computes theoretical uncached throughput fields when the rate is valid', () => {
+ it('computes prompt-suffix throughput fields when the rate is valid', () => {
const e = entry({
tput_per_gpu: 900,
output_tput_per_gpu: 600,
@@ -702,30 +702,30 @@ describe('createChartDataPoint', () => {
theoretical_cache_hit_rate: 0.8,
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- // uncached input = 300 x (1 - 0.8); uncached total = 900 - 300 x 0.8
- expect(point.uncachedInputTputPerGpu?.y).toBeCloseTo(60);
- expect(point.uncachedTputPerGpu?.y).toBeCloseTo(660);
+ // prompt suffix = 300 x (1 - 0.8); suffix + output = 900 - 300 x 0.8
+ expect(point.promptSuffixTputPerGpu?.y).toBeCloseTo(60);
+ expect(point.promptSuffixOutputTputPerGpu?.y).toBeCloseTo(660);
});
- it('omits theoretical uncached throughput fields when the rate is missing', () => {
+ it('omits prompt-suffix throughput fields when the rate is missing', () => {
const e = entry({ tput_per_gpu: 900, input_tput_per_gpu: 300 });
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- expect(point.uncachedInputTputPerGpu).toBeUndefined();
- expect(point.uncachedTputPerGpu).toBeUndefined();
+ expect(point.promptSuffixTputPerGpu).toBeUndefined();
+ expect(point.promptSuffixOutputTputPerGpu).toBeUndefined();
});
- it('omits theoretical uncached throughput fields when the rate is out of range', () => {
+ it('omits prompt-suffix throughput fields when the rate is out of range', () => {
const e = entry({
tput_per_gpu: 900,
input_tput_per_gpu: 300,
theoretical_cache_hit_rate: 1.2,
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- expect(point.uncachedInputTputPerGpu).toBeUndefined();
- expect(point.uncachedTputPerGpu).toBeUndefined();
+ expect(point.promptSuffixTputPerGpu).toBeUndefined();
+ expect(point.promptSuffixOutputTputPerGpu).toBeUndefined();
});
- it('reports zero uncached input and output-only total at a full theoretical hit rate', () => {
+ it('reports zero prompt suffix and output-only suffix+output at a full theoretical hit rate', () => {
const e = entry({
tput_per_gpu: 900,
output_tput_per_gpu: 600,
@@ -733,8 +733,8 @@ describe('createChartDataPoint', () => {
theoretical_cache_hit_rate: 1,
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- expect(point.uncachedInputTputPerGpu?.y).toBe(0);
- expect(point.uncachedTputPerGpu?.y).toBeCloseTo(600);
+ expect(point.promptSuffixTputPerGpu?.y).toBe(0);
+ expect(point.promptSuffixOutputTputPerGpu?.y).toBeCloseTo(600);
});
it('computes tpPerMw from throughput and hardware power', () => {
diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts
index fef68ed4..544acd72 100644
--- a/packages/app/src/lib/chart-utils.ts
+++ b/packages/app/src/lib/chart-utils.ts
@@ -327,24 +327,24 @@ export function buildDerivedChartFields(
if (wants('inputTputPerGpu') && inputTputPerGpu) {
fields.inputTputPerGpu = chartMetric(inputTputPerGpu);
}
- // Theoretical uncached throughput: subtract the infinite-cache theoretical
- // prefix share of input throughput (rate derived from the trace by the
- // harness). Uses the theoretical rate rather than server-observed cache
- // hits so systems with good cache storage aren't penalized — only agentic
- // trace points carry the rate, so fixed-sequence points omit these fields.
+ // Prompt-suffix throughput: subtract the infinite-cache theoretical prefix
+ // share of input throughput (rate derived from the trace by the harness).
+ // Uses the theoretical rate rather than server-observed cache hits so
+ // systems with good cache storage aren't penalized — only agentic trace
+ // points carry the rate, so fixed-sequence points omit these fields.
const theoreticalHitRate = entry.theoretical_cache_hit_rate;
const hasTheoreticalHitRate =
typeof theoreticalHitRate === 'number' &&
Number.isFinite(theoreticalHitRate) &&
theoreticalHitRate >= 0 &&
theoreticalHitRate <= 1;
- if (wants('uncachedTputPerGpu') && hasTheoreticalHitRate && tputPerGpu) {
- fields.uncachedTputPerGpu = chartMetric(
+ if (wants('promptSuffixOutputTputPerGpu') && hasTheoreticalHitRate && tputPerGpu) {
+ fields.promptSuffixOutputTputPerGpu = chartMetric(
Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate),
);
}
- if (wants('uncachedInputTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) {
- fields.uncachedInputTputPerGpu = chartMetric(inputTputPerGpu * (1 - theoreticalHitRate));
+ if (wants('promptSuffixTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) {
+ fields.promptSuffixTputPerGpu = chartMetric(inputTputPerGpu * (1 - theoreticalHitRate));
}
if (wants('tpPerMw')) fields.tpPerMw = chartMetric((tputPerGpu * 1000) / hardwarePower);
if (wants('inputTputPerMw') && inputTputPerGpu) {
From 55ef345d237a902306ec4270d43fb5da5371115b Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 21:28:07 +0000
Subject: [PATCH 05/13] feat(inference): rename metrics to new input suffix
naming
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Per thread, call the metric "New Input Suffix + Output Token Throughput":
the new (theoretically uncachable) input suffix plus output tokens.
- Prompt Suffix + Output Token Throughput per Chip -> New Input Suffix +
Output Token Throughput per Chip (key y_newInputSuffixOutputTputPerGpu)
- Prompt Suffix Token Throughput per Chip -> New Input Suffix Token
Throughput per Chip (key y_newInputSuffixTputPerGpu)
中文：按讨论将指标更名为 "New Input Suffix + Output Token Throughput"：
新增（理论上无法缓存的）输入 suffix 加输出 token。
- 每芯片 prompt suffix + 输出 token 吞吐量 -> 每芯片新输入 suffix + 输出 token 吞吐量
- 每芯片 prompt suffix token 吞吐量 -> 每芯片新输入 suffix token 吞吐量
---
.../components/inference/metric-registry.ts | 30 +++++++++----------
.../app/src/components/inference/types.ts | 6 ++--
packages/app/src/lib/chart-utils.test.ts | 26 ++++++++--------
packages/app/src/lib/chart-utils.ts | 10 +++----
4 files changed, 36 insertions(+), 36 deletions(-)
diff --git a/packages/app/src/components/inference/metric-registry.ts b/packages/app/src/components/inference/metric-registry.ts
index 706130f7..65ff5d6b 100644
--- a/packages/app/src/components/inference/metric-registry.ts
+++ b/packages/app/src/components/inference/metric-registry.ts
@@ -43,26 +43,26 @@ export const METRIC_REGISTRY = {
titleZh: '每芯片输出 token 吞吐量',
polarity: 'higher',
},
- // Prompt-suffix throughput pair: total / input throughput minus the
+ // New-input-suffix throughput pair: total / input throughput minus the
// infinite-cache theoretical prefix share (trace-derived, agentic only).
// Deliberately based on the theoretical rate rather than server-observed
// cache hits so systems with good cache storage aren't penalized — like
// MFU vs HFU, actual (uncached + output) throughput is always at least the
// theoretical (uncachable suffix + output) throughput.
- promptSuffixOutputTputPerGpu: {
- field: 'promptSuffixOutputTputPerGpu.y',
- label: 'Prompt Suffix + Output Token Throughput per Chip (tok/s/chip)',
- labelZh: '每芯片 prompt suffix + 输出 token 吞吐量（tok/s/chip）',
- title: 'Prompt Suffix + Output Token Throughput per Chip',
- titleZh: '每芯片 prompt suffix + 输出 token 吞吐量',
+ newInputSuffixOutputTputPerGpu: {
+ field: 'newInputSuffixOutputTputPerGpu.y',
+ label: 'New Input Suffix + Output Token Throughput per Chip (tok/s/chip)',
+ labelZh: '每芯片新输入 suffix + 输出 token 吞吐量（tok/s/chip）',
+ title: 'New Input Suffix + Output Token Throughput per Chip',
+ titleZh: '每芯片新输入 suffix + 输出 token 吞吐量',
polarity: 'higher',
},
- promptSuffixTputPerGpu: {
- field: 'promptSuffixTputPerGpu.y',
- label: 'Prompt Suffix Token Throughput per Chip (tok/s/chip)',
- labelZh: '每芯片 prompt suffix token 吞吐量（tok/s/chip）',
- title: 'Prompt Suffix Token Throughput per Chip',
- titleZh: '每芯片 prompt suffix token 吞吐量',
+ newInputSuffixTputPerGpu: {
+ field: 'newInputSuffixTputPerGpu.y',
+ label: 'New Input Suffix Token Throughput per Chip (tok/s/chip)',
+ labelZh: '每芯片新输入 suffix token 吞吐量（tok/s/chip）',
+ title: 'New Input Suffix Token Throughput per Chip',
+ titleZh: '每芯片新输入 suffix token 吞吐量',
polarity: 'higher',
x: 'p90_ttft',
xLabel: 'P90 Time To First Token (s)',
@@ -498,8 +498,8 @@ export const METRIC_CONTROL_GROUPS: readonly MetricControlGroup[] = [
'y_tpPerGpu',
'y_inputTputPerGpu',
'y_outputTputPerGpu',
- 'y_promptSuffixOutputTputPerGpu',
- 'y_promptSuffixTputPerGpu',
+ 'y_newInputSuffixOutputTputPerGpu',
+ 'y_newInputSuffixTputPerGpu',
'y_tpPerMw',
'y_inputTputPerMw',
'y_outputTputPerMw',
diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts
index 43c68fe9..af1f214f 100644
--- a/packages/app/src/components/inference/types.ts
+++ b/packages/app/src/components/inference/types.ts
@@ -274,10 +274,10 @@ export interface InferenceData extends Partial {
expect(point.inputTputPerGpu).toEqual({ y: 300, roof: false });
});
- it('computes prompt-suffix throughput fields when the rate is valid', () => {
+ it('computes new-input-suffix throughput fields when the rate is valid', () => {
const e = entry({
tput_per_gpu: 900,
output_tput_per_gpu: 600,
@@ -702,30 +702,30 @@ describe('createChartDataPoint', () => {
theoretical_cache_hit_rate: 0.8,
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- // prompt suffix = 300 x (1 - 0.8); suffix + output = 900 - 300 x 0.8
- expect(point.promptSuffixTputPerGpu?.y).toBeCloseTo(60);
- expect(point.promptSuffixOutputTputPerGpu?.y).toBeCloseTo(660);
+ // input suffix = 300 x (1 - 0.8); suffix + output = 900 - 300 x 0.8
+ expect(point.newInputSuffixTputPerGpu?.y).toBeCloseTo(60);
+ expect(point.newInputSuffixOutputTputPerGpu?.y).toBeCloseTo(660);
});
- it('omits prompt-suffix throughput fields when the rate is missing', () => {
+ it('omits new-input-suffix throughput fields when the rate is missing', () => {
const e = entry({ tput_per_gpu: 900, input_tput_per_gpu: 300 });
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- expect(point.promptSuffixTputPerGpu).toBeUndefined();
- expect(point.promptSuffixOutputTputPerGpu).toBeUndefined();
+ expect(point.newInputSuffixTputPerGpu).toBeUndefined();
+ expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined();
});
- it('omits prompt-suffix throughput fields when the rate is out of range', () => {
+ it('omits new-input-suffix throughput fields when the rate is out of range', () => {
const e = entry({
tput_per_gpu: 900,
input_tput_per_gpu: 300,
theoretical_cache_hit_rate: 1.2,
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- expect(point.promptSuffixTputPerGpu).toBeUndefined();
- expect(point.promptSuffixOutputTputPerGpu).toBeUndefined();
+ expect(point.newInputSuffixTputPerGpu).toBeUndefined();
+ expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined();
});
- it('reports zero prompt suffix and output-only suffix+output at a full theoretical hit rate', () => {
+ it('reports zero input suffix and output-only suffix+output at a full theoretical hit rate', () => {
const e = entry({
tput_per_gpu: 900,
output_tput_per_gpu: 600,
@@ -733,8 +733,8 @@ describe('createChartDataPoint', () => {
theoretical_cache_hit_rate: 1,
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- expect(point.promptSuffixTputPerGpu?.y).toBe(0);
- expect(point.promptSuffixOutputTputPerGpu?.y).toBeCloseTo(600);
+ expect(point.newInputSuffixTputPerGpu?.y).toBe(0);
+ expect(point.newInputSuffixOutputTputPerGpu?.y).toBeCloseTo(600);
});
it('computes tpPerMw from throughput and hardware power', () => {
diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts
index 544acd72..45586727 100644
--- a/packages/app/src/lib/chart-utils.ts
+++ b/packages/app/src/lib/chart-utils.ts
@@ -327,7 +327,7 @@ export function buildDerivedChartFields(
if (wants('inputTputPerGpu') && inputTputPerGpu) {
fields.inputTputPerGpu = chartMetric(inputTputPerGpu);
}
- // Prompt-suffix throughput: subtract the infinite-cache theoretical prefix
+ // New-input-suffix throughput: subtract the infinite-cache theoretical prefix
// share of input throughput (rate derived from the trace by the harness).
// Uses the theoretical rate rather than server-observed cache hits so
// systems with good cache storage aren't penalized — only agentic trace
@@ -338,13 +338,13 @@ export function buildDerivedChartFields(
Number.isFinite(theoreticalHitRate) &&
theoreticalHitRate >= 0 &&
theoreticalHitRate <= 1;
- if (wants('promptSuffixOutputTputPerGpu') && hasTheoreticalHitRate && tputPerGpu) {
- fields.promptSuffixOutputTputPerGpu = chartMetric(
+ if (wants('newInputSuffixOutputTputPerGpu') && hasTheoreticalHitRate && tputPerGpu) {
+ fields.newInputSuffixOutputTputPerGpu = chartMetric(
Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate),
);
}
- if (wants('promptSuffixTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) {
- fields.promptSuffixTputPerGpu = chartMetric(inputTputPerGpu * (1 - theoreticalHitRate));
+ if (wants('newInputSuffixTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) {
+ fields.newInputSuffixTputPerGpu = chartMetric(inputTputPerGpu * (1 - theoreticalHitRate));
}
if (wants('tpPerMw')) fields.tpPerMw = chartMetric((tputPerGpu * 1000) / hardwarePower);
if (wants('inputTputPerMw') && inputTputPerGpu) {
From cc252f0a8dac28a27b41d369a419d83661ee0573 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 21:39:43 +0000
Subject: [PATCH 06/13] fix(inference): require input throughput for suffix
throughput metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Bugbot: missing input throughput is normalized to 0 upstream, so the
suffix+output metric would silently equal total throughput. Omit both
fields when input throughput is absent.
中文：Bugbot 发现上游会将缺失的输入吞吐量归一化为 0，导致 suffix + 输出
指标退化为总吞吐量。现在输入吞吐量缺失时两个字段均省略。
---
packages/app/src/lib/chart-utils.test.ts | 11 +++++++++++
packages/app/src/lib/chart-utils.ts | 9 ++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/packages/app/src/lib/chart-utils.test.ts b/packages/app/src/lib/chart-utils.test.ts
index cbaea2ac..9daf22eb 100644
--- a/packages/app/src/lib/chart-utils.test.ts
+++ b/packages/app/src/lib/chart-utils.test.ts
@@ -714,6 +714,17 @@ describe('createChartDataPoint', () => {
expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined();
});
+ it('omits new-input-suffix throughput fields when input throughput is missing', () => {
+ const e = entry({
+ tput_per_gpu: 900,
+ input_tput_per_gpu: 0,
+ theoretical_cache_hit_rate: 0.8,
+ });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ expect(point.newInputSuffixTputPerGpu).toBeUndefined();
+ expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined();
+ });
+
it('omits new-input-suffix throughput fields when the rate is out of range', () => {
const e = entry({
tput_per_gpu: 900,
diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts
index 45586727..74c6d249 100644
--- a/packages/app/src/lib/chart-utils.ts
+++ b/packages/app/src/lib/chart-utils.ts
@@ -338,7 +338,14 @@ export function buildDerivedChartFields(
Number.isFinite(theoreticalHitRate) &&
theoreticalHitRate >= 0 &&
theoreticalHitRate <= 1;
- if (wants('newInputSuffixOutputTputPerGpu') && hasTheoreticalHitRate && tputPerGpu) {
+ // Both metrics require input throughput: without it the prefix share can't
+ // be subtracted and suffix+output would silently equal total throughput.
+ if (
+ wants('newInputSuffixOutputTputPerGpu') &&
+ hasTheoreticalHitRate &&
+ tputPerGpu &&
+ inputTputPerGpu
+ ) {
fields.newInputSuffixOutputTputPerGpu = chartMetric(
Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate),
);
From 9f749c816c2779d5769d3dfed9d2ccb59bf67714 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:01:36 +0000
Subject: [PATCH 07/13] feat(inference): new input suffix + output TFLOP/s per
chip y-metric
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Achieved model TFLOP/s on the theoretically necessary tokens only:
FLOPs/token = 2 x active params (GEMM-only Kaplan/PaLM convention),
times the new-input-suffix + output token throughput, so cached prefix
tokens contribute no compute credit. Attention-score FLOPs are excluded:
they depend on per-request context lengths (only aggregate token counts
reach the chart layer) and on the attention implementation
(MHA/GQA/MLA/linear), so 2N_active is the comparable cross-model lower
bound - same spirit as MFU counting only theoretically required work.
Active params come from the model-architectures registry; points whose
model lacks an entry omit the field.
中文：新增每芯片新输入 suffix + 输出 TFLOP/s 指标，仅统计理论上必需的
token：FLOPs/token = 2 x 激活参数量（GEMM-only Kaplan/PaLM 口径），
乘以新输入 suffix + 输出 token 吞吐量，缓存的 prefix token 不计入算力。
不含 attention score FLOPs：其依赖每个请求的上下文长度（图表层只有
聚合 token 数）及 attention 实现（MHA/GQA/MLA/linear），2N_active 是
可跨模型比较的下界，与 MFU 只统计理论必需计算量的思路一致。激活参数量
取自 model-architectures 注册表；缺少架构条目的模型省略该字段。
---
.../components/inference/metric-registry.ts | 17 +++++++++++++
.../app/src/components/inference/types.ts | 2 ++
packages/app/src/lib/chart-utils.test.ts | 24 +++++++++++++++++++
packages/app/src/lib/chart-utils.ts | 20 ++++++++++++++++
4 files changed, 63 insertions(+)
diff --git a/packages/app/src/components/inference/metric-registry.ts b/packages/app/src/components/inference/metric-registry.ts
index 65ff5d6b..144f59d5 100644
--- a/packages/app/src/components/inference/metric-registry.ts
+++ b/packages/app/src/components/inference/metric-registry.ts
@@ -68,6 +68,22 @@ export const METRIC_REGISTRY = {
xLabel: 'P90 Time To First Token (s)',
heading: 'vs. P90 Time To First Token',
},
+ // Achieved model TFLOP/s on the theoretically necessary tokens only:
+ // 2 × active params per token (GEMM-only Kaplan/PaLM convention) times the
+ // new-input-suffix + output token throughput above. Attention-score FLOPs
+ // are deliberately excluded — they depend on each request's context length
+ // (only aggregate token counts reach the chart layer) and on the attention
+ // implementation (MHA/GQA/MLA/linear), so 2N_active is the comparable
+ // cross-model lower bound, in the same spirit as MFU counting only
+ // theoretically required work.
+ newInputSuffixOutputTflopsPerGpu: {
+ field: 'newInputSuffixOutputTflopsPerGpu.y',
+ label: 'New Input Suffix + Output TFLOP/s per Chip (TFLOP/s/chip)',
+ labelZh: '每芯片新输入 suffix + 输出 TFLOP/s（TFLOP/s/chip）',
+ title: 'New Input Suffix + Output TFLOP/s per Chip',
+ titleZh: '每芯片新输入 suffix + 输出 TFLOP/s',
+ polarity: 'higher',
+ },
tpPerMw: {
field: 'tpPerMw.y',
label: 'Token Throughput per All in Utility MW (tok/s/MW)',
@@ -500,6 +516,7 @@ export const METRIC_CONTROL_GROUPS: readonly MetricControlGroup[] = [
'y_outputTputPerGpu',
'y_newInputSuffixOutputTputPerGpu',
'y_newInputSuffixTputPerGpu',
+ 'y_newInputSuffixOutputTflopsPerGpu',
'y_tpPerMw',
'y_inputTputPerMw',
'y_outputTputPerMw',
diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts
index af1f214f..b7010759 100644
--- a/packages/app/src/components/inference/types.ts
+++ b/packages/app/src/components/inference/types.ts
@@ -278,6 +278,8 @@ export interface InferenceData extends Partial {
expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined();
});
+ it('derives suffix+output TFLOP/s from 2 × active params for known models', () => {
+ const e = entry({
+ model: 'DeepSeek-R1-0528', // 37B active params
+ tput_per_gpu: 900,
+ input_tput_per_gpu: 600,
+ output_tput_per_gpu: 300,
+ theoretical_cache_hit_rate: 0.8,
+ });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ // suffix+output = 900 - 600 × 0.8 = 420 tok/s → 2 × 37e9 × 420 / 1e12
+ expect(point.newInputSuffixOutputTflopsPerGpu?.y).toBeCloseTo(31.08, 5);
+ });
+
+ it('omits suffix+output TFLOP/s for models without architecture data', () => {
+ const e = entry({
+ model: 'not-a-real-model',
+ tput_per_gpu: 900,
+ input_tput_per_gpu: 600,
+ theoretical_cache_hit_rate: 0.8,
+ });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ expect(point.newInputSuffixOutputTflopsPerGpu).toBeUndefined();
+ });
+
it('omits new-input-suffix throughput fields when input throughput is missing', () => {
const e = entry({
tput_per_gpu: 900,
diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts
index 74c6d249..5c9e477d 100644
--- a/packages/app/src/lib/chart-utils.ts
+++ b/packages/app/src/lib/chart-utils.ts
@@ -18,6 +18,8 @@ import {
type BenchmarkMetricKey,
} from '@/components/inference/metric-registry';
import { getGpuSpecs, isKnownGpu } from '@/lib/constants';
+import type { Model } from '@/lib/data-mappings';
+import { getModelArchitecture } from '@/lib/model-architectures';
import { getVendor, type Vendor } from '@/lib/dynamic-colors';
import type { Locale } from '@/lib/i18n';
@@ -350,6 +352,24 @@ export function buildDerivedChartFields(
Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate),
);
}
+ // Achieved model TFLOP/s per chip on the theoretically necessary tokens:
+ // FLOPs/token = 2 × active params (GEMM-only Kaplan/PaLM convention;
+ // attention-score FLOPs excluded — see metric-registry doc comment).
+ // activeParams is in billions, so tok/s × 2 × 1e9 params / 1e12 = ÷1000.
+ if (
+ wants('newInputSuffixOutputTflopsPerGpu') &&
+ hasTheoreticalHitRate &&
+ tputPerGpu &&
+ inputTputPerGpu
+ ) {
+ const activeParams = getModelArchitecture(entry.model as Model)?.activeParams;
+ if (activeParams) {
+ const suffixOutputTput = Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate);
+ fields.newInputSuffixOutputTflopsPerGpu = chartMetric(
+ (2 * activeParams * suffixOutputTput) / 1000,
+ );
+ }
+ }
if (wants('newInputSuffixTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) {
fields.newInputSuffixTputPerGpu = chartMetric(inputTputPerGpu * (1 - theoreticalHitRate));
}
From 94de05e33637f5fbfd8b75e73d8362e31ccb84f1 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:19:28 +0000
Subject: [PATCH 08/13] fix(zh-copy): exempt TFLOP/s/chip as an English unit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The chip-untranslated rule only knew tok/s/chip; the new TFLOP/s/chip
y-metric unit tripped it. Units stay English per AGENTS.md rule 6, so
extend the exemption to [KMGT]FLOP/s/chip and add a fixture.
中文：chip-untranslated 规则原本只豁免 tok/s/chip，新增的 TFLOP/s/chip
Y 轴指标单位触发了误报。按 AGENTS.md 第 6 条单位保留英文，故将豁免
扩展到 [KMGT]FLOP/s/chip 并补充用例。
---
packages/app/src/lib/zh-copy-mechanical-regressions.jsonl | 1 +
packages/app/src/lib/zh-copy.test.ts | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl b/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl
index 8b9e22d1..ce9920f0 100644
--- a/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl
+++ b/packages/app/src/lib/zh-copy-mechanical-regressions.jsonl
@@ -16,3 +16,4 @@
{"rule": "duplicated-technical-loanword", "kind": "exemption", "text": "'这里所说的预热（warmup）发生在正式测量之前。'", "note": "Chinese-first parenthetical explanation is intentional for a broader audience"}
{"rule": "duplicated-technical-loanword", "kind": "exemption", "text": "'随机种子（seed）用于固定会话采样。'", "note": "Chinese-first parenthetical explanation remains allowed"}
{"rule": "duplicated-technical-loanword", "kind": "exemption", "text": "'将 KV cache 卸载（offload）到 DRAM。'", "note": "Chinese-first parenthetical explanation remains allowed"}
+{"rule": "chip-untranslated", "kind": "exemption", "text": "labelZh: '每芯片新输入 suffix + 输出 TFLOP/s（TFLOP/s/chip）',", "note": "TFLOP/s/chip is a unit and stays English like tok/s/chip"}
diff --git a/packages/app/src/lib/zh-copy.test.ts b/packages/app/src/lib/zh-copy.test.ts
index 9f47e170..27c6b39a 100644
--- a/packages/app/src/lib/zh-copy.test.ts
+++ b/packages/app/src/lib/zh-copy.test.ts
@@ -84,7 +84,8 @@ function segment(raw: string, isProse: boolean): string[] {
// "Chip" is an ordinary English noun whose Chinese equivalent (芯片) is what
// readers actually use, so it cannot inherit GPU's exemption. Units keep the
// English form per AGENTS.md rule 6 — that is the one recorded exception.
-const CHIP_UNITS = /(?:tok|tokens?)\/s\/chip|\$\/chip[/-](?:hr|hour)|[A-Za-z]Chip\b|\bChip[A-Z]/giu;
+const CHIP_UNITS =
+ /(?:tok|tokens?|[KMGT]?FLOPs?)\/s\/chip|\$\/chip[/-](?:hr|hour)|[A-Za-z]Chip\b|\bChip[A-Z]/giu;
const RULES: Rule[] = [
{
From 502c3b83e095df5b9c59129b40c5972acd6ee428 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:17:36 +0000
Subject: [PATCH 09/13] feat(metrics): price per-model attention FLOPs into
suffix+output TFLOP/s
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Replace the GEMM-only FLOP estimate with GEMM + per-architecture attention
FLOPs integrated in closed form over each run's true (ISL, OSL) request
distribution.
- db: stats v9 adds exact joint request-length moment sums
(n, ΣP, ΣP², ΣO, ΣO², ΣPO) to the aggregate-stats bundle and to the
derived-agentic-metrics fallback/self-heal path.
- app: new attention-flops module prices any affine+capped per-context
cost F(L) = lin·L + coeff·min(L, cap) + const exactly from the moments,
under the theoretical infinite-cache prefix (cached prefix excluded from
computed tokens but still attended by the suffix).
- model-architectures: cited attention cost specs for DeepSeek-R1 (absorbed
MLA), DeepSeek-V4-Pro (HCA + capped CSA + indexer), Llama 3.x 70B (GQA),
gpt-oss (full + sliding-window), Kimi K2.5 (MLA), Kimi K3 (KDA + gated
MLA NoPE), MiniMax M2.5 (GQA), MiniMax M3 (dense + capped MSA).
- chart-utils: TFLOP/s/chip = suffix+output tput × (2·N_active + attention
FLOPs/token) and is omitted when moments or an attention spec are missing.
中文：将 TFLOP/s/chip 指标从仅 GEMM 估算升级为 GEMM + 各模型注意力 FLOPs。
数据库统计 v9 新增请求长度联合矩（n、ΣP、ΣP²、ΣO、ΣO²、ΣPO），前端按各架构
的注意力成本公式 F(L)=lin·L+coeff·min(L,cap)+const 在理论无限缓存前缀假设下
闭式积分。覆盖 DeepSeek-R1/V4-Pro、Llama 3.x 70B、gpt-oss、Kimi K2.5/K3、
MiniMax M2.5/M3 八个架构；缺少矩数据或注意力规格时该指标不显示。
---
.../api/v1/derived-agentic-metrics/route.ts | 2 +
.../inference/hooks/useChartData.ts | 32 +++-
.../components/inference/metric-registry.ts | 15 +-
.../app/src/components/inference/types.ts | 9 ++
.../inference/utils/canonicalFrontier.test.ts | 10 +-
.../hooks/api/use-derived-agentic-metrics.ts | 4 +
packages/app/src/lib/attention-flops.test.ts | 143 ++++++++++++++++++
packages/app/src/lib/attention-flops.ts | 142 +++++++++++++++++
packages/app/src/lib/benchmark-transform.ts | 10 ++
packages/app/src/lib/chart-utils.test.ts | 40 ++++-
packages/app/src/lib/chart-utils.ts | 30 +++-
packages/app/src/lib/model-architectures.ts | 114 ++++++++++++++
.../src/etl/compute-aggregate-stats.test.ts | 12 ++
.../db/src/etl/compute-aggregate-stats.ts | 19 ++-
packages/db/src/queries/agentic-aggregates.ts | 5 +-
packages/db/src/queries/agentic-shared.ts | 69 ++++++++-
.../queries/derived-agentic-metrics.test.ts | 42 ++++-
.../db/src/queries/derived-agentic-metrics.ts | 28 +++-
18 files changed, 690 insertions(+), 36 deletions(-)
create mode 100644 packages/app/src/lib/attention-flops.test.ts
create mode 100644 packages/app/src/lib/attention-flops.ts
diff --git a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts
index aa483fc3..2564d33e 100644
--- a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts
+++ b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts
@@ -38,6 +38,8 @@ const getCachedDerivedAgenticMetrics = cachedQuery(
* - p75/p90_e2e_norm_intvty: slow-tail OSL / E2E-latency in tok/s/user,
* computed as 1 / pXX(per-request E2EL/OSL) across every turn in every
* session — plain interactivity charged for the prefill wait.
+ * - request_length_moments: exact joint (ISL, OSL) moment sums over profiling
+ * requests, used client-side for closed-form attention-FLOP pricing.
*
* Ids without a trace_replay blob or with unparseable records are omitted.
*/
diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts
index e293ca1c..796c45aa 100644
--- a/packages/app/src/components/inference/hooks/useChartData.ts
+++ b/packages/app/src/components/inference/hooks/useChartData.ts
@@ -18,7 +18,10 @@ import {
resolveComparisonEntries,
} from '@/components/inference/utils/comparisonEntry';
import { useBenchmarks, benchmarkQueryOptions } from '@/hooks/api/use-benchmarks';
+import { useDerivedAgenticMetrics } from '@/hooks/api/use-derived-agentic-metrics';
import type { BenchmarkRow } from '@/lib/api';
+import type { RequestLengthMoments } from '@/lib/attention-flops';
+import { isPersistedBenchmarkId } from '@/lib/benchmark-id';
import {
GPU_ALIAS_TO_CANONICAL,
getModelSortIndex,
@@ -384,6 +387,31 @@ export function useChartData(
overviewHistoryPair?.baselineConfigKey,
]);
+ // Request-length moment sums for agentic points — the attention-FLOPs input
+ // of the TFLOP/s-per-chip y-metric. Shares the 'derived-agentic-metrics'
+ // react-query cache with ChartDisplay's derived-x fetch, so switching between
+ // the two costs one request. The chart renders without them first; the
+ // TFLOP/s metric fills in when they arrive.
+ const momentTargetIds = useMemo(() => {
+ const ids = new Set();
+ for (const r of rows) {
+ const numericId = typeof r.id === 'number' ? r.id : Number(r.id);
+ if (r.benchmark_type === 'agentic_traces' && isPersistedBenchmarkId(numericId)) {
+ ids.add(numericId);
+ }
+ }
+ return [...ids].sort((a, b) => a - b);
+ }, [rows]);
+ const derivedAgenticQuery = useDerivedAgenticMetrics(momentTargetIds, momentTargetIds.length > 0);
+ const requestLengthMomentsById = useMemo(() => {
+ if (!derivedAgenticQuery.data) return undefined;
+ const map: Record = {};
+ for (const metric of Object.values(derivedAgenticQuery.data)) {
+ map[metric.id] = metric.request_length_moments;
+ }
+ return map;
+ }, [derivedAgenticQuery.data]);
+
// Transform filtered rows into chart data
const { chartData, hardwareConfig: rawHardwareConfig } = useMemo(() => {
if (rows.length === 0)
@@ -391,8 +419,8 @@ export function useChartData(
chartData: [] as InferenceData[][],
hardwareConfig: {} as HardwareConfig,
};
- return transformBenchmarkRows(rows, selectedPercentile);
- }, [rows, selectedPercentile]);
+ return transformBenchmarkRows(rows, selectedPercentile, requestLengthMomentsById);
+ }, [rows, selectedPercentile, requestLengthMomentsById]);
// Sort hardware config — stabilize reference when keys haven't changed.
// Different sequences for the same model often have the same GPU configs,
diff --git a/packages/app/src/components/inference/metric-registry.ts b/packages/app/src/components/inference/metric-registry.ts
index 144f59d5..6701385c 100644
--- a/packages/app/src/components/inference/metric-registry.ts
+++ b/packages/app/src/components/inference/metric-registry.ts
@@ -69,13 +69,14 @@ export const METRIC_REGISTRY = {
heading: 'vs. P90 Time To First Token',
},
// Achieved model TFLOP/s on the theoretically necessary tokens only:
- // 2 × active params per token (GEMM-only Kaplan/PaLM convention) times the
- // new-input-suffix + output token throughput above. Attention-score FLOPs
- // are deliberately excluded — they depend on each request's context length
- // (only aggregate token counts reach the chart layer) and on the attention
- // implementation (MHA/GQA/MLA/linear), so 2N_active is the comparable
- // cross-model lower bound, in the same spirit as MFU counting only
- // theoretically required work.
+ // (2 × active params + per-model attention FLOPs per computed token) times
+ // the new-input-suffix + output token throughput above. The attention term
+ // integrates each architecture's mechanism (MLA, CSA/HCA + indexers, MSA,
+ // KDA + gated MLA, sliding/full GQA — specs in model-architectures.ts,
+ // math in attention-flops.ts) over the run's exact per-request (ISL, OSL)
+ // sums at the theoretical cache hit rate, so different request-length
+ // distributions are priced individually and summed. Points without stored
+ // request-length moments or an attention spec omit the metric.
newInputSuffixOutputTflopsPerGpu: {
field: 'newInputSuffixOutputTflopsPerGpu.y',
label: 'New Input Suffix + Output TFLOP/s per Chip (TFLOP/s/chip)',
diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts
index b7010759..b8d6000f 100644
--- a/packages/app/src/components/inference/types.ts
+++ b/packages/app/src/components/inference/types.ts
@@ -1,6 +1,7 @@
import type React from 'react';
import type { WorkerPower } from '@semianalysisai/inferencex-db/queries/benchmarks';
+import type { RequestLengthMoments } from '@/lib/attention-flops';
import type { HardwareEntry } from '@/lib/constants';
import type { Model, Sequence } from '@/lib/data-mappings';
import type { MetricKey } from './metric-registry';
@@ -219,6 +220,14 @@ export interface AggDataEntry {
total_prompt_tokens?: number;
/** Total generated (output) tokens. */
total_generation_tokens?: number;
+ /**
+ * Exact joint (ISL, OSL) sums over the run's request population, fetched
+ * from the derived-agentic-metrics endpoint and merged in by
+ * `transformBenchmarkRows`. Feeds the attention-FLOPs term of the
+ * TFLOP/s-per-chip y-metric; absent on fixed-seq points, unofficial-run
+ * overlays, and rows whose stats haven't been fetched yet.
+ */
+ request_length_moments?: RequestLengthMoments | null;
}
/**
diff --git a/packages/app/src/components/inference/utils/canonicalFrontier.test.ts b/packages/app/src/components/inference/utils/canonicalFrontier.test.ts
index 90ddb549..9896b2a4 100644
--- a/packages/app/src/components/inference/utils/canonicalFrontier.test.ts
+++ b/packages/app/src/components/inference/utils/canonicalFrontier.test.ts
@@ -25,9 +25,9 @@ describe('canonicalNormalizedFrontierIds', () => {
it('computes the true normalized-interactivity frontier', () => {
const points = [point(1, 100), point(2, 150), point(3, 120)];
const metrics = {
- 1: { id: 1, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10 },
- 2: { id: 2, p75_e2e_norm_intvty: 20, p90_e2e_norm_intvty: 20 },
- 3: { id: 3, p75_e2e_norm_intvty: 30, p90_e2e_norm_intvty: 30 },
+ 1: { id: 1, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10, request_length_moments: null },
+ 2: { id: 2, p75_e2e_norm_intvty: 20, p90_e2e_norm_intvty: 20, request_length_moments: null },
+ 3: { id: 3, p75_e2e_norm_intvty: 30, p90_e2e_norm_intvty: 30, request_length_moments: null },
};
expect(
@@ -38,8 +38,8 @@ describe('canonicalNormalizedFrontierIds', () => {
it('keeps frontiers independent across dates', () => {
const points = [point(1, 500, { date: '2026-08-01' }), point(2, 100, { date: '2026-08-02' })];
const metrics = {
- 1: { id: 1, p75_e2e_norm_intvty: 50, p90_e2e_norm_intvty: 50 },
- 2: { id: 2, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10 },
+ 1: { id: 1, p75_e2e_norm_intvty: 50, p90_e2e_norm_intvty: 50, request_length_moments: null },
+ 2: { id: 2, p75_e2e_norm_intvty: 10, p90_e2e_norm_intvty: 10, request_length_moments: null },
};
expect(
[...canonicalNormalizedFrontierIds(points, metrics, 'p90', 'upper_left')!].toSorted(),
diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts
index aa2c659c..82831ee7 100644
--- a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts
+++ b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts
@@ -1,3 +1,5 @@
+import type { RequestLengthMoments } from '@/lib/attention-flops';
+
import { bulkIdsFetcher, useBulkIdsQuery } from './benchmark-id-query';
export interface DerivedAgenticMetric {
@@ -7,6 +9,8 @@ export interface DerivedAgenticMetric {
p75_e2e_norm_intvty: number | null;
/** Slow-tail P90 E2E Normalized Interactivity in tok/s/user — 1 / p90(per-request E2EL/OSL). */
p90_e2e_norm_intvty: number | null;
+ /** Exact joint (ISL, OSL) sums over the run's requests — attention-FLOPs input. */
+ request_length_moments: RequestLengthMoments | null;
}
export type DerivedAgenticMetricMap = Record;
diff --git a/packages/app/src/lib/attention-flops.test.ts b/packages/app/src/lib/attention-flops.test.ts
new file mode 100644
index 00000000..43c33b82
--- /dev/null
+++ b/packages/app/src/lib/attention-flops.test.ts
@@ -0,0 +1,143 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+ attentionFlopsPerComputedToken,
+ sumComputedTokens,
+ sumContexts,
+ type AttentionCostSpec,
+ type RequestLengthMoments,
+} from './attention-flops';
+
+/** Build exact moments from raw (ISL, OSL) pairs. */
+function momentsOf(pairs: [number, number][]): RequestLengthMoments {
+ return {
+ n: pairs.length,
+ sumIsl: pairs.reduce((s, [p]) => s + p, 0),
+ sumIslSq: pairs.reduce((s, [p]) => s + p * p, 0),
+ sumOsl: pairs.reduce((s, [, o]) => s + o, 0),
+ sumOslSq: pairs.reduce((s, [, o]) => s + o * o, 0),
+ sumIslOsl: pairs.reduce((s, [p, o]) => s + p * o, 0),
+ };
+}
+
+/**
+ * Brute-force reference: enumerate every computed token's context. A request
+ * (P, O) at rate r computes its suffix at contexts r·P+1 … P, then decodes at
+ * P+1 … P+O. Use rates where r·P is an integer so enumeration is exact.
+ */
+function bruteForceContexts(pairs: [number, number][], rate: number): number[] {
+ const contexts: number[] = [];
+ for (const [p, o] of pairs) {
+ const cached = rate * p;
+ for (let t = cached + 1; t <= p + o; t++) contexts.push(t);
+ }
+ return contexts;
+}
+
+describe('sumContexts / sumComputedTokens', () => {
+ it('matches brute-force enumeration at rate 0', () => {
+ const pairs: [number, number][] = [
+ [4, 2],
+ [10, 3],
+ [1, 1],
+ ];
+ const contexts = bruteForceContexts(pairs, 0);
+ const m = momentsOf(pairs);
+ expect(sumContexts(m, 0)).toBe(contexts.reduce((a, b) => a + b, 0));
+ expect(sumComputedTokens(m, 0)).toBe(contexts.length);
+ });
+
+ it('matches brute-force enumeration at rate 0.5 (integer cached prefixes)', () => {
+ const pairs: [number, number][] = [
+ [4, 2],
+ [10, 3],
+ [100, 7],
+ ];
+ const contexts = bruteForceContexts(pairs, 0.5);
+ const m = momentsOf(pairs);
+ expect(sumContexts(m, 0.5)).toBe(contexts.reduce((a, b) => a + b, 0));
+ expect(sumComputedTokens(m, 0.5)).toBe(contexts.length);
+ });
+
+ it('reduces to decode-only sums at rate 1', () => {
+ const pairs: [number, number][] = [
+ [4, 2],
+ [10, 3],
+ ];
+ const contexts = bruteForceContexts(pairs, 1);
+ const m = momentsOf(pairs);
+ expect(sumContexts(m, 1)).toBe(contexts.reduce((a, b) => a + b, 0));
+ expect(sumComputedTokens(m, 1)).toBe(contexts.length);
+ });
+});
+
+describe('attentionFlopsPerComputedToken', () => {
+ const pairs: [number, number][] = [
+ [4, 2],
+ [10, 3],
+ ];
+ const m = momentsOf(pairs);
+
+ it('prices a pure linear spec exactly (per-context coefficient)', () => {
+ const spec: AttentionCostSpec = {
+ groups: [{ label: 'dense', layers: 5, linPerCtx: 1000 }],
+ };
+ const contexts = bruteForceContexts(pairs, 0.5);
+ const expected = (5 * 1000 * contexts.reduce((a, b) => a + b, 0)) / contexts.length;
+ expect(attentionFlopsPerComputedToken(spec, m, 0.5)).toBeCloseTo(expected, 8);
+ });
+
+ it('prices a constant spec independently of context', () => {
+ const spec: AttentionCostSpec = {
+ groups: [{ label: 'linear-attn', layers: 3, constPerToken: 7 }],
+ };
+ expect(attentionFlopsPerComputedToken(spec, m, 0)).toBeCloseTo(21, 10);
+ expect(attentionFlopsPerComputedToken(spec, m, 0.5)).toBeCloseTo(21, 10);
+ });
+
+ it('saturates capped terms at cap × tokens when every context exceeds the cap', () => {
+ // Single request with a long prompt: all computed contexts are > 2.
+ const big = momentsOf([[100, 10]]);
+ const spec: AttentionCostSpec = {
+ groups: [{ label: 'window', layers: 2, capped: { coeff: 10, cap: 2 } }],
+ };
+ const nTokens = sumComputedTokens(big, 0);
+ expect(attentionFlopsPerComputedToken(spec, big, 0)).toBeCloseTo(
+ (2 * 10 * 2 * nTokens) / nTokens,
+ 10,
+ );
+ });
+
+ it('falls back to Σctx for capped terms when every context is under the cap', () => {
+ const spec: AttentionCostSpec = {
+ groups: [{ label: 'budget', layers: 1, capped: { coeff: 4, cap: 1_000_000 } }],
+ };
+ const contexts = bruteForceContexts(pairs, 0);
+ const expected = (4 * contexts.reduce((a, b) => a + b, 0)) / contexts.length;
+ expect(attentionFlopsPerComputedToken(spec, m, 0)).toBeCloseTo(expected, 8);
+ });
+
+ it('combines linear + capped + constant groups additively', () => {
+ const spec: AttentionCostSpec = {
+ groups: [
+ { label: 'a', layers: 2, linPerCtx: 3 },
+ { label: 'b', layers: 1, capped: { coeff: 5, cap: 4 } },
+ { label: 'c', layers: 4, constPerToken: 11 },
+ ],
+ };
+ const nTokens = sumComputedTokens(m, 0);
+ const ctx = sumContexts(m, 0);
+ const expected = (2 * 3 * ctx + 5 * Math.min(4 * nTokens, ctx) + 4 * 11 * nTokens) / nTokens;
+ expect(attentionFlopsPerComputedToken(spec, m, 0)).toBeCloseTo(expected, 8);
+ });
+
+ it('returns null for invalid rates, empty moments, and zero computed tokens', () => {
+ const spec: AttentionCostSpec = { groups: [{ label: 'x', layers: 1, linPerCtx: 1 }] };
+ expect(attentionFlopsPerComputedToken(spec, m, -0.1)).toBeNull();
+ expect(attentionFlopsPerComputedToken(spec, m, 1.1)).toBeNull();
+ expect(attentionFlopsPerComputedToken(spec, m, Number.NaN)).toBeNull();
+ expect(attentionFlopsPerComputedToken(spec, momentsOf([]), 0.5)).toBeNull();
+ // rate 1 with zero output tokens → nothing is computed.
+ expect(attentionFlopsPerComputedToken(spec, momentsOf([[10, 0]]), 1)).toBeNull();
+ });
+});
diff --git a/packages/app/src/lib/attention-flops.ts b/packages/app/src/lib/attention-flops.ts
new file mode 100644
index 00000000..b41fa7d8
--- /dev/null
+++ b/packages/app/src/lib/attention-flops.ts
@@ -0,0 +1,142 @@
+/**
+ * Model-specific attention-FLOPs accounting for the
+ * "New Input Suffix + Output TFLOP/s per Chip" y-metric.
+ *
+ * ## What is counted
+ *
+ * Activation–activation attention compute only: QK^T score matmuls,
+ * attention-weighted value aggregation (AV), sparse-attention indexer
+ * scoring, and linear-attention (KDA) state update/readout. Weight matmuls
+ * (Q/K/V/O projections, indexer weight projections, MLA up/down projections)
+ * are EXCLUDED — they are covered by the separate `2 × N_active_params`
+ * GEMM term (Kaplan et al. / PaLM appendix convention,
+ * https://arxiv.org/pdf/2204.02311). One multiply-accumulate = 2 FLOPs.
+ *
+ * ## Per-layer-group cost model
+ *
+ * Every supported architecture's per-computed-token attention cost at
+ * context length L reduces to the affine-plus-capped form
+ *
+ * F_group(L) = linPerCtx · L + capped.coeff · min(L, capped.cap) + constPerToken
+ *
+ * summed over layer groups (× layer count). Examples:
+ * - Dense GQA/MHA/absorbed-MLA: pure `linPerCtx` (4·H·d·L or 2·H·(d_s+d_v)·L).
+ * - Sliding-window layers (gpt-oss): capped term with cap = window.
+ * - Top-k sparse attention (DSA/MSA): dense indexer scoring is `linPerCtx`,
+ * the main attention over the selected set is a capped term with
+ * cap = token budget (e.g. DeepSeek-V4-Pro CSA's
+ * 262,144·min(1024, L/4) rewrites to 65,536·min(L, 4096)).
+ * - Linear attention (Kimi K3 KDA): `constPerToken`, independent of L.
+ *
+ * ## Integration over the request population
+ *
+ * The DB layer stores exact joint sums over each run's per-request
+ * (ISL, OSL) pairs (`RequestLengthMoments`). With the theoretical cache hit
+ * rate r (infinite-cache prefix share of the prompt, so a request with
+ * prompt P computes its suffix of (1−r)·P tokens at contexts r·P+1 … P,
+ * then decodes O tokens at contexts P+1 … P+O — cached-prefix tokens are
+ * NOT recomputed but ARE attended over by every computed token):
+ *
+ * Σ contexts = (1−r²)/2·ΣP² + (1−r)/2·ΣP + Σ(P·O) + (ΣO² + ΣO)/2
+ * Σ tokens = (1−r)·ΣP + ΣO
+ *
+ * both exact given the stored moments. Capped terms use
+ * Σ min(L, cap) ≈ min(cap · Σtokens, Σcontexts) — an upper bound (min is
+ * concave) that is exact when every context is on the same side of the cap;
+ * agentic traces overwhelmingly sit far above the small caps (128-token
+ * windows) and the error is bounded by the capped term itself for the
+ * large caps (2048/4096-token budgets).
+ *
+ * Per-model formulas and dimensions were verified against HF configs and
+ * tech reports; sources are cited on each spec in model-architectures.ts.
+ */
+
+/**
+ * Exact joint (ISL, OSL) sums over a run's request population. Mirrors
+ * `RequestLengthMoments` in packages/db/src/queries/agentic-shared.ts
+ * (served through the derived-agentic-metrics endpoint).
+ */
+export interface RequestLengthMoments {
+ /** Number of requests with both ISL and OSL present. */
+ n: number;
+ /** Σ ISL_i */
+ sumIsl: number;
+ /** Σ ISL_i² */
+ sumIslSq: number;
+ /** Σ OSL_i */
+ sumOsl: number;
+ /** Σ OSL_i² */
+ sumOslSq: number;
+ /** Σ ISL_i·OSL_i */
+ sumIslOsl: number;
+}
+
+/**
+ * One group of identical attention layers. Per computed token at context L
+ * the group costs
+ * layers × (linPerCtx·L + capped.coeff·min(L, capped.cap) + constPerToken)
+ * FLOPs (MAC = 2 FLOPs; omitted fields default to 0).
+ */
+export interface AttentionLayerGroup {
+ /** Human-readable mechanism label (documentation only). */
+ label: string;
+ /** Number of layers of this type. */
+ layers: number;
+ /** FLOPs per computed token per unit of context (the a·L term). */
+ linPerCtx?: number;
+ /** Capped term b·min(L, cap): sliding windows and top-k token budgets. */
+ capped?: { coeff: number; cap: number };
+ /** Context-independent FLOPs per computed token (linear attention). */
+ constPerToken?: number;
+}
+
+/** Full attention-cost specification for one model. */
+export interface AttentionCostSpec {
+ groups: AttentionLayerGroup[];
+}
+
+/** Closed-form Σ over all computed tokens of their context lengths. */
+export function sumContexts(moments: RequestLengthMoments, rate: number): number {
+ const oneMinusR = 1 - rate;
+ return (
+ ((1 - rate * rate) / 2) * moments.sumIslSq +
+ (oneMinusR / 2) * moments.sumIsl +
+ moments.sumIslOsl +
+ (moments.sumOslSq + moments.sumOsl) / 2
+ );
+}
+
+/** Total computed (non-cached) tokens: suffix prefill + decode. */
+export function sumComputedTokens(moments: RequestLengthMoments, rate: number): number {
+ return (1 - rate) * moments.sumIsl + moments.sumOsl;
+}
+
+/**
+ * Average attention FLOPs per computed token for a run, integrating the
+ * model's per-layer-group cost over the request-length distribution at the
+ * given theoretical cache hit rate.
+ *
+ * Returns null when the moments are unusable (no paired requests, or no
+ * computed tokens — e.g. rate = 1 with zero output tokens).
+ */
+export function attentionFlopsPerComputedToken(
+ spec: AttentionCostSpec,
+ moments: RequestLengthMoments,
+ rate: number,
+): number | null {
+ if (!Number.isFinite(rate) || rate < 0 || rate > 1) return null;
+ if (!moments || moments.n <= 0) return null;
+ const nTokens = sumComputedTokens(moments, rate);
+ if (!Number.isFinite(nTokens) || nTokens <= 0) return null;
+ const sumCtx = sumContexts(moments, rate);
+ if (!Number.isFinite(sumCtx) || sumCtx < 0) return null;
+
+ let totalFlops = 0;
+ for (const g of spec.groups) {
+ const lin = (g.linPerCtx ?? 0) * sumCtx;
+ const capped = g.capped ? g.capped.coeff * Math.min(g.capped.cap * nTokens, sumCtx) : 0;
+ const constant = (g.constPerToken ?? 0) * nTokens;
+ totalFlops += g.layers * (lin + capped + constant);
+ }
+ return totalFlops / nTokens;
+}
diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts
index 4036af80..f1827035 100644
--- a/packages/app/src/lib/benchmark-transform.ts
+++ b/packages/app/src/lib/benchmark-transform.ts
@@ -19,6 +19,7 @@ import {
} from '@/lib/chart-utils';
import { getHardwareConfig } from '@/lib/constants';
import { isPersistedBenchmarkId } from '@/lib/benchmark-id';
+import type { RequestLengthMoments } from '@/lib/attention-flops';
import type { BenchmarkRow } from '@/lib/api';
/**
@@ -362,10 +363,16 @@ export function mergeRunScopedRows(
* (default 'median'). Swaps `median_intvty`/`median_e2el` in the chart
* definition for the chosen percentile — only agentic rows carry the
* full set (median/p90/p99/p99.9) so this mainly affects that scenario.
+ * @param requestLengthMomentsById Optional per-persisted-id (ISL, OSL) moment
+ * sums from the derived-agentic-metrics endpoint. Merged into each entry
+ * before derived fields are built so the TFLOP/s-per-chip metric can price
+ * attention FLOPs; callers without them (unofficial-run overlays, AI chart)
+ * simply omit that metric.
*/
export function transformBenchmarkRows(
rows: BenchmarkRow[],
percentile = 'median',
+ requestLengthMomentsById?: Record,
): {
chartData: InferenceData[][];
hardwareConfig: HardwareConfig;
@@ -378,6 +385,9 @@ export function transformBenchmarkRows(
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const entry = rowToAggDataEntry(row);
+ if (requestLengthMomentsById && entry.id !== undefined) {
+ entry.request_length_moments = requestLengthMomentsById[entry.id] ?? null;
+ }
const hwKey = getHardwareKey(entry);
entry.hwKey = hwKey;
diff --git a/packages/app/src/lib/chart-utils.test.ts b/packages/app/src/lib/chart-utils.test.ts
index 7d0e2c0e..3d74836c 100644
--- a/packages/app/src/lib/chart-utils.test.ts
+++ b/packages/app/src/lib/chart-utils.test.ts
@@ -714,17 +714,30 @@ describe('createChartDataPoint', () => {
expect(point.newInputSuffixOutputTputPerGpu).toBeUndefined();
});
- it('derives suffix+output TFLOP/s from 2 × active params for known models', () => {
+ it('derives suffix+output TFLOP/s from GEMM + attention FLOPs for known models', () => {
const e = entry({
- model: 'DeepSeek-R1-0528', // 37B active params
+ model: 'DeepSeek-R1-0528', // 37B active params, 61 absorbed-MLA layers
tput_per_gpu: 900,
input_tput_per_gpu: 600,
output_tput_per_gpu: 300,
theoretical_cache_hit_rate: 0.8,
+ // One request: ISL 1000, OSL 100.
+ request_length_moments: {
+ n: 1,
+ sumIsl: 1000,
+ sumIslSq: 1_000_000,
+ sumOsl: 100,
+ sumOslSq: 10_000,
+ sumIslOsl: 100_000,
+ },
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
- // suffix+output = 900 - 600 × 0.8 = 420 tok/s → 2 × 37e9 × 420 / 1e12
- expect(point.newInputSuffixOutputTflopsPerGpu?.y).toBeCloseTo(31.08, 5);
+ // suffix+output = 900 - 600 × 0.8 = 420 tok/s.
+ // Σctx = (1-0.64)/2·1e6 + 0.1·1000 + 1e5 + (1e4+100)/2 = 285,150;
+ // Σtokens = 0.2·1000 + 100 = 300;
+ // attention = 61·278,528·285,150/300 = 16,149,192,704 FLOPs/token;
+ // (2·37e9 + 16,149,192,704) × 420 / 1e12 = 37.86266… TFLOP/s.
+ expect(point.newInputSuffixOutputTflopsPerGpu?.y).toBeCloseTo(37.86266093568, 5);
});
it('omits suffix+output TFLOP/s for models without architecture data', () => {
@@ -733,6 +746,25 @@ describe('createChartDataPoint', () => {
tput_per_gpu: 900,
input_tput_per_gpu: 600,
theoretical_cache_hit_rate: 0.8,
+ request_length_moments: {
+ n: 1,
+ sumIsl: 1000,
+ sumIslSq: 1_000_000,
+ sumOsl: 100,
+ sumOslSq: 10_000,
+ sumIslOsl: 100_000,
+ },
+ });
+ const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
+ expect(point.newInputSuffixOutputTflopsPerGpu).toBeUndefined();
+ });
+
+ it('omits suffix+output TFLOP/s when request-length moments are missing', () => {
+ const e = entry({
+ model: 'DeepSeek-R1-0528',
+ tput_per_gpu: 900,
+ input_tput_per_gpu: 600,
+ theoretical_cache_hit_rate: 0.8,
});
const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100');
expect(point.newInputSuffixOutputTflopsPerGpu).toBeUndefined();
diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts
index 5c9e477d..e39fa2f5 100644
--- a/packages/app/src/lib/chart-utils.ts
+++ b/packages/app/src/lib/chart-utils.ts
@@ -19,6 +19,7 @@ import {
} from '@/components/inference/metric-registry';
import { getGpuSpecs, isKnownGpu } from '@/lib/constants';
import type { Model } from '@/lib/data-mappings';
+import { attentionFlopsPerComputedToken } from '@/lib/attention-flops';
import { getModelArchitecture } from '@/lib/model-architectures';
import { getVendor, type Vendor } from '@/lib/dynamic-colors';
import type { Locale } from '@/lib/i18n';
@@ -353,21 +354,34 @@ export function buildDerivedChartFields(
);
}
// Achieved model TFLOP/s per chip on the theoretically necessary tokens:
- // FLOPs/token = 2 × active params (GEMM-only Kaplan/PaLM convention;
- // attention-score FLOPs excluded — see metric-registry doc comment).
- // activeParams is in billions, so tok/s × 2 × 1e9 params / 1e12 = ÷1000.
+ // FLOPs/token = 2 × active params (Kaplan/PaLM GEMM convention) + the
+ // model-specific attention FLOPs per computed token, integrated over the
+ // run's exact per-request (ISL, OSL) sums at the theoretical hit rate
+ // (see attention-flops.ts). Requires the moments and an attention spec —
+ // points without them (fixed-seq, unofficial overlays, models outside the
+ // architecture registry) omit the metric rather than showing a lower bound
+ // inconsistent with other points.
if (
wants('newInputSuffixOutputTflopsPerGpu') &&
hasTheoreticalHitRate &&
tputPerGpu &&
inputTputPerGpu
) {
- const activeParams = getModelArchitecture(entry.model as Model)?.activeParams;
- if (activeParams) {
- const suffixOutputTput = Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate);
- fields.newInputSuffixOutputTflopsPerGpu = chartMetric(
- (2 * activeParams * suffixOutputTput) / 1000,
+ const arch = getModelArchitecture(entry.model as Model);
+ const moments = entry.request_length_moments;
+ if (arch?.activeParams && arch.attention && moments) {
+ const attnFlopsPerToken = attentionFlopsPerComputedToken(
+ arch.attention,
+ moments,
+ theoreticalHitRate,
);
+ if (attnFlopsPerToken !== null) {
+ const suffixOutputTput = Math.max(0, tputPerGpu - inputTputPerGpu * theoreticalHitRate);
+ const flopsPerToken = 2 * arch.activeParams * 1e9 + attnFlopsPerToken;
+ fields.newInputSuffixOutputTflopsPerGpu = chartMetric(
+ (suffixOutputTput * flopsPerToken) / 1e12,
+ );
+ }
}
}
if (wants('newInputSuffixTputPerGpu') && hasTheoreticalHitRate && inputTputPerGpu) {
diff --git a/packages/app/src/lib/model-architectures.ts b/packages/app/src/lib/model-architectures.ts
index f30393bb..1ae9fa90 100644
--- a/packages/app/src/lib/model-architectures.ts
+++ b/packages/app/src/lib/model-architectures.ts
@@ -1,4 +1,5 @@
import { Model } from '@/lib/data-mappings';
+import type { AttentionCostSpec } from '@/lib/attention-flops';
/**
* Model architecture types
@@ -134,6 +135,13 @@ export interface ModelArchitecture {
ffnVariant?: string;
/** Elementwise activation applied to the gate projection. Defaults to `SiLU`. */
ffnGateActivation?: string;
+ /**
+ * Attention-FLOPs cost model for the TFLOP/s-per-chip y-metric — see
+ * attention-flops.ts for the accounting conventions (activation–activation
+ * ops only, MAC = 2 FLOPs, absorbed MLA form). Models without a spec are
+ * simply omitted from that metric.
+ */
+ attention?: AttentionCostSpec;
}
/**
@@ -177,6 +185,15 @@ export const MODEL_ARCHITECTURES: Partial> = {
],
developer: 'DeepSeek',
sourceUrl: 'https://huggingface.co/deepseek-ai/DeepSeek-R1-0528',
+ // Absorbed (MQA-mode) MLA — the form that executes at decode: all 128
+ // query heads score against the shared 576-dim cached latent (512
+ // kv_lora_rank + 64 rope) and aggregate 512-dim latent values.
+ // 2·128·(576+512)·L = 278,528·L per layer. Dims:
+ // https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/config.json;
+ // MQA-mode description: https://arxiv.org/html/2512.02556v1.
+ attention: {
+ groups: [{ label: 'MLA (absorbed)', layers: 61, linPerCtx: 278528 }],
+ },
},
[Model.DeepSeek_V4_Pro]: {
model: Model.DeepSeek_V4_Pro,
@@ -242,6 +259,36 @@ export const MODEL_ARCHITECTURES: Partial> = {
],
developer: 'DeepSeek',
sourceUrl: 'https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro',
+ // Shared single-latent MQA (d_score = d_v = 512, RoPE in-place in the last
+ // 64 dims). Config `compress_ratios` decodes the full stack as 31 HCA +
+ // 30 CSA layers (the 3 hash-routed MoE layers are HCA, HCA, CSA).
+ // HCA: dense over L/128 pooled entries + 128-token window, no indexer →
+ // 2·128·(512+512)·(L/128 + 128) = 2,048·L + 33,554,432.
+ // CSA: FP4 lightning indexer (64 heads × dim 128) over L/4 pooled keys +
+ // core attention over top-1024 pooled entries + 128-token window →
+ // 4,096·L + 262,144·(min(1024, L/4) + 128), and
+ // 262,144·min(1024, L/4) ≡ 65,536·min(L, 4096).
+ // Attention sink = one learnable per-head logit (0 KV entries, ~0 FLOPs).
+ // Sources: https://arxiv.org/html/2606.19348v1;
+ // https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/config.json;
+ // https://huggingface.co/docs/transformers/en/model_doc/deepseek_v4.
+ attention: {
+ groups: [
+ {
+ label: 'HCA (128:1 pooled, dense)',
+ layers: 31,
+ linPerCtx: 2048,
+ constPerToken: 33554432,
+ },
+ {
+ label: 'CSA (4:1 pooled, indexer top-1024)',
+ layers: 30,
+ linPerCtx: 4096,
+ capped: { coeff: 65536, cap: 4096 },
+ constPerToken: 33554432,
+ },
+ ],
+ },
},
[Model.Llama3_3_70B]: {
model: Model.Llama3_3_70B,
@@ -259,6 +306,12 @@ export const MODEL_ARCHITECTURES: Partial> = {
features: ['Grouped Query Attention', 'RoPE'],
developer: 'Meta',
sourceUrl: 'https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct',
+ // Dense causal GQA: 4·H·d·L = 4·64·128·L per layer (GQA KV sharing saves
+ // cache, not score/AV compute). Dims:
+ // https://huggingface.co/unsloth/Llama-3.3-70B-Instruct/raw/main/config.json.
+ attention: {
+ groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 65536 }],
+ },
},
[Model.Llama3_1_70B]: {
model: Model.Llama3_1_70B,
@@ -276,6 +329,11 @@ export const MODEL_ARCHITECTURES: Partial> = {
features: ['Grouped Query Attention', 'RoPE'],
developer: 'Meta',
sourceUrl: 'https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct',
+ // Identical attention geometry to Llama 3.3 70B (80 × 64 heads × dim 128):
+ // https://huggingface.co/unsloth/Meta-Llama-3.1-70B-Instruct/raw/main/config.json.
+ attention: {
+ groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 65536 }],
+ },
},
[Model.GptOss]: {
model: Model.GptOss,
@@ -318,6 +376,17 @@ export const MODEL_ARCHITECTURES: Partial> = {
],
developer: 'OpenAI',
sourceUrl: 'https://huggingface.co/openai/gpt-oss-120b',
+ // 18 full + 18 sliding-128 GQA layers, 64 heads × dim 64 → 4·64·64 =
+ // 16,384 FLOPs/token per ctx unit; sliding layers cap L at 128. The
+ // learnable sink is one extra softmax logit per head (~0 FLOPs).
+ // https://huggingface.co/openai/gpt-oss-120b/raw/main/config.json;
+ // https://arxiv.org/abs/2508.10925.
+ attention: {
+ groups: [
+ { label: 'Full GQA', layers: 18, linPerCtx: 16384 },
+ { label: 'Sliding-128 GQA', layers: 18, capped: { coeff: 16384, cap: 128 } },
+ ],
+ },
},
[Model.Kimi_K2_5]: {
model: Model.Kimi_K2_5,
@@ -339,6 +408,13 @@ export const MODEL_ARCHITECTURES: Partial> = {
features: ['Multi-head Latent Attention', 'DeepSeek-style MoE', 'YaRN RoPE'],
developer: 'Moonshot AI',
sourceUrl: 'https://huggingface.co/moonshotai/Kimi-K2.5',
+ // DeepSeek-V3-style absorbed MLA on all 61 layers, 64 heads: score dim
+ // 576 (512 latent + 64 rope), value dim 512 → 2·64·(576+512)·L =
+ // 139,264·L per layer.
+ // https://huggingface.co/moonshotai/Kimi-K2.5/raw/main/config.json.
+ attention: {
+ groups: [{ label: 'MLA (absorbed)', layers: 61, linPerCtx: 139264 }],
+ },
},
[Model.Kimi_K3]: {
model: Model.Kimi_K3,
@@ -407,6 +483,20 @@ export const MODEL_ARCHITECTURES: Partial> = {
],
developer: 'Moonshot AI',
sourceUrl: 'https://huggingface.co/moonshotai/Kimi-K3',
+ // 69 KDA linear-attention layers: gated delta-rule update + readout on a
+ // 128×128 state per head × 96 heads ≈ 7·H·d² = 11,010,048 FLOPs/token,
+ // independent of L (recurrent form; the chunked prefill kernel is ~11%
+ // higher — within noise of this metric). 24 gated-MLA layers are NoPE
+ // (`mla_use_nope`), so score dim = value dim = 512 → 2·96·(512+512)·L =
+ // 196,608·L per layer.
+ // https://huggingface.co/moonshotai/Kimi-K3/raw/main/config.json;
+ // https://arxiv.org/abs/2510.26692.
+ attention: {
+ groups: [
+ { label: 'KDA (linear)', layers: 69, constPerToken: 11010048 },
+ { label: 'Gated MLA (NoPE, absorbed)', layers: 24, linPerCtx: 196608 },
+ ],
+ },
},
[Model.MiniMax_M2_5]: {
model: Model.MiniMax_M2_5,
@@ -434,6 +524,12 @@ export const MODEL_ARCHITECTURES: Partial> = {
],
developer: 'MiniMax',
sourceUrl: 'https://huggingface.co/MiniMaxAI/MiniMax-M2',
+ // Dense causal GQA on all 62 layers (config `attn_type_list` is all 1s):
+ // 4·48·128·L = 24,576·L per layer.
+ // https://huggingface.co/MiniMaxAI/MiniMax-M2.5/raw/main/config.json.
+ attention: {
+ groups: [{ label: 'Full GQA', layers: 62, linPerCtx: 24576 }],
+ },
},
[Model.MiniMax_M3]: {
model: Model.MiniMax_M3,
@@ -467,6 +563,24 @@ export const MODEL_ARCHITECTURES: Partial> = {
],
developer: 'MiniMax',
sourceUrl: 'https://huggingface.co/MiniMaxAI/MiniMax-M3',
+ // Layers 0–2 dense GQA (4·64·128·L = 32,768·L); layers 3–59 MSA: per-token
+ // index scoring (4 group heads × dim 128, no value head) over all L
+ // positions = 2·4·128·L = 1,024·L, then exact GQA over the top-16
+ // 128-token blocks (2048-token budget) = 32,768·min(L, 2048). Reproduces
+ // the paper's 28.4× attention-compute reduction at 1M.
+ // https://huggingface.co/MiniMaxAI/MiniMax-M3/raw/main/config.json;
+ // https://arxiv.org/html/2606.13392v2.
+ attention: {
+ groups: [
+ { label: 'Dense GQA (layers 0-2)', layers: 3, linPerCtx: 32768 },
+ {
+ label: 'MSA (top-16 blocks of 128)',
+ layers: 57,
+ linPerCtx: 1024,
+ capped: { coeff: 32768, cap: 2048 },
+ },
+ ],
+ },
},
};
diff --git a/packages/db/src/etl/compute-aggregate-stats.test.ts b/packages/db/src/etl/compute-aggregate-stats.test.ts
index d2119c7b..e9b2d40d 100644
--- a/packages/db/src/etl/compute-aggregate-stats.test.ts
+++ b/packages/db/src/etl/compute-aggregate-stats.test.ts
@@ -97,6 +97,16 @@ describe('computeAggregateStats', () => {
expect(stats.e2elPerOsl?.p50).toBeCloseTo(2 / 75, 6);
// p90 of 3 values (linear interpolation): pos=1.8 → 0.02667 + 0.8×(0.03-0.02667)
expect(stats.e2elPerOsl?.p90).toBeCloseTo(2 / 75 + 0.8 * (0.03 - 2 / 75), 6);
+
+ // Exact joint (ISL, OSL) moments for closed-form attention-FLOP pricing.
+ expect(stats.requestLengthMoments).toEqual({
+ n: 3,
+ sumIsl: 600,
+ sumIslSq: 100 * 100 + 200 * 200 + 300 * 300,
+ sumOsl: 225,
+ sumOslSq: 50 * 50 + 75 * 75 + 100 * 100,
+ sumIslOsl: 100 * 50 + 200 * 75 + 300 * 100,
+ });
});
it('excludes cancelled requests from every streamed profile distribution', async () => {
@@ -114,6 +124,8 @@ describe('computeAggregateStats', () => {
expect(stats.e2elPerOsl?.n).toBe(1);
expect(stats.sequenceLengths.isl?.n).toBe(1);
expect(stats.sequenceLengths.osl?.n).toBe(1);
+ expect(stats.requestLengthMoments?.n).toBe(1);
+ expect(stats.requestLengthMoments?.sumIslSq).toBe(100 * 100);
});
it('accepts a gzip profile split across arbitrarily small compressed chunks', async () => {
diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts
index 07729b59..5b301e25 100644
--- a/packages/db/src/etl/compute-aggregate-stats.ts
+++ b/packages/db/src/etl/compute-aggregate-stats.ts
@@ -22,6 +22,7 @@ import {
type MetricPercentiles,
type SequenceLengthSketches,
} from '../queries/agentic-aggregates';
+import { requestLengthMomentsOf, type RequestLengthMoments } from '../queries/agentic-shared';
export { STATS_VERSION };
@@ -39,6 +40,12 @@ export interface AggregateStats {
e2elPerOsl: MetricPercentiles | null;
/** Bounded mergeable distributions used by the chart-level subtitle. */
sequenceLengths: SequenceLengthSketches;
+ /**
+ * Exact joint (ISL, OSL) sums over the request population — the sufficient
+ * statistics the frontend integrates model-specific attention-FLOPs
+ * formulas over (see agentic-shared.ts).
+ */
+ requestLengthMoments: RequestLengthMoments | null;
}
interface ProfileMetricEnvelope {
@@ -75,12 +82,14 @@ async function extractProfileSamples(
isl: number[];
osl: number[];
e2elPerOsl: number[];
+ pairs: { isl: number; osl: number }[];
}> {
const input = Readable.from(compressedChunks).pipe(createGunzip());
const lines = createInterface({ input, crlfDelay: Infinity });
const isl: number[] = [];
const osl: number[] = [];
const e2elPerOsl: number[] = [];
+ const pairs: { isl: number; osl: number }[] = [];
for await (const line of lines) {
if (!line) continue;
@@ -100,6 +109,9 @@ async function extractProfileSamples(
const outputLength = profileMetricValue(metrics.output_sequence_length);
if (inputLength !== undefined) isl.push(inputLength);
if (outputLength !== undefined) osl.push(outputLength);
+ if (inputLength !== undefined && outputLength !== undefined) {
+ pairs.push({ isl: inputLength, osl: outputLength });
+ }
const requestLatencyMs = profileMetricValue(metrics.request_latency);
const ttftMs = profileMetricValue(metrics.time_to_first_token);
@@ -117,7 +129,7 @@ async function extractProfileSamples(
}
}
- return { isl, osl, e2elPerOsl };
+ return { isl, osl, e2elPerOsl, pairs };
}
/**
@@ -132,13 +144,15 @@ export async function computeProfileAggregateStatsFromCompressedChunks(
let oslPct: MetricPercentiles | null = null;
let e2elPerOsl: MetricPercentiles | null = null;
let sequenceLengths: SequenceLengthSketches = { isl: null, osl: null };
+ let requestLengthMoments: RequestLengthMoments | null = null;
try {
- const { isl, osl, e2elPerOsl: ratios } = await extractProfileSamples(compressedChunks);
+ const { isl, osl, e2elPerOsl: ratios, pairs } = await extractProfileSamples(compressedChunks);
islPct = percentilesOf(isl);
oslPct = percentilesOf(osl);
sequenceLengths = sequenceLengthSketches(isl, osl);
e2elPerOsl = percentilesOf(ratios);
+ requestLengthMoments = requestLengthMomentsOf(pairs);
} catch {
// Ignore malformed blobs and leave the profile-derived fields null.
}
@@ -151,6 +165,7 @@ export async function computeProfileAggregateStatsFromCompressedChunks(
prefixCacheHitRate: null,
e2elPerOsl,
sequenceLengths,
+ requestLengthMoments,
};
}
diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts
index 9511d248..033c7d73 100644
--- a/packages/db/src/queries/agentic-aggregates.ts
+++ b/packages/db/src/queries/agentic-aggregates.ts
@@ -33,6 +33,7 @@ import {
STATS_VERSION,
writeBackTraceReplayJsonb,
type MetricPercentiles,
+ type RequestLengthMoments,
type SequenceLengthSketches,
} from './agentic-shared';
@@ -315,7 +316,7 @@ export async function getAgenticAggregates(
if (row.profile_blob) {
try {
const jsonl = gunzipSync(row.profile_blob).toString('utf8');
- const { isl, osl } = extractIslOsl(jsonl);
+ const { isl, osl, requestLengthMoments } = extractIslOsl(jsonl);
const islPct = percentilesOf(isl);
const oslPct = percentilesOf(osl);
result[id].isl = islPct;
@@ -334,6 +335,7 @@ export async function getAgenticAggregates(
prefixCacheHitRate: null,
e2elPerOsl: derived.e2el_per_osl,
sequenceLengths: sequenceLengthSketches(isl, osl),
+ requestLengthMoments,
},
});
} catch {
@@ -420,6 +422,7 @@ interface FullAggregateStats {
prefixCacheHitRate: MetricPercentiles | null;
e2elPerOsl: MetricPercentiles | null;
sequenceLengths: SequenceLengthSketches;
+ requestLengthMoments: RequestLengthMoments | null;
}
function blankAggregate(id: number): AgenticAggregate {
diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts
index c242e4b8..0823d532 100644
--- a/packages/db/src/queries/agentic-shared.ts
+++ b/packages/db/src/queries/agentic-shared.ts
@@ -48,8 +48,60 @@ import {
* v8: add p95 and bounded mergeable ISL/OSL sketches. The dashboard merges
* the sketches for all resident chart points instead of loading request-level
* timelines or attempting to combine per-point percentiles.
+ *
+ * v9: add `requestLengthMoments` — exact joint moments of the per-request
+ * (ISL, OSL) pairs (n, ΣISL, ΣISL², ΣOSL, ΣOSL², ΣISL·OSL). The frontend
+ * integrates model-specific attention-FLOPs formulas over the true request
+ * population from these sums (prefill attention is quadratic in context, so
+ * E[ISL²] ≠ E[ISL]² matters); marginal percentiles/sketches can't provide the
+ * joint ISL·OSL term the decode integral needs.
*/
-export const STATS_VERSION = 8;
+export const STATS_VERSION = 9;
+
+/**
+ * Exact sums over the per-request (ISL, OSL) pairs of one benchmark point.
+ * Only records carrying BOTH sequence lengths contribute, so every sum is
+ * over the same request population and cross-terms stay consistent.
+ *
+ * These six sums are sufficient statistics for any attention-cost integral
+ * that is polynomial (≤ quadratic) in per-request context length: e.g.
+ * Σᵢ suffix-prefill context = (1−r²)/2 · ΣISL² and Σᵢ decode context =
+ * ΣISL·OSL + (ΣOSL² + ΣOSL)/2, with r the point's theoretical cache hit rate.
+ */
+export interface RequestLengthMoments {
+ /** Number of requests with both ISL and OSL present. */
+ n: number;
+ sumIsl: number;
+ sumIslSq: number;
+ sumOsl: number;
+ sumOslSq: number;
+ sumIslOsl: number;
+}
+
+/** Accumulate the joint moments for paired per-request (ISL, OSL) samples. */
+export function requestLengthMomentsOf(
+ pairs: readonly { isl: number; osl: number }[],
+): RequestLengthMoments | null {
+ if (pairs.length === 0) return null;
+ const m: RequestLengthMoments = {
+ n: 0,
+ sumIsl: 0,
+ sumIslSq: 0,
+ sumOsl: 0,
+ sumOslSq: 0,
+ sumIslOsl: 0,
+ };
+ for (const { isl, osl } of pairs) {
+ if (!Number.isFinite(isl) || !Number.isFinite(osl) || isl < 0 || osl < 0) continue;
+ m.n += 1;
+ m.sumIsl += isl;
+ m.sumIslSq += isl * isl;
+ m.sumOsl += osl;
+ m.sumOslSq += osl * osl;
+ m.sumIslOsl += isl * osl;
+ }
+ return m.n > 0 ? m : null;
+}
interface ProfileRecord {
metadata?: { benchmark_phase?: string; was_cancelled?: boolean };
@@ -59,10 +111,18 @@ interface ProfileRecord {
};
}
-/** Parse the profile_export.jsonl → per-request ISL + OSL arrays. */
-export function extractIslOsl(jsonl: string): { isl: number[]; osl: number[] } {
+/**
+ * Parse the profile_export.jsonl → per-request ISL + OSL arrays, plus the
+ * joint (ISL, OSL) moments over records carrying both lengths.
+ */
+export function extractIslOsl(jsonl: string): {
+ isl: number[];
+ osl: number[];
+ requestLengthMoments: RequestLengthMoments | null;
+} {
const isl: number[] = [];
const osl: number[] = [];
+ const pairs: { isl: number; osl: number }[] = [];
for (const line of jsonl.split('\n')) {
if (!line) continue;
let rec: ProfileRecord;
@@ -78,8 +138,9 @@ export function extractIslOsl(jsonl: string): { isl: number[]; osl: number[] } {
const o = readNum(m.output_sequence_length);
if (typeof i === 'number') isl.push(i);
if (typeof o === 'number') osl.push(o);
+ if (typeof i === 'number' && typeof o === 'number') pairs.push({ isl: i, osl: o });
}
- return { isl, osl };
+ return { isl, osl, requestLengthMoments: requestLengthMomentsOf(pairs) };
}
export interface MetricPercentiles {
diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts
index 091fb81e..ad91b2cd 100644
--- a/packages/db/src/queries/derived-agentic-metrics.test.ts
+++ b/packages/db/src/queries/derived-agentic-metrics.test.ts
@@ -28,6 +28,41 @@ describe('computeDerivedFromBlob', () => {
it('returns null when no usable records', () => {
const out = computeDerivedFromBlob('');
expect(out.e2el_per_osl).toBeNull();
+ expect(out.request_length_moments).toBeNull();
+ });
+
+ it('accumulates exact joint (ISL, OSL) moments over profiling records', () => {
+ const lines = [
+ rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }),
+ rec('s2', 0, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 4000 }),
+ // Missing TTFT breaks the latency ratio but NOT the length moments.
+ JSON.stringify({
+ metadata: { conversation_id: 's3', turn_index: 0, benchmark_phase: 'profiling' },
+ metrics: {
+ request_latency: { value: 1000, unit: 'ms' },
+ input_sequence_length: { value: 10, unit: 'tokens' },
+ output_sequence_length: { value: 20, unit: 'tokens' },
+ },
+ }),
+ // Warmup phase is excluded from both.
+ JSON.stringify({
+ metadata: { conversation_id: 's4', turn_index: 0, benchmark_phase: 'warmup' },
+ metrics: {
+ input_sequence_length: { value: 9999, unit: 'tokens' },
+ output_sequence_length: { value: 9999, unit: 'tokens' },
+ },
+ }),
+ ];
+ const out = computeDerivedFromBlob(lines.join('\n'));
+ expect(out.request_length_moments).toEqual({
+ n: 3,
+ sumIsl: 310,
+ sumIslSq: 100 * 100 + 200 * 200 + 10 * 10,
+ sumOsl: 170,
+ sumOslSq: 50 * 50 + 100 * 100 + 20 * 20,
+ sumIslOsl: 100 * 50 + 200 * 100 + 10 * 20,
+ });
+ expect(out.e2el_per_osl?.n).toBe(2);
});
it('computes per-request E2EL/OSL ratios pooled across sessions', () => {
@@ -258,6 +293,11 @@ describe('getDerivedAgenticMetrics write-back', () => {
]);
const result = await getDerivedAgenticMetrics(sql, [7]);
- expect(result[7]).toEqual({ id: 7, p75_e2e_norm_intvty: null, p90_e2e_norm_intvty: null });
+ expect(result[7]).toEqual({
+ id: 7,
+ p75_e2e_norm_intvty: null,
+ p90_e2e_norm_intvty: null,
+ request_length_moments: null,
+ });
});
});
diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts
index 5685ef2e..df94cec9 100644
--- a/packages/db/src/queries/derived-agentic-metrics.ts
+++ b/packages/db/src/queries/derived-agentic-metrics.ts
@@ -29,10 +29,12 @@ import {
fetchAggregateStatsRows,
percentilesOf,
readNum,
+ requestLengthMomentsOf,
sequenceLengthSketches,
STATS_VERSION,
writeBackTraceReplayJsonb,
type MetricPercentiles,
+ type RequestLengthMoments,
type SequenceLengthSketches,
} from './agentic-shared';
@@ -43,6 +45,13 @@ export interface DerivedAgenticMetric {
p75_e2e_norm_intvty: number | null;
/** Slow-tail P90 E2E Normalized Interactivity in tok/s/user — 1 / p90(per-request E2EL/OSL). */
p90_e2e_norm_intvty: number | null;
+ /**
+ * Exact joint (ISL, OSL) sums over the request population. The frontend
+ * integrates per-model attention-FLOPs formulas over these for the
+ * TFLOP/s-per-chip y-metric. Null when the blob had no usable records or
+ * the stored bundle predates v9 and hasn't self-healed yet.
+ */
+ request_length_moments: RequestLengthMoments | null;
}
export type DerivedAgenticMetricMap = Record;
@@ -65,6 +74,7 @@ interface StoredAggregateStats {
prefixCacheHitRate: MetricPercentiles | null;
e2elPerOsl: MetricPercentiles | null;
sequenceLengths: SequenceLengthSketches;
+ requestLengthMoments?: RequestLengthMoments | null;
}
/**
@@ -123,8 +133,10 @@ function invertRatio(v: number | null | undefined): number | null {
*/
export function computeDerivedFromBlob(jsonl: string): {
e2el_per_osl: MetricPercentiles | null;
+ request_length_moments: RequestLengthMoments | null;
} {
const ratios: number[] = [];
+ const pairs: { isl: number; osl: number }[] = [];
for (const line of jsonl.split('\n')) {
if (!line) continue;
let rec: ProfileRecord;
@@ -134,11 +146,20 @@ export function computeDerivedFromBlob(jsonl: string): {
continue;
}
if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue;
+ // Moments only need the sequence-length pair — keep them even when the
+ // latency fields extractTurn requires are missing or non-positive.
+ const m = rec.metrics ?? {};
+ const isl = readNum(m.input_sequence_length);
+ const osl = readNum(m.output_sequence_length);
+ if (typeof isl === 'number' && typeof osl === 'number') pairs.push({ isl, osl });
const turn = extractTurn(rec);
if (!turn) continue;
ratios.push(turn.request_latency_ms / 1000 / turn.osl);
}
- return { e2el_per_osl: percentilesOf(ratios) };
+ return {
+ e2el_per_osl: percentilesOf(ratios),
+ request_length_moments: requestLengthMomentsOf(pairs),
+ };
}
export async function getDerivedAgenticMetrics(
@@ -167,6 +188,7 @@ export async function getDerivedAgenticMetrics(
id,
p75_e2e_norm_intvty: invertRatio(row.stats.e2elPerOsl?.p75),
p90_e2e_norm_intvty: invertRatio(row.stats.e2elPerOsl?.p90),
+ request_length_moments: row.stats.requestLengthMoments ?? null,
};
} else {
idsNeedingBlob.push(id);
@@ -205,11 +227,12 @@ export async function getDerivedAgenticMetrics(
const id = Number(row.benchmark_result_id);
try {
const jsonl = gunzipSync(row.blob).toString('utf8');
- const { e2el_per_osl } = computeDerivedFromBlob(jsonl);
+ const { e2el_per_osl, request_length_moments } = computeDerivedFromBlob(jsonl);
result[id] = {
id,
p75_e2e_norm_intvty: invertRatio(e2el_per_osl?.p75),
p90_e2e_norm_intvty: invertRatio(e2el_per_osl?.p90),
+ request_length_moments,
};
// Self-heal the shared `aggregate_stats` bundle. We only have the profile
@@ -237,6 +260,7 @@ export async function getDerivedAgenticMetrics(
prefixCacheHitRate: prior?.prefixCacheHitRate ?? null,
e2elPerOsl: e2el_per_osl,
sequenceLengths: sequenceLengthSketches(isl, osl),
+ requestLengthMoments: request_length_moments,
};
writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged);
}
From 6b62f9a6ef003733452dce3891c30a949d704683 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:25:32 +0000
Subject: [PATCH 10/13] docs(api): document request_length_moments in
derived-agentic-metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Add the joint (ISL, OSL) request-length moment sums to the public API
documentation for GET /api/v1/derived-agentic-metrics (schema + example)
and refresh the route/shared-source SHA-256 digests in the catalog.
中文：在 GET /api/v1/derived-agentic-metrics 的公开 API 文档中补充
(ISL, OSL) 请求长度联合矩和字段（schema 与示例），并更新路由及共享
源码的 SHA-256 摘要。
---
packages/app/src/lib/api-documentation.ts | 37 ++++++++++++++++++++---
packages/app/src/lib/api-route-catalog.ts | 4 +--
2 files changed, 34 insertions(+), 7 deletions(-)
diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts
index 4a22d16c..5081bcd8 100644
--- a/packages/app/src/lib/api-documentation.ts
+++ b/packages/app/src/lib/api-documentation.ts
@@ -1849,8 +1849,8 @@ export const apiOperations: readonly ApiOperation[] = [
path: '/api/v1/derived-agentic-metrics',
summary: text('Read derived agentic metrics', '读取派生智能体指标'),
description: text(
- 'Returns normalized interactivity percentiles keyed by benchmark result ID. IDs are deduplicated and at most 200 are accepted.',
- '按基准结果 ID 返回归一化交互性百分位。ID 会去重，最多接受 200 个。',
+ 'Returns normalized interactivity percentiles and joint request-length moment sums keyed by benchmark result ID. IDs are deduplicated and at most 200 are accepted.',
+ '按基准结果 ID 返回归一化交互性百分位及请求长度联合矩和。ID 会去重，最多接受 200 个。',
),
audience: 'public',
stability: 'beta',
@@ -1868,16 +1868,43 @@ export const apiOperations: readonly ApiOperation[] = [
],
responses: [
success(
- 'Result IDs mapped to p75 and p90 normalized interactivity.',
- '结果 ID 映射到 p75 和 p90 归一化交互性。',
+ 'Result IDs mapped to p75/p90 normalized interactivity and exact joint (ISL, OSL) request-length moment sums (null when the profile blob is missing).',
+ '结果 ID 映射到 p75/p90 归一化交互性及精确的 (ISL, OSL) 请求长度联合矩和（缺少 profile blob 时为 null）。',
mapSchema(
objectSchema({
id: integerSchema,
p75_e2e_norm_intvty: nullableNumberSchema,
p90_e2e_norm_intvty: nullableNumberSchema,
+ request_length_moments: {
+ type: ['object', 'null'],
+ properties: {
+ n: integerSchema,
+ sumIsl: { type: 'number' },
+ sumIslSq: { type: 'number' },
+ sumOsl: { type: 'number' },
+ sumOslSq: { type: 'number' },
+ sumIslOsl: { type: 'number' },
+ },
+ required: ['n', 'sumIsl', 'sumIslSq', 'sumOsl', 'sumOslSq', 'sumIslOsl'],
+ additionalProperties: false,
+ },
}),
),
- { '421': { id: 421, p75_e2e_norm_intvty: 31.2, p90_e2e_norm_intvty: 24.8 } },
+ {
+ '421': {
+ id: 421,
+ p75_e2e_norm_intvty: 31.2,
+ p90_e2e_norm_intvty: 24.8,
+ request_length_moments: {
+ n: 2,
+ sumIsl: 300,
+ sumIslSq: 50_000,
+ sumOsl: 150,
+ sumOslSq: 12_500,
+ sumIslOsl: 25_000,
+ },
+ },
+ },
),
errorResponse(
'400',
diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts
index bc49b9bb..0135c745 100644
--- a/packages/app/src/lib/api-route-catalog.ts
+++ b/packages/app/src/lib/api-route-catalog.ts
@@ -191,7 +191,7 @@ export const apiRouteCatalog = [
method: 'GET',
classification: 'published-read',
operationId: 'get-derived-agentic-metrics',
- sourceSha256: '68013b8e1a0354e677c075b5ab31df4be5889fdc0dc2eb82dbe52a108f4d1db2',
+ sourceSha256: '13c6ba2b0b4d827745336b016728970651580a79bec59bf4fae9743d35d8eb19',
},
{
source: 'src/app/api/v1/eval-samples-live/route.ts',
@@ -711,7 +711,7 @@ export const apiContractSourceDigests = [
},
{
source: '../db/src/queries/agentic-aggregates.ts',
- sourceSha256: 'fae8d19971730132cb30cd781f677562bfc6328b1f4e35a8268a8391ad187c18',
+ sourceSha256: '807bb1349b7adf67ff9b699b8c9bb518932ebeaf3a6a46cb37160e078abb1863',
reviewArea: {
en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.',
zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。',
From 3513940d060d066f0c64a60c9e00ea0e6a0d615c Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:51:28 +0000
Subject: [PATCH 11/13] fix: stream oversized blobs in agentic fallbacks to
survive Neon's 64 MB response cap
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Production profile_export blobs reach 248 MB compressed while Neon's
serverless HTTP driver rejects any response over 64 MB (HTTP 507). The
derived-agentic-metrics fallback selected 6 whole blobs per query and the
agentic-aggregates fallback 8 per query, so after the v9 stats bump routed
every pre-v9 row through the fallback, one oversized batch failed the whole
query, the endpoint 500'd, and the new TFLOP/s per chip metric (plus p75/p90
E2E-normalized interactivity) showed "No data available" on existing data.
- agentic-shared: add streamTraceReplayBlob (bounded, self-terminating
substring chunk reads) + a single shared streaming profile-sample
extractor used by ingest, backfill, and both query fallbacks
- derived-agentic-metrics + agentic-aggregates: metadata query first, then
per-row streamed recompute with per-row error isolation so one bad blob
can never blank the whole response; server_metrics blobs are also read
via bounded chunks (they can exceed the cap too)
- etl/compute-aggregate-stats: delegate to the shared extractor
- verified against prod: the previously-507ing 35 MB row returns full
moments in <2 s; the worst-case 237 MB blob streams in 11 s at 93 MB RSS
中文：
生产环境的 profile_export blob 压缩后最大达 248 MB，而 Neon serverless HTTP
驱动拒绝超过 64 MB 的响应（HTTP 507）。derived-agentic-metrics 回退路径每次
查询内联拉取 6 个完整 blob，agentic-aggregates 回退每次 8 个；v9 版本升级后
所有旧行都走回退路径，一个超大批次即导致整个查询失败、接口 500，新的
TFLOP/s per chip 指标（以及 p75/p90 端到端归一化交互性）在现有数据上显示
"No data available"。
- agentic-shared：新增 streamTraceReplayBlob（有界、自终止的 substring 分块
读取）及统一的流式 profile 样本提取器，供摄取、回填和两个查询回退共用
- derived-agentic-metrics 与 agentic-aggregates：先做元数据查询，再逐行流式
重算并按行隔离错误，单个坏 blob 不再拖垮整个响应；server_metrics blob 同样
改为分块读取（也可能超过上限）
- etl/compute-aggregate-stats：委托给共享提取器
- 已用生产数据验证：此前 507 的 35 MB 行 <2 秒返回完整矩量；最坏情况 237 MB
blob 11 秒流式完成，内存峰值 93 MB
---
packages/app/src/lib/api-route-catalog.ts | 2 +-
.../db/src/etl/compute-aggregate-stats.ts | 94 +--------
.../db/src/queries/agentic-aggregates.test.ts | 48 +++--
packages/db/src/queries/agentic-aggregates.ts | 188 +++++++++---------
.../db/src/queries/agentic-shared.test.ts | 93 ++++++++-
packages/db/src/queries/agentic-shared.ts | 159 +++++++++++++--
.../queries/derived-agentic-metrics.test.ts | 29 ++-
.../db/src/queries/derived-agentic-metrics.ts | 133 ++++---------
8 files changed, 414 insertions(+), 332 deletions(-)
diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts
index 0135c745..08450d9f 100644
--- a/packages/app/src/lib/api-route-catalog.ts
+++ b/packages/app/src/lib/api-route-catalog.ts
@@ -711,7 +711,7 @@ export const apiContractSourceDigests = [
},
{
source: '../db/src/queries/agentic-aggregates.ts',
- sourceSha256: '807bb1349b7adf67ff9b699b8c9bb518932ebeaf3a6a46cb37160e078abb1863',
+ sourceSha256: '1fb18a9e81a77a556fffcf16327c2975bdeb07b5594350a1208bc65bacc27ae9',
reviewArea: {
en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.',
zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。',
diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts
index 5b301e25..0c32b753 100644
--- a/packages/db/src/etl/compute-aggregate-stats.ts
+++ b/packages/db/src/etl/compute-aggregate-stats.ts
@@ -9,10 +9,6 @@
* computation changes so the backfill script knows which rows to recompute.
*/
-import { createInterface } from 'node:readline';
-import { Readable } from 'node:stream';
-import { createGunzip } from 'node:zlib';
-
import { gunzipJsonWithinLimit, streamCollectKeys } from './gzip-json-stream';
import {
STATS_VERSION,
@@ -22,7 +18,11 @@ import {
type MetricPercentiles,
type SequenceLengthSketches,
} from '../queries/agentic-aggregates';
-import { requestLengthMomentsOf, type RequestLengthMoments } from '../queries/agentic-shared';
+import {
+ extractProfileSamples,
+ requestLengthMomentsOf,
+ type RequestLengthMoments,
+} from '../queries/agentic-shared';
export { STATS_VERSION };
@@ -48,90 +48,6 @@ export interface AggregateStats {
requestLengthMoments: RequestLengthMoments | null;
}
-interface ProfileMetricEnvelope {
- value?: number;
-}
-
-interface ProfileRecord {
- metadata?: {
- benchmark_phase?: string;
- was_cancelled?: boolean;
- };
- metrics?: {
- input_sequence_length?: ProfileMetricEnvelope | number;
- output_sequence_length?: ProfileMetricEnvelope | number;
- request_latency?: ProfileMetricEnvelope | number;
- time_to_first_token?: ProfileMetricEnvelope | number;
- };
-}
-
-function profileMetricValue(value: ProfileMetricEnvelope | number | undefined): number | undefined {
- const number = typeof value === 'number' ? value : value?.value;
- return typeof number === 'number' && Number.isFinite(number) ? number : undefined;
-}
-
-/**
- * Stream a profile export line by line so exceptionally large traces never
- * materialize their multi-gigabyte decompressed JSONL as one string. The
- * numeric sample arrays are tiny relative to the source and are needed for
- * exact percentile calculation.
- */
-async function extractProfileSamples(
- compressedChunks: Iterable | AsyncIterable,
-): Promise<{
- isl: number[];
- osl: number[];
- e2elPerOsl: number[];
- pairs: { isl: number; osl: number }[];
-}> {
- const input = Readable.from(compressedChunks).pipe(createGunzip());
- const lines = createInterface({ input, crlfDelay: Infinity });
- const isl: number[] = [];
- const osl: number[] = [];
- const e2elPerOsl: number[] = [];
- const pairs: { isl: number; osl: number }[] = [];
-
- for await (const line of lines) {
- if (!line) continue;
- let record: ProfileRecord;
- try {
- record = JSON.parse(line) as ProfileRecord;
- } catch {
- continue;
- }
- if (record.metadata?.benchmark_phase && record.metadata.benchmark_phase !== 'profiling') {
- continue;
- }
- if (record.metadata?.was_cancelled === true) continue;
-
- const metrics = record.metrics ?? {};
- const inputLength = profileMetricValue(metrics.input_sequence_length);
- const outputLength = profileMetricValue(metrics.output_sequence_length);
- if (inputLength !== undefined) isl.push(inputLength);
- if (outputLength !== undefined) osl.push(outputLength);
- if (inputLength !== undefined && outputLength !== undefined) {
- pairs.push({ isl: inputLength, osl: outputLength });
- }
-
- const requestLatencyMs = profileMetricValue(metrics.request_latency);
- const ttftMs = profileMetricValue(metrics.time_to_first_token);
- if (
- requestLatencyMs !== undefined &&
- ttftMs !== undefined &&
- inputLength !== undefined &&
- outputLength !== undefined &&
- requestLatencyMs > 0 &&
- ttftMs > 0 &&
- inputLength > 0 &&
- outputLength > 0
- ) {
- e2elPerOsl.push(requestLatencyMs / 1000 / outputLength);
- }
- }
-
- return { isl, osl, e2elPerOsl, pairs };
-}
-
/**
* Compute the profile-derived half of the aggregate bundle from compressed
* chunks. Backfills use this for oversized TOAST values so neither Postgres
diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts
index 89ec4b91..96da7913 100644
--- a/packages/db/src/queries/agentic-aggregates.test.ts
+++ b/packages/db/src/queries/agentic-aggregates.test.ts
@@ -211,10 +211,19 @@ describe('getAgenticAggregates write-back', () => {
const { sql, calls } = mockSql([
// fetchAggregateStatsRows
[{ benchmark_result_id: 7, stats: staleStats }],
- // Pass 1: profile blob (+ trace_replay_id for write-back)
- [{ benchmark_result_id: 7, trace_replay_id: 870, profile_blob: profileBlob }],
- // Pass 2: server blob
- [{ benchmark_result_id: 7, server_blob: serverBlob }],
+ // metadata query: ids → trace_replay rows + blob presence (no blob inline)
+ [
+ {
+ benchmark_result_id: 7,
+ trace_replay_id: 870,
+ has_profile_blob: true,
+ has_server_blob: true,
+ },
+ ],
+ // Pass 1: profile blob substring chunk (short → stream terminates)
+ [{ chunk: profileBlob }],
+ // Pass 2: server blob substring chunk
+ [{ chunk: serverBlob }],
]);
const result = await getAgenticAggregates(sql, [7]);
@@ -223,14 +232,17 @@ describe('getAgenticAggregates write-back', () => {
expect(result[7]?.isl?.n).toBe(2);
expect(result[7]?.kvCacheUtil?.mean).toBeCloseTo(0.25, 6);
- // 4 calls: stats read, profile read, server read, write-back UPDATE.
- expect(calls).toHaveLength(4);
- expect(calls[3]!.text).toContain('update agentic_trace_replay set aggregate_stats');
- expect(calls[3]!.text).toContain('::jsonb where id');
+ // 5 calls: stats read, metadata read, profile chunk, server chunk, write-back UPDATE.
+ expect(calls).toHaveLength(5);
+ // Blobs are streamed via bounded substring chunks — never selected whole.
+ expect(calls[2]!.text).toContain('substring(profile_export_jsonl_gz from');
+ expect(calls[3]!.text).toContain('substring(server_metrics_json_gz from');
+ expect(calls[4]!.text).toContain('update agentic_trace_replay set aggregate_stats');
+ expect(calls[4]!.text).toContain('::jsonb where id');
// The payload OBJECT is bound directly (not stringified — that would
// double-encode into a JSONB string).
- const [written, traceReplayId] = calls[3]!.values as [WrittenStats, number];
+ const [written, traceReplayId] = calls[4]!.values as [WrittenStats, number];
expect(traceReplayId).toBe(870);
expect(written.version).toBe(STATS_VERSION);
// Server field FRESHLY recomputed (0.25), not the stale 0.9 carried forward.
@@ -258,16 +270,22 @@ describe('getAgenticAggregates write-back', () => {
};
const { sql, calls } = mockSql([
[{ benchmark_result_id: 7, stats: staleStats }],
- // Pass 1: no profile blob → nothing to recompute, nothing to heal.
- [{ benchmark_result_id: 7, trace_replay_id: 870, profile_blob: null }],
- // Pass 2: no server blob either.
- [{ benchmark_result_id: 7, server_blob: null }],
+ // Metadata: both blobs missing → nothing to recompute, nothing to heal,
+ // and no substring reads are even issued.
+ [
+ {
+ benchmark_result_id: 7,
+ trace_replay_id: 870,
+ has_profile_blob: false,
+ has_server_blob: false,
+ },
+ ],
]);
await getAgenticAggregates(sql, [7]);
- // stats read + 2 blob reads only — no write-back (profile parse never succeeded).
- expect(calls).toHaveLength(3);
+ // stats read + metadata read only — no write-back (profile parse never ran).
+ expect(calls).toHaveLength(2);
expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false);
});
});
diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts
index 033c7d73..4a36eb24 100644
--- a/packages/db/src/queries/agentic-aggregates.ts
+++ b/packages/db/src/queries/agentic-aggregates.ts
@@ -14,7 +14,7 @@
*/
import { Readable } from 'node:stream';
-import { createGunzip, gunzipSync } from 'node:zlib';
+import { createGunzip } from 'node:zlib';
import { chain } from 'stream-chain';
@@ -24,13 +24,15 @@ import { streamObject } from 'stream-json/streamers/stream-object.js';
import { gunzipJsonWithinLimit } from '../etl/gzip-json-stream';
import type { DbClient } from '../connection.js';
-import { computeDerivedFromBlob } from './derived-agentic-metrics';
import {
- extractIslOsl,
+ extractProfileSamples,
fetchAggregateStatsRows,
percentilesOf,
+ readTraceReplayBlob,
+ requestLengthMomentsOf,
sequenceLengthSketches,
STATS_VERSION,
+ streamTraceReplayBlob,
writeBackTraceReplayJsonb,
type MetricPercentiles,
type RequestLengthMoments,
@@ -60,17 +62,6 @@ export interface AgenticAggregate {
export type AgenticAggregateMap = Record;
-/**
- * `profile_export_jsonl_gz` is small (~1-3 MB) so we can batch many per
- * round-trip. `server_metrics_json_gz` is much bigger (~17 MB compressed
- * for high-conc TP+EP runs; Neon encodes bytea over HTTP at ~1.6× wire
- * size, so two of those = ~50 MB and three already trips the 64 MB cap).
- * We fetch the two blob types in separate queries with different chunk
- * sizes.
- */
-const PROFILE_CHUNK_SIZE = 8;
-const SERVER_CHUNK_SIZE = 1;
-
interface TimeSlice {
start_ns?: number;
end_ns?: number;
@@ -260,7 +251,7 @@ export async function getAgenticAggregates(
const statsRows = await fetchAggregateStatsRows(sql, benchmarkResultIds);
const idsNeedingProfile: number[] = [];
- const idsNeedingServer: number[] = [];
+
for (const row of statsRows) {
const id = Number(row.benchmark_result_id);
const agg = blankAggregate(id);
@@ -273,7 +264,6 @@ export async function getAgenticAggregates(
// No stats (or stale version) — schedule the blob-parse fallback below
// so the response still surfaces data. Backfill should drain these.
idsNeedingProfile.push(id);
- idsNeedingServer.push(id);
}
result[id] = agg;
}
@@ -283,7 +273,7 @@ export async function getAgenticAggregates(
if (!(id in result)) result[id] = blankAggregate(id);
}
- if (idsNeedingProfile.length === 0 && idsNeedingServer.length === 0) {
+ if (idsNeedingProfile.length === 0) {
return result;
}
@@ -294,94 +284,94 @@ export async function getAgenticAggregates(
// good stored data.
const pendingById = new Map();
- // ── Fallback Pass 1: profile_export blobs (cheap; large batches). ──────
- for (let i = 0; i < idsNeedingProfile.length; i += PROFILE_CHUNK_SIZE) {
- const chunk = idsNeedingProfile.slice(i, i + PROFILE_CHUNK_SIZE);
- const rows = (await sql`
- select
- br.id as benchmark_result_id,
- atr.id as trace_replay_id,
- atr.profile_export_jsonl_gz as profile_blob
- from benchmark_results br
- join agentic_trace_replay atr on atr.id = br.trace_replay_id
- where br.id = any(${chunk}::bigint[])
- `) as {
- benchmark_result_id: number;
- trace_replay_id: number;
- profile_blob: Buffer | null;
- }[];
- for (const row of rows) {
- const id = Number(row.benchmark_result_id);
- result[id] ??= blankAggregate(id);
- if (row.profile_blob) {
- try {
- const jsonl = gunzipSync(row.profile_blob).toString('utf8');
- const { isl, osl, requestLengthMoments } = extractIslOsl(jsonl);
- const islPct = percentilesOf(isl);
- const oslPct = percentilesOf(osl);
- result[id].isl = islPct;
- result[id].osl = oslPct;
- // Recompute every profile-derived field from this same JSONL so the
- // self-healed bundle is complete at the new version. Server-derived
- // fields are filled in Pass 2 (or stay null without a server blob).
- const derived = computeDerivedFromBlob(jsonl);
- pendingById.set(id, {
- traceReplayId: Number(row.trace_replay_id),
- stats: {
- version: STATS_VERSION,
- isl: islPct,
- osl: oslPct,
- kvCacheUtil: null,
- prefixCacheHitRate: null,
- e2elPerOsl: derived.e2el_per_osl,
- sequenceLengths: sequenceLengthSketches(isl, osl),
- requestLengthMoments,
- },
- });
- } catch {
- // ignore malformed blob
- }
- }
+ // Both passes stream blobs through bounded `substring` chunks instead of
+ // selecting whole bytea columns: production profile blobs reach >240 MB
+ // compressed while Neon's serverless HTTP driver caps a single response at
+ // 64 MB (HTTP 507 above that), so an inline blob select can fail the whole
+ // query — the failure mode that blanked derived metrics for stale-version
+ // rows. One cheap metadata query maps ids to trace_replay rows first.
+ const metaRows = (await sql`
+ select
+ br.id as benchmark_result_id,
+ atr.id as trace_replay_id,
+ (atr.profile_export_jsonl_gz is not null) as has_profile_blob,
+ (atr.server_metrics_json_gz is not null) as has_server_blob
+ from benchmark_results br
+ join agentic_trace_replay atr on atr.id = br.trace_replay_id
+ where br.id = any(${idsNeedingProfile}::bigint[])
+ `) as {
+ benchmark_result_id: number;
+ trace_replay_id: number;
+ has_profile_blob: boolean;
+ has_server_blob: boolean;
+ }[];
+
+ // ── Fallback Pass 1: profile_export blobs (streamed line parse). ──────
+ for (const row of metaRows) {
+ const id = Number(row.benchmark_result_id);
+ result[id] ??= blankAggregate(id);
+ if (!row.has_profile_blob) continue;
+ try {
+ // One pass yields every profile-derived field so the self-healed
+ // bundle is complete at the new version. Server-derived fields are
+ // filled in Pass 2 (or stay null without a server blob).
+ const { isl, osl, e2elPerOsl, pairs } = await extractProfileSamples(
+ streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', Number(row.trace_replay_id)),
+ );
+ const islPct = percentilesOf(isl);
+ const oslPct = percentilesOf(osl);
+ result[id].isl = islPct;
+ result[id].osl = oslPct;
+ pendingById.set(id, {
+ traceReplayId: Number(row.trace_replay_id),
+ stats: {
+ version: STATS_VERSION,
+ isl: islPct,
+ osl: oslPct,
+ kvCacheUtil: null,
+ prefixCacheHitRate: null,
+ e2elPerOsl: percentilesOf(e2elPerOsl),
+ sequenceLengths: sequenceLengthSketches(isl, osl),
+ requestLengthMoments: requestLengthMomentsOf(pairs),
+ },
+ });
+ } catch {
+ // ignore malformed/unreadable blob — never fail the whole response
}
}
// ── Fallback Pass 2: server_metrics blobs (huge; one at a time). ───────
// Serial to avoid OOM on the decompressed JSON of a high-conc TP+EP row
// (>500 MB raw). The aggregator is fronted by a blob cache, so the slow
// path runs at most once per sibling set.
- for (let i = 0; i < idsNeedingServer.length; i += SERVER_CHUNK_SIZE) {
- const chunk = idsNeedingServer.slice(i, i + SERVER_CHUNK_SIZE);
- const rows = (await sql`
- select
- br.id as benchmark_result_id,
- atr.server_metrics_json_gz as server_blob
- from benchmark_results br
- join agentic_trace_replay atr on atr.id = br.trace_replay_id
- where br.id = any(${chunk}::bigint[])
- `) as { benchmark_result_id: number; server_blob: Buffer | null }[];
- for (const row of rows) {
- const id = Number(row.benchmark_result_id);
- result[id] ??= blankAggregate(id);
- if (!row.server_blob) continue;
- let parsed: { kvCacheUtil: number[]; prefixCacheHitRate: number[] } | null = null;
- try {
- const json = gunzipJsonWithinLimit(row.server_blob);
- parsed =
- json === null
- ? await streamExtractServerMetricSamples(row.server_blob)
- : extractServerMetricSamples(json);
- } catch {
- // malformed blob or failed stream fallback — leave nulls
- }
- if (parsed) {
- const kvPct = percentilesOf(parsed.kvCacheUtil);
- const prefixPct = percentilesOf(parsed.prefixCacheHitRate);
- result[id].kvCacheUtil = kvPct;
- result[id].prefixCacheHitRate = prefixPct;
- const pending = pendingById.get(id);
- if (pending) {
- pending.stats.kvCacheUtil = kvPct;
- pending.stats.prefixCacheHitRate = prefixPct;
- }
+ for (const row of metaRows) {
+ const id = Number(row.benchmark_result_id);
+ result[id] ??= blankAggregate(id);
+ if (!row.has_server_blob) continue;
+ let parsed: { kvCacheUtil: number[]; prefixCacheHitRate: number[] } | null = null;
+ try {
+ const serverBlob = await readTraceReplayBlob(
+ sql,
+ 'server_metrics_json_gz',
+ Number(row.trace_replay_id),
+ );
+ if (!serverBlob) continue;
+ const json = gunzipJsonWithinLimit(serverBlob);
+ parsed =
+ json === null
+ ? await streamExtractServerMetricSamples(serverBlob)
+ : extractServerMetricSamples(json);
+ } catch {
+ // malformed blob or failed stream fallback — leave nulls
+ }
+ if (parsed) {
+ const kvPct = percentilesOf(parsed.kvCacheUtil);
+ const prefixPct = percentilesOf(parsed.prefixCacheHitRate);
+ result[id].kvCacheUtil = kvPct;
+ result[id].prefixCacheHitRate = prefixPct;
+ const pending = pendingById.get(id);
+ if (pending) {
+ pending.stats.kvCacheUtil = kvPct;
+ pending.stats.prefixCacheHitRate = prefixPct;
}
}
}
diff --git a/packages/db/src/queries/agentic-shared.test.ts b/packages/db/src/queries/agentic-shared.test.ts
index 35a25d97..44b4c16e 100644
--- a/packages/db/src/queries/agentic-shared.test.ts
+++ b/packages/db/src/queries/agentic-shared.test.ts
@@ -1,8 +1,17 @@
+import { randomBytes } from 'node:crypto';
+import { gzipSync } from 'node:zlib';
+
import { afterEach, describe, expect, it, vi } from 'vitest';
import type { DbClient } from '../connection.js';
-import { _resetWriteBackWarned, writeBackTraceReplayJsonb } from './agentic-shared';
+import {
+ _resetWriteBackWarned,
+ BLOB_CHUNK_BYTES,
+ extractProfileSamples,
+ streamTraceReplayBlob,
+ writeBackTraceReplayJsonb,
+} from './agentic-shared';
/**
* Capture every SQL call: the joined template text plus the bound values, so we
@@ -77,3 +86,85 @@ describe('writeBackTraceReplayJsonb', () => {
expect(warn.mock.calls[0]![0]).toContain('could not persist chart_series');
});
});
+
+/** Serve `substring(col from ? for ?)` reads out of an in-memory buffer. */
+function blobSql(blob: Buffer): { sql: DbClient; chunkReads: number[] } {
+ const chunkReads: number[] = [];
+ const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => {
+ const text = strings.join('?');
+ expect(text).toContain('substring(profile_export_jsonl_gz from');
+ const [offset, length] = values as [number, number, number];
+ chunkReads.push(offset);
+ // SQL substring is 1-based; slice is 0-based.
+ const chunk = blob.subarray(offset - 1, offset - 1 + length);
+ return Promise.resolve([{ chunk }]);
+ }) as unknown as DbClient;
+ return { sql, chunkReads };
+}
+
+/** One profiling record padded with poorly compressible bytes. */
+function paddedRec(isl: number, osl: number): string {
+ return JSON.stringify({
+ metadata: { benchmark_phase: 'profiling' },
+ metrics: {
+ request_latency: { value: 1000, unit: 'ms' },
+ time_to_first_token: { value: 100, unit: 'ms' },
+ input_sequence_length: { value: isl, unit: 'tokens' },
+ output_sequence_length: { value: osl, unit: 'tokens' },
+ padding: randomBytes(4 * 1024 * 1024).toString('base64'),
+ },
+ });
+}
+
+describe('streamTraceReplayBlob + extractProfileSamples', () => {
+ it('reassembles a blob larger than one chunk across multiple substring reads', async () => {
+ // Poorly compressible padding pushes the gzip past BLOB_CHUNK_BYTES so the
+ // reader must issue several bounded reads and the gunzip stream must
+ // reassemble records that straddle chunk boundaries.
+ const jsonl = [paddedRec(100, 50), paddedRec(200, 100), paddedRec(300, 25)].join('\n');
+ const blob = gzipSync(Buffer.from(jsonl));
+ expect(blob.length).toBeGreaterThan(BLOB_CHUNK_BYTES);
+
+ const { sql, chunkReads } = blobSql(blob);
+ const samples = await extractProfileSamples(
+ streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', 42),
+ );
+
+ expect(chunkReads.length).toBeGreaterThan(1);
+ expect(chunkReads[0]).toBe(1);
+ expect(chunkReads[1]).toBe(1 + BLOB_CHUNK_BYTES);
+ expect(samples.isl).toEqual([100, 200, 300]);
+ expect(samples.osl).toEqual([50, 100, 25]);
+ expect(samples.e2elPerOsl).toEqual([1 / 50, 1 / 100, 1 / 25]);
+ expect(samples.pairs).toHaveLength(3);
+ });
+
+ it('terminates without reads beyond a short final chunk and skips cancelled records', async () => {
+ const lines = [
+ JSON.stringify({
+ metadata: { benchmark_phase: 'profiling' },
+ metrics: {
+ input_sequence_length: { value: 10, unit: 'tokens' },
+ output_sequence_length: { value: 5, unit: 'tokens' },
+ },
+ }),
+ JSON.stringify({
+ metadata: { benchmark_phase: 'profiling', was_cancelled: true },
+ metrics: {
+ input_sequence_length: { value: 7777, unit: 'tokens' },
+ output_sequence_length: { value: 7777, unit: 'tokens' },
+ },
+ }),
+ ];
+ const blob = gzipSync(Buffer.from(lines.join('\n')));
+ const { sql, chunkReads } = blobSql(blob);
+ const samples = await extractProfileSamples(
+ streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', 42),
+ );
+ // Single short chunk — the reader must not issue a second round-trip.
+ expect(chunkReads).toEqual([1]);
+ expect(samples.pairs).toEqual([{ isl: 10, osl: 5 }]);
+ // No latency fields → no ratio samples; cancelled record fully ignored.
+ expect(samples.e2elPerOsl).toEqual([]);
+ });
+});
diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts
index 0823d532..073c4dcb 100644
--- a/packages/db/src/queries/agentic-shared.ts
+++ b/packages/db/src/queries/agentic-shared.ts
@@ -13,6 +13,10 @@
* write-back. (agentic-aggregates re-exports both for existing importers.)
*/
+import { createInterface } from 'node:readline';
+import { Readable } from 'node:stream';
+import { createGunzip } from 'node:zlib';
+
import type { DbClient } from '../connection.js';
import {
@@ -106,11 +110,83 @@ export function requestLengthMomentsOf(
interface ProfileRecord {
metadata?: { benchmark_phase?: string; was_cancelled?: boolean };
metrics?: {
+ request_latency?: { value?: number; unit?: string } | number;
+ time_to_first_token?: { value?: number; unit?: string } | number;
input_sequence_length?: { value?: number } | number;
output_sequence_length?: { value?: number } | number;
};
}
+/**
+ * Per-request samples pulled from a profile_export.jsonl blob in one pass —
+ * the raw material every profile-derived aggregate is computed from. Both
+ * query fallbacks and the ingest/backfill path share this single extractor so
+ * the fast (stored bundle) and slow (blob recompute) paths can never drift.
+ */
+export interface ProfileSamples {
+ isl: number[];
+ osl: number[];
+ /** Per-request E2E latency / OSL ratios (seconds per output token). */
+ e2elPerOsl: number[];
+ /** (ISL, OSL) pairs over records carrying both lengths. */
+ pairs: { isl: number; osl: number }[];
+}
+
+function addProfileSampleLine(acc: ProfileSamples, line: string): void {
+ if (!line) return;
+ let rec: ProfileRecord;
+ try {
+ rec = JSON.parse(line) as ProfileRecord;
+ } catch {
+ return;
+ }
+ if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') return;
+ if (rec.metadata?.was_cancelled === true) return;
+ const m = rec.metrics ?? {};
+ const isl = readNum(m.input_sequence_length);
+ const osl = readNum(m.output_sequence_length);
+ if (isl !== undefined) acc.isl.push(isl);
+ if (osl !== undefined) acc.osl.push(osl);
+ if (isl !== undefined && osl !== undefined) acc.pairs.push({ isl, osl });
+ const rl = readNum(m.request_latency);
+ const tt = readNum(m.time_to_first_token);
+ if (
+ rl !== undefined &&
+ tt !== undefined &&
+ isl !== undefined &&
+ osl !== undefined &&
+ rl > 0 &&
+ tt > 0 &&
+ isl > 0 &&
+ osl > 0
+ ) {
+ acc.e2elPerOsl.push(rl / 1000 / osl);
+ }
+}
+
+/** Collect profile samples from an already-decompressed JSONL string. */
+export function collectProfileSamplesFromJsonl(jsonl: string): ProfileSamples {
+ const acc: ProfileSamples = { isl: [], osl: [], e2elPerOsl: [], pairs: [] };
+ for (const line of jsonl.split('\n')) addProfileSampleLine(acc, line);
+ return acc;
+}
+
+/**
+ * Stream a gzipped profile export line by line so exceptionally large traces
+ * never materialize their multi-gigabyte decompressed JSONL as one string.
+ * The numeric sample arrays are tiny relative to the source and are needed
+ * for exact percentile calculation.
+ */
+export async function extractProfileSamples(
+ compressedChunks: Iterable | AsyncIterable,
+): Promise {
+ const input = Readable.from(compressedChunks).pipe(createGunzip());
+ const lines = createInterface({ input, crlfDelay: Infinity });
+ const acc: ProfileSamples = { isl: [], osl: [], e2elPerOsl: [], pairs: [] };
+ for await (const line of lines) addProfileSampleLine(acc, line);
+ return acc;
+}
+
/**
* Parse the profile_export.jsonl → per-request ISL + OSL arrays, plus the
* joint (ISL, OSL) moments over records carrying both lengths.
@@ -120,29 +196,72 @@ export function extractIslOsl(jsonl: string): {
osl: number[];
requestLengthMoments: RequestLengthMoments | null;
} {
- const isl: number[] = [];
- const osl: number[] = [];
- const pairs: { isl: number; osl: number }[] = [];
- for (const line of jsonl.split('\n')) {
- if (!line) continue;
- let rec: ProfileRecord;
- try {
- rec = JSON.parse(line) as ProfileRecord;
- } catch {
- continue;
- }
- if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue;
- if (rec.metadata?.was_cancelled === true) continue;
- const m = rec.metrics ?? {};
- const i = readNum(m.input_sequence_length);
- const o = readNum(m.output_sequence_length);
- if (typeof i === 'number') isl.push(i);
- if (typeof o === 'number') osl.push(o);
- if (typeof i === 'number' && typeof o === 'number') pairs.push({ isl: i, osl: o });
- }
+ const { isl, osl, pairs } = collectProfileSamplesFromJsonl(jsonl);
return { isl, osl, requestLengthMoments: requestLengthMomentsOf(pairs) };
}
+/**
+ * 8 MiB of bytea per `substring` read — the hex wire encoding doubles it, so
+ * each response stays far under Neon's serverless-HTTP 64 MB response cap.
+ * Production profile blobs reach >240 MB compressed (server blobs are of the
+ * same order), so selecting a whole blob column inline is NEVER safe: the
+ * driver rejects the response (HTTP 507) and the whole query fails.
+ */
+export const BLOB_CHUNK_BYTES = 8 * 1024 * 1024;
+
+export type TraceReplayBlobColumn = 'profile_export_jsonl_gz' | 'server_metrics_json_gz';
+
+/** Normalize a driver-returned bytea value (Buffer, Uint8Array, or hex text). */
+function asBuffer(v: unknown): Buffer | null {
+ if (v === null || v === undefined) return null;
+ if (Buffer.isBuffer(v)) return v;
+ if (v instanceof Uint8Array) return Buffer.from(v);
+ if (typeof v === 'string' && v.startsWith(String.raw`\x`)) return Buffer.from(v.slice(2), 'hex');
+ return null;
+}
+
+/**
+ * Stream a gzipped blob column in bounded `substring` chunks. Self-terminating
+ * on the first short/empty chunk, so no size pre-query is needed and a
+ * `pg_column_size` vs `octet_length` mismatch can never truncate the stream.
+ */
+export async function* streamTraceReplayBlob(
+ sql: DbClient,
+ column: TraceReplayBlobColumn,
+ traceReplayId: number,
+): AsyncGenerator {
+ for (let offset = 1; ; offset += BLOB_CHUNK_BYTES) {
+ // Static SQL per column (no dynamic identifiers) — `column` is a
+ // closed union, not caller-supplied text.
+ const rows = (await (column === 'profile_export_jsonl_gz'
+ ? sql`
+ select substring(profile_export_jsonl_gz from ${offset} for ${BLOB_CHUNK_BYTES}) as chunk
+ from agentic_trace_replay
+ where id = ${traceReplayId}
+ `
+ : sql`
+ select substring(server_metrics_json_gz from ${offset} for ${BLOB_CHUNK_BYTES}) as chunk
+ from agentic_trace_replay
+ where id = ${traceReplayId}
+ `)) as { chunk: unknown }[];
+ const chunk = asBuffer(rows[0]?.chunk);
+ if (!chunk || chunk.length === 0) break;
+ yield chunk;
+ if (chunk.length < BLOB_CHUNK_BYTES) break;
+ }
+}
+
+/** Read a whole blob column via bounded chunks; null when absent/empty. */
+export async function readTraceReplayBlob(
+ sql: DbClient,
+ column: TraceReplayBlobColumn,
+ traceReplayId: number,
+): Promise {
+ const chunks: Buffer[] = [];
+ for await (const chunk of streamTraceReplayBlob(sql, column, traceReplayId)) chunks.push(chunk);
+ return chunks.length > 0 ? Buffer.concat(chunks) : null;
+}
+
export interface MetricPercentiles {
mean: number;
p50: number;
diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts
index ad91b2cd..d5eaa16a 100644
--- a/packages/db/src/queries/derived-agentic-metrics.test.ts
+++ b/packages/db/src/queries/derived-agentic-metrics.test.ts
@@ -191,8 +191,11 @@ describe('getDerivedAgenticMetrics write-back', () => {
const { sql, calls } = mockSql([
// fetchAggregateStatsRows
[{ benchmark_result_id: 7, stats: staleStats }],
- // fallback profile-blob query
- [{ benchmark_result_id: 7, trace_replay_id: 870, blob }],
+ // fallback metadata query (ids → trace_replay rows; no blob inline)
+ [{ benchmark_result_id: 7, trace_replay_id: 870 }],
+ // first substring chunk — shorter than BLOB_CHUNK_BYTES, so the
+ // streaming reader terminates without another round-trip
+ [{ chunk: blob }],
]);
const result = await getDerivedAgenticMetrics(sql, [7]);
@@ -201,10 +204,12 @@ describe('getDerivedAgenticMetrics write-back', () => {
expect(result[7]?.p90_e2e_norm_intvty).toBeCloseTo(1 / 0.038, 6);
expect(result[7]?.p75_e2e_norm_intvty).toBeCloseTo(1 / 0.035, 6);
- // 3 calls: stats read, blob read, write-back UPDATE.
- expect(calls).toHaveLength(3);
- expect(calls[2]!.text).toContain('update agentic_trace_replay set aggregate_stats');
- expect(calls[2]!.text).toContain('::jsonb where id');
+ // 4 calls: stats read, metadata read, blob chunk read, write-back UPDATE.
+ expect(calls).toHaveLength(4);
+ // The blob is streamed via bounded substring chunks — never selected whole.
+ expect(calls[2]!.text).toContain('substring(profile_export_jsonl_gz from');
+ expect(calls[3]!.text).toContain('update agentic_trace_replay set aggregate_stats');
+ expect(calls[3]!.text).toContain('::jsonb where id');
// The write-back binds a COMPLETE, version-stamped bundle at the new version,
// recomputing profile fields and carrying server fields forward untouched.
@@ -217,7 +222,7 @@ describe('getDerivedAgenticMetrics write-back', () => {
kvCacheUtil: unknown;
e2elPerOsl: { p75: number; p90: number; n: number } | null;
}
- const [written, traceReplayId] = calls[2]!.values as [WrittenStats, number];
+ const [written, traceReplayId] = calls[3]!.values as [WrittenStats, number];
expect(traceReplayId).toBe(870);
expect(written.version).toBe(STATS_VERSION);
expect(written.e2elPerOsl?.n).toBe(2);
@@ -245,16 +250,18 @@ describe('getDerivedAgenticMetrics write-back', () => {
const { sql, calls } = mockSql([
// fetchAggregateStatsRows — no stored bundle at all
[{ benchmark_result_id: 7, stats: null }],
- // fallback profile-blob query
- [{ benchmark_result_id: 7, trace_replay_id: 870, blob }],
+ // fallback metadata query
+ [{ benchmark_result_id: 7, trace_replay_id: 870 }],
+ // single substring chunk
+ [{ chunk: blob }],
]);
const result = await getDerivedAgenticMetrics(sql, [7]);
// Caller still gets the freshly computed metric (1 / 0.02 s-per-token).
expect(result[7]?.p90_e2e_norm_intvty).toBeCloseTo(50, 6);
- // Stats read + blob read only — no write-back UPDATE.
- expect(calls).toHaveLength(2);
+ // Stats read + metadata read + chunk read only — no write-back UPDATE.
+ expect(calls).toHaveLength(3);
expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false);
});
diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts
index df94cec9..a0a8464c 100644
--- a/packages/db/src/queries/derived-agentic-metrics.ts
+++ b/packages/db/src/queries/derived-agentic-metrics.ts
@@ -21,17 +21,16 @@
* WORST request's effective token rate.
*/
-import { gunzipSync } from 'node:zlib';
-
import type { DbClient } from '../connection.js';
import {
- extractIslOsl,
+ collectProfileSamplesFromJsonl,
+ extractProfileSamples,
fetchAggregateStatsRows,
percentilesOf,
- readNum,
requestLengthMomentsOf,
sequenceLengthSketches,
STATS_VERSION,
+ streamTraceReplayBlob,
writeBackTraceReplayJsonb,
type MetricPercentiles,
type RequestLengthMoments,
@@ -77,48 +76,6 @@ interface StoredAggregateStats {
requestLengthMoments?: RequestLengthMoments | null;
}
-/**
- * JSONL blobs can be ~1-2 MB compressed (~5-10 MB raw) and Neon's serverless
- * HTTP driver caps responses at 64 MB — chunk to stay well under.
- */
-const QUERY_CHUNK_SIZE = 6;
-
-interface RecordMetrics {
- request_latency?: { value?: number; unit?: string } | number;
- time_to_first_token?: { value?: number; unit?: string } | number;
- input_sequence_length?: { value?: number } | number;
- output_sequence_length?: { value?: number } | number;
-}
-
-interface RecordMetadata {
- conversation_id?: string;
- turn_index?: number;
- benchmark_phase?: string;
-}
-
-interface ProfileRecord {
- metadata?: RecordMetadata;
- metrics?: RecordMetrics;
-}
-
-interface TurnFields {
- request_latency_ms: number;
- ttft_ms: number;
- isl: number;
- osl: number;
-}
-
-function extractTurn(rec: ProfileRecord): TurnFields | null {
- const m = rec.metrics ?? {};
- const rl = readNum(m.request_latency);
- const tt = readNum(m.time_to_first_token);
- const isl = readNum(m.input_sequence_length);
- const osl = readNum(m.output_sequence_length);
- if (rl === undefined || tt === undefined || isl === undefined || osl === undefined) return null;
- if (rl <= 0 || tt <= 0 || isl <= 0 || osl <= 0) return null;
- return { request_latency_ms: rl, ttft_ms: tt, isl, osl };
-}
-
/** 1/x for a positive stored ratio; null when the bundle/percentile is absent. */
function invertRatio(v: number | null | undefined): number | null {
return typeof v === 'number' && Number.isFinite(v) && v > 0 ? 1 / v : null;
@@ -135,29 +92,12 @@ export function computeDerivedFromBlob(jsonl: string): {
e2el_per_osl: MetricPercentiles | null;
request_length_moments: RequestLengthMoments | null;
} {
- const ratios: number[] = [];
- const pairs: { isl: number; osl: number }[] = [];
- for (const line of jsonl.split('\n')) {
- if (!line) continue;
- let rec: ProfileRecord;
- try {
- rec = JSON.parse(line) as ProfileRecord;
- } catch {
- continue;
- }
- if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue;
- // Moments only need the sequence-length pair — keep them even when the
- // latency fields extractTurn requires are missing or non-positive.
- const m = rec.metrics ?? {};
- const isl = readNum(m.input_sequence_length);
- const osl = readNum(m.output_sequence_length);
- if (typeof isl === 'number' && typeof osl === 'number') pairs.push({ isl, osl });
- const turn = extractTurn(rec);
- if (!turn) continue;
- ratios.push(turn.request_latency_ms / 1000 / turn.osl);
- }
+ // Moments only need the sequence-length pair — the shared collector keeps
+ // them even when the latency fields the ratio requires are missing or
+ // non-positive.
+ const { e2elPerOsl, pairs } = collectProfileSamplesFromJsonl(jsonl);
return {
- e2el_per_osl: percentilesOf(ratios),
+ e2el_per_osl: percentilesOf(e2elPerOsl),
request_length_moments: requestLengthMomentsOf(pairs),
};
}
@@ -198,36 +138,37 @@ export async function getDerivedAgenticMetrics(
if (idsNeedingBlob.length === 0) return result;
- // Fallback: parse the profile blob directly. Used for rows whose
+ // Fallback: recompute from the profile blob. Used for rows whose
// `aggregate_stats` is null or computed by an older STATS_VERSION; the
// backfill script drains the population so this path should be rare.
- // `trace_replay_id` + the (small) stale `aggregate_stats` come along on the
- // same join — no extra round-trip — so we can self-heal after recompute.
- const rows: {
- benchmark_result_id: number;
- trace_replay_id: number;
- blob: Buffer;
- }[] = [];
- for (let i = 0; i < idsNeedingBlob.length; i += QUERY_CHUNK_SIZE) {
- const chunk = idsNeedingBlob.slice(i, i + QUERY_CHUNK_SIZE);
- const chunkRows = (await sql`
- select
- br.id as benchmark_result_id,
- atr.id as trace_replay_id,
- atr.profile_export_jsonl_gz as blob
- from benchmark_results br
- join agentic_trace_replay atr on atr.id = br.trace_replay_id
- where br.id = any(${chunk}::bigint[])
- and atr.profile_export_jsonl_gz is not null
- `) as { benchmark_result_id: number; trace_replay_id: number; blob: Buffer }[];
- rows.push(...chunkRows);
- }
-
- for (const row of rows) {
+ //
+ // The blob is NEVER selected whole: production profile exports reach
+ // >240 MB compressed while Neon's serverless HTTP driver caps a response at
+ // 64 MB (HTTP 507 above that — the failure that blanked the TFLOP/s
+ // metric for every pre-v9 row). Instead a cheap metadata query maps ids to
+ // trace_replay rows, then each blob streams through bounded `substring`
+ // chunks into a streaming gunzip line parser — the same pattern
+ // backfill-aggregate-stats.ts uses for oversized TOAST values.
+ const metaRows = (await sql`
+ select
+ br.id as benchmark_result_id,
+ atr.id as trace_replay_id
+ from benchmark_results br
+ join agentic_trace_replay atr on atr.id = br.trace_replay_id
+ where br.id = any(${idsNeedingBlob}::bigint[])
+ and atr.profile_export_jsonl_gz is not null
+ `) as { benchmark_result_id: number; trace_replay_id: number }[];
+
+ // Serial on purpose: each blob already parallelizes nothing and bounding
+ // concurrency keeps peak memory at one decompression stream.
+ for (const row of metaRows) {
const id = Number(row.benchmark_result_id);
try {
- const jsonl = gunzipSync(row.blob).toString('utf8');
- const { e2el_per_osl, request_length_moments } = computeDerivedFromBlob(jsonl);
+ const { isl, osl, e2elPerOsl, pairs } = await extractProfileSamples(
+ streamTraceReplayBlob(sql, 'profile_export_jsonl_gz', Number(row.trace_replay_id)),
+ );
+ const e2el_per_osl = percentilesOf(e2elPerOsl);
+ const request_length_moments = requestLengthMomentsOf(pairs);
result[id] = {
id,
p75_e2e_norm_intvty: invertRatio(e2el_per_osl?.p75),
@@ -251,7 +192,6 @@ export async function getDerivedAgenticMetrics(
const prior = staleStatsById.get(id) ?? null;
const canPreserveServerFields = Boolean(prior?.kvCacheUtil || prior?.prefixCacheHitRate);
if (canPreserveServerFields) {
- const { isl, osl } = extractIslOsl(jsonl);
const merged: StoredAggregateStats = {
version: STATS_VERSION,
isl: percentilesOf(isl),
@@ -265,7 +205,8 @@ export async function getDerivedAgenticMetrics(
writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged);
}
} catch {
- // Skip malformed blobs silently — frontend treats missing ids as "no data".
+ // One malformed/unreadable blob must never take down the whole
+ // response — the frontend treats missing ids as "no data".
}
}
return result;
From f0422659f48da351f471ecc97dfbfa5a171378fe Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:57:43 +0000
Subject: [PATCH 12/13] chore: refresh agentic-aggregates shared-source digest
after format hook
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
中文：提交钩子二次格式化后刷新 agentic-aggregates 的共享源码 SHA-256 摘要。
---
packages/app/src/lib/api-route-catalog.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts
index 08450d9f..0f18edb6 100644
--- a/packages/app/src/lib/api-route-catalog.ts
+++ b/packages/app/src/lib/api-route-catalog.ts
@@ -711,7 +711,7 @@ export const apiContractSourceDigests = [
},
{
source: '../db/src/queries/agentic-aggregates.ts',
- sourceSha256: '1fb18a9e81a77a556fffcf16327c2975bdeb07b5594350a1208bc65bacc27ae9',
+ sourceSha256: 'f9745baf7d63b0a52081c4f1b587e9d4eb131b31a8127cd3d1281b87c8ec33cc',
reviewArea: {
en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.',
zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。',
From 728011ef96f0ac04ba8f8fa316c8177c73af8728 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 00:02:57 +0000
Subject: [PATCH 13/13] fix: correct Llama GQA attention constant; never
self-heal null server fields
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Bugbot findings on #849:
- Llama 3.1/3.3 70B linPerCtx was 65536 but 4·H·d with 64 heads × dim 128
is 32768 — attention FLOPs were doubled for both Llama models.
- agentic-aggregates pass 2: when a server blob exists but fails to read or
parse, drop the id from the self-heal set instead of stamping a
current-version bundle with null kvCacheUtil/prefixCacheHitRate, which
would permanently cache the miss (fast path would never retry the blob).
中文：
- Llama 3.1/3.3 70B 的 linPerCtx 误写为 65536；按 4·H·d（64 头 × 128 维）
应为 32768，此前两款 Llama 的注意力 FLOPs 被高估一倍。
- agentic-aggregates 第二遍：server blob 存在但读取/解析失败时，从自愈集合
中剔除该 id，而不是把 kvCacheUtil/prefixCacheHitRate 为 null 的当前版本
bundle 写回——否则缺失会被永久缓存，快路径不再重试。
---
packages/app/src/lib/api-route-catalog.ts | 2 +-
packages/app/src/lib/model-architectures.ts | 4 +-
.../db/src/queries/agentic-aggregates.test.ts | 38 +++++++++++++++++++
packages/db/src/queries/agentic-aggregates.ts | 19 +++++++---
4 files changed, 54 insertions(+), 9 deletions(-)
diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts
index 0f18edb6..b3549e5c 100644
--- a/packages/app/src/lib/api-route-catalog.ts
+++ b/packages/app/src/lib/api-route-catalog.ts
@@ -711,7 +711,7 @@ export const apiContractSourceDigests = [
},
{
source: '../db/src/queries/agentic-aggregates.ts',
- sourceSha256: 'f9745baf7d63b0a52081c4f1b587e9d4eb131b31a8127cd3d1281b87c8ec33cc',
+ sourceSha256: 'd2ce8c8769dd38012ba22054017f600b8e6e978a54a9248dd6085d5fc6b4b642',
reviewArea: {
en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.',
zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。',
diff --git a/packages/app/src/lib/model-architectures.ts b/packages/app/src/lib/model-architectures.ts
index 1ae9fa90..daef3b00 100644
--- a/packages/app/src/lib/model-architectures.ts
+++ b/packages/app/src/lib/model-architectures.ts
@@ -310,7 +310,7 @@ export const MODEL_ARCHITECTURES: Partial> = {
// cache, not score/AV compute). Dims:
// https://huggingface.co/unsloth/Llama-3.3-70B-Instruct/raw/main/config.json.
attention: {
- groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 65536 }],
+ groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 32768 }],
},
},
[Model.Llama3_1_70B]: {
@@ -332,7 +332,7 @@ export const MODEL_ARCHITECTURES: Partial> = {
// Identical attention geometry to Llama 3.3 70B (80 × 64 heads × dim 128):
// https://huggingface.co/unsloth/Meta-Llama-3.1-70B-Instruct/raw/main/config.json.
attention: {
- groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 65536 }],
+ groups: [{ label: 'Full GQA', layers: 80, linPerCtx: 32768 }],
},
},
[Model.GptOss]: {
diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts
index 96da7913..ba3ccf67 100644
--- a/packages/db/src/queries/agentic-aggregates.test.ts
+++ b/packages/db/src/queries/agentic-aggregates.test.ts
@@ -288,4 +288,42 @@ describe('getAgenticAggregates write-back', () => {
expect(calls).toHaveLength(2);
expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false);
});
+
+ it('does not stamp a bundle with null server fields when the server blob fails to parse', async () => {
+ const profileBlob = gzipSync(
+ Buffer.from(profileRec({ cid: 's1', isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 })),
+ );
+ const staleStats = {
+ version: STATS_VERSION - 1,
+ isl: null,
+ osl: null,
+ kvCacheUtil: null,
+ prefixCacheHitRate: null,
+ };
+ const { sql, calls } = mockSql([
+ [{ benchmark_result_id: 7, stats: staleStats }],
+ [
+ {
+ benchmark_result_id: 7,
+ trace_replay_id: 870,
+ has_profile_blob: true,
+ has_server_blob: true,
+ },
+ ],
+ // Pass 1: profile blob parses fine.
+ [{ chunk: profileBlob }],
+ // Pass 2: server blob chunk is corrupt — gunzip fails.
+ [{ chunk: Buffer.from('not gzip at all') }],
+ ]);
+
+ const result = await getAgenticAggregates(sql, [7]);
+
+ // Profile-derived fields are still served for this request…
+ expect(result[7]?.isl?.n).toBe(1);
+ expect(result[7]?.kvCacheUtil).toBeNull();
+ // …but nothing is written back: stamping a current-version bundle with
+ // null server fields would permanently cache the miss.
+ expect(calls).toHaveLength(4);
+ expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false);
+ });
});
diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts
index 4a36eb24..343b1f82 100644
--- a/packages/db/src/queries/agentic-aggregates.ts
+++ b/packages/db/src/queries/agentic-aggregates.ts
@@ -354,12 +354,13 @@ export async function getAgenticAggregates(
'server_metrics_json_gz',
Number(row.trace_replay_id),
);
- if (!serverBlob) continue;
- const json = gunzipJsonWithinLimit(serverBlob);
- parsed =
- json === null
- ? await streamExtractServerMetricSamples(serverBlob)
- : extractServerMetricSamples(json);
+ if (serverBlob) {
+ const json = gunzipJsonWithinLimit(serverBlob);
+ parsed =
+ json === null
+ ? await streamExtractServerMetricSamples(serverBlob)
+ : extractServerMetricSamples(json);
+ }
} catch {
// malformed blob or failed stream fallback — leave nulls
}
@@ -373,6 +374,12 @@ export async function getAgenticAggregates(
pending.stats.kvCacheUtil = kvPct;
pending.stats.prefixCacheHitRate = prefixPct;
}
+ } else {
+ // A server blob exists but couldn't be read or parsed this time
+ // (transient stream error, oversized JSON, ...). Don't self-heal a
+ // version-stamped bundle with null server fields — that would
+ // permanently cache the miss and the fast path would never retry.
+ pendingById.delete(id);
}
}