{
+ const interactivity = new Set(
+ paretoFrontUpperLeft(siblingParetoPoints(siblings, 'p90_intvty')).map((point) => point.id),
+ );
+ const ttft = new Set(
+ paretoFrontUpperRight(siblingParetoPoints(siblings, 'p90_ttft')).map((point) => point.id),
+ );
+ return new Set([...interactivity].filter((id) => ttft.has(id)));
}
type SortMode = 'default' | 'conc' | 'parallelism' | 'tput' | 'requests';
@@ -182,9 +226,8 @@ const isSortMode = (v: string | null): v is SortMode =>
export function SiblingNav({ sku, siblings }: { sku: BenchmarkSku; siblings: BenchmarkSibling[] }) {
const router = useRouter();
const pathname = usePathname();
- const agenticBase = isZhPathname(pathname)
- ? `${ZH_PREFIX}/inference/agentic`
- : '/inference/agentic';
+ const isZh = isZhPathname(pathname);
+ const agenticBase = isZh ? `${ZH_PREFIX}/inference/agentic` : '/inference/agentic';
// Persist the sort in the URL so clicking a point (which remounts this
// component on the new route) keeps the chosen order instead of resetting.
// Read it once from the URL on mount — this component only renders after the
@@ -197,6 +240,7 @@ export function SiblingNav({ sku, siblings }: { sku: BenchmarkSku; siblings: Ben
});
const sorted = useMemo(() => sortSiblings(siblings, sortMode), [siblings, sortMode]);
+ const dualParetoIds = useMemo(() => dualParetoSiblingIds(siblings), [siblings]);
// prev/next follow the displayed (sorted) order so navigation matches the row.
const currentIdx = sorted.findIndex((s) => s.is_current);
@@ -272,6 +316,17 @@ export function SiblingNav({ sku, siblings }: { sku: BenchmarkSku; siblings: Ben
{sorted.map((s) => {
const active = s.is_current;
+ const isDualPareto = dualParetoIds.has(s.id);
+ const title = [
+ isDualPareto
+ ? isZh
+ ? '同时位于 P90 交互性与 TTFT Pareto 前沿'
+ : 'On both P90 interactivity and TTFT Pareto frontiers'
+ : null,
+ s.has_trace ? null : isZh ? '无已存储的追踪数据' : 'No stored trace data',
+ ]
+ .filter(Boolean)
+ .join(' · ');
return (
@@ -309,6 +374,15 @@ export function SiblingNav({ sku, siblings }: { sku: BenchmarkSku; siblings: Ben
next
+ {dualParetoIds.size > 0 && (
+
+
+ {isZh ? 'P90 交互性 + TTFT Pareto' : 'P90 interactivity + TTFT Pareto'}
+
+ )}
);
}
diff --git a/packages/app/src/components/inference/agentic-point/time-series-chart.tsx b/packages/app/src/components/inference/agentic-point/time-series-chart.tsx
index f112538f1..ef8b7e53e 100644
--- a/packages/app/src/components/inference/agentic-point/time-series-chart.tsx
+++ b/packages/app/src/components/inference/agentic-point/time-series-chart.tsx
@@ -3,14 +3,34 @@
import { useMemo } from 'react';
import type { TimeSeriesPoint } from '@/hooks/api/use-trace-server-metrics';
+import { useLocale } from '@/lib/use-locale';
import { ChartHover, type HoverItem } from './chart-hover';
import { CHART_PAD, ChartEmpty, fmtCount, fmtSeconds } from './chart-shared';
import { interpAt, maxTimeSeriesValue, type ChartSeries } from './time-series-math';
-// Historical entry point: the pure data-shaping helpers lived in this module
-// before being extracted; re-export them so both import paths stay valid.
-export * from './time-series-math';
+// Historical entry point: keep the extracted data-shaping helpers available
+// here through named exports, which also compile in the Cypress component app.
+export { interpAt, maxTimeSeriesValue };
+export type { ChartSeries };
+export {
+ averageSequenceLengthInFlight,
+ buildThroughputChartSeries,
+ cumulativeAverage,
+ cumulativeCompletedRequests,
+ cumulativeDifferenceMonotonic,
+ cumulativeTimeAverage,
+ cumulativeUniqueInputTokens,
+ inflightUniqueTokens,
+ quantile,
+ rollingAverage,
+ rollingRatioFromComponents,
+ rollingRatioOfSums,
+ rollingRequestMetric,
+ timeRollingAverage,
+ toggleThroughputSeries,
+} from './time-series-math';
+export type { RequestMetric, RequestPercentile, ThroughputSeriesKey } from './time-series-math';
/** A constant horizontal reference line (e.g. a capacity ceiling). */
export interface ReferenceLine {
@@ -284,17 +304,64 @@ const KNOWN_SOURCE_COLORS: Record = {
miss: '#f97316',
'cache hit (HBM)': '#3b82f6',
'cache hit (CPU offload)': '#22c55e',
+ 'cache hit (NVMe offload)': '#a855f7',
+ 'cache hit (P2P)': '#06b6d4',
+ 'cache hit (filesystem)': '#14b8a6',
+ 'cache hit (object store)': '#ec4899',
+ 'cache hit (mixed tiers)': '#ef4444',
+ 'cache hit (external)': '#64748b',
'cache hit': '#3b82f6',
'compute (miss)': '#f97316',
};
-const SOURCE_LABELS: Record = {
- local_compute: 'Prefill',
- local_cache_hit: 'HBM Cache Hit',
- external_kv_transfer: 'Offload Cache Hit',
- miss: 'Miss',
+const SOURCE_LABELS = {
+ en: {
+ local_compute: 'Prefill',
+ local_cache_hit: 'HBM Cache Hit',
+ external_kv_transfer: 'Offload Cache Hit',
+ miss: 'Miss',
+ 'cache hit (HBM)': 'HBM Cache Hit',
+ 'cache hit (CPU offload)': 'CPU Offload Cache Hit',
+ 'cache hit (NVMe offload)': 'NVMe Offload Cache Hit',
+ 'cache hit (P2P)': 'P2P Cache Hit',
+ 'cache hit (filesystem)': 'Filesystem Cache Hit',
+ 'cache hit (object store)': 'Object Store Cache Hit',
+ 'cache hit (mixed tiers)': 'Mixed-Tier Cache Hit',
+ 'cache hit (external)': 'External Cache Hit',
+ 'cache hit': 'Cache Hit',
+ 'compute (miss)': 'Prefill',
+ },
+ zh: {
+ local_compute: 'Prefill',
+ local_cache_hit: 'HBM cache 命中',
+ external_kv_transfer: 'Offload cache 命中',
+ miss: '未命中',
+ 'cache hit (HBM)': 'HBM cache 命中',
+ 'cache hit (CPU offload)': 'CPU offload cache 命中',
+ 'cache hit (NVMe offload)': 'NVMe offload cache 命中',
+ 'cache hit (P2P)': 'P2P cache 命中',
+ 'cache hit (filesystem)': '文件系统 cache 命中',
+ 'cache hit (object store)': '对象存储 cache 命中',
+ 'cache hit (mixed tiers)': '混合层级 cache 命中',
+ 'cache hit (external)': '外部 cache 命中',
+ 'cache hit': 'Cache 命中',
+ 'compute (miss)': 'Prefill',
+ },
+} as const;
+
+const STACKED_AREA_STRINGS = {
+ en: { time: 'time', share: '% of prefill tokens' },
+ zh: { time: '时间', share: 'prefill token 占比' },
};
+function sourceLabel(source: string, locale: 'en' | 'zh'): string {
+ const known = SOURCE_LABELS[locale] as Record;
+ if (known[source]) return known[source];
+ const customTier = /^cache hit \((?.+)\)$/u.exec(source)?.groups?.tier;
+ if (!customTier) return source;
+ return locale === 'zh' ? `${customTier} cache 命中` : `Cache Hit (${customTier})`;
+}
+
// Fallback palette for any source name not in KNOWN_SOURCE_COLORS so we never
// emit two layers in the same shade. Cycles by stack (insertion) order.
const FALLBACK_PALETTE = [
@@ -320,6 +387,8 @@ export function StackedAreaChart({
width?: number;
height?: number;
}) {
+ const locale = useLocale();
+ const strings = STACKED_AREA_STRINGS[locale];
const W = width;
const H = height;
@@ -421,7 +490,7 @@ export function StackedAreaChart({
}
const items: HoverItem[] = stackOrder.map((name) => ({
color: colorFor(name),
- label: SOURCE_LABELS[name] ?? name,
+ label: sourceLabel(name, locale),
value: `${((shares[name]?.[idx] ?? 0) * 100).toFixed(1)}%`,
}));
return { items, title: fmtSeconds(t) };
@@ -493,7 +562,7 @@ export function StackedAreaChart({
opacity={0.55}
textAnchor="middle"
>
- time
+ {strings.time}
- % of prefill tokens
+ {strings.share}
{(() => {
const chipY = H - 8;
@@ -515,7 +584,7 @@ export function StackedAreaChart({
- {SOURCE_LABELS[l.name] ?? l.name}
+ {sourceLabel(l.name, locale)}
);
diff --git a/packages/app/src/components/inference/utils/runtime-metadata-labels.test.ts b/packages/app/src/components/inference/utils/runtime-metadata-labels.test.ts
index 5f4fd776d..9f4eae73f 100644
--- a/packages/app/src/components/inference/utils/runtime-metadata-labels.test.ts
+++ b/packages/app/src/components/inference/utils/runtime-metadata-labels.test.ts
@@ -21,6 +21,8 @@ describe('runtime metadata labels', () => {
it('formats DRAM and other offload types consistently', () => {
expect(offloadTypeLabel('dram')).toBe('DRAM');
+ expect(offloadTypeLabel('nvme')).toBe('NVMe');
+ expect(offloadTypeLabel('dram+nvme')).toBe('DRAM+NVMe');
expect(offloadTypeLabel('cpu')).toBe('CPU');
});
});
diff --git a/packages/app/src/components/inference/utils/runtime-metadata-labels.ts b/packages/app/src/components/inference/utils/runtime-metadata-labels.ts
index 3c151b9a0..c9da5bdd9 100644
--- a/packages/app/src/components/inference/utils/runtime-metadata-labels.ts
+++ b/packages/app/src/components/inference/utils/runtime-metadata-labels.ts
@@ -16,11 +16,20 @@ const CACHE_IMPLEMENTATION_LABELS: Record = {
export const cacheImplementationLabel = (value: string): string =>
CACHE_IMPLEMENTATION_LABELS[value.toLowerCase()] ?? value;
-export const offloadTypeLabel = (value: string): string => {
- if (value.toLowerCase() === 'dram') return 'DRAM';
- return value.toUpperCase();
+const OFFLOAD_TIER_LABELS: Record = {
+ dram: 'DRAM',
+ nvme: 'NVMe',
};
+export const offloadTypeLabel = (value: string): string =>
+ value
+ .split('+')
+ .map((tier) => {
+ const normalized = tier.trim().toLowerCase();
+ return OFFLOAD_TIER_LABELS[normalized] ?? normalized.toUpperCase();
+ })
+ .join('+');
+
export const versionedComponentLabel = (
name: string | null | undefined,
version: string | null | undefined,
diff --git a/packages/app/src/hooks/api/use-benchmark-siblings.ts b/packages/app/src/hooks/api/use-benchmark-siblings.ts
index 7471dd45b..0e76e25ea 100644
--- a/packages/app/src/hooks/api/use-benchmark-siblings.ts
+++ b/packages/app/src/hooks/api/use-benchmark-siblings.ts
@@ -4,6 +4,7 @@ export interface BenchmarkSibling {
id: number;
conc: number;
offload_mode: string | null;
+ kv_offloading: string | null;
decode_tp: number;
decode_ep: number;
decode_pp: number | null;
@@ -23,6 +24,8 @@ export interface BenchmarkSibling {
disagg: boolean;
is_multinode: boolean;
tput_per_gpu: number | null;
+ p90_intvty: number | null;
+ p90_ttft: number | null;
total_requests: number | null;
is_current: boolean;
has_trace: boolean;
diff --git a/packages/app/src/hooks/api/use-trace-server-metrics.ts b/packages/app/src/hooks/api/use-trace-server-metrics.ts
index 68ad6d441..439bf36ce 100644
--- a/packages/app/src/hooks/api/use-trace-server-metrics.ts
+++ b/packages/app/src/hooks/api/use-trace-server-metrics.ts
@@ -57,6 +57,7 @@ export interface MetricSourceSeries {
kvCacheUsage: TimeSeriesPoint[];
prefixCacheHitRate: TimeSeriesPoint[];
queueDepth: QueueDepthPoint[];
+ /** Fresh prefill plus physical cache tiers when available; logical fallback otherwise. */
promptTokensBySource: Record;
promptTps: TimeSeriesPoint[];
generationTps: TimeSeriesPoint[];
diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts
index 4a22d16c5..bbea35a63 100644
--- a/packages/app/src/lib/api-documentation.ts
+++ b/packages/app/src/lib/api-documentation.ts
@@ -1794,6 +1794,7 @@ export const apiOperations: readonly ApiOperation[] = [
id: 421,
conc: 32,
offload_mode: 'off',
+ kv_offloading: 'none',
decode_tp: 8,
decode_ep: 1,
decode_pp: null,
@@ -1813,6 +1814,8 @@ export const apiOperations: readonly ApiOperation[] = [
disagg: false,
is_multinode: false,
tput_per_gpu: 128.4,
+ p90_intvty: 7.2,
+ p90_ttft: 4.8,
total_requests: 320,
is_current: true,
has_trace: true,
@@ -2408,8 +2411,8 @@ export const apiOperations: readonly ApiOperation[] = [
path: '/api/v1/trace-server-metrics',
summary: text('Read trace server metrics', '读取跟踪服务器指标'),
description: text(
- 'Returns point metadata and chart-ready aggregate time series for cache usage, queue depth, prefill and decode throughput, and prompt-token sources. metricSources contains source descriptors; source-specific arrays are loaded by the point-detail UI only when selected.',
- '返回点元数据，以及缓存使用率、队列深度、预填充和解码吞吐、提示 token 来源的图表聚合时间序列。metricSources 仅包含来源描述信息；详情页只会在用户选中某个来源时加载其专属数组。',
+ 'Returns point metadata and chart-ready aggregate time series for cache usage, queue depth, prefill and decode throughput, and prompt-token sources. When vLLM exports physical cache-tier attribution, promptTokensBySource separates HBM, CPU, NVMe, and connector-defined hits while retaining fresh prefill; older rows keep the logical cache-hit fallback. metricSources contains source descriptors; source-specific arrays are loaded by the point-detail UI only when selected.',
+ '返回点元数据，以及 cache 使用率、队列深度、prefill 和 decode 吞吐、prompt token 来源的图表聚合时间序列。当 vLLM 导出物理 cache 层级归因时，promptTokensBySource 会区分 HBM、CPU、NVMe 和 connector 自定义命中来源，同时保留新计算的 prefill；旧数据继续使用逻辑 cache 命中回退。metricSources 仅包含来源描述信息；详情页只会在用户选中某个来源时加载其专属数组。',
),
audience: 'public',
stability: 'beta',
@@ -2463,7 +2466,11 @@ export const apiOperations: readonly ApiOperation[] = [
kvCacheUsage: [{ t: 0, v: 0.44 }],
prefixCacheHitRate: [],
queueDepth: [],
- promptTokensBySource: {},
+ promptTokensBySource: {
+ local_compute: [{ t: 0, value: 2200 }],
+ 'cache hit (HBM)': [{ t: 0, value: 6800 }],
+ 'cache hit (NVMe offload)': [{ t: 0, value: 1000 }],
+ },
prefillTps: [],
decodeTps: [],
prefixCacheHitsTps: [],
diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts
index d60e6fdba..55fdf628b 100644
--- a/packages/app/src/lib/api-route-catalog.ts
+++ b/packages/app/src/lib/api-route-catalog.ts
@@ -100,7 +100,7 @@ export const apiRouteCatalog = [
method: 'GET',
classification: 'published-read',
operationId: 'get-benchmark-siblings',
- sourceSha256: '2b20de8b2b67ed53027eba478068f8f22ae7e3843ac38423e8ce6b27c1f74fd6',
+ sourceSha256: '72a4dd160a940a95b9ce7fd27b8f272421178889b8ecae298211174d5c7f506d',
},
{
source: 'src/app/api/v1/benchmarks/route.ts',
@@ -719,7 +719,7 @@ export const apiContractSourceDigests = [
},
{
source: '../db/src/queries/benchmark-siblings.ts',
- sourceSha256: '07d3d1bf93820091d1014b14bf954555edcec422c05e575ad87142f9845e4156',
+ sourceSha256: '9ce28ba60ddcfbbba740036060baa7645f2421e3eb4c73db15a6ff3db415452c',
reviewArea: {
en: 'Benchmark sibling SKU metadata and sibling navigation row shape.',
zh: '基准同组 SKU 元数据和同组导航行结构。',
@@ -815,7 +815,7 @@ export const apiContractSourceDigests = [
},
{
source: '../db/src/queries/trace-server-metrics.ts',
- sourceSha256: 'da987d22521dc63da34dcacf3caaad6adb21aa6e5e7a65d1a4fa495a71e9f1f0',
+ sourceSha256: 'b7ded0f27e9328f9d0e1c2bd349600cdfd93ba6da91b3b9ac479cb0770042d9a',
reviewArea: {
en: 'Trace server metric metadata, time-series groups, source labels, and units.',
zh: '跟踪服务器指标元数据、时间序列分组、来源标签和单位。',
diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts
index 7d3271656..dcbc3e6e1 100644
--- a/packages/app/src/lib/chart-utils.ts
+++ b/packages/app/src/lib/chart-utils.ts
@@ -637,10 +637,15 @@ export const getNestedYValue = (point: T, key: string):
*/
export const isFrontierEligible = (p: { x: number }): boolean => Number.isFinite(p.x) && p.x > 0;
+interface ParetoPoint {
+ x: number;
+ y: number;
+}
+
/**
* Calculates the Pareto front (upper right) for a given set of points.
*/
-export const paretoFrontUpperRight = (points: InferenceData[]): InferenceData[] => {
+export const paretoFrontUpperRight = (points: T[]): T[] => {
if (points.length === 0) {
return [];
}
@@ -652,7 +657,7 @@ export const paretoFrontUpperRight = (points: InferenceData[]): InferenceData[]
return a.x - b.x;
});
- const front: InferenceData[] = [];
+ const front: T[] = [];
let maxY = -Infinity;
for (const point of points) {
@@ -671,7 +676,7 @@ export const paretoFrontUpperRight = (points: InferenceData[]): InferenceData[]
/**
* Calculates the Pareto front (upper left) for a given set of points.
*/
-export const paretoFrontUpperLeft = (points: InferenceData[]): InferenceData[] => {
+export const paretoFrontUpperLeft = (points: T[]): T[] => {
if (points.length === 0) {
return [];
}
@@ -683,7 +688,7 @@ export const paretoFrontUpperLeft = (points: InferenceData[]): InferenceData[] =
return a.x - b.x;
});
- const front: InferenceData[] = [];
+ const front: T[] = [];
for (const point of points) {
if (front.length > 0 && point.x === front.at(-1)!.x) {
@@ -704,7 +709,7 @@ export const paretoFrontUpperLeft = (points: InferenceData[]): InferenceData[] =
/**
* Calculates the Pareto front (lower left) for a given set of points.
*/
-export const paretoFrontLowerLeft = (points: InferenceData[]): InferenceData[] => {
+export const paretoFrontLowerLeft = (points: T[]): T[] => {
if (points.length === 0) {
return [];
}
@@ -716,7 +721,7 @@ export const paretoFrontLowerLeft = (points: InferenceData[]): InferenceData[] =
return a.x - b.x;
});
- const front: InferenceData[] = [];
+ const front: T[] = [];
let minY = Infinity;
for (const point of points) {
@@ -731,7 +736,7 @@ export const paretoFrontLowerLeft = (points: InferenceData[]): InferenceData[] =
/**
* Calculates the Pareto front (lower right) for a given set of points.
*/
-export const paretoFrontLowerRight = (points: InferenceData[]): InferenceData[] => {
+export const paretoFrontLowerRight = (points: T[]): T[] => {
if (points.length === 0) {
return [];
}
@@ -743,7 +748,7 @@ export const paretoFrontLowerRight = (points: InferenceData[]): InferenceData[]
return b.x - a.x;
});
- const front: InferenceData[] = [];
+ const front: T[] = [];
let minY = Infinity;
for (const point of points) {
@@ -767,7 +772,7 @@ export type ParetoDirection = keyof typeof PARETO_BY_DIRECTION;
/** Look up the Pareto frontier function for a roofline direction. */
export const paretoFrontForDirection = (
dir: ParetoDirection,
-): ((points: InferenceData[]) => InferenceData[]) => PARETO_BY_DIRECTION[dir];
+): ((points: T[]) => T[]) => PARETO_BY_DIRECTION[dir];
// ---------------------------------------------------------------------------
// Locale-aware metric label/title helpers
diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts
index 54295cc77..2527f40da 100644
--- a/packages/db/src/etl/compute-chart-series.test.ts
+++ b/packages/db/src/etl/compute-chart-series.test.ts
@@ -63,6 +63,13 @@ function makeBlob(opts?: {
return gzipSync(Buffer.from(json));
}
+function sourceRateSeries(source: string, rate: number) {
+ return {
+ labels: { source },
+ timeslices: [{ start_ns: 0, end_ns: 1e9, rate }],
+ };
+}
+
/** Build a synthetic per-engine vLLM metric series for the multi-engine test. */
function buildEngineSeries(engineId: number, baseRunning: number) {
const labels = { engine: String(engineId) };
@@ -239,6 +246,81 @@ describe('computeChartSeries', () => {
expect(series!.promptTokensBySource['miss']).toEqual([{ t: 0, value: 800 }]);
});
+ it('combines vLLM fresh prefill with physical cache tiers without double counting', async () => {
+ const blob = gzipSync(
+ Buffer.from(
+ JSON.stringify({
+ metrics: {
+ 'vllm:prompt_tokens_by_source': {
+ series: [
+ sourceRateSeries('local_compute', 300),
+ sourceRateSeries('local_cache_hit', 400),
+ sourceRateSeries('external_kv_transfer', 300),
+ ],
+ },
+ 'vllm:prompt_tokens_cached_by_source': {
+ series: [
+ sourceRateSeries('device', 400),
+ sourceRateSeries('cpu', 200),
+ sourceRateSeries('disk', 100),
+ sourceRateSeries('external', 0),
+ sourceRateSeries('mixed', 0),
+ ],
+ },
+ },
+ }),
+ ),
+ );
+
+ const series = await computeChartSeries(blob);
+ expect(series?.promptTokensBySource).toEqual({
+ local_compute: [{ t: 0, value: 300 }],
+ 'cache hit (HBM)': [{ t: 0, value: 400 }],
+ 'cache hit (CPU offload)': [{ t: 0, value: 200 }],
+ 'cache hit (NVMe offload)': [{ t: 0, value: 100 }],
+ });
+ expect(total(Object.values(series!.promptTokensBySource).flat())).toBe(1000);
+ expect(series?.promptTokensBySource).not.toHaveProperty('local_cache_hit');
+ expect(series?.promptTokensBySource).not.toHaveProperty('external_kv_transfer');
+ });
+
+ it('preserves connector-defined vLLM tiers and treats a missing label as external', async () => {
+ const blob = gzipSync(
+ Buffer.from(
+ JSON.stringify({
+ metrics: {
+ 'vllm:prompt_tokens_by_source': {
+ series: [
+ {
+ labels: { source: 'local_compute' },
+ timeslices: [{ start_ns: 0, end_ns: 1e9, rate: 5 }],
+ },
+ ],
+ },
+ 'vllm:prompt_tokens_cached_by_source': {
+ series: [
+ {
+ labels: { source: 'weka' },
+ timeslices: [{ start_ns: 0, end_ns: 1e9, rate: 7 }],
+ },
+ {
+ timeslices: [{ start_ns: 0, end_ns: 1e9, rate: 3 }],
+ },
+ ],
+ },
+ },
+ }),
+ ),
+ );
+
+ const series = await computeChartSeries(blob);
+ expect(series?.promptTokensBySource).toEqual({
+ local_compute: [{ t: 0, value: 5 }],
+ 'cache hit (weka)': [{ t: 0, value: 7 }],
+ 'cache hit (external)': [{ t: 0, value: 3 }],
+ });
+ });
+
it('computes timing metadata from the widest metric window', async () => {
const series = await computeChartSeries(makeBlob());
// kvCacheUsage has the widest window (0 → 3e9), so startNs=0, endNs=3e9.
diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts
index 82878ad86..aa660e787 100644
--- a/packages/db/src/etl/compute-chart-series.ts
+++ b/packages/db/src/etl/compute-chart-series.ts
@@ -120,8 +120,15 @@ import {
* per-rank detail stays reachable as that source's own
* `kvCacheUsageByEngine`.
*
+ * v16: consume vLLM's physical cached-token source metric. The existing
+ * `prompt_tokens_by_source` metric remains the source of freshly computed
+ * prompt tokens, while `prompt_tokens_cached_by_source` replaces its coarse
+ * local/external cache-hit buckets with device, CPU, disk, and connector-tier
+ * attribution. Keeping only the logical compute bucket prevents double
+ * counting and makes the stacked breakdown sum to total prompt-token volume.
+ *
*/
-export const CHART_SERIES_VERSION = 15;
+export const CHART_SERIES_VERSION = 16;
export interface TimeSeriesPoint {
/** Seconds from benchmark start. */
@@ -218,6 +225,22 @@ export interface RawMetric {
export type MetricsMap = Record;
+const VLLM_CACHE_SOURCE_BUCKETS: Record = {
+ device: 'cache hit (HBM)',
+ cpu: 'cache hit (CPU offload)',
+ disk: 'cache hit (NVMe offload)',
+ p2p: 'cache hit (P2P)',
+ fs: 'cache hit (filesystem)',
+ obj: 'cache hit (object store)',
+ mixed: 'cache hit (mixed tiers)',
+ external: 'cache hit (external)',
+};
+
+/** Canonical chart bucket for vLLM's physical cached-token source labels. */
+function vllmCacheSourceBucket(source: string): string {
+ return VLLM_CACHE_SOURCE_BUCKETS[source] ?? `cache hit (${source})`;
+}
+
/**
* The set of metric subtrees the chart consumes. Includes both vllm:* and
* sglang:* names so the stream-parse fallback collects whichever framework
@@ -234,6 +257,7 @@ export const CHART_METRIC_KEYS = new Set([
'vllm:prompt_tokens',
'vllm:generation_tokens',
'vllm:prompt_tokens_by_source',
+ 'vllm:prompt_tokens_cached_by_source',
// SGLang
'sglang:token_usage',
'sglang:cached_tokens',
@@ -1051,9 +1075,10 @@ function buildSeriesFromMetrics(
}
// Per-source prompt tokens — sum across engines per source label.
- // vllm: vllm:prompt_tokens_by_source has one series per source label
- // (local_cache_hit, external_cache_hit, miss, ...). Use the
- // `source`/`reason`/`kind` label as the breakdown key.
+ // vllm: the logical prompt_tokens_by_source metric supplies fresh
+ // prefill. When prompt_tokens_cached_by_source is available, its
+ // physical cache tiers replace the logical local/external hit
+ // buckets so the two metrics are combined without double counting.
// sglang: sglang:realtime_tokens uses a `mode` label with values
// {prefill_cache, prefill_compute, decode}. Filter to prefill_*
// since decode isn't prompt-token volume.
@@ -1070,10 +1095,27 @@ function buildSeriesFromMetrics(
if (existing) existing.push(series);
else promptBySrc.set(label, [series]);
};
- for (const series of metrics['vllm:prompt_tokens_by_source']?.series ?? []) {
- const labels = series.labels ?? {};
- const source = labels['source'] ?? labels['reason'] ?? labels['kind'] ?? JSON.stringify(labels);
- addSeriesRates(source, series);
+ const logicalVllmSeries = metrics['vllm:prompt_tokens_by_source']?.series ?? [];
+ const physicalVllmSeries = metrics['vllm:prompt_tokens_cached_by_source']?.series ?? [];
+ if (physicalVllmSeries.length > 0) {
+ for (const series of logicalVllmSeries) {
+ const labels = series.labels ?? {};
+ const source = labels['source'] ?? labels['reason'] ?? labels['kind'] ?? '';
+ if (source === 'local_compute' || source === 'miss') {
+ addSeriesRates(source, series);
+ }
+ }
+ for (const series of physicalVllmSeries) {
+ const source = series.labels?.['source'] || 'external';
+ addSeriesRates(vllmCacheSourceBucket(source), series);
+ }
+ } else {
+ for (const series of logicalVllmSeries) {
+ const labels = series.labels ?? {};
+ const source =
+ labels['source'] ?? labels['reason'] ?? labels['kind'] ?? JSON.stringify(labels);
+ addSeriesRates(source, series);
+ }
}
// SGLang fallback: only consider when the vllm metric wasn't found.
// - Cache misses (fresh prefill): `sglang:realtime_tokens[mode=prefill_compute]`
diff --git a/packages/db/src/queries/benchmark-siblings.test.ts b/packages/db/src/queries/benchmark-siblings.test.ts
new file mode 100644
index 000000000..a1b07eb5c
--- /dev/null
+++ b/packages/db/src/queries/benchmark-siblings.test.ts
@@ -0,0 +1,84 @@
+import { describe, expect, it } from 'vitest';
+
+import type { DbClient } from '../connection.js';
+
+import { getBenchmarkSiblings } from './benchmark-siblings.js';
+
+function mockSql(queue: unknown[][]): {
+ sql: DbClient;
+ calls: string[];
+} {
+ const responses = [...queue];
+ const calls: string[] = [];
+ const sql = ((strings: TemplateStringsArray) => {
+ calls.push(strings.join('?'));
+ return Promise.resolve(responses.shift() ?? []);
+ }) as unknown as DbClient;
+ return { sql, calls };
+}
+
+describe('getBenchmarkSiblings', () => {
+ it('returns physical offload tiers and P90 Pareto coordinates', async () => {
+ const { sql, calls } = mockSql([
+ [
+ {
+ hardware: 'h100',
+ framework: 'vllm',
+ model: 'minimaxm3',
+ precision: 'fp8',
+ spec_method: 'mtp',
+ benchmark_type: 'agentic_traces',
+ workflow_run_id: 12,
+ date: '2026-08-28',
+ github_run_id: 33169766010,
+ dataset_slug: 'agentx-minimaxm3',
+ },
+ ],
+ [
+ {
+ id: 440549,
+ conc: 7,
+ offload_mode: 'on',
+ kv_offloading: 'dram+nvme',
+ decode_tp: 8,
+ decode_ep: 1,
+ decode_pp: 1,
+ decode_dcp_size: null,
+ decode_pcp_size: null,
+ decode_dp_attention: false,
+ decode_num_workers: 1,
+ prefill_tp: 8,
+ prefill_ep: 1,
+ prefill_pp: 1,
+ prefill_dcp_size: null,
+ prefill_pcp_size: null,
+ prefill_dp_attention: false,
+ prefill_num_workers: 1,
+ num_prefill_gpu: 0,
+ num_decode_gpu: 8,
+ disagg: false,
+ is_multinode: false,
+ tput_per_gpu: '128.4',
+ p90_intvty: '7.2',
+ p90_ttft: '4.8',
+ total_requests: '320',
+ has_trace: true,
+ },
+ ],
+ ]);
+
+ const result = await getBenchmarkSiblings(sql, 440549);
+
+ expect(result?.siblings[0]).toMatchObject({
+ id: 440549,
+ kv_offloading: 'dram+nvme',
+ tput_per_gpu: 128.4,
+ p90_intvty: 7.2,
+ p90_ttft: 4.8,
+ is_current: true,
+ });
+ expect(calls[1]).toContain("br.metrics->>'kv_offloading'");
+ expect(calls[1]).toContain("br.metrics->>'p90_intvty'");
+ expect(calls[1]).toContain("br.metrics->>'p90_ttft'");
+ });
+});
diff --git a/packages/db/src/queries/benchmark-siblings.ts b/packages/db/src/queries/benchmark-siblings.ts
index a6a6c41f8..1d9246f99 100644
--- a/packages/db/src/queries/benchmark-siblings.ts
+++ b/packages/db/src/queries/benchmark-siblings.ts
@@ -12,6 +12,8 @@ export interface BenchmarkSibling {
conc: number;
/** "on" | "off" | null. */
offload_mode: string | null;
+ /** Physical KV-cache tier selection reported by the benchmark runner. */
+ kv_offloading: string | null;
decode_tp: number;
decode_ep: number;
decode_pp: number | null;
@@ -32,6 +34,10 @@ export interface BenchmarkSibling {
is_multinode: boolean;
/** Throughput per GPU (tok/s/gpu) for this point; null if the metric is absent. */
tput_per_gpu: number | null;
+ /** P90 interactivity (tok/s/user), used by the AgentX Pareto navigator. */
+ p90_intvty: number | null;
+ /** P90 time to first token (s), used by the AgentX Pareto navigator. */
+ p90_ttft: number | null;
/**
* Total requests for this point — `total_requests_completed` (aiperf runner)
* falling back to the legacy `num_requests_total`; null if neither is present.
@@ -96,6 +102,7 @@ export async function getBenchmarkSiblings(
const rows = (await sql`
select
br.id, br.conc, br.offload_mode,
+ nullif(br.metrics->>'kv_offloading', '') as kv_offloading,
c.decode_tp, c.decode_ep, (br.metrics->>'decode_pp')::int as decode_pp,
coalesce(
(br.metrics->>'decode_dcp_size')::int,
@@ -118,6 +125,8 @@ export async function getBenchmarkSiblings(
c.prefill_dp_attention, c.prefill_num_workers,
c.num_prefill_gpu, c.num_decode_gpu, c.disagg, c.is_multinode,
(br.metrics->>'tput_per_gpu')::float8 as tput_per_gpu,
+ (br.metrics->>'p90_intvty')::float8 as p90_intvty,
+ (br.metrics->>'p90_ttft')::float8 as p90_ttft,
coalesce(
(br.metrics->>'total_requests_completed')::float8,
(br.metrics->>'num_requests_total')::float8
@@ -137,6 +146,7 @@ export async function getBenchmarkSiblings(
id: number;
conc: number;
offload_mode: string | null;
+ kv_offloading: string | null;
decode_tp: number;
decode_ep: number;
decode_pp: number | null;
@@ -156,6 +166,8 @@ export async function getBenchmarkSiblings(
disagg: boolean;
is_multinode: boolean;
tput_per_gpu: number | null;
+ p90_intvty: number | null;
+ p90_ttft: number | null;
total_requests: number | null;
has_trace: boolean;
}[];
@@ -164,6 +176,7 @@ export async function getBenchmarkSiblings(
id: Number(r.id),
conc: r.conc,
offload_mode: r.offload_mode,
+ kv_offloading: r.kv_offloading,
decode_tp: r.decode_tp,
decode_ep: r.decode_ep,
decode_pp: r.decode_pp === null ? null : Number(r.decode_pp),
@@ -183,6 +196,8 @@ export async function getBenchmarkSiblings(
disagg: r.disagg,
is_multinode: r.is_multinode,
tput_per_gpu: r.tput_per_gpu === null ? null : Number(r.tput_per_gpu),
+ p90_intvty: r.p90_intvty === null ? null : Number(r.p90_intvty),
+ p90_ttft: r.p90_ttft === null ? null : Number(r.p90_ttft),
total_requests: r.total_requests === null ? null : Number(r.total_requests),
is_current: Number(r.id) === benchmarkResultId,
has_trace: r.has_trace,
diff --git a/packages/db/src/queries/trace-server-metrics.ts b/packages/db/src/queries/trace-server-metrics.ts
index 5e3f5f076..374865118 100644
--- a/packages/db/src/queries/trace-server-metrics.ts
+++ b/packages/db/src/queries/trace-server-metrics.ts
@@ -86,8 +86,8 @@ export interface TraceServerMetrics {
queueDepth: QueueDepthPoint[];
/**
* Per-source prompt-token counts over time (counter rate per scrape).
- * Keyed by the value of the `source` label (typically `local_cache_hit`,
- * `external_cache_hit`, `miss`, etc.). Plot as stacked area.
+ * Fresh prefill is combined with physical cache-tier hits when the producer
+ * exposes them; older rows retain their logical cache-hit buckets.
*/
promptTokensBySource: Record;
/** Prefill throughput: vllm:prompt_tokens rate (tokens/sec) per scrape. */