From bc2ba7bf6b01c7f50783d9505cb6c13aad784df4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 16 Aug 2026 15:03:11 -0500 Subject: [PATCH 1/3] feat(agentx): display TensorRT-LLM server metrics --- .../db/src/etl/compute-aggregate-stats.ts | 4 + .../db/src/etl/compute-chart-series.test.ts | 77 +++++++++++++ packages/db/src/etl/compute-chart-series.ts | 101 ++++++++++++++++-- .../db/src/etl/server-metrics-adapters.ts | 33 +++++- .../db/src/queries/agentic-aggregates.test.ts | 41 +++++++ packages/db/src/queries/agentic-aggregates.ts | 16 +++ 6 files changed, 263 insertions(+), 9 deletions(-) diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index 07729b59..54a89644 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -193,6 +193,10 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([ 'vllm:prefix_cache_queries', 'vllm:gpu_prefix_cache_hits', 'vllm:gpu_prefix_cache_queries', + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens_total', ]); /** diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 54295cc7..4b93ae0e 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -146,6 +146,19 @@ function kvBlob(profiling: unknown[], warmup: unknown[] = []) { ); } +function buildTrtllmSeries( + endpoint_url: string, + dynamo_component: 'prefill' | 'backend', + value: number, + field: 'rate' | 'avg', +) { + return { + endpoint_url, + labels: { dynamo_component, worker_id: `${dynamo_component}-worker` }, + timeslices: [{ start_ns: 0, end_ns: 1e9, [field]: value }], + }; +} + describe('computeChartSeries', () => { it('returns null when the blob is null', async () => { expect(await computeChartSeries(null)).toBeNull(); @@ -708,6 +721,70 @@ describe('computeChartSeries', () => { expect(result?.metricSources).toEqual([]); }); + + it('extracts native TensorRT-LLM metrics and preserves disaggregated worker roles', async () => { + const prefillUrl = 'http://prefill-a.internal.test:7500/metrics'; + const decodeUrl = 'http://decode-a.internal.test:7501/metrics'; + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 0.3, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 0.7, 'avg'), + ], + }, + trtllm_kv_cache_host_utilization: { + series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], + }, + trtllm_prompt_tokens_total: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 100, 'rate'), + buildTrtllmSeries(decodeUrl, 'backend', 200, 'rate'), + ], + }, + trtllm_prompt_cached_tokens_total: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'), + buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'), + ], + }, + trtllm_generation_tokens_total: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')], + }, + trtllm_num_requests_running: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 2, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 3, 'avg'), + ], + }, + trtllm_num_requests_waiting: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 4, 'avg')], + }, + }, + }); + + const result = await computeChartSeries(gzipSync(Buffer.from(json)), { + framework: 'trtllm', + disagg: true, + }); + + expect(result?.kvCacheUsage).toEqual([{ t: 0, value: 0.5 }]); + expect(result?.hostKvCacheUsage).toEqual([{ t: 0, value: 0.25 }]); + expect(result?.prefixCacheHitRate).toEqual([{ t: 0, value: 0.4 }]); + expect(result?.queueDepth).toEqual([{ t: 0, running: 5, waiting: 4, total: 9 }]); + expect(result?.prefillTps).toEqual([{ t: 0, value: 300 }]); + expect(result?.decodeTps).toEqual([{ t: 0, value: 50 }]); + expect(result?.promptTokensBySource).toEqual({ + 'cache hit (HBM)': [{ t: 0, value: 120 }], + 'compute (miss)': [{ t: 0, value: 180 }], + }); + expect(result?.metricSources.map(({ source }) => [source.role, source.endpointUrl])).toEqual([ + ['prefill', prefillUrl], + ['decode', decodeUrl], + ]); + expect(result?.metricSources[0]?.promptTps).toEqual([{ t: 0, value: 100 }]); + expect(result?.metricSources[1]?.generationTps).toEqual([{ t: 0, value: 50 }]); + }); }); // ── Summed series on the canonical grid (v14) ─────────────────────────── diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 82878ad8..39d4aac6 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -120,8 +120,10 @@ import { * per-rank detail stays reachable as that source's own * `kvCacheUsageByEngine`. * + * v16: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV + * metrics, including per-prefill/decode source series for Dynamo disaggregation. */ -export const CHART_SERIES_VERSION = 15; +export const CHART_SERIES_VERSION = 16; export interface TimeSeriesPoint { /** Seconds from benchmark start. */ @@ -244,6 +246,15 @@ export const CHART_METRIC_KEYS = new Set([ 'sglang:realtime_tokens', 'sglang:hicache_host_used_tokens', 'sglang:hicache_host_total_tokens', + // TensorRT-LLM + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_host_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens_total', + 'trtllm_generation_tokens_total', + 'trtllm_num_requests_running', + 'trtllm_num_requests_waiting', ]); /** @@ -976,6 +987,7 @@ function buildSeriesFromMetrics( 'vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage', + 'trtllm_kv_cache_utilization', ); // One entry per logical engine (v13) — mirrored API-server frontends and the // warmup/profiling phase split are collapsed here rather than showing up as @@ -989,11 +1001,16 @@ function buildSeriesFromMetrics( // Prefix cache hit rate per scrape: Σhits.rate / Σqueries.rate across // engines, joined on start_ns. SGLang names: cached_tokens / prompt_tokens. - const hitsSeries = pickSeries('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const hitsSeries = pickSeries( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); const qsSeries = pickSeries( 'vllm:prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsOnGrid = summedSeries(hitsSeries, tOf, 'rate', tickS); const qsOnGrid = summedSeries(qsSeries, tOf, 'rate', tickS); @@ -1003,10 +1020,25 @@ function buildSeriesFromMetrics( const q = qsByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push({ t, value: h / q }); } + if (prefixCacheHitRate.length === 0) { + for (const [t, value] of sortedEntries( + aggregateByStart(metrics['trtllm_kv_cache_hit_rate']?.series, 'avg', 'avg'), + )) { + prefixCacheHitRate.push({ t: tOf(t), value }); + } + } // Queue depth: sum running + waiting across engines per timeslice. - const runSeries = pickSeries('vllm:num_requests_running', 'sglang:num_running_reqs'); - const waitSeries = pickSeries('vllm:num_requests_waiting', 'sglang:num_queue_reqs'); + const runSeries = pickSeries( + 'vllm:num_requests_running', + 'sglang:num_running_reqs', + 'trtllm_num_requests_running', + ); + const waitSeries = pickSeries( + 'vllm:num_requests_waiting', + 'sglang:num_queue_reqs', + 'trtllm_num_requests_waiting', + ); const runOnGrid = summedSeries(runSeries, tOf, 'avg', tickS); const waitByT = byT(summedSeries(waitSeries, tOf, 'avg', tickS)); const runByT = byT(runOnGrid); @@ -1025,11 +1057,23 @@ function buildSeriesFromMetrics( // work. const counterRate = (...names: string[]): TimeSeriesPoint[] => summedSeries(pickSeries(...names), tOf, 'rate', tickS); - const prefillTps = counterRate('vllm:prompt_tokens', 'sglang:prompt_tokens'); - const decodeTps = counterRate('vllm:generation_tokens', 'sglang:generation_tokens'); + const prefillTps = counterRate( + 'vllm:prompt_tokens', + 'sglang:prompt_tokens', + 'trtllm_prompt_tokens_total', + ); + const decodeTps = counterRate( + 'vllm:generation_tokens', + 'sglang:generation_tokens', + 'trtllm_generation_tokens_total', + ); // Tokens served from prefix cache per scrape. Lets the frontend derive // "cumulative unique input tokens served" = cumsum(prefillTps) − cumsum(hits). - const prefixCacheHitsTps = counterRate('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const prefixCacheHitsTps = counterRate( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); // SGLang hicache: host-pool KV cache utilization as used/total per // timeslice. Both metrics are gauges in absolute tokens. Total stays @@ -1049,6 +1093,13 @@ function buildSeriesFromMetrics( hostKvCacheUsage.push({ t, value: used / total }); } } + if (hostKvCacheUsage.length === 0) { + for (const [t, value] of sortedEntries( + aggregateByStart(metrics['trtllm_kv_cache_host_utilization']?.series, 'avg', 'avg'), + )) { + hostKvCacheUsage.push({ t: tOf(t), value }); + } + } // Per-source prompt tokens — sum across engines per source label. // vllm: vllm:prompt_tokens_by_source has one series per source label @@ -1108,6 +1159,27 @@ function buildSeriesFromMetrics( addSeriesRates(label, series); } } + if (promptBySrcByT.size === 0) { + const promptByT = aggregateByStart( + metrics['trtllm_prompt_tokens_total']?.series, + 'rate', + 'sum', + ); + const cachedByT = aggregateByStart( + metrics['trtllm_prompt_cached_tokens_total']?.series, + 'rate', + 'sum', + ); + const cachedSeries: RawSeries = { timeslices: [] }; + const computedSeries: RawSeries = { timeslices: [] }; + for (const [t, prompt] of promptByT) { + const cached = Math.max(0, cachedByT.get(t) ?? 0); + cachedSeries.timeslices!.push({ start_ns: t, rate: cached }); + computedSeries.timeslices!.push({ start_ns: t, rate: Math.max(0, prompt - cached) }); + } + addSeriesRates('cache hit (HBM)', cachedSeries); + addSeriesRates('compute (miss)', computedSeries); + } const promptTokensBySource: Record = {}; for (const [source, seriesForSource] of promptBySrc) { // Idle ticks are dropped rather than emitted as zeros: this feeds a @@ -1119,10 +1191,23 @@ function buildSeriesFromMetrics( const metricSources: MetricSourceSeries[] = []; const adapter = selectServerMetricsAdapter(context); if (includeMetricSources && context.disagg && adapter.id !== 'generic') { + const endpointRoles = new Map(); + for (const metric of Object.values(metrics)) { + for (const series of metric.series ?? []) { + const endpointUrl = series.endpoint_url; + const role = series.labels?.['disaggregation_mode'] ?? series.labels?.['dynamo_component']; + if (endpointUrl && role) endpointRoles.set(endpointUrl, role); + } + } const grouped = new Map(); for (const [metricName, metric] of Object.entries(metrics)) { for (const series of metric.series ?? []) { - const source = adapter.identifySource(series); + const roleHint = series.endpoint_url ? endpointRoles.get(series.endpoint_url) : undefined; + const identifiedSeries = + roleHint && !series.labels?.['disaggregation_mode'] + ? { ...series, labels: { ...series.labels, disaggregation_mode: roleHint } } + : series; + const source = adapter.identifySource(identifiedSeries); let group = grouped.get(source.id); if (!group) { group = { source, metrics: {} }; diff --git a/packages/db/src/etl/server-metrics-adapters.ts b/packages/db/src/etl/server-metrics-adapters.ts index 5e80e8bd..45f3bacc 100644 --- a/packages/db/src/etl/server-metrics-adapters.ts +++ b/packages/db/src/etl/server-metrics-adapters.ts @@ -78,6 +78,37 @@ const dynamoAdapter: ServerMetricsAdapter = { }, }; +const trtllmAdapter: ServerMetricsAdapter = { + id: 'trtllm', + matches: ({ framework }) => framework?.toLowerCase().includes('trt') ?? false, + identifySource(series) { + const labels = series.labels ?? {}; + const nativeRole = labels['disaggregation_mode'] ?? labels['dynamo_component'] ?? null; + const role: MetricSourceRole = + nativeRole === 'prefill' + ? 'prefill' + : nativeRole === 'decode' || nativeRole === 'backend' + ? 'decode' + : nativeRole === 'aggregated' + ? 'combined' + : 'unknown'; + const endpointUrl = series.endpoint_url ?? null; + const workerId = labels['worker_id'] ?? null; + const dpRank = labels['dp_rank'] ?? null; + const engine = labels['engine'] ?? labels['engine_idx'] ?? null; + return { + id: stableId('trtllm', [role, endpointUrl, workerId, dpRank, engine]), + adapter: 'trtllm', + role, + endpointUrl, + nativeRole, + workerId, + dpRank, + engine, + }; + }, +}; + const genericAdapter: ServerMetricsAdapter = { id: 'generic', matches: () => true, @@ -100,7 +131,7 @@ const genericAdapter: ServerMetricsAdapter = { }, }; -const ADAPTERS: readonly ServerMetricsAdapter[] = [dynamoAdapter, genericAdapter]; +const ADAPTERS: readonly ServerMetricsAdapter[] = [trtllmAdapter, dynamoAdapter, genericAdapter]; export function selectServerMetricsAdapter(context: ServerMetricsContext): ServerMetricsAdapter { return ADAPTERS.find((adapter) => adapter.matches(context)) ?? genericAdapter; diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 89ec4b91..7882c074 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,6 +129,47 @@ describe('extractServerMetricSamples', () => { expect(out.kvCacheUtil).toEqual([]); expect(out.prefixCacheHitRate).toEqual([]); }); + + it('extracts TensorRT-LLM KV utilization and prefix cache hit rate', () => { + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + { + timeslices: [ + { start_ns: 0, avg: 0.2 }, + { start_ns: 1, avg: 0.6 }, + ], + }, + ], + }, + trtllm_prompt_cached_tokens_total: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 70 }, + { start_ns: 1, rate: 20 }, + ], + }, + ], + }, + trtllm_prompt_tokens_total: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 100 }, + { start_ns: 1, rate: 50 }, + ], + }, + ], + }, + }, + }); + + const out = extractServerMetricSamples(json); + expect(out.kvCacheUtil).toEqual([0.2, 0.6]); + expect(out.prefixCacheHitRate).toEqual([0.7, 0.4]); + }); }); /** The write-back payload as bound to the UPDATE (a partial aggregate_stats). */ diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 9511d248..b9d1a3e9 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -157,6 +157,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage', + 'trtllm_kv_cache_utilization', ); const kvCacheUtil = [...aggregateSeriesByStart(kvSeriesAll, 'avg', 'avg').values()]; @@ -167,6 +168,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:prefix_cache_hits', 'vllm:gpu_prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens_total', ); const queriesAll = pickFirstNonEmpty( metrics, @@ -174,6 +176,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:gpu_prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsByT = aggregateSeriesByStart(hitsAll, 'rate', 'sum'); const qByT = aggregateSeriesByStart(queriesAll, 'rate', 'sum'); @@ -182,6 +185,14 @@ export function extractServerMetricSamples(json: string): { const q = qByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push(h / q); } + if (prefixCacheHitRate.length === 0) { + const directRate = aggregateSeriesByStart( + metrics['trtllm_kv_cache_hit_rate']?.series ?? [], + 'avg', + 'avg', + ); + prefixCacheHitRate.push(...directRate.values()); + } return { kvCacheUtil, prefixCacheHitRate }; } @@ -200,6 +211,11 @@ const TARGET_METRIC_KEYS = new Set([ 'sglang:token_usage', 'sglang:cached_tokens', 'sglang:prompt_tokens', + // TensorRT-LLM + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens_total', ]); /** From 1e4878604ac641741c06d8d8ea40c2608483cca4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 17 Aug 2026 10:28:20 -0500 Subject: [PATCH 2/3] fix(agentx): handle AIPerf-normalized TRT metric names --- packages/app/src/lib/api-route-catalog.ts | 2 +- .../db/src/etl/compute-aggregate-stats.ts | 2 + .../db/src/etl/compute-chart-series.test.ts | 6 +- packages/db/src/etl/compute-chart-series.ts | 75 +++++++++++-------- .../db/src/queries/agentic-aggregates.test.ts | 4 +- packages/db/src/queries/agentic-aggregates.ts | 4 + 6 files changed, 56 insertions(+), 37 deletions(-) diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index f93535dc..66705247 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -711,7 +711,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/agentic-aggregates.ts', - sourceSha256: 'fae8d19971730132cb30cd781f677562bfc6328b1f4e35a8268a8391ad187c18', + sourceSha256: 'b8b72a37ca7a67a1f234036fcbd9e3edacbd7031e0a68fb793944fc4de7030da', reviewArea: { en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.', zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。', diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index 54a89644..85cd7c1a 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -195,7 +195,9 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([ 'vllm:gpu_prefix_cache_queries', 'trtllm_kv_cache_utilization', 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ]); diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 4b93ae0e..97766cc7 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -736,19 +736,19 @@ describe('computeChartSeries', () => { trtllm_kv_cache_host_utilization: { series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], }, - trtllm_prompt_tokens_total: { + trtllm_prompt_tokens: { series: [ buildTrtllmSeries(prefillUrl, 'prefill', 100, 'rate'), buildTrtllmSeries(decodeUrl, 'backend', 200, 'rate'), ], }, - trtllm_prompt_cached_tokens_total: { + trtllm_prompt_cached_tokens: { series: [ buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'), buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'), ], }, - trtllm_generation_tokens_total: { + trtllm_generation_tokens: { series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')], }, trtllm_num_requests_running: { diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 39d4aac6..416c9532 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -250,8 +250,11 @@ export const CHART_METRIC_KEYS = new Set([ 'trtllm_kv_cache_utilization', 'trtllm_kv_cache_host_utilization', 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', + 'trtllm_generation_tokens', 'trtllm_generation_tokens_total', 'trtllm_num_requests_running', 'trtllm_num_requests_waiting', @@ -1004,12 +1007,14 @@ function buildSeriesFromMetrics( const hitsSeries = pickSeries( 'vllm:prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); const qsSeries = pickSeries( 'vllm:prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ); const hitsOnGrid = summedSeries(hitsSeries, tOf, 'rate', tickS); @@ -1021,11 +1026,11 @@ function buildSeriesFromMetrics( if (q !== undefined && q > 0) prefixCacheHitRate.push({ t, value: h / q }); } if (prefixCacheHitRate.length === 0) { - for (const [t, value] of sortedEntries( - aggregateByStart(metrics['trtllm_kv_cache_hit_rate']?.series, 'avg', 'avg'), - )) { - prefixCacheHitRate.push({ t: tOf(t), value }); - } + prefixCacheHitRate.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_hit_rate']?.series, tOf), + ), + ); } // Queue depth: sum running + waiting across engines per timeslice. @@ -1060,11 +1065,13 @@ function buildSeriesFromMetrics( const prefillTps = counterRate( 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ); const decodeTps = counterRate( 'vllm:generation_tokens', 'sglang:generation_tokens', + 'trtllm_generation_tokens', 'trtllm_generation_tokens_total', ); // Tokens served from prefix cache per scrape. Lets the frontend derive @@ -1072,6 +1079,7 @@ function buildSeriesFromMetrics( const prefixCacheHitsTps = counterRate( 'vllm:prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); @@ -1094,11 +1102,11 @@ function buildSeriesFromMetrics( } } if (hostKvCacheUsage.length === 0) { - for (const [t, value] of sortedEntries( - aggregateByStart(metrics['trtllm_kv_cache_host_utilization']?.series, 'avg', 'avg'), - )) { - hostKvCacheUsage.push({ t: tOf(t), value }); - } + hostKvCacheUsage.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_host_utilization']?.series, tOf), + ), + ); } // Per-source prompt tokens — sum across engines per source label. @@ -1159,27 +1167,6 @@ function buildSeriesFromMetrics( addSeriesRates(label, series); } } - if (promptBySrcByT.size === 0) { - const promptByT = aggregateByStart( - metrics['trtllm_prompt_tokens_total']?.series, - 'rate', - 'sum', - ); - const cachedByT = aggregateByStart( - metrics['trtllm_prompt_cached_tokens_total']?.series, - 'rate', - 'sum', - ); - const cachedSeries: RawSeries = { timeslices: [] }; - const computedSeries: RawSeries = { timeslices: [] }; - for (const [t, prompt] of promptByT) { - const cached = Math.max(0, cachedByT.get(t) ?? 0); - cachedSeries.timeslices!.push({ start_ns: t, rate: cached }); - computedSeries.timeslices!.push({ start_ns: t, rate: Math.max(0, prompt - cached) }); - } - addSeriesRates('cache hit (HBM)', cachedSeries); - addSeriesRates('compute (miss)', computedSeries); - } const promptTokensBySource: Record = {}; for (const [source, seriesForSource] of promptBySrc) { // Idle ticks are dropped rather than emitted as zeros: this feeds a @@ -1187,6 +1174,32 @@ function buildSeriesFromMetrics( const arr = summedSeries(seriesForSource, tOf, 'rate', tickS).filter((p) => p.value > 0); if (arr.length > 0) promptTokensBySource[source] = arr; } + if (Object.keys(promptTokensBySource).length === 0) { + const promptPoints = summedSeries( + pickSeries('trtllm_prompt_tokens', 'trtllm_prompt_tokens_total'), + tOf, + 'rate', + tickS, + ); + const cachedByT = byT( + summedSeries( + pickSeries('trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total'), + tOf, + 'rate', + tickS, + ), + ); + const cached: TimeSeriesPoint[] = []; + const computed: TimeSeriesPoint[] = []; + for (const { t, value: prompt } of promptPoints) { + const cachedValue = Math.max(0, cachedByT.get(t) ?? 0); + if (cachedValue > 0) cached.push({ t, value: cachedValue }); + const computedValue = Math.max(0, prompt - cachedValue); + if (computedValue > 0) computed.push({ t, value: computedValue }); + } + if (cached.length > 0) promptTokensBySource['cache hit (HBM)'] = cached; + if (computed.length > 0) promptTokensBySource['compute (miss)'] = computed; + } const metricSources: MetricSourceSeries[] = []; const adapter = selectServerMetricsAdapter(context); diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 7882c074..945b515a 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -143,7 +143,7 @@ describe('extractServerMetricSamples', () => { }, ], }, - trtllm_prompt_cached_tokens_total: { + trtllm_prompt_cached_tokens: { series: [ { timeslices: [ @@ -153,7 +153,7 @@ describe('extractServerMetricSamples', () => { }, ], }, - trtllm_prompt_tokens_total: { + trtllm_prompt_tokens: { series: [ { timeslices: [ diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index b9d1a3e9..aecb3261 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -168,6 +168,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:prefix_cache_hits', 'vllm:gpu_prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); const queriesAll = pickFirstNonEmpty( @@ -176,6 +177,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:gpu_prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ); const hitsByT = aggregateSeriesByStart(hitsAll, 'rate', 'sum'); @@ -214,7 +216,9 @@ const TARGET_METRIC_KEYS = new Set([ // TensorRT-LLM 'trtllm_kv_cache_utilization', 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', ]); From 10cdc42f057262f31ccb438fbcbc440f7d698964 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 28 Aug 2026 10:22:16 -0500 Subject: [PATCH 3/3] fix(agentx): derive TRT prompt throughput from prefill histogram --- .../db/src/etl/compute-chart-series.test.ts | 11 +-- packages/db/src/etl/compute-chart-series.ts | 82 ++++++++++++------- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 97766cc7..9edb72f9 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -150,12 +150,13 @@ function buildTrtllmSeries( endpoint_url: string, dynamo_component: 'prefill' | 'backend', value: number, - field: 'rate' | 'avg', + field: 'rate' | 'avg' | 'sum', + durationNs = 1e9, ) { return { endpoint_url, labels: { dynamo_component, worker_id: `${dynamo_component}-worker` }, - timeslices: [{ start_ns: 0, end_ns: 1e9, [field]: value }], + timeslices: [{ start_ns: 0, end_ns: durationNs, [field]: value }], }; } @@ -736,10 +737,10 @@ describe('computeChartSeries', () => { trtllm_kv_cache_host_utilization: { series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], }, - trtllm_prompt_tokens: { + trtllm_prefill_batch_tokens: { series: [ - buildTrtllmSeries(prefillUrl, 'prefill', 100, 'rate'), - buildTrtllmSeries(decodeUrl, 'backend', 200, 'rate'), + buildTrtllmSeries(prefillUrl, 'prefill', 30, 'sum', 0.5e9), + buildTrtllmSeries(decodeUrl, 'backend', 60, 'sum', 0.5e9), ], }, trtllm_prompt_cached_tokens: { diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 416c9532..c6a88f3f 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -122,8 +122,13 @@ import { * * v16: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV * metrics, including per-prefill/decode source series for Dynamo disaggregation. + * + * v17: derive TensorRT-LLM prompt throughput from its actual metric shape: + * cached-token counter rate plus the prefill-batch-token histogram sum per + * timeslice. TRT-LLM does not currently expose the prompt-token counter that + * v16 expected. */ -export const CHART_SERIES_VERSION = 16; +export const CHART_SERIES_VERSION = 17; export interface TimeSeriesPoint { /** Seconds from benchmark start. */ @@ -206,8 +211,11 @@ interface RawSlice { end_ns?: number; avg?: number; rate?: number; + sum?: number; } +type RawSliceField = 'avg' | 'rate' | 'sumRate'; + interface RawSeries { endpoint_url?: string; labels?: Record; @@ -254,6 +262,7 @@ export const CHART_METRIC_KEYS = new Set([ 'trtllm_prompt_cached_tokens_total', 'trtllm_prompt_tokens', 'trtllm_prompt_tokens_total', + 'trtllm_prefill_batch_tokens', 'trtllm_generation_tokens', 'trtllm_generation_tokens_total', 'trtllm_num_requests_running', @@ -487,7 +496,7 @@ const MIRROR_RATE_RELATIVE_TOLERANCE = 0.05; * request counts) whose mirrors agree almost exactly. Rates need a relative * test because their magnitude is unbounded. */ -function looksMirrored(means: readonly number[], field: 'avg' | 'rate'): boolean { +function looksMirrored(means: readonly number[], field: RawSliceField): boolean { const spread = Math.max(...means) - Math.min(...means); if (field === 'avg') return spread <= MIRROR_MEAN_TOLERANCE; const scale = Math.max(...means.map((m) => Math.abs(m))); @@ -569,7 +578,7 @@ function spanOf(scrapes: ScrapeMap): number { function resolveComponents( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate' = 'avg', + field: RawSliceField = 'avg', ): ResolvedEngine[] { const groups = new Map(); for (const s of series ?? []) { @@ -587,7 +596,16 @@ function resolveComponents( } for (const ts of s.timeslices ?? []) { if (typeof ts.start_ns !== 'number' || !Number.isFinite(ts.start_ns)) continue; - const value = ts[field]; + const durationS = + typeof ts.start_ns === 'number' && typeof ts.end_ns === 'number' + ? (ts.end_ns - ts.start_ns) / 1e9 + : 0; + const value = + field === 'sumRate' + ? typeof ts.sum === 'number' && durationS > 0 + ? ts.sum / durationS + : undefined + : ts[field]; if (typeof value !== 'number' || !Number.isFinite(value)) continue; const at = scrapes.get(ts.start_ns); if (at) { @@ -929,7 +947,7 @@ function sumOntoGrid( function summedSeries( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate', + field: RawSliceField, tickS: number | null, ): TimeSeriesPoint[] { const components = resolveComponents(series, tOf, field).map((c) => c.points); @@ -1062,7 +1080,7 @@ function buildSeriesFromMetrics( // work. const counterRate = (...names: string[]): TimeSeriesPoint[] => summedSeries(pickSeries(...names), tOf, 'rate', tickS); - const prefillTps = counterRate( + const promptCounterTps = counterRate( 'vllm:prompt_tokens', 'sglang:prompt_tokens', 'trtllm_prompt_tokens', @@ -1082,6 +1100,25 @@ function buildSeriesFromMetrics( 'trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total', ); + const trtllmComputedPromptTps = summedSeries( + metrics['trtllm_prefill_batch_tokens']?.series, + tOf, + 'sumRate', + tickS, + ); + const prefillTps = + promptCounterTps.length > 0 + ? promptCounterTps + : sumOntoGrid([trtllmComputedPromptTps, prefixCacheHitsTps], tickS); + if (prefixCacheHitRate.length === 0 && (!qsSeries || qsSeries.length === 0)) { + const prefillByT = byT(prefillTps); + for (const { t, value: cached } of prefixCacheHitsTps) { + const prompt = prefillByT.get(t); + if (prompt !== undefined && prompt > 0) { + prefixCacheHitRate.push({ t, value: cached / prompt }); + } + } + } // SGLang hicache: host-pool KV cache utilization as used/total per // timeslice. Both metrics are gauges in absolute tokens. Total stays @@ -1175,28 +1212,17 @@ function buildSeriesFromMetrics( if (arr.length > 0) promptTokensBySource[source] = arr; } if (Object.keys(promptTokensBySource).length === 0) { - const promptPoints = summedSeries( - pickSeries('trtllm_prompt_tokens', 'trtllm_prompt_tokens_total'), - tOf, - 'rate', - tickS, - ); - const cachedByT = byT( - summedSeries( - pickSeries('trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total'), - tOf, - 'rate', - tickS, - ), - ); - const cached: TimeSeriesPoint[] = []; - const computed: TimeSeriesPoint[] = []; - for (const { t, value: prompt } of promptPoints) { - const cachedValue = Math.max(0, cachedByT.get(t) ?? 0); - if (cachedValue > 0) cached.push({ t, value: cachedValue }); - const computedValue = Math.max(0, prompt - cachedValue); - if (computedValue > 0) computed.push({ t, value: computedValue }); - } + const cached = prefixCacheHitsTps.filter((point) => point.value > 0); + const cachedByT = byT(prefixCacheHitsTps); + const computed = + trtllmComputedPromptTps.length > 0 + ? trtllmComputedPromptTps.filter((point) => point.value > 0) + : promptCounterTps + .map(({ t, value: prompt }) => ({ + t, + value: Math.max(0, prompt - (cachedByT.get(t) ?? 0)), + })) + .filter((point) => point.value > 0); if (cached.length > 0) promptTokensBySource['cache hit (HBM)'] = cached; if (computed.length > 0) promptTokensBySource['compute (miss)'] = computed; }