diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index f93535dc..66705247 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -711,7 +711,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/agentic-aggregates.ts', - sourceSha256: 'fae8d19971730132cb30cd781f677562bfc6328b1f4e35a8268a8391ad187c18', + sourceSha256: 'b8b72a37ca7a67a1f234036fcbd9e3edacbd7031e0a68fb793944fc4de7030da', reviewArea: { en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.', zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。', diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index 07729b59..85cd7c1a 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -193,6 +193,12 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([ 'vllm:prefix_cache_queries', 'vllm:gpu_prefix_cache_hits', 'vllm:gpu_prefix_cache_queries', + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ]); /** diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 54295cc7..9edb72f9 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -146,6 +146,20 @@ function kvBlob(profiling: unknown[], warmup: unknown[] = []) { ); } +function buildTrtllmSeries( + endpoint_url: string, + dynamo_component: 'prefill' | 'backend', + value: number, + field: 'rate' | 'avg' | 'sum', + durationNs = 1e9, +) { + return { + endpoint_url, + labels: { dynamo_component, worker_id: `${dynamo_component}-worker` }, + timeslices: [{ start_ns: 0, end_ns: durationNs, [field]: value }], + }; +} + describe('computeChartSeries', () => { it('returns null when the blob is null', async () => { expect(await computeChartSeries(null)).toBeNull(); @@ -708,6 +722,70 @@ describe('computeChartSeries', () => { expect(result?.metricSources).toEqual([]); }); + + it('extracts native TensorRT-LLM metrics and preserves disaggregated worker roles', async () => { + const prefillUrl = 'http://prefill-a.internal.test:7500/metrics'; + const decodeUrl = 'http://decode-a.internal.test:7501/metrics'; + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 0.3, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 0.7, 'avg'), + ], + }, + trtllm_kv_cache_host_utilization: { + series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')], + }, + trtllm_prefill_batch_tokens: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 30, 'sum', 0.5e9), + buildTrtllmSeries(decodeUrl, 'backend', 60, 'sum', 0.5e9), + ], + }, + trtllm_prompt_cached_tokens: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'), + buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'), + ], + }, + trtllm_generation_tokens: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')], + }, + trtllm_num_requests_running: { + series: [ + buildTrtllmSeries(prefillUrl, 'prefill', 2, 'avg'), + buildTrtllmSeries(decodeUrl, 'backend', 3, 'avg'), + ], + }, + trtllm_num_requests_waiting: { + series: [buildTrtllmSeries(decodeUrl, 'backend', 4, 'avg')], + }, + }, + }); + + const result = await computeChartSeries(gzipSync(Buffer.from(json)), { + framework: 'trtllm', + disagg: true, + }); + + expect(result?.kvCacheUsage).toEqual([{ t: 0, value: 0.5 }]); + expect(result?.hostKvCacheUsage).toEqual([{ t: 0, value: 0.25 }]); + expect(result?.prefixCacheHitRate).toEqual([{ t: 0, value: 0.4 }]); + expect(result?.queueDepth).toEqual([{ t: 0, running: 5, waiting: 4, total: 9 }]); + expect(result?.prefillTps).toEqual([{ t: 0, value: 300 }]); + expect(result?.decodeTps).toEqual([{ t: 0, value: 50 }]); + expect(result?.promptTokensBySource).toEqual({ + 'cache hit (HBM)': [{ t: 0, value: 120 }], + 'compute (miss)': [{ t: 0, value: 180 }], + }); + expect(result?.metricSources.map(({ source }) => [source.role, source.endpointUrl])).toEqual([ + ['prefill', prefillUrl], + ['decode', decodeUrl], + ]); + expect(result?.metricSources[0]?.promptTps).toEqual([{ t: 0, value: 100 }]); + expect(result?.metricSources[1]?.generationTps).toEqual([{ t: 0, value: 50 }]); + }); }); // ── Summed series on the canonical grid (v14) ─────────────────────────── diff --git a/packages/db/src/etl/compute-chart-series.ts b/packages/db/src/etl/compute-chart-series.ts index 82878ad8..c6a88f3f 100644 --- a/packages/db/src/etl/compute-chart-series.ts +++ b/packages/db/src/etl/compute-chart-series.ts @@ -120,8 +120,15 @@ import { * per-rank detail stays reachable as that source's own * `kvCacheUsageByEngine`. * + * v16: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV + * metrics, including per-prefill/decode source series for Dynamo disaggregation. + * + * v17: derive TensorRT-LLM prompt throughput from its actual metric shape: + * cached-token counter rate plus the prefill-batch-token histogram sum per + * timeslice. TRT-LLM does not currently expose the prompt-token counter that + * v16 expected. */ -export const CHART_SERIES_VERSION = 15; +export const CHART_SERIES_VERSION = 17; export interface TimeSeriesPoint { /** Seconds from benchmark start. */ @@ -204,8 +211,11 @@ interface RawSlice { end_ns?: number; avg?: number; rate?: number; + sum?: number; } +type RawSliceField = 'avg' | 'rate' | 'sumRate'; + interface RawSeries { endpoint_url?: string; labels?: Record; @@ -244,6 +254,19 @@ export const CHART_METRIC_KEYS = new Set([ 'sglang:realtime_tokens', 'sglang:hicache_host_used_tokens', 'sglang:hicache_host_total_tokens', + // TensorRT-LLM + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_host_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', + 'trtllm_prefill_batch_tokens', + 'trtllm_generation_tokens', + 'trtllm_generation_tokens_total', + 'trtllm_num_requests_running', + 'trtllm_num_requests_waiting', ]); /** @@ -473,7 +496,7 @@ const MIRROR_RATE_RELATIVE_TOLERANCE = 0.05; * request counts) whose mirrors agree almost exactly. Rates need a relative * test because their magnitude is unbounded. */ -function looksMirrored(means: readonly number[], field: 'avg' | 'rate'): boolean { +function looksMirrored(means: readonly number[], field: RawSliceField): boolean { const spread = Math.max(...means) - Math.min(...means); if (field === 'avg') return spread <= MIRROR_MEAN_TOLERANCE; const scale = Math.max(...means.map((m) => Math.abs(m))); @@ -555,7 +578,7 @@ function spanOf(scrapes: ScrapeMap): number { function resolveComponents( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate' = 'avg', + field: RawSliceField = 'avg', ): ResolvedEngine[] { const groups = new Map(); for (const s of series ?? []) { @@ -573,7 +596,16 @@ function resolveComponents( } for (const ts of s.timeslices ?? []) { if (typeof ts.start_ns !== 'number' || !Number.isFinite(ts.start_ns)) continue; - const value = ts[field]; + const durationS = + typeof ts.start_ns === 'number' && typeof ts.end_ns === 'number' + ? (ts.end_ns - ts.start_ns) / 1e9 + : 0; + const value = + field === 'sumRate' + ? typeof ts.sum === 'number' && durationS > 0 + ? ts.sum / durationS + : undefined + : ts[field]; if (typeof value !== 'number' || !Number.isFinite(value)) continue; const at = scrapes.get(ts.start_ns); if (at) { @@ -915,7 +947,7 @@ function sumOntoGrid( function summedSeries( series: readonly RawSeries[] | undefined, tOf: (ns: number) => number, - field: 'avg' | 'rate', + field: RawSliceField, tickS: number | null, ): TimeSeriesPoint[] { const components = resolveComponents(series, tOf, field).map((c) => c.points); @@ -976,6 +1008,7 @@ function buildSeriesFromMetrics( 'vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage', + 'trtllm_kv_cache_utilization', ); // One entry per logical engine (v13) — mirrored API-server frontends and the // warmup/profiling phase split are collapsed here rather than showing up as @@ -989,11 +1022,18 @@ function buildSeriesFromMetrics( // Prefix cache hit rate per scrape: Σhits.rate / Σqueries.rate across // engines, joined on start_ns. SGLang names: cached_tokens / prompt_tokens. - const hitsSeries = pickSeries('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const hitsSeries = pickSeries( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); const qsSeries = pickSeries( 'vllm:prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsOnGrid = summedSeries(hitsSeries, tOf, 'rate', tickS); const qsOnGrid = summedSeries(qsSeries, tOf, 'rate', tickS); @@ -1003,10 +1043,25 @@ function buildSeriesFromMetrics( const q = qsByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push({ t, value: h / q }); } + if (prefixCacheHitRate.length === 0) { + prefixCacheHitRate.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_hit_rate']?.series, tOf), + ), + ); + } // Queue depth: sum running + waiting across engines per timeslice. - const runSeries = pickSeries('vllm:num_requests_running', 'sglang:num_running_reqs'); - const waitSeries = pickSeries('vllm:num_requests_waiting', 'sglang:num_queue_reqs'); + const runSeries = pickSeries( + 'vllm:num_requests_running', + 'sglang:num_running_reqs', + 'trtllm_num_requests_running', + ); + const waitSeries = pickSeries( + 'vllm:num_requests_waiting', + 'sglang:num_queue_reqs', + 'trtllm_num_requests_waiting', + ); const runOnGrid = summedSeries(runSeries, tOf, 'avg', tickS); const waitByT = byT(summedSeries(waitSeries, tOf, 'avg', tickS)); const runByT = byT(runOnGrid); @@ -1025,11 +1080,45 @@ function buildSeriesFromMetrics( // work. const counterRate = (...names: string[]): TimeSeriesPoint[] => summedSeries(pickSeries(...names), tOf, 'rate', tickS); - const prefillTps = counterRate('vllm:prompt_tokens', 'sglang:prompt_tokens'); - const decodeTps = counterRate('vllm:generation_tokens', 'sglang:generation_tokens'); + const promptCounterTps = counterRate( + 'vllm:prompt_tokens', + 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', + ); + const decodeTps = counterRate( + 'vllm:generation_tokens', + 'sglang:generation_tokens', + 'trtllm_generation_tokens', + 'trtllm_generation_tokens_total', + ); // Tokens served from prefix cache per scrape. Lets the frontend derive // "cumulative unique input tokens served" = cumsum(prefillTps) − cumsum(hits). - const prefixCacheHitsTps = counterRate('vllm:prefix_cache_hits', 'sglang:cached_tokens'); + const prefixCacheHitsTps = counterRate( + 'vllm:prefix_cache_hits', + 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + ); + const trtllmComputedPromptTps = summedSeries( + metrics['trtllm_prefill_batch_tokens']?.series, + tOf, + 'sumRate', + tickS, + ); + const prefillTps = + promptCounterTps.length > 0 + ? promptCounterTps + : sumOntoGrid([trtllmComputedPromptTps, prefixCacheHitsTps], tickS); + if (prefixCacheHitRate.length === 0 && (!qsSeries || qsSeries.length === 0)) { + const prefillByT = byT(prefillTps); + for (const { t, value: cached } of prefixCacheHitsTps) { + const prompt = prefillByT.get(t); + if (prompt !== undefined && prompt > 0) { + prefixCacheHitRate.push({ t, value: cached / prompt }); + } + } + } // SGLang hicache: host-pool KV cache utilization as used/total per // timeslice. Both metrics are gauges in absolute tokens. Total stays @@ -1049,6 +1138,13 @@ function buildSeriesFromMetrics( hostKvCacheUsage.push({ t, value: used / total }); } } + if (hostKvCacheUsage.length === 0) { + hostKvCacheUsage.push( + ...averageAcrossEngines( + resolveLogicalEngines(metrics['trtllm_kv_cache_host_utilization']?.series, tOf), + ), + ); + } // Per-source prompt tokens — sum across engines per source label. // vllm: vllm:prompt_tokens_by_source has one series per source label @@ -1115,14 +1211,42 @@ function buildSeriesFromMetrics( const arr = summedSeries(seriesForSource, tOf, 'rate', tickS).filter((p) => p.value > 0); if (arr.length > 0) promptTokensBySource[source] = arr; } + if (Object.keys(promptTokensBySource).length === 0) { + const cached = prefixCacheHitsTps.filter((point) => point.value > 0); + const cachedByT = byT(prefixCacheHitsTps); + const computed = + trtllmComputedPromptTps.length > 0 + ? trtllmComputedPromptTps.filter((point) => point.value > 0) + : promptCounterTps + .map(({ t, value: prompt }) => ({ + t, + value: Math.max(0, prompt - (cachedByT.get(t) ?? 0)), + })) + .filter((point) => point.value > 0); + if (cached.length > 0) promptTokensBySource['cache hit (HBM)'] = cached; + if (computed.length > 0) promptTokensBySource['compute (miss)'] = computed; + } const metricSources: MetricSourceSeries[] = []; const adapter = selectServerMetricsAdapter(context); if (includeMetricSources && context.disagg && adapter.id !== 'generic') { + const endpointRoles = new Map(); + for (const metric of Object.values(metrics)) { + for (const series of metric.series ?? []) { + const endpointUrl = series.endpoint_url; + const role = series.labels?.['disaggregation_mode'] ?? series.labels?.['dynamo_component']; + if (endpointUrl && role) endpointRoles.set(endpointUrl, role); + } + } const grouped = new Map(); for (const [metricName, metric] of Object.entries(metrics)) { for (const series of metric.series ?? []) { - const source = adapter.identifySource(series); + const roleHint = series.endpoint_url ? endpointRoles.get(series.endpoint_url) : undefined; + const identifiedSeries = + roleHint && !series.labels?.['disaggregation_mode'] + ? { ...series, labels: { ...series.labels, disaggregation_mode: roleHint } } + : series; + const source = adapter.identifySource(identifiedSeries); let group = grouped.get(source.id); if (!group) { group = { source, metrics: {} }; diff --git a/packages/db/src/etl/server-metrics-adapters.ts b/packages/db/src/etl/server-metrics-adapters.ts index 5e80e8bd..45f3bacc 100644 --- a/packages/db/src/etl/server-metrics-adapters.ts +++ b/packages/db/src/etl/server-metrics-adapters.ts @@ -78,6 +78,37 @@ const dynamoAdapter: ServerMetricsAdapter = { }, }; +const trtllmAdapter: ServerMetricsAdapter = { + id: 'trtllm', + matches: ({ framework }) => framework?.toLowerCase().includes('trt') ?? false, + identifySource(series) { + const labels = series.labels ?? {}; + const nativeRole = labels['disaggregation_mode'] ?? labels['dynamo_component'] ?? null; + const role: MetricSourceRole = + nativeRole === 'prefill' + ? 'prefill' + : nativeRole === 'decode' || nativeRole === 'backend' + ? 'decode' + : nativeRole === 'aggregated' + ? 'combined' + : 'unknown'; + const endpointUrl = series.endpoint_url ?? null; + const workerId = labels['worker_id'] ?? null; + const dpRank = labels['dp_rank'] ?? null; + const engine = labels['engine'] ?? labels['engine_idx'] ?? null; + return { + id: stableId('trtllm', [role, endpointUrl, workerId, dpRank, engine]), + adapter: 'trtllm', + role, + endpointUrl, + nativeRole, + workerId, + dpRank, + engine, + }; + }, +}; + const genericAdapter: ServerMetricsAdapter = { id: 'generic', matches: () => true, @@ -100,7 +131,7 @@ const genericAdapter: ServerMetricsAdapter = { }, }; -const ADAPTERS: readonly ServerMetricsAdapter[] = [dynamoAdapter, genericAdapter]; +const ADAPTERS: readonly ServerMetricsAdapter[] = [trtllmAdapter, dynamoAdapter, genericAdapter]; export function selectServerMetricsAdapter(context: ServerMetricsContext): ServerMetricsAdapter { return ADAPTERS.find((adapter) => adapter.matches(context)) ?? genericAdapter; diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 89ec4b91..945b515a 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,6 +129,47 @@ describe('extractServerMetricSamples', () => { expect(out.kvCacheUtil).toEqual([]); expect(out.prefixCacheHitRate).toEqual([]); }); + + it('extracts TensorRT-LLM KV utilization and prefix cache hit rate', () => { + const json = JSON.stringify({ + metrics: { + trtllm_kv_cache_utilization: { + series: [ + { + timeslices: [ + { start_ns: 0, avg: 0.2 }, + { start_ns: 1, avg: 0.6 }, + ], + }, + ], + }, + trtllm_prompt_cached_tokens: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 70 }, + { start_ns: 1, rate: 20 }, + ], + }, + ], + }, + trtllm_prompt_tokens: { + series: [ + { + timeslices: [ + { start_ns: 0, rate: 100 }, + { start_ns: 1, rate: 50 }, + ], + }, + ], + }, + }, + }); + + const out = extractServerMetricSamples(json); + expect(out.kvCacheUtil).toEqual([0.2, 0.6]); + expect(out.prefixCacheHitRate).toEqual([0.7, 0.4]); + }); }); /** The write-back payload as bound to the UPDATE (a partial aggregate_stats). */ diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 9511d248..aecb3261 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -157,6 +157,7 @@ export function extractServerMetricSamples(json: string): { 'vllm:kv_cache_usage_perc', 'vllm:gpu_cache_usage_perc', 'sglang:token_usage', + 'trtllm_kv_cache_utilization', ); const kvCacheUtil = [...aggregateSeriesByStart(kvSeriesAll, 'avg', 'avg').values()]; @@ -167,6 +168,8 @@ export function extractServerMetricSamples(json: string): { 'vllm:prefix_cache_hits', 'vllm:gpu_prefix_cache_hits', 'sglang:cached_tokens', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', ); const queriesAll = pickFirstNonEmpty( metrics, @@ -174,6 +177,8 @@ export function extractServerMetricSamples(json: string): { 'vllm:gpu_prefix_cache_queries', 'vllm:prompt_tokens', 'sglang:prompt_tokens', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ); const hitsByT = aggregateSeriesByStart(hitsAll, 'rate', 'sum'); const qByT = aggregateSeriesByStart(queriesAll, 'rate', 'sum'); @@ -182,6 +187,14 @@ export function extractServerMetricSamples(json: string): { const q = qByT.get(t); if (q !== undefined && q > 0) prefixCacheHitRate.push(h / q); } + if (prefixCacheHitRate.length === 0) { + const directRate = aggregateSeriesByStart( + metrics['trtllm_kv_cache_hit_rate']?.series ?? [], + 'avg', + 'avg', + ); + prefixCacheHitRate.push(...directRate.values()); + } return { kvCacheUtil, prefixCacheHitRate }; } @@ -200,6 +213,13 @@ const TARGET_METRIC_KEYS = new Set([ 'sglang:token_usage', 'sglang:cached_tokens', 'sglang:prompt_tokens', + // TensorRT-LLM + 'trtllm_kv_cache_utilization', + 'trtllm_kv_cache_hit_rate', + 'trtllm_prompt_cached_tokens', + 'trtllm_prompt_cached_tokens_total', + 'trtllm_prompt_tokens', + 'trtllm_prompt_tokens_total', ]); /**