Display TensorRT-LLM AgentX server metrics

packages/app/src/lib/api-route-catalog.ts

@@ -711,7 +711,7 @@ export const apiContractSourceDigests = [
   },
   {
     source: '../db/src/queries/agentic-aggregates.ts',
-    sourceSha256: 'fae8d19971730132cb30cd781f677562bfc6328b1f4e35a8268a8391ad187c18',
+    sourceSha256: 'b8b72a37ca7a67a1f234036fcbd9e3edacbd7031e0a68fb793944fc4de7030da',
     reviewArea: {
       en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.',
       zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。',

packages/db/src/etl/compute-aggregate-stats.ts

@@ -193,6 +193,12 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([
   'vllm:prefix_cache_queries',
   'vllm:gpu_prefix_cache_hits',
   'vllm:gpu_prefix_cache_queries',
+  'trtllm_kv_cache_utilization',
+  'trtllm_kv_cache_hit_rate',
+  'trtllm_prompt_cached_tokens',
+  'trtllm_prompt_cached_tokens_total',
+  'trtllm_prompt_tokens',
+  'trtllm_prompt_tokens_total',
 ]);
 
 /**

packages/db/src/etl/compute-chart-series.test.ts

@@ -146,6 +146,20 @@ function kvBlob(profiling: unknown[], warmup: unknown[] = []) {
   );
 }
 
+function buildTrtllmSeries(
+  endpoint_url: string,
+  dynamo_component: 'prefill' | 'backend',
+  value: number,
+  field: 'rate' | 'avg' | 'sum',
+  durationNs = 1e9,
+) {
+  return {
+    endpoint_url,
+    labels: { dynamo_component, worker_id: `${dynamo_component}-worker` },
+    timeslices: [{ start_ns: 0, end_ns: durationNs, [field]: value }],
+  };
+}
+
 describe('computeChartSeries', () => {
   it('returns null when the blob is null', async () => {
     expect(await computeChartSeries(null)).toBeNull();
@@ -708,6 +722,70 @@ describe('computeChartSeries', () => {
 
     expect(result?.metricSources).toEqual([]);
   });
+
+  it('extracts native TensorRT-LLM metrics and preserves disaggregated worker roles', async () => {
+    const prefillUrl = 'http://prefill-a.internal.test:7500/metrics';
+    const decodeUrl = 'http://decode-a.internal.test:7501/metrics';
+    const json = JSON.stringify({
+      metrics: {
+        trtllm_kv_cache_utilization: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 0.3, 'avg'),
+            buildTrtllmSeries(decodeUrl, 'backend', 0.7, 'avg'),
+          ],
+        },
+        trtllm_kv_cache_host_utilization: {
+          series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')],
+        },
+        trtllm_prefill_batch_tokens: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 30, 'sum', 0.5e9),
+            buildTrtllmSeries(decodeUrl, 'backend', 60, 'sum', 0.5e9),
+          ],
+        },
+        trtllm_prompt_cached_tokens: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'),
+            buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'),
+          ],
+        },
+        trtllm_generation_tokens: {
+          series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')],
+        },
+        trtllm_num_requests_running: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 2, 'avg'),
+            buildTrtllmSeries(decodeUrl, 'backend', 3, 'avg'),
+          ],
+        },
+        trtllm_num_requests_waiting: {
+          series: [buildTrtllmSeries(decodeUrl, 'backend', 4, 'avg')],
+        },
+      },
+    });
+
+    const result = await computeChartSeries(gzipSync(Buffer.from(json)), {
+      framework: 'trtllm',
+      disagg: true,
+    });
+
+    expect(result?.kvCacheUsage).toEqual([{ t: 0, value: 0.5 }]);
+    expect(result?.hostKvCacheUsage).toEqual([{ t: 0, value: 0.25 }]);
+    expect(result?.prefixCacheHitRate).toEqual([{ t: 0, value: 0.4 }]);
+    expect(result?.queueDepth).toEqual([{ t: 0, running: 5, waiting: 4, total: 9 }]);
+    expect(result?.prefillTps).toEqual([{ t: 0, value: 300 }]);
+    expect(result?.decodeTps).toEqual([{ t: 0, value: 50 }]);
+    expect(result?.promptTokensBySource).toEqual({
+      'cache hit (HBM)': [{ t: 0, value: 120 }],
+      'compute (miss)': [{ t: 0, value: 180 }],
+    });
+    expect(result?.metricSources.map(({ source }) => [source.role, source.endpointUrl])).toEqual([
+      ['prefill', prefillUrl],
+      ['decode', decodeUrl],
+    ]);
+    expect(result?.metricSources[0]?.promptTps).toEqual([{ t: 0, value: 100 }]);
+    expect(result?.metricSources[1]?.generationTps).toEqual([{ t: 0, value: 50 }]);
+  });
 });
 
 // ── Summed series on the canonical grid (v14) ───────────────────────────

packages/db/src/etl/compute-chart-series.ts

@@ -120,8 +120,15 @@ import {
  * per-rank detail stays reachable as that source's own
  * `kvCacheUsageByEngine`.
  *
+ * v16: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV
+ * metrics, including per-prefill/decode source series for Dynamo disaggregation.
+ *
+ * v17: derive TensorRT-LLM prompt throughput from its actual metric shape:
+ * cached-token counter rate plus the prefill-batch-token histogram sum per
+ * timeslice. TRT-LLM does not currently expose the prompt-token counter that
+ * v16 expected.
  */
-export const CHART_SERIES_VERSION = 15;
+export const CHART_SERIES_VERSION = 17;
 
 export interface TimeSeriesPoint {
   /** Seconds from benchmark start. */
@@ -204,8 +211,11 @@ interface RawSlice {
   end_ns?: number;
   avg?: number;
   rate?: number;
+  sum?: number;
 }
 
+type RawSliceField = 'avg' | 'rate' | 'sumRate';
+
 interface RawSeries {
   endpoint_url?: string;
   labels?: Record<string, string>;
@@ -244,6 +254,19 @@ export const CHART_METRIC_KEYS = new Set([
   'sglang:realtime_tokens',
   'sglang:hicache_host_used_tokens',
   'sglang:hicache_host_total_tokens',
+  // TensorRT-LLM
+  'trtllm_kv_cache_utilization',
+  'trtllm_kv_cache_host_utilization',
+  'trtllm_kv_cache_hit_rate',
+  'trtllm_prompt_cached_tokens',
+  'trtllm_prompt_cached_tokens_total',
+  'trtllm_prompt_tokens',
+  'trtllm_prompt_tokens_total',
+  'trtllm_prefill_batch_tokens',
+  'trtllm_generation_tokens',
+  'trtllm_generation_tokens_total',
+  'trtllm_num_requests_running',
+  'trtllm_num_requests_waiting',
 ]);
 
 /**
@@ -473,7 +496,7 @@ const MIRROR_RATE_RELATIVE_TOLERANCE = 0.05;
  * request counts) whose mirrors agree almost exactly. Rates need a relative
  * test because their magnitude is unbounded.
  */
-function looksMirrored(means: readonly number[], field: 'avg' | 'rate'): boolean {
+function looksMirrored(means: readonly number[], field: RawSliceField): boolean {
   const spread = Math.max(...means) - Math.min(...means);
   if (field === 'avg') return spread <= MIRROR_MEAN_TOLERANCE;
   const scale = Math.max(...means.map((m) => Math.abs(m)));
@@ -555,7 +578,7 @@ function spanOf(scrapes: ScrapeMap): number {
 function resolveComponents(
   series: readonly RawSeries[] | undefined,
   tOf: (ns: number) => number,
-  field: 'avg' | 'rate' = 'avg',
+  field: RawSliceField = 'avg',
 ): ResolvedEngine[] {
   const groups = new Map<string, EngineGroup>();
   for (const s of series ?? []) {
@@ -573,7 +596,16 @@ function resolveComponents(
     }
     for (const ts of s.timeslices ?? []) {
       if (typeof ts.start_ns !== 'number' || !Number.isFinite(ts.start_ns)) continue;
-      const value = ts[field];
+      const durationS =
+        typeof ts.start_ns === 'number' && typeof ts.end_ns === 'number'
+          ? (ts.end_ns - ts.start_ns) / 1e9
+          : 0;
+      const value =
+        field === 'sumRate'
+          ? typeof ts.sum === 'number' && durationS > 0
+            ? ts.sum / durationS
+            : undefined
+          : ts[field];
       if (typeof value !== 'number' || !Number.isFinite(value)) continue;
       const at = scrapes.get(ts.start_ns);
       if (at) {
@@ -915,7 +947,7 @@ function sumOntoGrid(
 function summedSeries(
   series: readonly RawSeries[] | undefined,
   tOf: (ns: number) => number,
-  field: 'avg' | 'rate',
+  field: RawSliceField,
   tickS: number | null,
 ): TimeSeriesPoint[] {
   const components = resolveComponents(series, tOf, field).map((c) => c.points);
@@ -976,6 +1008,7 @@ function buildSeriesFromMetrics(
     'vllm:kv_cache_usage_perc',
     'vllm:gpu_cache_usage_perc',
     'sglang:token_usage',
+    'trtllm_kv_cache_utilization',
   );
   // One entry per logical engine (v13) — mirrored API-server frontends and the
   // warmup/profiling phase split are collapsed here rather than showing up as
@@ -989,11 +1022,18 @@ function buildSeriesFromMetrics(
 
   // Prefix cache hit rate per scrape: Σhits.rate / Σqueries.rate across
   // engines, joined on start_ns. SGLang names: cached_tokens / prompt_tokens.
-  const hitsSeries = pickSeries('vllm:prefix_cache_hits', 'sglang:cached_tokens');
+  const hitsSeries = pickSeries(
+    'vllm:prefix_cache_hits',
+    'sglang:cached_tokens',
+    'trtllm_prompt_cached_tokens',
+    'trtllm_prompt_cached_tokens_total',
+  );
   const qsSeries = pickSeries(
     'vllm:prefix_cache_queries',
     'vllm:prompt_tokens',
     'sglang:prompt_tokens',
+    'trtllm_prompt_tokens',
+    'trtllm_prompt_tokens_total',
   );
   const hitsOnGrid = summedSeries(hitsSeries, tOf, 'rate', tickS);
   const qsOnGrid = summedSeries(qsSeries, tOf, 'rate', tickS);
@@ -1003,10 +1043,25 @@ function buildSeriesFromMetrics(
     const q = qsByT.get(t);
     if (q !== undefined && q > 0) prefixCacheHitRate.push({ t, value: h / q });
   }
+  if (prefixCacheHitRate.length === 0) {
+    prefixCacheHitRate.push(
+      ...averageAcrossEngines(
+        resolveLogicalEngines(metrics['trtllm_kv_cache_hit_rate']?.series, tOf),
+      ),
+    );
+  }
 
   // Queue depth: sum running + waiting across engines per timeslice.
-  const runSeries = pickSeries('vllm:num_requests_running', 'sglang:num_running_reqs');
-  const waitSeries = pickSeries('vllm:num_requests_waiting', 'sglang:num_queue_reqs');
+  const runSeries = pickSeries(
+    'vllm:num_requests_running',
+    'sglang:num_running_reqs',
+    'trtllm_num_requests_running',
+  );
+  const waitSeries = pickSeries(
+    'vllm:num_requests_waiting',
+    'sglang:num_queue_reqs',
+    'trtllm_num_requests_waiting',
+  );
   const runOnGrid = summedSeries(runSeries, tOf, 'avg', tickS);
   const waitByT = byT(summedSeries(waitSeries, tOf, 'avg', tickS));
   const runByT = byT(runOnGrid);
@@ -1025,11 +1080,45 @@ function buildSeriesFromMetrics(
   // work.
   const counterRate = (...names: string[]): TimeSeriesPoint[] =>
     summedSeries(pickSeries(...names), tOf, 'rate', tickS);
-  const prefillTps = counterRate('vllm:prompt_tokens', 'sglang:prompt_tokens');
-  const decodeTps = counterRate('vllm:generation_tokens', 'sglang:generation_tokens');
+  const promptCounterTps = counterRate(
+    'vllm:prompt_tokens',
+    'sglang:prompt_tokens',
+    'trtllm_prompt_tokens',
+    'trtllm_prompt_tokens_total',
+  );
+  const decodeTps = counterRate(
+    'vllm:generation_tokens',
+    'sglang:generation_tokens',
+    'trtllm_generation_tokens',
+    'trtllm_generation_tokens_total',
+  );
   // Tokens served from prefix cache per scrape. Lets the frontend derive
   // "cumulative unique input tokens served" = cumsum(prefillTps) − cumsum(hits).
-  const prefixCacheHitsTps = counterRate('vllm:prefix_cache_hits', 'sglang:cached_tokens');
+  const prefixCacheHitsTps = counterRate(
+    'vllm:prefix_cache_hits',
+    'sglang:cached_tokens',
+    'trtllm_prompt_cached_tokens',
+    'trtllm_prompt_cached_tokens_total',
+  );
+  const trtllmComputedPromptTps = summedSeries(
+    metrics['trtllm_prefill_batch_tokens']?.series,
+    tOf,
+    'sumRate',
+    tickS,
+  );
+  const prefillTps =
+    promptCounterTps.length > 0
+      ? promptCounterTps
+      : sumOntoGrid([trtllmComputedPromptTps, prefixCacheHitsTps], tickS);
+  if (prefixCacheHitRate.length === 0 && (!qsSeries || qsSeries.length === 0)) {
+    const prefillByT = byT(prefillTps);
+    for (const { t, value: cached } of prefixCacheHitsTps) {
+      const prompt = prefillByT.get(t);
+      if (prompt !== undefined && prompt > 0) {
+        prefixCacheHitRate.push({ t, value: cached / prompt });
+      }
+    }
+  }
 
   // SGLang hicache: host-pool KV cache utilization as used/total per
   // timeslice. Both metrics are gauges in absolute tokens. Total stays
@@ -1049,6 +1138,13 @@ function buildSeriesFromMetrics(
       hostKvCacheUsage.push({ t, value: used / total });
     }
   }
+  if (hostKvCacheUsage.length === 0) {
+    hostKvCacheUsage.push(
+      ...averageAcrossEngines(
+        resolveLogicalEngines(metrics['trtllm_kv_cache_host_utilization']?.series, tOf),
+      ),
+    );
+  }
 
   // Per-source prompt tokens — sum across engines per source label.
   //   vllm: vllm:prompt_tokens_by_source has one series per source label
@@ -1115,14 +1211,42 @@ function buildSeriesFromMetrics(
     const arr = summedSeries(seriesForSource, tOf, 'rate', tickS).filter((p) => p.value > 0);
     if (arr.length > 0) promptTokensBySource[source] = arr;
   }
+  if (Object.keys(promptTokensBySource).length === 0) {
+    const cached = prefixCacheHitsTps.filter((point) => point.value > 0);
+    const cachedByT = byT(prefixCacheHitsTps);
+    const computed =
+      trtllmComputedPromptTps.length > 0
+        ? trtllmComputedPromptTps.filter((point) => point.value > 0)
+        : promptCounterTps
+            .map(({ t, value: prompt }) => ({
+              t,
+              value: Math.max(0, prompt - (cachedByT.get(t) ?? 0)),
+            }))
+            .filter((point) => point.value > 0);
+    if (cached.length > 0) promptTokensBySource['cache hit (HBM)'] = cached;
+    if (computed.length > 0) promptTokensBySource['compute (miss)'] = computed;
+  }
 
   const metricSources: MetricSourceSeries[] = [];
   const adapter = selectServerMetricsAdapter(context);
   if (includeMetricSources && context.disagg && adapter.id !== 'generic') {
+    const endpointRoles = new Map<string, string>();
+    for (const metric of Object.values(metrics)) {
+      for (const series of metric.series ?? []) {
+        const endpointUrl = series.endpoint_url;
+        const role = series.labels?.['disaggregation_mode'] ?? series.labels?.['dynamo_component'];
+        if (endpointUrl && role) endpointRoles.set(endpointUrl, role);
+      }
+    }
     const grouped = new Map<string, { source: MetricSource; metrics: MetricsMap }>();
     for (const [metricName, metric] of Object.entries(metrics)) {
       for (const series of metric.series ?? []) {
-        const source = adapter.identifySource(series);
+        const roleHint = series.endpoint_url ? endpointRoles.get(series.endpoint_url) : undefined;
+        const identifiedSeries =
+          roleHint && !series.labels?.['disaggregation_mode']
+            ? { ...series, labels: { ...series.labels, disaggregation_mode: roleHint } }
+            : series;
+        const source = adapter.identifySource(identifiedSeries);
         let group = grouped.get(source.id);
         if (!group) {
           group = { source, metrics: {} };