Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions packages/app/cypress/e2e/agentic-measured-power.cy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
// AgentX measured power (PLAN-10 / gap G15): the role-local energy axes
// (Measured Prefill/Decode J per token) render for agentic runs, and pinned
// tooltips carry the per-worker power drilldown when the row ships workers[].
//
// The shared cypress/fixtures/api/*.json files contain ZERO agentic rows (by
// design), so this spec injects agentic availability + benchmark rows via
// spec-scoped intercepts, following ttft-x-axis-toggle.cy.ts. Production
// AgentX rows currently ship workers: null (producer emission lands with
// PLAN-09), so the synthetic workers here verify the UI ahead of real data —
// and the workers-less series proves the graceful-absence path.
import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e';
import {
agenticMetrics,
measuredPowerMetrics,
syntheticWorkers,
} from '../support/agentic-fixtures';

const MODEL_DB_KEY = 'dsv4'; // DeepSeek-V4-Pro
const AGENTIC_DATE = '2026-06-12';

// One disaggregated series carrying role energy + workers, one aggregate
// series with whole-run measured metrics only and no workers.
const POWER_GPUS = [
{ hardware: 'b200', framework: 'vllm', disagg: true },
{ hardware: 'b300', framework: 'vllm', disagg: false },
];

const agenticAvailability = POWER_GPUS.flatMap((g) => [
{
model: MODEL_DB_KEY,
isl: null,
osl: null,
precision: 'fp4',
hardware: g.hardware,
framework: g.framework,
spec_method: 'none',
disagg: g.disagg,
benchmark_type: 'agentic_traces',
date: AGENTIC_DATE,
},
{
model: MODEL_DB_KEY,
isl: 8192,
osl: 1024,
precision: 'fp4',
hardware: g.hardware,
framework: g.framework,
spec_method: 'none',
disagg: g.disagg,
benchmark_type: 'single_turn',
date: AGENTIC_DATE,
},
]);

let benchIdCursor = 930000;
const agenticBenchmarks = POWER_GPUS.flatMap((g) =>
[16, 64, 128].map((conc) => ({
id: benchIdCursor++,
hardware: g.hardware,
framework: g.framework,
model: MODEL_DB_KEY,
precision: 'fp4',
spec_method: 'none',
disagg: g.disagg,
is_multinode: g.disagg,
prefill_tp: 8,
prefill_ep: 1,
prefill_dp_attention: false,
prefill_num_workers: g.disagg ? 1 : 0,
decode_tp: 8,
decode_ep: 1,
decode_dp_attention: false,
decode_num_workers: g.disagg ? 1 : 0,
num_prefill_gpu: 8,
num_decode_gpu: 8,
isl: null,
osl: null,
conc,
offload_mode: 'off',
benchmark_type: 'agentic_traces',
image: 'vllm/vllm-openai:v0.9.0',
metrics: { ...agenticMetrics(conc), ...measuredPowerMetrics(conc, { disagg: g.disagg }) },
workers: g.disagg ? syntheticWorkers(true) : null,
date: AGENTIC_DATE,
run_url: null,
})),
);

const DISAGG_DOTS = '.dot-group[data-hw-key^="b200"]';
const AGGREGATE_DOTS = '.dot-group[data-hw-key^="b300"]';

describe('AgentX measured power (role energy axes + worker drilldown)', () => {
beforeEach(() => {
cy.intercept('GET', '/api/v1/availability', { body: agenticAvailability }).as('availability');
cy.intercept('GET', '/api/v1/benchmarks*', { body: agenticBenchmarks }).as('benchmarks');
interceptDerivedAgenticMetrics();
// No stored traces or logs for the synthetic ids: keeps the pinned
// tooltip free of the "View charts/logs" actions this spec doesn't test.
cy.intercept('GET', '/api/v1/trace-availability*', (request) => {
const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? [];
request.reply({ body: Object.fromEntries(ids.map((id) => [id, false])) });
});
cy.intercept('GET', '/api/v1/log-availability*', (request) => {
const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? [];
request.reply({ body: Object.fromEntries(ids.map((id) => [id, false])) });
});
cy.visit('/inference?i_seq=agentic-traces', {
onBeforeLoad(win) {
win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now()));
unlockAgenticGate(win);
},
});
cy.get('[data-testid="scatter-graph"] .dot-group').should('have.length.greaterThan', 0);
});

it('offers the role-energy axes and coverage-filters series without role energy', () => {
cy.get('[data-testid="yaxis-metric-selector"]').click();
cy.contains('[data-slot="select-content"]', 'Measured Energy')
.scrollIntoView()
.should('be.visible');
cy.contains('[role="option"]', 'Measured Prefill Joules per Input Token')
.scrollIntoView()
.should('be.visible');
cy.contains('[role="option"]', 'Measured Decode Joules per Output Token')
.scrollIntoView()
.should('be.visible');

cy.contains('[role="option"]', 'Measured Prefill Joules per Input Token').click();
cy.get('[data-slot="select-content"]').should('not.exist');

// Only the disagg series carries prefill_joules_per_input_token; the
// aggregate series is coverage-filtered off the chart AND out of the
// rendered legend (hwTypesWithData is metric-aware — InferenceContext).
cy.get(DISAGG_DOTS).should('have.length', 3);
cy.get(AGGREGATE_DOTS).should('not.exist');
cy.get('[data-testid="chart-legend"]').should('contain.text', 'B200');
cy.get('[data-testid="chart-legend"]').should('not.contain.text', 'B300');

// The switch is non-destructive: the selection universe never intersects
// metric coverage (selectableHwTypes), so picking a whole-run measured
// axis both series carry brings the aggregate series straight back.
cy.get('[data-testid="yaxis-metric-selector"]').click();
cy.contains('[role="option"]', 'Measured Joules per Output Token').scrollIntoView().click();
cy.get('[data-slot="select-content"]').should('not.exist');
cy.get(AGGREGATE_DOTS).should('have.length', 3);
cy.get('[data-testid="chart-legend"]').should('contain.text', 'B300');
});

it('renders the per-worker power table on a pinned agentic tooltip', () => {
cy.get(`${DISAGG_DOTS} .visible-shape`).first().click({ force: true });

cy.get('[data-chart-tooltip]:visible').should('have.length', 1);
cy.get('[data-chart-tooltip]:visible [data-testid="tooltip-worker-power"]')
.should('exist')
.and('contain.text', 'prefill[0]')
.and('contain.text', '612.3 W')
.and('contain.text', 'decode[0]');
});

it('stays graceful when the pinned point has no workers payload', () => {
cy.get(`${AGGREGATE_DOTS} .visible-shape`).first().click({ force: true });

cy.get('[data-chart-tooltip]:visible').should('have.length', 1);
cy.get('[data-testid="tooltip-worker-power"]').should('not.exist');
});
});
55 changes: 55 additions & 0 deletions packages/app/cypress/support/agentic-fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,61 @@ export function percentileLadder(prefix: string, base: number): Record<string, n
};
}

/**
* Deterministic measured-power telemetry (schema v2, validated) so axis
* positions are stable across runs. `disagg` adds the role splits — role
* watts plus the role-local energy scalars behind the
* Measured Prefill/Decode J-per-token axes.
*/
export function measuredPowerMetrics(
concurrency: number,
opts: { disagg?: boolean } = {},
): Record<string, number> {
const scale = concurrency / 16;
return {
power_valid: 1,
power_metric_schema_version: 2,
avg_power_w: 600 + 10 * scale,
joules_per_input_token: 0.3 / scale,
joules_per_output_token: 8 / scale,
joules_per_total_token: 0.9 / scale,
joules_per_successful_query: 1500 / scale,
avg_temp_c: 65 + scale,
avg_util_pct: 80 + scale,
...(opts.disagg
? {
prefill_avg_power_w: 612.3,
decode_avg_power_w: 701.5,
prefill_joules_per_input_token: 0.4 / scale,
decode_joules_per_output_token: 5.1 / scale,
}
: {}),
};
}

/**
* WorkerPower-shaped rows for the pinned-tooltip drilldown. Plain object
* literals — cypress support files never import types from src.
*/
export function syntheticWorkers(disagg: boolean) {
if (!disagg) {
return [{ role: 'agg', worker_idx: 0, hosts: ['n0'], num_gpus: 8, avg_power_w: 640.2 }];
}
return [
{ role: 'frontend', worker_idx: 0, hosts: ['fe0'], num_gpus: 0, avg_power_w: 120 },
{
role: 'prefill',
worker_idx: 0,
hosts: ['pn0'],
num_gpus: 8,
avg_power_w: 612.3,
avg_temp_c: 68.4,
avg_util_pct: 88.5,
},
{ role: 'decode', worker_idx: 0, hosts: ['dn0'], num_gpus: 8, avg_power_w: 701.5 },
];
}

export function agenticMetrics(concurrency: number): Record<string, number> {
const scale = concurrency / 16;
const itl = 0.011 * scale;
Expand Down
37 changes: 37 additions & 0 deletions packages/app/src/components/inference/axis-metric-explanations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,41 @@ function measuredJoulesPerToken(tokenType: TokenType): MetricExplanation {
};
}

const MEASURED_ROLE_EN: Record<'prefill' | 'decode', { tokens: string; isolates: string }> = {
prefill: { tokens: 'input (prompt)', isolates: 'prompt-processing' },
decode: { tokens: 'output', isolates: 'token-generation' },
};

const MEASURED_ROLE_ZH: Record<'prefill' | 'decode', { tokens: string; isolates: string }> = {
prefill: { tokens: '输入', isolates: '提示词处理' },
decode: { tokens: '输出', isolates: 'token 生成' },
};

function measuredRoleJoulesPerToken(role: 'prefill' | 'decode'): MetricExplanation {
return {
description: {
en:
`Measured accelerator energy consumed by the ${role} workers per ` +
`${MEASURED_ROLE_EN[role].tokens} token, from runner power telemetry integrated over ` +
`the run. Unlike the whole-deployment J/tok metrics, only that role's energy is ` +
`charged, so it isolates ${MEASURED_ROLE_EN[role].isolates} efficiency in ` +
`disaggregated deployments.${MEASURED_TIER_NOTE_EN}`,
zh:
`每个${MEASURED_ROLE_ZH[role].tokens} token 由 ${MEASURED_PHASE_ZH[role]}工作进程消耗的` +
`加速器实测能耗,由运行器功耗遥测在整个运行期间积分得到。与全部署 J/tok 指标不同,` +
`它只计入该角色的能耗,因此可以在分离式部署中单独衡量${
MEASURED_ROLE_ZH[role].isolates
}效率。${MEASURED_TIER_NOTE_ZH}`,
},
formula: {
en:
`J/tok = measured ${role}-worker energy over the run ÷ ` +
`${role === 'prefill' ? 'input' : 'output'} tokens processed`,
zh: `J/tok = 运行期间 ${role} 工作进程实测能耗 ÷ 处理的${MEASURED_ROLE_ZH[role].tokens} token 数`,
},
};
}

/**
* Every `METRIC_REGISTRY` key gets a bilingual explanation and a structural
* formula. Grounded in `buildDerivedChartFields` (src/lib/chart-utils.ts) and
Expand Down Expand Up @@ -352,6 +387,8 @@ export const METRIC_EXPLANATIONS: Record<MetricKey, MetricExplanation> = {
measuredJPerOutputToken: measuredJoulesPerToken('output'),
measuredJPerInputToken: measuredJoulesPerToken('input'),
measuredJPerTotalToken: measuredJoulesPerToken('total'),
measuredPrefillJPerInputToken: measuredRoleJoulesPerToken('prefill'),
measuredDecodeJPerOutputToken: measuredRoleJoulesPerToken('decode'),
measuredJPerSuccessfulQuery: {
description: {
en:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ const QUERY_ENERGY_METRICS = [
'y_measuredWhPerSuccessfulQuery',
] as const;

const ROLE_ENERGY_METRICS = [
'y_measuredPrefillJPerInputToken',
'y_measuredDecodeJPerOutputToken',
] as const;

const defs = chartDefinitions as unknown as ChartDefinition[];
const interactivityDef = defs.find((d) => d.chartType === 'interactivity')!;
const e2eDef = defs.find((d) => d.chartType === 'e2e')!;
Expand Down Expand Up @@ -149,6 +154,16 @@ describe('measured-power Pareto direction', () => {
}
});

it.each(ROLE_ENERGY_METRICS)('%s is bilingual and lower-is-better', (metric) => {
expect(declaredDirection(interactivityDef, metric)).toBe('lower_right');
expect(declaredDirection(e2eDef, metric)).toBe('lower_left');
for (const chartDef of [interactivityDef, e2eDef]) {
expect(chartDef[metric]).toMatch(/\.y$/u);
expect(chartDef[`${metric}_label`]).toBeTruthy();
expect(chartDef[`${metric}_labelZh`]).toBeTruthy();
}
});

it('leaves %TDP without a Pareto direction on either block', () => {
// %TDP is a utilization gauge, not an efficiency frontier: a config running
// hotter is not "worse" along an axis the roofline can order, so declaring a
Expand Down
8 changes: 8 additions & 0 deletions packages/app/src/components/inference/metric-registry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,12 @@ describe('metric compatibility', () => {
expect(resolveMetricConfigKey('y_measuredJPerSuccessfulQuery')).toBe(
'y_measuredJPerSuccessfulQuery',
);
expect(resolveMetricConfigKey('y_measuredPrefillJPerInputToken')).toBe(
'y_measuredPrefillJPerInputToken',
);
expect(resolveMetricConfigKey('y_measuredDecodeJPerOutputToken')).toBe(
'y_measuredDecodeJPerOutputToken',
);
expect(resolveMetricConfigKey('y_costUser')).toBe('y_costUser');
expect(isBenchmarkMetricKey('tpPerGpu')).toBe(true);
expect(isBenchmarkMetricKey('tokenRevenuePerGpuHour')).toBe(true);
Expand All @@ -101,5 +107,7 @@ describe('metric compatibility', () => {
expect(tokenMetricTypeForConfigKey('y_tpPerGpu')).toBe('total');
expect(tokenMetricTypeForConfigKey('y_tokenRevenuePerGpuHour')).toBe('total');
expect(tokenMetricTypeForConfigKey('y_measuredAvgPower')).toBe('total');
expect(tokenMetricTypeForConfigKey('y_measuredPrefillJPerInputToken')).toBe('input');
expect(tokenMetricTypeForConfigKey('y_measuredDecodeJPerOutputToken')).toBe('output');
});
});
20 changes: 19 additions & 1 deletion packages/app/src/components/inference/metric-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,14 @@ export const METRIC_REGISTRY = {
titleZh: '每输出 token 实测焦耳能耗',
polarity: 'lower',
},
measuredDecodeJPerOutputToken: {
field: 'measuredDecodeJPerOutputToken.y',
label: 'Measured Decode J per Output Token (J/tok)',
labelZh: '每输出 token 实测 Decode 能耗(J/tok)',
title: 'Measured Decode Joules per Output Token',
titleZh: '每输出 token 实测 Decode 焦耳能耗',
polarity: 'lower',
},
measuredJPerInputToken: {
field: 'measuredJPerInputToken.y',
label: 'Measured J per Input Token (J/tok)',
Expand All @@ -383,6 +391,14 @@ export const METRIC_REGISTRY = {
titleZh: '每输入 token 实测焦耳能耗',
polarity: 'lower',
},
measuredPrefillJPerInputToken: {
field: 'measuredPrefillJPerInputToken.y',
label: 'Measured Prefill J per Input Token (J/tok)',
labelZh: '每输入 token 实测 Prefill 能耗(J/tok)',
title: 'Measured Prefill Joules per Input Token',
titleZh: '每输入 token 实测 Prefill 焦耳能耗',
polarity: 'lower',
},
measuredJPerTotalToken: {
field: 'measuredJPerTotalToken.y',
label: 'Measured J per Token (J/tok)',
Expand Down Expand Up @@ -484,7 +500,7 @@ export interface MetricControlGroup {
}

/**
* The nine runner-telemetry y-axes in the "Measured Energy" control group.
* The runner-telemetry y-axes in the "Measured Energy" control group.
* Exported (and referenced by the group below, so the two cannot drift) for
* consumers that treat measured axes specially — the legacy-power point ring,
* tooltip tier line, and footer legend key.
Expand All @@ -494,7 +510,9 @@ export const MEASURED_ENERGY_METRIC_CONFIG_KEYS = [
'y_measuredDecodeAvgPower',
'y_measuredAvgPower',
'y_measuredJPerInputToken',
'y_measuredPrefillJPerInputToken',
'y_measuredJPerOutputToken',
'y_measuredDecodeJPerOutputToken',
'y_measuredJPerTotalToken',
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/components/inference/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,9 @@ export interface InferenceData extends Partial<Omit<AggDataEntry, AggDataConflic
measuredJPerOutputToken?: { y: number; roof: boolean };
measuredJPerTotalToken?: { y: number; roof: boolean };
measuredJPerInputToken?: { y: number; roof: boolean };
// Role-local energy (prefill/decode workers only) — disagg-only in practice.
measuredPrefillJPerInputToken?: { y: number; roof: boolean };
measuredDecodeJPerOutputToken?: { y: number; roof: boolean };
measuredJPerSuccessfulQuery?: { y: number; roof: boolean };
measuredWhPerSuccessfulQuery?: { y: number; roof: boolean };
measuredPowerPercentTdp?: { y: number; roof: boolean };
Expand Down
Loading