Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/app/cypress/component/quick-filters-dialog.cy.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const availableQuickFilters: QuickFilters = {
frameworks: ['vllm', 'sglang'],
deployment: ['single-node', 'multi-node', 'disagg'],
spec: ['mtp', 'stp'],
power: ['certified', 'legacy'],
};

describe('QuickFiltersDialog', () => {
Expand Down Expand Up @@ -47,6 +48,7 @@ describe('QuickFiltersDialog', () => {
frameworks: ['sglang'],
deployment: ['disagg'],
spec: ['mtp'],
power: ['certified'],
},
availableQuickFilters,
},
Expand All @@ -57,5 +59,6 @@ describe('QuickFiltersDialog', () => {
cy.get('@setQuickFilterFrameworks').should('have.been.calledWith', []);
cy.get('@setQuickFilterDeployment').should('have.been.calledWith', []);
cy.get('@setQuickFilterSpec').should('have.been.calledWith', []);
cy.get('@setQuickFilterPower').should('have.been.calledWith', []);
});
});
174 changes: 174 additions & 0 deletions packages/app/cypress/e2e/certified-power-filter.cy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
// Certified-vs-legacy measured power (PLAN-02 / gap G2): on the Measured
// Energy y-axes, points without a producer validation verdict carry a dotted
// ring and a footer legend key, and Quick Filters gains a "Measured Power"
// category (Certified/Legacy pills, `i_power` share-link param). Fixture rows
// are intercepted so one config is certified (power_valid=1) and one is
// legacy (no verdict), keeping every assertion deterministic regardless of
// what the production dataset contains.

const POWER_MODEL = 'dsv4';
const POWER_DATE = '2026-08-20';

const powerPercentileLadder = (prefix: string, base: number): Record<string, number> => ({
[`median_${prefix}`]: base,
[`p75_${prefix}`]: base * 1.2,
[`p90_${prefix}`]: base * 1.5,
[`p95_${prefix}`]: base * 1.7,
[`p99_${prefix}`]: base * 2.2,
[`std_${prefix}`]: base * 0.3,
});

const powerMetrics = (conc: number): Record<string, number> => {
const scale = conc / 16;
const itl = 0.011 * scale;
return {
...powerPercentileLadder('ttft', 0.4 * scale),
...powerPercentileLadder('tpot', 0.012 * scale),
...powerPercentileLadder('itl', itl),
...powerPercentileLadder('e2el', 8 * scale),
median_intvty: 1 / itl,
p75_intvty: 1 / (itl * 1.2),
p90_intvty: 1 / (itl * 1.5),
p99_intvty: 1 / (itl * 2.2),
std_intvty: (1 / itl) * 0.1,
tput_per_gpu: 950 / Math.sqrt(scale),
output_tput_per_gpu: 210,
input_tput_per_gpu: 740,
};
};

const powerConfigs = [
// Legacy telemetry: measured watts without a producer verdict.
{ hardware: 'b200', framework: 'vllm', power: { avg_power_w: 560 } },
// Certified telemetry: explicit power_valid=1 verdict.
{ hardware: 'mi300x', framework: 'sglang', power: { power_valid: 1, avg_power_w: 685.5 } },
];

const powerAvailability = powerConfigs.map((config) => ({
model: POWER_MODEL,
isl: 8192,
osl: 1024,
precision: 'fp4',
hardware: config.hardware,
framework: config.framework,
spec_method: 'none',
disagg: false,
benchmark_type: 'single_turn',
date: POWER_DATE,
}));

let powerBenchmarkId = 990000;
const powerBenchmarks = powerConfigs.flatMap((config) =>
[16, 64, 128].map((conc) => ({
id: powerBenchmarkId++,
hardware: config.hardware,
framework: config.framework,
model: POWER_MODEL,
precision: 'fp4',
spec_method: 'none',
disagg: false,
is_multinode: false,
prefill_tp: 8,
decode_tp: 8,
num_prefill_gpu: 8,
num_decode_gpu: 8,
isl: 8192,
osl: 1024,
conc,
offload_mode: 'off',
benchmark_type: 'single_turn',
image: `${config.framework}/server:test`,
metrics: { ...powerMetrics(conc), ...config.power },
workers: null,
date: POWER_DATE,
run_url: null,
})),
);

function visitCertifiedPowerChart(extraParams = '') {
cy.intercept('GET', '/api/v1/availability', { body: powerAvailability }).as('availability');
cy.intercept('GET', '/api/v1/benchmarks*', { body: powerBenchmarks }).as('benchmarks');
cy.visit(`/inference?g_model=DeepSeek-V4-Pro&i_seq=8k/1k&i_prec=fp4${extraParams}`, {
onBeforeLoad(win) {
win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now()));
},
});
cy.wait(['@availability', '@benchmarks']);
cy.get('[data-testid="inference-chart-display"]', { timeout: 30_000 }).should('exist');
cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1);
}

describe('Certified vs legacy measured power', () => {
it('rings legacy points on a measured axis and filters them via Quick Filters', () => {
visitCertifiedPowerChart();

// No decorations off the Measured Energy axes.
cy.get('.legacy-power-ring').should('not.exist');
cy.get('[data-testid="legacy-power-key"]').should('not.exist');

// Pick "Measured Average Power per Chip" from the y-axis dropdown. The
// select list is a scroll container and Measured Energy sits below the
// fold, so scroll before clicking.
cy.get('[data-testid="yaxis-metric-selector"]').click();
cy.contains('[role="option"]', 'Measured Average Power per Chip')
.scrollIntoView()
.should('be.visible')
.click();
cy.get('[data-slot="select-content"]').should('not.exist');

// The no-verdict config is ringed; the certified one is not. The footer
// legend key appears with the ringed points.
cy.get('.dot-group[data-hw-key^="b200"] .legacy-power-ring').should('exist');
cy.get('.dot-group[data-hw-key^="mi300x"] .legacy-power-ring').should('not.exist');
cy.get('[data-testid="legacy-power-key"]').should('be.visible');
cy.screenshot('legacy-power-rings', { capture: 'viewport' });

// Quick Filters gains the Measured Power category with both pills enabled.
cy.get('[data-testid="scatter-quick-filters"]').click();
cy.get('[data-testid="quick-filters-dialog"]').should('be.visible');
cy.get('[data-testid="quick-filter-power-certified"]').should('be.enabled');
cy.get('[data-testid="quick-filter-power-legacy"]').should('be.enabled');

// Certified-only: legacy points (and with them every ring and the legend
// key) leave the chart while the certified series stays.
cy.get('[data-testid="quick-filter-power-certified"]').click();
cy.get('[data-testid="quick-filters-selected-count"]').should('contain.text', '1 selected');
cy.get('.dot-group[data-hw-key^="b200"]').should('not.exist');
cy.get('.dot-group[data-hw-key^="mi300x"]').should('exist');
cy.get('.legacy-power-ring').should('not.exist');
cy.get('[data-testid="legacy-power-key"]').should('not.exist');
cy.get('[data-testid="inference-chart-display"] svg').should('exist');
cy.screenshot('certified-only-filter', { capture: 'viewport' });

// Clear filters restores the legacy series, rings, and legend key.
cy.contains('button', 'Clear filters').click();
cy.get('[data-testid="quick-filters-selected-count"]').should('not.exist');
cy.get('[data-testid="quick-filter-power-certified"]').should(
'have.attr',
'aria-pressed',
'false',
);
cy.contains('button', 'Done').click();
cy.get('.dot-group[data-hw-key^="b200"] .legacy-power-ring').should('exist');
cy.get('[data-testid="legacy-power-key"]').should('be.visible');
});

it('restores a shared i_power=certified link with the pill pre-selected', () => {
// Note: filter writes live in the in-memory share-link store (the address
// bar is deliberately stripped after load — see url-state.ts), so the
// durable observable behavior is the restore direction tested here.
visitCertifiedPowerChart('&i_metric=y_measuredAvgPower&i_power=certified');

cy.get('.dot-group[data-hw-key^="mi300x"]').should('exist');
cy.get('.dot-group[data-hw-key^="b200"]').should('not.exist');
cy.get('[data-testid="legacy-power-key"]').should('not.exist');

cy.get('[data-testid="scatter-quick-filters"]').click();
cy.get('[data-testid="quick-filter-power-certified"]').should(
'have.attr',
'aria-pressed',
'true',
);
cy.get('[data-testid="quick-filters-selected-count"]').should('contain.text', '1 selected');
});
});
5 changes: 3 additions & 2 deletions packages/app/cypress/support/mock-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,12 +230,13 @@ export function createMockInferenceContextValues(
setSelectedXAxisMode: namedStub('setSelectedXAxisMode'),
scaleType: 'auto',
setScaleType: namedStub('setScaleType'),
quickFilters: { vendors: [], frameworks: [], deployment: [], spec: [] },
availableQuickFilters: { vendors: [], frameworks: [], deployment: [], spec: [] },
quickFilters: { vendors: [], frameworks: [], deployment: [], spec: [], power: [] },
availableQuickFilters: { vendors: [], frameworks: [], deployment: [], spec: [], power: [] },
setQuickFilterVendors: namedStub('setQuickFilterVendors'),
setQuickFilterFrameworks: namedStub('setQuickFilterFrameworks'),
setQuickFilterDeployment: namedStub('setQuickFilterDeployment'),
setQuickFilterSpec: namedStub('setQuickFilterSpec'),
setQuickFilterPower: namedStub('setQuickFilterPower'),
isLegendExpanded: true,
setIsLegendExpanded: namedStub('setIsLegendExpanded'),
hideNonOptimal: false,
Expand Down
21 changes: 18 additions & 3 deletions packages/app/src/components/inference/InferenceContext.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ import { bestSeriesPerSku } from './utils/best-series-per-sku';
import {
EMPTY_QUICK_FILTERS,
parseDeploymentModes,
parsePowerTiers,
type DeploymentMode,
type PowerTier,
type QuickFilters,
type SpecMode,
} from './utils/quickFilters';
Expand Down Expand Up @@ -425,7 +427,7 @@ export function InferenceProvider({
() => (getUrlParam('i_scale') as 'auto' | 'linear' | 'log') || 'auto',
);

// ── Quick filters (vendor / framework / deployment / mtp-stp) ───────────────
// ── Quick filters (vendor / framework / deployment / mtp-stp / power tier) ──
// Coarse pre-filters applied to the point set. Empty = no constraint.
//
// Initialized empty rather than from the URL so the first client render matches
Expand All @@ -438,8 +440,9 @@ export function InferenceProvider({
const [quickFilterFrameworks, setQuickFilterFrameworks] = useState<string[]>([]);
const [quickFilterDeployment, setQuickFilterDeployment] = useState<DeploymentMode[]>([]);
const [quickFilterSpec, setQuickFilterSpec] = useState<SpecMode[]>([]);
const [quickFilterPower, setQuickFilterPower] = useState<PowerTier[]>([]);
useEffect(() => {
const parse = (key: 'i_vendor' | 'i_fw' | 'i_disagg' | 'i_spec') => {
const parse = (key: 'i_vendor' | 'i_fw' | 'i_disagg' | 'i_spec' | 'i_power') => {
const v = getUrlParam(key);
return v ? v.split(',').filter(Boolean) : [];
};
Expand All @@ -449,19 +452,28 @@ export function InferenceProvider({
// point, so expand it to both aggregate deployment modes.
const deployment = parseDeploymentModes(parse('i_disagg'));
const spec = parse('i_spec') as SpecMode[];
const power = parsePowerTiers(parse('i_power'));
if (vendors.length > 0) setQuickFilterVendors(vendors);
if (frameworks.length > 0) setQuickFilterFrameworks(frameworks);
if (deployment.length > 0) setQuickFilterDeployment(deployment);
if (spec.length > 0) setQuickFilterSpec(spec);
if (power.length > 0) setQuickFilterPower(power);
}, [getUrlParam]);
const quickFilters = useMemo<QuickFilters>(
() => ({
vendors: quickFilterVendors,
frameworks: quickFilterFrameworks,
deployment: quickFilterDeployment,
spec: quickFilterSpec,
power: quickFilterPower,
}),
[quickFilterVendors, quickFilterFrameworks, quickFilterDeployment, quickFilterSpec],
[
quickFilterVendors,
quickFilterFrameworks,
quickFilterDeployment,
quickFilterSpec,
quickFilterPower,
],
);
// Historical Trends hides Quick Filters, so never apply invisible selections there.
// Agentic charts expose vendor, framework, and deployment filters, but speculative
Expand Down Expand Up @@ -1473,6 +1485,7 @@ export function InferenceProvider({
i_fw: quickFilterFrameworks.join(','),
i_disagg: quickFilterDeployment.join(','),
i_spec: quickFilterSpec.join(','),
i_power: quickFilterPower.join(','),
},
[
selectedYAxisMetric,
Expand All @@ -1499,6 +1512,7 @@ export function InferenceProvider({
quickFilterFrameworks,
quickFilterDeployment,
quickFilterSpec,
quickFilterPower,
],
);

Expand Down Expand Up @@ -1763,6 +1777,7 @@ export function InferenceProvider({
setQuickFilterFrameworks,
setQuickFilterDeployment,
setQuickFilterSpec,
setQuickFilterPower,
setIsLegendExpanded,
setHideNonOptimal,
setShowPointLabels,
Expand Down
50 changes: 33 additions & 17 deletions packages/app/src/components/inference/axis-metric-explanations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,18 @@ function provisionedJoules(tokenType: TokenType): MetricExplanation {
};
}

/**
* Certification-tier note appended to every Measured Energy explanation:
* pairs the dotted legacy-power point ring and the Quick Filters → Measured
* Power pills with the footer text that explains them.
*/
const MEASURED_TIER_NOTE_EN =
' Points without a producer validation verdict are legacy measurements — marked with a ' +
'dotted ring on the chart and filterable via Quick Filters → Measured Power.';
const MEASURED_TIER_NOTE_ZH =
'没有生产端验证结论的数据点属于旧版测量——在图表上以虚线圆环标记,' +
'并可通过快捷筛选中的“实测功耗”进行筛选。';

type MeasuredPhase = 'run' | 'prefill' | 'decode';

const MEASURED_PHASE_EN: Record<MeasuredPhase, string> = {
Expand All @@ -216,11 +228,13 @@ function measuredPower(phase: MeasuredPhase): MetricExplanation {
description: {
en:
`Average per-chip accelerator power actually drawn during ${MEASURED_PHASE_EN[phase]}, ` +
'read from runner telemetry. Unlike the all-in provisioned metrics, this reflects real ' +
'measured draw, not the provisioned budget.',
`read from runner telemetry. Unlike the all-in provisioned metrics, this reflects real ` +
`measured draw, not the provisioned budget.${MEASURED_TIER_NOTE_EN}`,
zh:
`${MEASURED_PHASE_ZH[phase]}每块加速器芯片的实际平均功耗,来自运行器遥测数据。` +
'与全电源配置类指标不同,它反映的是真实实测功耗,而不是按配置计算的预算值。',
`与全电源配置类指标不同,它反映的是真实实测功耗,而不是按配置计算的预算值。${
MEASURED_TIER_NOTE_ZH
}`,
},
formula: {
en: `W = mean of sampled per-chip accelerator power draw over ${MEASURED_PHASE_EN[phase]}`,
Expand All @@ -236,10 +250,10 @@ function measuredJoulesPerToken(tokenType: TokenType): MetricExplanation {
`Measured accelerator energy consumed per ${
tokenType === 'total' ? 'token (including prompt tokens)' : `${tokenType} token`
}, from runner power telemetry integrated over the run. Lower means the system converts ` +
'electricity into tokens more efficiently.',
`electricity into tokens more efficiently.${MEASURED_TIER_NOTE_EN}`,
zh:
`每个${TOKEN_TYPE_ZH[tokenType]}消耗的加速器实测能耗,由运行器功耗遥测在整个运行期间积分得到。` +
'数值越低,说明系统把电能转化为 token 的效率越高。',
`数值越低,说明系统把电能转化为 token 的效率越高。${MEASURED_TIER_NOTE_ZH}`,
},
formula: {
en: `J/tok = measured accelerator energy over the run ÷ ${TOKEN_TYPE_EN[tokenType]} processed`,
Expand Down Expand Up @@ -341,12 +355,12 @@ export const METRIC_EXPLANATIONS: Record<MetricKey, MetricExplanation> = {
measuredJPerSuccessfulQuery: {
description: {
en:
'Measured accelerator energy consumed per successfully completed request, from runner ' +
'power telemetry. It charges the energy of the whole run only to requests that finished ' +
'successfully.',
`Measured accelerator energy consumed per successfully completed request, from runner ` +
`power telemetry. It charges the energy of the whole run only to requests that finished ` +
`successfully.${MEASURED_TIER_NOTE_EN}`,
zh:
'每个成功完成的请求消耗的加速器实测能耗,来自运行器功耗遥测。' +
'整个运行的能耗只计入成功完成的请求。',
`每个成功完成的请求消耗的加速器实测能耗,来自运行器功耗遥测。` +
`整个运行的能耗只计入成功完成的请求。${MEASURED_TIER_NOTE_ZH}`,
},
formula: {
en: 'J/query = measured accelerator energy over the run ÷ successfully completed requests',
Expand All @@ -356,9 +370,9 @@ export const METRIC_EXPLANATIONS: Record<MetricKey, MetricExplanation> = {
measuredWhPerSuccessfulQuery: {
description: {
en:
'The same measured energy per successful request expressed in watt-hours, a more ' +
'familiar household unit (1 Wh = 3,600 J).',
zh: '与每次成功请求实测能耗相同的量,换算成更直观的瓦时单位(1 Wh = 3,600 J)。',
`The same measured energy per successful request expressed in watt-hours, a more ` +
`familiar household unit (1 Wh = 3,600 J).${MEASURED_TIER_NOTE_EN}`,
zh: `与每次成功请求实测能耗相同的量,换算成更直观的瓦时单位(1 Wh = 3,600 J)。${MEASURED_TIER_NOTE_ZH}`,
},
formula: {
en: 'Wh/query = measured J per successful query ÷ 3,600',
Expand All @@ -368,11 +382,13 @@ export const METRIC_EXPLANATIONS: Record<MetricKey, MetricExplanation> = {
measuredPowerPercentTdp: {
description: {
en:
'Measured average per-chip power as a share of the accelerator’s rated TDP. Values well ' +
'below 100% suggest the workload leaves thermal or power headroom on the table.',
`Measured average per-chip power as a share of the accelerator’s rated TDP. Values well ` +
`below 100% suggest the workload leaves thermal or power headroom on the table.${
MEASURED_TIER_NOTE_EN
}`,
zh:
'每芯片实测平均功耗占加速器额定 TDP 的百分比。' +
'明显低于 100% 说明该工作负载没有用满散热或供电余量。',
`每芯片实测平均功耗占加速器额定 TDP 的百分比。` +
`明显低于 100% 说明该工作负载没有用满散热或供电余量。${MEASURED_TIER_NOTE_ZH}`,
},
formula: {
en: '% TDP = measured average power per chip (W) ÷ rated TDP (W) × 100',
Expand Down
Loading