diff --git a/packages/app/cypress/e2e/api-documentation.cy.ts b/packages/app/cypress/e2e/api-documentation.cy.ts index f22f9e9b5..fe0503a63 100644 --- a/packages/app/cypress/e2e/api-documentation.cy.ts +++ b/packages/app/cypress/e2e/api-documentation.cy.ts @@ -9,7 +9,9 @@ describe('API documentation', () => { .and('contain.text', 'Quickstart') .and('contain.text', 'curl') .and('contain.text', '/api/v1/availability') - .and('contain.text', 'Endpoint reference'); + .and('contain.text', 'Endpoint reference') + .and('contain.text', 'Measured power') + .and('contain.text', 'powerValid'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('[data-testid="api-spec-version"]').should('have.text', 'v1 · OpenAPI 3.1'); cy.get('[data-testid="api-endpoint-list-benchmarks"]') @@ -45,6 +47,10 @@ describe('API documentation', () => { 'operationId', 'list-benchmarks', ); + const benchmarkParameterNames = body.paths['/api/v1/benchmarks'].get.parameters.map( + (parameter: { name: string }) => parameter.name, + ); + expect(benchmarkParameterNames).to.include('powerValid'); expect(body.paths['/api/v1/collectivex/runs/{runId}'].get).to.have.property( 'operationId', 'get-collectivex-run', @@ -58,7 +64,8 @@ describe('API documentation', () => { .and('contain.text', '快速入门') .and('contain.text', '约定') .and('contain.text', '端点参考') - .and('contain.text', 'BenchmarkRow 与指标'); + .and('contain.text', 'BenchmarkRow 与指标') + .and('contain.text', '实测功率'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('link[rel="alternate"][hreflang="en"]').should('have.attr', 'href', `${SITE_URL}/api`); cy.get('link[rel="alternate"][hreflang="zh-CN"]').should( diff --git a/packages/app/src/app/api/v1/benchmarks/route.test.ts b/packages/app/src/app/api/v1/benchmarks/route.test.ts index 629339016..e7f679e35 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.test.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.test.ts @@ -242,4 +242,148 @@ describe('GET /api/v1/benchmarks', () => { const body = await res.json(); expect(body).toEqual([]); }); + + describe('powerValid filter', () => { + const validatedV2 = { + id: 1, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, power_metric_schema_version: 2, avg_power_w: 700 }, + }; + const validatedUnversioned = { + id: 2, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, avg_power_w: 650 }, + }; + const invalidated = { + id: 3, + benchmark_type: 'single_turn', + metrics: { power_valid: 0 }, + }; + const legacy = { + id: 4, + benchmark_type: 'single_turn', + metrics: { tput_per_gpu: 100 }, + }; + const powerRows = [validatedV2, validatedUnversioned, invalidated, legacy]; + + it('powerValid=1 keeps only rows with a validated verdict', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([validatedV2, validatedUnversioned]); + }); + + it('powerValid=0 keeps only explicitly invalidated rows', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=0')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([invalidated]); + }); + + it('powerValid=strictV2 requires a validated verdict plus schema version 2', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([validatedV2]); + }); + + it('powerValid=any matches the response with the param absent (backward compat)', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + const withParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any')); + + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + const withoutParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528')); + + expect(withParam.status).toBe(200); + expect(withoutParam.status).toBe(200); + const bodyWithParam = await withParam.json(); + expect(bodyWithParam).toEqual(await withoutParam.json()); + expect(bodyWithParam).toEqual(powerRows); + }); + + it('rejects an unknown powerValid value without querying', async () => { + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=garbage')); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Unknown powerValid filter' }); + expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); + }); + + it('rejects powerValid combined with view=calculator without querying', async () => { + const res = await GET( + req( + '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1&view=calculator&sequence=1k%2F1k', + ), + ); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: 'powerValid cannot be combined with view=calculator', + }); + expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); + }); + + it('allows powerValid=any with view=calculator (no-op filter)', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + benchmark_type: 'single_turn', + isl: 1024, + osl: 1024, + metrics: { tput_per_gpu: 100, avg_power_w: 700 }, + }, + ]); + + const res = await GET( + req( + '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any&view=calculator&sequence=1k%2F1k', + ), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual([ + { benchmark_type: 'single_turn', isl: 1024, osl: 1024, metrics: { tput_per_gpu: 100 } }, + ]); + }); + + it('composes with the agentic workflow-metadata trim', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1 }, + }, + { + id: 2, + benchmark_type: 'single_turn', + workflow_run_id: 43, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1 }, + }, + { + id: 3, + benchmark_type: 'agentic_traces', + workflow_run_id: 44, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 0 }, + }, + ]); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1')); + expect(await res.json()).toEqual([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1 }, + }, + { id: 2, benchmark_type: 'single_turn', metrics: { power_valid: 1 } }, + ]); + }); + }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/route.ts b/packages/app/src/app/api/v1/benchmarks/route.ts index a75260d7b..887bb2a07 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.ts @@ -11,6 +11,7 @@ import { import { cachedJson, cachedQuery } from '@/lib/api-cache'; import { toCalculatorBenchmarkRows } from '@/lib/benchmark-api-view'; +import { filterByPowerValidity, parsePowerValidityFilter } from '@/lib/benchmark-power-validity'; import { PUBLIC_API_ERRORS, publicApiError } from '@/lib/public-api-errors'; import { agenticWorkflowMetadataOnly } from '@/lib/agentic-workflow-metadata'; import { loadFixture } from '@/lib/test-fixtures'; @@ -52,6 +53,7 @@ export async function GET(request: NextRequest) { const exactRun = params.get('exactRun') === 'true'; const view = params.get('view'); const sequence = params.get('sequence') ?? ''; + const powerValidFilter = parsePowerValidityFilter(params.get('powerValid')); const dbModelKeys = DISPLAY_MODEL_TO_DB[model]; if (!dbModelKeys || dbModelKeys.length === 0) { return publicApiError(PUBLIC_API_ERRORS.unknownModel, 400); @@ -59,10 +61,23 @@ export async function GET(request: NextRequest) { if (view === 'calculator' && !['1k/1k', '1k/8k', '8k/1k', 'agentic-traces'].includes(sequence)) { return NextResponse.json({ error: 'Unknown calculator sequence' }, { status: 400 }); } + if (powerValidFilter === undefined) { + return NextResponse.json({ error: 'Unknown powerValid filter' }, { status: 400 }); + } + // The calculator cache stores rows already trimmed to an allowlist that + // excludes power_valid, so post-cache filtering cannot work there. + if (view === 'calculator' && powerValidFilter !== 'any') { + return NextResponse.json( + { error: 'powerValid cannot be combined with view=calculator' }, + { status: 400 }, + ); + } if (FIXTURES_MODE) { const fixture = loadFixture('benchmarks'); return cachedJson( - view === 'calculator' ? toCalculatorBenchmarkRows(fixture, sequence) : fixture, + view === 'calculator' + ? toCalculatorBenchmarkRows(fixture, sequence) + : filterByPowerValidity(fixture, powerValidFilter), ); } @@ -73,7 +88,11 @@ export async function GET(request: NextRequest) { : exactRun && runId ? await getCachedBenchmarksForRun(dbModelKeys, runId) : await getCachedBenchmarks(dbModelKeys, date, exact || undefined, runId); - return cachedJson(agenticWorkflowMetadataOnly(rows)); + return cachedJson( + agenticWorkflowMetadataOnly( + view === 'calculator' ? rows : filterByPowerValidity(rows, powerValidFilter), + ), + ); } catch (error) { console.error('Error fetching benchmarks:', error); return publicApiError(PUBLIC_API_ERRORS.internal, 500); diff --git a/packages/app/src/lib/api-documentation.power.test.ts b/packages/app/src/lib/api-documentation.power.test.ts new file mode 100644 index 000000000..94fd654b4 --- /dev/null +++ b/packages/app/src/lib/api-documentation.power.test.ts @@ -0,0 +1,83 @@ +import { POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; +import { describe, expect, it } from 'vitest'; + +import { apiOperations, buildOpenApiDocument, getApiDocumentation } from './api-documentation'; +import { POWER_VALIDITY_FILTERS } from './benchmark-power-validity'; + +const listBenchmarks = apiOperations.find((operation) => operation.id === 'list-benchmarks'); +const benchmarkRowSchema = listBenchmarks?.responses.find((response) => response.status === '200') + ?.schema.items; + +describe('measured-power API documentation', () => { + it('types every power metric key in the benchmarks metrics schema', () => { + const metricsSchema = benchmarkRowSchema?.properties?.metrics; + expect(metricsSchema).toBeDefined(); + expect(metricsSchema?.additionalProperties).toEqual({ type: 'number' }); + for (const key of POWER_METRIC_KEYS) { + const property = metricsSchema?.properties?.[key]; + expect(property?.type, `${key} must be a typed number property`).toBe('number'); + expect( + property?.description?.trim(), + `${key} must carry a nonempty description`, + ).toBeTruthy(); + } + }); + + it('reserves optional power_invalid_reasons and power_audit row fields', () => { + const reasons = benchmarkRowSchema?.properties?.power_invalid_reasons; + expect(reasons?.type).toBe('array'); + expect(reasons?.items).toEqual({ type: 'string' }); + + const audit = benchmarkRowSchema?.properties?.power_audit; + expect(Object.keys(audit?.properties ?? {}).toSorted()).toEqual( + [ + 'window_start_unix', + 'window_end_unix', + 'expected_gpu_count', + 'observed_gpu_count', + 'sample_count', + 'max_sample_gap_s', + 'producer_sha', + 'exporter_image_sha256', + ].toSorted(), + ); + // Producers may emit partial audits, so individual audit fields remain optional. + expect(audit?.required).toBeUndefined(); + + expect(benchmarkRowSchema?.required).not.toContain('power_invalid_reasons'); + expect(benchmarkRowSchema?.required).not.toContain('power_audit'); + }); + + it('projects the view, sequence, and powerValid parameters into OpenAPI', () => { + const document = buildOpenApiDocument('https://api-docs.test'); + const operation = (document.paths['/api/v1/benchmarks'] as Record).get as { + parameters: readonly { name: string; schema: unknown }[]; + }; + const names = operation.parameters.map((parameter) => parameter.name); + expect(names).toContain('view'); + expect(names).toContain('sequence'); + + const powerValid = operation.parameters.find((parameter) => parameter.name === 'powerValid'); + expect(powerValid?.schema).toEqual({ + type: 'string', + enum: ['1', '0', 'any', 'strictV2'], + default: 'any', + }); + expect([...POWER_VALIDITY_FILTERS]).toEqual(['1', '0', 'any', 'strictV2']); + }); + + it('renders a bilingual measured-power schema note', () => { + for (const locale of ['en', 'zh'] as const) { + const note = getApiDocumentation(locale).schemaNotes.find( + (candidate) => candidate.id === 'measured-power', + ); + expect(note, `${locale} must expose a measured-power schema note`).toBeDefined(); + expect(note?.title.trim()).toBeTruthy(); + expect(note?.description.trim()).toBeTruthy(); + } + const zhNote = getApiDocumentation('zh').schemaNotes.find( + (candidate) => candidate.id === 'measured-power', + ); + expect(zhNote?.description).toMatch(/[㐀-鿿]/u); + }); +}); diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 4a22d16c5..bb51fa166 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -1,6 +1,11 @@ -import { DB_MODEL_TO_DISPLAY, DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; +import { + DB_MODEL_TO_DISPLAY, + DISPLAY_MODEL_TO_DB, + POWER_METRIC_KEYS, +} from '@semianalysisai/inferencex-constants'; import { COLLECTIVEX_VERSIONS } from '@semianalysisai/inferencex-db/collectivex/types'; +import { POWER_VALIDITY_FILTERS } from './benchmark-power-validity'; import { PUBLIC_API_ERRORS } from './public-api-errors'; export type ApiDocumentationLocale = 'en' | 'zh'; @@ -192,6 +197,59 @@ const errorResponse = ( mediaType: 'application/json', }); +const powerMetricDescriptions: Readonly> = { + power_valid: + 'Publication verdict: 1 = validated measurement window; 0 = failed validation — measured power/energy values are withheld from this row end-to-end, so treat any that remain as unreliable; absent = legacy row predating validation.', + power_metric_schema_version: + 'Power schema version. Version 2 defines every unprefixed joules_per_* field as whole-deployment energy, including on disaggregated runs.', + avg_power_w: 'Mean per-GPU power draw in watts during the measured load window.', + joules_per_successful_query: 'Whole-deployment energy in joules divided by successful requests.', + joules_per_output_token: + 'Energy per generated output token in joules; cluster-wide on schema-version-2 rows, including disaggregated runs.', + joules_per_total_token: + 'Total system energy divided by input plus output tokens; a workload-shape-fair view that does not treat prompt tokens as free.', + prefill_avg_power_w: + 'Mean per-GPU power draw in watts across prefill workers; emitted only for deployments with distinct prefill and decode roles.', + decode_avg_power_w: + 'Mean per-GPU power draw in watts across decode workers; emitted only for deployments with distinct prefill and decode roles.', + joules_per_input_token: + 'Energy per input token in joules; cluster-wide on schema-version-2 rows.', + prefill_joules_per_input_token: 'Role-local prefill energy per input token in joules.', + decode_joules_per_output_token: 'Role-local decode energy per generated output token in joules.', + avg_temp_c: 'Mean per-GPU temperature in degrees Celsius during the load window.', + peak_temp_c: + 'Maximum instantaneous per-GPU temperature in degrees Celsius during the load window.', + avg_util_pct: 'Mean per-GPU utilization percentage (0-100) during the load window.', + avg_mem_used_mb: 'Mean per-GPU memory used in MB during the load window.', +}; +const benchmarkMetricsSchema: ApiSchema = { + type: 'object', + additionalProperties: numberSchema, + description: + 'Scalar metric map. Keys evolve independently; measured power / energy / GPU-telemetry keys are typed below.', + properties: Object.fromEntries( + POWER_METRIC_KEYS.map((key): [string, ApiSchema] => [ + key, + { type: 'number', description: powerMetricDescriptions[key] }, + ]), + ), +}; +const powerAuditSchema: ApiSchema = { + type: 'object', + properties: { + window_start_unix: numberSchema, + window_end_unix: numberSchema, + expected_gpu_count: integerSchema, + observed_gpu_count: integerSchema, + sample_count: integerSchema, + max_sample_gap_s: numberSchema, + producer_sha: nullableStringSchema, + exporter_image_sha256: nullableStringSchema, + }, + additionalProperties: true, + description: + 'Optional power measurement-window audit. Individual fields may be absent; legacy rows omit the object.', +}; const workerPowerSchema = objectSchemaWithOptional( { role: stringSchema, @@ -234,14 +292,20 @@ const benchmarkRowSchema = objectSchemaWithOptional( offload_mode: stringSchema, image: nullableStringSchema, recipe_fingerprint: nullableStringSchema, - metrics: metricMapSchema, + metrics: benchmarkMetricsSchema, workers: arraySchema(workerPowerSchema), + power_invalid_reasons: { + ...arraySchema(stringSchema), + description: + 'Optional snake_case validation reason codes when metrics.power_valid == 0. Absent on legacy rows.', + }, + power_audit: powerAuditSchema, date: { type: 'string', format: 'date' }, workflow_run_id: integerSchema, run_started_at: { type: ['string', 'null'], format: 'date-time' }, run_url: nullableStringSchema, }, - ['workers', 'workflow_run_id', 'run_started_at'], + ['workers', 'power_invalid_reasons', 'power_audit', 'workflow_run_id', 'run_started_at'], ); const benchmarkRowsSchema = arraySchema(benchmarkRowSchema); const benchmarkExample = [ @@ -271,7 +335,17 @@ const benchmarkExample = [ offload_mode: 'off', image: 'vllm/vllm-openai:v0.10.2', recipe_fingerprint: '7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d', - metrics: { median_ttft: 0.42, median_tpot: 0.018, tput_per_gpu: 128.4 }, + metrics: { + median_ttft: 0.42, + median_tpot: 0.018, + tput_per_gpu: 128.4, + power_valid: 1, + power_metric_schema_version: 2, + avg_power_w: 678.5, + joules_per_output_token: 5.3, + joules_per_total_token: 2.65, + avg_temp_c: 61.2, + }, date: '2026-08-08', run_url: 'https://github.com/semianalysis/inference-benchmarks/actions/runs/123456789', }, @@ -589,8 +663,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/benchmarks', summary: text('Read benchmark results', '读取基准结果'), description: text( - 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. The page-owned calculator view is not part of this public contract.', - '按展示模型返回原始基准行。可用 date 获取截至该日的快照,exact=true 限定该日,runId 约束最新查询,或将 exactRun=true 与数字 runId 组合以仅返回该工作流运行。页面专用的计算器视图不属于此公开契约。', + 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change), and powerValid filters rows by measured-power validity and cannot be combined with view=calculator (except powerValid=any, which is a no-op).', + '按展示模型返回原始基准行。可用 date 获取截至该日的快照,exact=true 限定该日,runId 约束最新查询,或将 exactRun=true 与数字 runId 组合以仅返回该工作流运行。view=calculator 返回页面专用的裁剪投影(会移除实测功率指标和 workers,其允许列表可能变化);powerValid 按实测功率有效性筛选行,且不能与 view=calculator 组合使用(powerValid=any 除外,等同于不筛选)。', ), audience: 'public', stability: 'stable', @@ -645,6 +719,36 @@ export const apiOperations: readonly ApiOperation[] = [ { type: 'boolean', default: false }, false, ), + parameter( + 'view', + 'query', + false, + 'enum', + 'calculator trims each row to the page-owned metric allowlist the throughput calculator consumes and removes workers; measured power metrics are excluded from this view. Requires sequence. Omit for every stored metric, including measured power.', + 'calculator 会将每行裁剪为吞吐量计算器所需的页面专用指标允许列表并移除 workers;此视图不包含实测功率指标。需要同时提供 sequence。省略则返回全部已存储指标,包括实测功率。', + { type: 'string', enum: ['calculator'] }, + 'calculator', + ), + parameter( + 'sequence', + 'query', + false, + 'enum', + 'Required when view=calculator and ignored otherwise. Unknown values yield 400 Unknown calculator sequence.', + '当 view=calculator 时必填,其余情况会被忽略。未知值返回 400 Unknown calculator sequence。', + { type: 'string', enum: ['1k/1k', '1k/8k', '8k/1k', 'agentic-traces'] }, + '1k/1k', + ), + parameter( + 'powerValid', + 'query', + false, + 'enum', + '1 keeps only rows with a validated power measurement (metrics.power_valid == 1); 0 keeps only explicitly invalidated rows; any applies no filter (default; includes legacy rows without a verdict); strictV2 keeps rows with power_valid == 1 and power_metric_schema_version == 2 (whole-deployment energy semantics) — stricter than the InferenceX UI, which also displays validated rows that predate schema versioning. Unknown values yield 400 Unknown powerValid filter; cannot be combined with view=calculator (except any, which is a no-op).', + '1 仅保留具有已验证功率测量的行(metrics.power_valid == 1);0 仅保留被明确判定无效的行;any 不做筛选(默认值;包含没有判定结果的旧数据行);strictV2 保留 power_valid == 1 且 power_metric_schema_version == 2(全部署能耗语义)的行——比 InferenceX 界面更严格,界面还会展示早于版本标注机制的已验证行。未知值返回 400 Unknown powerValid filter;不能与 view=calculator 组合使用(any 除外,等同于不筛选)。', + { type: 'string', enum: POWER_VALIDITY_FILTERS, default: 'any' }, + 'strictV2', + ), ], responses: [ success( @@ -655,8 +759,8 @@ export const apiOperations: readonly ApiOperation[] = [ ), errorResponse( '400', - 'The model is missing or unsupported.', - '模型缺失或不受支持。', + 'The model is missing or unsupported, the calculator sequence is unknown, the powerValid filter is unknown, or a non-any powerValid is combined with view=calculator.', + '模型缺失或不受支持、计算器序列未知、powerValid 筛选值未知,或非 any 的 powerValid 与 view=calculator 组合使用。', PUBLIC_API_ERRORS.unknownModel, ), errorResponse( @@ -2555,6 +2659,21 @@ const overview = { shape: 'BenchmarkRows', example: benchmarkExample[0], }, + { + id: 'measured-power', + title: text('Measured power', '实测功率'), + description: text( + 'Benchmark rows may carry measured power, energy, and GPU-telemetry metric keys (avg_power_w, joules_per_*, avg_temp_c, peak_temp_c, avg_util_pct, avg_mem_used_mb). power_valid is tri-state: 1 means the measurement window was validated; 0 means validation failed and measured values are withheld end-to-end (the producer strips them and ingest scrubs them — treat any that remain as unreliable); absent marks a legacy row predating validation. power_metric_schema_version == 2 defines every unprefixed joules_per_* field as whole-deployment energy — unversioned disaggregated joules are ambiguous because those fields previously carried role-local values. workers[] carries the per-worker power/telemetry breakdown on multinode and disaggregated runs. power_invalid_reasons and power_audit provide optional producer validation details. Filter rows with the powerValid request parameter; its strict value is named strictV2 (power_valid == 1 and power_metric_schema_version == 2) rather than "certified" because it is stricter than the InferenceX UI, which also displays validated rows that predate schema versioning.', + '基准行可能携带实测功率、能耗和 GPU 遥测指标键(avg_power_w、joules_per_*、avg_temp_c、peak_temp_c、avg_util_pct、avg_mem_used_mb)。power_valid 为三态:1 表示测量窗口已通过验证;0 表示验证失败,实测值会被全链路扣留(生产端剥离、摄取端再次清除——若仍残留请视为不可靠);缺失表示早于验证机制的旧数据行。power_metric_schema_version == 2 将所有无前缀的 joules_per_* 字段定义为全部署能耗——未标注版本的分离式 joules 含义不明确,因为这些字段曾承载角色本地值。多节点和分离式运行的每 worker 功率/遥测明细位于 workers[]。power_invalid_reasons 与 power_audit 可提供生产端的可选验证详情。可使用 powerValid 请求参数筛选行;其严格取值命名为 strictV2(power_valid == 1 且 power_metric_schema_version == 2)而非 "certified",因为它比 InferenceX 界面更严格——界面还会展示早于版本标注机制的已验证行。', + ), + shape: 'BenchmarkRows', + example: { + power_valid: 1, + power_metric_schema_version: 2, + avg_power_w: 678.5, + joules_per_output_token: 5.3, + }, + }, { id: 'metric-maps', title: text('ID-keyed maps', '以 ID 为键的映射'), diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index f93535dca..912518db8 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -108,7 +108,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-benchmarks', - sourceSha256: 'c6a5b78108b7e0d523b11590e1e34ef2e8c2d5457673eb41338d93d3d8f04909', + sourceSha256: 'b736b302cdb294bb316ff4666a64b47432655772f442aa8f5c95fe1f3cc9ed38', }, { source: 'src/app/api/v1/benchmarks/history/route.ts', @@ -443,7 +443,7 @@ export const stablePublicApiContracts = [ }, { operationId: 'list-benchmarks', - parameters: ['model', 'date', 'exact', 'runId', 'exactRun'], + parameters: ['model', 'date', 'exact', 'runId', 'exactRun', 'view', 'sequence', 'powerValid'], statuses: ['200', '400', '500'], auth: 'none', cachePolicy: 'public-db-day', diff --git a/packages/app/src/lib/benchmark-power-validity.test.ts b/packages/app/src/lib/benchmark-power-validity.test.ts new file mode 100644 index 000000000..d8b685726 --- /dev/null +++ b/packages/app/src/lib/benchmark-power-validity.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +import { + filterByPowerValidity, + parsePowerValidityFilter, + POWER_VALIDITY_FILTERS, +} from './benchmark-power-validity'; + +describe('parsePowerValidityFilter', () => { + it('treats an absent param as any', () => { + expect(parsePowerValidityFilter(null)).toBe('any'); + }); + + it('accepts every listed filter value', () => { + for (const filter of POWER_VALIDITY_FILTERS) { + expect(parsePowerValidityFilter(filter)).toBe(filter); + } + }); + + it('rejects unknown values', () => { + expect(parsePowerValidityFilter('garbage')).toBeUndefined(); + expect(parsePowerValidityFilter('')).toBeUndefined(); + // `certified` is a UI display tier, not an API filter value. + expect(parsePowerValidityFilter('certified')).toBeUndefined(); + }); + + it('is case-sensitive', () => { + expect(parsePowerValidityFilter('ANY')).toBeUndefined(); + expect(parsePowerValidityFilter('strictv2')).toBeUndefined(); + }); +}); + +describe('filterByPowerValidity', () => { + const validatedV2 = { id: 1, metrics: { power_valid: 1, power_metric_schema_version: 2 } }; + const validatedUnversioned = { id: 2, metrics: { power_valid: 1 } }; + const invalidated = { id: 3, metrics: { power_valid: 0, power_metric_schema_version: 2 } }; + const legacy = { id: 4, metrics: { tput_per_gpu: 100 } }; + const noMetrics = { id: 5 } as { id: number; metrics?: Record }; + const rows = [validatedV2, validatedUnversioned, invalidated, legacy, noMetrics]; + + it('returns every row unchanged for any', () => { + const result = filterByPowerValidity(rows, 'any'); + expect(result).toEqual(rows); + expect(result).not.toBe(rows); + }); + + it('keeps only explicitly validated rows for 1', () => { + expect(filterByPowerValidity(rows, '1')).toEqual([validatedV2, validatedUnversioned]); + }); + + it('keeps only explicitly invalidated rows for 0', () => { + expect(filterByPowerValidity(rows, '0')).toEqual([invalidated]); + }); + + it('requires a validated verdict and schema version 2 for strictV2', () => { + expect(filterByPowerValidity(rows, 'strictV2')).toEqual([validatedV2]); + }); + + it('excludes legacy and metric-less rows from every filter except any', () => { + for (const filter of ['1', '0', 'strictV2'] as const) { + const surviving = filterByPowerValidity([legacy, noMetrics], filter); + expect(surviving).toEqual([]); + } + }); +}); diff --git a/packages/app/src/lib/benchmark-power-validity.ts b/packages/app/src/lib/benchmark-power-validity.ts new file mode 100644 index 000000000..6b7d98ec6 --- /dev/null +++ b/packages/app/src/lib/benchmark-power-validity.ts @@ -0,0 +1,42 @@ +/** + * Measured-power validity filtering for the public benchmarks API. + * + * `metrics.power_valid` is tri-state: 1 means the measurement window was + * validated, an explicit 0 is an authoritative invalid verdict (measured + * values are withheld end-to-end), and an absent key marks a legacy row that + * predates validation. `strictV2` additionally requires + * `power_metric_schema_version === 2`, mirroring + * `WHOLE_DEPLOYMENT_ENERGY_SCHEMA_VERSION` in `benchmark-transform.ts` and + * `POWER_METRIC_SCHEMA_VERSION` in the runner's `utils/aggregate_power.py` — + * only version 2 defines unprefixed `joules_per_*` fields as whole-deployment + * energy. The name is deliberately not `certified`: the UI's certified tier is + * a display rule that also admits validated legacy rows without a schema + * version, which `strictV2` excludes. + */ +export const POWER_VALIDITY_FILTERS = ['1', '0', 'any', 'strictV2'] as const; +export type PowerValidityFilter = (typeof POWER_VALIDITY_FILTERS)[number]; + +/** Absent param means no filtering; unknown values return undefined so the caller can 400. */ +export function parsePowerValidityFilter(raw: string | null): PowerValidityFilter | undefined { + if (raw === null) return 'any'; + return (POWER_VALIDITY_FILTERS as readonly string[]).includes(raw) + ? (raw as PowerValidityFilter) + : undefined; +} + +/** + * Pure post-cache row filter. Rows without `metrics` or without a + * `power_valid` verdict (legacy rows) match only `any`. + */ +export function filterByPowerValidity }>( + rows: readonly T[], + filter: PowerValidityFilter, +): T[] { + if (filter === 'any') return [...rows]; + return rows.filter((row) => { + const powerValid = row.metrics?.power_valid; + if (filter === '1') return powerValid === 1; + if (filter === '0') return powerValid === 0; + return powerValid === 1 && row.metrics?.power_metric_schema_version === 2; + }); +} diff --git a/packages/constants/src/metric-keys.test.ts b/packages/constants/src/metric-keys.test.ts index fc606267a..1253ba7f0 100644 --- a/packages/constants/src/metric-keys.test.ts +++ b/packages/constants/src/metric-keys.test.ts @@ -4,6 +4,7 @@ import { MEASURED_POWER_METRIC_KEY_LIST, MEASURED_POWER_METRIC_KEYS, METRIC_KEYS, + POWER_METRIC_KEYS, } from './metric-keys'; describe('MEASURED_POWER_METRIC_KEYS', () => { @@ -35,7 +36,6 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { }); it('never contains the contract discriminators or invalid-verdict companion fields', () => { - // These fields describe or explain withholding; they are not measurements. for (const key of [ 'power_valid', 'power_metric_schema_version', @@ -46,3 +46,24 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { } }); }); + +describe('POWER_METRIC_KEYS', () => { + it('is a subset of METRIC_KEYS', () => { + for (const key of POWER_METRIC_KEYS) { + expect(METRIC_KEYS.has(key)).toBe(true); + } + }); + + it('has no duplicate keys', () => { + expect(new Set(POWER_METRIC_KEYS).size).toBe(POWER_METRIC_KEYS.length); + }); + + it('contains exactly the contract discriminators plus the 13 measured keys', () => { + // The public API documentation types every one of these keys on + // BenchmarkRow.metrics, so membership changes are contract changes. + expect(new Set(POWER_METRIC_KEYS)).toEqual( + new Set(['power_valid', 'power_metric_schema_version', ...MEASURED_POWER_METRIC_KEY_LIST]), + ); + expect(POWER_METRIC_KEYS).toHaveLength(15); + }); +}); diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 70cf69f6c..55e376c6a 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -45,6 +45,23 @@ export const MEASURED_POWER_METRIC_KEYS: ReadonlySet = new Set( MEASURED_POWER_METRIC_KEY_LIST, ); +/** + * Complete measured-power contract surface on `metrics`: the contract + * discriminators plus every measured power / energy / GPU-telemetry key. + * This is the set the public API documentation types on + * `BenchmarkRow.metrics`; it feeds METRIC_KEYS automatically. + */ +export const POWER_METRIC_KEYS = [ + // measured power / energy publication contract (aggregate_power.py) + // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power + // power_metric_schema_version: version 2 defines every unprefixed + // joules_per_* field as whole-deployment energy + 'power_valid', + 'power_metric_schema_version', + // measured power / energy / telemetry values, withheld when power_valid = 0 + ...MEASURED_POWER_METRIC_KEY_LIST, +] as const; + /** * Canonical set of metric keys stored in the benchmark_results.metrics JSONB column. * @@ -177,13 +194,7 @@ export const METRIC_KEYS = new Set([ // profiling window (agentic aiperf; flat in v2 artifacts, mapped from // server_metrics.kv_cache.gpu_usage_pct in v3) 'gpu_kv_cache_usage_pct', - // measured power / energy publication contract (aggregate_power.py) - // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power - // power_metric_schema_version: version 2 defines every unprefixed - // joules_per_* field as whole-deployment energy - 'power_valid', - 'power_metric_schema_version', - ...MEASURED_POWER_METRIC_KEY_LIST, + ...POWER_METRIC_KEYS, // extended parallelism dimensions (2026-07+ artifacts): pipeline parallelism // and decode/prefill context parallelism per role. These are config // dimensions, not measurements, but the configs table has no columns for