Skip to content
Open
11 changes: 9 additions & 2 deletions packages/app/cypress/e2e/api-documentation.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ describe('API documentation', () => {
.and('contain.text', 'Quickstart')
.and('contain.text', 'curl')
.and('contain.text', '/api/v1/availability')
.and('contain.text', 'Endpoint reference');
.and('contain.text', 'Endpoint reference')
.and('contain.text', 'Measured power')
.and('contain.text', 'powerValid');
cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json');
cy.get('[data-testid="api-spec-version"]').should('have.text', 'v1 · OpenAPI 3.1');
cy.get('[data-testid="api-endpoint-list-benchmarks"]')
Expand Down Expand Up @@ -45,6 +47,10 @@ describe('API documentation', () => {
'operationId',
'list-benchmarks',
);
const benchmarkParameterNames = body.paths['/api/v1/benchmarks'].get.parameters.map(
(parameter: { name: string }) => parameter.name,
);
expect(benchmarkParameterNames).to.include('powerValid');
expect(body.paths['/api/v1/collectivex/runs/{runId}'].get).to.have.property(
'operationId',
'get-collectivex-run',
Expand All @@ -58,7 +64,8 @@ describe('API documentation', () => {
.and('contain.text', '快速入门')
.and('contain.text', '约定')
.and('contain.text', '端点参考')
.and('contain.text', 'BenchmarkRow 与指标');
.and('contain.text', 'BenchmarkRow 与指标')
.and('contain.text', '实测功率');
cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json');
cy.get('link[rel="alternate"][hreflang="en"]').should('have.attr', 'href', `${SITE_URL}/api`);
cy.get('link[rel="alternate"][hreflang="zh-CN"]').should(
Expand Down
144 changes: 144 additions & 0 deletions packages/app/src/app/api/v1/benchmarks/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,4 +242,148 @@ describe('GET /api/v1/benchmarks', () => {
const body = await res.json();
expect(body).toEqual([]);
});

describe('powerValid filter', () => {
const validatedV2 = {
id: 1,
benchmark_type: 'single_turn',
metrics: { power_valid: 1, power_metric_schema_version: 2, avg_power_w: 700 },
};
const validatedUnversioned = {
id: 2,
benchmark_type: 'single_turn',
metrics: { power_valid: 1, avg_power_w: 650 },
};
const invalidated = {
id: 3,
benchmark_type: 'single_turn',
metrics: { power_valid: 0 },
};
const legacy = {
id: 4,
benchmark_type: 'single_turn',
metrics: { tput_per_gpu: 100 },
};
const powerRows = [validatedV2, validatedUnversioned, invalidated, legacy];

it('powerValid=1 keeps only rows with a validated verdict', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows);

const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1'));
expect(res.status).toBe(200);
expect(await res.json()).toEqual([validatedV2, validatedUnversioned]);
});

it('powerValid=0 keeps only explicitly invalidated rows', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows);

const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=0'));
expect(res.status).toBe(200);
expect(await res.json()).toEqual([invalidated]);
});

it('powerValid=strictV2 requires a validated verdict plus schema version 2', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows);

const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2'));
expect(res.status).toBe(200);
expect(await res.json()).toEqual([validatedV2]);
});

it('powerValid=any matches the response with the param absent (backward compat)', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows);
const withParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any'));

mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows);
const withoutParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528'));

expect(withParam.status).toBe(200);
expect(withoutParam.status).toBe(200);
const bodyWithParam = await withParam.json();
expect(bodyWithParam).toEqual(await withoutParam.json());
expect(bodyWithParam).toEqual(powerRows);
});

it('rejects an unknown powerValid value without querying', async () => {
const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=garbage'));

expect(res.status).toBe(400);
expect(await res.json()).toEqual({ error: 'Unknown powerValid filter' });
expect(mockGetLatestBenchmarks).not.toHaveBeenCalled();
});

it('rejects powerValid combined with view=calculator without querying', async () => {
const res = await GET(
req(
'/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1&view=calculator&sequence=1k%2F1k',
),
);

expect(res.status).toBe(400);
expect(await res.json()).toEqual({
error: 'powerValid cannot be combined with view=calculator',
});
expect(mockGetLatestBenchmarks).not.toHaveBeenCalled();
});

it('allows powerValid=any with view=calculator (no-op filter)', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce([
{
benchmark_type: 'single_turn',
isl: 1024,
osl: 1024,
metrics: { tput_per_gpu: 100, avg_power_w: 700 },
},
]);

const res = await GET(
req(
'/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any&view=calculator&sequence=1k%2F1k',
),
);

expect(res.status).toBe(200);
expect(await res.json()).toEqual([
{ benchmark_type: 'single_turn', isl: 1024, osl: 1024, metrics: { tput_per_gpu: 100 } },
]);
});

it('composes with the agentic workflow-metadata trim', async () => {
mockGetLatestBenchmarks.mockResolvedValueOnce([
{
id: 1,
benchmark_type: 'agentic_traces',
workflow_run_id: 42,
run_started_at: '2026-08-12T10:00:00Z',
metrics: { power_valid: 1 },
},
{
id: 2,
benchmark_type: 'single_turn',
workflow_run_id: 43,
run_started_at: '2026-08-12T10:00:00Z',
metrics: { power_valid: 1 },
},
{
id: 3,
benchmark_type: 'agentic_traces',
workflow_run_id: 44,
run_started_at: '2026-08-12T10:00:00Z',
metrics: { power_valid: 0 },
},
]);

const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1'));
expect(await res.json()).toEqual([
{
id: 1,
benchmark_type: 'agentic_traces',
workflow_run_id: 42,
run_started_at: '2026-08-12T10:00:00Z',
metrics: { power_valid: 1 },
},
{ id: 2, benchmark_type: 'single_turn', metrics: { power_valid: 1 } },
]);
});
});
});
23 changes: 21 additions & 2 deletions packages/app/src/app/api/v1/benchmarks/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {

import { cachedJson, cachedQuery } from '@/lib/api-cache';
import { toCalculatorBenchmarkRows } from '@/lib/benchmark-api-view';
import { filterByPowerValidity, parsePowerValidityFilter } from '@/lib/benchmark-power-validity';
import { PUBLIC_API_ERRORS, publicApiError } from '@/lib/public-api-errors';
import { agenticWorkflowMetadataOnly } from '@/lib/agentic-workflow-metadata';
import { loadFixture } from '@/lib/test-fixtures';
Expand Down Expand Up @@ -52,17 +53,31 @@ export async function GET(request: NextRequest) {
const exactRun = params.get('exactRun') === 'true';
const view = params.get('view');
const sequence = params.get('sequence') ?? '';
const powerValidFilter = parsePowerValidityFilter(params.get('powerValid'));
const dbModelKeys = DISPLAY_MODEL_TO_DB[model];
if (!dbModelKeys || dbModelKeys.length === 0) {
return publicApiError(PUBLIC_API_ERRORS.unknownModel, 400);
}
if (view === 'calculator' && !['1k/1k', '1k/8k', '8k/1k', 'agentic-traces'].includes(sequence)) {
return NextResponse.json({ error: 'Unknown calculator sequence' }, { status: 400 });
}
if (powerValidFilter === undefined) {
return NextResponse.json({ error: 'Unknown powerValid filter' }, { status: 400 });
}
// The calculator cache stores rows already trimmed to an allowlist that
// excludes power_valid, so post-cache filtering cannot work there.
if (view === 'calculator' && powerValidFilter !== 'any') {
return NextResponse.json(
{ error: 'powerValid cannot be combined with view=calculator' },
{ status: 400 },
);
}
if (FIXTURES_MODE) {
const fixture = loadFixture<BenchmarkRow[]>('benchmarks');
return cachedJson(
view === 'calculator' ? toCalculatorBenchmarkRows(fixture, sequence) : fixture,
view === 'calculator'
? toCalculatorBenchmarkRows(fixture, sequence)
: filterByPowerValidity(fixture, powerValidFilter),
);
}

Expand All @@ -73,7 +88,11 @@ export async function GET(request: NextRequest) {
: exactRun && runId
? await getCachedBenchmarksForRun(dbModelKeys, runId)
: await getCachedBenchmarks(dbModelKeys, date, exact || undefined, runId);
return cachedJson(agenticWorkflowMetadataOnly(rows));
return cachedJson(
agenticWorkflowMetadataOnly(
view === 'calculator' ? rows : filterByPowerValidity(rows, powerValidFilter),
),
);
} catch (error) {
console.error('Error fetching benchmarks:', error);
return publicApiError(PUBLIC_API_ERRORS.internal, 500);
Expand Down
83 changes: 83 additions & 0 deletions packages/app/src/lib/api-documentation.power.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import { POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants';
import { describe, expect, it } from 'vitest';

import { apiOperations, buildOpenApiDocument, getApiDocumentation } from './api-documentation';
import { POWER_VALIDITY_FILTERS } from './benchmark-power-validity';

const listBenchmarks = apiOperations.find((operation) => operation.id === 'list-benchmarks');
const benchmarkRowSchema = listBenchmarks?.responses.find((response) => response.status === '200')
?.schema.items;

describe('measured-power API documentation', () => {
it('types every power metric key in the benchmarks metrics schema', () => {
const metricsSchema = benchmarkRowSchema?.properties?.metrics;
expect(metricsSchema).toBeDefined();
expect(metricsSchema?.additionalProperties).toEqual({ type: 'number' });
for (const key of POWER_METRIC_KEYS) {
const property = metricsSchema?.properties?.[key];
expect(property?.type, `${key} must be a typed number property`).toBe('number');
expect(
property?.description?.trim(),
`${key} must carry a nonempty description`,
).toBeTruthy();
}
});

it('reserves optional power_invalid_reasons and power_audit row fields', () => {
const reasons = benchmarkRowSchema?.properties?.power_invalid_reasons;
expect(reasons?.type).toBe('array');
expect(reasons?.items).toEqual({ type: 'string' });

const audit = benchmarkRowSchema?.properties?.power_audit;
expect(Object.keys(audit?.properties ?? {}).toSorted()).toEqual(
[
'window_start_unix',
'window_end_unix',
'expected_gpu_count',
'observed_gpu_count',
'sample_count',
'max_sample_gap_s',
'producer_sha',
'exporter_image_sha256',
].toSorted(),
);
// Producers may emit partial audits, so individual audit fields remain optional.
expect(audit?.required).toBeUndefined();

expect(benchmarkRowSchema?.required).not.toContain('power_invalid_reasons');
expect(benchmarkRowSchema?.required).not.toContain('power_audit');
});

it('projects the view, sequence, and powerValid parameters into OpenAPI', () => {
const document = buildOpenApiDocument('https://api-docs.test');
const operation = (document.paths['/api/v1/benchmarks'] as Record<string, unknown>).get as {
parameters: readonly { name: string; schema: unknown }[];
};
const names = operation.parameters.map((parameter) => parameter.name);
expect(names).toContain('view');
expect(names).toContain('sequence');

const powerValid = operation.parameters.find((parameter) => parameter.name === 'powerValid');
expect(powerValid?.schema).toEqual({
type: 'string',
enum: ['1', '0', 'any', 'strictV2'],
default: 'any',
});
expect([...POWER_VALIDITY_FILTERS]).toEqual(['1', '0', 'any', 'strictV2']);
});

it('renders a bilingual measured-power schema note', () => {
for (const locale of ['en', 'zh'] as const) {
const note = getApiDocumentation(locale).schemaNotes.find(
(candidate) => candidate.id === 'measured-power',
);
expect(note, `${locale} must expose a measured-power schema note`).toBeDefined();
expect(note?.title.trim()).toBeTruthy();
expect(note?.description.trim()).toBeTruthy();
}
const zhNote = getApiDocumentation('zh').schemaNotes.find(
(candidate) => candidate.id === 'measured-power',
);
expect(zhNote?.description).toMatch(/[㐀-鿿]/u);
});
});
Loading