diff --git a/packages/app/cypress/e2e/api-documentation.cy.ts b/packages/app/cypress/e2e/api-documentation.cy.ts index f2ccdeb23..919e13231 100644 --- a/packages/app/cypress/e2e/api-documentation.cy.ts +++ b/packages/app/cypress/e2e/api-documentation.cy.ts @@ -1,3 +1,5 @@ +import type { BenchmarkRow } from '@semianalysisai/inferencex-db/queries/benchmarks'; + const SITE_URL = 'https://inferencex.semianalysis.com'; describe('API documentation', () => { @@ -33,7 +35,9 @@ describe('API documentation', () => { .and('contain.text', 'Quickstart') .and('contain.text', 'curl') .and('contain.text', '/api/v1/availability') - .and('contain.text', 'Endpoint reference'); + .and('contain.text', 'Endpoint reference') + .and('contain.text', 'Measured power') + .and('contain.text', 'powerValid=strictV2'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('[data-testid="api-spec-version"]').should('have.text', 'v1 · OpenAPI 3.1'); cy.get('[data-testid="api-endpoint-list-benchmarks"]') @@ -69,6 +73,11 @@ describe('API documentation', () => { 'operationId', 'list-benchmarks', ); + const powerValid = body.paths['/api/v1/benchmarks'].get.parameters.find( + (parameter: { name: string }) => parameter.name === 'powerValid', + ); + expect(powerValid.required).to.equal(false); + expect(powerValid.schema).to.deep.equal({ type: 'string', enum: ['strictV2'] }); expect(body.paths['/api/v1/collectivex/runs/{runId}'].get).to.have.property( 'operationId', 'get-collectivex-run', @@ -82,7 +91,9 @@ describe('API documentation', () => { .and('contain.text', '快速入门') .and('contain.text', '约定') .and('contain.text', '端点参考') - .and('contain.text', 'BenchmarkRow 与指标'); + .and('contain.text', 'BenchmarkRow 与指标') + .and('contain.text', '实测功率') + .and('contain.text', 'powerValid=strictV2'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('link[rel="alternate"][hreflang="en"]').should('have.attr', 'href', `${SITE_URL}/api`); cy.get('link[rel="alternate"][hreflang="zh-CN"]').should( @@ -122,4 +133,43 @@ describe('API documentation', () => { ); }); }); + + it('supports strictV2 power requests while preserving ordinary benchmark requests', () => { + const url = '/api/v1/benchmarks?model=DeepSeek-R1-0528'; + cy.request(url).then(({ body, status }) => { + expect(status).to.equal(200); + expect(body).to.be.an('array'); + expect(body.length).to.be.greaterThan(0); + const expected = body.filter( + (row) => row.metrics.power_valid === 1 && row.metrics.power_metric_schema_version === 2, + ); + cy.request(`${url}&powerValid=strictV2`).then((response) => { + expect(response.status).to.equal(200); + expect(response.body).to.deep.equal(expected); + }); + }); + + for (const value of ['1', '0', 'any', 'certified', '']) { + cy.request({ url: `${url}&powerValid=${value}`, failOnStatusCode: false }).then( + ({ body, status }) => { + expect(status).to.equal(400); + expect(body).to.deep.equal({ error: 'Unknown powerValid filter' }); + }, + ); + } + + const calculatorUrl = `${url}&view=calculator&sequence=8k%2F1k`; + cy.request(calculatorUrl).then(({ body, status }) => { + expect(status).to.equal(200); + expect(body).to.be.an('array'); + expect(body.length).to.be.greaterThan(0); + }); + cy.request({ + url: `${calculatorUrl}&powerValid=strictV2`, + failOnStatusCode: false, + }).then(({ body, status }) => { + expect(status).to.equal(400); + expect(body).to.deep.equal({ error: 'powerValid cannot be combined with view=calculator' }); + }); + }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/route.test.ts b/packages/app/src/app/api/v1/benchmarks/route.test.ts index 629339016..fc35be4b6 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.test.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.test.ts @@ -242,4 +242,130 @@ describe('GET /api/v1/benchmarks', () => { const body = await res.json(); expect(body).toEqual([]); }); + + describe('powerValid filter', () => { + const validatedV2 = { + id: 1, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, power_metric_schema_version: 2, avg_power_w: 700 }, + }; + const validatedUnversioned = { + id: 2, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, avg_power_w: 650 }, + }; + const invalidated = { + id: 3, + benchmark_type: 'single_turn', + metrics: { power_valid: 0 }, + }; + const legacy = { + id: 4, + benchmark_type: 'single_turn', + metrics: { tput_per_gpu: 100 }, + }; + const powerRows = [validatedV2, validatedUnversioned, invalidated, legacy]; + + it('powerValid=strictV2 requires a validated verdict plus schema version 2', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([validatedV2]); + }); + + it('keeps every general benchmark row when powerValid is omitted', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual(powerRows); + }); + + it.each(['1', '0', 'any', 'certified', 'garbage', '', 'strictv2'])( + 'rejects unsupported powerValid=%s without querying', + async (value) => { + const res = await GET(req(`/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=${value}`)); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Unknown powerValid filter' }); + expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); + expect(mockGetBenchmarksForRun).not.toHaveBeenCalled(); + }, + ); + + it('rejects powerValid combined with view=calculator without querying', async () => { + const res = await GET( + req( + '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2&view=calculator&sequence=1k%2F1k', + ), + ); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: 'powerValid cannot be combined with view=calculator', + }); + expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); + }); + + it('keeps calculator requests working when powerValid is omitted', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + benchmark_type: 'single_turn', + isl: 1024, + osl: 1024, + metrics: { tput_per_gpu: 100, avg_power_w: 700 }, + }, + ]); + + const res = await GET( + req('/api/v1/benchmarks?model=DeepSeek-R1-0528&view=calculator&sequence=1k%2F1k'), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual([ + { benchmark_type: 'single_turn', isl: 1024, osl: 1024, metrics: { tput_per_gpu: 100 } }, + ]); + }); + + it('composes with the agentic workflow-metadata trim', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1, power_metric_schema_version: 2 }, + }, + { + id: 2, + benchmark_type: 'single_turn', + workflow_run_id: 43, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1, power_metric_schema_version: 2 }, + }, + { + id: 3, + benchmark_type: 'agentic_traces', + workflow_run_id: 44, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 0 }, + }, + ]); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2')); + expect(await res.json()).toEqual([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1, power_metric_schema_version: 2 }, + }, + { + id: 2, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, power_metric_schema_version: 2 }, + }, + ]); + }); + }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/route.ts b/packages/app/src/app/api/v1/benchmarks/route.ts index a75260d7b..d69efe5c7 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.ts @@ -11,6 +11,7 @@ import { import { cachedJson, cachedQuery } from '@/lib/api-cache'; import { toCalculatorBenchmarkRows } from '@/lib/benchmark-api-view'; +import { filterByPowerValidity, parsePowerValidityFilter } from '@/lib/benchmark-power-validity'; import { PUBLIC_API_ERRORS, publicApiError } from '@/lib/public-api-errors'; import { agenticWorkflowMetadataOnly } from '@/lib/agentic-workflow-metadata'; import { loadFixture } from '@/lib/test-fixtures'; @@ -52,6 +53,7 @@ export async function GET(request: NextRequest) { const exactRun = params.get('exactRun') === 'true'; const view = params.get('view'); const sequence = params.get('sequence') ?? ''; + const powerValidFilter = parsePowerValidityFilter(params.get('powerValid')); const dbModelKeys = DISPLAY_MODEL_TO_DB[model]; if (!dbModelKeys || dbModelKeys.length === 0) { return publicApiError(PUBLIC_API_ERRORS.unknownModel, 400); @@ -59,10 +61,23 @@ export async function GET(request: NextRequest) { if (view === 'calculator' && !['1k/1k', '1k/8k', '8k/1k', 'agentic-traces'].includes(sequence)) { return NextResponse.json({ error: 'Unknown calculator sequence' }, { status: 400 }); } + if (powerValidFilter === undefined) { + return NextResponse.json({ error: 'Unknown powerValid filter' }, { status: 400 }); + } + // The calculator cache stores rows already trimmed to an allowlist that + // excludes power_valid, so post-cache filtering cannot work there. + if (view === 'calculator' && powerValidFilter !== null) { + return NextResponse.json( + { error: 'powerValid cannot be combined with view=calculator' }, + { status: 400 }, + ); + } if (FIXTURES_MODE) { const fixture = loadFixture('benchmarks'); return cachedJson( - view === 'calculator' ? toCalculatorBenchmarkRows(fixture, sequence) : fixture, + view === 'calculator' + ? toCalculatorBenchmarkRows(fixture, sequence) + : filterByPowerValidity(fixture, powerValidFilter), ); } @@ -73,7 +88,11 @@ export async function GET(request: NextRequest) { : exactRun && runId ? await getCachedBenchmarksForRun(dbModelKeys, runId) : await getCachedBenchmarks(dbModelKeys, date, exact || undefined, runId); - return cachedJson(agenticWorkflowMetadataOnly(rows)); + return cachedJson( + agenticWorkflowMetadataOnly( + view === 'calculator' ? rows : filterByPowerValidity(rows, powerValidFilter), + ), + ); } catch (error) { console.error('Error fetching benchmarks:', error); return publicApiError(PUBLIC_API_ERRORS.internal, 500); diff --git a/packages/app/src/lib/api-documentation.power.test.ts b/packages/app/src/lib/api-documentation.power.test.ts new file mode 100644 index 000000000..d5d80bdd0 --- /dev/null +++ b/packages/app/src/lib/api-documentation.power.test.ts @@ -0,0 +1,89 @@ +import { POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; +import { describe, expect, it } from 'vitest'; + +import { apiOperations, buildOpenApiDocument, getApiDocumentation } from './api-documentation'; +import { POWER_VALIDITY_FILTERS } from './benchmark-power-validity'; + +const listBenchmarks = apiOperations.find((operation) => operation.id === 'list-benchmarks'); +const benchmarkRowSchema = listBenchmarks?.responses.find((response) => response.status === '200') + ?.schema.items; + +describe('measured-power API documentation', () => { + it('types every power metric key in the benchmarks metrics schema', () => { + const metricsSchema = benchmarkRowSchema?.properties?.metrics; + expect(metricsSchema).toBeDefined(); + expect(metricsSchema?.additionalProperties).toEqual({ type: 'number' }); + for (const key of POWER_METRIC_KEYS) { + const property = metricsSchema?.properties?.[key]; + expect(property?.type, `${key} must be a typed number property`).toBe('number'); + expect( + property?.description?.trim(), + `${key} must carry a nonempty description`, + ).toBeTruthy(); + } + }); + + it('reserves optional power_invalid_reasons and power_audit row fields', () => { + const reasons = benchmarkRowSchema?.properties?.power_invalid_reasons; + expect(reasons?.type).toBe('array'); + expect(reasons?.items).toEqual({ type: 'string' }); + + const audit = benchmarkRowSchema?.properties?.power_audit; + expect(Object.keys(audit?.properties ?? {}).toSorted()).toEqual( + [ + 'window_start_unix', + 'window_end_unix', + 'expected_gpu_count', + 'observed_gpu_count', + 'sample_count', + 'max_sample_gap_s', + 'producer_sha', + 'exporter_image_sha256', + ].toSorted(), + ); + // Producers may emit partial audits, so individual audit fields remain optional. + expect(audit?.required).toBeUndefined(); + + expect(benchmarkRowSchema?.required).not.toContain('power_invalid_reasons'); + expect(benchmarkRowSchema?.required).not.toContain('power_audit'); + }); + + it('documents strictV2 as the only optional power filter without changing ordinary requests', () => { + const document = buildOpenApiDocument('https://api-docs.test'); + const operation = (document.paths['/api/v1/benchmarks'] as Record).get as { + parameters: readonly { name: string; required: boolean; schema: unknown }[]; + }; + const names = operation.parameters.map((parameter) => parameter.name); + expect(names).toContain('view'); + expect(names).toContain('sequence'); + + const powerValid = operation.parameters.find((parameter) => parameter.name === 'powerValid'); + expect(powerValid?.schema).toEqual({ + type: 'string', + enum: ['strictV2'], + }); + expect(powerValid?.required).toBe(false); + expect([...POWER_VALIDITY_FILTERS]).toEqual(['strictV2']); + expect(listBenchmarks?.curlUrl).toBe( + 'https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-R1-0528', + ); + }); + + it('renders a bilingual measured-power schema note', () => { + for (const locale of ['en', 'zh'] as const) { + const note = getApiDocumentation(locale).schemaNotes.find( + (candidate) => candidate.id === 'measured-power', + ); + expect(note, `${locale} must expose a measured-power schema note`).toBeDefined(); + expect(note?.title.trim()).toBeTruthy(); + expect(note?.description.trim()).toBeTruthy(); + expect(note?.description).toContain('powerValid=strictV2'); + expect(note?.description).toContain('power_valid == 1'); + expect(note?.description).toContain('power_metric_schema_version == 2'); + } + const zhNote = getApiDocumentation('zh').schemaNotes.find( + (candidate) => candidate.id === 'measured-power', + ); + expect(zhNote?.description).toMatch(/[㐀-鿿]/u); + }); +}); diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 77afc3334..21386dc70 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -1,6 +1,11 @@ -import { DB_MODEL_TO_DISPLAY, DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; +import { + DB_MODEL_TO_DISPLAY, + DISPLAY_MODEL_TO_DB, + POWER_METRIC_KEYS, +} from '@semianalysisai/inferencex-constants'; import { COLLECTIVEX_VERSIONS } from '@semianalysisai/inferencex-db/collectivex/types'; +import { POWER_VALIDITY_FILTERS } from './benchmark-power-validity'; import { PUBLIC_API_ERRORS } from './public-api-errors'; export type ApiDocumentationLocale = 'en' | 'zh'; @@ -192,6 +197,59 @@ const errorResponse = ( mediaType: 'application/json', }); +const powerMetricDescriptions: Readonly> = { + power_valid: + 'Publication verdict: 1 = validated measurement window; 0 = failed validation — measured power/energy values are withheld from this row end-to-end, so treat any that remain as unreliable; absent = legacy row predating validation.', + power_metric_schema_version: + 'Power schema version. Version 2 defines every unprefixed joules_per_* field as whole-deployment energy, including on disaggregated runs.', + avg_power_w: 'Mean per-GPU power draw in watts during the measured load window.', + joules_per_successful_query: 'Whole-deployment energy in joules divided by successful requests.', + joules_per_output_token: + 'Energy per generated output token in joules; cluster-wide on schema-version-2 rows, including disaggregated runs.', + joules_per_total_token: + 'Total system energy divided by input plus output tokens; a workload-shape-fair view that does not treat prompt tokens as free.', + prefill_avg_power_w: + 'Mean per-GPU power draw in watts across prefill workers; emitted only for deployments with distinct prefill and decode roles.', + decode_avg_power_w: + 'Mean per-GPU power draw in watts across decode workers; emitted only for deployments with distinct prefill and decode roles.', + joules_per_input_token: + 'Energy per input token in joules; cluster-wide on schema-version-2 rows.', + prefill_joules_per_input_token: 'Role-local prefill energy per input token in joules.', + decode_joules_per_output_token: 'Role-local decode energy per generated output token in joules.', + avg_temp_c: 'Mean per-GPU temperature in degrees Celsius during the load window.', + peak_temp_c: + 'Maximum instantaneous per-GPU temperature in degrees Celsius during the load window.', + avg_util_pct: 'Mean per-GPU utilization percentage (0-100) during the load window.', + avg_mem_used_mb: 'Mean per-GPU memory used in MB during the load window.', +}; +const benchmarkMetricsSchema: ApiSchema = { + type: 'object', + additionalProperties: numberSchema, + description: + 'Scalar metric map. Keys evolve independently; measured power / energy / GPU-telemetry keys are typed below.', + properties: Object.fromEntries( + POWER_METRIC_KEYS.map((key): [string, ApiSchema] => [ + key, + { type: 'number', description: powerMetricDescriptions[key] }, + ]), + ), +}; +const powerAuditSchema: ApiSchema = { + type: 'object', + properties: { + window_start_unix: numberSchema, + window_end_unix: numberSchema, + expected_gpu_count: integerSchema, + observed_gpu_count: integerSchema, + sample_count: integerSchema, + max_sample_gap_s: numberSchema, + producer_sha: nullableStringSchema, + exporter_image_sha256: nullableStringSchema, + }, + additionalProperties: true, + description: + 'Optional power measurement-window audit. Individual fields may be absent; legacy rows omit the object.', +}; const workerPowerSchema = objectSchemaWithOptional( { role: stringSchema, @@ -234,14 +292,20 @@ const benchmarkRowSchema = objectSchemaWithOptional( offload_mode: stringSchema, image: nullableStringSchema, recipe_fingerprint: nullableStringSchema, - metrics: metricMapSchema, + metrics: benchmarkMetricsSchema, workers: arraySchema(workerPowerSchema), + power_invalid_reasons: { + ...arraySchema(stringSchema), + description: + 'Optional snake_case validation reason codes when metrics.power_valid == 0. Absent on legacy rows.', + }, + power_audit: powerAuditSchema, date: { type: 'string', format: 'date' }, workflow_run_id: integerSchema, run_started_at: { type: ['string', 'null'], format: 'date-time' }, run_url: nullableStringSchema, }, - ['workers', 'workflow_run_id', 'run_started_at'], + ['workers', 'power_invalid_reasons', 'power_audit', 'workflow_run_id', 'run_started_at'], ); const benchmarkRowsSchema = arraySchema(benchmarkRowSchema); const benchmarkExample = [ @@ -271,7 +335,17 @@ const benchmarkExample = [ offload_mode: 'off', image: 'vllm/vllm-openai:v0.10.2', recipe_fingerprint: '7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d', - metrics: { median_ttft: 0.42, median_tpot: 0.018, tput_per_gpu: 128.4 }, + metrics: { + median_ttft: 0.42, + median_tpot: 0.018, + tput_per_gpu: 128.4, + power_valid: 1, + power_metric_schema_version: 2, + avg_power_w: 678.5, + joules_per_output_token: 5.3, + joules_per_total_token: 2.65, + avg_temp_c: 61.2, + }, date: '2026-08-08', run_url: 'https://github.com/semianalysis/inference-benchmarks/actions/runs/123456789', }, @@ -590,8 +664,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/benchmarks', summary: text('Read benchmark results', '读取基准结果'), description: text( - 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. The page-owned calculator view is not part of this public contract.', - '返回指定展示模型的原始基准测试数据行。使用 date 可获取截至指定日期的快照；exact=true 仅返回该日期的数据；runId 用于限定最新结果的查询范围；将 exactRun=true 与数值型 runId 搭配使用，则只返回该工作流运行的数据。页面内部使用的 calculator 视图不属于此公开契约。', + 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change). powerValid=strictV2 selects validated schema-v2 power measurements and cannot be combined with view=calculator. Omit powerValid to keep general benchmark results regardless of power validity.', + '返回指定展示模型的原始基准测试数据行。使用 date 可获取截至指定日期的快照；exact=true 仅返回该日期的数据；runId 用于限定最新结果的查询范围；将 exactRun=true 与数值型 runId 搭配使用，则只返回该工作流运行的数据。view=calculator 返回页面专用的裁剪投影（会移除实测功率指标和 workers，其允许列表可能变化）。powerValid=strictV2 仅返回采用 schema v2 且功率测量通过验证的数据行，不能与 view=calculator 组合使用。省略 powerValid 则保留常规基准测试结果，不按功率有效性筛选。', ), audience: 'public', stability: 'stable', @@ -646,6 +720,36 @@ export const apiOperations: readonly ApiOperation[] = [ { type: 'boolean', default: false }, false, ), + parameter( + 'view', + 'query', + false, + 'enum', + 'calculator trims each row to the page-owned metric allowlist the throughput calculator consumes and removes workers; measured power metrics are excluded from this view. Requires sequence. Omit for every stored metric, including measured power.', + 'calculator 会将每行裁剪为吞吐量计算器所需的页面专用指标允许列表并移除 workers；此视图不包含实测功率指标。需要同时提供 sequence。省略则返回全部已存储指标，包括实测功率。', + { type: 'string', enum: ['calculator'] }, + 'calculator', + ), + parameter( + 'sequence', + 'query', + false, + 'enum', + 'Required when view=calculator and ignored otherwise. Unknown values yield 400 Unknown calculator sequence.', + '当 view=calculator 时必填，其余情况会被忽略。未知值返回 400 Unknown calculator sequence。', + { type: 'string', enum: ['1k/1k', '1k/8k', '8k/1k', 'agentic-traces'] }, + '1k/1k', + ), + parameter( + 'powerValid', + 'query', + false, + 'enum', + 'Only strictV2 is accepted. It keeps rows whose metrics.power_valid is the number 1 and metrics.power_metric_schema_version is the number 2 (whole-deployment energy semantics). Omit this parameter to apply no power filter, preserving throughput and latency results even when power is missing or invalid. Other values, including an empty value, yield 400 Unknown powerValid filter. Cannot be combined with view=calculator.', + '仅接受 strictV2：保留 metrics.power_valid 为数字 1、且 metrics.power_metric_schema_version 为数字 2 的数据行，其能耗指标采用整个部署的统计口径。省略此参数则不按功率筛选，即使功率缺失或无效，也会保留吞吐量和延迟结果。其他取值（包括空值）返回 400 Unknown powerValid filter。不能与 view=calculator 组合使用。', + { type: 'string', enum: POWER_VALIDITY_FILTERS }, + 'strictV2', + ), ], responses: [ success( @@ -656,8 +760,8 @@ export const apiOperations: readonly ApiOperation[] = [ ), errorResponse( '400', - 'The model is missing or unsupported.', - '模型缺失或不受支持。', + 'The model is missing or unsupported, the calculator sequence is unknown, a supplied powerValid value is not strictV2, or powerValid=strictV2 is combined with view=calculator.', + '模型缺失或不受支持、计算器序列未知、提供的 powerValid 取值不是 strictV2，或 powerValid=strictV2 与 view=calculator 组合使用。', PUBLIC_API_ERRORS.unknownModel, ), errorResponse( @@ -2570,6 +2674,21 @@ const overview = { shape: 'BenchmarkRows', example: benchmarkExample[0], }, + { + id: 'measured-power', + title: text('Measured power', '实测功率'), + description: text( + 'Benchmark rows may carry measured power, energy, and GPU-telemetry metric keys (avg_power_w, joules_per_*, avg_temp_c, peak_temp_c, avg_util_pct, avg_mem_used_mb). power_valid is tri-state: 1 means the measurement window was validated; 0 means validation failed and measured values are withheld end-to-end (the producer strips them and ingest scrubs them — treat any that remain as unreliable); absent marks a legacy row predating validation. power_metric_schema_version == 2 defines every unprefixed joules_per_* field as whole-deployment energy — unversioned disaggregated joules are ambiguous because those fields previously carried role-local values. workers[] carries the per-worker power/telemetry breakdown on multinode and disaggregated runs. power_invalid_reasons and power_audit provide optional producer validation details. For measured-power requests, use powerValid=strictV2 to require power_valid == 1 and power_metric_schema_version == 2. It is the only supported power filter. Omit powerValid for general benchmark requests so results remain available even when they lack valid power measurements.', + '基准测试数据行可能包含实测功率、能耗和 GPU 遥测指标（avg_power_w、joules_per_*、avg_temp_c、peak_temp_c、avg_util_pct、avg_mem_used_mb）。power_valid 有三种状态：1 表示测量窗口已通过验证；0 表示验证失败，生产端会移除实测值，摄取端也会再次清除，若仍有残留，应视为不可靠；缺失表示该数据行早于验证机制。power_metric_schema_version == 2 规定所有无前缀的 joules_per_* 字段均按整个部署统计能耗。未标注版本的分离式部署数据中，这些字段曾记录单个角色的能耗，因此其统计口径不明确。多节点和分离式运行中，各 worker 的功率和遥测明细位于 workers[]。power_invalid_reasons 与 power_audit 可包含生产端的验证详情。查询实测功率时，使用 powerValid=strictV2，仅保留 power_valid == 1 且 power_metric_schema_version == 2 的行。这是唯一支持的功率筛选值。常规基准测试请求应省略 powerValid，以保留缺少有效功率测量的结果。', + ), + shape: 'BenchmarkRows', + example: { + power_valid: 1, + power_metric_schema_version: 2, + avg_power_w: 678.5, + joules_per_output_token: 5.3, + }, + }, { id: 'metric-maps', title: text('ID-keyed maps', '以 ID 为键的映射'), diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index 7ef0e6297..e01c8e5a7 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -108,7 +108,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-benchmarks', - sourceSha256: 'c6a5b78108b7e0d523b11590e1e34ef2e8c2d5457673eb41338d93d3d8f04909', + sourceSha256: '7b251598bf9e4e181834311a554aa8ef7a9bc39d605eed243364ce5c3f5cc43e', }, { source: 'src/app/api/v1/benchmarks/history/route.ts', @@ -443,7 +443,7 @@ export const stablePublicApiContracts = [ }, { operationId: 'list-benchmarks', - parameters: ['model', 'date', 'exact', 'runId', 'exactRun'], + parameters: ['model', 'date', 'exact', 'runId', 'exactRun', 'view', 'sequence', 'powerValid'], statuses: ['200', '400', '500'], auth: 'none', cachePolicy: 'public-db-day', diff --git a/packages/app/src/lib/benchmark-power-validity.test.ts b/packages/app/src/lib/benchmark-power-validity.test.ts new file mode 100644 index 000000000..1c7b0f131 --- /dev/null +++ b/packages/app/src/lib/benchmark-power-validity.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest'; + +import { filterByPowerValidity, parsePowerValidityFilter } from './benchmark-power-validity'; + +describe('parsePowerValidityFilter', () => { + it('preserves an absent param as no filtering', () => { + expect(parsePowerValidityFilter(null)).toBeNull(); + }); + + it('accepts strictV2', () => { + expect(parsePowerValidityFilter('strictV2')).toBe('strictV2'); + }); + + it.each(['1', '0', 'any', 'certified', 'garbage', '', 'strictv2', 'strictV2 '])( + 'rejects unsupported value %j', + (value) => { + expect(parsePowerValidityFilter(value)).toBeUndefined(); + }, + ); +}); + +describe('filterByPowerValidity', () => { + const validatedV2 = { id: 1, metrics: { power_valid: 1, power_metric_schema_version: 2 } }; + const validatedUnversioned = { id: 2, metrics: { power_valid: 1 } }; + const invalidated = { id: 3, metrics: { power_valid: 0, power_metric_schema_version: 2 } }; + const legacy = { id: 4, metrics: { tput_per_gpu: 100 } }; + const noMetrics: { id: number; metrics?: Record } = { id: 5 }; + const rows = [validatedV2, validatedUnversioned, invalidated, legacy, noMetrics]; + + it('preserves all general benchmark rows when the parameter is omitted', () => { + const result = filterByPowerValidity(rows, null); + expect(result).toEqual(rows); + expect(result).not.toBe(rows); + }); + + it('requires a validated verdict and schema version 2 for strictV2', () => { + expect(filterByPowerValidity(rows, 'strictV2')).toEqual([validatedV2]); + }); + + it('excludes unsupported versions and malformed verdicts from strictV2', () => { + const unsupported = [ + { metrics: { power_valid: 1, power_metric_schema_version: 1 } }, + { metrics: { power_valid: 1, power_metric_schema_version: 3 } }, + { metrics: { power_valid: '1', power_metric_schema_version: 2 } }, + { metrics: { power_valid: 1, power_metric_schema_version: '2' } }, + { metrics: { power_valid: true, power_metric_schema_version: 2 } }, + ]; + expect(filterByPowerValidity(unsupported, 'strictV2')).toEqual([]); + }); +}); diff --git a/packages/app/src/lib/benchmark-power-validity.ts b/packages/app/src/lib/benchmark-power-validity.ts new file mode 100644 index 000000000..5842e30c5 --- /dev/null +++ b/packages/app/src/lib/benchmark-power-validity.ts @@ -0,0 +1,38 @@ +/** + * Measured-power validity filtering for the public benchmarks API. + * + * `metrics.power_valid` is tri-state: 1 means the measurement window was + * validated, an explicit 0 is an authoritative invalid verdict (measured + * values are withheld end-to-end), and an absent key marks a legacy row that + * predates validation. The only public filter, `strictV2`, requires a valid + * verdict and `power_metric_schema_version === 2`, mirroring + * `WHOLE_DEPLOYMENT_ENERGY_SCHEMA_VERSION` in `benchmark-transform.ts` and + * `POWER_METRIC_SCHEMA_VERSION` in the runner's `utils/aggregate_power.py` — + * only version 2 defines unprefixed `joules_per_*` fields as whole-deployment + * energy. The name is deliberately not `certified`: the UI's certified tier is + * a display rule that also admits validated legacy rows without a schema + * version, which `strictV2` excludes. + */ +export const POWER_VALIDITY_FILTERS = ['strictV2'] as const; +export type PowerValidityFilter = (typeof POWER_VALIDITY_FILTERS)[number]; + +/** Absent param means no filtering; unknown values return undefined so the caller can 400. */ +export function parsePowerValidityFilter( + raw: string | null, +): PowerValidityFilter | null | undefined { + return raw === null || raw === 'strictV2' ? raw : undefined; +} + +/** + * Pure post-cache row filter. An omitted parameter preserves general benchmark + * results, including rows with no power measurement. + */ +export function filterByPowerValidity }>( + rows: readonly T[], + filter: PowerValidityFilter | null, +): T[] { + if (filter === null) return [...rows]; + return rows.filter( + (row) => row.metrics?.power_valid === 1 && row.metrics?.power_metric_schema_version === 2, + ); +} diff --git a/packages/constants/src/metric-keys.test.ts b/packages/constants/src/metric-keys.test.ts index fc606267a..279879f74 100644 --- a/packages/constants/src/metric-keys.test.ts +++ b/packages/constants/src/metric-keys.test.ts @@ -4,6 +4,7 @@ import { MEASURED_POWER_METRIC_KEY_LIST, MEASURED_POWER_METRIC_KEYS, METRIC_KEYS, + POWER_METRIC_KEYS, } from './metric-keys'; describe('MEASURED_POWER_METRIC_KEYS', () => { @@ -46,3 +47,24 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { } }); }); + +describe('POWER_METRIC_KEYS', () => { + it('is a subset of METRIC_KEYS', () => { + for (const key of POWER_METRIC_KEYS) { + expect(METRIC_KEYS.has(key)).toBe(true); + } + }); + + it('has no duplicate keys', () => { + expect(new Set(POWER_METRIC_KEYS).size).toBe(POWER_METRIC_KEYS.length); + }); + + it('contains exactly the contract discriminators plus the 13 measured keys', () => { + // The public API documentation types every one of these keys on + // BenchmarkRow.metrics, so membership changes are contract changes. + expect(new Set(POWER_METRIC_KEYS)).toEqual( + new Set(['power_valid', 'power_metric_schema_version', ...MEASURED_POWER_METRIC_KEY_LIST]), + ); + expect(POWER_METRIC_KEYS).toHaveLength(15); + }); +}); diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 70cf69f6c..55e376c6a 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -45,6 +45,23 @@ export const MEASURED_POWER_METRIC_KEYS: ReadonlySet = new Set( MEASURED_POWER_METRIC_KEY_LIST, ); +/** + * Complete measured-power contract surface on `metrics`: the contract + * discriminators plus every measured power / energy / GPU-telemetry key. + * This is the set the public API documentation types on + * `BenchmarkRow.metrics`; it feeds METRIC_KEYS automatically. + */ +export const POWER_METRIC_KEYS = [ + // measured power / energy publication contract (aggregate_power.py) + // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power + // power_metric_schema_version: version 2 defines every unprefixed + // joules_per_* field as whole-deployment energy + 'power_valid', + 'power_metric_schema_version', + // measured power / energy / telemetry values, withheld when power_valid = 0 + ...MEASURED_POWER_METRIC_KEY_LIST, +] as const; + /** * Canonical set of metric keys stored in the benchmark_results.metrics JSONB column. * @@ -177,13 +194,7 @@ export const METRIC_KEYS = new Set([ // profiling window (agentic aiperf; flat in v2 artifacts, mapped from // server_metrics.kv_cache.gpu_usage_pct in v3) 'gpu_kv_cache_usage_pct', - // measured power / energy publication contract (aggregate_power.py) - // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power - // power_metric_schema_version: version 2 defines every unprefixed - // joules_per_* field as whole-deployment energy - 'power_valid', - 'power_metric_schema_version', - ...MEASURED_POWER_METRIC_KEY_LIST, + ...POWER_METRIC_KEYS, // extended parallelism dimensions (2026-07+ artifacts): pipeline parallelism // and decode/prefill context parallelism per role. These are config // dimensions, not measurements, but the configs table has no columns for