From 9f2cb57ac7c821606f92ff1372fcb4b559467be3 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 12:56:25 -0700 Subject: [PATCH 01/11] =?UTF-8?q?feat(constants):=20extract=20MEASURED=5FP?= =?UTF-8?q?OWER=5FMETRIC=5FKEYS=20withheld-power=20set=20|=20=E5=B8=B8?= =?UTF-8?q?=E9=87=8F=EF=BC=9A=E6=8F=90=E5=8F=96=20MEASURED=5FPOWER=5FMETRI?= =?UTF-8?q?C=5FKEYS=20=E4=BD=9C=E4=B8=BA=E9=9C=80=E6=89=A3=E7=95=99?= =?UTF-8?q?=E7=9A=84=E5=AE=9E=E6=B5=8B=E5=8A=9F=E8=80=97=E6=8C=87=E6=A0=87?= =?UTF-8?q?=E9=9B=86=E5=90=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the 13 measured power / energy / GPU-telemetry keys out of the METRIC_KEYS literal into an exported MEASURED_POWER_METRIC_KEY_LIST (and ReadonlySet MEASURED_POWER_METRIC_KEYS) so the ingest scrub and the display-layer withholding share one source of truth. METRIC_KEYS membership is unchanged (spread keeps the same entries); the contract discriminators power_valid / power_metric_schema_version and the invalid-verdict companion fields power_invalid_reasons / power_audit are codified as never part of the withheld set. 将 13 个实测功耗/能耗/GPU 遥测指标键从 METRIC_KEYS 字面量中提取为导出的 MEASURED_POWER_METRIC_KEY_LIST（及 ReadonlySet 形式的 MEASURED_POWER_METRIC_KEYS），使摄取端剥离逻辑与前端展示层扣留逻辑共享 单一事实来源。METRIC_KEYS 成员保持不变；契约判别字段与无效判定伴随字段 经测试固定永不进入扣留集合。 --- packages/constants/src/metric-keys.test.ts | 55 ++++++++++++++ packages/constants/src/metric-keys.ts | 88 +++++++++++++--------- 2 files changed, 109 insertions(+), 34 deletions(-) create mode 100644 packages/constants/src/metric-keys.test.ts diff --git a/packages/constants/src/metric-keys.test.ts b/packages/constants/src/metric-keys.test.ts new file mode 100644 index 000000000..b7213086a --- /dev/null +++ b/packages/constants/src/metric-keys.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest'; + +import { + MEASURED_POWER_METRIC_KEY_LIST, + MEASURED_POWER_METRIC_KEYS, + METRIC_KEYS, +} from './metric-keys'; + +describe('MEASURED_POWER_METRIC_KEYS', () => { + it('is a subset of METRIC_KEYS', () => { + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(METRIC_KEYS.has(key)).toBe(true); + } + }); + + it('contains exactly the 13 measured power / energy / telemetry keys', () => { + // Guards accidental additions/removals: the ingest scrub and the display + // withholding both key off this set, so membership changes are policy + // changes and must be deliberate. + expect(new Set(MEASURED_POWER_METRIC_KEY_LIST)).toEqual( + new Set([ + 'avg_power_w', + 'joules_per_successful_query', + 'joules_per_output_token', + 'joules_per_total_token', + 'prefill_avg_power_w', + 'decode_avg_power_w', + 'joules_per_input_token', + 'prefill_joules_per_input_token', + 'decode_joules_per_output_token', + 'avg_temp_c', + 'peak_temp_c', + 'avg_util_pct', + 'avg_mem_used_mb', + ]), + ); + expect(MEASURED_POWER_METRIC_KEYS.size).toBe(13); + }); + + it('never contains the contract discriminators or invalid-verdict companion fields', () => { + // power_valid / power_metric_schema_version are the verdict itself and + // must survive a scrub; power_invalid_reasons / power_audit are the + // producer's explanation of an invalid verdict (persisted app-side by + // PLAN-07) and are only meaningful on scrubbed rows. None of them may + // ever be added to the withheld set. + for (const key of [ + 'power_valid', + 'power_metric_schema_version', + 'power_invalid_reasons', + 'power_audit', + ]) { + expect(MEASURED_POWER_METRIC_KEYS.has(key)).toBe(false); + } + }); +}); diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 370ea7f50..fac99b513 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -1,3 +1,54 @@ +/** + * Measured power / energy / GPU-telemetry metrics that MUST be withheld + * whenever a row carries an invalid power verdict (power_valid = 0). + * Excludes the contract discriminators (power_valid, + * power_metric_schema_version) and the invalid-verdict companion fields + * (power_invalid_reasons, power_audit), which are always kept. Add any new + * measured power/telemetry key here — it feeds METRIC_KEYS automatically. + * Mirrored by the display-layer withholding in + * packages/app/src/lib/benchmark-transform.ts (rowToAggDataEntry). + */ +export const MEASURED_POWER_METRIC_KEY_LIST = [ + // measured power / energy (emitted by runner's aggregate_power.py) + // avg_power_w: mean per-GPU draw (W) during the load window + // joules_per_successful_query: whole-deployment energy / successful requests + // joules_per_output_token: energy / total_output_tokens. CLUSTER-WIDE on + // schema-version-2 rows, including disaggregated runs. + // joules_per_total_token: total_system_energy / (total_input + total_output) + // — cluster-wide; workload-shape-fair view that + // doesn't treat prompt as free. + 'avg_power_w', + 'joules_per_successful_query', + 'joules_per_output_token', + 'joules_per_total_token', + // multinode / disagg role splits (emitted only when the deployment has + // distinct prefill / decode workers) + // prefill_avg_power_w / decode_avg_power_w: mean per-GPU draw within each role + // Explicit role-local energy remains separate from the version-2 unprefixed + // whole-deployment fields. + 'prefill_avg_power_w', + 'decode_avg_power_w', + 'joules_per_input_token', + 'prefill_joules_per_input_token', + 'decode_joules_per_output_token', + // cluster-wide GPU telemetry beyond power (emitted by aggregate_power.py when + // the perfmon CSVs include temperature, utilization, or memory samples). + // avg_temp_c: mean per-GPU temperature (Celsius) during load window + // peak_temp_c: max instantaneous per-GPU temperature in window + // avg_util_pct: mean per-GPU GPU-utilization percent (0-100) + // avg_mem_used_mb: mean per-GPU memory used (MiB / MB) + // Single-node and multinode runs both surface these as flat scalars; the + // per-worker breakdown carries the same fields on each entry in workers[]. + 'avg_temp_c', + 'peak_temp_c', + 'avg_util_pct', + 'avg_mem_used_mb', +] as const; + +export const MEASURED_POWER_METRIC_KEYS: ReadonlySet = new Set( + MEASURED_POWER_METRIC_KEY_LIST, +); + /** * Canonical set of metric keys stored in the benchmark_results.metrics JSONB column. * @@ -130,45 +181,14 @@ export const METRIC_KEYS = new Set([ // profiling window (agentic aiperf; flat in v2 artifacts, mapped from // server_metrics.kv_cache.gpu_usage_pct in v3) 'gpu_kv_cache_usage_pct', - // measured power / energy (emitted by runner's aggregate_power.py) + // measured power / energy publication contract (aggregate_power.py) // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power // power_metric_schema_version: version 2 defines every unprefixed // joules_per_* field as whole-deployment energy - // avg_power_w: mean per-GPU draw (W) during the load window - // joules_per_successful_query: whole-deployment energy / successful requests - // joules_per_output_token: energy / total_output_tokens. CLUSTER-WIDE on - // schema-version-2 rows, including disaggregated runs. - // joules_per_total_token: total_system_energy / (total_input + total_output) - // — cluster-wide; workload-shape-fair view that - // doesn't treat prompt as free. 'power_valid', 'power_metric_schema_version', - 'avg_power_w', - 'joules_per_successful_query', - 'joules_per_output_token', - 'joules_per_total_token', - // multinode / disagg role splits (emitted only when the deployment has - // distinct prefill / decode workers) - // prefill_avg_power_w / decode_avg_power_w: mean per-GPU draw within each role - // Explicit role-local energy remains separate from the version-2 unprefixed - // whole-deployment fields. - 'prefill_avg_power_w', - 'decode_avg_power_w', - 'joules_per_input_token', - 'prefill_joules_per_input_token', - 'decode_joules_per_output_token', - // cluster-wide GPU telemetry beyond power (emitted by aggregate_power.py when - // the perfmon CSVs include temperature, utilization, or memory samples). - // avg_temp_c: mean per-GPU temperature (Celsius) during load window - // peak_temp_c: max instantaneous per-GPU temperature in window - // avg_util_pct: mean per-GPU GPU-utilization percent (0-100) - // avg_mem_used_mb: mean per-GPU memory used (MiB / MB) - // Single-node and multinode runs both surface these as flat scalars; the - // per-worker breakdown carries the same fields on each entry in workers[]. - 'avg_temp_c', - 'peak_temp_c', - 'avg_util_pct', - 'avg_mem_used_mb', + // measured power / energy / telemetry values, withheld when power_valid = 0 + ...MEASURED_POWER_METRIC_KEY_LIST, // extended parallelism dimensions (2026-07+ artifacts): pipeline parallelism // and decode/prefill context parallelism per role. These are config // dimensions, not measurements, but the configs table has no columns for From 74ab22530469db3f25dc394e8f289449b624ef32 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 12:56:51 -0700 Subject: [PATCH 02/11] =?UTF-8?q?fix(etl):=20strip=20measured=20power=20me?= =?UTF-8?q?trics=20at=20ingest=20when=20power=5Fvalid=3D0=20|=20ETL?= =?UTF-8?q?=EF=BC=9Apower=5Fvalid=3D0=20=E6=97=B6=E5=9C=A8=E6=91=84?= =?UTF-8?q?=E5=8F=96=E9=98=B6=E6=AE=B5=E5=89=A5=E7=A6=BB=E5=AE=9E=E6=B5=8B?= =?UTF-8?q?=E5=8A=9F=E8=80=97=E6=8C=87=E6=A0=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defense-in-depth for the power publication contract (G8). Today the only protections are producer-side stripping (aggregate_power.py) and the frontend display withholding — the ETL persists, and the raw API serves, whatever measured values a power_valid=0 artifact carries. Add scrubWithheldPowerMetrics to mapBenchmarkRow: after the normalized verdict fails closed to 0, delete every MEASURED_POWER_METRIC_KEYS entry and drop the workers payload. The scrub runs after the last metrics mutation (agentic preferFullResponseMetrics reassignment + extractRuntimeMetadata merge), covers every ingest path (CI ingest and both re-mapping backfills), is idempotent, and converges re-ingested dirty artifacts to exactly what a clean producer would ship. Verdict discriminators, companion fields (power_invalid_reasons / power_audit), legacy no-verdict rows, and pv=1 rows are untouched. The query layer deliberately stays unfiltered (single enforcement point at ETL). 针对功耗发布契约的纵深防御（G8）。此前仅有生产端剥离与前端展示扣留两道 防线：power_valid=0 的工件若携带实测值，ETL 会照常入库、原始 API 会照常 返回。本次在 mapBenchmarkRow 中新增 scrubWithheldPowerMetrics：判定值 归一化为 0 后，删除 MEASURED_POWER_METRIC_KEYS 中的全部指标并丢弃 workers 载荷。剥离在最后一次 metrics 变更之后执行，覆盖所有摄取路径， 幂等且使脏工件重摄取后与清洁生产端输出完全一致。判别字段、伴随字段、 无判定的历史行及 power_valid=1 的行均不受影响；查询层有意保持不过滤 （ETL 为唯一强制点）。 --- packages/db/src/etl/benchmark-mapper.test.ts | 166 +++++++++++++++++++ packages/db/src/etl/benchmark-mapper.ts | 35 +++- 2 files changed, 198 insertions(+), 3 deletions(-) diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 03f65ba64..4bc840dd8 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect } from 'vitest'; +import { MEASURED_POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; import { extractWorkers, mapBenchmarkRow } from './benchmark-mapper'; import { createSkipTracker } from './skip-tracker'; @@ -61,6 +62,33 @@ function makeV2Row(overrides: Record = {}): Record { }; } +/** + * All 13 measured power/energy/telemetry keys with distinct sentinel values, + * plus a valid 2-entry per-worker payload — the shape a producer regression + * would ship if it stopped stripping measurements on power_valid=0 rows. + */ +function dirtyPowerPayload(): Record { + return { + avg_power_w: 685.5, + joules_per_successful_query: 1542.75, + joules_per_output_token: 8.4, + joules_per_total_token: 0.8, + prefill_avg_power_w: 612.3, + decode_avg_power_w: 701.5, + joules_per_input_token: 1.2, + prefill_joules_per_input_token: 0.4, + decode_joules_per_output_token: 5.1, + avg_temp_c: 68.4, + peak_temp_c: 79.2, + avg_util_pct: 88.5, + avg_mem_used_mb: 71234.5, + workers: [ + { role: 'prefill', worker_idx: 0, hosts: ['pn0'], num_gpus: 4, avg_power_w: 612.3 }, + { role: 'decode', worker_idx: 0, hosts: ['dn0'], num_gpus: 8, avg_power_w: 701.5 }, + ], + }; +} + describe('mapBenchmarkRow', () => { describe('v1 schema', () => { it('maps a valid v1 row to BenchmarkParams', () => { @@ -281,6 +309,144 @@ describe('mapBenchmarkRow', () => { }); }); + describe('power_valid=0 measured-power scrub (defense-in-depth)', () => { + it('strips every measured key and the workers payload on an explicit invalid verdict', () => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeV2Row({ + power_valid: 0, + power_metric_schema_version: 2, + median_ttft: 50.2, + ...dirtyPowerPayload(), + }), + tracker, + ); + + expect(result).not.toBeNull(); + expect(result!.metrics.power_valid).toBe(0); + expect(result!.metrics.power_metric_schema_version).toBe(2); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(result!.metrics).not.toHaveProperty(key); + } + expect(result!.workers).toBeUndefined(); + // Non-power metrics survive untouched — a power_valid=0 row is still a + // perfectly valid performance result. + expect(result!.metrics.tput_per_gpu).toBe(567.8); + expect(result!.metrics.median_ttft).toBe(50.2); + }); + + it('keeps every measured key and the workers payload on a valid verdict', () => { + const tracker = createSkipTracker(); + const dirty = dirtyPowerPayload(); + const result = mapBenchmarkRow( + makeV2Row({ power_valid: 1, power_metric_schema_version: 2, ...dirty }), + tracker, + ); + + expect(result!.metrics.power_valid).toBe(1); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(result!.metrics[key]).toBe(dirty[key]); + } + expect(result!.workers).toHaveLength(2); + }); + + it('leaves legacy rows without a verdict untouched (historical measurements kept)', () => { + const tracker = createSkipTracker(); + const dirty = dirtyPowerPayload(); + const result = mapBenchmarkRow(makeV2Row(dirty), tracker); + + expect(result!.metrics).not.toHaveProperty('power_valid'); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(result!.metrics[key]).toBe(dirty[key]); + } + expect(result!.workers).toHaveLength(2); + }); + + it.each([true, 'garbage', 2, Number.NaN])( + 'scrubs after failing closed on malformed verdict %j', + (powerValid) => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeV2Row({ power_valid: powerValid, ...dirtyPowerPayload() }), + tracker, + ); + + expect(result!.metrics.power_valid).toBe(0); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(result!.metrics).not.toHaveProperty(key); + } + expect(result!.workers).toBeUndefined(); + }, + ); + + it('converges a dirty pv=0 artifact to exactly what a clean producer would ship', () => { + // Re-ingest replaces metrics and workers wholesale (ON CONFLICT DO + // UPDATE in benchmark-ingest.ts), so a scrubbed dirty row must be a + // fixed point identical to the producer-clean mapping. + const tracker = createSkipTracker(); + const dirtyResult = mapBenchmarkRow( + makeV2Row({ power_valid: 0, power_metric_schema_version: 2, ...dirtyPowerPayload() }), + tracker, + ); + const cleanResult = mapBenchmarkRow( + makeV2Row({ power_valid: 0, power_metric_schema_version: 2 }), + tracker, + ); + + expect(dirtyResult!.metrics).toEqual(cleanResult!.metrics); + expect(dirtyResult!.workers).toBeUndefined(); + expect(cleanResult!.workers).toBeUndefined(); + }); + + it('scrubs agentic rows after the preferFullResponseMetrics reassignment', () => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeAgenticRow({ + power_valid: 0, + ...dirtyPowerPayload(), + mean_full_response_itl: 0.02, + }), + tracker, + ); + + expect(result!.metrics.power_valid).toBe(0); + // The reassignment ran: full-response ITL became canonical. + expect(result!.metrics.mean_itl).toBe(0.02); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(result!.metrics).not.toHaveProperty(key); + } + expect(result!.workers).toBeUndefined(); + }); + + it('tolerates the invalid-verdict companion fields without scrubbing them', () => { + // PLAN-06 producers ship power_invalid_reasons / power_audit alongside + // power_valid=0. The scrub must never touch them; persisting them + // app-side is PLAN-07 — today's pinned behavior is that non-numeric + // fields simply never land in the numeric metrics record (parseNum + // drops arrays/objects). + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeV2Row({ + power_valid: 0, + power_metric_schema_version: 2, + ...dirtyPowerPayload(), + power_invalid_reasons: ['window_too_short'], + power_audit: { window_start_unix: 1, window_end_unix: 2 }, + }), + tracker, + ); + + expect(result).not.toBeNull(); + expect(result!.metrics.power_valid).toBe(0); + expect(result!.metrics.power_metric_schema_version).toBe(2); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(result!.metrics).not.toHaveProperty(key); + } + expect(result!.metrics).not.toHaveProperty('power_invalid_reasons'); + expect(result!.metrics).not.toHaveProperty('power_audit'); + }); + }); + describe('skip tracking', () => { it('skips and counts unmapped model', () => { const tracker = createSkipTracker(); diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index 544c6ab16..97fb0da6b 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -6,7 +6,11 @@ import type { ConfigParams } from './config-cache'; import type { SkipTracker } from './skip-tracker'; -import { METRIC_KEYS, PRECISION_KEYS } from '@semianalysisai/inferencex-constants'; +import { + MEASURED_POWER_METRIC_KEYS, + METRIC_KEYS, + PRECISION_KEYS, +} from '@semianalysisai/inferencex-constants'; import { flattenAgenticAggRow } from './agentic-v3-flatten'; import { preferFullResponseMetrics } from './full-response-interactivity'; import { @@ -343,12 +347,17 @@ export function mapBenchmarkRow( ? rawRecipeFingerprint.trim() : null; + // Scrub AFTER the last mutation of `metrics` (the agentic + // preferFullResponseMetrics reassignment and the extractRuntimeMetadata + // merge above) so no later step can resurrect a withheld key. + const powerWithheld = scrubWithheldPowerMetrics(metrics); // Per-worker measured-power breakdown. The runner emits this as an array // of objects sibling to the scalar metrics; we surface it on a dedicated // BenchmarkParams.workers field so downstream consumers can treat it as // structured data without polluting the flat metrics record. Defensive - // narrowing — anything other than a non-empty array of objects is dropped. - const workers = extractWorkers(row.workers); + // narrowing — anything other than a non-empty array of objects is dropped, + // and a withheld power verdict drops the payload entirely. + const workers = powerWithheld ? undefined : extractWorkers(row.workers); return { config: { @@ -489,6 +498,26 @@ function normalizePowerContractMetrics( } } +/** + * Defense-in-depth for the power publication contract: when the + * normalized verdict is an explicit invalid (power_valid === 0), delete + * every measured power/energy/telemetry metric so withheld measurements + * can never be persisted or served, even if a producer regression ships + * them. Keeps power_valid and power_metric_schema_version (and any + * future companion fields such as power_invalid_reasons / power_audit). + * Legacy rows without a verdict are untouched. Returns true when the + * row's power is withheld so the caller also drops the workers payload. + * This is the single enforcement point — the query layer intentionally + * serves metrics unfiltered (see queries/benchmarks.ts rawMetrics), and + * the frontend independently withholds at display + * (benchmark-transform.ts rowToAggDataEntry). + */ +function scrubWithheldPowerMetrics(metrics: Record): boolean { + if (metrics.power_valid !== 0) return false; + for (const key of MEASURED_POWER_METRIC_KEYS) delete metrics[key]; + return true; +} + /** * Narrow a raw `workers` value from the artifact JSON to `WorkerPower[]` or * undefined. Each entry must have a string `role`, a numeric `worker_idx`, From f1b2c9d38e0e1154007b60cd8896f2880b77f6ef Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 12:57:12 -0700 Subject: [PATCH 03/11] =?UTF-8?q?test(app):=20pin=20display=20withholding?= =?UTF-8?q?=20parity=20with=20MEASURED=5FPOWER=5FMETRIC=5FKEYS=20|=20?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=EF=BC=9A=E5=89=8D=E7=AB=AF=E5=B1=95=E7=A4=BA?= =?UTF-8?q?=E5=B1=82=E6=89=A3=E7=95=99=E9=80=BB=E8=BE=91=E4=B8=8E=20MEASUR?= =?UTF-8?q?ED=5FPOWER=5FMETRIC=5FKEYS=20=E4=BF=9D=E6=8C=81=E4=B8=80?= =?UTF-8?q?=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive rowToAggDataEntry with a power_valid=0 row carrying every key in the shared constant and assert each corresponding output field (and workers) comes back undefined — adding a key to the constant that the frontend forgets to withhold now fails this test. The reverse direction (frontend withholding a key missing from the constant) stays hand-audited, as noted in the test comment. 以携带共享常量中全部键的 power_valid=0 行驱动 rowToAggDataEntry，断言 对应输出字段（含 workers）均为 undefined —— 若向常量新增了前端未扣留的 键，此测试即失败。反向情况（前端扣留了常量之外的键）仍由人工审计，测试 注释中已注明。 --- .../app/src/lib/benchmark-transform.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index fac8734d8..a95a9ddf1 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -1,3 +1,4 @@ +import { MEASURED_POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; import { describe, it, expect, vi } from 'vitest'; import { getPointLabel } from '@/components/inference/utils/tooltipUtils'; @@ -424,6 +425,28 @@ describe('rowToAggDataEntry', () => { expect(point.measuredPowerPercentTdp).toBeUndefined(); }); + it('withholds every MEASURED_POWER_METRIC_KEYS field when power_valid=0 (ETL parity)', () => { + // Parity with the ingest scrub (benchmark-mapper.ts + // scrubWithheldPowerMetrics): the shared constant is the single source of + // truth for the withheld set, so a key added there without a matching + // display-layer withholding fails here. The reverse direction — the + // display layer withholding a key missing from the constant — stays + // hand-audited. + const metrics: BenchmarkRow['metrics'] = { power_valid: 0 }; + for (const key of MEASURED_POWER_METRIC_KEYS) metrics[key] = 123.45; + const entry = rowToAggDataEntry( + makeRow({ + metrics, + workers: [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 123.45 }], + }), + ); + + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect((entry as unknown as Record)[key]).toBeUndefined(); + } + expect(entry.workers).toBeUndefined(); + }); + it('keeps measured telemetry compatible when a legacy row omits power_valid', () => { const workers = [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 560 }]; const row = makeRow({ From 25172a63f74c51df8512881a4f3cde4b20ecbfc1 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 13:15:59 -0700 Subject: [PATCH 04/11] =?UTF-8?q?fix(etl):=20apply=20power=20scrub=20to=20?= =?UTF-8?q?supplemental=20ingest=20bypass=20|=20ETL=EF=BC=9A=E5=AF=B9?= =?UTF-8?q?=E7=BB=95=E8=BF=87=20mapBenchmarkRow=20=E7=9A=84=E8=A1=A5?= =?UTF-8?q?=E5=85=85=E6=95=B0=E6=8D=AE=E6=91=84=E5=8F=96=E5=90=8C=E6=A0=B7?= =?UTF-8?q?=E5=BA=94=E7=94=A8=E5=8A=9F=E8=80=97=E5=89=A5=E7=A6=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found ingest-supplemental.ts persists metrics verbatim via bulkIngestBenchmarkRows without mapBenchmarkRow, so a supplemental entry carrying power_valid=0 plus measured values would have bypassed the scrub. Export normalizePowerContractMetrics / scrubWithheldPowerMetrics and run the same fail-closed normalize+scrub pair on supplemental metrics; correct the scrub docstring to name the path; pin the direct-call usage with unit tests. --- packages/db/src/etl/benchmark-mapper.test.ts | 50 +++++++++++++++++++- packages/db/src/etl/benchmark-mapper.ts | 13 +++-- packages/db/src/ingest-supplemental.ts | 7 +++ 3 files changed, 64 insertions(+), 6 deletions(-) diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 4bc840dd8..7ca31c88d 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -1,6 +1,11 @@ import { describe, it, expect } from 'vitest'; import { MEASURED_POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; -import { extractWorkers, mapBenchmarkRow } from './benchmark-mapper'; +import { + extractWorkers, + mapBenchmarkRow, + normalizePowerContractMetrics, + scrubWithheldPowerMetrics, +} from './benchmark-mapper'; import { createSkipTracker } from './skip-tracker'; /** Minimal valid v1 benchmark row. */ @@ -861,6 +866,49 @@ describe('mapBenchmarkRow', () => { }); }); +describe('scrubWithheldPowerMetrics (direct — supplemental ingest path)', () => { + // ingest-supplemental.ts persists metrics without mapBenchmarkRow, calling + // normalizePowerContractMetrics + scrubWithheldPowerMetrics on the raw + // record directly. Pin that usage pattern here. + function supplementalMetrics(overrides: Record = {}): Record { + const { workers: _workers, ...measured } = dirtyPowerPayload(); + return { tput_per_gpu: 567.8, ...measured, ...overrides }; + } + + it('strips every measured key on power_valid=0 and reports withheld', () => { + const metrics = supplementalMetrics({ power_valid: 0, power_metric_schema_version: 2 }); + + expect(scrubWithheldPowerMetrics(metrics)).toBe(true); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(metrics).not.toHaveProperty(key); + } + expect(metrics.power_valid).toBe(0); + expect(metrics.power_metric_schema_version).toBe(2); + expect(metrics.tput_per_gpu).toBe(567.8); + }); + + it('leaves power_valid=1 and legacy no-verdict records untouched', () => { + for (const metrics of [supplementalMetrics({ power_valid: 1 }), supplementalMetrics()]) { + const before = { ...metrics }; + expect(scrubWithheldPowerMetrics(metrics)).toBe(false); + expect(metrics).toEqual(before); + } + }); + + it('fails closed on a malformed verdict when composed with normalization', () => { + const metrics = supplementalMetrics({ power_valid: 2 }); + // The exact ingest-supplemental.ts call sequence: same object as row + // and metrics. + normalizePowerContractMetrics(metrics, metrics); + expect(scrubWithheldPowerMetrics(metrics)).toBe(true); + + expect(metrics.power_valid).toBe(0); + for (const key of MEASURED_POWER_METRIC_KEYS) { + expect(metrics).not.toHaveProperty(key); + } + }); +}); + describe('extractWorkers', () => { it('returns undefined for non-array input', () => { expect(extractWorkers(undefined)).toBeUndefined(); diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index 97fb0da6b..e79945aff 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -467,7 +467,7 @@ function captureNumericMetrics(row: Record): Record * two fields are semantic discriminators, so loose numeric coercion must never * turn malformed producer output into an affirmative verdict or schema. */ -function normalizePowerContractMetrics( +export function normalizePowerContractMetrics( row: Record, metrics: Record, ): void { @@ -507,12 +507,15 @@ function normalizePowerContractMetrics( * future companion fields such as power_invalid_reasons / power_audit). * Legacy rows without a verdict are untouched. Returns true when the * row's power is withheld so the caller also drops the workers payload. - * This is the single enforcement point — the query layer intentionally - * serves metrics unfiltered (see queries/benchmarks.ts rawMetrics), and - * the frontend independently withholds at display + * This is the single enforcement policy at ingest: every mapBenchmarkRow + * path runs it here, and ingest-supplemental.ts — the one persistence + * path that bypasses mapBenchmarkRow — applies the same normalize+scrub + * pair to its verbatim metrics. The query layer intentionally serves + * metrics unfiltered (see queries/benchmarks.ts rawMetrics), and the + * frontend independently withholds at display * (benchmark-transform.ts rowToAggDataEntry). */ -function scrubWithheldPowerMetrics(metrics: Record): boolean { +export function scrubWithheldPowerMetrics(metrics: Record): boolean { if (metrics.power_valid !== 0) return false; for (const key of MEASURED_POWER_METRIC_KEYS) delete metrics[key]; return true; diff --git a/packages/db/src/ingest-supplemental.ts b/packages/db/src/ingest-supplemental.ts index 4483d4397..18a2a8b55 100644 --- a/packages/db/src/ingest-supplemental.ts +++ b/packages/db/src/ingest-supplemental.ts @@ -27,6 +27,7 @@ import { bulkUpsertAvailability, type BenchmarkPersistenceInput, } from './etl/benchmark-ingest'; +import { normalizePowerContractMetrics, scrubWithheldPowerMetrics } from './etl/benchmark-mapper'; import { ingestEvalRow } from './etl/eval-ingest'; const sql = createAdminSql({ @@ -265,6 +266,12 @@ async function ingestSupplementalBmk( numDecodeGpu: entry.tp * entry.ep, }); + // Supplemental metrics bypass mapBenchmarkRow, so apply the power + // publication contract here too: fail-closed verdict normalization, + // then strip withheld measurements when power_valid=0. + normalizePowerContractMetrics(entry.metrics, entry.metrics); + scrubWithheldPowerMetrics(entry.metrics); + rows.push({ configId, benchmarkType: 'single_turn', From ac7a23ccfb6937b4452e1b269dfea94cb9a9f342 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 15:51:42 -0700 Subject: [PATCH 05/11] =?UTF-8?q?feat(constants):=20export=20POWER=5FMETRI?= =?UTF-8?q?C=5FKEYS=20power-contract=20key=20list=20|=20=E5=B8=B8=E9=87=8F?= =?UTF-8?q?=EF=BC=9A=E5=AF=BC=E5=87=BA=20POWER=5FMETRIC=5FKEYS=20=E5=8A=9F?= =?UTF-8?q?=E7=8E=87=E5=A5=91=E7=BA=A6=E6=8C=87=E6=A0=87=E9=94=AE=E5=88=97?= =?UTF-8?q?=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Derives the full documented power surface (discriminators + MEASURED_POWER_METRIC_KEY_LIST) as one exported constant and spreads it into METRIC_KEYS, keeping set membership provably unchanged. The public API documentation layer types BenchmarkRow.metrics from this export. --- packages/constants/src/metric-keys.test.ts | 22 ++++++++++++++++++ packages/constants/src/metric-keys.ts | 27 +++++++++++++++------- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/packages/constants/src/metric-keys.test.ts b/packages/constants/src/metric-keys.test.ts index b7213086a..979afe302 100644 --- a/packages/constants/src/metric-keys.test.ts +++ b/packages/constants/src/metric-keys.test.ts @@ -4,6 +4,7 @@ import { MEASURED_POWER_METRIC_KEY_LIST, MEASURED_POWER_METRIC_KEYS, METRIC_KEYS, + POWER_METRIC_KEYS, } from './metric-keys'; describe('MEASURED_POWER_METRIC_KEYS', () => { @@ -53,3 +54,24 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { } }); }); + +describe('POWER_METRIC_KEYS', () => { + it('is a subset of METRIC_KEYS', () => { + for (const key of POWER_METRIC_KEYS) { + expect(METRIC_KEYS.has(key)).toBe(true); + } + }); + + it('has no duplicate keys', () => { + expect(new Set(POWER_METRIC_KEYS).size).toBe(POWER_METRIC_KEYS.length); + }); + + it('contains exactly the contract discriminators plus the 13 measured keys', () => { + // The public API documentation types every one of these keys on + // BenchmarkRow.metrics, so membership changes are contract changes. + expect(new Set(POWER_METRIC_KEYS)).toEqual( + new Set(['power_valid', 'power_metric_schema_version', ...MEASURED_POWER_METRIC_KEY_LIST]), + ); + expect(POWER_METRIC_KEYS).toHaveLength(15); + }); +}); diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index fac99b513..73b75ea42 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -49,6 +49,23 @@ export const MEASURED_POWER_METRIC_KEYS: ReadonlySet = new Set( MEASURED_POWER_METRIC_KEY_LIST, ); +/** + * Complete measured-power contract surface on `metrics`: the contract + * discriminators plus every measured power / energy / GPU-telemetry key. + * This is the set the public API documentation types on + * `BenchmarkRow.metrics`; it feeds METRIC_KEYS automatically. + */ +export const POWER_METRIC_KEYS = [ + // measured power / energy publication contract (aggregate_power.py) + // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power + // power_metric_schema_version: version 2 defines every unprefixed + // joules_per_* field as whole-deployment energy + 'power_valid', + 'power_metric_schema_version', + // measured power / energy / telemetry values, withheld when power_valid = 0 + ...MEASURED_POWER_METRIC_KEY_LIST, +] as const; + /** * Canonical set of metric keys stored in the benchmark_results.metrics JSONB column. * @@ -181,14 +198,8 @@ export const METRIC_KEYS = new Set([ // profiling window (agentic aiperf; flat in v2 artifacts, mapped from // server_metrics.kv_cache.gpu_usage_pct in v3) 'gpu_kv_cache_usage_pct', - // measured power / energy publication contract (aggregate_power.py) - // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power - // power_metric_schema_version: version 2 defines every unprefixed - // joules_per_* field as whole-deployment energy - 'power_valid', - 'power_metric_schema_version', - // measured power / energy / telemetry values, withheld when power_valid = 0 - ...MEASURED_POWER_METRIC_KEY_LIST, + // measured power / energy contract: discriminators + measured values + ...POWER_METRIC_KEYS, // extended parallelism dimensions (2026-07+ artifacts): pipeline parallelism // and decode/prefill context parallelism per role. These are config // dimensions, not measurements, but the configs table has no columns for From ca703a8d1f057f6a7a08956ff0031966cf920c55 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 15:53:51 -0700 Subject: [PATCH 06/11] =?UTF-8?q?feat(api):=20add=20powerValid=20measured-?= =?UTF-8?q?power=20validity=20filter=20to=20/api/v1/benchmarks=20|=20API?= =?UTF-8?q?=EF=BC=9A=E4=B8=BA=20/api/v1/benchmarks=20=E6=96=B0=E5=A2=9E=20?= =?UTF-8?q?powerValid=20=E5=AE=9E=E6=B5=8B=E5=8A=9F=E7=8E=87=E6=9C=89?= =?UTF-8?q?=E6=95=88=E6=80=A7=E7=AD=9B=E9=80=89=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure post-cache row filter (cache keys unchanged): 1 keeps validated rows, 0 keeps explicitly invalidated rows, any is the default identity (legacy rows included), and strictV2 additionally requires power_metric_schema_version == 2 (whole-deployment energy semantics). Named strictV2 rather than certified to avoid colliding with the UI tier's product meaning. Unknown values 400; the combination with view=calculator 400s because the calculator cache stores rows already trimmed past power_valid. Route digest bumped in the review ledger. --- .../src/app/api/v1/benchmarks/route.test.ts | 144 ++++++++++++++++++ .../app/src/app/api/v1/benchmarks/route.ts | 23 ++- packages/app/src/lib/api-route-catalog.ts | 2 +- .../src/lib/benchmark-power-validity.test.ts | 65 ++++++++ .../app/src/lib/benchmark-power-validity.ts | 42 +++++ 5 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 packages/app/src/lib/benchmark-power-validity.test.ts create mode 100644 packages/app/src/lib/benchmark-power-validity.ts diff --git a/packages/app/src/app/api/v1/benchmarks/route.test.ts b/packages/app/src/app/api/v1/benchmarks/route.test.ts index 629339016..e7f679e35 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.test.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.test.ts @@ -242,4 +242,148 @@ describe('GET /api/v1/benchmarks', () => { const body = await res.json(); expect(body).toEqual([]); }); + + describe('powerValid filter', () => { + const validatedV2 = { + id: 1, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, power_metric_schema_version: 2, avg_power_w: 700 }, + }; + const validatedUnversioned = { + id: 2, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, avg_power_w: 650 }, + }; + const invalidated = { + id: 3, + benchmark_type: 'single_turn', + metrics: { power_valid: 0 }, + }; + const legacy = { + id: 4, + benchmark_type: 'single_turn', + metrics: { tput_per_gpu: 100 }, + }; + const powerRows = [validatedV2, validatedUnversioned, invalidated, legacy]; + + it('powerValid=1 keeps only rows with a validated verdict', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([validatedV2, validatedUnversioned]); + }); + + it('powerValid=0 keeps only explicitly invalidated rows', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=0')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([invalidated]); + }); + + it('powerValid=strictV2 requires a validated verdict plus schema version 2', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([validatedV2]); + }); + + it('powerValid=any matches the response with the param absent (backward compat)', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + const withParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any')); + + mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); + const withoutParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528')); + + expect(withParam.status).toBe(200); + expect(withoutParam.status).toBe(200); + const bodyWithParam = await withParam.json(); + expect(bodyWithParam).toEqual(await withoutParam.json()); + expect(bodyWithParam).toEqual(powerRows); + }); + + it('rejects an unknown powerValid value without querying', async () => { + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=garbage')); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Unknown powerValid filter' }); + expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); + }); + + it('rejects powerValid combined with view=calculator without querying', async () => { + const res = await GET( + req( + '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1&view=calculator&sequence=1k%2F1k', + ), + ); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: 'powerValid cannot be combined with view=calculator', + }); + expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); + }); + + it('allows powerValid=any with view=calculator (no-op filter)', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + benchmark_type: 'single_turn', + isl: 1024, + osl: 1024, + metrics: { tput_per_gpu: 100, avg_power_w: 700 }, + }, + ]); + + const res = await GET( + req( + '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any&view=calculator&sequence=1k%2F1k', + ), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual([ + { benchmark_type: 'single_turn', isl: 1024, osl: 1024, metrics: { tput_per_gpu: 100 } }, + ]); + }); + + it('composes with the agentic workflow-metadata trim', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1 }, + }, + { + id: 2, + benchmark_type: 'single_turn', + workflow_run_id: 43, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1 }, + }, + { + id: 3, + benchmark_type: 'agentic_traces', + workflow_run_id: 44, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 0 }, + }, + ]); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1')); + expect(await res.json()).toEqual([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + metrics: { power_valid: 1 }, + }, + { id: 2, benchmark_type: 'single_turn', metrics: { power_valid: 1 } }, + ]); + }); + }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/route.ts b/packages/app/src/app/api/v1/benchmarks/route.ts index a75260d7b..887bb2a07 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.ts @@ -11,6 +11,7 @@ import { import { cachedJson, cachedQuery } from '@/lib/api-cache'; import { toCalculatorBenchmarkRows } from '@/lib/benchmark-api-view'; +import { filterByPowerValidity, parsePowerValidityFilter } from '@/lib/benchmark-power-validity'; import { PUBLIC_API_ERRORS, publicApiError } from '@/lib/public-api-errors'; import { agenticWorkflowMetadataOnly } from '@/lib/agentic-workflow-metadata'; import { loadFixture } from '@/lib/test-fixtures'; @@ -52,6 +53,7 @@ export async function GET(request: NextRequest) { const exactRun = params.get('exactRun') === 'true'; const view = params.get('view'); const sequence = params.get('sequence') ?? ''; + const powerValidFilter = parsePowerValidityFilter(params.get('powerValid')); const dbModelKeys = DISPLAY_MODEL_TO_DB[model]; if (!dbModelKeys || dbModelKeys.length === 0) { return publicApiError(PUBLIC_API_ERRORS.unknownModel, 400); @@ -59,10 +61,23 @@ export async function GET(request: NextRequest) { if (view === 'calculator' && !['1k/1k', '1k/8k', '8k/1k', 'agentic-traces'].includes(sequence)) { return NextResponse.json({ error: 'Unknown calculator sequence' }, { status: 400 }); } + if (powerValidFilter === undefined) { + return NextResponse.json({ error: 'Unknown powerValid filter' }, { status: 400 }); + } + // The calculator cache stores rows already trimmed to an allowlist that + // excludes power_valid, so post-cache filtering cannot work there. + if (view === 'calculator' && powerValidFilter !== 'any') { + return NextResponse.json( + { error: 'powerValid cannot be combined with view=calculator' }, + { status: 400 }, + ); + } if (FIXTURES_MODE) { const fixture = loadFixture('benchmarks'); return cachedJson( - view === 'calculator' ? toCalculatorBenchmarkRows(fixture, sequence) : fixture, + view === 'calculator' + ? toCalculatorBenchmarkRows(fixture, sequence) + : filterByPowerValidity(fixture, powerValidFilter), ); } @@ -73,7 +88,11 @@ export async function GET(request: NextRequest) { : exactRun && runId ? await getCachedBenchmarksForRun(dbModelKeys, runId) : await getCachedBenchmarks(dbModelKeys, date, exact || undefined, runId); - return cachedJson(agenticWorkflowMetadataOnly(rows)); + return cachedJson( + agenticWorkflowMetadataOnly( + view === 'calculator' ? rows : filterByPowerValidity(rows, powerValidFilter), + ), + ); } catch (error) { console.error('Error fetching benchmarks:', error); return publicApiError(PUBLIC_API_ERRORS.internal, 500); diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index f93535dca..a272f824b 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -108,7 +108,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-benchmarks', - sourceSha256: 'c6a5b78108b7e0d523b11590e1e34ef2e8c2d5457673eb41338d93d3d8f04909', + sourceSha256: 'b736b302cdb294bb316ff4666a64b47432655772f442aa8f5c95fe1f3cc9ed38', }, { source: 'src/app/api/v1/benchmarks/history/route.ts', diff --git a/packages/app/src/lib/benchmark-power-validity.test.ts b/packages/app/src/lib/benchmark-power-validity.test.ts new file mode 100644 index 000000000..5d11230ee --- /dev/null +++ b/packages/app/src/lib/benchmark-power-validity.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +import { + filterByPowerValidity, + parsePowerValidityFilter, + POWER_VALIDITY_FILTERS, +} from './benchmark-power-validity'; + +describe('parsePowerValidityFilter', () => { + it('treats an absent param as any', () => { + expect(parsePowerValidityFilter(null)).toBe('any'); + }); + + it('accepts every listed filter value', () => { + for (const filter of POWER_VALIDITY_FILTERS) { + expect(parsePowerValidityFilter(filter)).toBe(filter); + } + }); + + it('rejects unknown values', () => { + expect(parsePowerValidityFilter('garbage')).toBeUndefined(); + expect(parsePowerValidityFilter('')).toBeUndefined(); + // The pre-rename spelling must not silently alias to strictV2. + expect(parsePowerValidityFilter('certified')).toBeUndefined(); + }); + + it('is case-sensitive', () => { + expect(parsePowerValidityFilter('ANY')).toBeUndefined(); + expect(parsePowerValidityFilter('strictv2')).toBeUndefined(); + }); +}); + +describe('filterByPowerValidity', () => { + const validatedV2 = { id: 1, metrics: { power_valid: 1, power_metric_schema_version: 2 } }; + const validatedUnversioned = { id: 2, metrics: { power_valid: 1 } }; + const invalidated = { id: 3, metrics: { power_valid: 0, power_metric_schema_version: 2 } }; + const legacy = { id: 4, metrics: { tput_per_gpu: 100 } }; + const noMetrics = { id: 5 } as { id: number; metrics?: Record }; + const rows = [validatedV2, validatedUnversioned, invalidated, legacy, noMetrics]; + + it('returns every row unchanged for any', () => { + const result = filterByPowerValidity(rows, 'any'); + expect(result).toEqual(rows); + expect(result).not.toBe(rows); + }); + + it('keeps only explicitly validated rows for 1', () => { + expect(filterByPowerValidity(rows, '1')).toEqual([validatedV2, validatedUnversioned]); + }); + + it('keeps only explicitly invalidated rows for 0', () => { + expect(filterByPowerValidity(rows, '0')).toEqual([invalidated]); + }); + + it('requires a validated verdict and schema version 2 for strictV2', () => { + expect(filterByPowerValidity(rows, 'strictV2')).toEqual([validatedV2]); + }); + + it('excludes legacy and metric-less rows from every filter except any', () => { + for (const filter of ['1', '0', 'strictV2'] as const) { + const surviving = filterByPowerValidity([legacy, noMetrics], filter); + expect(surviving).toEqual([]); + } + }); +}); diff --git a/packages/app/src/lib/benchmark-power-validity.ts b/packages/app/src/lib/benchmark-power-validity.ts new file mode 100644 index 000000000..6b7d98ec6 --- /dev/null +++ b/packages/app/src/lib/benchmark-power-validity.ts @@ -0,0 +1,42 @@ +/** + * Measured-power validity filtering for the public benchmarks API. + * + * `metrics.power_valid` is tri-state: 1 means the measurement window was + * validated, an explicit 0 is an authoritative invalid verdict (measured + * values are withheld end-to-end), and an absent key marks a legacy row that + * predates validation. `strictV2` additionally requires + * `power_metric_schema_version === 2`, mirroring + * `WHOLE_DEPLOYMENT_ENERGY_SCHEMA_VERSION` in `benchmark-transform.ts` and + * `POWER_METRIC_SCHEMA_VERSION` in the runner's `utils/aggregate_power.py` — + * only version 2 defines unprefixed `joules_per_*` fields as whole-deployment + * energy. The name is deliberately not `certified`: the UI's certified tier is + * a display rule that also admits validated legacy rows without a schema + * version, which `strictV2` excludes. + */ +export const POWER_VALIDITY_FILTERS = ['1', '0', 'any', 'strictV2'] as const; +export type PowerValidityFilter = (typeof POWER_VALIDITY_FILTERS)[number]; + +/** Absent param means no filtering; unknown values return undefined so the caller can 400. */ +export function parsePowerValidityFilter(raw: string | null): PowerValidityFilter | undefined { + if (raw === null) return 'any'; + return (POWER_VALIDITY_FILTERS as readonly string[]).includes(raw) + ? (raw as PowerValidityFilter) + : undefined; +} + +/** + * Pure post-cache row filter. Rows without `metrics` or without a + * `power_valid` verdict (legacy rows) match only `any`. + */ +export function filterByPowerValidity }>( + rows: readonly T[], + filter: PowerValidityFilter, +): T[] { + if (filter === 'any') return [...rows]; + return rows.filter((row) => { + const powerValid = row.metrics?.power_valid; + if (filter === '1') return powerValid === 1; + if (filter === '0') return powerValid === 0; + return powerValid === 1 && row.metrics?.power_metric_schema_version === 2; + }); +} From ad561480d42d9dd31544cb791aa269bbf49713b2 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 15:57:07 -0700 Subject: [PATCH 07/11] =?UTF-8?q?docs(api):=20document=20the=20measured-po?= =?UTF-8?q?wer=20contract=20on=20/api/v1/benchmarks=20|=20=E6=96=87?= =?UTF-8?q?=E6=A1=A3=EF=BC=9A=E4=B8=BA=20/api/v1/benchmarks=20=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E5=8C=96=E5=AE=9E=E6=B5=8B=E5=8A=9F=E7=8E=87=E5=A5=91?= =?UTF-8?q?=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Types every POWER_METRIC_KEYS entry on the BenchmarkRow metrics schema (additionalProperties still admits non-power keys), reserves the forthcoming power_invalid_reasons / power_audit row fields from the cross-plan producer contract, documents the previously-undocumented view/sequence params plus the new powerValid param, adds a bilingual measured-power schema note (tri-state power_valid, schema-version-2 whole-deployment energy, workers[] relationship, strictV2-vs-UI divergence), and mirrors the parameter list in the stable contract ledger. Cypress asserts the note and param render on /api and /zh/api. --- .../app/cypress/e2e/api-documentation.cy.ts | 11 +- .../src/lib/api-documentation.power.test.ts | 85 +++++++++++ packages/app/src/lib/api-documentation.ts | 135 ++++++++++++++++-- packages/app/src/lib/api-route-catalog.ts | 2 +- 4 files changed, 222 insertions(+), 11 deletions(-) create mode 100644 packages/app/src/lib/api-documentation.power.test.ts diff --git a/packages/app/cypress/e2e/api-documentation.cy.ts b/packages/app/cypress/e2e/api-documentation.cy.ts index f22f9e9b5..fe0503a63 100644 --- a/packages/app/cypress/e2e/api-documentation.cy.ts +++ b/packages/app/cypress/e2e/api-documentation.cy.ts @@ -9,7 +9,9 @@ describe('API documentation', () => { .and('contain.text', 'Quickstart') .and('contain.text', 'curl') .and('contain.text', '/api/v1/availability') - .and('contain.text', 'Endpoint reference'); + .and('contain.text', 'Endpoint reference') + .and('contain.text', 'Measured power') + .and('contain.text', 'powerValid'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('[data-testid="api-spec-version"]').should('have.text', 'v1 · OpenAPI 3.1'); cy.get('[data-testid="api-endpoint-list-benchmarks"]') @@ -45,6 +47,10 @@ describe('API documentation', () => { 'operationId', 'list-benchmarks', ); + const benchmarkParameterNames = body.paths['/api/v1/benchmarks'].get.parameters.map( + (parameter: { name: string }) => parameter.name, + ); + expect(benchmarkParameterNames).to.include('powerValid'); expect(body.paths['/api/v1/collectivex/runs/{runId}'].get).to.have.property( 'operationId', 'get-collectivex-run', @@ -58,7 +64,8 @@ describe('API documentation', () => { .and('contain.text', '快速入门') .and('contain.text', '约定') .and('contain.text', '端点参考') - .and('contain.text', 'BenchmarkRow 与指标'); + .and('contain.text', 'BenchmarkRow 与指标') + .and('contain.text', '实测功率'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('link[rel="alternate"][hreflang="en"]').should('have.attr', 'href', `${SITE_URL}/api`); cy.get('link[rel="alternate"][hreflang="zh-CN"]').should( diff --git a/packages/app/src/lib/api-documentation.power.test.ts b/packages/app/src/lib/api-documentation.power.test.ts new file mode 100644 index 000000000..32f26b72e --- /dev/null +++ b/packages/app/src/lib/api-documentation.power.test.ts @@ -0,0 +1,85 @@ +import { POWER_METRIC_KEYS } from '@semianalysisai/inferencex-constants'; +import { describe, expect, it } from 'vitest'; + +import { apiOperations, buildOpenApiDocument, getApiDocumentation } from './api-documentation'; +import { POWER_VALIDITY_FILTERS } from './benchmark-power-validity'; + +const listBenchmarks = apiOperations.find((operation) => operation.id === 'list-benchmarks'); +const benchmarkRowSchema = listBenchmarks?.responses.find((response) => response.status === '200') + ?.schema.items; + +describe('measured-power API documentation', () => { + it('types every power metric key in the benchmarks metrics schema', () => { + const metricsSchema = benchmarkRowSchema?.properties?.metrics; + expect(metricsSchema).toBeDefined(); + // Non-power keys stay admitted alongside the typed power properties. + expect(metricsSchema?.additionalProperties).toEqual({ type: 'number' }); + for (const key of POWER_METRIC_KEYS) { + const property = metricsSchema?.properties?.[key]; + expect(property?.type, `${key} must be a typed number property`).toBe('number'); + expect( + property?.description?.trim(), + `${key} must carry a nonempty description`, + ).toBeTruthy(); + } + }); + + it('reserves optional power_invalid_reasons and power_audit row fields', () => { + const reasons = benchmarkRowSchema?.properties?.power_invalid_reasons; + expect(reasons?.type).toBe('array'); + expect(reasons?.items).toEqual({ type: 'string' }); + + const audit = benchmarkRowSchema?.properties?.power_audit; + expect(Object.keys(audit?.properties ?? {}).toSorted()).toEqual( + [ + 'window_start_unix', + 'window_end_unix', + 'expected_gpu_count', + 'observed_gpu_count', + 'sample_count', + 'max_sample_gap_s', + 'producer_sha', + 'exporter_image_sha256', + ].toSorted(), + ); + // PLAN-07's mapper stores partial audits, so no audit field may be required. + expect(audit?.required).toBeUndefined(); + + expect(benchmarkRowSchema?.required).not.toContain('power_invalid_reasons'); + expect(benchmarkRowSchema?.required).not.toContain('power_audit'); + }); + + it('projects the view, sequence, and powerValid parameters into OpenAPI', () => { + const document = buildOpenApiDocument('https://api-docs.test'); + const operation = (document.paths['/api/v1/benchmarks'] as Record).get as { + parameters: readonly { name: string; schema: unknown }[]; + }; + const names = operation.parameters.map((parameter) => parameter.name); + expect(names).toContain('view'); + expect(names).toContain('sequence'); + + const powerValid = operation.parameters.find((parameter) => parameter.name === 'powerValid'); + expect(powerValid?.schema).toEqual({ + type: 'string', + enum: ['1', '0', 'any', 'strictV2'], + default: 'any', + }); + // The documented enum comes from the filter module, so route and docs cannot drift. + expect([...POWER_VALIDITY_FILTERS]).toEqual(['1', '0', 'any', 'strictV2']); + }); + + it('renders a bilingual measured-power schema note', () => { + for (const locale of ['en', 'zh'] as const) { + const note = getApiDocumentation(locale).schemaNotes.find( + (candidate) => candidate.id === 'measured-power', + ); + expect(note, `${locale} must expose a measured-power schema note`).toBeDefined(); + expect(note?.title.trim()).toBeTruthy(); + expect(note?.description.trim()).toBeTruthy(); + } + const zhNote = getApiDocumentation('zh').schemaNotes.find( + (candidate) => candidate.id === 'measured-power', + ); + expect(zhNote?.description).toMatch(/[㐀-鿿]/u); + }); +}); diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 4a22d16c5..cff686565 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -1,6 +1,11 @@ -import { DB_MODEL_TO_DISPLAY, DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; +import { + DB_MODEL_TO_DISPLAY, + DISPLAY_MODEL_TO_DB, + POWER_METRIC_KEYS, +} from '@semianalysisai/inferencex-constants'; import { COLLECTIVEX_VERSIONS } from '@semianalysisai/inferencex-db/collectivex/types'; +import { POWER_VALIDITY_FILTERS } from './benchmark-power-validity'; import { PUBLIC_API_ERRORS } from './public-api-errors'; export type ApiDocumentationLocale = 'en' | 'zh'; @@ -192,6 +197,59 @@ const errorResponse = ( mediaType: 'application/json', }); +const powerMetricDescriptions: Readonly> = { + power_valid: + 'Publication verdict: 1 = validated measurement window; 0 = failed validation — measured power/energy values are withheld from this row end-to-end, so treat any that remain as unreliable; absent = legacy row predating validation.', + power_metric_schema_version: + 'Power schema version. Version 2 defines every unprefixed joules_per_* field as whole-deployment energy, including on disaggregated runs.', + avg_power_w: 'Mean per-GPU power draw in watts during the measured load window.', + joules_per_successful_query: 'Whole-deployment energy in joules divided by successful requests.', + joules_per_output_token: + 'Energy per generated output token in joules; cluster-wide on schema-version-2 rows, including disaggregated runs.', + joules_per_total_token: + 'Total system energy divided by input plus output tokens; a workload-shape-fair view that does not treat prompt tokens as free.', + prefill_avg_power_w: + 'Mean per-GPU power draw in watts across prefill workers; emitted only for deployments with distinct prefill and decode roles.', + decode_avg_power_w: + 'Mean per-GPU power draw in watts across decode workers; emitted only for deployments with distinct prefill and decode roles.', + joules_per_input_token: + 'Energy per input token in joules; cluster-wide on schema-version-2 rows.', + prefill_joules_per_input_token: 'Role-local prefill energy per input token in joules.', + decode_joules_per_output_token: 'Role-local decode energy per generated output token in joules.', + avg_temp_c: 'Mean per-GPU temperature in degrees Celsius during the load window.', + peak_temp_c: + 'Maximum instantaneous per-GPU temperature in degrees Celsius during the load window.', + avg_util_pct: 'Mean per-GPU utilization percentage (0-100) during the load window.', + avg_mem_used_mb: 'Mean per-GPU memory used in MB during the load window.', +}; +const benchmarkMetricsSchema: ApiSchema = { + type: 'object', + additionalProperties: numberSchema, + description: + 'Scalar metric map. Keys evolve independently; measured power / energy / GPU-telemetry keys are typed below.', + properties: Object.fromEntries( + POWER_METRIC_KEYS.map((key): [string, ApiSchema] => [ + key, + { type: 'number', description: powerMetricDescriptions[key] }, + ]), + ), +}; +const powerAuditSchema: ApiSchema = { + type: 'object', + properties: { + window_start_unix: numberSchema, + window_end_unix: numberSchema, + expected_gpu_count: integerSchema, + observed_gpu_count: integerSchema, + sample_count: integerSchema, + max_sample_gap_s: numberSchema, + producer_sha: nullableStringSchema, + exporter_image_sha256: nullableStringSchema, + }, + additionalProperties: true, + description: + 'Reserved (forthcoming): power measurement-window audit emitted by newer producers. Absent on legacy rows.', +}; const workerPowerSchema = objectSchemaWithOptional( { role: stringSchema, @@ -234,14 +292,20 @@ const benchmarkRowSchema = objectSchemaWithOptional( offload_mode: stringSchema, image: nullableStringSchema, recipe_fingerprint: nullableStringSchema, - metrics: metricMapSchema, + metrics: benchmarkMetricsSchema, workers: arraySchema(workerPowerSchema), + power_invalid_reasons: { + ...arraySchema(stringSchema), + description: + 'Reserved (forthcoming): snake_case reason codes, present when metrics.power_valid == 0. Absent on legacy rows.', + }, + power_audit: powerAuditSchema, date: { type: 'string', format: 'date' }, workflow_run_id: integerSchema, run_started_at: { type: ['string', 'null'], format: 'date-time' }, run_url: nullableStringSchema, }, - ['workers', 'workflow_run_id', 'run_started_at'], + ['workers', 'power_invalid_reasons', 'power_audit', 'workflow_run_id', 'run_started_at'], ); const benchmarkRowsSchema = arraySchema(benchmarkRowSchema); const benchmarkExample = [ @@ -271,7 +335,17 @@ const benchmarkExample = [ offload_mode: 'off', image: 'vllm/vllm-openai:v0.10.2', recipe_fingerprint: '7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d7d72a33d', - metrics: { median_ttft: 0.42, median_tpot: 0.018, tput_per_gpu: 128.4 }, + metrics: { + median_ttft: 0.42, + median_tpot: 0.018, + tput_per_gpu: 128.4, + power_valid: 1, + power_metric_schema_version: 2, + avg_power_w: 678.5, + joules_per_output_token: 5.3, + joules_per_total_token: 2.65, + avg_temp_c: 61.2, + }, date: '2026-08-08', run_url: 'https://github.com/semianalysis/inference-benchmarks/actions/runs/123456789', }, @@ -589,8 +663,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/benchmarks', summary: text('Read benchmark results', '读取基准结果'), description: text( - 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. The page-owned calculator view is not part of this public contract.', - '按展示模型返回原始基准行。可用 date 获取截至该日的快照，exact=true 限定该日，runId 约束最新查询，或将 exactRun=true 与数字 runId 组合以仅返回该工作流运行。页面专用的计算器视图不属于此公开契约。', + 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change), and powerValid filters rows by measured-power validity and cannot be combined with view=calculator.', + '按展示模型返回原始基准行。可用 date 获取截至该日的快照，exact=true 限定该日，runId 约束最新查询，或将 exactRun=true 与数字 runId 组合以仅返回该工作流运行。view=calculator 返回页面专用的裁剪投影（会移除实测功率指标和 workers，其允许列表可能变化）；powerValid 按实测功率有效性筛选行，且不能与 view=calculator 组合使用。', ), audience: 'public', stability: 'stable', @@ -645,6 +719,36 @@ export const apiOperations: readonly ApiOperation[] = [ { type: 'boolean', default: false }, false, ), + parameter( + 'view', + 'query', + false, + 'enum', + 'calculator trims each row to the page-owned metric allowlist the throughput calculator consumes and removes workers; measured power metrics are excluded from this view. Requires sequence. Omit for every stored metric, including measured power.', + 'calculator 会将每行裁剪为吞吐量计算器所需的页面专用指标允许列表并移除 workers；此视图不包含实测功率指标。需要同时提供 sequence。省略则返回全部已存储指标，包括实测功率。', + { type: 'string', enum: ['calculator'] }, + 'calculator', + ), + parameter( + 'sequence', + 'query', + false, + 'enum', + 'Required when view=calculator and ignored otherwise. Unknown values yield 400 Unknown calculator sequence.', + '当 view=calculator 时必填，其余情况会被忽略。未知值返回 400 Unknown calculator sequence。', + { type: 'string', enum: ['1k/1k', '1k/8k', '8k/1k', 'agentic-traces'] }, + '1k/1k', + ), + parameter( + 'powerValid', + 'query', + false, + 'enum', + '1 keeps only rows with a validated power measurement (metrics.power_valid == 1); 0 keeps only explicitly invalidated rows; any applies no filter (default; includes legacy rows without a verdict); strictV2 keeps rows with power_valid == 1 and power_metric_schema_version == 2 (whole-deployment energy semantics) — stricter than the InferenceX UI, which also displays validated rows that predate schema versioning. Unknown values yield 400 Unknown powerValid filter; cannot be combined with view=calculator.', + '1 仅保留具有已验证功率测量的行（metrics.power_valid == 1）；0 仅保留被明确判定无效的行；any 不做筛选（默认值；包含没有判定结果的旧数据行）；strictV2 保留 power_valid == 1 且 power_metric_schema_version == 2（全部署能耗语义）的行——比 InferenceX 界面更严格，界面还会展示早于版本标注机制的已验证行。未知值返回 400 Unknown powerValid filter；不能与 view=calculator 组合使用。', + { type: 'string', enum: POWER_VALIDITY_FILTERS, default: 'any' }, + 'strictV2', + ), ], responses: [ success( @@ -655,8 +759,8 @@ export const apiOperations: readonly ApiOperation[] = [ ), errorResponse( '400', - 'The model is missing or unsupported.', - '模型缺失或不受支持。', + 'The model is missing or unsupported, the calculator sequence is unknown, the powerValid filter is unknown, or powerValid is combined with view=calculator.', + '模型缺失或不受支持、计算器序列未知、powerValid 筛选值未知，或 powerValid 与 view=calculator 组合使用。', PUBLIC_API_ERRORS.unknownModel, ), errorResponse( @@ -2555,6 +2659,21 @@ const overview = { shape: 'BenchmarkRows', example: benchmarkExample[0], }, + { + id: 'measured-power', + title: text('Measured power', '实测功率'), + description: text( + 'Benchmark rows may carry measured power, energy, and GPU-telemetry metric keys (avg_power_w, joules_per_*, avg_temp_c, peak_temp_c, avg_util_pct, avg_mem_used_mb). power_valid is tri-state: 1 means the measurement window was validated; 0 means validation failed and measured values are withheld end-to-end (the producer strips them and ingest scrubs them — treat any that remain as unreliable); absent marks a legacy row predating validation. power_metric_schema_version == 2 defines every unprefixed joules_per_* field as whole-deployment energy — unversioned disaggregated joules are ambiguous because those fields previously carried role-local values. workers[] carries the per-worker power/telemetry breakdown on multinode and disaggregated runs. power_invalid_reasons and power_audit are reserved forthcoming fields. Filter rows with the powerValid request parameter; its strict value is named strictV2 (power_valid == 1 and power_metric_schema_version == 2) rather than "certified" because it is stricter than the InferenceX UI, which also displays validated rows that predate schema versioning.', + '基准行可能携带实测功率、能耗和 GPU 遥测指标键（avg_power_w、joules_per_*、avg_temp_c、peak_temp_c、avg_util_pct、avg_mem_used_mb）。power_valid 为三态：1 表示测量窗口已通过验证；0 表示验证失败，实测值会被全链路扣留（生产端剥离、摄取端再次清除——若仍残留请视为不可靠）；缺失表示早于验证机制的旧数据行。power_metric_schema_version == 2 将所有无前缀的 joules_per_* 字段定义为全部署能耗——未标注版本的分离式 joules 含义不明确，因为这些字段曾承载角色本地值。多节点和分离式运行的每 worker 功率/遥测明细位于 workers[]。power_invalid_reasons 与 power_audit 为预留的即将推出字段。可使用 powerValid 请求参数筛选行；其严格取值命名为 strictV2（power_valid == 1 且 power_metric_schema_version == 2）而非 "certified"，因为它比 InferenceX 界面更严格——界面还会展示早于版本标注机制的已验证行。', + ), + shape: 'BenchmarkRows', + example: { + power_valid: 1, + power_metric_schema_version: 2, + avg_power_w: 678.5, + joules_per_output_token: 5.3, + }, + }, { id: 'metric-maps', title: text('ID-keyed maps', '以 ID 为键的映射'), diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index a272f824b..912518db8 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -443,7 +443,7 @@ export const stablePublicApiContracts = [ }, { operationId: 'list-benchmarks', - parameters: ['model', 'date', 'exact', 'runId', 'exactRun'], + parameters: ['model', 'date', 'exact', 'runId', 'exactRun', 'view', 'sequence', 'powerValid'], statuses: ['200', '400', '500'], auth: 'none', cachePolicy: 'public-db-day', From c848016af0248889295c6a0a6205656d164aad39 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 27 Aug 2026 16:15:28 -0700 Subject: [PATCH 08/11] =?UTF-8?q?docs(api):=20note=20the=20powerValid=3Dan?= =?UTF-8?q?y=20exception=20to=20the=20calculator-view=20400=20|=20?= =?UTF-8?q?=E6=96=87=E6=A1=A3=EF=BC=9A=E8=AF=B4=E6=98=8E=20powerValid=3Dan?= =?UTF-8?q?y=20=E5=8F=AF=E4=B8=8E=20view=3Dcalculator=20=E7=BB=84=E5=90=88?= =?UTF-8?q?=E7=9A=84=E4=BE=8B=E5=A4=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding: the route only rejects non-any powerValid with view=calculator, but the published copy claimed the combination is always invalid. Align the operation description, the powerValid param description, and the 400 response description (EN + ZH) with actual behavior. --- packages/app/src/lib/api-documentation.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index cff686565..190b7d542 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -663,8 +663,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/benchmarks', summary: text('Read benchmark results', '读取基准结果'), description: text( - 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change), and powerValid filters rows by measured-power validity and cannot be combined with view=calculator.', - '按展示模型返回原始基准行。可用 date 获取截至该日的快照，exact=true 限定该日，runId 约束最新查询，或将 exactRun=true 与数字 runId 组合以仅返回该工作流运行。view=calculator 返回页面专用的裁剪投影（会移除实测功率指标和 workers，其允许列表可能变化）；powerValid 按实测功率有效性筛选行，且不能与 view=calculator 组合使用。', + 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change), and powerValid filters rows by measured-power validity and cannot be combined with view=calculator (except powerValid=any, which is a no-op).', + '按展示模型返回原始基准行。可用 date 获取截至该日的快照，exact=true 限定该日，runId 约束最新查询，或将 exactRun=true 与数字 runId 组合以仅返回该工作流运行。view=calculator 返回页面专用的裁剪投影（会移除实测功率指标和 workers，其允许列表可能变化）；powerValid 按实测功率有效性筛选行，且不能与 view=calculator 组合使用（powerValid=any 除外，等同于不筛选）。', ), audience: 'public', stability: 'stable', @@ -744,8 +744,8 @@ export const apiOperations: readonly ApiOperation[] = [ 'query', false, 'enum', - '1 keeps only rows with a validated power measurement (metrics.power_valid == 1); 0 keeps only explicitly invalidated rows; any applies no filter (default; includes legacy rows without a verdict); strictV2 keeps rows with power_valid == 1 and power_metric_schema_version == 2 (whole-deployment energy semantics) — stricter than the InferenceX UI, which also displays validated rows that predate schema versioning. Unknown values yield 400 Unknown powerValid filter; cannot be combined with view=calculator.', - '1 仅保留具有已验证功率测量的行（metrics.power_valid == 1）；0 仅保留被明确判定无效的行；any 不做筛选（默认值；包含没有判定结果的旧数据行）；strictV2 保留 power_valid == 1 且 power_metric_schema_version == 2（全部署能耗语义）的行——比 InferenceX 界面更严格，界面还会展示早于版本标注机制的已验证行。未知值返回 400 Unknown powerValid filter；不能与 view=calculator 组合使用。', + '1 keeps only rows with a validated power measurement (metrics.power_valid == 1); 0 keeps only explicitly invalidated rows; any applies no filter (default; includes legacy rows without a verdict); strictV2 keeps rows with power_valid == 1 and power_metric_schema_version == 2 (whole-deployment energy semantics) — stricter than the InferenceX UI, which also displays validated rows that predate schema versioning. Unknown values yield 400 Unknown powerValid filter; cannot be combined with view=calculator (except any, which is a no-op).', + '1 仅保留具有已验证功率测量的行（metrics.power_valid == 1）；0 仅保留被明确判定无效的行；any 不做筛选（默认值；包含没有判定结果的旧数据行）；strictV2 保留 power_valid == 1 且 power_metric_schema_version == 2（全部署能耗语义）的行——比 InferenceX 界面更严格，界面还会展示早于版本标注机制的已验证行。未知值返回 400 Unknown powerValid filter；不能与 view=calculator 组合使用（any 除外，等同于不筛选）。', { type: 'string', enum: POWER_VALIDITY_FILTERS, default: 'any' }, 'strictV2', ), @@ -759,8 +759,8 @@ export const apiOperations: readonly ApiOperation[] = [ ), errorResponse( '400', - 'The model is missing or unsupported, the calculator sequence is unknown, the powerValid filter is unknown, or powerValid is combined with view=calculator.', - '模型缺失或不受支持、计算器序列未知、powerValid 筛选值未知，或 powerValid 与 view=calculator 组合使用。', + 'The model is missing or unsupported, the calculator sequence is unknown, the powerValid filter is unknown, or a non-any powerValid is combined with view=calculator.', + '模型缺失或不受支持、计算器序列未知、powerValid 筛选值未知，或非 any 的 powerValid 与 view=calculator 组合使用。', PUBLIC_API_ERRORS.unknownModel, ), errorResponse( From 701fbbacc823f7101c2abb147a7f1c788bd611a1 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Mon, 31 Aug 2026 13:50:06 -0700 Subject: [PATCH 09/11] chore(etl): clarify power scrub comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：清理功耗剥离相关测试与实现注释，移除内部计划编号，并保留关键的故障关闭与摄取顺序语义。 --- .../app/src/lib/benchmark-transform.test.ts | 6 ----- packages/constants/src/metric-keys.test.ts | 9 +------ packages/constants/src/metric-keys.ts | 13 +++------- packages/db/src/etl/benchmark-mapper.test.ts | 25 +++--------------- packages/db/src/etl/benchmark-mapper.ts | 26 ++++++------------- 5 files changed, 16 insertions(+), 63 deletions(-) diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index a95a9ddf1..c4c28c67d 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -426,12 +426,6 @@ describe('rowToAggDataEntry', () => { }); it('withholds every MEASURED_POWER_METRIC_KEYS field when power_valid=0 (ETL parity)', () => { - // Parity with the ingest scrub (benchmark-mapper.ts - // scrubWithheldPowerMetrics): the shared constant is the single source of - // truth for the withheld set, so a key added there without a matching - // display-layer withholding fails here. The reverse direction — the - // display layer withholding a key missing from the constant — stays - // hand-audited. const metrics: BenchmarkRow['metrics'] = { power_valid: 0 }; for (const key of MEASURED_POWER_METRIC_KEYS) metrics[key] = 123.45; const entry = rowToAggDataEntry( diff --git a/packages/constants/src/metric-keys.test.ts b/packages/constants/src/metric-keys.test.ts index b7213086a..fc606267a 100644 --- a/packages/constants/src/metric-keys.test.ts +++ b/packages/constants/src/metric-keys.test.ts @@ -14,9 +14,6 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { }); it('contains exactly the 13 measured power / energy / telemetry keys', () => { - // Guards accidental additions/removals: the ingest scrub and the display - // withholding both key off this set, so membership changes are policy - // changes and must be deliberate. expect(new Set(MEASURED_POWER_METRIC_KEY_LIST)).toEqual( new Set([ 'avg_power_w', @@ -38,11 +35,7 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { }); it('never contains the contract discriminators or invalid-verdict companion fields', () => { - // power_valid / power_metric_schema_version are the verdict itself and - // must survive a scrub; power_invalid_reasons / power_audit are the - // producer's explanation of an invalid verdict (persisted app-side by - // PLAN-07) and are only meaningful on scrubbed rows. None of them may - // ever be added to the withheld set. + // These fields describe or explain withholding; they are not measurements. for (const key of [ 'power_valid', 'power_metric_schema_version', diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index fac99b513..70cf69f6c 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -1,12 +1,8 @@ /** - * Measured power / energy / GPU-telemetry metrics that MUST be withheld - * whenever a row carries an invalid power verdict (power_valid = 0). - * Excludes the contract discriminators (power_valid, - * power_metric_schema_version) and the invalid-verdict companion fields - * (power_invalid_reasons, power_audit), which are always kept. Add any new - * measured power/telemetry key here — it feeds METRIC_KEYS automatically. - * Mirrored by the display-layer withholding in - * packages/app/src/lib/benchmark-transform.ts (rowToAggDataEntry). + * Power, energy, and GPU telemetry withheld at ingest and display when the + * normalized `power_valid` verdict is 0. Contract and diagnostic fields are + * excluded so the invalid verdict remains auditable. Add new measured fields + * here; `METRIC_KEYS` derives from this list. */ export const MEASURED_POWER_METRIC_KEY_LIST = [ // measured power / energy (emitted by runner's aggregate_power.py) @@ -187,7 +183,6 @@ export const METRIC_KEYS = new Set([ // joules_per_* field as whole-deployment energy 'power_valid', 'power_metric_schema_version', - // measured power / energy / telemetry values, withheld when power_valid = 0 ...MEASURED_POWER_METRIC_KEY_LIST, // extended parallelism dimensions (2026-07+ artifacts): pipeline parallelism // and decode/prefill context parallelism per role. These are config diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 7ca31c88d..0bc4f3b1a 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -67,11 +67,6 @@ function makeV2Row(overrides: Record = {}): Record { }; } -/** - * All 13 measured power/energy/telemetry keys with distinct sentinel values, - * plus a valid 2-entry per-worker payload — the shape a producer regression - * would ship if it stopped stripping measurements on power_valid=0 rows. - */ function dirtyPowerPayload(): Record { return { avg_power_w: 685.5, @@ -334,8 +329,6 @@ describe('mapBenchmarkRow', () => { expect(result!.metrics).not.toHaveProperty(key); } expect(result!.workers).toBeUndefined(); - // Non-power metrics survive untouched — a power_valid=0 row is still a - // perfectly valid performance result. expect(result!.metrics.tput_per_gpu).toBe(567.8); expect(result!.metrics.median_ttft).toBe(50.2); }); @@ -385,9 +378,8 @@ describe('mapBenchmarkRow', () => { ); it('converges a dirty pv=0 artifact to exactly what a clean producer would ship', () => { - // Re-ingest replaces metrics and workers wholesale (ON CONFLICT DO - // UPDATE in benchmark-ingest.ts), so a scrubbed dirty row must be a - // fixed point identical to the producer-clean mapping. + // Upserts replace metrics and workers wholesale, so re-ingest must + // converge to the same row as a producer-clean artifact. const tracker = createSkipTracker(); const dirtyResult = mapBenchmarkRow( makeV2Row({ power_valid: 0, power_metric_schema_version: 2, ...dirtyPowerPayload() }), @@ -415,7 +407,6 @@ describe('mapBenchmarkRow', () => { ); expect(result!.metrics.power_valid).toBe(0); - // The reassignment ran: full-response ITL became canonical. expect(result!.metrics.mean_itl).toBe(0.02); for (const key of MEASURED_POWER_METRIC_KEYS) { expect(result!.metrics).not.toHaveProperty(key); @@ -423,12 +414,7 @@ describe('mapBenchmarkRow', () => { expect(result!.workers).toBeUndefined(); }); - it('tolerates the invalid-verdict companion fields without scrubbing them', () => { - // PLAN-06 producers ship power_invalid_reasons / power_audit alongside - // power_valid=0. The scrub must never touch them; persisting them - // app-side is PLAN-07 — today's pinned behavior is that non-numeric - // fields simply never land in the numeric metrics record (parseNum - // drops arrays/objects). + it('keeps structured invalid-verdict companions outside the numeric metrics record', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow( makeV2Row({ @@ -867,9 +853,6 @@ describe('mapBenchmarkRow', () => { }); describe('scrubWithheldPowerMetrics (direct — supplemental ingest path)', () => { - // ingest-supplemental.ts persists metrics without mapBenchmarkRow, calling - // normalizePowerContractMetrics + scrubWithheldPowerMetrics on the raw - // record directly. Pin that usage pattern here. function supplementalMetrics(overrides: Record = {}): Record { const { workers: _workers, ...measured } = dirtyPowerPayload(); return { tput_per_gpu: 567.8, ...measured, ...overrides }; @@ -897,8 +880,6 @@ describe('scrubWithheldPowerMetrics (direct — supplemental ingest path)', () = it('fails closed on a malformed verdict when composed with normalization', () => { const metrics = supplementalMetrics({ power_valid: 2 }); - // The exact ingest-supplemental.ts call sequence: same object as row - // and metrics. normalizePowerContractMetrics(metrics, metrics); expect(scrubWithheldPowerMetrics(metrics)).toBe(true); diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index e79945aff..0e6958f3b 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -347,9 +347,8 @@ export function mapBenchmarkRow( ? rawRecipeFingerprint.trim() : null; - // Scrub AFTER the last mutation of `metrics` (the agentic - // preferFullResponseMetrics reassignment and the extractRuntimeMetadata - // merge above) so no later step can resurrect a withheld key. + // Keep this after agentic reassignment and runtime metadata merging so no + // later mutation can reintroduce a withheld key. const powerWithheld = scrubWithheldPowerMetrics(metrics); // Per-worker measured-power breakdown. The runner emits this as an array // of objects sibling to the scalar metrics; we surface it on a dedicated @@ -499,21 +498,12 @@ export function normalizePowerContractMetrics( } /** - * Defense-in-depth for the power publication contract: when the - * normalized verdict is an explicit invalid (power_valid === 0), delete - * every measured power/energy/telemetry metric so withheld measurements - * can never be persisted or served, even if a producer regression ships - * them. Keeps power_valid and power_metric_schema_version (and any - * future companion fields such as power_invalid_reasons / power_audit). - * Legacy rows without a verdict are untouched. Returns true when the - * row's power is withheld so the caller also drops the workers payload. - * This is the single enforcement policy at ingest: every mapBenchmarkRow - * path runs it here, and ingest-supplemental.ts — the one persistence - * path that bypasses mapBenchmarkRow — applies the same normalize+scrub - * pair to its verbatim metrics. The query layer intentionally serves - * metrics unfiltered (see queries/benchmarks.ts rawMetrics), and the - * frontend independently withholds at display - * (benchmark-transform.ts rowToAggDataEntry). + * Enforces fail-closed power publication at ingest. An explicit normalized + * invalid verdict removes every measured field while preserving the contract + * and diagnostic fields; legacy rows without a verdict remain unchanged. + * Returns true so callers also drop worker telemetry. Paths that bypass + * `mapBenchmarkRow` must normalize the verdict before calling this function. + * Queries intentionally remain raw; the frontend withholds independently. */ export function scrubWithheldPowerMetrics(metrics: Record): boolean { if (metrics.power_valid !== 0) return false; From f8fb515eacc9068e750a8d8f03d6ab657a2dcc31 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Mon, 31 Aug 2026 13:53:03 -0700 Subject: [PATCH 10/11] chore(api): clarify measured-power contract comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：清理内部规划标签和冗余叙述，并以当前契约语义明确功率审计字段说明。 --- packages/app/src/lib/api-documentation.power.test.ts | 4 +--- packages/app/src/lib/api-documentation.ts | 8 ++++---- packages/app/src/lib/benchmark-power-validity.test.ts | 2 +- packages/constants/src/metric-keys.test.ts | 9 +-------- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/packages/app/src/lib/api-documentation.power.test.ts b/packages/app/src/lib/api-documentation.power.test.ts index 32f26b72e..94fd654b4 100644 --- a/packages/app/src/lib/api-documentation.power.test.ts +++ b/packages/app/src/lib/api-documentation.power.test.ts @@ -12,7 +12,6 @@ describe('measured-power API documentation', () => { it('types every power metric key in the benchmarks metrics schema', () => { const metricsSchema = benchmarkRowSchema?.properties?.metrics; expect(metricsSchema).toBeDefined(); - // Non-power keys stay admitted alongside the typed power properties. expect(metricsSchema?.additionalProperties).toEqual({ type: 'number' }); for (const key of POWER_METRIC_KEYS) { const property = metricsSchema?.properties?.[key]; @@ -42,7 +41,7 @@ describe('measured-power API documentation', () => { 'exporter_image_sha256', ].toSorted(), ); - // PLAN-07's mapper stores partial audits, so no audit field may be required. + // Producers may emit partial audits, so individual audit fields remain optional. expect(audit?.required).toBeUndefined(); expect(benchmarkRowSchema?.required).not.toContain('power_invalid_reasons'); @@ -64,7 +63,6 @@ describe('measured-power API documentation', () => { enum: ['1', '0', 'any', 'strictV2'], default: 'any', }); - // The documented enum comes from the filter module, so route and docs cannot drift. expect([...POWER_VALIDITY_FILTERS]).toEqual(['1', '0', 'any', 'strictV2']); }); diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 190b7d542..bb51fa166 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -248,7 +248,7 @@ const powerAuditSchema: ApiSchema = { }, additionalProperties: true, description: - 'Reserved (forthcoming): power measurement-window audit emitted by newer producers. Absent on legacy rows.', + 'Optional power measurement-window audit. Individual fields may be absent; legacy rows omit the object.', }; const workerPowerSchema = objectSchemaWithOptional( { @@ -297,7 +297,7 @@ const benchmarkRowSchema = objectSchemaWithOptional( power_invalid_reasons: { ...arraySchema(stringSchema), description: - 'Reserved (forthcoming): snake_case reason codes, present when metrics.power_valid == 0. Absent on legacy rows.', + 'Optional snake_case validation reason codes when metrics.power_valid == 0. Absent on legacy rows.', }, power_audit: powerAuditSchema, date: { type: 'string', format: 'date' }, @@ -2663,8 +2663,8 @@ const overview = { id: 'measured-power', title: text('Measured power', '实测功率'), description: text( - 'Benchmark rows may carry measured power, energy, and GPU-telemetry metric keys (avg_power_w, joules_per_*, avg_temp_c, peak_temp_c, avg_util_pct, avg_mem_used_mb). power_valid is tri-state: 1 means the measurement window was validated; 0 means validation failed and measured values are withheld end-to-end (the producer strips them and ingest scrubs them — treat any that remain as unreliable); absent marks a legacy row predating validation. power_metric_schema_version == 2 defines every unprefixed joules_per_* field as whole-deployment energy — unversioned disaggregated joules are ambiguous because those fields previously carried role-local values. workers[] carries the per-worker power/telemetry breakdown on multinode and disaggregated runs. power_invalid_reasons and power_audit are reserved forthcoming fields. Filter rows with the powerValid request parameter; its strict value is named strictV2 (power_valid == 1 and power_metric_schema_version == 2) rather than "certified" because it is stricter than the InferenceX UI, which also displays validated rows that predate schema versioning.', - '基准行可能携带实测功率、能耗和 GPU 遥测指标键（avg_power_w、joules_per_*、avg_temp_c、peak_temp_c、avg_util_pct、avg_mem_used_mb）。power_valid 为三态：1 表示测量窗口已通过验证；0 表示验证失败，实测值会被全链路扣留（生产端剥离、摄取端再次清除——若仍残留请视为不可靠）；缺失表示早于验证机制的旧数据行。power_metric_schema_version == 2 将所有无前缀的 joules_per_* 字段定义为全部署能耗——未标注版本的分离式 joules 含义不明确，因为这些字段曾承载角色本地值。多节点和分离式运行的每 worker 功率/遥测明细位于 workers[]。power_invalid_reasons 与 power_audit 为预留的即将推出字段。可使用 powerValid 请求参数筛选行；其严格取值命名为 strictV2（power_valid == 1 且 power_metric_schema_version == 2）而非 "certified"，因为它比 InferenceX 界面更严格——界面还会展示早于版本标注机制的已验证行。', + 'Benchmark rows may carry measured power, energy, and GPU-telemetry metric keys (avg_power_w, joules_per_*, avg_temp_c, peak_temp_c, avg_util_pct, avg_mem_used_mb). power_valid is tri-state: 1 means the measurement window was validated; 0 means validation failed and measured values are withheld end-to-end (the producer strips them and ingest scrubs them — treat any that remain as unreliable); absent marks a legacy row predating validation. power_metric_schema_version == 2 defines every unprefixed joules_per_* field as whole-deployment energy — unversioned disaggregated joules are ambiguous because those fields previously carried role-local values. workers[] carries the per-worker power/telemetry breakdown on multinode and disaggregated runs. power_invalid_reasons and power_audit provide optional producer validation details. Filter rows with the powerValid request parameter; its strict value is named strictV2 (power_valid == 1 and power_metric_schema_version == 2) rather than "certified" because it is stricter than the InferenceX UI, which also displays validated rows that predate schema versioning.', + '基准行可能携带实测功率、能耗和 GPU 遥测指标键（avg_power_w、joules_per_*、avg_temp_c、peak_temp_c、avg_util_pct、avg_mem_used_mb）。power_valid 为三态：1 表示测量窗口已通过验证；0 表示验证失败，实测值会被全链路扣留（生产端剥离、摄取端再次清除——若仍残留请视为不可靠）；缺失表示早于验证机制的旧数据行。power_metric_schema_version == 2 将所有无前缀的 joules_per_* 字段定义为全部署能耗——未标注版本的分离式 joules 含义不明确，因为这些字段曾承载角色本地值。多节点和分离式运行的每 worker 功率/遥测明细位于 workers[]。power_invalid_reasons 与 power_audit 可提供生产端的可选验证详情。可使用 powerValid 请求参数筛选行；其严格取值命名为 strictV2（power_valid == 1 且 power_metric_schema_version == 2）而非 "certified"，因为它比 InferenceX 界面更严格——界面还会展示早于版本标注机制的已验证行。', ), shape: 'BenchmarkRows', example: { diff --git a/packages/app/src/lib/benchmark-power-validity.test.ts b/packages/app/src/lib/benchmark-power-validity.test.ts index 5d11230ee..d8b685726 100644 --- a/packages/app/src/lib/benchmark-power-validity.test.ts +++ b/packages/app/src/lib/benchmark-power-validity.test.ts @@ -20,7 +20,7 @@ describe('parsePowerValidityFilter', () => { it('rejects unknown values', () => { expect(parsePowerValidityFilter('garbage')).toBeUndefined(); expect(parsePowerValidityFilter('')).toBeUndefined(); - // The pre-rename spelling must not silently alias to strictV2. + // `certified` is a UI display tier, not an API filter value. expect(parsePowerValidityFilter('certified')).toBeUndefined(); }); diff --git a/packages/constants/src/metric-keys.test.ts b/packages/constants/src/metric-keys.test.ts index 979afe302..deb21dfeb 100644 --- a/packages/constants/src/metric-keys.test.ts +++ b/packages/constants/src/metric-keys.test.ts @@ -15,9 +15,6 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { }); it('contains exactly the 13 measured power / energy / telemetry keys', () => { - // Guards accidental additions/removals: the ingest scrub and the display - // withholding both key off this set, so membership changes are policy - // changes and must be deliberate. expect(new Set(MEASURED_POWER_METRIC_KEY_LIST)).toEqual( new Set([ 'avg_power_w', @@ -39,11 +36,7 @@ describe('MEASURED_POWER_METRIC_KEYS', () => { }); it('never contains the contract discriminators or invalid-verdict companion fields', () => { - // power_valid / power_metric_schema_version are the verdict itself and - // must survive a scrub; power_invalid_reasons / power_audit are the - // producer's explanation of an invalid verdict (persisted app-side by - // PLAN-07) and are only meaningful on scrubbed rows. None of them may - // ever be added to the withheld set. + // Verdict and audit metadata must survive measured-value scrubbing. for (const key of [ 'power_valid', 'power_metric_schema_version', From 98143219b47df6b01ff4da489107818e6d0f43dd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 4 Sep 2026 15:27:28 -0700 Subject: [PATCH 11/11] fix(api): restrict power filtering to strictV2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅接受 strictV2 作为公开功率筛选值；省略参数时保留常规基准结果。同步更新中英文文档、OpenAPI 契约和回归测试。 --- .../app/cypress/e2e/api-documentation.cy.ts | 53 ++++++++++++-- .../src/app/api/v1/benchmarks/route.test.ts | 70 +++++++------------ .../app/src/app/api/v1/benchmarks/route.ts | 2 +- .../src/lib/api-documentation.power.test.ts | 16 +++-- packages/app/src/lib/api-documentation.ts | 18 ++--- packages/app/src/lib/api-route-catalog.ts | 2 +- .../src/lib/benchmark-power-validity.test.ts | 61 ++++++---------- .../app/src/lib/benchmark-power-validity.ts | 32 ++++----- 8 files changed, 133 insertions(+), 121 deletions(-) diff --git a/packages/app/cypress/e2e/api-documentation.cy.ts b/packages/app/cypress/e2e/api-documentation.cy.ts index 6272ca794..919e13231 100644 --- a/packages/app/cypress/e2e/api-documentation.cy.ts +++ b/packages/app/cypress/e2e/api-documentation.cy.ts @@ -1,3 +1,5 @@ +import type { BenchmarkRow } from '@semianalysisai/inferencex-db/queries/benchmarks'; + const SITE_URL = 'https://inferencex.semianalysis.com'; describe('API documentation', () => { @@ -35,7 +37,7 @@ describe('API documentation', () => { .and('contain.text', '/api/v1/availability') .and('contain.text', 'Endpoint reference') .and('contain.text', 'Measured power') - .and('contain.text', 'powerValid'); + .and('contain.text', 'powerValid=strictV2'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('[data-testid="api-spec-version"]').should('have.text', 'v1 · OpenAPI 3.1'); cy.get('[data-testid="api-endpoint-list-benchmarks"]') @@ -71,10 +73,11 @@ describe('API documentation', () => { 'operationId', 'list-benchmarks', ); - const benchmarkParameterNames = body.paths['/api/v1/benchmarks'].get.parameters.map( - (parameter: { name: string }) => parameter.name, + const powerValid = body.paths['/api/v1/benchmarks'].get.parameters.find( + (parameter: { name: string }) => parameter.name === 'powerValid', ); - expect(benchmarkParameterNames).to.include('powerValid'); + expect(powerValid.required).to.equal(false); + expect(powerValid.schema).to.deep.equal({ type: 'string', enum: ['strictV2'] }); expect(body.paths['/api/v1/collectivex/runs/{runId}'].get).to.have.property( 'operationId', 'get-collectivex-run', @@ -89,7 +92,8 @@ describe('API documentation', () => { .and('contain.text', '约定') .and('contain.text', '端点参考') .and('contain.text', 'BenchmarkRow 与指标') - .and('contain.text', '实测功率'); + .and('contain.text', '实测功率') + .and('contain.text', 'powerValid=strictV2'); cy.get('[data-testid="api-openapi-link"]').should('have.attr', 'href', '/api/openapi.json'); cy.get('link[rel="alternate"][hreflang="en"]').should('have.attr', 'href', `${SITE_URL}/api`); cy.get('link[rel="alternate"][hreflang="zh-CN"]').should( @@ -129,4 +133,43 @@ describe('API documentation', () => { ); }); }); + + it('supports strictV2 power requests while preserving ordinary benchmark requests', () => { + const url = '/api/v1/benchmarks?model=DeepSeek-R1-0528'; + cy.request(url).then(({ body, status }) => { + expect(status).to.equal(200); + expect(body).to.be.an('array'); + expect(body.length).to.be.greaterThan(0); + const expected = body.filter( + (row) => row.metrics.power_valid === 1 && row.metrics.power_metric_schema_version === 2, + ); + cy.request(`${url}&powerValid=strictV2`).then((response) => { + expect(response.status).to.equal(200); + expect(response.body).to.deep.equal(expected); + }); + }); + + for (const value of ['1', '0', 'any', 'certified', '']) { + cy.request({ url: `${url}&powerValid=${value}`, failOnStatusCode: false }).then( + ({ body, status }) => { + expect(status).to.equal(400); + expect(body).to.deep.equal({ error: 'Unknown powerValid filter' }); + }, + ); + } + + const calculatorUrl = `${url}&view=calculator&sequence=8k%2F1k`; + cy.request(calculatorUrl).then(({ body, status }) => { + expect(status).to.equal(200); + expect(body).to.be.an('array'); + expect(body.length).to.be.greaterThan(0); + }); + cy.request({ + url: `${calculatorUrl}&powerValid=strictV2`, + failOnStatusCode: false, + }).then(({ body, status }) => { + expect(status).to.equal(400); + expect(body).to.deep.equal({ error: 'powerValid cannot be combined with view=calculator' }); + }); + }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/route.test.ts b/packages/app/src/app/api/v1/benchmarks/route.test.ts index e7f679e35..fc35be4b6 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.test.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.test.ts @@ -266,22 +266,6 @@ describe('GET /api/v1/benchmarks', () => { }; const powerRows = [validatedV2, validatedUnversioned, invalidated, legacy]; - it('powerValid=1 keeps only rows with a validated verdict', async () => { - mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); - - const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1')); - expect(res.status).toBe(200); - expect(await res.json()).toEqual([validatedV2, validatedUnversioned]); - }); - - it('powerValid=0 keeps only explicitly invalidated rows', async () => { - mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); - - const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=0')); - expect(res.status).toBe(200); - expect(await res.json()).toEqual([invalidated]); - }); - it('powerValid=strictV2 requires a validated verdict plus schema version 2', async () => { mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); @@ -290,32 +274,28 @@ describe('GET /api/v1/benchmarks', () => { expect(await res.json()).toEqual([validatedV2]); }); - it('powerValid=any matches the response with the param absent (backward compat)', async () => { + it('keeps every general benchmark row when powerValid is omitted', async () => { mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); - const withParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any')); - - mockGetLatestBenchmarks.mockResolvedValueOnce(powerRows); - const withoutParam = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528')); - - expect(withParam.status).toBe(200); - expect(withoutParam.status).toBe(200); - const bodyWithParam = await withParam.json(); - expect(bodyWithParam).toEqual(await withoutParam.json()); - expect(bodyWithParam).toEqual(powerRows); + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual(powerRows); }); - it('rejects an unknown powerValid value without querying', async () => { - const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=garbage')); - - expect(res.status).toBe(400); - expect(await res.json()).toEqual({ error: 'Unknown powerValid filter' }); - expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); - }); + it.each(['1', '0', 'any', 'certified', 'garbage', '', 'strictv2'])( + 'rejects unsupported powerValid=%s without querying', + async (value) => { + const res = await GET(req(`/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=${value}`)); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Unknown powerValid filter' }); + expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); + expect(mockGetBenchmarksForRun).not.toHaveBeenCalled(); + }, + ); it('rejects powerValid combined with view=calculator without querying', async () => { const res = await GET( req( - '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1&view=calculator&sequence=1k%2F1k', + '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2&view=calculator&sequence=1k%2F1k', ), ); @@ -326,7 +306,7 @@ describe('GET /api/v1/benchmarks', () => { expect(mockGetLatestBenchmarks).not.toHaveBeenCalled(); }); - it('allows powerValid=any with view=calculator (no-op filter)', async () => { + it('keeps calculator requests working when powerValid is omitted', async () => { mockGetLatestBenchmarks.mockResolvedValueOnce([ { benchmark_type: 'single_turn', @@ -337,9 +317,7 @@ describe('GET /api/v1/benchmarks', () => { ]); const res = await GET( - req( - '/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=any&view=calculator&sequence=1k%2F1k', - ), + req('/api/v1/benchmarks?model=DeepSeek-R1-0528&view=calculator&sequence=1k%2F1k'), ); expect(res.status).toBe(200); @@ -355,14 +333,14 @@ describe('GET /api/v1/benchmarks', () => { benchmark_type: 'agentic_traces', workflow_run_id: 42, run_started_at: '2026-08-12T10:00:00Z', - metrics: { power_valid: 1 }, + metrics: { power_valid: 1, power_metric_schema_version: 2 }, }, { id: 2, benchmark_type: 'single_turn', workflow_run_id: 43, run_started_at: '2026-08-12T10:00:00Z', - metrics: { power_valid: 1 }, + metrics: { power_valid: 1, power_metric_schema_version: 2 }, }, { id: 3, @@ -373,16 +351,20 @@ describe('GET /api/v1/benchmarks', () => { }, ]); - const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=1')); + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528&powerValid=strictV2')); expect(await res.json()).toEqual([ { id: 1, benchmark_type: 'agentic_traces', workflow_run_id: 42, run_started_at: '2026-08-12T10:00:00Z', - metrics: { power_valid: 1 }, + metrics: { power_valid: 1, power_metric_schema_version: 2 }, + }, + { + id: 2, + benchmark_type: 'single_turn', + metrics: { power_valid: 1, power_metric_schema_version: 2 }, }, - { id: 2, benchmark_type: 'single_turn', metrics: { power_valid: 1 } }, ]); }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/route.ts b/packages/app/src/app/api/v1/benchmarks/route.ts index 887bb2a07..d69efe5c7 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.ts @@ -66,7 +66,7 @@ export async function GET(request: NextRequest) { } // The calculator cache stores rows already trimmed to an allowlist that // excludes power_valid, so post-cache filtering cannot work there. - if (view === 'calculator' && powerValidFilter !== 'any') { + if (view === 'calculator' && powerValidFilter !== null) { return NextResponse.json( { error: 'powerValid cannot be combined with view=calculator' }, { status: 400 }, diff --git a/packages/app/src/lib/api-documentation.power.test.ts b/packages/app/src/lib/api-documentation.power.test.ts index 94fd654b4..d5d80bdd0 100644 --- a/packages/app/src/lib/api-documentation.power.test.ts +++ b/packages/app/src/lib/api-documentation.power.test.ts @@ -48,10 +48,10 @@ describe('measured-power API documentation', () => { expect(benchmarkRowSchema?.required).not.toContain('power_audit'); }); - it('projects the view, sequence, and powerValid parameters into OpenAPI', () => { + it('documents strictV2 as the only optional power filter without changing ordinary requests', () => { const document = buildOpenApiDocument('https://api-docs.test'); const operation = (document.paths['/api/v1/benchmarks'] as Record).get as { - parameters: readonly { name: string; schema: unknown }[]; + parameters: readonly { name: string; required: boolean; schema: unknown }[]; }; const names = operation.parameters.map((parameter) => parameter.name); expect(names).toContain('view'); @@ -60,10 +60,13 @@ describe('measured-power API documentation', () => { const powerValid = operation.parameters.find((parameter) => parameter.name === 'powerValid'); expect(powerValid?.schema).toEqual({ type: 'string', - enum: ['1', '0', 'any', 'strictV2'], - default: 'any', + enum: ['strictV2'], }); - expect([...POWER_VALIDITY_FILTERS]).toEqual(['1', '0', 'any', 'strictV2']); + expect(powerValid?.required).toBe(false); + expect([...POWER_VALIDITY_FILTERS]).toEqual(['strictV2']); + expect(listBenchmarks?.curlUrl).toBe( + 'https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-R1-0528', + ); }); it('renders a bilingual measured-power schema note', () => { @@ -74,6 +77,9 @@ describe('measured-power API documentation', () => { expect(note, `${locale} must expose a measured-power schema note`).toBeDefined(); expect(note?.title.trim()).toBeTruthy(); expect(note?.description.trim()).toBeTruthy(); + expect(note?.description).toContain('powerValid=strictV2'); + expect(note?.description).toContain('power_valid == 1'); + expect(note?.description).toContain('power_metric_schema_version == 2'); } const zhNote = getApiDocumentation('zh').schemaNotes.find( (candidate) => candidate.id === 'measured-power', diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 782999a0a..21386dc70 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -664,8 +664,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/benchmarks', summary: text('Read benchmark results', '读取基准结果'), description: text( - 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change), and powerValid filters rows by measured-power validity and cannot be combined with view=calculator (except powerValid=any, which is a no-op).', - '返回指定展示模型的原始基准测试数据行。使用 date 可获取截至指定日期的快照；exact=true 仅返回该日期的数据；runId 用于限定最新结果的查询范围；将 exactRun=true 与数值型 runId 搭配使用，则只返回该工作流运行的数据。view=calculator 返回页面专用的裁剪投影（会移除实测功率指标和 workers，其允许列表可能变化）；powerValid 按实测功率有效性筛选行，且不能与 view=calculator 组合使用（powerValid=any 除外，等同于不筛选）。', + 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId to return only that workflow run. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change). powerValid=strictV2 selects validated schema-v2 power measurements and cannot be combined with view=calculator. Omit powerValid to keep general benchmark results regardless of power validity.', + '返回指定展示模型的原始基准测试数据行。使用 date 可获取截至指定日期的快照；exact=true 仅返回该日期的数据；runId 用于限定最新结果的查询范围；将 exactRun=true 与数值型 runId 搭配使用，则只返回该工作流运行的数据。view=calculator 返回页面专用的裁剪投影（会移除实测功率指标和 workers，其允许列表可能变化）。powerValid=strictV2 仅返回采用 schema v2 且功率测量通过验证的数据行，不能与 view=calculator 组合使用。省略 powerValid 则保留常规基准测试结果，不按功率有效性筛选。', ), audience: 'public', stability: 'stable', @@ -745,9 +745,9 @@ export const apiOperations: readonly ApiOperation[] = [ 'query', false, 'enum', - '1 keeps only rows with a validated power measurement (metrics.power_valid == 1); 0 keeps only explicitly invalidated rows; any applies no filter (default; includes legacy rows without a verdict); strictV2 keeps rows with power_valid == 1 and power_metric_schema_version == 2 (whole-deployment energy semantics) — stricter than the InferenceX UI, which also displays validated rows that predate schema versioning. Unknown values yield 400 Unknown powerValid filter; cannot be combined with view=calculator (except any, which is a no-op).', - '1 仅保留具有已验证功率测量的行（metrics.power_valid == 1）；0 仅保留被明确判定无效的行；any 不做筛选（默认值；包含没有判定结果的旧数据行）；strictV2 保留 power_valid == 1 且 power_metric_schema_version == 2（全部署能耗语义）的行——比 InferenceX 界面更严格，界面还会展示早于版本标注机制的已验证行。未知值返回 400 Unknown powerValid filter；不能与 view=calculator 组合使用（any 除外，等同于不筛选）。', - { type: 'string', enum: POWER_VALIDITY_FILTERS, default: 'any' }, + 'Only strictV2 is accepted. It keeps rows whose metrics.power_valid is the number 1 and metrics.power_metric_schema_version is the number 2 (whole-deployment energy semantics). Omit this parameter to apply no power filter, preserving throughput and latency results even when power is missing or invalid. Other values, including an empty value, yield 400 Unknown powerValid filter. Cannot be combined with view=calculator.', + '仅接受 strictV2：保留 metrics.power_valid 为数字 1、且 metrics.power_metric_schema_version 为数字 2 的数据行，其能耗指标采用整个部署的统计口径。省略此参数则不按功率筛选，即使功率缺失或无效，也会保留吞吐量和延迟结果。其他取值（包括空值）返回 400 Unknown powerValid filter。不能与 view=calculator 组合使用。', + { type: 'string', enum: POWER_VALIDITY_FILTERS }, 'strictV2', ), ], @@ -760,8 +760,8 @@ export const apiOperations: readonly ApiOperation[] = [ ), errorResponse( '400', - 'The model is missing or unsupported, the calculator sequence is unknown, the powerValid filter is unknown, or a non-any powerValid is combined with view=calculator.', - '模型缺失或不受支持、计算器序列未知、powerValid 筛选值未知，或非 any 的 powerValid 与 view=calculator 组合使用。', + 'The model is missing or unsupported, the calculator sequence is unknown, a supplied powerValid value is not strictV2, or powerValid=strictV2 is combined with view=calculator.', + '模型缺失或不受支持、计算器序列未知、提供的 powerValid 取值不是 strictV2，或 powerValid=strictV2 与 view=calculator 组合使用。', PUBLIC_API_ERRORS.unknownModel, ), errorResponse( @@ -2678,8 +2678,8 @@ const overview = { id: 'measured-power', title: text('Measured power', '实测功率'), description: text( - 'Benchmark rows may carry measured power, energy, and GPU-telemetry metric keys (avg_power_w, joules_per_*, avg_temp_c, peak_temp_c, avg_util_pct, avg_mem_used_mb). power_valid is tri-state: 1 means the measurement window was validated; 0 means validation failed and measured values are withheld end-to-end (the producer strips them and ingest scrubs them — treat any that remain as unreliable); absent marks a legacy row predating validation. power_metric_schema_version == 2 defines every unprefixed joules_per_* field as whole-deployment energy — unversioned disaggregated joules are ambiguous because those fields previously carried role-local values. workers[] carries the per-worker power/telemetry breakdown on multinode and disaggregated runs. power_invalid_reasons and power_audit provide optional producer validation details. Filter rows with the powerValid request parameter; its strict value is named strictV2 (power_valid == 1 and power_metric_schema_version == 2) rather than "certified" because it is stricter than the InferenceX UI, which also displays validated rows that predate schema versioning.', - '基准行可能携带实测功率、能耗和 GPU 遥测指标键（avg_power_w、joules_per_*、avg_temp_c、peak_temp_c、avg_util_pct、avg_mem_used_mb）。power_valid 为三态：1 表示测量窗口已通过验证；0 表示验证失败，实测值会被全链路扣留（生产端剥离、摄取端再次清除——若仍残留请视为不可靠）；缺失表示早于验证机制的旧数据行。power_metric_schema_version == 2 将所有无前缀的 joules_per_* 字段定义为全部署能耗——未标注版本的分离式 joules 含义不明确，因为这些字段曾承载角色本地值。多节点和分离式运行的每 worker 功率/遥测明细位于 workers[]。power_invalid_reasons 与 power_audit 可提供生产端的可选验证详情。可使用 powerValid 请求参数筛选行；其严格取值命名为 strictV2（power_valid == 1 且 power_metric_schema_version == 2）而非 "certified"，因为它比 InferenceX 界面更严格——界面还会展示早于版本标注机制的已验证行。', + 'Benchmark rows may carry measured power, energy, and GPU-telemetry metric keys (avg_power_w, joules_per_*, avg_temp_c, peak_temp_c, avg_util_pct, avg_mem_used_mb). power_valid is tri-state: 1 means the measurement window was validated; 0 means validation failed and measured values are withheld end-to-end (the producer strips them and ingest scrubs them — treat any that remain as unreliable); absent marks a legacy row predating validation. power_metric_schema_version == 2 defines every unprefixed joules_per_* field as whole-deployment energy — unversioned disaggregated joules are ambiguous because those fields previously carried role-local values. workers[] carries the per-worker power/telemetry breakdown on multinode and disaggregated runs. power_invalid_reasons and power_audit provide optional producer validation details. For measured-power requests, use powerValid=strictV2 to require power_valid == 1 and power_metric_schema_version == 2. It is the only supported power filter. Omit powerValid for general benchmark requests so results remain available even when they lack valid power measurements.', + '基准测试数据行可能包含实测功率、能耗和 GPU 遥测指标（avg_power_w、joules_per_*、avg_temp_c、peak_temp_c、avg_util_pct、avg_mem_used_mb）。power_valid 有三种状态：1 表示测量窗口已通过验证；0 表示验证失败，生产端会移除实测值，摄取端也会再次清除，若仍有残留，应视为不可靠；缺失表示该数据行早于验证机制。power_metric_schema_version == 2 规定所有无前缀的 joules_per_* 字段均按整个部署统计能耗。未标注版本的分离式部署数据中，这些字段曾记录单个角色的能耗，因此其统计口径不明确。多节点和分离式运行中，各 worker 的功率和遥测明细位于 workers[]。power_invalid_reasons 与 power_audit 可包含生产端的验证详情。查询实测功率时，使用 powerValid=strictV2，仅保留 power_valid == 1 且 power_metric_schema_version == 2 的行。这是唯一支持的功率筛选值。常规基准测试请求应省略 powerValid，以保留缺少有效功率测量的结果。', ), shape: 'BenchmarkRows', example: { diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index f59ed11aa..e01c8e5a7 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -108,7 +108,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-benchmarks', - sourceSha256: 'b736b302cdb294bb316ff4666a64b47432655772f442aa8f5c95fe1f3cc9ed38', + sourceSha256: '7b251598bf9e4e181834311a554aa8ef7a9bc39d605eed243364ce5c3f5cc43e', }, { source: 'src/app/api/v1/benchmarks/history/route.ts', diff --git a/packages/app/src/lib/benchmark-power-validity.test.ts b/packages/app/src/lib/benchmark-power-validity.test.ts index d8b685726..1c7b0f131 100644 --- a/packages/app/src/lib/benchmark-power-validity.test.ts +++ b/packages/app/src/lib/benchmark-power-validity.test.ts @@ -1,33 +1,22 @@ import { describe, expect, it } from 'vitest'; -import { - filterByPowerValidity, - parsePowerValidityFilter, - POWER_VALIDITY_FILTERS, -} from './benchmark-power-validity'; +import { filterByPowerValidity, parsePowerValidityFilter } from './benchmark-power-validity'; describe('parsePowerValidityFilter', () => { - it('treats an absent param as any', () => { - expect(parsePowerValidityFilter(null)).toBe('any'); + it('preserves an absent param as no filtering', () => { + expect(parsePowerValidityFilter(null)).toBeNull(); }); - it('accepts every listed filter value', () => { - for (const filter of POWER_VALIDITY_FILTERS) { - expect(parsePowerValidityFilter(filter)).toBe(filter); - } + it('accepts strictV2', () => { + expect(parsePowerValidityFilter('strictV2')).toBe('strictV2'); }); - it('rejects unknown values', () => { - expect(parsePowerValidityFilter('garbage')).toBeUndefined(); - expect(parsePowerValidityFilter('')).toBeUndefined(); - // `certified` is a UI display tier, not an API filter value. - expect(parsePowerValidityFilter('certified')).toBeUndefined(); - }); - - it('is case-sensitive', () => { - expect(parsePowerValidityFilter('ANY')).toBeUndefined(); - expect(parsePowerValidityFilter('strictv2')).toBeUndefined(); - }); + it.each(['1', '0', 'any', 'certified', 'garbage', '', 'strictv2', 'strictV2 '])( + 'rejects unsupported value %j', + (value) => { + expect(parsePowerValidityFilter(value)).toBeUndefined(); + }, + ); }); describe('filterByPowerValidity', () => { @@ -35,31 +24,27 @@ describe('filterByPowerValidity', () => { const validatedUnversioned = { id: 2, metrics: { power_valid: 1 } }; const invalidated = { id: 3, metrics: { power_valid: 0, power_metric_schema_version: 2 } }; const legacy = { id: 4, metrics: { tput_per_gpu: 100 } }; - const noMetrics = { id: 5 } as { id: number; metrics?: Record }; + const noMetrics: { id: number; metrics?: Record } = { id: 5 }; const rows = [validatedV2, validatedUnversioned, invalidated, legacy, noMetrics]; - it('returns every row unchanged for any', () => { - const result = filterByPowerValidity(rows, 'any'); + it('preserves all general benchmark rows when the parameter is omitted', () => { + const result = filterByPowerValidity(rows, null); expect(result).toEqual(rows); expect(result).not.toBe(rows); }); - it('keeps only explicitly validated rows for 1', () => { - expect(filterByPowerValidity(rows, '1')).toEqual([validatedV2, validatedUnversioned]); - }); - - it('keeps only explicitly invalidated rows for 0', () => { - expect(filterByPowerValidity(rows, '0')).toEqual([invalidated]); - }); - it('requires a validated verdict and schema version 2 for strictV2', () => { expect(filterByPowerValidity(rows, 'strictV2')).toEqual([validatedV2]); }); - it('excludes legacy and metric-less rows from every filter except any', () => { - for (const filter of ['1', '0', 'strictV2'] as const) { - const surviving = filterByPowerValidity([legacy, noMetrics], filter); - expect(surviving).toEqual([]); - } + it('excludes unsupported versions and malformed verdicts from strictV2', () => { + const unsupported = [ + { metrics: { power_valid: 1, power_metric_schema_version: 1 } }, + { metrics: { power_valid: 1, power_metric_schema_version: 3 } }, + { metrics: { power_valid: '1', power_metric_schema_version: 2 } }, + { metrics: { power_valid: 1, power_metric_schema_version: '2' } }, + { metrics: { power_valid: true, power_metric_schema_version: 2 } }, + ]; + expect(filterByPowerValidity(unsupported, 'strictV2')).toEqual([]); }); }); diff --git a/packages/app/src/lib/benchmark-power-validity.ts b/packages/app/src/lib/benchmark-power-validity.ts index 6b7d98ec6..5842e30c5 100644 --- a/packages/app/src/lib/benchmark-power-validity.ts +++ b/packages/app/src/lib/benchmark-power-validity.ts @@ -4,8 +4,8 @@ * `metrics.power_valid` is tri-state: 1 means the measurement window was * validated, an explicit 0 is an authoritative invalid verdict (measured * values are withheld end-to-end), and an absent key marks a legacy row that - * predates validation. `strictV2` additionally requires - * `power_metric_schema_version === 2`, mirroring + * predates validation. The only public filter, `strictV2`, requires a valid + * verdict and `power_metric_schema_version === 2`, mirroring * `WHOLE_DEPLOYMENT_ENERGY_SCHEMA_VERSION` in `benchmark-transform.ts` and * `POWER_METRIC_SCHEMA_VERSION` in the runner's `utils/aggregate_power.py` — * only version 2 defines unprefixed `joules_per_*` fields as whole-deployment @@ -13,30 +13,26 @@ * a display rule that also admits validated legacy rows without a schema * version, which `strictV2` excludes. */ -export const POWER_VALIDITY_FILTERS = ['1', '0', 'any', 'strictV2'] as const; +export const POWER_VALIDITY_FILTERS = ['strictV2'] as const; export type PowerValidityFilter = (typeof POWER_VALIDITY_FILTERS)[number]; /** Absent param means no filtering; unknown values return undefined so the caller can 400. */ -export function parsePowerValidityFilter(raw: string | null): PowerValidityFilter | undefined { - if (raw === null) return 'any'; - return (POWER_VALIDITY_FILTERS as readonly string[]).includes(raw) - ? (raw as PowerValidityFilter) - : undefined; +export function parsePowerValidityFilter( + raw: string | null, +): PowerValidityFilter | null | undefined { + return raw === null || raw === 'strictV2' ? raw : undefined; } /** - * Pure post-cache row filter. Rows without `metrics` or without a - * `power_valid` verdict (legacy rows) match only `any`. + * Pure post-cache row filter. An omitted parameter preserves general benchmark + * results, including rows with no power measurement. */ export function filterByPowerValidity }>( rows: readonly T[], - filter: PowerValidityFilter, + filter: PowerValidityFilter | null, ): T[] { - if (filter === 'any') return [...rows]; - return rows.filter((row) => { - const powerValid = row.metrics?.power_valid; - if (filter === '1') return powerValid === 1; - if (filter === '0') return powerValid === 0; - return powerValid === 1 && row.metrics?.power_metric_schema_version === 2; - }); + if (filter === null) return [...rows]; + return rows.filter( + (row) => row.metrics?.power_valid === 1 && row.metrics?.power_metric_schema_version === 2, + ); }