From 061dd6ebb8cf142bb8d9fbd0844f4e27608d69d9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Jul 2026 05:06:19 +0200 Subject: [PATCH] fix(eval): separate parity from score eligibility --- CHANGELOG.md | 2 ++ scripts/native_eval/aggregate.py | 27 +++++++++++++++++++-------- tests/test_native_eval_aggregate.py | 21 ++++++++++++++------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f91a99f..0815d4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). - Scope native Harbor parity claims to the validated harness/model route instead of applying one fleet-wide boolean. +- Report scoped Harbor parity independently from native leaderboard eligibility, + so complete runs remain scoreable without an unsupported parity claim. ### Added diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index 43a6efe..d316090 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -921,7 +921,6 @@ def _summarize_run( and not harness_wide_failure and canonical_model_identity and trajectory_complete - and parity_validated ) if incomplete: exclusion_reason = "incomplete" @@ -933,8 +932,6 @@ def _summarize_run( exclusion_reason = "canonical_model_identity_not_preserved" elif not trajectory_complete: exclusion_reason = "trajectory_unavailable" - elif not parity_validated: - exclusion_reason = "parity_not_validated" else: exclusion_reason = "" if manifest.get("leaderboard_eligible") is False: @@ -999,6 +996,9 @@ def _pair_summaries(runs: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: ), ) eligible = [run for run in ordered if run["eligible"]] + parity_validated_runs = [ + run for run in ordered if run["parity_validated"] is True + ] scores = [float(run["score"]) for run in eligible] exact_passes = [int(run["exact_passes"]) for run in eligible] total_expected = sum(int(run["expected_task_count"]) for run in eligible) @@ -1015,6 +1015,12 @@ def _pair_summaries(runs: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: "excluded_run_labels": [ str(run["run_label"]) for run in ordered if not run["eligible"] ], + "parity_validated_repetitions": len(parity_validated_runs), + "parity_unvalidated_run_labels": [ + str(run["run_label"]) + for run in ordered + if run["parity_validated"] is not True + ], "mean_score": mean_score, "score_stdev": score_stdev, "min_score": min_score, @@ -1178,19 +1184,22 @@ def _write_leaderboard(path: Path, pairs: Sequence[dict[str, Any]]) -> None: lines = [ "# Cleaned Native ShellBench Leaderboard", "", - "Incomplete, infra-dominated, and harness-wide failure repetitions are excluded.", + "Incomplete, infra-dominated, harness-wide failure, identity-invalid, and " + "trajectory-incomplete repetitions are excluded. Harbor parity validation is " + "reported separately and is not an eligibility gate.", "", - "| Rank | Pair | Repetitions | Mean | Stdev | Min | Max | Mean exact passes | Pass rate | Clean complete |", - "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + "| Rank | Pair | Repetitions | Parity validated | Mean | Stdev | Min | Max | Mean exact passes | Pass rate | Clean complete |", + "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ] for rank, pair in enumerate(ranked, start=1): lines.append( - "| {rank} | {label} | {eligible}/{total} | {mean} | {stdev} | " + "| {rank} | {label} | {eligible}/{total} | {parity}/{total} | {mean} | {stdev} | " "{minimum} | {maximum} | {passes} | {pass_rate} | {clean} |".format( rank=rank, label=_markdown_label(pair["pair_label"]), eligible=pair["eligible_repetitions"], total=pair["total_repetitions"], + parity=pair["parity_validated_repetitions"], mean=_format_metric(pair["mean_score"]), stdev=_format_metric(pair["score_stdev"]), minimum=_format_metric(pair["min_score"]), @@ -1201,7 +1210,9 @@ def _write_leaderboard(path: Path, pairs: Sequence[dict[str, Any]]) -> None: ) ) if not ranked: - lines.append("| - | No eligible repetitions | 0/0 | - | - | - | - | - | - | 0 |") + lines.append( + "| - | No eligible repetitions | 0/0 | 0/0 | - | - | - | - | - | - | 0 |" + ) path.write_text("\n".join(lines) + "\n") diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index 63f2f43..c1e26d1 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -308,7 +308,7 @@ def test_pair_label_uses_manifest_harness_and_model(tmp_path: Path): assert len(rerun["runs"]) == 2 -def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path): +def test_native_run_reports_missing_parity_without_excluding_score(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" _write_run( @@ -323,8 +323,15 @@ def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path): report = aggregate(jobs_root, summaries_dir) - assert report["runs"][0]["eligible"] is False - assert report["runs"][0]["exclusion_reason"] == "parity_not_validated" + run = report["runs"][0] + assert run["parity_validated"] is False + assert run["parity_validation_status"] == "not_validated" + assert run["eligible"] is True + assert run["exclusion_reason"] == "" + assert report["pairs"][0]["parity_validated_repetitions"] == 0 + assert report["pairs"][0]["parity_unvalidated_run_labels"] == [ + "codex-gpt55-calibration" + ] def test_native_run_rejects_legacy_unscoped_parity_claim(tmp_path: Path): @@ -347,7 +354,7 @@ def test_native_run_rejects_legacy_unscoped_parity_claim(tmp_path: Path): run = report["runs"][0] assert run["parity_validated"] is False assert run["parity_validation_status"] == "legacy_unscoped" - assert run["eligible"] is False + assert run["eligible"] is True def test_native_run_accepts_known_legacy_codex_gpt55_route(tmp_path: Path): @@ -393,8 +400,8 @@ def test_native_run_rejects_legacy_codex_gpt55_full_suite(tmp_path: Path): run = report["runs"][0] assert run["parity_validated"] is False assert run["parity_validation_status"] == "legacy_scope_mismatch" - assert run["eligible"] is False - assert run["exclusion_reason"] == "parity_not_validated" + assert run["eligible"] is True + assert run["exclusion_reason"] == "" def test_native_run_rejects_mismatched_parity_scope(tmp_path: Path): @@ -420,7 +427,7 @@ def test_native_run_rejects_mismatched_parity_scope(tmp_path: Path): run = report["runs"][0] assert run["parity_validated"] is False assert run["parity_validation_status"] == "scope_mismatch" - assert run["eligible"] is False + assert run["eligible"] is True def test_native_identity_checks_ignore_pre_agent_infra_failures(tmp_path: Path):