fix(eval): separate parity from score eligibility

CHANGELOG.md

@@ -21,6 +21,8 @@
 - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali).
 - Scope native Harbor parity claims to the validated harness/model route instead
   of applying one fleet-wide boolean.
+- Report scoped Harbor parity independently from native leaderboard eligibility,
+  so complete runs remain scoreable without an unsupported parity claim.
 
 ### Added
 

scripts/native_eval/aggregate.py

@@ -921,7 +921,6 @@ def _summarize_run(
         and not harness_wide_failure
         and canonical_model_identity
         and trajectory_complete
-        and parity_validated
     )
     if incomplete:
         exclusion_reason = "incomplete"
@@ -933,8 +932,6 @@ def _summarize_run(
         exclusion_reason = "canonical_model_identity_not_preserved"
     elif not trajectory_complete:
         exclusion_reason = "trajectory_unavailable"
-    elif not parity_validated:
-        exclusion_reason = "parity_not_validated"
     else:
         exclusion_reason = ""
     if manifest.get("leaderboard_eligible") is False:
@@ -999,6 +996,9 @@ def _pair_summaries(runs: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
             ),
         )
         eligible = [run for run in ordered if run["eligible"]]
+        parity_validated_runs = [
+            run for run in ordered if run["parity_validated"] is True
+        ]
         scores = [float(run["score"]) for run in eligible]
         exact_passes = [int(run["exact_passes"]) for run in eligible]
         total_expected = sum(int(run["expected_task_count"]) for run in eligible)
@@ -1015,6 +1015,12 @@ def _pair_summaries(runs: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
                 "excluded_run_labels": [
                     str(run["run_label"]) for run in ordered if not run["eligible"]
                 ],
+                "parity_validated_repetitions": len(parity_validated_runs),
+                "parity_unvalidated_run_labels": [
+                    str(run["run_label"])
+                    for run in ordered
+                    if run["parity_validated"] is not True
+                ],
                 "mean_score": mean_score,
                 "score_stdev": score_stdev,
                 "min_score": min_score,
@@ -1178,19 +1184,22 @@ def _write_leaderboard(path: Path, pairs: Sequence[dict[str, Any]]) -> None:
     lines = [
         "# Cleaned Native ShellBench Leaderboard",
         "",
-        "Incomplete, infra-dominated, and harness-wide failure repetitions are excluded.",
+        "Incomplete, infra-dominated, harness-wide failure, identity-invalid, and "
+        "trajectory-incomplete repetitions are excluded. Harbor parity validation is "
+        "reported separately and is not an eligibility gate.",
         "",
-        "| Rank | Pair | Repetitions | Mean | Stdev | Min | Max | Mean exact passes | Pass rate | Clean complete |",
-        "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+        "| Rank | Pair | Repetitions | Parity validated | Mean | Stdev | Min | Max | Mean exact passes | Pass rate | Clean complete |",
+        "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
     ]
     for rank, pair in enumerate(ranked, start=1):
         lines.append(
-            "| {rank} | {label} | {eligible}/{total} | {mean} | {stdev} | "
+            "| {rank} | {label} | {eligible}/{total} | {parity}/{total} | {mean} | {stdev} | "
             "{minimum} | {maximum} | {passes} | {pass_rate} | {clean} |".format(
                 rank=rank,
                 label=_markdown_label(pair["pair_label"]),
                 eligible=pair["eligible_repetitions"],
                 total=pair["total_repetitions"],
+                parity=pair["parity_validated_repetitions"],
                 mean=_format_metric(pair["mean_score"]),
                 stdev=_format_metric(pair["score_stdev"]),
                 minimum=_format_metric(pair["min_score"]),
@@ -1201,7 +1210,9 @@ def _write_leaderboard(path: Path, pairs: Sequence[dict[str, Any]]) -> None:
             )
         )
     if not ranked:
-        lines.append("| - | No eligible repetitions | 0/0 | - | - | - | - | - | - | 0 |")
+        lines.append(
+            "| - | No eligible repetitions | 0/0 | 0/0 | - | - | - | - | - | - | 0 |"
+        )
     path.write_text("\n".join(lines) + "\n")
 
 

tests/test_native_eval_aggregate.py

@@ -308,7 +308,7 @@ def test_pair_label_uses_manifest_harness_and_model(tmp_path: Path):
     assert len(rerun["runs"]) == 2
 
 
-def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path):
+def test_native_run_reports_missing_parity_without_excluding_score(tmp_path: Path):
     jobs_root = tmp_path / "native"
     summaries_dir = tmp_path / "summaries"
     _write_run(
@@ -323,8 +323,15 @@ def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path):
 
     report = aggregate(jobs_root, summaries_dir)
 
-    assert report["runs"][0]["eligible"] is False
-    assert report["runs"][0]["exclusion_reason"] == "parity_not_validated"
+    run = report["runs"][0]
+    assert run["parity_validated"] is False
+    assert run["parity_validation_status"] == "not_validated"
+    assert run["eligible"] is True
+    assert run["exclusion_reason"] == ""
+    assert report["pairs"][0]["parity_validated_repetitions"] == 0
+    assert report["pairs"][0]["parity_unvalidated_run_labels"] == [
+        "codex-gpt55-calibration"
+    ]
 
 
 def test_native_run_rejects_legacy_unscoped_parity_claim(tmp_path: Path):
@@ -347,7 +354,7 @@ def test_native_run_rejects_legacy_unscoped_parity_claim(tmp_path: Path):
     run = report["runs"][0]
     assert run["parity_validated"] is False
     assert run["parity_validation_status"] == "legacy_unscoped"
-    assert run["eligible"] is False
+    assert run["eligible"] is True
 
 
 def test_native_run_accepts_known_legacy_codex_gpt55_route(tmp_path: Path):
@@ -393,8 +400,8 @@ def test_native_run_rejects_legacy_codex_gpt55_full_suite(tmp_path: Path):
     run = report["runs"][0]
     assert run["parity_validated"] is False
     assert run["parity_validation_status"] == "legacy_scope_mismatch"
-    assert run["eligible"] is False
-    assert run["exclusion_reason"] == "parity_not_validated"
+    assert run["eligible"] is True
+    assert run["exclusion_reason"] == ""
 
 
 def test_native_run_rejects_mismatched_parity_scope(tmp_path: Path):
@@ -420,7 +427,7 @@ def test_native_run_rejects_mismatched_parity_scope(tmp_path: Path):
     run = report["runs"][0]
     assert run["parity_validated"] is False
     assert run["parity_validation_status"] == "scope_mismatch"
-    assert run["eligible"] is False
+    assert run["eligible"] is True
 
 
 def test_native_identity_checks_ignore_pre_agent_infra_failures(tmp_path: Path):