From 1550d11c0312e1d08fc4f3039df7c2a8e4d2e035 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 04:42:20 +0200 Subject: [PATCH 01/28] fix(eval): stabilize GPT-5.5 native harness runs --- CHANGELOG.md | 2 + scripts/native_eval/aggregate.py | 6 +++ scripts/native_eval/fleet.py | 31 ++++++++++++- scripts/native_eval/harnesses.py | 23 +++++++-- scripts/native_eval/plan.py | 17 +++++++ scripts/native_eval/proxy.py | 8 ++++ scripts/native_eval/run_job.py | 4 ++ scripts/native_eval/runtime.py | 27 ++++++++++- tests/test_native_eval_aggregate.py | 39 ++++++++++++++++ tests/test_native_eval_fleet.py | 6 ++- tests/test_native_eval_proxy.py | 9 +++- tests/test_native_eval_runner.py | 72 ++++++++++++++++++++++++++++- 12 files changed, 235 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1432a8..b08dd37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,4 +4,6 @@ ### Fixed +- Record and enforce GPT reasoning effort in native runs, preserve it across + reruns, and stabilize OpenClaw and Hermes trace completion. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index 8a656cc..1cdf606 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -34,6 +34,7 @@ } INFRA_EXCEPTION_TYPES = { + "AgentSetupError", "AgentSetupTimeoutError", "DockerStartupError", "EnvironmentStartTimeoutError", @@ -805,6 +806,11 @@ def _summarize_run( exclusion_reason = "parity_not_validated" else: exclusion_reason = "" + if manifest.get("leaderboard_eligible") is False: + eligible = False + exclusion_reason = ( + str(manifest.get("exclusion_reason") or "") or "explicitly_excluded" + ) return { "run_label": run_label, diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index f30866a..142e882 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -47,6 +47,7 @@ RERUN_STATUSES = {"failed", "lease_lost"} ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} +REASONING_EFFORTS = {"low", "medium", "high"} class CommandExecutor(Protocol): @@ -324,6 +325,21 @@ def _validate_plan(self) -> None: raise FleetError( f"{run.run_label} expects {run.expected_task_count}, index expects {expected}" ) + if run.provider == "openai": + reasoning_effort = str(entry.get("reasoning_effort") or "") + if reasoning_effort not in REASONING_EFFORTS: + raise FleetError( + f"{run.run_label} must set reasoning_effort to " + "low, medium, or high" + ) + judge_reasoning_effort = str( + entry.get("judge_reasoning_effort") or "" + ) + if judge_reasoning_effort not in REASONING_EFFORTS: + raise FleetError( + f"{run.run_label} must set judge_reasoning_effort to " + "low, medium, or high" + ) def _prepare_inputs(self) -> None: for path, description in ( @@ -764,7 +780,9 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: harbor_reference_commit=$1 judge_model_id=$2 execution_mode=$3 -shift 3 +reasoning_effort=$4 +judge_reasoning_effort=$5 +shift 5 mkdir -p "$root/run-logs" stdout="$root/run-logs/$label.stdout.log" stderr="$root/run-logs/$label.stderr.log" @@ -779,6 +797,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: "SHELLBENCH_HARBOR_REFERENCE_COMMIT=$harbor_reference_commit" \ "SHELLBENCH_JUDGE_MODEL_ID=$judge_model_id" \ "SHELLBENCH_EXECUTION_MODE=$execution_mode" \ + "SHELLBENCH_REASONING_EFFORT=$reasoning_effort" \ + "SHELLBENCH_JUDGE_REASONING_EFFORT=$judge_reasoning_effort" \ "$root/runner/scripts/native_eval/remote_run.sh" "$@" \ >"$stdout" 2>"$stderr" None: run.provider, run.proxy_model_name, ] + entry = self._store.get(run.run_label) + reasoning_effort = str(entry.get("reasoning_effort") or "") + judge_reasoning_effort = str( + entry.get("judge_reasoning_effort") or reasoning_effort + ) self._checked( self._ssh_command( lease, @@ -823,6 +848,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: self.config.harbor_reference_commit, self.config.judge_model_id, self.config.execution_mode, + reasoning_effort, + judge_reasoning_effort, *args, ], ), @@ -1162,6 +1189,8 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: rerun = { **run.to_dict(), "run_label": label, + "reasoning_effort": entry.get("reasoning_effort"), + "judge_reasoning_effort": entry.get("judge_reasoning_effort"), "attempt": next_attempt, "status": "planned", "leaderboard_eligible": None, diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 519487f..4de7025 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -99,11 +99,24 @@ def _openclaw( ) run_command = ( f"export PATH={_base_path()}; export HOME={home}; " + "log=/logs/agent/openclaw.txt; " "openclaw agent --local --json --agent main --thinking off " f"--model {shlex.quote(model)} " "--message \"$(cat /tmp/shellbench-instruction.md)\" " - ">/logs/agent/openclaw.txt 2>&1 \"$log\" 2>&1 /dev/null; do " + "if grep -Eq \"\\[agents/agent-command\\] \\[agent\\] run .* " + "ended with stopReason=\" \"$log\"; then " + "sleep 2; kill \"$pid\" 2>/dev/null || true; sleep 1; " + "kill -KILL \"$pid\" 2>/dev/null || true; " + "for f in /proc/[0-9]*/comm; do " + "name=$(cat \"$f\" 2>/dev/null || true); " + "case \"$name\" in openclaw-agent|\"npm exec chrome\"|chrome-devtools) " + "child=${f#/proc/}; child=${child%/comm}; " + "kill -KILL \"$child\" 2>/dev/null || true;; esac; done; " + "wait \"$pid\" 2>/dev/null || true; cat \"$log\"; exit 0; " + "fi; sleep 1; done; " + "wait \"$pid\"; status=$?; cat \"$log\"; exit \"$status\"" ) cleanup = ( "python3 - <<'PY'\n" @@ -189,8 +202,12 @@ def _hermes( ) cleanup = ( f"export PATH={_base_path()}; export HERMES_HOME={home}; " + "session_id=$(sed -n 's/^session_id: //p' " + "/logs/agent/hermes.txt | tail -1); " + "if [ -n \"$session_id\" ]; then " "hermes sessions export /logs/agent/hermes-session.jsonl " - "--source cli 2>/dev/null || true" + "--session-id \"$session_id\" --yes --redact; " + "fi" ) return HarnessCommand( setup_command=setup, diff --git a/scripts/native_eval/plan.py b/scripts/native_eval/plan.py index 1fdc1ad..d5544f2 100644 --- a/scripts/native_eval/plan.py +++ b/scripts/native_eval/plan.py @@ -14,12 +14,16 @@ def write_run_index( output: Path, public_tasks_commit: str, run_date: str, + reasoning_effort: str, + judge_reasoning_effort: str, ) -> list[dict[str, object]]: tasks = validate_suite(tasks_root) runs = build_matrix_plan(len(tasks), run_date=run_date) entries = [ { **run.to_dict(), + "reasoning_effort": reasoning_effort, + "judge_reasoning_effort": judge_reasoning_effort, "attempt": 0, "status": "planned", "leaderboard_eligible": None, @@ -49,12 +53,25 @@ def main() -> None: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--public-tasks-commit", required=True) parser.add_argument("--run-date", required=True) + parser.add_argument( + "--reasoning-effort", + choices=("low", "medium", "high"), + required=True, + ) + parser.add_argument( + "--judge-reasoning-effort", + choices=("low", "medium", "high"), + ) args = parser.parse_args() entries = write_run_index( tasks_root=args.tasks_root, output=args.output, public_tasks_commit=args.public_tasks_commit, run_date=args.run_date, + reasoning_effort=args.reasoning_effort, + judge_reasoning_effort=( + args.judge_reasoning_effort or args.reasoning_effort + ), ) print(f"wrote {len(entries)} planned runs to {args.output}") diff --git a/scripts/native_eval/proxy.py b/scripts/native_eval/proxy.py index 5f26b74..96b19c0 100644 --- a/scripts/native_eval/proxy.py +++ b/scripts/native_eval/proxy.py @@ -1,12 +1,18 @@ from __future__ import annotations import json +import os from pathlib import Path from scripts.native_eval.models import LITELLM_VERSION, MODELS def write_proxy_config(path: Path) -> None: + reasoning_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip() + if reasoning_effort not in {"low", "medium", "high"}: + raise ValueError( + "SHELLBENCH_REASONING_EFFORT must be low, medium, or high" + ) model_list = [] for model in MODELS: litellm_params = { @@ -15,6 +21,7 @@ def write_proxy_config(path: Path) -> None: } if model.slug in {"gpt55", "gpt56-sol"}: litellm_params["additional_drop_params"] = ["temperature"] + litellm_params["reasoning_effort"] = reasoning_effort model_list.append( { "model_name": model.proxy_model_name, @@ -44,6 +51,7 @@ def write_proxy_config(path: Path) -> None: }, "shellbench_native": { "litellm_version": LITELLM_VERSION, + "reasoning_effort": reasoning_effort, "models": [ { "slug": model.slug, diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index 0b1f8e0..3b64fa8 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -244,6 +244,10 @@ def _run_manifest( "SHELLBENCH_HARBOR_REFERENCE_COMMIT" ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), + "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), + "judge_reasoning_effort": os.environ.get( + "SHELLBENCH_JUDGE_REASONING_EFFORT" + ), "runner_commit": _git_commit(), "runner_patch_hash": _runner_patch_hash(), "public_tasks_commit": public_tasks_commit, diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index be2c475..088e786 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -31,6 +31,10 @@ class AgentSetupTimeoutError(NativeEvalError): pass +class AgentSetupError(NativeEvalError): + pass + + class NonZeroAgentExitCodeError(NativeEvalError): pass @@ -215,7 +219,26 @@ async def _discover_workdir(self) -> None: self.container_id, ] ) - self.workdir = output.strip() or "/app" + configured = output.strip() + if configured: + self.workdir = configured + return + self.workdir = ( + await capture_process( + [ + "docker", + "exec", + self.container_id, + "sh", + "-lc", + ( + "if [ -d /app ]; then printf /app; " + "elif [ -d /workspace ]; then printf /workspace; " + "else pwd; fi" + ), + ] + ) + ).strip() or "/" async def copy_instruction(self, instruction: str) -> None: local = self.trial_dir / "instruction.md" @@ -444,7 +467,7 @@ async def run_trial( stderr_path=trial_dir / "agent" / "setup-stderr.txt", ) if setup.returncode: - raise NativeEvalError(f"Agent setup exited {setup.returncode}") + raise AgentSetupError(f"Agent setup exited {setup.returncode}") except TimeoutError as exc: raise AgentSetupTimeoutError("Agent setup timed out after 300s") from exc finally: diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index c8d17d0..2623fb7 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -20,6 +20,8 @@ def _write_run( model_slug: str = "model", native: bool = False, parity_validated: bool = False, + leaderboard_eligible: bool | None = None, + exclusion_reason: str = "", ) -> None: run_dir = jobs_root / "jobs" / run_label run_dir.mkdir(parents=True) @@ -41,6 +43,9 @@ def _write_run( ) if pair_label is not None: manifest["pair_label"] = pair_label + if leaderboard_eligible is not None: + manifest["leaderboard_eligible"] = leaderboard_eligible + manifest["exclusion_reason"] = exclusion_reason if tasks is not None: manifest["tasks"] = tasks (run_dir / "run_manifest.json").write_text(json.dumps(manifest)) @@ -287,6 +292,40 @@ def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path): assert report["runs"][0]["exclusion_reason"] == "parity_not_validated" +def test_manifest_can_explicitly_exclude_complete_run(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-rerun1", + expected_task_count=1, + results=[_result("a", reward=1.0)], + leaderboard_eligible=False, + exclusion_reason="manual_intervention", + ) + + report = aggregate(jobs_root, summaries_dir) + + assert report["runs"][0]["eligible"] is False + assert report["runs"][0]["exclusion_reason"] == "manual_intervention" + + +def test_agent_setup_error_is_classified_as_infrastructure(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-setup-failure", + expected_task_count=1, + results=[_result("a", exception_type="AgentSetupError")], + ) + + report = aggregate(jobs_root, summaries_dir) + + assert report["runs"][0]["infra"] == 1 + assert report["runs"][0]["agent_exits"] == 0 + + def test_unknown_exception_is_agent_exit_and_gateway_signature_is_excluded( tmp_path: Path, ): diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 9ecba5d..212a1a4 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -59,6 +59,8 @@ def _write_index( def _planned(run: RunSpec) -> dict[str, object]: return { **run.to_dict(), + "reasoning_effort": "high", + "judge_reasoning_effort": "high", "attempt": 0, "status": "planned", "leaderboard_eligible": None, @@ -249,7 +251,7 @@ def run( ) remote_command = shlex.split(argv[-1]) dispatch_marker = remote_command.index("fleet-dispatch") - remote_run_args = remote_command[dispatch_marker + 13 :] + remote_run_args = remote_command[dispatch_marker + 15 :] with self._dispatch_condition: self.dispatches.append(label) self.dispatch_concurrency[label] = int(remote_run_args[10]) @@ -774,6 +776,8 @@ def test_failed_run_is_preserved_and_suffixed_rerun_completes( assert runs[0]["final_result_count"] == 1 assert runs[1]["status"] == "completed" assert runs[1]["rerun_of"] == base + assert runs[1]["reasoning_effort"] == "high" + assert runs[1]["judge_reasoning_effort"] == "high" assert executor.dispatches == [base, rerun] assert (config.local_root / "raw" / f"{base}-final-artifacts.tar.gz").is_file() assert (config.local_root / "raw" / f"{rerun}-final-artifacts.tar.gz").is_file() diff --git a/tests/test_native_eval_proxy.py b/tests/test_native_eval_proxy.py index f2866d4..ddb86c9 100644 --- a/tests/test_native_eval_proxy.py +++ b/tests/test_native_eval_proxy.py @@ -4,7 +4,11 @@ from scripts.native_eval.proxy import write_proxy_config -def test_openai_proxy_models_drop_unsupported_temperature(tmp_path: Path): +def test_openai_proxy_models_enforce_reasoning_effort( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high") config_path = tmp_path / "proxy.json" write_proxy_config(config_path) @@ -17,3 +21,6 @@ def test_openai_proxy_models_drop_unsupported_temperature(tmp_path: Path): assert models["gpt-5.6-sol"]["litellm_params"]["additional_drop_params"] == [ "temperature" ] + assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == "high" + assert models["gpt-5.6-sol"]["litellm_params"]["reasoning_effort"] == "high" + assert config["shellbench_native"]["reasoning_effort"] == "high" diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 49be82c..acc9a9e 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import json import tarfile from argparse import Namespace @@ -16,13 +17,16 @@ RunSpec, build_matrix_plan, ) +from scripts.native_eval import plan as native_plan from scripts.native_eval.proxy import write_proxy_config from scripts.native_eval.run_job import _git_commit, _run_manifest, build_run_spec from scripts.native_eval.runtime import ( + DockerTaskEnvironment, collect_agent_metrics, read_reward, write_agent_trajectory, ) +from scripts.native_eval import runtime as native_runtime from scripts.native_eval.tasks import TaskSpec, validate_suite @@ -36,6 +40,26 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: assert {run.repetition for run in plan} == {1, 2, 3} +def test_run_index_records_agent_and_judge_reasoning( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setattr(native_plan, "validate_suite", lambda _root: [object()]) + + entries = native_plan.write_run_index( + tasks_root=tmp_path, + output=tmp_path / "run-index.json", + public_tasks_commit="tasks-commit", + run_date="20260728", + reasoning_effort="high", + judge_reasoning_effort="high", + ) + + assert len(entries) == 72 + assert {entry["reasoning_effort"] for entry in entries} == {"high"} + assert {entry["judge_reasoning_effort"] for entry in entries} == {"high"} + + def test_task_loader_accepts_rich_manifest_and_compose(tmp_path: Path) -> None: task_dir = tmp_path / "browser-task" (task_dir / "environment").mkdir(parents=True) @@ -102,7 +126,11 @@ def test_validate_suite_reports_missing_required_file(tmp_path: Path) -> None: raise AssertionError("expected suite validation to fail") -def test_proxy_config_pins_only_requested_upstreams(tmp_path: Path) -> None: +def test_proxy_config_pins_only_requested_upstreams( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high") path = tmp_path / "config.json" write_proxy_config(path) config = json.loads(path.read_text(encoding="utf-8")) @@ -117,6 +145,11 @@ def test_proxy_config_pins_only_requested_upstreams(tmp_path: Path) -> None: assert "OPENAI_API_KEY" in serialized assert "ANTHROPIC_API_KEY" in serialized assert "sk-" not in serialized + assert all( + item["litellm_params"].get("reasoning_effort") == "high" + for item in config["model_list"] + if item["model_info"]["provider"] == "openai" + ) def test_run_manifest_records_native_audit_metadata( @@ -126,6 +159,8 @@ def test_run_manifest_records_native_audit_metadata( monkeypatch.setenv("SHELLBENCH_EXECUTION_MODE", "native") monkeypatch.setenv("SHELLBENCH_HARBOR_REFERENCE_COMMIT", "harbor-commit") monkeypatch.setenv("SHELLBENCH_JUDGE_MODEL_ID", "gpt-5.5") + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high") + monkeypatch.setenv("SHELLBENCH_JUDGE_REASONING_EFFORT", "high") run = RunSpec( run_label="openclaw-gpt55-full-2-r1-20260727", harness="openclaw", @@ -155,6 +190,8 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["trajectory_mode"] == "unsupported" assert manifest["canonical_model_identity"] is None assert manifest["provider_model_id"] == "gpt-5.5" + assert manifest["reasoning_effort"] == "high" + assert manifest["judge_reasoning_effort"] == "high" def test_run_spec_preserves_explicit_planned_identity() -> None: @@ -207,6 +244,9 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "sb-opus5" not in command.run_command assert "OPENROUTER_API_KEY" not in command.env assert 'exit "$status"' in command.run_command + if harness.name == "openclaw": + assert "ended with stopReason=" in command.run_command + assert "--thinking off" in command.run_command def test_hermes_uses_named_local_proxy_provider() -> None: @@ -236,6 +276,36 @@ def test_hermes_uses_named_local_proxy_provider() -> None: assert command.env == {"SHELLBENCH_PROXY_KEY": "local-proxy-key"} assert "custom:shellbench" in command.setup_command assert "http://host.docker.internal:4000/v1" in command.setup_command + assert '--session-id "$session_id" --yes --redact' in command.cleanup_command + + +def test_workdir_falls_back_to_existing_container_directory( + tmp_path: Path, + monkeypatch, +) -> None: + commands: list[list[str]] = [] + + async def fake_capture(command: list[str]) -> str: + commands.append(command) + return "" if command[1] == "inspect" else "/workspace" + + monkeypatch.setattr(native_runtime, "capture_process", fake_capture) + environment = DockerTaskEnvironment( + task=object(), # type: ignore[arg-type] + trial_dir=tmp_path, + container_name="trial", + project_name="trial", + toolchain_root=tmp_path, + container_id="container-id", + ) + + asyncio.run(environment._discover_workdir()) + + assert environment.workdir == "/workspace" + assert commands[1][-1] == ( + "if [ -d /app ]; then printf /app; " + "elif [ -d /workspace ]; then printf /workspace; else pwd; fi" + ) def test_claude_code_selects_canonical_model_explicitly() -> None: From ef0cbb8b78018b10b6c83c0943de4fd10332cff9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 06:45:17 +0200 Subject: [PATCH 02/28] feat(eval): add GPT-5.6 Luna and Terra models --- CHANGELOG.md | 4 ++++ scripts/native_eval/models.py | 14 ++++++++++++++ scripts/native_eval/proxy.py | 2 +- tests/test_native_eval_proxy.py | 2 ++ tests/test_native_eval_runner.py | 6 +++--- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b08dd37..5a2971f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,3 +7,7 @@ - Record and enforce GPT reasoning effort in native runs, preserve it across reruns, and stabilize OpenClaw and Hermes trace completion. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). + +### Added + +- Add `gpt-5.6-luna` and `gpt-5.6-terra` to the native evaluation catalog. diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index dbc29ab..dbbc424 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -46,6 +46,20 @@ def to_dict(self) -> dict[str, object]: "gpt-5.6-sol", "gpt-5.6-sol", ), + ModelSpec( + "gpt56-luna", + "gpt-5.6-luna", + "openai", + "gpt-5.6-luna", + "gpt-5.6-luna", + ), + ModelSpec( + "gpt56-terra", + "gpt-5.6-terra", + "openai", + "gpt-5.6-terra", + "gpt-5.6-terra", + ), ModelSpec( "fable5", "fable-5", diff --git a/scripts/native_eval/proxy.py b/scripts/native_eval/proxy.py index 96b19c0..3532d11 100644 --- a/scripts/native_eval/proxy.py +++ b/scripts/native_eval/proxy.py @@ -19,7 +19,7 @@ def write_proxy_config(path: Path) -> None: "model": f"{model.provider}/{model.provider_model_id}", "api_key": f"os.environ/{model.provider.upper()}_API_KEY", } - if model.slug in {"gpt55", "gpt56-sol"}: + if model.provider == "openai": litellm_params["additional_drop_params"] = ["temperature"] litellm_params["reasoning_effort"] = reasoning_effort model_list.append( diff --git a/tests/test_native_eval_proxy.py b/tests/test_native_eval_proxy.py index ddb86c9..8eb6f85 100644 --- a/tests/test_native_eval_proxy.py +++ b/tests/test_native_eval_proxy.py @@ -23,4 +23,6 @@ def test_openai_proxy_models_enforce_reasoning_effort( ] assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == "high" assert models["gpt-5.6-sol"]["litellm_params"]["reasoning_effort"] == "high" + assert models["gpt-5.6-luna"]["litellm_params"]["reasoning_effort"] == "high" + assert models["gpt-5.6-terra"]["litellm_params"]["reasoning_effort"] == "high" assert config["shellbench_native"]["reasoning_effort"] == "high" diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index acc9a9e..e0fb54e 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -33,8 +33,8 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: plan = build_matrix_plan(116, run_date="20260727") - assert len(plan) == 72 - assert len({run.run_label for run in plan}) == 72 + assert len(plan) == 96 + assert len({run.run_label for run in plan}) == 96 assert {run.harness for run in plan} == {harness.name for harness in HARNESSES} assert {run.model_slug for run in plan} == {model.slug for model in MODELS} assert {run.repetition for run in plan} == {1, 2, 3} @@ -55,7 +55,7 @@ def test_run_index_records_agent_and_judge_reasoning( judge_reasoning_effort="high", ) - assert len(entries) == 72 + assert len(entries) == 96 assert {entry["reasoning_effort"] for entry in entries} == {"high"} assert {entry["judge_reasoning_effort"] for entry in entries} == {"high"} From 903362c15f24d5e278b07f6f5203cf33a68add5b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 07:35:39 +0200 Subject: [PATCH 03/28] fix(eval): preserve OpenClaw and Hermes trajectories --- CHANGELOG.md | 2 + scripts/native_eval/fleet.py | 8 +- scripts/native_eval/harness_trajectories.py | 765 ++++++++++++++++++++ scripts/native_eval/harnesses.py | 43 +- scripts/native_eval/models.py | 2 +- scripts/native_eval/runtime.py | 15 +- tests/test_native_eval_fleet.py | 4 +- tests/test_native_eval_runner.py | 266 ++++++- 8 files changed, 1093 insertions(+), 12 deletions(-) create mode 100644 scripts/native_eval/harness_trajectories.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2971f..e5bbdcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Record and enforce GPT reasoning effort in native runs, preserve it across reruns, and stabilize OpenClaw and Hermes trace completion. +- Export OpenClaw and Hermes sessions reliably and convert their native traces + to Harbor-compatible ATIF trajectories with canonical model identity. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). ### Added diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 142e882..21aa89b 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -164,6 +164,7 @@ class FleetConfig: harbor_reference_commit: str = "" judge_model_id: str = "" execution_mode: str = "native" + parity_validated: bool = False class RunIndexStore: @@ -782,7 +783,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: execution_mode=$3 reasoning_effort=$4 judge_reasoning_effort=$5 -shift 5 +parity_validated=$6 +shift 6 mkdir -p "$root/run-logs" stdout="$root/run-logs/$label.stdout.log" stderr="$root/run-logs/$label.stderr.log" @@ -799,6 +801,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: "SHELLBENCH_EXECUTION_MODE=$execution_mode" \ "SHELLBENCH_REASONING_EFFORT=$reasoning_effort" \ "SHELLBENCH_JUDGE_REASONING_EFFORT=$judge_reasoning_effort" \ + "SHELLBENCH_PARITY_VALIDATED=$parity_validated" \ "$root/runner/scripts/native_eval/remote_run.sh" "$@" \ >"$stdout" 2>"$stderr" None: self.config.execution_mode, reasoning_effort, judge_reasoning_effort, + str(self.config.parity_validated).lower(), *args, ], ), @@ -1376,6 +1380,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--harbor-reference-commit", default="") parser.add_argument("--judge-model-id", default="") parser.add_argument("--execution-mode", default="native") + parser.add_argument("--parity-validated", action="store_true") args = parser.parse_args(argv) args.model_max_runs = _parse_model_values( parser, @@ -1426,6 +1431,7 @@ def main(argv: list[str] | None = None) -> int: harbor_reference_commit=args.harbor_reference_commit, judge_model_id=args.judge_model_id, execution_mode=args.execution_mode, + parity_validated=args.parity_validated, ) try: return FleetController(config).run() diff --git a/scripts/native_eval/harness_trajectories.py b/scripts/native_eval/harness_trajectories.py new file mode 100644 index 0000000..d247add --- /dev/null +++ b/scripts/native_eval/harness_trajectories.py @@ -0,0 +1,765 @@ +from __future__ import annotations + +import json +import os +import re +import uuid +from pathlib import Path +from typing import Any + +from scripts.native_eval.models import RunSpec + + +def load_openclaw_envelope(path: Path) -> dict[str, Any] | None: + if not path.is_file(): + return None + text = path.read_text(encoding="utf-8", errors="replace").strip() + decoder = json.JSONDecoder() + for start in range(len(text) - 1, -1, -1): + if text[start] != "{": + continue + try: + value, _ = decoder.raw_decode(text[start:]) + except (json.JSONDecodeError, ValueError): + continue + if ( + isinstance(value, dict) + and isinstance(value.get("payloads"), list) + and isinstance(value.get("meta"), dict) + ): + return value + return None + + +def write_openclaw_trajectory( + instruction: str, + run: RunSpec, + agent_dir: Path, +) -> dict[str, Any]: + log_path = agent_dir / "openclaw.txt" + session_path = agent_dir / "openclaw.session.jsonl" + envelope = load_openclaw_envelope(log_path) + meta = envelope.get("meta") if envelope else {} + if not isinstance(meta, dict): + meta = {} + agent_meta = meta.get("agentMeta") + if not isinstance(agent_meta, dict): + agent_meta = {} + envelope_models = { + value + for value in ( + agent_meta.get("model"), + (meta.get("executionTrace") or {}).get("winnerModel") + if isinstance(meta.get("executionTrace"), dict) + else None, + ) + if isinstance(value, str) and value + } + session_models = _openclaw_session_models(session_path) + log_models = _openclaw_log_models(log_path) + observed_models = session_models or envelope_models + if not observed_models and run.model_id in log_models: + observed_models = {run.model_id} + runtime_model_name = ( + next(iter(observed_models)) if len(observed_models) == 1 else None + ) + canonical_model_identity = observed_models == {run.model_id} + + steps = _openclaw_session_steps( + session_path, + instruction=instruction, + model_name=f"{run.provider}/{run.model_id}", + ) + trace_fidelity = "session" + source_path = session_path + if not steps: + if envelope is None: + return _unavailable( + session_path, + runtime_model_name=runtime_model_name, + canonical_model_identity=canonical_model_identity, + observed_models=observed_models, + ) + steps = _openclaw_envelope_steps( + envelope, + instruction=instruction, + model_name=f"{run.provider}/{run.model_id}", + ) + trace_fidelity = "envelope" + source_path = log_path + if len(steps) < 2: + return _unavailable( + source_path, + runtime_model_name=runtime_model_name, + canonical_model_identity=canonical_model_identity, + observed_models=observed_models, + ) + + usage = agent_meta.get("usage") + if not isinstance(usage, dict): + usage = _openclaw_session_usage(session_path) + input_tokens = _int(usage.get("input")) + output_tokens = _int(usage.get("output")) + cache_read = _int(usage.get("cacheRead")) + session_id = str( + agent_meta.get("sessionId") + or _openclaw_session_id(session_path) + or uuid.uuid4() + ) + model_name = runtime_model_name or run.model_id + trajectory = { + "schema_version": "ATIF-v1.7", + "session_id": session_id, + "agent": { + "name": run.harness, + "version": run.harness_version, + "model_name": f"{run.provider}/{model_name}", + }, + "steps": steps, + "final_metrics": _without_none( + { + "total_prompt_tokens": input_tokens + cache_read or None, + "total_completion_tokens": output_tokens or None, + "total_cached_tokens": cache_read or None, + "total_steps": len(steps), + } + ), + "extra": { + "native_raw_trace_file": source_path.name, + "trace_fidelity": trace_fidelity, + "observed_models": sorted(observed_models), + "log_models": sorted(log_models), + "stop_reason": _nested_string(meta, "completion", "stopReason"), + "aborted": meta.get("aborted"), + }, + } + _atomic_write_json(agent_dir / "trajectory.json", trajectory) + return { + "trajectory_status": "real", + "trajectory_source": str(source_path), + "trajectory_event_count": len(steps), + "runtime_model_name": runtime_model_name, + "canonical_model_identity": canonical_model_identity, + "trajectory_validation": { + "trace_fidelity": trace_fidelity, + "observed_models": sorted(observed_models), + "log_models": sorted(log_models), + "session_id": session_id, + }, + } + + +def write_hermes_trajectory( + instruction: str, + run: RunSpec, + agent_dir: Path, +) -> dict[str, Any]: + source_path = agent_dir / "hermes-session.jsonl" + sessions = _load_hermes_sessions(source_path) + if not sessions: + return _unavailable(source_path) + session = next( + ( + item + for item in reversed(sessions) + if item.get("model") == run.model_id and item.get("messages") + ), + sessions[-1], + ) + observed_models = { + value + for value in [session.get("model"), *_hermes_message_models(session)] + if isinstance(value, str) and value + } + runtime_model_name = ( + next(iter(observed_models)) if len(observed_models) == 1 else None + ) + canonical_model_identity = observed_models == {run.model_id} + validation = _validate_hermes_session(session, instruction) + steps = _hermes_steps( + session, + instruction, + model_name=f"{run.provider}/{run.model_id}", + ) + if len(steps) < 2: + return _unavailable( + source_path, + runtime_model_name=runtime_model_name, + canonical_model_identity=canonical_model_identity, + observed_models=observed_models, + ) + + session_id = str(session.get("id") or session.get("session_id") or uuid.uuid4()) + trajectory = { + "schema_version": "ATIF-v1.7", + "session_id": session_id, + "agent": { + "name": run.harness, + "version": run.harness_version, + "model_name": f"{run.provider}/{run.model_id}", + }, + "steps": steps, + "final_metrics": _without_none( + { + "total_prompt_tokens": _int(session.get("input_tokens")) or None, + "total_completion_tokens": _int(session.get("output_tokens")) or None, + "total_cached_tokens": ( + _int(session.get("cache_read_tokens")) or None + ), + "total_steps": len(steps), + } + ), + "extra": { + "native_raw_trace_file": source_path.name, + "trace_fidelity": "session", + "observed_models": sorted(observed_models), + "exported_session_count": len(sessions), + "reasoning_tokens": _int(session.get("reasoning_tokens")) or None, + "cache_write_tokens": _int(session.get("cache_write_tokens")) or None, + "api_call_count": _int(session.get("api_call_count")) or None, + "tool_call_count": _int(session.get("tool_call_count")) or None, + }, + } + _atomic_write_json(agent_dir / "trajectory.json", trajectory) + return { + "trajectory_status": "real", + "trajectory_source": str(source_path), + "trajectory_event_count": len(steps), + "runtime_model_name": runtime_model_name, + "canonical_model_identity": canonical_model_identity, + "trajectory_validation": { + "trace_fidelity": "session", + "observed_models": sorted(observed_models), + "session_id": session_id, + **validation, + }, + } + + +def _openclaw_envelope_steps( + envelope: dict[str, Any], + *, + instruction: str, + model_name: str, +) -> list[dict[str, Any]]: + meta = envelope.get("meta") + if not isinstance(meta, dict): + meta = {} + payloads = envelope.get("payloads") + if not isinstance(payloads, list): + payloads = [] + visible: list[str] = [] + reasoning: list[str] = [] + for item in payloads: + if not isinstance(item, dict): + continue + text = item.get("text") + if not isinstance(text, str) or not text.strip(): + continue + (reasoning if item.get("isReasoning") is True else visible).append(text.strip()) + assistant_text = "\n\n".join(visible) + if not assistant_text and isinstance(meta.get("finalAssistantVisibleText"), str): + assistant_text = meta["finalAssistantVisibleText"].strip() + + agent_meta = meta.get("agentMeta") + usage = agent_meta.get("usage") if isinstance(agent_meta, dict) else None + if not isinstance(usage, dict): + usage = {} + input_tokens = _int(usage.get("input")) + output_tokens = _int(usage.get("output")) + cache_read = _int(usage.get("cacheRead")) + cache_write = _int(usage.get("cacheWrite")) + metrics = _without_none( + { + "prompt_tokens": input_tokens + cache_read or None, + "completion_tokens": output_tokens or None, + "cached_tokens": cache_read or None, + "extra": {"cache_write_tokens": cache_write} if cache_write else None, + } + ) + agent_step = _without_none( + { + "step_id": 2, + "source": "agent", + "message": assistant_text or "(no assistant text in OpenClaw output)", + "model_name": model_name, + "reasoning_content": "\n\n".join(reasoning) or None, + "metrics": metrics or None, + "llm_call_count": 1, + } + ) + return [ + {"step_id": 1, "source": "user", "message": instruction}, + agent_step, + ] + + +def _openclaw_session_steps( + path: Path, + *, + instruction: str, + model_name: str, +) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + rows: list[tuple[dict[str, Any], dict[str, Any]]] = [] + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(record, dict) or record.get("type") != "message": + continue + message = record.get("message") + if isinstance(message, dict) and message.get("role") in { + "user", + "assistant", + "toolResult", + }: + rows.append((record, message)) + if not rows: + return [] + + steps: list[dict[str, Any]] = [] + first_user = True + index = 0 + while index < len(rows): + record, message = rows[index] + timestamp = record.get("timestamp") + timestamp = timestamp if isinstance(timestamp, str) else None + role = message.get("role") + if role == "user": + body = _content_text(message.get("content")) + text = instruction.strip() if first_user and instruction.strip() else body + first_user = False + steps.append( + _without_none( + { + "step_id": len(steps) + 1, + "source": "user", + "message": text or "(empty user message)", + "timestamp": timestamp, + } + ) + ) + index += 1 + continue + if role != "assistant": + index += 1 + continue + + text, tool_calls = _openclaw_assistant_content(message.get("content")) + error = message.get("errorMessage") + if text.strip(): + agent_text = text.strip() + elif isinstance(error, str) and error.strip(): + agent_text = f"(error) {error.strip()}" + else: + agent_text = "(no assistant text)" + + pending = { + call["tool_call_id"] for call in tool_calls if call.get("tool_call_id") + } + observations: list[dict[str, Any]] = [] + cursor = index + 1 + while cursor < len(rows) and rows[cursor][1].get("role") == "toolResult": + tool_result = rows[cursor][1] + call_id = str(tool_result.get("toolCallId") or "") + if call_id not in pending: + break + details = tool_result.get("details") + content = details.get("aggregated") if isinstance(details, dict) else None + if not isinstance(content, str) or not content.strip(): + content = _content_text(tool_result.get("content")) + observations.append( + _without_none( + { + "source_call_id": call_id or None, + "content": content or None, + } + ) + ) + pending.discard(call_id) + cursor += 1 + if not pending: + break + + usage = message.get("usage") + metrics = _openclaw_usage_metrics(usage) + steps.append( + _without_none( + { + "step_id": len(steps) + 1, + "source": "agent", + "message": agent_text, + "timestamp": timestamp, + "model_name": model_name, + "tool_calls": tool_calls or None, + "observation": ( + {"results": observations} if observations else None + ), + "metrics": metrics, + "llm_call_count": 1, + } + ) + ) + index = cursor + return steps if len(steps) >= 2 else [] + + +def _openclaw_session_records(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + records: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + records.append(value) + return records + + +def _openclaw_session_models(path: Path) -> set[str]: + models: set[str] = set() + for record in _openclaw_session_records(path): + if record.get("type") == "model_change": + model_id = record.get("modelId") + if isinstance(model_id, str) and model_id: + models.add(model_id) + message = record.get("message") + if ( + record.get("type") == "message" + and isinstance(message, dict) + and message.get("role") == "assistant" + and isinstance(message.get("model"), str) + and message["model"] + ): + models.add(message["model"]) + return models + + +def _openclaw_log_models(path: Path) -> set[str]: + if not path.is_file(): + return set() + return set( + re.findall( + r"\[model-fetch\].*?\bmodel=([^\s]+)", + path.read_text(encoding="utf-8", errors="replace"), + ) + ) + + +def _openclaw_session_id(path: Path) -> str | None: + for record in _openclaw_session_records(path): + if record.get("type") == "session" and isinstance(record.get("id"), str): + return record["id"] + return path.stem if path.is_file() else None + + +def _openclaw_session_usage(path: Path) -> dict[str, int]: + totals = {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0} + for record in _openclaw_session_records(path): + message = record.get("message") + if not isinstance(message, dict) or message.get("role") != "assistant": + continue + usage = message.get("usage") + if not isinstance(usage, dict): + continue + for key in totals: + totals[key] += _int(usage.get(key)) + return totals + + +def _openclaw_assistant_content( + content: Any, +) -> tuple[str, list[dict[str, Any]]]: + if not isinstance(content, list): + return "", [] + text: list[str] = [] + tools: list[dict[str, Any]] = [] + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") == "text" and isinstance(part.get("text"), str): + text.append(part["text"]) + elif part.get("type") == "toolCall" and isinstance(part.get("name"), str): + tools.append( + { + "tool_call_id": str(part.get("id") or ""), + "function_name": part["name"], + "arguments": _arguments(part.get("arguments")), + } + ) + return "".join(text), tools + + +def _openclaw_usage_metrics(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + input_tokens = _int(value.get("input")) + output_tokens = _int(value.get("output")) + cache_read = _int(value.get("cacheRead")) + cache_write = _int(value.get("cacheWrite")) + metrics = _without_none( + { + "prompt_tokens": input_tokens + cache_read or None, + "completion_tokens": output_tokens or None, + "cached_tokens": cache_read or None, + "extra": {"cache_write_tokens": cache_write} if cache_write else None, + } + ) + return metrics or None + + +def _load_hermes_sessions(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + sessions: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and isinstance(value.get("messages"), list): + sessions.append(value) + return sessions + + +def _hermes_steps( + session: dict[str, Any], + instruction: str, + *, + model_name: str, +) -> list[dict[str, Any]]: + messages = [ + item for item in session.get("messages", []) if isinstance(item, dict) + ] + steps: list[dict[str, Any]] = [] + first_user = True + index = 0 + while index < len(messages): + message = messages[index] + role = message.get("role") + if role == "user": + content = _content_text(message.get("content")) + if first_user and instruction.strip(): + content = instruction.strip() + first_user = False + if content: + steps.append( + { + "step_id": len(steps) + 1, + "source": "user", + "message": content, + } + ) + elif role == "assistant": + content = _content_text(message.get("content")) + tool_calls = _hermes_tool_calls(message.get("tool_calls")) + observations: list[dict[str, Any]] = [] + cursor = index + 1 + while cursor < len(messages) and messages[cursor].get("role") == "tool": + tool_message = messages[cursor] + observations.append( + _without_none( + { + "source_call_id": tool_message.get("tool_call_id"), + "content": _content_text(tool_message.get("content")) + or None, + } + ) + ) + cursor += 1 + if content or tool_calls: + steps.append( + _without_none( + { + "step_id": len(steps) + 1, + "source": "agent", + "message": content or "[tool call]", + "model_name": model_name, + "reasoning_content": ( + message.get("reasoning_content") + if isinstance( + message.get("reasoning_content"), str + ) + else None + ), + "tool_calls": tool_calls or None, + "observation": ( + {"results": observations} if observations else None + ), + "llm_call_count": 1, + } + ) + ) + index = cursor - 1 + index += 1 + return steps + + +def _validate_hermes_session( + session: dict[str, Any], + instruction: str, +) -> dict[str, Any]: + messages = [ + item for item in session.get("messages", []) if isinstance(item, dict) + ] + message_ids = [item.get("id") for item in messages] + unique_message_ids = all( + value is not None for value in message_ids + ) and len(message_ids) == len(set(message_ids)) + timestamps = [ + float(item["timestamp"]) + for item in messages + if isinstance(item.get("timestamp"), (int, float)) + ] + timestamps_ordered = timestamps == sorted(timestamps) + flattened_tool_calls = sum( + len(item.get("tool_calls") or []) + for item in messages + if isinstance(item.get("tool_calls"), list) + ) + tool_call_ids = { + str(call.get("id")) + for item in messages + if isinstance(item.get("tool_calls"), list) + for call in item["tool_calls"] + if isinstance(call, dict) and call.get("id") + } + tool_result_ids = { + str(item.get("tool_call_id")) + for item in messages + if item.get("role") == "tool" and item.get("tool_call_id") + } + first_user = next( + (item for item in messages if item.get("role") == "user"), + None, + ) + instruction_matches = first_user is not None and _normalize_text( + _content_text(first_user.get("content")) + ) == _normalize_text(instruction) + final_message = messages[-1] if messages else {} + terminal_event_seen = ( + final_message.get("role") == "assistant" + and bool(_content_text(final_message.get("content")).strip()) + and final_message.get("finish_reason") == "stop" + and tool_call_ids <= tool_result_ids + ) + return { + "message_count_matches": session.get("message_count") == len(messages), + "tool_call_count_matches": ( + session.get("tool_call_count") == flattened_tool_calls + ), + "message_ids_unique": unique_message_ids, + "timestamps_ordered": timestamps_ordered, + "instruction_matches": instruction_matches, + "unmatched_tool_call_ids": sorted(tool_call_ids - tool_result_ids), + "orphan_tool_result_ids": sorted(tool_result_ids - tool_call_ids), + "terminal_event_seen": terminal_event_seen, + } + + +def _hermes_tool_calls(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + calls: list[dict[str, Any]] = [] + for item in value: + if not isinstance(item, dict): + continue + function = item.get("function") + if not isinstance(function, dict): + continue + calls.append( + { + "tool_call_id": str(item.get("id") or uuid.uuid4().hex[:8]), + "function_name": str(function.get("name") or "unknown"), + "arguments": _arguments(function.get("arguments")), + } + ) + return calls + + +def _hermes_message_models(session: dict[str, Any]) -> list[str]: + models: list[str] = [] + for message in session.get("messages", []): + if isinstance(message, dict) and isinstance(message.get("model"), str): + models.append(message["model"]) + return models + + +def _content_text(content: Any) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + return " ".join( + str(part.get("text") or part.get("content") or "") + for part in content + if isinstance(part, dict) + and isinstance(part.get("text") or part.get("content"), str) + ) + + +def _arguments(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {"raw": value} + return parsed if isinstance(parsed, dict) else {"value": parsed} + return {} + + +def _unavailable( + source_path: Path, + *, + runtime_model_name: str | None = None, + canonical_model_identity: bool = False, + observed_models: set[str] | None = None, +) -> dict[str, Any]: + return { + "trajectory_status": "unavailable", + "trajectory_source": str(source_path), + "trajectory_event_count": 0, + "runtime_model_name": runtime_model_name, + "canonical_model_identity": canonical_model_identity, + "trajectory_validation": { + "observed_models": sorted(observed_models or set()), + }, + } + + +def _nested_string(value: dict[str, Any], *keys: str) -> str | None: + current: Any = value + for key in keys: + if not isinstance(current, dict): + return None + current = current.get(key) + return current if isinstance(current, str) else None + + +def _without_none(value: dict[str, Any]) -> dict[str, Any]: + return {key: item for key, item in value.items() if item is not None} + + +def _int(value: Any) -> int: + return int(value) if isinstance(value, (int, float)) else 0 + + +def _normalize_text(value: str) -> str: + return "\n".join(line.rstrip() for line in value.strip().splitlines()) + + +def _atomic_write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text( + json.dumps(value, indent=2, sort_keys=False) + "\n", + encoding="utf-8", + ) + temporary.replace(path) diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 4de7025..bcaf4a2 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -122,13 +122,45 @@ def _openclaw( "python3 - <<'PY'\n" "import json, pathlib, shutil\n" "p=pathlib.Path('/logs/agent/openclaw.txt')\n" + "sessions=pathlib.Path(" + "'/tmp/shellbench-openclaw/.openclaw/agents/main/sessions')\n" + "if sessions.is_dir():\n" + " shutil.copytree(sessions, '/logs/agent/openclaw.sessions', " + "dirs_exist_ok=True)\n" + "src=None\n" "try:\n" - " d=json.loads(p.read_text())\n" - " src=((d.get('meta') or {}).get('agentMeta') or {}).get('sessionFile')\n" - " if src and pathlib.Path(src).is_file():\n" - " shutil.copy2(src, '/logs/agent/openclaw.session.jsonl')\n" + " raw=p.read_text(encoding='utf-8', errors='replace').strip()\n" + " dec=json.JSONDecoder(); d=None\n" + " for start in range(len(raw)-1, -1, -1):\n" + " if raw[start] != '{': continue\n" + " try: candidate, _ = dec.raw_decode(raw[start:])\n" + " except (json.JSONDecodeError, ValueError): continue\n" + " if isinstance(candidate, dict) and " + "isinstance(candidate.get('meta'), dict):\n" + " d=candidate; break\n" + " if d:\n" + " src=((d.get('meta') or {}).get('agentMeta') or {}).get('sessionFile')\n" "except Exception:\n" " pass\n" + "if not src and sessions.is_dir():\n" + " try:\n" + " store=json.loads((sessions/'sessions.json').read_text())\n" + " entry=store.get('agent:main:main') if isinstance(store, dict) else None\n" + " if isinstance(entry, dict):\n" + " src=entry.get('sessionFile')\n" + " if not src and entry.get('sessionId'):\n" + " src=str(sessions/f\"{entry['sessionId']}.jsonl\")\n" + " except Exception:\n" + " pass\n" + "source=pathlib.Path(src) if src else None\n" + "if source and not source.is_absolute(): source=sessions/source\n" + "if not source or not source.is_file():\n" + " candidates=[f for f in sessions.glob('*.jsonl') " + "if '.trajectory.' not in f.name] if sessions.is_dir() else []\n" + " source=max(candidates, key=lambda f: f.stat().st_mtime, " + "default=None)\n" + "if source and source.is_file():\n" + " shutil.copy2(source, '/logs/agent/openclaw.session.jsonl')\n" "PY" ) return HarnessCommand( @@ -207,6 +239,9 @@ def _hermes( "if [ -n \"$session_id\" ]; then " "hermes sessions export /logs/agent/hermes-session.jsonl " "--session-id \"$session_id\" --yes --redact; " + "else " + "hermes sessions export /logs/agent/hermes-session.jsonl " + "--yes --redact; " "fi" ) return HarnessCommand( diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index dbbc424..338b3e4 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -99,7 +99,7 @@ def to_dict(self) -> dict[str, object]: NODE_VERSION = "22.23.1" LITELLM_VERSION = "1.93.0" -REAL_TRAJECTORY_HARNESSES = frozenset({"codex"}) +REAL_TRAJECTORY_HARNESSES = frozenset({"openclaw", "hermes", "codex"}) def model_by_slug(slug: str) -> ModelSpec: diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index 088e786..f6b9e39 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -10,6 +10,11 @@ from pathlib import Path from typing import Any +from scripts.native_eval.harness_trajectories import ( + load_openclaw_envelope, + write_hermes_trajectory, + write_openclaw_trajectory, +) from scripts.native_eval.harnesses import TOOLCHAIN_ROOT, build_harness_command from scripts.native_eval.models import RunSpec from scripts.native_eval.tasks import TaskSpec @@ -704,7 +709,11 @@ def collect_agent_metrics(harness: str, agent_dir: Path) -> dict[str, Any]: path = candidates.get(harness) if path is None or not path.is_file(): return metrics - events = _load_json_events(path) + if harness == "openclaw": + envelope = load_openclaw_envelope(path) + events = [envelope] if envelope is not None else [] + else: + events = _load_json_events(path) for event in events: usage = _find_usage(event) if usage: @@ -855,6 +864,10 @@ def write_agent_trajectory( run: RunSpec, agent_dir: Path, ) -> dict[str, Any]: + if run.harness == "openclaw": + return write_openclaw_trajectory(task.instruction, run, agent_dir) + if run.harness == "hermes": + return write_hermes_trajectory(task.instruction, run, agent_dir) if run.harness != "codex": return { "trajectory_status": "unsupported", diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 212a1a4..c5ea06d 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -251,7 +251,7 @@ def run( ) remote_command = shlex.split(argv[-1]) dispatch_marker = remote_command.index("fleet-dispatch") - remote_run_args = remote_command[dispatch_marker + 15 :] + remote_run_args = remote_command[dispatch_marker + 16 :] with self._dispatch_condition: self.dispatches.append(label) self.dispatch_concurrency[label] = int(remote_run_args[10]) @@ -705,6 +705,7 @@ def test_parse_args_accepts_repeatable_model_limits(tmp_path: Path) -> None: "gpt-5.5", "--execution-mode", "native", + "--parity-validated", "--model-task-concurrency", "fable5=2", ] @@ -715,6 +716,7 @@ def test_parse_args_accepts_repeatable_model_limits(tmp_path: Path) -> None: assert args.harbor_reference_commit == "harbor-commit" assert args.judge_model_id == "gpt-5.5" assert args.execution_mode == "native" + assert args.parity_validated is True assert args.model_task_concurrency == {"fable5": 2} diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index e0fb54e..64f2e79 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -187,7 +187,7 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["execution_mode"] == "native" assert manifest["harbor_reference_commit"] == "harbor-commit" assert manifest["judge_model_id"] == "gpt-5.5" - assert manifest["trajectory_mode"] == "unsupported" + assert manifest["trajectory_mode"] == "real_harness_events" assert manifest["canonical_model_identity"] is None assert manifest["provider_model_id"] == "gpt-5.5" assert manifest["reasoning_effort"] == "high" @@ -277,6 +277,7 @@ def test_hermes_uses_named_local_proxy_provider() -> None: assert "custom:shellbench" in command.setup_command assert "http://host.docker.internal:4000/v1" in command.setup_command assert '--session-id "$session_id" --yes --redact' in command.cleanup_command + assert "else hermes sessions export" in command.cleanup_command def test_workdir_falls_back_to_existing_container_directory( @@ -521,10 +522,10 @@ def test_codex_trajectory_rejects_truncated_or_mismatched_stream(tmp_path: Path) assert metadata["trajectory_validation"]["terminal_event_seen"] is False -def test_non_codex_trajectory_is_explicitly_unranked(tmp_path: Path) -> None: +def test_unsupported_trajectory_is_explicitly_unranked(tmp_path: Path) -> None: run = RunSpec( - run_label="openclaw-gpt55", - harness="openclaw", + run_label="claude-code-gpt55", + harness="claude-code", harness_version="test", model_slug="gpt55", model_id="gpt-5.5", @@ -562,6 +563,263 @@ def test_non_codex_trajectory_is_explicitly_unranked(tmp_path: Path) -> None: assert metadata["canonical_model_identity"] is False +def test_openclaw_mixed_log_converts_harbor_envelope_to_atif( + tmp_path: Path, +) -> None: + agent_dir = tmp_path / "agent" + agent_dir.mkdir() + envelope = { + "payloads": [{"text": "done", "mediaUrl": None}], + "meta": { + "agentMeta": { + "sessionId": "session-123", + "model": "gpt-5.6-luna", + "usage": { + "input": 10, + "output": 20, + "cacheRead": 30, + "cacheWrite": 4, + }, + }, + "executionTrace": { + "winnerProvider": "openai", + "winnerModel": "gpt-5.6-luna", + }, + "completion": {"stopReason": "stop"}, + "aborted": False, + }, + } + (agent_dir / "openclaw.txt").write_text( + "debug preamble\n" + + json.dumps(envelope, indent=2) + + "\n[agents/agent-command] ended with stopReason=stop\n", + encoding="utf-8", + ) + run = RunSpec( + run_label="openclaw-gpt56-luna", + harness="openclaw", + harness_version="test", + model_slug="gpt56-luna", + model_id="gpt-5.6-luna", + provider="openai", + proxy_model_name="gpt-5.6-luna", + repetition=1, + expected_task_count=1, + run_date="20260728", + ) + task = _trajectory_task(tmp_path, "do the task") + + metadata = write_agent_trajectory(task, run, agent_dir) + trajectory = json.loads((agent_dir / "trajectory.json").read_text()) + metrics = collect_agent_metrics("openclaw", agent_dir) + + assert metadata["trajectory_status"] == "real" + assert metadata["canonical_model_identity"] is True + assert metadata["trajectory_validation"]["trace_fidelity"] == "envelope" + assert trajectory["schema_version"] == "ATIF-v1.7" + assert trajectory["session_id"] == "session-123" + assert trajectory["agent"]["model_name"] == "openai/gpt-5.6-luna" + assert trajectory["steps"][1]["message"] == "done" + assert trajectory["final_metrics"]["total_prompt_tokens"] == 40 + assert metrics["n_input_tokens"] == 10 + assert metrics["n_cache_tokens"] == 30 + assert metrics["n_output_tokens"] == 20 + + +def test_openclaw_session_without_envelope_converts_to_atif( + tmp_path: Path, +) -> None: + agent_dir = tmp_path / "agent" + agent_dir.mkdir() + records = [ + { + "type": "session", + "id": "session-only-123", + "timestamp": "2026-07-28T00:00:00Z", + "cwd": "/app", + }, + { + "type": "model_change", + "id": "model-1", + "parentId": None, + "timestamp": "2026-07-28T00:00:01Z", + "provider": "openai", + "modelId": "gpt-5.6-terra", + }, + { + "type": "message", + "id": "user-1", + "parentId": "model-1", + "timestamp": "2026-07-28T00:00:02Z", + "message": {"role": "user", "content": "do the task"}, + }, + { + "type": "message", + "id": "assistant-1", + "parentId": "user-1", + "timestamp": "2026-07-28T00:00:03Z", + "message": { + "role": "assistant", + "provider": "openai", + "model": "gpt-5.6-terra", + "content": [{"type": "text", "text": "done"}], + "usage": { + "input": 12, + "output": 7, + "cacheRead": 30, + "cacheWrite": 2, + }, + "stopReason": "stop", + }, + }, + ] + (agent_dir / "openclaw.session.jsonl").write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + (agent_dir / "openclaw.txt").write_text( + "[provider-transport-fetch] [model-fetch] response " + "provider=openai api=openai-responses model=gpt-5.6-terra status=200\n" + "run ended before JSON envelope\n", + encoding="utf-8", + ) + run = RunSpec( + run_label="openclaw-gpt56-terra", + harness="openclaw", + harness_version="test", + model_slug="gpt56-terra", + model_id="gpt-5.6-terra", + provider="openai", + proxy_model_name="gpt-5.6-terra", + repetition=1, + expected_task_count=1, + run_date="20260728", + ) + task = _trajectory_task(tmp_path, "do the task") + + metadata = write_agent_trajectory(task, run, agent_dir) + trajectory = json.loads((agent_dir / "trajectory.json").read_text()) + + assert metadata["trajectory_status"] == "real" + assert metadata["canonical_model_identity"] is True + assert metadata["trajectory_validation"]["trace_fidelity"] == "session" + assert metadata["trajectory_validation"]["log_models"] == ["gpt-5.6-terra"] + assert trajectory["session_id"] == "session-only-123" + assert trajectory["agent"]["model_name"] == "openai/gpt-5.6-terra" + assert trajectory["final_metrics"]["total_prompt_tokens"] == 42 + assert trajectory["final_metrics"]["total_completion_tokens"] == 7 + assert trajectory["final_metrics"]["total_cached_tokens"] == 30 + + +def test_hermes_session_converts_parallel_tools_to_atif(tmp_path: Path) -> None: + agent_dir = tmp_path / "agent" + agent_dir.mkdir() + session = { + "id": "hermes-session-123", + "model": "gpt-5.6-sol", + "message_count": 4, + "tool_call_count": 1, + "input_tokens": 100, + "output_tokens": 25, + "cache_read_tokens": 75, + "reasoning_tokens": 8, + "api_call_count": 2, + "messages": [ + { + "id": 1, + "role": "user", + "content": "do the task", + "timestamp": 1.0, + }, + { + "id": 2, + "role": "assistant", + "content": "", + "timestamp": 2.0, + "finish_reason": "tool_calls", + "tool_calls": [ + { + "id": "call-1", + "function": { + "name": "read_file", + "arguments": '{"path":"/app/input.txt"}', + }, + } + ], + }, + { + "id": 3, + "role": "tool", + "content": "input", + "tool_call_id": "call-1", + "tool_name": "read_file", + "timestamp": 3.0, + }, + { + "id": 4, + "role": "assistant", + "content": "done", + "timestamp": 4.0, + "finish_reason": "stop", + }, + ], + } + (agent_dir / "hermes-session.jsonl").write_text( + json.dumps(session) + "\n", + encoding="utf-8", + ) + run = RunSpec( + run_label="hermes-gpt56-sol", + harness="hermes", + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=1, + expected_task_count=1, + run_date="20260728", + ) + task = _trajectory_task(tmp_path, "do the task") + + metadata = write_agent_trajectory(task, run, agent_dir) + trajectory = json.loads((agent_dir / "trajectory.json").read_text()) + + assert metadata["trajectory_status"] == "real" + assert metadata["canonical_model_identity"] is True + assert metadata["trajectory_validation"]["terminal_event_seen"] is True + assert metadata["trajectory_validation"]["instruction_matches"] is True + assert len(trajectory["steps"]) == 3 + assert trajectory["steps"][1]["tool_calls"][0]["function_name"] == "read_file" + assert trajectory["steps"][1]["observation"]["results"][0]["content"] == "input" + assert trajectory["final_metrics"]["total_prompt_tokens"] == 100 + assert trajectory["final_metrics"]["total_cached_tokens"] == 75 + assert trajectory["final_metrics"]["total_completion_tokens"] == 25 + + +def _trajectory_task(tmp_path: Path, instruction: str) -> TaskSpec: + task_dir = tmp_path / "task" + task_dir.mkdir(exist_ok=True) + return TaskSpec( + name="task", + title="task", + path=task_dir, + instruction=instruction, + raw_config={}, + checksum="abc", + dockerfile=task_dir / "Dockerfile", + build_context=task_dir, + compose_file=None, + verifier_command="bash /tests/test.sh", + agent_timeout_sec=900, + verifier_timeout_sec=300, + build_timeout_sec=1800, + mcp_servers=(), + environment_env={}, + verifier_env={}, + ) + + def test_codex_metrics_include_cached_input_tokens(tmp_path: Path) -> None: agent_dir = tmp_path / "agent" agent_dir.mkdir() From 1e7eaf7bfe95ff883956a12fd4b5707f5c67515f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 07:56:30 +0200 Subject: [PATCH 04/28] fix(eval): preserve restricted trace archives --- CHANGELOG.md | 1 + scripts/native_eval/harnesses.py | 10 +++++++--- scripts/native_eval/remote_checkpoint.sh | 5 +++-- scripts/native_eval/remote_run.sh | 3 ++- tests/test_native_eval_runner.py | 4 ++++ 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5bbdcb..2c002df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ reruns, and stabilize OpenClaw and Hermes trace completion. - Export OpenClaw and Hermes sessions reliably and convert their native traces to Harbor-compatible ATIF trajectories with canonical model identity. +- Preserve restricted harness trace files in checkpoint and final archives. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). ### Added diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index bcaf4a2..47c017c 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -124,9 +124,11 @@ def _openclaw( "p=pathlib.Path('/logs/agent/openclaw.txt')\n" "sessions=pathlib.Path(" "'/tmp/shellbench-openclaw/.openclaw/agents/main/sessions')\n" + "archive=pathlib.Path('/logs/agent/openclaw.sessions')\n" "if sessions.is_dir():\n" - " shutil.copytree(sessions, '/logs/agent/openclaw.sessions', " - "dirs_exist_ok=True)\n" + " shutil.copytree(sessions, archive, dirs_exist_ok=True)\n" + " for item in [archive, *archive.rglob('*')]:\n" + " item.chmod(0o755 if item.is_dir() else 0o644)\n" "src=None\n" "try:\n" " raw=p.read_text(encoding='utf-8', errors='replace').strip()\n" @@ -160,7 +162,9 @@ def _openclaw( " source=max(candidates, key=lambda f: f.stat().st_mtime, " "default=None)\n" "if source and source.is_file():\n" - " shutil.copy2(source, '/logs/agent/openclaw.session.jsonl')\n" + " destination=pathlib.Path('/logs/agent/openclaw.session.jsonl')\n" + " shutil.copy2(source, destination)\n" + " destination.chmod(0o644)\n" "PY" ) return HarnessCommand( diff --git a/scripts/native_eval/remote_checkpoint.sh b/scripts/native_eval/remote_checkpoint.sh index 085d786..38e17a4 100755 --- a/scripts/native_eval/remote_checkpoint.sh +++ b/scripts/native_eval/remote_checkpoint.sh @@ -34,14 +34,15 @@ mkdir -p \ "$SNAPSHOT_DIR/shellbench_meta-$RUN_LABEL" if [[ -d "$JOB_DIR" ]]; then - rsync -a --exclude '*.tmp' "$JOB_DIR/" \ + sudo rsync -a --exclude '*.tmp' "$JOB_DIR/" \ "$SNAPSHOT_DIR/results/jobs/$RUN_LABEL/" fi if [[ -d "$PROXY_DIR" ]]; then mkdir -p "$SNAPSHOT_DIR/proxy/$RUN_LABEL" - rsync -a --exclude '*.tmp' "$PROXY_DIR/" \ + sudo rsync -a --exclude '*.tmp' "$PROXY_DIR/" \ "$SNAPSHOT_DIR/proxy/$RUN_LABEL/" fi +sudo chown -R "$(id -u):$(id -g)" "$SNAPSHOT_DIR" if [[ -d "$ROOT/run-logs" ]]; then mkdir -p "$SNAPSHOT_DIR/run-logs" for suffix in stdout stderr; do diff --git a/scripts/native_eval/remote_run.sh b/scripts/native_eval/remote_run.sh index 185fe5f..0f33021 100755 --- a/scripts/native_eval/remote_run.sh +++ b/scripts/native_eval/remote_run.sh @@ -161,6 +161,7 @@ for suffix in stdout stderr; do fi done ARCHIVE_ITEMS+=(-C /tmp "shellbench_meta-$RUN_LABEL") -tar -czf "/tmp/$RUN_LABEL-final-artifacts.tar.gz" "${ARCHIVE_ITEMS[@]}" +sudo tar -czf "/tmp/$RUN_LABEL-final-artifacts.tar.gz" "${ARCHIVE_ITEMS[@]}" +sudo chown "$(id -u):$(id -g)" "/tmp/$RUN_LABEL-final-artifacts.tar.gz" exit "$RUN_STATUS" diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 64f2e79..c530304 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -247,6 +247,10 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: if harness.name == "openclaw": assert "ended with stopReason=" in command.run_command assert "--thinking off" in command.run_command + assert "item.chmod(0o755 if item.is_dir() else 0o644)" in ( + command.cleanup_command + ) + assert "destination.chmod(0o644)" in command.cleanup_command def test_hermes_uses_named_local_proxy_provider() -> None: From e6c523643bacaa099a20abb2d7e190e70e9bcf2b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 08:36:46 +0200 Subject: [PATCH 05/28] fix(eval): ignore pre-agent infra in trace eligibility --- CHANGELOG.md | 2 ++ scripts/native_eval/aggregate.py | 11 ++++++---- tests/test_native_eval_aggregate.py | 32 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c002df..66c2eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ - Export OpenClaw and Hermes sessions reliably and convert their native traces to Harbor-compatible ATIF trajectories with canonical model identity. - Preserve restricted harness trace files in checkpoint and final archives. +- Keep pre-agent infrastructure failures from invalidating otherwise verified + native model identity and trajectory coverage. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). ### Added diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index 1cdf606..2589772 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -767,20 +767,23 @@ def _summarize_run( manifest.get("runner") == "shellbench-native" or any(row.get("source") == "shellbench-native" for row in rows) ) + agent_attempt_rows = [ + row + for row in scored_rows + if row.get("_valid_result") and row.get("classification") != "infra" + ] canonical_model_identity = not native_run or ( manifest.get("canonical_model_identity") is True and all( row.get("canonical_model_identity") is True - for row in scored_rows - if row.get("_valid_result") + for row in agent_attempt_rows ) ) trajectory_complete = not native_run or ( manifest.get("trajectory_mode") == "real_harness_events" and all( row.get("trajectory_status") == "real" - for row in scored_rows - if row.get("_valid_result") + for row in agent_attempt_rows ) ) parity_validated = not native_run or manifest.get("parity_validated") is True diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index 2623fb7..8a2a354 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -292,6 +292,38 @@ def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path): assert report["runs"][0]["exclusion_reason"] == "parity_not_validated" +def test_native_identity_checks_ignore_pre_agent_infra_failures(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "hermes-gpt56-terra", + expected_task_count=3, + results=[ + _native_result("a"), + _native_result("b", reward=0.0), + _result( + "environment-timeout", + source="shellbench-native", + exception_type="EnvironmentStartTimeoutError", + ), + ], + harness="hermes", + model_slug="gpt56-terra", + native=True, + parity_validated=True, + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["infra"] == 1 + assert run["infra_dominated"] is False + assert run["canonical_model_identity"] is True + assert run["trajectory_complete"] is True + assert run["eligible"] is True + + def test_manifest_can_explicitly_exclude_complete_run(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" From 3949474e02c4524de169f32d6569bf4e6ee44ce3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 09:29:11 +0200 Subject: [PATCH 06/28] fix(eval): preserve Codex JSONL trajectories --- CHANGELOG.md | 2 + scripts/native_eval/harnesses.py | 6 ++- scripts/native_eval/runtime.py | 29 +++++++++++-- tests/test_native_eval_runner.py | 72 +++++++++++++++++++++++++++++++- 4 files changed, 102 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c2eaa..ae5b113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ - Preserve restricted harness trace files in checkpoint and final archives. - Keep pre-agent infrastructure failures from invalidating otherwise verified native model identity and trajectory coverage. +- Keep Codex stderr diagnostics out of JSONL traces and recover known + diagnostics from previously combined streams. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). ### Added diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 47c017c..dd21147 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -290,8 +290,10 @@ def _codex( f"--model {shlex.quote(run.model_id)} " "--json --enable unified_exec -- " "\"$(cat /tmp/shellbench-instruction.md)\" " - ">/logs/agent/codex.txt 2>&1 /logs/agent/codex.txt 2>/logs/agent/codex-stderr.txt " + "&2; exit \"$status\"" ) cleanup = ( "rm -rf /logs/agent/sessions; " diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index f6b9e39..4ac55e0 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -3,6 +3,7 @@ import asyncio import json import os +import re import traceback import uuid from dataclasses import dataclass @@ -48,6 +49,12 @@ class RewardFileNotFoundError(NativeEvalError): pass +CODEX_DIAGNOSTIC_LINE = re.compile( + r"^\d{4}-\d{2}-\d{2}T\S+\s+" + r"(?:TRACE|DEBUG|INFO|WARN|ERROR)\s+codex[\w:.-]*:" +) + + @dataclass(frozen=True) class CommandResult: returncode: int @@ -745,9 +752,12 @@ def _load_json_events(path: Path) -> list[dict[str, Any]]: return events -def _load_codex_stream(path: Path) -> tuple[list[dict[str, Any]], int, int]: +def _load_codex_stream( + path: Path, +) -> tuple[list[dict[str, Any]], int, int, int]: events: list[dict[str, Any]] = [] preamble_lines = 0 + diagnostic_lines = 0 malformed_event_lines = 0 stream_started = False for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): @@ -756,6 +766,9 @@ def _load_codex_stream(path: Path) -> tuple[list[dict[str, Any]], int, int]: try: item = json.loads(line) except json.JSONDecodeError: + if CODEX_DIAGNOSTIC_LINE.match(line): + diagnostic_lines += 1 + continue if stream_started: malformed_event_lines += 1 else: @@ -766,7 +779,7 @@ def _load_codex_stream(path: Path) -> tuple[list[dict[str, Any]], int, int]: events.append(item) else: malformed_event_lines += 1 - return events, preamble_lines, malformed_event_lines + return events, preamble_lines, diagnostic_lines, malformed_event_lines def _observed_codex_models(agent_dir: Path) -> set[str]: @@ -879,9 +892,14 @@ def write_agent_trajectory( source_path = agent_dir / "codex.txt" if source_path.is_file(): - events, preamble_lines, malformed_event_lines = _load_codex_stream(source_path) + ( + events, + preamble_lines, + diagnostic_lines, + malformed_event_lines, + ) = _load_codex_stream(source_path) else: - events, preamble_lines, malformed_event_lines = [], 0, 0 + events, preamble_lines, diagnostic_lines, malformed_event_lines = [], 0, 0, 0 observed_models = _observed_codex_models(agent_dir) runtime_model_name = ( next(iter(observed_models)) if len(observed_models) == 1 else None @@ -921,6 +939,7 @@ def write_agent_trajectory( "trajectory_validation": { "terminal_event_seen": terminal_event_seen, "preamble_lines": preamble_lines, + "diagnostic_lines": diagnostic_lines, "malformed_event_lines": malformed_event_lines, "observed_models": sorted(observed_models), }, @@ -953,6 +972,7 @@ def write_agent_trajectory( "native_raw_trace_file": source_path.name, "native_raw_event_count": len(events), "native_raw_preamble_lines": preamble_lines, + "native_raw_diagnostic_lines": diagnostic_lines, "native_raw_malformed_event_lines": malformed_event_lines, "observed_models": sorted(observed_models), "terminal_event_seen": terminal_event_seen, @@ -968,6 +988,7 @@ def write_agent_trajectory( "trajectory_validation": { "terminal_event_seen": terminal_event_seen, "preamble_lines": preamble_lines, + "diagnostic_lines": diagnostic_lines, "malformed_event_lines": malformed_event_lines, "observed_models": sorted(observed_models), }, diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index c530304..f693c30 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -251,6 +251,9 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: command.cleanup_command ) assert "destination.chmod(0o644)" in command.cleanup_command + if harness.name == "codex": + assert "2>/logs/agent/codex-stderr.txt" in command.run_command + assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command def test_hermes_uses_named_local_proxy_provider() -> None: @@ -389,7 +392,17 @@ def test_codex_trajectory_uses_real_stream_events(tmp_path: Path) -> None: }, ] (agent_dir / "codex.txt").write_text( - "\n".join(json.dumps(event) for event in events) + "\n", + "\n".join( + [ + json.dumps(events[0]), + ( + "2026-07-28T07:10:46.280064Z ERROR " + "codex_core::tools::router: apply_patch verification failed" + ), + *(json.dumps(event) for event in events[1:]), + ] + ) + + "\n", encoding="utf-8", ) session_dir = agent_dir / "sessions" @@ -443,6 +456,8 @@ def test_codex_trajectory_uses_real_stream_events(tmp_path: Path) -> None: assert metadata["trajectory_status"] == "real" assert metadata["runtime_model_name"] == "gpt-5.5" assert metadata["canonical_model_identity"] is True + assert metadata["trajectory_validation"]["diagnostic_lines"] == 1 + assert metadata["trajectory_validation"]["malformed_event_lines"] == 0 assert trajectory["session_id"] == "thread-123" assert len(trajectory["steps"]) == 6 assert trajectory["steps"][2]["tool_calls"][0]["function_name"] == "shell" @@ -526,6 +541,61 @@ def test_codex_trajectory_rejects_truncated_or_mismatched_stream(tmp_path: Path) assert metadata["trajectory_validation"]["terminal_event_seen"] is False +def test_codex_trajectory_rejects_unknown_interleaved_output(tmp_path: Path) -> None: + agent_dir = tmp_path / "agent" + session_dir = agent_dir / "sessions" + session_dir.mkdir(parents=True) + events = [ + {"type": "thread.started", "thread_id": "thread-123"}, + "not a recognized codex diagnostic", + { + "type": "item.completed", + "item": {"id": "message-1", "type": "agent_message", "text": "done"}, + }, + {"type": "turn.completed", "usage": {"input_tokens": 1}}, + ] + (agent_dir / "codex.txt").write_text( + "\n".join( + event if isinstance(event, str) else json.dumps(event) + for event in events + ) + + "\n", + encoding="utf-8", + ) + (session_dir / "rollout.jsonl").write_text( + json.dumps( + { + "type": "turn_context", + "payload": {"model": "gpt-5.5"}, + } + ) + + "\n", + encoding="utf-8", + ) + run = RunSpec( + run_label="codex-gpt55-calibration", + harness="codex", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=1, + run_date="20260727", + ) + + metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + + assert metadata["trajectory_status"] == "unavailable" + assert metadata["trajectory_validation"]["diagnostic_lines"] == 0 + assert metadata["trajectory_validation"]["malformed_event_lines"] == 1 + + def test_unsupported_trajectory_is_explicitly_unranked(tmp_path: Path) -> None: run = RunSpec( run_label="claude-code-gpt55", From 9319c36c6314f64f8b9422ee8550a3e04fb61a5b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 11:17:28 +0200 Subject: [PATCH 07/28] fix(eval): recover active AWS leases --- CHANGELOG.md | 2 ++ scripts/native_eval/fleet.py | 46 +++++++++++++++++++++++++---- tests/test_native_eval_fleet.py | 52 +++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae5b113..9fbf6bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ native model identity and trajectory coverage. - Keep Codex stderr diagnostics out of JSONL traces and recover known diagnostics from previously combined streams. +- Reattach active AWS leases through their recorded SSH endpoint when Crabbox + readiness probing is stale. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). ### Added diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 21aa89b..5969955 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -83,6 +83,10 @@ class LeaseUnavailableError(FleetError): pass +class LeaseNotReadyError(FleetError): + pass + + @dataclass(frozen=True) class Lease: lease_id: str @@ -107,7 +111,7 @@ def from_inspect(cls, value: dict[str, Any], *, region: str) -> Lease: if value.get("state") != "active": raise LeaseUnavailableError(f"lease {value['id']} is no longer active") if not value.get("ready"): - raise FleetError(f"lease {value['id']} is active but not ready") + raise LeaseNotReadyError(f"lease {value['id']} is active but not ready") return cls( lease_id=str(value["id"]), slug=str(value["slug"]), @@ -119,6 +123,23 @@ def from_inspect(cls, value: dict[str, Any], *, region: str) -> Lease: region=region, ) + @classmethod + def from_manifest(cls, value: dict[str, Any]) -> Lease: + required = ("id", "slug", "host", "ssh_user", "ssh_port", "identity_file") + missing = [field for field in required if not value.get(field)] + if missing: + raise FleetError(f"stored lease omitted fields: {', '.join(missing)}") + return cls( + lease_id=str(value["id"]), + slug=str(value["slug"]), + host=str(value["host"]), + user=str(value["ssh_user"]), + port=int(value["ssh_port"]), + identity_file=Path(str(value["identity_file"])), + instance_type=str(value.get("instance_type") or ""), + region=str(value.get("region") or ""), + ) + def to_dict(self) -> dict[str, Any]: return { "id": self.lease_id, @@ -519,11 +540,17 @@ def _ensure_lease(self, entry: dict[str, Any]) -> Lease: lease_value = entry.get("lease") if lease_value: identifier = str(lease_value["id"]) - return self._inspect_lease( - identifier, - required=True, - region_hint=str(lease_value.get("region") or ""), - ) + try: + return self._inspect_lease( + identifier, + required=True, + region_hint=str(lease_value.get("region") or ""), + ) + except LeaseNotReadyError: + stored = Lease.from_manifest(lease_value) + if self._ssh_reachable(stored): + return self._detect_region(stored) + raise slug = str(entry.get("requested_lease_slug") or self._lease_slug(entry["run_label"])) self._store.update( @@ -621,6 +648,13 @@ def _inspect_lease( ) return self._detect_region(lease) + def _ssh_reachable(self, lease: Lease) -> bool: + result = self.executor.run( + self._ssh_command(lease, ["true"]), + capture_output=True, + ) + return result.returncode == 0 + def _detect_region(self, lease: Lease) -> Lease: script = """ set -eu diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index c5ea06d..48a719f 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -457,6 +457,58 @@ def test_model_cap_counts_adopted_run_before_dispatching_same_model( assert adopted_stop < pending_fable_dispatch +def test_recovery_uses_recorded_ssh_endpoint_when_crabbox_ready_probe_is_stale( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run = _planned(_run_spec(label)) + run["status"] = "recovery_required" + run["lease"] = { + "id": "cbx_existing", + "slug": "existing", + "provider": "aws", + "instance_type": "c7a.24xlarge", + "region": "eu-west-1", + "host": "192.0.2.50", + "ssh_user": "crabbox", + "ssh_port": 22, + "identity_file": "/tmp/cbx_existing.key", + "state": "active", + } + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + config = _config(tmp_path, run_index) + executor = FakeExecutor( + config.local_root, + expected_counts={label: 2}, + running_labels={label}, + ) + executor.leases["cbx_existing"] = { + "id": "cbx_existing", + "slug": "existing", + "state": "active", + "ready": False, + "serverType": "c7a.24xlarge", + "sshHost": "192.0.2.50", + "sshUser": "crabbox", + "sshPort": "2222", + "sshKey": "/tmp/cbx_existing.key", + } + executor.active_leases = 1 + + assert FleetController(config, executor=executor).run() == 0 + + recovered = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] + assert recovered["status"] == "completed" + assert executor.dispatches == [] + reachability_probe = next( + command + for command in executor.commands + if command and command[0] == "ssh" and command[-1] == "true" + ) + assert "22" in reachability_probe + + def test_model_task_concurrency_override_is_used_at_dispatch(tmp_path: Path) -> None: fable_label = "openclaw-fable5-full-2-r1-20260727" gpt_label = "openclaw-gpt55-full-2-r1-20260727" From 5c86a89544540178235c404fc84b863bf3044b00 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 12:59:16 +0200 Subject: [PATCH 08/28] fix(eval): parse Hermes JSONL by LF records --- CHANGELOG.md | 1 + scripts/native_eval/harness_trajectories.py | 2 +- tests/test_native_eval_runner.py | 50 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fbf6bd..198bbdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ diagnostics from previously combined streams. - Reattach active AWS leases through their recorded SSH endpoint when Crabbox readiness probing is stale. +- Preserve Hermes JSONL sessions containing literal Unicode line separators. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). ### Added diff --git a/scripts/native_eval/harness_trajectories.py b/scripts/native_eval/harness_trajectories.py index d247add..1159717 100644 --- a/scripts/native_eval/harness_trajectories.py +++ b/scripts/native_eval/harness_trajectories.py @@ -517,7 +517,7 @@ def _load_hermes_sessions(path: Path) -> list[dict[str, Any]]: if not path.is_file(): return [] sessions: list[dict[str, Any]] = [] - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + for line in path.read_text(encoding="utf-8", errors="replace").split("\n"): try: value = json.loads(line) except json.JSONDecodeError: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index f693c30..f2e8241 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -871,6 +871,56 @@ def test_hermes_session_converts_parallel_tools_to_atif(tmp_path: Path) -> None: assert trajectory["final_metrics"]["total_completion_tokens"] == 25 +def test_hermes_session_preserves_unicode_line_separator(tmp_path: Path) -> None: + agent_dir = tmp_path / "agent" + agent_dir.mkdir() + session = { + "id": "hermes-session-unicode", + "model": "gpt-5.5", + "message_count": 2, + "tool_call_count": 0, + "messages": [ + { + "id": 1, + "role": "user", + "content": "do the task", + "timestamp": 1.0, + }, + { + "id": 2, + "role": "assistant", + "content": "line one\u2028line two", + "timestamp": 2.0, + "finish_reason": "stop", + }, + ], + } + (agent_dir / "hermes-session.jsonl").write_text( + json.dumps(session, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + run = RunSpec( + run_label="hermes-gpt55", + harness="hermes", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=1, + run_date="20260728", + ) + task = _trajectory_task(tmp_path, "do the task") + + metadata = write_agent_trajectory(task, run, agent_dir) + trajectory = json.loads((agent_dir / "trajectory.json").read_text()) + + assert metadata["trajectory_status"] == "real" + assert metadata["canonical_model_identity"] is True + assert trajectory["steps"][-1]["message"] == "line one\u2028line two" + + def _trajectory_task(tmp_path: Path, instruction: str) -> TaskSpec: task_dir = tmp_path / "task" task_dir.mkdir(exist_ok=True) From 5412856f12b57dcace741633baeb8112017f7dbe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 13:10:23 +0200 Subject: [PATCH 09/28] feat(eval): support xhigh reasoning effort --- scripts/native_eval/fleet.py | 6 +++--- scripts/native_eval/plan.py | 4 ++-- scripts/native_eval/proxy.py | 4 ++-- tests/test_native_eval_fleet.py | 17 +++++++++++++++++ tests/test_native_eval_proxy.py | 15 +++++++++++++++ 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 5969955..6d5b763 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -47,7 +47,7 @@ RERUN_STATUSES = {"failed", "lease_lost"} ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} -REASONING_EFFORTS = {"low", "medium", "high"} +REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} class CommandExecutor(Protocol): @@ -352,7 +352,7 @@ def _validate_plan(self) -> None: if reasoning_effort not in REASONING_EFFORTS: raise FleetError( f"{run.run_label} must set reasoning_effort to " - "low, medium, or high" + "low, medium, high, or xhigh" ) judge_reasoning_effort = str( entry.get("judge_reasoning_effort") or "" @@ -360,7 +360,7 @@ def _validate_plan(self) -> None: if judge_reasoning_effort not in REASONING_EFFORTS: raise FleetError( f"{run.run_label} must set judge_reasoning_effort to " - "low, medium, or high" + "low, medium, high, or xhigh" ) def _prepare_inputs(self) -> None: diff --git a/scripts/native_eval/plan.py b/scripts/native_eval/plan.py index d5544f2..06c7327 100644 --- a/scripts/native_eval/plan.py +++ b/scripts/native_eval/plan.py @@ -55,12 +55,12 @@ def main() -> None: parser.add_argument("--run-date", required=True) parser.add_argument( "--reasoning-effort", - choices=("low", "medium", "high"), + choices=("low", "medium", "high", "xhigh"), required=True, ) parser.add_argument( "--judge-reasoning-effort", - choices=("low", "medium", "high"), + choices=("low", "medium", "high", "xhigh"), ) args = parser.parse_args() entries = write_run_index( diff --git a/scripts/native_eval/proxy.py b/scripts/native_eval/proxy.py index 3532d11..bb3607b 100644 --- a/scripts/native_eval/proxy.py +++ b/scripts/native_eval/proxy.py @@ -9,9 +9,9 @@ def write_proxy_config(path: Path) -> None: reasoning_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip() - if reasoning_effort not in {"low", "medium", "high"}: + if reasoning_effort not in {"low", "medium", "high", "xhigh"}: raise ValueError( - "SHELLBENCH_REASONING_EFFORT must be low, medium, or high" + "SHELLBENCH_REASONING_EFFORT must be low, medium, high, or xhigh" ) model_list = [] for model in MODELS: diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 48a719f..b2c01c7 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -361,6 +361,23 @@ def test_controller_runs_bounded_wave_and_stops_after_verified_export( assert "TOPSECRET" not in "\n".join(" ".join(command) for command in executor.commands) +def test_controller_accepts_xhigh_reasoning_effort(tmp_path: Path) -> None: + label = "openclaw-gpt56-sol-xhigh-full-2-r1-20260728" + run = _planned(_run_spec(label, model_slug="gpt56-sol")) + run["reasoning_effort"] = "xhigh" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + + completed = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] + assert completed["status"] == "completed" + assert completed["reasoning_effort"] == "xhigh" + assert completed["judge_reasoning_effort"] == "high" + + def test_capacity_warmup_retries_same_run_without_recovery_churn( tmp_path: Path, ) -> None: diff --git a/tests/test_native_eval_proxy.py b/tests/test_native_eval_proxy.py index 8eb6f85..36ec843 100644 --- a/tests/test_native_eval_proxy.py +++ b/tests/test_native_eval_proxy.py @@ -26,3 +26,18 @@ def test_openai_proxy_models_enforce_reasoning_effort( assert models["gpt-5.6-luna"]["litellm_params"]["reasoning_effort"] == "high" assert models["gpt-5.6-terra"]["litellm_params"]["reasoning_effort"] == "high" assert config["shellbench_native"]["reasoning_effort"] == "high" + + +def test_openai_proxy_accepts_xhigh_reasoning_effort( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "xhigh") + config_path = tmp_path / "proxy.json" + + write_proxy_config(config_path) + + config = json.loads(config_path.read_text()) + models = {item["model_name"]: item for item in config["model_list"]} + assert models["gpt-5.6-sol"]["litellm_params"]["reasoning_effort"] == "xhigh" + assert config["shellbench_native"]["reasoning_effort"] == "xhigh" From 0070f7057e9edf87d8223538a1f27db618ca71ca Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 13:12:12 +0200 Subject: [PATCH 10/28] fix(eval): isolate judge reasoning effort --- scripts/native_eval/proxy.py | 44 +++++++++++++++++++++++++++++++- scripts/native_eval/runtime.py | 27 +++++++++++--------- tests/test_native_eval_proxy.py | 24 ++++++++++++++++- tests/test_native_eval_runner.py | 18 ++++++++++--- 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/scripts/native_eval/proxy.py b/scripts/native_eval/proxy.py index bb3607b..fe14142 100644 --- a/scripts/native_eval/proxy.py +++ b/scripts/native_eval/proxy.py @@ -7,12 +7,32 @@ from scripts.native_eval.models import LITELLM_VERSION, MODELS +JUDGE_PROXY_MODEL_NAME = "shellbench-judge" +REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} + + def write_proxy_config(path: Path) -> None: reasoning_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip() - if reasoning_effort not in {"low", "medium", "high", "xhigh"}: + if reasoning_effort not in REASONING_EFFORTS: raise ValueError( "SHELLBENCH_REASONING_EFFORT must be low, medium, high, or xhigh" ) + judge_model_id = os.environ.get("SHELLBENCH_JUDGE_MODEL_ID", "gpt-5.5").strip() + judge_reasoning_effort = os.environ.get( + "SHELLBENCH_JUDGE_REASONING_EFFORT", + reasoning_effort, + ).strip() + if judge_reasoning_effort not in REASONING_EFFORTS: + raise ValueError( + "SHELLBENCH_JUDGE_REASONING_EFFORT must be low, medium, high, or xhigh" + ) + judge_model = next( + (model for model in MODELS if model.provider_model_id == judge_model_id), + None, + ) + if judge_model is None: + raise ValueError(f"unknown SHELLBENCH_JUDGE_MODEL_ID: {judge_model_id}") + model_list = [] for model in MODELS: litellm_params = { @@ -33,6 +53,25 @@ def write_proxy_config(path: Path) -> None: }, } ) + judge_litellm_params = { + "model": f"{judge_model.provider}/{judge_model.provider_model_id}", + "api_key": f"os.environ/{judge_model.provider.upper()}_API_KEY", + } + if judge_model.provider == "openai": + judge_litellm_params["additional_drop_params"] = ["temperature"] + judge_litellm_params["reasoning_effort"] = judge_reasoning_effort + model_list.append( + { + "model_name": JUDGE_PROXY_MODEL_NAME, + "litellm_params": judge_litellm_params, + "model_info": { + "friendly_name": judge_model.friendly_name, + "provider": judge_model.provider, + "provider_model_id": judge_model.provider_model_id, + "role": "judge", + }, + } + ) config = { "model_list": model_list, "general_settings": { @@ -52,6 +91,9 @@ def write_proxy_config(path: Path) -> None: "shellbench_native": { "litellm_version": LITELLM_VERSION, "reasoning_effort": reasoning_effort, + "judge_model_id": judge_model_id, + "judge_proxy_model_name": JUDGE_PROXY_MODEL_NAME, + "judge_reasoning_effort": judge_reasoning_effort, "models": [ { "slug": model.slug, diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index 4ac55e0..153369c 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -18,6 +18,7 @@ ) from scripts.native_eval.harnesses import TOOLCHAIN_ROOT, build_harness_command from scripts.native_eval.models import RunSpec +from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME from scripts.native_eval.tasks import TaskSpec @@ -55,6 +56,19 @@ class RewardFileNotFoundError(NativeEvalError): ) +def build_judge_env(proxy_url: str, proxy_key: str) -> dict[str, str]: + return { + "AGENT_JUDGE_API_URL": f"{proxy_url.rstrip('/')}/v1/chat/completions", + "AGENT_JUDGE_MODEL": JUDGE_PROXY_MODEL_NAME, + "AGENT_JUDGE_API_KEY": proxy_key, + "LLM_JUDGE_API_URL": f"{proxy_url.rstrip('/')}/v1", + "LLM_JUDGE_MODEL": JUDGE_PROXY_MODEL_NAME, + "LLM_JUDGE_API_KEY": proxy_key, + "OPENAI_BASE_URL": f"{proxy_url.rstrip('/')}/v1", + "OPENROUTER_API_KEY": proxy_key, + } + + @dataclass(frozen=True) class CommandResult: returncode: int @@ -535,18 +549,7 @@ async def run_trial( await environment.install_tests() verifier_started = utc_now() verifier_env = task.resolved_verifier_env() - judge_env = { - "AGENT_JUDGE_API_URL": ( - f"{proxy_url.rstrip('/')}/v1/chat/completions" - ), - "AGENT_JUDGE_MODEL": "gpt-5.5", - "AGENT_JUDGE_API_KEY": proxy_key, - "LLM_JUDGE_API_URL": f"{proxy_url.rstrip('/')}/v1", - "LLM_JUDGE_MODEL": "gpt-5.5", - "LLM_JUDGE_API_KEY": proxy_key, - "OPENAI_BASE_URL": f"{proxy_url.rstrip('/')}/v1", - "OPENROUTER_API_KEY": proxy_key, - } + judge_env = build_judge_env(proxy_url, proxy_key) for key, value in judge_env.items(): if not verifier_env.get(key): verifier_env[key] = value diff --git a/tests/test_native_eval_proxy.py b/tests/test_native_eval_proxy.py index 36ec843..01cc785 100644 --- a/tests/test_native_eval_proxy.py +++ b/tests/test_native_eval_proxy.py @@ -1,7 +1,7 @@ import json from pathlib import Path -from scripts.native_eval.proxy import write_proxy_config +from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME, write_proxy_config def test_openai_proxy_models_enforce_reasoning_effort( @@ -41,3 +41,25 @@ def test_openai_proxy_accepts_xhigh_reasoning_effort( models = {item["model_name"]: item for item in config["model_list"]} assert models["gpt-5.6-sol"]["litellm_params"]["reasoning_effort"] == "xhigh" assert config["shellbench_native"]["reasoning_effort"] == "xhigh" + + +def test_proxy_keeps_judge_reasoning_independent( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "low") + monkeypatch.setenv("SHELLBENCH_JUDGE_MODEL_ID", "gpt-5.5") + monkeypatch.setenv("SHELLBENCH_JUDGE_REASONING_EFFORT", "high") + config_path = tmp_path / "proxy.json" + + write_proxy_config(config_path) + + config = json.loads(config_path.read_text()) + models = {item["model_name"]: item for item in config["model_list"]} + assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == "low" + assert ( + models[JUDGE_PROXY_MODEL_NAME]["litellm_params"]["reasoning_effort"] + == "high" + ) + assert config["shellbench_native"]["judge_model_id"] == "gpt-5.5" + assert config["shellbench_native"]["judge_reasoning_effort"] == "high" diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index f2e8241..260de2f 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -18,10 +18,11 @@ build_matrix_plan, ) from scripts.native_eval import plan as native_plan -from scripts.native_eval.proxy import write_proxy_config +from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME, write_proxy_config from scripts.native_eval.run_job import _git_commit, _run_manifest, build_run_spec from scripts.native_eval.runtime import ( DockerTaskEnvironment, + build_judge_env, collect_agent_metrics, read_reward, write_agent_trajectory, @@ -135,12 +136,13 @@ def test_proxy_config_pins_only_requested_upstreams( write_proxy_config(path) config = json.loads(path.read_text(encoding="utf-8")) - assert [item["model_name"] for item in config["model_list"]] == [ + assert [item["model_name"] for item in config["model_list"][:-1]] == [ model.provider_model_id for model in MODELS ] - assert [item["litellm_params"]["model"] for item in config["model_list"]] == [ + assert [item["litellm_params"]["model"] for item in config["model_list"][:-1]] == [ f"{model.provider}/{model.provider_model_id}" for model in MODELS ] + assert config["model_list"][-1]["model_name"] == JUDGE_PROXY_MODEL_NAME serialized = path.read_text(encoding="utf-8") assert "OPENAI_API_KEY" in serialized assert "ANTHROPIC_API_KEY" in serialized @@ -152,6 +154,16 @@ def test_proxy_config_pins_only_requested_upstreams( ) +def test_judge_env_uses_dedicated_proxy_alias() -> None: + judge_env = build_judge_env("http://proxy:4000/", "proxy-key") + + assert judge_env["AGENT_JUDGE_MODEL"] == JUDGE_PROXY_MODEL_NAME + assert judge_env["LLM_JUDGE_MODEL"] == JUDGE_PROXY_MODEL_NAME + assert judge_env["AGENT_JUDGE_API_URL"] == ( + "http://proxy:4000/v1/chat/completions" + ) + + def test_run_manifest_records_native_audit_metadata( tmp_path: Path, monkeypatch, From ccff4cb5e241ee1cd1b1dd2fcbfea7f275933cbe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 13:25:34 +0200 Subject: [PATCH 11/28] fix(eval): skip OpenClaw onboarding bootstrap --- scripts/native_eval/harnesses.py | 10 ++++++++-- tests/test_native_eval_runner.py | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index dd21147..00be9eb 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -68,7 +68,12 @@ def _openclaw( "transport": server.transport, } config = { - "agents": {"defaults": {"workspace": "."}}, + "agents": { + "defaults": { + "workspace": ".", + "skipBootstrap": True, + } + }, "gateway": {"mode": "local"}, "models": { "providers": { @@ -94,7 +99,8 @@ def _openclaw( setup = ( f"export PATH={_base_path()}; export HOME={home}; " "rm -rf \"$HOME\"; mkdir -p \"$HOME/.openclaw\"; " - "openclaw setup --baseline --workspace . >/logs/agent/setup.log 2>&1; " + "openclaw setup --baseline --skip-bootstrap --workspace . " + ">/logs/agent/setup.log 2>&1; " f"printf %s {config_json} > \"$HOME/.openclaw/openclaw.json\"" ) run_command = ( diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 260de2f..114a716 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -259,6 +259,8 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: if harness.name == "openclaw": assert "ended with stopReason=" in command.run_command assert "--thinking off" in command.run_command + assert "setup --baseline --skip-bootstrap" in command.setup_command + assert '"skipBootstrap":true' in command.setup_command assert "item.chmod(0o755 if item.is_dir() else 0o644)" in ( command.cleanup_command ) From b6732a6d00ce302c229305a47c1fa4302a480612 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 13:33:58 +0200 Subject: [PATCH 12/28] fix(eval): remove OpenClaw bootstrap file --- scripts/native_eval/harnesses.py | 1 + tests/test_native_eval_runner.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 00be9eb..476561a 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -101,6 +101,7 @@ def _openclaw( "rm -rf \"$HOME\"; mkdir -p \"$HOME/.openclaw\"; " "openclaw setup --baseline --skip-bootstrap --workspace . " ">/logs/agent/setup.log 2>&1; " + "rm -f BOOTSTRAP.md; " f"printf %s {config_json} > \"$HOME/.openclaw/openclaw.json\"" ) run_command = ( diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 114a716..ab10250 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -260,6 +260,7 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "ended with stopReason=" in command.run_command assert "--thinking off" in command.run_command assert "setup --baseline --skip-bootstrap" in command.setup_command + assert "rm -f BOOTSTRAP.md" in command.setup_command assert '"skipBootstrap":true' in command.setup_command assert "item.chmod(0o755 if item.is_dir() else 0o644)" in ( command.cleanup_command From 86eff1cbb1e6ff7ba57613ad74e108b32602e7fd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 14:48:21 +0200 Subject: [PATCH 13/28] fix(eval): detect markerless OpenClaw completion --- scripts/native_eval/harnesses.py | 34 ++++++++++++++++++++++- tests/test_native_eval_runner.py | 46 +++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 476561a..2e4d45f 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -14,6 +14,36 @@ NPM_BIN = TOOLCHAIN_ROOT / "npm-packages" / "node_modules" / ".bin" HERMES_BIN = TOOLCHAIN_ROOT / "home" / ".local" / "bin" +_OPENCLAW_COMPLETION_PROBE = """\ +import json +import pathlib +import sys + +try: + text = pathlib.Path(sys.argv[1]).read_text( + encoding="utf-8", + errors="replace", + ).strip() +except OSError: + sys.exit(1) + +decoder = json.JSONDecoder() +for start in range(len(text) - 1, -1, -1): + if text[start] != "{": + continue + try: + value, _ = decoder.raw_decode(text[start:]) + except (json.JSONDecodeError, ValueError): + continue + if ( + isinstance(value, dict) + and isinstance(value.get("payloads"), list) + and isinstance(value.get("meta"), dict) + ): + sys.exit(0) +sys.exit(1) +""" + @dataclass(frozen=True) class HarnessCommand: @@ -95,6 +125,7 @@ def _openclaw( if servers: config["mcp"] = {"servers": servers} config_json = shlex.quote(json.dumps(config, separators=(",", ":"))) + completion_probe = shlex.quote(_OPENCLAW_COMPLETION_PROBE) home = "/tmp/shellbench-openclaw" setup = ( f"export PATH={_base_path()}; export HOME={home}; " @@ -113,7 +144,8 @@ def _openclaw( ">\"$log\" 2>&1 /dev/null; do " "if grep -Eq \"\\[agents/agent-command\\] \\[agent\\] run .* " - "ended with stopReason=\" \"$log\"; then " + f"ended with stopReason=\" \"$log\" || python3 -c {completion_probe} " + "\"$log\"; then " "sleep 2; kill \"$pid\" 2>/dev/null || true; sleep 1; " "kill -KILL \"$pid\" 2>/dev/null || true; " "for f in /proc/[0-9]*/comm; do " diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index ab10250..2ef67e5 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -2,6 +2,8 @@ import asyncio import json +import subprocess +import sys import tarfile from argparse import Namespace from pathlib import Path @@ -10,7 +12,10 @@ count_result_json, next_checkpoint_sequence, ) -from scripts.native_eval.harnesses import build_harness_command +from scripts.native_eval.harnesses import ( + _OPENCLAW_COMPLETION_PROBE, + build_harness_command, +) from scripts.native_eval.models import ( HARNESSES, MODELS, @@ -258,6 +263,7 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert 'exit "$status"' in command.run_command if harness.name == "openclaw": assert "ended with stopReason=" in command.run_command + assert "python3 -c" in command.run_command assert "--thinking off" in command.run_command assert "setup --baseline --skip-bootstrap" in command.setup_command assert "rm -f BOOTSTRAP.md" in command.setup_command @@ -271,6 +277,44 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command +def test_openclaw_completion_probe_accepts_markerless_final_envelope( + tmp_path: Path, +) -> None: + log_path = tmp_path / "openclaw.txt" + log_path.write_text( + "debug preamble\n" + + json.dumps( + { + "payloads": [{"text": "done"}], + "meta": {"aborted": False}, + } + ), + encoding="utf-8", + ) + + completed = subprocess.run( + [sys.executable, "-c", _OPENCLAW_COMPLETION_PROBE, str(log_path)], + check=False, + ) + + assert completed.returncode == 0 + + +def test_openclaw_completion_probe_rejects_partial_log(tmp_path: Path) -> None: + log_path = tmp_path / "openclaw.txt" + log_path.write_text( + 'debug preamble\n{"payloads":[{"text":"still writing"}]', + encoding="utf-8", + ) + + completed = subprocess.run( + [sys.executable, "-c", _OPENCLAW_COMPLETION_PROBE, str(log_path)], + check=False, + ) + + assert completed.returncode == 1 + + def test_hermes_uses_named_local_proxy_provider() -> None: run = RunSpec( run_label="hermes-test", From 102841297addb91e57841b3f6f16669cbbd8bf26 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 16:39:54 +0200 Subject: [PATCH 14/28] fix(eval): keep agent exits leaderboard eligible --- scripts/native_eval/aggregate.py | 19 +++++-- tests/test_native_eval_aggregate.py | 81 +++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index 2589772..ebee368 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -772,18 +772,31 @@ def _summarize_run( for row in scored_rows if row.get("_valid_result") and row.get("classification") != "infra" ] + # Forced agent exits can retain raw logs without producing a final + # structured trajectory. They remain scored failures, not run invalidators. + trajectory_required_rows = [ + row + for row in agent_attempt_rows + if row.get("classification") != "agent_exit" + ] + identity_rows = [ + row + for row in agent_attempt_rows + if row.get("classification") != "agent_exit" + or row.get("trajectory_status") == "real" + ] canonical_model_identity = not native_run or ( - manifest.get("canonical_model_identity") is True + bool(identity_rows) and all( row.get("canonical_model_identity") is True - for row in agent_attempt_rows + for row in identity_rows ) ) trajectory_complete = not native_run or ( manifest.get("trajectory_mode") == "real_harness_events" and all( row.get("trajectory_status") == "real" - for row in agent_attempt_rows + for row in trajectory_required_rows ) ) parity_validated = not native_run or manifest.get("parity_validated") is True diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index 8a2a354..e1773ad 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -324,6 +324,87 @@ def test_native_identity_checks_ignore_pre_agent_infra_failures(tmp_path: Path): assert run["eligible"] is True +def test_native_identity_checks_ignore_unavailable_agent_exit_trajectory( + tmp_path: Path, +): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + timed_out = _result( + "agent-timeout", + source="shellbench-native", + exception_type="NonZeroAgentExitCodeError", + exception_message="Agent timed out after 900.0s", + ) + timed_out["agent_result"].update( + { + "trajectory_status": "unavailable", + "runtime_model_name": None, + "canonical_model_identity": False, + } + ) + _write_run( + jobs_root, + "hermes-gpt56-sol-timeout", + expected_task_count=2, + results=[_native_result("a"), timed_out], + harness="hermes", + model_slug="gpt56-sol", + native=True, + parity_validated=True, + ) + manifest_path = ( + jobs_root + / "jobs" + / "hermes-gpt56-sol-timeout" + / "run_manifest.json" + ) + manifest = json.loads(manifest_path.read_text()) + manifest["canonical_model_identity"] = False + manifest_path.write_text(json.dumps(manifest)) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["agent_exits"] == 1 + assert run["canonical_model_identity"] is True + assert run["trajectory_complete"] is True + assert run["eligible"] is True + + +def test_native_identity_checks_real_agent_exit_trajectory(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + wrong_model = _result( + "agent-exit", + source="shellbench-native", + exception_type="NonZeroAgentExitCodeError", + ) + wrong_model["agent_result"].update( + { + "trajectory_status": "real", + "runtime_model_name": "wrong-model", + "canonical_model_identity": False, + } + ) + _write_run( + jobs_root, + "hermes-gpt56-sol-wrong-model", + expected_task_count=2, + results=[_native_result("a"), wrong_model], + harness="hermes", + model_slug="gpt56-sol", + native=True, + parity_validated=True, + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["canonical_model_identity"] is False + assert run["eligible"] is False + assert run["exclusion_reason"] == "canonical_model_identity_not_preserved" + + def test_manifest_can_explicitly_exclude_complete_run(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" From b4414bbdf023f11a6e198c81e979e0aba7113830 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 20:55:20 +0200 Subject: [PATCH 15/28] fix(eval): recover Codex session trajectories --- CHANGELOG.md | 2 + scripts/native_eval/runtime.py | 196 ++++++++++++++++++++++++++++++- tests/test_native_eval_runner.py | 165 ++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 198bbdf..b7d4ab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ native model identity and trajectory coverage. - Keep Codex stderr diagnostics out of JSONL traces and recover known diagnostics from previously combined streams. +- Recover Codex trajectories from complete native session rollouts when its + CLI JSONL stream is transiently incomplete or contains binary tool output. - Reattach active AWS leases through their recorded SSH endpoint when Crabbox readiness probing is stale. - Preserve Hermes JSONL sessions containing literal Unicode line separators. diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index 153369c..21ee8cc 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -4,6 +4,7 @@ import json import os import re +import time import traceback import uuid from dataclasses import dataclass @@ -54,6 +55,8 @@ class RewardFileNotFoundError(NativeEvalError): r"^\d{4}-\d{2}-\d{2}T\S+\s+" r"(?:TRACE|DEBUG|INFO|WARN|ERROR)\s+codex[\w:.-]*:" ) +CODEX_STREAM_READ_ATTEMPTS = 20 +CODEX_STREAM_READ_DELAY_SECONDS = 0.1 def build_judge_env(proxy_url: str, proxy_key: str) -> dict[str, str]: @@ -785,6 +788,22 @@ def _load_codex_stream( return events, preamble_lines, diagnostic_lines, malformed_event_lines +def _load_stable_codex_stream( + path: Path, +) -> tuple[list[dict[str, Any]], int, int, int, int]: + best = _load_codex_stream(path) + attempts = 1 + while best[3] and attempts < CODEX_STREAM_READ_ATTEMPTS: + time.sleep(CODEX_STREAM_READ_DELAY_SECONDS) + candidate = _load_codex_stream(path) + attempts += 1 + if (candidate[3], -len(candidate[0])) < (best[3], -len(best[0])): + best = candidate + if best[3] == 0: + break + return (*best, attempts) + + def _observed_codex_models(agent_dir: Path) -> set[str]: models: set[str] = set() sessions_dir = agent_dir / "sessions" @@ -802,6 +821,143 @@ def _observed_codex_models(agent_dir: Path) -> set[str]: return models +def _codex_session_events(agent_dir: Path) -> tuple[Path | None, list[dict[str, Any]]]: + sessions_dir = agent_dir / "sessions" + if not sessions_dir.is_dir(): + return None, [] + candidates: list[tuple[Path, list[dict[str, Any]]]] = [] + for path in sorted(sessions_dir.rglob("*.jsonl")): + events = _load_json_events(path) + if events: + candidates.append((path, events)) + if not candidates: + return None, [] + return max(candidates, key=lambda item: len(item[1])) + + +def _codex_session_text(value: Any) -> str: + if isinstance(value, str): + return value + if isinstance(value, list): + return "\n".join(filter(None, (_codex_session_text(item) for item in value))) + if not isinstance(value, dict): + return "" + for key in ("text", "output_text", "input_text"): + text = value.get(key) + if isinstance(text, str): + return text + return _codex_session_text(value.get("content")) + + +def _codex_session_arguments(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {"value": value} + if isinstance(parsed, dict): + return parsed + return {"value": parsed} + return {"value": value} + + +def _codex_session_steps( + events: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], str | None, bool]: + steps: list[dict[str, Any]] = [] + session_id: str | None = None + terminal_event_seen = False + pending_calls: dict[str, tuple[str, dict[str, Any]]] = {} + + for event in events: + event_type = str(event.get("type") or "") + payload = event.get("payload") + if not isinstance(payload, dict): + continue + if event_type == "session_meta": + candidate = payload.get("session_id") or payload.get("id") + if candidate: + session_id = str(candidate) + continue + if event_type == "event_msg": + terminal_event_seen = terminal_event_seen or payload.get("type") == "task_complete" + continue + if event_type != "response_item": + continue + + item_type = str(payload.get("type") or "") + if item_type == "message" and payload.get("role") == "assistant": + text = _codex_session_text(payload.get("content")) + if text: + steps.append( + { + "source": "agent", + "message": text, + "llm_call_count": 1, + } + ) + continue + if item_type == "reasoning": + summary = _codex_session_text(payload.get("summary")) + if summary: + steps.append( + { + "source": "agent", + "message": "", + "reasoning_content": summary, + "llm_call_count": 1, + } + ) + continue + if item_type in {"custom_tool_call", "function_call"}: + call_id = str(payload.get("call_id") or payload.get("id") or uuid.uuid4()) + name = str(payload.get("name") or item_type) + arguments = _codex_session_arguments( + payload.get("input", payload.get("arguments")) + ) + pending_calls[call_id] = (name, arguments) + continue + if item_type not in {"custom_tool_call_output", "function_call_output"}: + continue + + call_id = str(payload.get("call_id") or "") + call = pending_calls.pop(call_id, None) + if call is None: + continue + name, arguments = call + output = payload.get("output") + output_text = ( + output + if isinstance(output, str) + else json.dumps(output, ensure_ascii=True, default=str) + ) + steps.append( + { + "source": "agent", + "message": "", + "tool_calls": [ + { + "tool_call_id": call_id, + "function_name": name, + "arguments": arguments, + } + ], + "observation": { + "results": [ + { + "source_call_id": call_id, + "content": output_text, + } + ] + }, + "llm_call_count": 1, + } + ) + return steps, session_id, terminal_event_seen + + def _completed_codex_items(events: list[dict[str, Any]]) -> list[dict[str, Any]]: order: list[str] = [] latest: dict[str, dict[str, Any]] = {} @@ -900,9 +1056,16 @@ def write_agent_trajectory( preamble_lines, diagnostic_lines, malformed_event_lines, - ) = _load_codex_stream(source_path) + stream_read_attempts, + ) = _load_stable_codex_stream(source_path) else: - events, preamble_lines, diagnostic_lines, malformed_event_lines = [], 0, 0, 0 + ( + events, + preamble_lines, + diagnostic_lines, + malformed_event_lines, + stream_read_attempts, + ) = ([], 0, 0, 0, 0) observed_models = _observed_codex_models(agent_dir) runtime_model_name = ( next(iter(observed_models)) if len(observed_models) == 1 else None @@ -927,6 +1090,30 @@ def write_agent_trajectory( step["step_id"] = len(steps) + 1 steps.append(step) + session_fallback = False + if ( + len(steps) == 1 + or malformed_event_lines + or not terminal_event_seen + or runtime_model_name is None + ): + session_path, session_events = _codex_session_events(agent_dir) + session_steps, fallback_session_id, fallback_terminal = _codex_session_steps( + session_events + ) + if session_steps and fallback_terminal and runtime_model_name is not None: + session_fallback = True + source_path = session_path or source_path + events = session_events + terminal_event_seen = True + malformed_event_lines = 0 + if fallback_session_id: + session_id = fallback_session_id + steps = steps[:1] + for step in session_steps: + step["step_id"] = len(steps) + 1 + steps.append(step) + if ( len(steps) == 1 or malformed_event_lines @@ -944,6 +1131,8 @@ def write_agent_trajectory( "preamble_lines": preamble_lines, "diagnostic_lines": diagnostic_lines, "malformed_event_lines": malformed_event_lines, + "stream_read_attempts": stream_read_attempts, + "session_fallback": session_fallback, "observed_models": sorted(observed_models), }, } @@ -977,6 +1166,7 @@ def write_agent_trajectory( "native_raw_preamble_lines": preamble_lines, "native_raw_diagnostic_lines": diagnostic_lines, "native_raw_malformed_event_lines": malformed_event_lines, + "native_session_fallback": session_fallback, "observed_models": sorted(observed_models), "terminal_event_seen": terminal_event_seen, }, @@ -993,6 +1183,8 @@ def write_agent_trajectory( "preamble_lines": preamble_lines, "diagnostic_lines": diagnostic_lines, "malformed_event_lines": malformed_event_lines, + "stream_read_attempts": stream_read_attempts, + "session_fallback": session_fallback, "observed_models": sorted(observed_models), }, } diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 2ef67e5..821baf0 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -655,6 +655,171 @@ def test_codex_trajectory_rejects_unknown_interleaved_output(tmp_path: Path) -> assert metadata["trajectory_validation"]["malformed_event_lines"] == 1 +def test_codex_trajectory_retries_transient_malformed_stream( + tmp_path: Path, + monkeypatch, +) -> None: + agent_dir = tmp_path / "agent" + session_dir = agent_dir / "sessions" + session_dir.mkdir(parents=True) + events = [ + {"type": "thread.started", "thread_id": "thread-123"}, + { + "type": "item.completed", + "item": {"id": "message-1", "type": "agent_message", "text": "done"}, + }, + {"type": "turn.completed", "usage": {"input_tokens": 1}}, + ] + (agent_dir / "codex.txt").write_text( + "\n".join(json.dumps(event) for event in events) + "\n", + encoding="utf-8", + ) + (session_dir / "rollout.jsonl").write_text( + json.dumps( + { + "type": "turn_context", + "payload": {"model": "gpt-5.5"}, + } + ) + + "\n", + encoding="utf-8", + ) + original_loader = native_runtime._load_codex_stream + calls = 0 + + def transient_loader(path: Path): + nonlocal calls + calls += 1 + loaded = original_loader(path) + if calls == 1: + return (*loaded[:3], 1) + return loaded + + monkeypatch.setattr(native_runtime, "_load_codex_stream", transient_loader) + monkeypatch.setattr(native_runtime.time, "sleep", lambda _seconds: None) + run = RunSpec( + run_label="codex-gpt55-calibration", + harness="codex", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=1, + run_date="20260727", + ) + + metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + + assert metadata["trajectory_status"] == "real" + assert metadata["trajectory_validation"]["malformed_event_lines"] == 0 + assert metadata["trajectory_validation"]["stream_read_attempts"] == 2 + + +def test_codex_trajectory_falls_back_to_complete_session_stream( + tmp_path: Path, + monkeypatch, +) -> None: + agent_dir = tmp_path / "agent" + session_dir = agent_dir / "sessions" + session_dir.mkdir(parents=True) + (agent_dir / "codex.txt").write_text( + "\n".join( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-123"}), + '{"type":"item.completed","item":{"id":"broken"', + json.dumps({"type": "turn.completed", "usage": {"input_tokens": 1}}), + ] + ) + + "\n", + encoding="utf-8", + ) + session_events = [ + { + "type": "session_meta", + "payload": {"session_id": "session-123"}, + }, + { + "type": "turn_context", + "payload": {"model": "gpt-5.5"}, + }, + { + "type": "response_item", + "payload": { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "inspect"}], + }, + }, + { + "type": "response_item", + "payload": { + "type": "custom_tool_call", + "call_id": "call-1", + "name": "shell", + "input": '{"command":"pwd"}', + }, + }, + { + "type": "response_item", + "payload": { + "type": "custom_tool_call_output", + "call_id": "call-1", + "output": "/app", + }, + }, + { + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "done"}], + }, + }, + { + "type": "event_msg", + "payload": {"type": "task_complete"}, + }, + ] + (session_dir / "rollout.jsonl").write_text( + "\n".join(json.dumps(event) for event in session_events) + "\n", + encoding="utf-8", + ) + monkeypatch.setattr(native_runtime.time, "sleep", lambda _seconds: None) + run = RunSpec( + run_label="codex-gpt55-calibration", + harness="codex", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=1, + run_date="20260727", + ) + + metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + trajectory = json.loads((agent_dir / "trajectory.json").read_text()) + + assert metadata["trajectory_status"] == "real" + assert metadata["trajectory_validation"]["session_fallback"] is True + assert metadata["trajectory_validation"]["malformed_event_lines"] == 0 + assert trajectory["session_id"] == "session-123" + assert trajectory["steps"][1]["reasoning_content"] == "inspect" + assert trajectory["steps"][2]["tool_calls"][0]["function_name"] == "shell" + assert trajectory["steps"][2]["observation"]["results"][0]["content"] == "/app" + assert trajectory["steps"][3]["message"] == "done" + + def test_unsupported_trajectory_is_explicitly_unranked(tmp_path: Path) -> None: run = RunSpec( run_label="claude-code-gpt55", From 3cf9e56e0ed6198507169b84d6294ece1a86cc21 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 22:36:17 +0200 Subject: [PATCH 16/28] feat(eval): support targeted canonical repairs --- scripts/native_eval/fleet.py | 33 ++++++++++++++++++++++--- scripts/native_eval/remote_run.sh | 10 +++++--- scripts/native_eval/run_job.py | 10 ++++++++ tests/test_native_eval_fleet.py | 39 +++++++++++++++++++++++++++++- tests/test_native_eval_runner.py | 40 +++++++++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 7 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 6d5b763..fc3ad5e 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -340,12 +340,30 @@ def _prepare_local_layout(self) -> None: (self.config.local_root / relative).mkdir(parents=True, exist_ok=True) def _validate_plan(self) -> None: - expected = int(self._store.data["expected_task_count"]) + suite_expected = int(self._store.data["expected_task_count"]) for entry in self._store.all_entries(): run = self._run_spec(entry) + task_names = entry.get("task_names") + if task_names is None: + expected = suite_expected + elif not isinstance(task_names, list) or not all( + isinstance(name, str) and name for name in task_names + ): + raise FleetError( + f"{run.run_label} task_names must be a list of non-empty strings" + ) + elif len(set(task_names)) != len(task_names): + raise FleetError(f"{run.run_label} task_names contains duplicates") + else: + expected = len(task_names) + if not entry.get("rerun_of_canonical_run"): + raise FleetError( + f"{run.run_label} task subset lacks rerun_of_canonical_run" + ) if run.expected_task_count != expected: raise FleetError( - f"{run.run_label} expects {run.expected_task_count}, index expects {expected}" + f"{run.run_label} expects {run.expected_task_count} tasks, " + f"plan selects {expected}" ) if run.provider == "openai": reasoning_effort = str(entry.get("reasoning_effort") or "") @@ -843,6 +861,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: kill -0 "$pid" printf '%s\n' "$pid" """ + entry = self._store.get(run.run_label) args = [ self.config.remote_root, f"{self.config.remote_root}/public-tasks/combined tasks/tasks", @@ -859,8 +878,9 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: run.model_id, run.provider, run.proxy_model_name, + str(entry.get("rerun_of_canonical_run") or ""), + *[str(name) for name in entry.get("task_names") or []], ] - entry = self._store.get(run.run_label) reasoning_effort = str(entry.get("reasoning_effort") or "") judge_reasoning_effort = str( entry.get("judge_reasoning_effort") or reasoning_effort @@ -1237,6 +1257,13 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: "artifacts": [], "created_at_utc": utc_now(), } + for metadata_field in ( + "task_names", + "rerun_of_canonical_run", + "repair_classifications", + ): + if metadata_field in entry: + rerun[metadata_field] = copy.deepcopy(entry[metadata_field]) self._store.append(rerun) return label diff --git a/scripts/native_eval/remote_run.sh b/scripts/native_eval/remote_run.sh index 0f33021..91c6f3c 100755 --- a/scripts/native_eval/remote_run.sh +++ b/scripts/native_eval/remote_run.sh @@ -2,11 +2,11 @@ set -euo pipefail usage() { - echo "usage: remote_run.sh ROOT TASKS_ROOT ENV_FILE RUN_LABEL HARNESS MODEL_SLUG REP EXPECTED_COUNT PUBLIC_TASKS_COMMIT RUN_DATE CONCURRENCY HARNESS_VERSION MODEL_ID MODEL_PROVIDER PROXY_MODEL_NAME [TASK_NAME ...]" >&2 + echo "usage: remote_run.sh ROOT TASKS_ROOT ENV_FILE RUN_LABEL HARNESS MODEL_SLUG REP EXPECTED_COUNT PUBLIC_TASKS_COMMIT RUN_DATE CONCURRENCY HARNESS_VERSION MODEL_ID MODEL_PROVIDER PROXY_MODEL_NAME RERUN_OF_CANONICAL_RUN [TASK_NAME ...]" >&2 exit 2 } -[[ $# -ge 15 ]] || usage +[[ $# -ge 16 ]] || usage ROOT="$1" TASKS_ROOT="$2" @@ -23,7 +23,8 @@ HARNESS_VERSION="${12}" MODEL_ID="${13}" MODEL_PROVIDER="${14}" PROXY_MODEL_NAME="${15}" -shift 15 +RERUN_OF_CANONICAL_RUN="${16}" +shift 16 TASK_NAMES=("$@") TASK_SUITE_PATH="combined tasks/tasks" TOOLCHAIN_ROOT="${TOOLCHAIN_ROOT:-/opt/shellbench-native}" @@ -127,6 +128,9 @@ RUN_COMMAND=( --proxy-url "http://host.docker.internal:4000" --concurrency "$CONCURRENCY" ) +if [[ -n "$RERUN_OF_CANONICAL_RUN" ]]; then + RUN_COMMAND+=(--rerun-of-canonical-run "$RERUN_OF_CANONICAL_RUN") +fi for task_name in "${TASK_NAMES[@]}"; do RUN_COMMAND+=(--task "$task_name") done diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index 3b64fa8..5945370 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -32,6 +32,7 @@ async def run_job( proxy_key: str, concurrency: int, task_names: set[str] | None = None, + rerun_of_canonical_run: str | None = None, ) -> dict[str, Any]: tasks = validate_suite(tasks_root) if task_names: @@ -60,6 +61,7 @@ async def run_job( started_at=started_at, tasks_root=tasks_root, tasks=tasks, + rerun_of_canonical_run=rerun_of_canonical_run, ) atomic_write_json(job_dir / "run_manifest.json", manifest) atomic_write_json( @@ -201,6 +203,7 @@ def _run_manifest( started_at: str, tasks_root: Path, tasks: list[TaskSpec], + rerun_of_canonical_run: str | None = None, ) -> dict[str, Any]: return { "run_label": run.run_label, @@ -223,6 +226,11 @@ def _run_manifest( } for task in tasks ], + "repair_mode": rerun_of_canonical_run is not None, + "rerun_of_canonical_run": rerun_of_canonical_run, + "repair_task_names": ( + [task.name for task in tasks] if rerun_of_canonical_run else [] + ), "task_concurrency": concurrency, "agent_concurrency": concurrency, "provider": "aws", @@ -336,6 +344,7 @@ def parse_args() -> argparse.Namespace: dest="task_names", help="Run only the named task. Repeat for multiple tasks.", ) + parser.add_argument("--rerun-of-canonical-run") return parser.parse_args() @@ -354,6 +363,7 @@ def main() -> None: proxy_key=proxy_key, concurrency=args.concurrency, task_names=set(args.task_names or []), + rerun_of_canonical_run=args.rerun_of_canonical_run, ) ) print(json.dumps(state, indent=2)) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index b2c01c7..3ac3dc4 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -10,7 +10,7 @@ import pytest -from scripts.native_eval.fleet import FleetConfig, FleetController, parse_args +from scripts.native_eval.fleet import FleetConfig, FleetController, FleetError, parse_args from scripts.native_eval.models import RunSpec @@ -378,6 +378,43 @@ def test_controller_accepts_xhigh_reasoning_effort(tmp_path: Path) -> None: assert completed["judge_reasoning_effort"] == "high" +def test_controller_dispatches_targeted_repair_tasks_with_lineage( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-low-full-116-r1-20260728-repairtasks1" + run = _planned(_run_spec(label, expected_task_count=2)) + run["task_names"] = ["task-a", "task-b"] + run["rerun_of_canonical_run"] = ( + "openclaw-gpt55-low-full-116-r1-20260728-runnerd676-lifecyclefix1" + ) + run["repair_classifications"] = ["infra", "agent_exit"] + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run], expected_task_count=116) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + + dispatched = executor.dispatch_arguments[label] + assert dispatched[15] == run["rerun_of_canonical_run"] + assert dispatched[16:] == ["task-a", "task-b"] + + +def test_controller_rejects_task_subset_without_canonical_parent( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-low-full-116-r1-20260728-repairtasks1" + run = _planned(_run_spec(label, expected_task_count=1)) + run["task_names"] = ["task-a"] + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run], expected_task_count=116) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 1}) + + with pytest.raises(FleetError, match="task subset lacks rerun_of_canonical_run"): + FleetController(config, executor=executor).run() + + def test_capacity_warmup_retries_same_run_without_recovery_churn( tmp_path: Path, ) -> None: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 821baf0..43e388d 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -7,6 +7,7 @@ import tarfile from argparse import Namespace from pathlib import Path +from types import SimpleNamespace from scripts.native_eval.checkpoint_loop import ( count_result_json, @@ -209,6 +210,45 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["provider_model_id"] == "gpt-5.5" assert manifest["reasoning_effort"] == "high" assert manifest["judge_reasoning_effort"] == "high" + assert manifest["repair_mode"] is False + assert manifest["rerun_of_canonical_run"] is None + assert manifest["repair_task_names"] == [] + + +def test_run_manifest_records_targeted_repair_lineage(tmp_path: Path) -> None: + run = RunSpec( + run_label="openclaw-gpt55-low-full-116-r1-parent-targetedfix1", + harness="openclaw", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="sb-gpt55", + repetition=1, + expected_task_count=2, + run_date="20260728", + ) + tasks = [ + SimpleNamespace(name="task-a", path=tmp_path / "task-a", checksum="a"), + SimpleNamespace(name="task-b", path=tmp_path / "task-b", checksum="b"), + ] + + manifest = _run_manifest( + run, + public_tasks_commit="tasks-commit", + task_suite_path="combined tasks/tasks", + concurrency=2, + started_at="2026-07-28T00:00:00Z", + tasks_root=tmp_path, + tasks=tasks, + rerun_of_canonical_run="openclaw-gpt55-low-full-116-r1-parent", + ) + + assert manifest["repair_mode"] is True + assert manifest["rerun_of_canonical_run"] == ( + "openclaw-gpt55-low-full-116-r1-parent" + ) + assert manifest["repair_task_names"] == ["task-a", "task-b"] def test_run_spec_preserves_explicit_planned_identity() -> None: From 3c4574c9daf51b2292906e2c1346dfda06a25934 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 22:49:33 +0200 Subject: [PATCH 17/28] fix(eval): recover transient Crabbox SSH dispatches --- scripts/native_eval/fleet.py | 104 +++++++++++++++++++++----------- tests/test_native_eval_fleet.py | 28 +++++++++ 2 files changed, 97 insertions(+), 35 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index fc3ad5e..3bb4952 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -286,6 +286,7 @@ def __init__( self.runner_archive_sha256 = "" self.task_archive_sha256 = "" self.crabbox_cli_version = "" + self._dispatch_lock = threading.Lock() def run(self) -> int: self._prepare_local_layout() @@ -502,7 +503,8 @@ def _execute_entry(self, run_label: str) -> bool: if remote_state == "missing": if self._local_artifacts(run.run_label): raise FleetError("local artifacts exist but the remote run state is missing") - self._hydrate_lease(lease) + if not entry.get("bootstrapped_at_utc"): + self._hydrate_lease(lease) self._dispatch(lease, run) elif remote_state == "stale": raise FleetError("remote run state is stale and cannot be overwritten") @@ -557,6 +559,12 @@ def _execute_entry(self, run_label: str) -> bool: def _ensure_lease(self, entry: dict[str, Any]) -> Lease: lease_value = entry.get("lease") if lease_value: + try: + stored = Lease.from_manifest(lease_value) + except FleetError: + stored = None + if stored is not None and self._ssh_reachable(stored): + return self._detect_region(stored) identifier = str(lease_value["id"]) try: return self._inspect_lease( @@ -565,7 +573,8 @@ def _ensure_lease(self, entry: dict[str, Any]) -> Lease: region_hint=str(lease_value.get("region") or ""), ) except LeaseNotReadyError: - stored = Lease.from_manifest(lease_value) + if stored is None: + stored = Lease.from_manifest(lease_value) if self._ssh_reachable(stored): return self._detect_region(stored) raise @@ -667,11 +676,17 @@ def _inspect_lease( return self._detect_region(lease) def _ssh_reachable(self, lease: Lease) -> bool: - result = self.executor.run( - self._ssh_command(lease, ["true"]), - capture_output=True, - ) - return result.returncode == 0 + with self._dispatch_lock: + for attempt in range(1, 5): + result = self.executor.run( + self._ssh_command(lease, ["true"]), + capture_output=True, + ) + if result.returncode == 0: + return True + if attempt < 4: + time.sleep(attempt * 5) + return False def _detect_region(self, lease: Lease) -> Lease: script = """ @@ -885,35 +900,54 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: judge_reasoning_effort = str( entry.get("judge_reasoning_effort") or reasoning_effort ) - self._checked( - self._ssh_command( - lease, - [ - "bash", - "-c", - script, - "fleet-dispatch", - self.config.remote_root, - run.run_label, - self.crabbox_cli_version, - lease.slug, - lease.lease_id, - lease.instance_type, - lease.host, - lease.region, - self.runner_commit, - self.config.harbor_reference_commit, - self.config.judge_model_id, - self.config.execution_mode, - reasoning_effort, - judge_reasoning_effort, - str(self.config.parity_validated).lower(), - *args, - ], - ), - capture_output=True, - description=f"dispatch {run.run_label}", + command = self._ssh_command( + lease, + [ + "bash", + "-c", + script, + "fleet-dispatch", + self.config.remote_root, + run.run_label, + self.crabbox_cli_version, + lease.slug, + lease.lease_id, + lease.instance_type, + lease.host, + lease.region, + self.runner_commit, + self.config.harbor_reference_commit, + self.config.judge_model_id, + self.config.execution_mode, + reasoning_effort, + judge_reasoning_effort, + str(self.config.parity_validated).lower(), + *args, + ], ) + with self._dispatch_lock: + for attempt in range(1, 5): + result = self.executor.run(command, capture_output=True) + if result.returncode == 0: + break + detail = (result.stderr or result.stdout or "").strip() + retryable = result.returncode == 255 and any( + marker in detail.lower() + for marker in ( + "operation timed out", + "connection timed out", + "connection refused", + "connection closed", + "banner exchange", + ) + ) + if not retryable or attempt == 4: + suffix = f": {detail[:500]}" if detail else "" + raise FleetError( + f"dispatch {run.run_label} failed with exit " + f"{result.returncode}{suffix}" + ) + time.sleep(attempt * 5) self._store.update( run.run_label, dispatched_at_utc=utc_now(), diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 3ac3dc4..3317e4b 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -114,6 +114,7 @@ def __init__( stop_code: int = 0, inspect_errors: set[str] | None = None, warmup_capacity_failures: dict[str, int] | None = None, + dispatch_failures: dict[str, int] | None = None, ) -> None: self.local_root = local_root self.expected_counts = expected_counts @@ -124,7 +125,9 @@ def __init__( self.stop_code = stop_code self.inspect_errors = inspect_errors or set() self.warmup_capacity_failures = warmup_capacity_failures or {} + self.dispatch_failures = dispatch_failures or {} self.warmup_attempts: dict[str, int] = {} + self.dispatch_attempts: dict[str, int] = {} self.leases: dict[str, dict[str, object]] = {} self.commands: list[list[str]] = [] self.events: list[tuple[str, str]] = [] @@ -253,6 +256,14 @@ def run( dispatch_marker = remote_command.index("fleet-dispatch") remote_run_args = remote_command[dispatch_marker + 16 :] with self._dispatch_condition: + attempt = self.dispatch_attempts.get(label, 0) + 1 + self.dispatch_attempts[label] = attempt + if attempt <= self.dispatch_failures.get(label, 0): + return _result( + argv, + 255, + stderr="ssh: connect to host 192.0.2.1 port 22: Operation timed out", + ) self.dispatches.append(label) self.dispatch_concurrency[label] = int(remote_run_args[10]) self.dispatch_arguments[label] = remote_run_args @@ -400,6 +411,23 @@ def test_controller_dispatches_targeted_repair_tasks_with_lineage( assert dispatched[16:] == ["task-a", "task-b"] +def test_controller_retries_transient_dispatch_timeout(tmp_path: Path) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [_planned(_run_spec(label))]) + config = _config(tmp_path, run_index) + executor = FakeExecutor( + config.local_root, + expected_counts={label: 2}, + dispatch_failures={label: 1}, + ) + + assert FleetController(config, executor=executor).run() == 0 + + assert executor.dispatch_attempts[label] == 2 + assert executor.dispatches == [label] + + def test_controller_rejects_task_subset_without_canonical_parent( tmp_path: Path, ) -> None: From f94b372a8ebef17e5845aceb17245ccc60b66b81 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:01:41 +0200 Subject: [PATCH 18/28] fix(eval): harden Crabbox lease cleanup --- scripts/native_eval/fleet.py | 88 +++++++++++++++++++++++++++------ tests/test_native_eval_fleet.py | 48 +++++++++++++++++- 2 files changed, 120 insertions(+), 16 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 3bb4952..8bcf039 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -74,6 +74,36 @@ def run( stderr=subprocess.PIPE if capture_output else None, ) + def run_with_timeout( + self, + command: Sequence[str], + *, + capture_output: bool, + timeout: float, + ) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run( + list(command), + check=False, + text=True, + stdout=subprocess.PIPE if capture_output else None, + stderr=subprocess.PIPE if capture_output else None, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or f"timed out after {timeout:g}s" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + return subprocess.CompletedProcess( + list(command), + 124, + stdout=stdout, + stderr=stderr, + ) + class FleetError(RuntimeError): pass @@ -104,14 +134,17 @@ def target(self) -> str: @classmethod def from_inspect(cls, value: dict[str, Any], *, region: str) -> Lease: + state = str(value.get("state") or "") + if not value.get("ready") and state in {"active", "provisioning"}: + lease_id = value.get("id") or value.get("slug") or "unknown" + raise LeaseNotReadyError(f"lease {lease_id} is {state} but not ready") + if state != "active": + lease_id = value.get("id") or value.get("slug") or "unknown" + raise LeaseUnavailableError(f"lease {lease_id} is no longer active") required = ("id", "slug", "sshHost", "sshUser", "sshPort", "sshKey") missing = [field for field in required if not value.get(field)] if missing: raise FleetError(f"Crabbox inspect omitted fields: {', '.join(missing)}") - if value.get("state") != "active": - raise LeaseUnavailableError(f"lease {value['id']} is no longer active") - if not value.get("ready"): - raise LeaseNotReadyError(f"lease {value['id']} is active but not ready") return cls( lease_id=str(value["id"]), slug=str(value["slug"]), @@ -1159,18 +1192,43 @@ def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: "checkpoint final event after remote done" ), } - stop = self.executor.run( - [ - self.config.crabbox_bin, - "stop", - "--provider", - "aws", - "--id", - str(lease_value["id"]), - ], - capture_output=True, + stop_command = [ + self.config.crabbox_bin, + "stop", + "--provider", + "aws", + "--id", + str(lease_value["id"]), + ] + try: + lease_still_exists = ( + self._inspect_lease( + str(lease_value["id"]), + required=False, + region_hint=str(lease_value.get("region") or ""), + ) + is not None + ) + except FleetError: + lease_still_exists = True + if lease_still_exists: + with self._dispatch_lock: + if isinstance(self.executor, SubprocessExecutor): + stop = self.executor.run_with_timeout( + stop_command, + capture_output=True, + timeout=45, + ) + else: + stop = self.executor.run(stop_command, capture_output=True) + else: + stop = subprocess.CompletedProcess(stop_command, 0, stdout="", stderr="") + stop_detail = (stop.stderr or stop.stdout or "").strip().lower() + already_stopped = any( + marker in stop_detail + for marker in ("not found", "not_found", "http 404", "already stopped") ) - if stop.returncode: + if stop.returncode and not already_stopped: self._store.update( run.run_label, status="stop_pending", diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 3317e4b..f767992 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -10,7 +10,15 @@ import pytest -from scripts.native_eval.fleet import FleetConfig, FleetController, FleetError, parse_args +from scripts.native_eval.fleet import ( + FleetConfig, + FleetController, + FleetError, + Lease, + LeaseNotReadyError, + SubprocessExecutor, + parse_args, +) from scripts.native_eval.models import RunSpec @@ -70,6 +78,44 @@ def _planned(run: RunSpec) -> dict[str, object]: } +def test_provisioning_lease_is_not_ready_before_ssh_details_exist() -> None: + with pytest.raises(LeaseNotReadyError, match="provisioning but not ready"): + Lease.from_inspect( + { + "id": "cbx_pending", + "slug": "sb-native-pending", + "state": "provisioning", + "ready": False, + "sshHost": "", + }, + region="eu-west-1", + ) + + +def test_subprocess_timeout_output_is_normalized_to_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def timeout(*args: object, **kwargs: object) -> None: + del args, kwargs + raise subprocess.TimeoutExpired( + ["crabbox", "stop"], + 45, + output=b"partial stdout", + stderr=b"partial stderr", + ) + + monkeypatch.setattr(subprocess, "run", timeout) + result = SubprocessExecutor().run_with_timeout( + ["crabbox", "stop"], + capture_output=True, + timeout=45, + ) + + assert result.returncode == 124 + assert result.stdout == "partial stdout" + assert result.stderr == "partial stderr" + + def _write_archive(path: Path) -> None: source = path.parent / f"{path.stem}-source" source.mkdir() From dc70121dabe0918068a178fd4b2df451b937951a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:11:46 +0200 Subject: [PATCH 19/28] fix(eval): reconcile timed-out lease stops --- scripts/native_eval/fleet.py | 22 +++++++++++++++++++++- tests/test_native_eval_fleet.py | 25 ++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 8bcf039..3083038 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -48,6 +48,7 @@ ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} +CRABBOX_STOP_TIMEOUT_SECONDS = 15 class CommandExecutor(Protocol): @@ -1217,12 +1218,31 @@ def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: stop = self.executor.run_with_timeout( stop_command, capture_output=True, - timeout=45, + timeout=CRABBOX_STOP_TIMEOUT_SECONDS, ) else: stop = self.executor.run(stop_command, capture_output=True) else: stop = subprocess.CompletedProcess(stop_command, 0, stdout="", stderr="") + if stop.returncode: + try: + stopped_after_error = ( + self._inspect_lease( + str(lease_value["id"]), + required=False, + region_hint=str(lease_value.get("region") or ""), + ) + is None + ) + except FleetError: + stopped_after_error = False + if stopped_after_error: + stop = subprocess.CompletedProcess( + stop_command, + 0, + stdout=stop.stdout, + stderr=stop.stderr, + ) stop_detail = (stop.stderr or stop.stdout or "").strip().lower() already_stopped = any( marker in stop_detail diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index f767992..487394b 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -158,6 +158,7 @@ def __init__( running_labels: set[str] | None = None, checkpoint_blocks: dict[str, threading.Event] | None = None, stop_code: int = 0, + stop_removes_lease_on_error: bool = False, inspect_errors: set[str] | None = None, warmup_capacity_failures: dict[str, int] | None = None, dispatch_failures: dict[str, int] | None = None, @@ -169,6 +170,7 @@ def __init__( self.remote_states = {label: "running" for label in (running_labels or set())} self.checkpoint_blocks = checkpoint_blocks or {} self.stop_code = stop_code + self.stop_removes_lease_on_error = stop_removes_lease_on_error self.inspect_errors = inspect_errors or set() self.warmup_capacity_failures = warmup_capacity_failures or {} self.dispatch_failures = dispatch_failures or {} @@ -264,8 +266,9 @@ def run( with self._lock: self.stops.append(lease_id) self.events.append(("stop", lease_id)) - if not self.stop_code: + if not self.stop_code or self.stop_removes_lease_on_error: self.active_leases -= 1 + self.leases.pop(lease_id, None) return _result(argv, self.stop_code) if "-m" in argv and "scripts.native_eval.checkpoint_loop" in argv: @@ -1154,6 +1157,26 @@ def test_stop_failure_is_left_pending_without_tight_retry(tmp_path: Path) -> Non assert len(executor.stops) == 1 +def test_stop_failure_is_reconciled_when_lease_disappeared(tmp_path: Path) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [_planned(_run_spec(label))]) + config = _config(tmp_path, run_index) + executor = FakeExecutor( + config.local_root, + expected_counts={label: 2}, + stop_code=124, + stop_removes_lease_on_error=True, + ) + + assert FleetController(config, executor=executor).run() == 0 + + run = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] + assert run["status"] == "completed" + assert run["verified_final_export"] is True + assert len(executor.stops) == 1 + + def test_ambiguous_lease_inspect_error_does_not_duplicate_run( tmp_path: Path, ) -> None: From 7bb5f54aa7ac8433f3ebe6fe048876a33946edb0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:16:38 +0200 Subject: [PATCH 20/28] fix(eval): recognize stopped Crabbox leases --- scripts/native_eval/fleet.py | 13 +++++++++---- tests/test_native_eval_fleet.py | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 3083038..c2c424e 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -703,10 +703,15 @@ def _inspect_lease( value = json.loads(result.stdout) except json.JSONDecodeError as exc: raise FleetError(f"invalid Crabbox inspect JSON for {identifier}") from exc - lease = Lease.from_inspect( - value, - region=region_hint or self.config.region, - ) + try: + lease = Lease.from_inspect( + value, + region=region_hint or self.config.region, + ) + except LeaseUnavailableError: + if required: + raise + return None return self._detect_region(lease) def _ssh_reachable(self, lease: Lease) -> bool: diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 487394b..3deadf1 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -16,6 +16,7 @@ FleetError, Lease, LeaseNotReadyError, + LeaseUnavailableError, SubprocessExecutor, parse_args, ) @@ -116,6 +117,29 @@ def timeout(*args: object, **kwargs: object) -> None: assert result.stderr == "partial stderr" +def test_optional_inspect_treats_stopped_lease_as_absent(tmp_path: Path) -> None: + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, []) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={}) + executor.leases["cbx_stopped"] = { + "id": "cbx_stopped", + "slug": "stopped", + "state": "stopped", + "ready": False, + "serverType": "c7a.24xlarge", + "sshHost": "", + "sshUser": "crabbox", + "sshPort": "22", + "sshKey": "/tmp/cbx_stopped.key", + } + controller = FleetController(config, executor=executor) + + assert controller._inspect_lease("cbx_stopped", required=False) is None + with pytest.raises(LeaseUnavailableError): + controller._inspect_lease("cbx_stopped", required=True) + + def _write_archive(path: Path) -> None: source = path.parent / f"{path.stem}-source" source.mkdir() From 8065360be02590abe1d3f4f26d498ddcb69ede29 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:31:30 +0200 Subject: [PATCH 21/28] fix(eval): allow Crabbox lease release retries --- scripts/native_eval/fleet.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index c2c424e..9ba6cc9 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -48,7 +48,10 @@ ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} -CRABBOX_STOP_TIMEOUT_SECONDS = 15 +# Crabbox's coordinator release path retries five 60-second requests with +# bounded backoff. Give it enough time to finish instead of leaking live AWS +# leases after a verified export. +CRABBOX_STOP_TIMEOUT_SECONDS = 6 * 60 class CommandExecutor(Protocol): From 543e7eb9806acb4aadad679015aa88ed4a6ddd3c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:35:09 +0200 Subject: [PATCH 22/28] fix(eval): wait for Crabbox lease readiness --- scripts/native_eval/fleet.py | 22 ++++++++++++++++++++-- tests/test_native_eval_fleet.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 9ba6cc9..71da196 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -52,6 +52,8 @@ # bounded backoff. Give it enough time to finish instead of leaking live AWS # leases after a verified export. CRABBOX_STOP_TIMEOUT_SECONDS = 6 * 60 +CRABBOX_READY_ATTEMPTS = 30 +CRABBOX_READY_BACKOFF_SECONDS = 10 class CommandExecutor(Protocol): @@ -622,7 +624,10 @@ def _ensure_lease(self, entry: dict[str, Any]) -> Lease: status="leasing", requested_lease_slug=slug, ) - existing = self._inspect_lease(slug, required=False) + try: + existing = self._inspect_lease(slug, required=False) + except LeaseNotReadyError: + existing = self._wait_for_lease_ready(slug) if existing is None: command = [ "env", @@ -643,7 +648,7 @@ def _ensure_lease(self, entry: dict[str, Any]) -> Lease: if self.config.instance_type: command.extend(["--type", self.config.instance_type]) self._warmup_lease(command, slug) - existing = self._inspect_lease(slug, required=True) + existing = self._wait_for_lease_ready(slug) self._store.update( entry["run_label"], status="bootstrapping", @@ -651,6 +656,19 @@ def _ensure_lease(self, entry: dict[str, Any]) -> Lease: ) return existing + def _wait_for_lease_ready(self, identifier: str) -> Lease: + for attempt in range(1, CRABBOX_READY_ATTEMPTS + 1): + try: + lease = self._inspect_lease(identifier, required=True) + except LeaseNotReadyError: + if attempt == CRABBOX_READY_ATTEMPTS: + raise + time.sleep(CRABBOX_READY_BACKOFF_SECONDS) + continue + assert lease is not None + return lease + raise AssertionError("unreachable") + def _warmup_lease(self, command: Sequence[str], slug: str) -> None: for attempt in range(1, self.config.warmup_capacity_attempts + 1): result = self.executor.run(command, capture_output=True) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 3deadf1..608cd44 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -93,6 +93,38 @@ def test_provisioning_lease_is_not_ready_before_ssh_details_exist() -> None: ) +def test_wait_for_lease_ready_retries_provisioning_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + controller = object.__new__(FleetController) + ready = Lease( + lease_id="cbx_ready", + slug="sb-native-ready", + host="192.0.2.10", + user="crabbox", + port=22, + identity_file=Path("/tmp/cbx_ready.key"), + instance_type="c7a.24xlarge", + region="eu-west-1", + ) + attempts = 0 + + def inspect(identifier: str, *, required: bool) -> Lease: + nonlocal attempts + assert identifier == "sb-native-ready" + assert required is True + attempts += 1 + if attempts < 3: + raise LeaseNotReadyError("lease is provisioning but not ready") + return ready + + monkeypatch.setattr(controller, "_inspect_lease", inspect) + monkeypatch.setattr("scripts.native_eval.fleet.time.sleep", lambda _: None) + + assert controller._wait_for_lease_ready("sb-native-ready") == ready + assert attempts == 3 + + def test_subprocess_timeout_output_is_normalized_to_text( monkeypatch: pytest.MonkeyPatch, ) -> None: From 8408ea333b02e267d88d0d9965f30954428060e4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:38:18 +0200 Subject: [PATCH 23/28] fix(eval): separate lease cleanup from dispatch --- scripts/native_eval/fleet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 71da196..4643a37 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -326,6 +326,7 @@ def __init__( self.task_archive_sha256 = "" self.crabbox_cli_version = "" self._dispatch_lock = threading.Lock() + self._cleanup_lock = threading.Lock() def run(self) -> int: self._prepare_local_layout() @@ -1239,7 +1240,7 @@ def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: except FleetError: lease_still_exists = True if lease_still_exists: - with self._dispatch_lock: + with self._cleanup_lock: if isinstance(self.executor, SubprocessExecutor): stop = self.executor.run_with_timeout( stop_command, From b1eabd8b9ea2b4ccdc06a6d581918e5e08cc8587 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:40:14 +0200 Subject: [PATCH 24/28] fix(eval): retry transient Crabbox inspect failures --- scripts/native_eval/fleet.py | 43 ++++++++++++++++++++++++--------- tests/test_native_eval_fleet.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 4643a37..2c07758 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -54,6 +54,7 @@ CRABBOX_STOP_TIMEOUT_SECONDS = 6 * 60 CRABBOX_READY_ATTEMPTS = 30 CRABBOX_READY_BACKOFF_SECONDS = 10 +CRABBOX_INSPECT_ATTEMPTS = 4 class CommandExecutor(Protocol): @@ -695,18 +696,36 @@ def _inspect_lease( required: bool, region_hint: str = "", ) -> Lease | None: - result = self.executor.run( - [ - self.config.crabbox_bin, - "inspect", - "--provider", - "aws", - "--id", - identifier, - "--json", - ], - capture_output=True, - ) + command = [ + self.config.crabbox_bin, + "inspect", + "--provider", + "aws", + "--id", + identifier, + "--json", + ] + for attempt in range(1, CRABBOX_INSPECT_ATTEMPTS + 1): + result = self.executor.run(command, capture_output=True) + if result.returncode == 0: + break + detail = (result.stderr or result.stdout or "").strip() + transient = any( + marker in detail.lower() + for marker in ( + "http 500", + "http 502", + "http 503", + "http 504", + "error code: 1101", + "context deadline exceeded", + "connection timed out", + "connection reset", + ) + ) + if not transient or attempt == CRABBOX_INSPECT_ATTEMPTS: + break + time.sleep(attempt * 5) if result.returncode: detail = (result.stderr or result.stdout or "").strip() missing = any( diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 608cd44..1838c49 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -172,6 +172,41 @@ def test_optional_inspect_treats_stopped_lease_as_absent(tmp_path: Path) -> None controller._inspect_lease("cbx_stopped", required=True) +def test_inspect_retries_transient_coordinator_failures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, []) + config = _config(tmp_path, run_index) + executor = FakeExecutor( + config.local_root, + expected_counts={}, + inspect_transient_failures={"cbx_ready": 2}, + ) + executor.leases["cbx_ready"] = { + "id": "cbx_ready", + "slug": "ready", + "state": "active", + "ready": True, + "serverType": "c7a.24xlarge", + "sshHost": "192.0.2.10", + "sshUser": "crabbox", + "sshPort": "22", + "sshKey": "/tmp/cbx_ready.key", + } + monkeypatch.setattr("scripts.native_eval.fleet.time.sleep", lambda _: None) + + lease = FleetController(config, executor=executor)._inspect_lease( + "cbx_ready", + required=True, + ) + + assert lease is not None + assert lease.lease_id == "cbx_ready" + assert executor.inspect_attempts["cbx_ready"] == 3 + + def _write_archive(path: Path) -> None: source = path.parent / f"{path.stem}-source" source.mkdir() @@ -216,6 +251,7 @@ def __init__( stop_code: int = 0, stop_removes_lease_on_error: bool = False, inspect_errors: set[str] | None = None, + inspect_transient_failures: dict[str, int] | None = None, warmup_capacity_failures: dict[str, int] | None = None, dispatch_failures: dict[str, int] | None = None, ) -> None: @@ -228,6 +264,8 @@ def __init__( self.stop_code = stop_code self.stop_removes_lease_on_error = stop_removes_lease_on_error self.inspect_errors = inspect_errors or set() + self.inspect_transient_failures = inspect_transient_failures or {} + self.inspect_attempts: dict[str, int] = {} self.warmup_capacity_failures = warmup_capacity_failures or {} self.dispatch_failures = dispatch_failures or {} self.warmup_attempts: dict[str, int] = {} @@ -266,6 +304,10 @@ def run( identifier = argv[argv.index("--id") + 1] if identifier in self.inspect_errors: return _result(argv, 1, stderr="broker request timed out") + attempt = self.inspect_attempts.get(identifier, 0) + 1 + self.inspect_attempts[identifier] = attempt + if attempt <= self.inspect_transient_failures.get(identifier, 0): + return _result(argv, 1, stderr="http 500: error code: 1101") lease = next( ( value From 7302a3d442258b4dd3d3aefd8e6c3b28a7a0bb04 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:47:04 +0200 Subject: [PATCH 25/28] fix(eval): preserve zero-result failure checkpoints --- scripts/native_eval/checkpoint_loop.py | 22 ++++++++++++++++++++++ scripts/native_eval/remote_checkpoint.sh | 5 +++++ tests/test_native_eval_checkpoint.py | 8 ++++++++ 3 files changed, 35 insertions(+) diff --git a/scripts/native_eval/checkpoint_loop.py b/scripts/native_eval/checkpoint_loop.py index 0618f24..e8d2660 100644 --- a/scripts/native_eval/checkpoint_loop.py +++ b/scripts/native_eval/checkpoint_loop.py @@ -369,6 +369,28 @@ def run_loop(args: argparse.Namespace) -> int: log_path=log_path, ) except (OSError, subprocess.SubprocessError, tarfile.TarError) as exc: + try: + pull_checkpoint( + client=client, + runner_root=args.runner_root, + remote_root=args.remote_root, + run_label=args.run_label, + raw_dir=raw_dir, + log_path=log_path, + sequence=sequence, + ) + except ( + OSError, + subprocess.SubprocessError, + tarfile.TarError, + ) as checkpoint_exc: + append_checkpoint_log( + log_path, + event="checkpoint_error", + archive_name="", + result_count=last_archive_count, + detail=str(checkpoint_exc).replace("\n", " ")[:500], + ) append_checkpoint_log( log_path, event="final_error", diff --git a/scripts/native_eval/remote_checkpoint.sh b/scripts/native_eval/remote_checkpoint.sh index 38e17a4..4695b6a 100755 --- a/scripts/native_eval/remote_checkpoint.sh +++ b/scripts/native_eval/remote_checkpoint.sh @@ -11,6 +11,7 @@ RUN_LABEL="$2" ARCHIVE_NAME="$3" JOB_DIR="$ROOT/results/jobs/$RUN_LABEL" PROXY_DIR="$ROOT/proxy/$RUN_LABEL" +RUN_STATE_DIR="/tmp/shellbench-runs/$RUN_LABEL" SNAPSHOT_DIR="$(mktemp -d "/tmp/$RUN_LABEL-checkpoint.XXXXXX")" ARCHIVE_PATH="/tmp/$ARCHIVE_NAME" TEMP_ARCHIVE="$ARCHIVE_PATH.tmp" @@ -62,6 +63,10 @@ find "$SNAPSHOT_DIR/results/jobs/$RUN_LABEL" \ cp "$JOB_DIR/run_manifest.json" "$META_DIR/run_manifest.json" 2>/dev/null || true cp /opt/shellbench-native/manifest.json \ "$META_DIR/toolchain_manifest.json" 2>/dev/null || true +if [[ -d "$RUN_STATE_DIR" ]]; then + mkdir -p "$META_DIR/run_state" + cp -a "$RUN_STATE_DIR/." "$META_DIR/run_state/" +fi tar -czf "$TEMP_ARCHIVE" -C "$SNAPSHOT_DIR" . tar -tzf "$TEMP_ARCHIVE" >/dev/null diff --git a/tests/test_native_eval_checkpoint.py b/tests/test_native_eval_checkpoint.py index 6b2a6f8..552e001 100644 --- a/tests/test_native_eval_checkpoint.py +++ b/tests/test_native_eval_checkpoint.py @@ -161,8 +161,16 @@ def test_done_final_copy_failure_logs_recovery_event( def fail_final(**_kwargs: object) -> int: raise subprocess.CalledProcessError(1, ["scp"]) + diagnostic_checkpoints: list[int] = [] + + def pull_diagnostic(**kwargs: object) -> int: + diagnostic_checkpoints.append(int(kwargs["sequence"])) + return 0 + monkeypatch.setattr(checkpoint_loop, "pull_final", fail_final) + monkeypatch.setattr(checkpoint_loop, "pull_checkpoint", pull_diagnostic) assert checkpoint_loop.run_loop(args) == 75 + assert diagnostic_checkpoints == [1] log_path = tmp_path / "logs" / f"{run_label}.checkpoints.log" assert "\tfinal_error\t" in log_path.read_text(encoding="utf-8") From 850ef1fc671a75e5519d9a38fa39499265201ab9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Jul 2026 23:53:14 +0200 Subject: [PATCH 26/28] fix(eval): replace degraded stored leases --- scripts/native_eval/fleet.py | 6 ++++-- tests/test_native_eval_fleet.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 2c07758..d9f9d0d 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -613,12 +613,14 @@ def _ensure_lease(self, entry: dict[str, Any]) -> Lease: required=True, region_hint=str(lease_value.get("region") or ""), ) - except LeaseNotReadyError: + except LeaseNotReadyError as exc: if stored is None: stored = Lease.from_manifest(lease_value) if self._ssh_reachable(stored): return self._detect_region(stored) - raise + raise LeaseUnavailableError( + f"stored lease {identifier} lost readiness and SSH access" + ) from exc slug = str(entry.get("requested_lease_slug") or self._lease_slug(entry["run_label"])) self._store.update( diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 1838c49..7139d3c 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -125,6 +125,33 @@ def inspect(identifier: str, *, required: bool) -> Lease: assert attempts == 3 +def test_stored_lease_losing_readiness_is_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + controller = object.__new__(FleetController) + entry = { + "lease": { + "id": "cbx_degraded", + "slug": "sb-native-degraded", + "host": "192.0.2.20", + "ssh_user": "crabbox", + "ssh_port": 22, + "identity_file": "/tmp/cbx_degraded.key", + "instance_type": "c7a.24xlarge", + "region": "eu-west-1", + } + } + monkeypatch.setattr(controller, "_ssh_reachable", lambda _lease: False) + + def inspect(*_args: object, **_kwargs: object) -> Lease: + raise LeaseNotReadyError("lease is active but not ready") + + monkeypatch.setattr(controller, "_inspect_lease", inspect) + + with pytest.raises(LeaseUnavailableError, match="lost readiness"): + controller._ensure_lease(entry) + + def test_subprocess_timeout_output_is_normalized_to_text( monkeypatch: pytest.MonkeyPatch, ) -> None: From a5d8f1e64114c9f92b1c49de8d50a076feae578f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Jul 2026 01:36:11 +0200 Subject: [PATCH 27/28] fix(eval): canonicalize aggregate task identities --- scripts/native_eval/aggregate.py | 4 +++- tests/test_native_eval_aggregate.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index ebee368..830b10f 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -395,8 +395,10 @@ def _task_identity(result: dict[str, Any], trial_dir: Path) -> tuple[str, str, s task_path = _nested(result, "task_id", "path") task_path = str(task_path or "") task_name = str(result.get("task_name") or "") - if not task_name and task_path: + if task_path: task_name = Path(task_path).name + elif task_name: + task_name = Path(task_name).name trial_name = str(result.get("trial_name") or trial_dir.name) return task_name or trial_name, task_path, trial_name diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index e1773ad..eb1a341 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -175,6 +175,27 @@ def test_aggregate_classifies_harbor_results_and_writes_all_outputs(tmp_path: Pa assert (summaries_dir / "cleaned_leaderboard.md").is_file() +def test_aggregate_uses_task_path_for_canonical_task_name(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + result = _result("display-name", reward=0.0) + result["task_name"] = "computer-use/display-name" + result["task_id"] = {"path": "/benchmark/tasks/canonical-task"} + _write_run( + jobs_root, + "model-rep-1", + expected_task_count=1, + results=[result], + ) + + aggregate(jobs_root, summaries_dir) + + with (summaries_dir / "per_task_results.csv").open(newline="") as handle: + row = next(csv.DictReader(handle)) + assert row["task_name"] == "canonical-task" + assert row["task_path"] == "/benchmark/tasks/canonical-task" + + def test_pair_aggregates_exclude_incomplete_and_infra_dominated_runs(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" From 40c6e928cf70dba3f9e093fb21c58aced3dd0cfa Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Jul 2026 04:49:27 +0200 Subject: [PATCH 28/28] fix(eval): scope parity and report repair sensitivity --- CHANGELOG.md | 4 + scripts/native_eval/aggregate.py | 243 +++++++++++++++++++++++++++- scripts/native_eval/fleet.py | 48 +++++- scripts/native_eval/run_job.py | 40 ++++- tests/test_native_eval_aggregate.py | 191 ++++++++++++++++++++++ tests/test_native_eval_fleet.py | 31 +++- tests/test_native_eval_runner.py | 104 ++++++++++++ 7 files changed, 654 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7d4ab6..f91a99f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,11 @@ readiness probing is stale. - Preserve Hermes JSONL sessions containing literal Unicode line separators. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). +- Scope native Harbor parity claims to the validated harness/model route instead + of applying one fleet-wide boolean. ### Added - Add `gpt-5.6-luna` and `gpt-5.6-terra` to the native evaluation catalog. +- Report repair-overlay task provenance and original-versus-repaired score + sensitivity when the original job is available. diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index 830b10f..43a6efe 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -69,6 +69,12 @@ "RewardFileNotFoundError", } +# Compatibility for manifests created before parity scopes were recorded. +# The pinned calibration validated only this exact native route and task count. +LEGACY_PARITY_VALIDATED_SCOPES = { + ("codex", "gpt55", 20), +} + TIMING_FIELDS = ( "environment_setup", "agent_setup", @@ -139,6 +145,8 @@ "canonical_model_identity", "trajectory_complete", "parity_validated", + "parity_validation_status", + "parity_validation_scope", "eligible", "exclusion_reason", "n_input_tokens", @@ -147,6 +155,23 @@ "cost_usd", ) +REPAIR_SENSITIVITY_FIELDS = ( + "run_label", + "pair_label", + "repetition", + "original_run_label", + "expected_task_count", + "original_task_count", + "repaired_task_count", + "unknown_source_task_count", + "original_result_count", + "repaired_result_count", + "original_score", + "repaired_score", + "score_delta", + "delta_status", +) + INFRA_FIELDS = ( "run_label", "pair_label", @@ -300,6 +325,88 @@ def _manifest_value(manifest: dict[str, Any], *keys: str) -> Any: return None +def _compact_json(value: Any) -> str: + return json.dumps(value, separators=(",", ":"), sort_keys=True) + + +def _scope_matches_value(actual: Any, expected: Any) -> bool: + if isinstance(expected, list): + return any(_scope_matches_value(actual, item) for item in expected) + if isinstance(actual, int) or isinstance(expected, int): + return _integer(actual) == _integer(expected) + return str(actual or "") == str(expected or "") + + +def _parity_validation( + manifest: dict[str, Any], + *, + native_run: bool, +) -> tuple[bool, str, str]: + if not native_run: + return True, "not_required", "" + + validation = manifest.get("parity_validation") + legacy_scope = manifest.get("parity_validation_scope") + if isinstance(validation, dict): + validated = validation.get("validated") is True + scope = validation.get("scope") + if scope is None: + scope = { + key: value + for key, value in validation.items() + if key not in {"validated", "evidence", "notes"} + } + elif isinstance(legacy_scope, dict): + validated = manifest.get("parity_validated") is True + scope = legacy_scope + elif manifest.get("parity_validated") is True: + legacy_scope = ( + str(manifest.get("harness") or ""), + str(manifest.get("model_slug") or ""), + _integer(manifest.get("expected_task_count")), + ) + scope = { + "expected_task_count": legacy_scope[2], + "harness": legacy_scope[0], + "model_slug": legacy_scope[1], + } + scope_text = _compact_json(scope) + if legacy_scope in LEGACY_PARITY_VALIDATED_SCOPES: + return True, "legacy_route_compatible", scope_text + if legacy_scope[:2] == ("codex", "gpt55"): + return False, "legacy_scope_mismatch", scope_text + return False, "legacy_unscoped", "" + else: + return False, "not_validated", "" + + if not isinstance(scope, dict): + return False, "scope_invalid", "" + scope_text = _compact_json(scope) + if not validated: + return False, "not_validated", scope_text + + model_keys = ("model_slug", "model_id", "provider_model_id") + if "harness" not in scope or not any(key in scope for key in model_keys): + return False, "scope_incomplete", scope_text + + comparable = { + "harness": manifest.get("harness"), + "model_slug": manifest.get("model_slug"), + "model_id": manifest.get("model_id"), + "provider_model_id": manifest.get("provider_model_id"), + "execution_mode": manifest.get("execution_mode"), + "task_suite": manifest.get("task_suite"), + "expected_task_count": manifest.get("expected_task_count"), + } + unsupported_keys = set(scope) - set(comparable) + if unsupported_keys: + return False, "scope_unsupported", scope_text + for key, expected in scope.items(): + if not _scope_matches_value(comparable[key], expected): + return False, "scope_mismatch", scope_text + return True, "validated", scope_text + + def _derive_repetition(run_label: str) -> int | None: match = re.search(r"(?:^|[-_])(?:rep(?:etition)?|repeat|r)[-_]?(\d+)$", run_label, re.I) return int(match.group(1)) if match else None @@ -537,7 +644,9 @@ def _matches_expected(row: dict[str, Any], task_name: str, task_path: str) -> bo return bool({task_name, task_path, Path(task_path).name if task_path else ""} & candidates) -def _load_run(run_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: +def _load_run( + run_dir: Path, +) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: run_label = run_dir.name manifest_path = run_dir / "run_manifest.json" manifest, manifest_error = _read_json(manifest_path) @@ -682,7 +791,7 @@ def _load_run(run_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: rows=rows, manifest=manifest, ) - return summary, rows + return summary, rows, manifest def _sum_present(rows: Iterable[dict[str, Any]], key: str) -> int | float | None: @@ -801,7 +910,11 @@ def _summarize_run( for row in trajectory_required_rows ) ) - parity_validated = not native_run or manifest.get("parity_validated") is True + ( + parity_validated, + parity_validation_status, + parity_validation_scope, + ) = _parity_validation(manifest, native_run=native_run) eligible = ( not incomplete and not infra_dominated @@ -855,6 +968,8 @@ def _summarize_run( "canonical_model_identity": canonical_model_identity, "trajectory_complete": trajectory_complete, "parity_validated": parity_validated, + "parity_validation_status": parity_validation_status, + "parity_validation_scope": parity_validation_scope, "eligible": eligible, "exclusion_reason": exclusion_reason, "n_input_tokens": _sum_present(scored_rows, "n_input_tokens"), @@ -929,6 +1044,114 @@ def _public_rows(rows: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: ] +def _repair_source_counts( + manifest: dict[str, Any], + expected_task_count: int, +) -> tuple[int, int, int] | None: + overlay = manifest.get("repair_overlay") + repair_names = manifest.get("repair_task_names") + if not isinstance(overlay, dict): + if not isinstance(repair_names, list) or not repair_names: + return None + repaired = len({str(name) for name in repair_names if name}) + return max(expected_task_count - repaired, 0), repaired, 0 + + lineage = overlay.get("task_lineage") + if isinstance(lineage, list): + source_kinds = [ + str(item.get("source_kind") or "") + for item in lineage + if isinstance(item, dict) + ] + if source_kinds: + original = source_kinds.count("canonical") + repaired = source_kinds.count("repair") + unknown = len(source_kinds) - original - repaired + return original, repaired, unknown + + repair_names = overlay.get("repair_task_names", repair_names) + if isinstance(repair_names, list): + repaired = len({str(name) for name in repair_names if name}) + return max(expected_task_count - repaired, 0), repaired, 0 + return 0, 0, expected_task_count + + +def _repair_overlay_sensitivity( + records: Sequence[tuple[dict[str, Any], dict[str, Any]]], +) -> dict[str, Any]: + summaries = { + str(summary["run_label"]): summary + for summary, _manifest in records + } + runs: list[dict[str, Any]] = [] + for summary, manifest in sorted( + records, + key=lambda record: str(record[0]["run_label"]), + ): + expected_task_count = int(summary["expected_task_count"]) + source_counts = _repair_source_counts(manifest, expected_task_count) + if source_counts is None: + continue + original_tasks, repaired_tasks, unknown_tasks = source_counts + overlay = manifest.get("repair_overlay") + overlay = overlay if isinstance(overlay, dict) else {} + original_run_label = str( + overlay.get("rerun_of_canonical_run") + or manifest.get("rerun_of_canonical_run") + or "" + ) + original_summary = summaries.get(original_run_label) + original_score = ( + float(original_summary["score"]) + if original_summary is not None + else None + ) + repaired_score = float(summary["score"]) + if not original_run_label: + delta_status = "original_label_unavailable" + elif original_summary is None: + delta_status = "original_unavailable" + elif ( + int(original_summary["expected_task_count"]) + != expected_task_count + ): + delta_status = "task_count_mismatch" + original_score = None + else: + delta_status = "available" + runs.append( + { + "run_label": summary["run_label"], + "pair_label": summary["pair_label"], + "repetition": summary["repetition"], + "original_run_label": original_run_label, + "expected_task_count": expected_task_count, + "original_task_count": original_tasks, + "repaired_task_count": repaired_tasks, + "unknown_source_task_count": unknown_tasks, + "original_result_count": ( + original_summary["result_file_count"] + if original_summary is not None + else None + ), + "repaired_result_count": summary["result_file_count"], + "original_score": original_score, + "repaired_score": repaired_score, + "score_delta": ( + repaired_score - original_score + if delta_status == "available" and original_score is not None + else None + ), + "delta_status": delta_status, + } + ) + return { + "schema_version": 1, + "overlay_run_count": len(runs), + "runs": runs, + } + + def _write_csv(path: Path, fieldnames: Sequence[str], rows: Iterable[dict[str, Any]]) -> None: with path.open("w", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore") @@ -998,6 +1221,7 @@ def aggregate(jobs_root: str | Path, summaries_dir: str | Path) -> dict[str, Any run_summaries: list[dict[str, Any]] = [] task_rows: list[dict[str, Any]] = [] + run_records: list[tuple[dict[str, Any], dict[str, Any]]] = [] run_dirs = sorted( path for path in jobs_dir.iterdir() @@ -1006,9 +1230,10 @@ def aggregate(jobs_root: str | Path, summaries_dir: str | Path) -> dict[str, Any and (path / "run_manifest.json").is_file() ) for run_dir in run_dirs: - summary, rows = _load_run(run_dir) + summary, rows, manifest = _load_run(run_dir) run_summaries.append(summary) task_rows.extend(rows) + run_records.append((summary, manifest)) run_summaries.sort(key=lambda run: str(run["run_label"])) task_rows.sort( @@ -1019,6 +1244,7 @@ def aggregate(jobs_root: str | Path, summaries_dir: str | Path) -> dict[str, Any ) ) pair_summaries = _pair_summaries(run_summaries) + repair_sensitivity = _repair_overlay_sensitivity(run_records) public_task_rows = _public_rows(task_rows) infra_rows = [row for row in public_task_rows if row["classification"] == "infra"] @@ -1035,6 +1261,7 @@ def aggregate(jobs_root: str | Path, summaries_dir: str | Path) -> dict[str, Any }, "runs": run_summaries, "pairs": pair_summaries, + "repair_overlay_sensitivity": repair_sensitivity, } _write_csv(output_dir / "aggregate_results.csv", RUN_FIELDS, run_summaries) @@ -1043,6 +1270,14 @@ def aggregate(jobs_root: str | Path, summaries_dir: str | Path) -> dict[str, Any ) _write_csv(output_dir / "per_task_results.csv", PER_TASK_FIELDS, public_task_rows) _write_csv(output_dir / "infra_failures.csv", INFRA_FIELDS, infra_rows) + _write_csv( + output_dir / "repair_overlay_sensitivity.csv", + REPAIR_SENSITIVITY_FIELDS, + repair_sensitivity["runs"], + ) + (output_dir / "repair_overlay_sensitivity.json").write_text( + json.dumps(repair_sensitivity, indent=2, sort_keys=True) + "\n" + ) _write_leaderboard(output_dir / "cleaned_leaderboard.md", pair_summaries) return report diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index d9f9d0d..c7af503 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -226,6 +226,7 @@ class FleetConfig: judge_model_id: str = "" execution_mode: str = "native" parity_validated: bool = False + parity_validated_routes: frozenset[tuple[str, str]] = frozenset() class RunIndexStore: @@ -933,7 +934,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: reasoning_effort=$4 judge_reasoning_effort=$5 parity_validated=$6 -shift 6 +parity_validation_json=$7 +shift 7 mkdir -p "$root/run-logs" stdout="$root/run-logs/$label.stdout.log" stderr="$root/run-logs/$label.stderr.log" @@ -951,6 +953,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: "SHELLBENCH_REASONING_EFFORT=$reasoning_effort" \ "SHELLBENCH_JUDGE_REASONING_EFFORT=$judge_reasoning_effort" \ "SHELLBENCH_PARITY_VALIDATED=$parity_validated" \ + "SHELLBENCH_PARITY_VALIDATION_JSON=$parity_validation_json" \ "$root/runner/scripts/native_eval/remote_run.sh" "$@" \ >"$stdout" 2>"$stderr" None: judge_reasoning_effort = str( entry.get("judge_reasoning_effort") or reasoning_effort ) + parity_validation = "" + if (run.harness, run.model_slug) in self.config.parity_validated_routes: + parity_validation = json.dumps( + { + "scope": { + "harness": run.harness, + "model_slug": run.model_slug, + }, + "validated": True, + }, + separators=(",", ":"), + sort_keys=True, + ) command = self._ssh_command( lease, [ @@ -1004,6 +1020,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: reasoning_effort, judge_reasoning_effort, str(self.config.parity_validated).lower(), + parity_validation, *args, ], ) @@ -1558,6 +1575,24 @@ def _parse_model_values( return parsed +def _parse_parity_routes( + parser: argparse.ArgumentParser, + values: list[str], +) -> frozenset[tuple[str, str]]: + parsed: set[tuple[str, str]] = set() + for raw in values: + harness, separator, model_slug = raw.partition("=") + if not separator or not harness or not model_slug: + parser.error( + f"--parity-validated-route must use HARNESS=MODEL: {raw!r}" + ) + route = (harness, model_slug) + if route in parsed: + parser.error(f"--parity-validated-route repeats {raw!r}") + parsed.add(route) + return frozenset(parsed) + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--run-index", type=Path, required=True) @@ -1602,6 +1637,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--judge-model-id", default="") parser.add_argument("--execution-mode", default="native") parser.add_argument("--parity-validated", action="store_true") + parser.add_argument( + "--parity-validated-route", + action="append", + default=[], + metavar="HARNESS=MODEL", + ) args = parser.parse_args(argv) args.model_max_runs = _parse_model_values( parser, @@ -1618,6 +1659,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: args.model_task_concurrency, "--model-task-concurrency", ) + args.parity_validated_routes = _parse_parity_routes( + parser, + args.parity_validated_route, + ) return args @@ -1653,6 +1698,7 @@ def main(argv: list[str] | None = None) -> int: judge_model_id=args.judge_model_id, execution_mode=args.execution_mode, parity_validated=args.parity_validated, + parity_validated_routes=args.parity_validated_routes, ) try: return FleetController(config).run() diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index 5945370..5658e8d 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -205,6 +205,7 @@ def _run_manifest( tasks: list[TaskSpec], rerun_of_canonical_run: str | None = None, ) -> dict[str, Any]: + parity_validation, parity_validated = _parity_metadata(run) return { "run_label": run.run_label, "harness": run.harness, @@ -245,7 +246,9 @@ def _run_manifest( "observed_model_ids": [], "trajectory_mode": trajectory_mode_for_harness(run.harness), "trajectory_complete": False, - "parity_validated": ( + "parity_validated": parity_validated, + "parity_validation": parity_validation, + "legacy_parity_validated_claim": ( os.environ.get("SHELLBENCH_PARITY_VALIDATED", "").lower() == "true" ), "harbor_reference_commit": os.environ.get( @@ -271,6 +274,41 @@ def _run_manifest( } +def _parity_metadata(run: RunSpec) -> tuple[dict[str, Any] | None, bool]: + raw = os.environ.get("SHELLBENCH_PARITY_VALIDATION_JSON", "").strip() + if not raw: + return None, False + try: + validation = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("SHELLBENCH_PARITY_VALIDATION_JSON is invalid JSON") from exc + if not isinstance(validation, dict): + raise ValueError("SHELLBENCH_PARITY_VALIDATION_JSON must be an object") + scope = validation.get("scope") + if not isinstance(scope, dict): + raise ValueError("parity validation scope must be an object") + if "harness" not in scope or not any( + key in scope for key in ("model_slug", "model_id", "provider_model_id") + ): + raise ValueError("parity validation scope must include harness and model") + + comparable = { + "harness": run.harness, + "model_slug": run.model_slug, + "model_id": run.model_id, + "provider_model_id": run.model_id, + } + unsupported_keys = set(scope) - set(comparable) + if unsupported_keys: + names = ", ".join(sorted(unsupported_keys)) + raise ValueError(f"unsupported parity validation scope keys: {names}") + matches = all( + str(comparable[key]) == str(expected) + for key, expected in scope.items() + ) + return validation, validation.get("validated") is True and matches + + def _git_commit() -> str: configured = os.environ.get("SHELLBENCH_RUNNER_COMMIT", "").strip() if configured: diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index eb1a341..63f2f43 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -20,8 +20,10 @@ def _write_run( model_slug: str = "model", native: bool = False, parity_validated: bool = False, + parity_validation: dict | None = None, leaderboard_eligible: bool | None = None, exclusion_reason: str = "", + manifest_extra: dict | None = None, ) -> None: run_dir = jobs_root / "jobs" / run_label run_dir.mkdir(parents=True) @@ -33,12 +35,22 @@ def _write_run( "expected_task_count": expected_task_count, } if native: + scoped_validation = parity_validation + if parity_validated and scoped_validation is None: + scoped_validation = { + "validated": True, + "scope": { + "harness": harness, + "model_slug": model_slug, + }, + } manifest.update( { "runner": "shellbench-native", "canonical_model_identity": True, "trajectory_mode": "real_harness_events", "parity_validated": parity_validated, + "parity_validation": scoped_validation, } ) if pair_label is not None: @@ -48,6 +60,8 @@ def _write_run( manifest["exclusion_reason"] = exclusion_reason if tasks is not None: manifest["tasks"] = tasks + if manifest_extra is not None: + manifest.update(manifest_extra) (run_dir / "run_manifest.json").write_text(json.dumps(manifest)) for index, result in enumerate(results, start=1): @@ -313,6 +327,102 @@ def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path): assert report["runs"][0]["exclusion_reason"] == "parity_not_validated" +def test_native_run_rejects_legacy_unscoped_parity_claim(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "hermes-gpt56-terra", + expected_task_count=1, + results=[_native_result("a")], + harness="hermes", + model_slug="gpt56-terra", + native=True, + parity_validated=True, + manifest_extra={"parity_validation": None}, + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["parity_validated"] is False + assert run["parity_validation_status"] == "legacy_unscoped" + assert run["eligible"] is False + + +def test_native_run_accepts_known_legacy_codex_gpt55_route(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "codex-gpt55-calibration", + expected_task_count=20, + results=[_native_result(f"task-{index}") for index in range(20)], + harness="codex", + model_slug="gpt55", + native=True, + parity_validated=True, + manifest_extra={"parity_validation": None}, + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["parity_validated"] is True + assert run["parity_validation_status"] == "legacy_route_compatible" + assert run["eligible"] is True + + +def test_native_run_rejects_legacy_codex_gpt55_full_suite(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "codex-gpt55-full-116", + expected_task_count=116, + results=[_native_result(f"task-{index}") for index in range(116)], + harness="codex", + model_slug="gpt55", + native=True, + parity_validated=True, + manifest_extra={"parity_validation": None}, + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["parity_validated"] is False + assert run["parity_validation_status"] == "legacy_scope_mismatch" + assert run["eligible"] is False + assert run["exclusion_reason"] == "parity_not_validated" + + +def test_native_run_rejects_mismatched_parity_scope(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "hermes-gpt55", + expected_task_count=1, + results=[_native_result("a")], + harness="hermes", + model_slug="gpt55", + native=True, + parity_validated=True, + parity_validation={ + "validated": True, + "scope": {"harness": "codex", "model_slug": "gpt55"}, + }, + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["parity_validated"] is False + assert run["parity_validation_status"] == "scope_mismatch" + assert run["eligible"] is False + + def test_native_identity_checks_ignore_pre_agent_infra_failures(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" @@ -606,3 +716,84 @@ def test_unexpected_task_result_is_excluded_from_reward_sum(tmp_path: Path): assert run["score"] == 0.5 assert run["eligible"] is False assert run["infra"] == 1 + + +def test_repair_overlay_sensitivity_reports_available_score_delta(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + original_label = "codex-gpt55-r1" + repaired_label = f"{original_label}-repair-abc" + _write_run( + jobs_root, + original_label, + expected_task_count=2, + results=[_result("a", reward=0.0), _result("b", reward=0.0)], + ) + _write_run( + jobs_root, + repaired_label, + expected_task_count=2, + results=[_result("a", reward=1.0), _result("b", reward=0.0)], + manifest_extra={ + "rerun_of_canonical_run": original_label, + "repair_task_names": ["a"], + "repair_overlay": { + "rerun_of_canonical_run": original_label, + "repair_task_names": ["a"], + "task_lineage": [ + {"task_name": "a", "source_kind": "repair"}, + {"task_name": "b", "source_kind": "canonical"}, + ], + }, + }, + ) + + report = aggregate(jobs_root, summaries_dir) + + sensitivity = report["repair_overlay_sensitivity"] + assert sensitivity["overlay_run_count"] == 1 + row = sensitivity["runs"][0] + assert row["run_label"] == repaired_label + assert row["original_task_count"] == 1 + assert row["repaired_task_count"] == 1 + assert row["original_result_count"] == 2 + assert row["repaired_result_count"] == 2 + assert row["original_score"] == 0.0 + assert row["repaired_score"] == 0.5 + assert row["score_delta"] == 0.5 + assert row["delta_status"] == "available" + + written = json.loads( + (summaries_dir / "repair_overlay_sensitivity.json").read_text() + ) + assert written == sensitivity + with (summaries_dir / "repair_overlay_sensitivity.csv").open( + newline="" + ) as handle: + csv_row = next(csv.DictReader(handle)) + assert csv_row["delta_status"] == "available" + assert csv_row["score_delta"] == "0.5" + + +def test_repair_overlay_sensitivity_marks_missing_original(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-r1-repair-abc", + expected_task_count=2, + results=[_result("a", reward=1.0), _result("b", reward=0.0)], + manifest_extra={ + "rerun_of_canonical_run": "openclaw-gpt55-r1", + "repair_task_names": ["a"], + }, + ) + + report = aggregate(jobs_root, summaries_dir) + + row = report["repair_overlay_sensitivity"]["runs"][0] + assert row["original_task_count"] == 1 + assert row["repaired_task_count"] == 1 + assert row["original_score"] is None + assert row["score_delta"] is None + assert row["delta_status"] == "original_unavailable" diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 7139d3c..d26aaa0 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -303,6 +303,7 @@ def __init__( self.dispatches: list[str] = [] self.dispatch_concurrency: dict[str, int] = {} self.dispatch_arguments: dict[str, list[str]] = {} + self.dispatch_parity_validation: dict[str, str] = {} self.stops: list[str] = [] self._lock = threading.Lock() self._dispatch_condition = threading.Condition(self._lock) @@ -428,7 +429,8 @@ def run( ) remote_command = shlex.split(argv[-1]) dispatch_marker = remote_command.index("fleet-dispatch") - remote_run_args = remote_command[dispatch_marker + 16 :] + parity_validation = remote_command[dispatch_marker + 16] + remote_run_args = remote_command[dispatch_marker + 17 :] with self._dispatch_condition: attempt = self.dispatch_attempts.get(label, 0) + 1 self.dispatch_attempts[label] = attempt @@ -441,6 +443,7 @@ def run( self.dispatches.append(label) self.dispatch_concurrency[label] = int(remote_run_args[10]) self.dispatch_arguments[label] = remote_run_args + self.dispatch_parity_validation[label] = parity_validation self.events.append(("dispatch", label)) self.remote_states[label] = "running" self._dispatch_condition.notify_all() @@ -483,6 +486,7 @@ def _config( model_task_concurrency: dict[str, int] | None = None, warmup_capacity_attempts: int = 12, warmup_capacity_backoff_seconds: float = 0, + parity_validated_routes: frozenset[tuple[str, str]] = frozenset(), ) -> FleetConfig: runner_archive = tmp_path / "runner.tar.gz" task_archive = tmp_path / "tasks.tar.gz" @@ -507,6 +511,7 @@ def _config( checkpoint_poll_seconds=1, warmup_capacity_attempts=warmup_capacity_attempts, warmup_capacity_backoff_seconds=warmup_capacity_backoff_seconds, + parity_validated_routes=parity_validated_routes, ) @@ -546,6 +551,25 @@ def test_controller_runs_bounded_wave_and_stops_after_verified_export( assert "TOPSECRET" not in "\n".join(" ".join(command) for command in executor.commands) +def test_controller_dispatches_only_matching_parity_scope(tmp_path: Path) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [_planned(_run_spec(label))]) + config = _config( + tmp_path, + run_index, + parity_validated_routes=frozenset({("openclaw", "gpt55")}), + ) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + + assert json.loads(executor.dispatch_parity_validation[label]) == { + "scope": {"harness": "openclaw", "model_slug": "gpt55"}, + "validated": True, + } + + def test_controller_accepts_xhigh_reasoning_effort(tmp_path: Path) -> None: label = "openclaw-gpt56-sol-xhigh-full-2-r1-20260728" run = _planned(_run_spec(label, model_slug="gpt56-sol")) @@ -1014,6 +1038,8 @@ def test_parse_args_accepts_repeatable_model_limits(tmp_path: Path) -> None: "--execution-mode", "native", "--parity-validated", + "--parity-validated-route", + "codex=gpt55", "--model-task-concurrency", "fable5=2", ] @@ -1025,6 +1051,7 @@ def test_parse_args_accepts_repeatable_model_limits(tmp_path: Path) -> None: assert args.judge_model_id == "gpt-5.5" assert args.execution_mode == "native" assert args.parity_validated is True + assert args.parity_validated_routes == frozenset({("codex", "gpt55")}) assert args.model_task_concurrency == {"fable5": 2} @@ -1038,6 +1065,8 @@ def test_parse_args_accepts_repeatable_model_limits(tmp_path: Path) -> None: ("--provider-max-runs", ["anthropic=0"]), ("--model-task-concurrency", ["fable5=-1"]), ("--model-task-concurrency", ["fable5=1", "fable5=2"]), + ("--parity-validated-route", ["codex"]), + ("--parity-validated-route", ["codex=gpt55", "codex=gpt55"]), ], ) def test_parse_args_rejects_invalid_model_values( diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 43e388d..34f13f7 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -174,6 +174,8 @@ def test_run_manifest_records_native_audit_metadata( tmp_path: Path, monkeypatch, ) -> None: + monkeypatch.delenv("SHELLBENCH_PARITY_VALIDATED", raising=False) + monkeypatch.delenv("SHELLBENCH_PARITY_VALIDATION_JSON", raising=False) monkeypatch.setenv("SHELLBENCH_EXECUTION_MODE", "native") monkeypatch.setenv("SHELLBENCH_HARBOR_REFERENCE_COMMIT", "harbor-commit") monkeypatch.setenv("SHELLBENCH_JUDGE_MODEL_ID", "gpt-5.5") @@ -213,6 +215,108 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["repair_mode"] is False assert manifest["rerun_of_canonical_run"] is None assert manifest["repair_task_names"] == [] + assert manifest["parity_validated"] is False + assert manifest["parity_validation"] is None + assert manifest["legacy_parity_validated_claim"] is False + + +def test_run_manifest_scopes_parity_to_matching_route( + tmp_path: Path, + monkeypatch, +) -> None: + validation = { + "validated": True, + "scope": {"harness": "codex", "model_slug": "gpt55"}, + "evidence": {"task_count": 20}, + } + monkeypatch.setenv( + "SHELLBENCH_PARITY_VALIDATION_JSON", + json.dumps(validation), + ) + monkeypatch.setenv("SHELLBENCH_PARITY_VALIDATED", "true") + codex_run = RunSpec( + run_label="codex-gpt55-cal20-native-r1", + harness="codex", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="sb-gpt55", + repetition=1, + expected_task_count=20, + run_date="20260727", + ) + hermes_run = RunSpec( + run_label="hermes-gpt55-cal20-native-r1", + harness="hermes", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="sb-gpt55", + repetition=1, + expected_task_count=20, + run_date="20260727", + ) + + codex_manifest = _run_manifest( + codex_run, + public_tasks_commit="tasks-commit", + task_suite_path="combined tasks/tasks", + concurrency=4, + started_at="2026-07-27T00:00:00Z", + tasks_root=tmp_path, + tasks=[], + ) + hermes_manifest = _run_manifest( + hermes_run, + public_tasks_commit="tasks-commit", + task_suite_path="combined tasks/tasks", + concurrency=4, + started_at="2026-07-27T00:00:00Z", + tasks_root=tmp_path, + tasks=[], + ) + + assert codex_manifest["parity_validated"] is True + assert codex_manifest["parity_validation"] == validation + assert codex_manifest["legacy_parity_validated_claim"] is True + assert hermes_manifest["parity_validated"] is False + assert hermes_manifest["parity_validation"] == validation + + +def test_legacy_global_parity_flag_does_not_publish_validated( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_PARITY_VALIDATED", "true") + monkeypatch.delenv("SHELLBENCH_PARITY_VALIDATION_JSON", raising=False) + run = RunSpec( + run_label="openclaw-gpt55-full-116-r1", + harness="openclaw", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="sb-gpt55", + repetition=1, + expected_task_count=116, + run_date="20260728", + ) + + manifest = _run_manifest( + run, + public_tasks_commit="tasks-commit", + task_suite_path="combined tasks/tasks", + concurrency=16, + started_at="2026-07-28T00:00:00Z", + tasks_root=tmp_path, + tasks=[], + ) + + assert manifest["parity_validated"] is False + assert manifest["parity_validation"] is None + assert manifest["legacy_parity_validated_claim"] is True def test_run_manifest_records_targeted_repair_lineage(tmp_path: Path) -> None: