diff --git a/CHANGELOG.md b/CHANGELOG.md index d1432a8..b08dd37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,4 +4,6 @@ ### Fixed +- Record and enforce GPT reasoning effort in native runs, preserve it across + reruns, and stabilize OpenClaw and Hermes trace completion. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index 8a656cc..1cdf606 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -34,6 +34,7 @@ } INFRA_EXCEPTION_TYPES = { + "AgentSetupError", "AgentSetupTimeoutError", "DockerStartupError", "EnvironmentStartTimeoutError", @@ -805,6 +806,11 @@ def _summarize_run( exclusion_reason = "parity_not_validated" else: exclusion_reason = "" + if manifest.get("leaderboard_eligible") is False: + eligible = False + exclusion_reason = ( + str(manifest.get("exclusion_reason") or "") or "explicitly_excluded" + ) return { "run_label": run_label, diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index f30866a..142e882 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -47,6 +47,7 @@ RERUN_STATUSES = {"failed", "lease_lost"} ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} +REASONING_EFFORTS = {"low", "medium", "high"} class CommandExecutor(Protocol): @@ -324,6 +325,21 @@ def _validate_plan(self) -> None: raise FleetError( f"{run.run_label} expects {run.expected_task_count}, index expects {expected}" ) + if run.provider == "openai": + reasoning_effort = str(entry.get("reasoning_effort") or "") + if reasoning_effort not in REASONING_EFFORTS: + raise FleetError( + f"{run.run_label} must set reasoning_effort to " + "low, medium, or high" + ) + judge_reasoning_effort = str( + entry.get("judge_reasoning_effort") or "" + ) + if judge_reasoning_effort not in REASONING_EFFORTS: + raise FleetError( + f"{run.run_label} must set judge_reasoning_effort to " + "low, medium, or high" + ) def _prepare_inputs(self) -> None: for path, description in ( @@ -764,7 +780,9 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: harbor_reference_commit=$1 judge_model_id=$2 execution_mode=$3 -shift 3 +reasoning_effort=$4 +judge_reasoning_effort=$5 +shift 5 mkdir -p "$root/run-logs" stdout="$root/run-logs/$label.stdout.log" stderr="$root/run-logs/$label.stderr.log" @@ -779,6 +797,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: "SHELLBENCH_HARBOR_REFERENCE_COMMIT=$harbor_reference_commit" \ "SHELLBENCH_JUDGE_MODEL_ID=$judge_model_id" \ "SHELLBENCH_EXECUTION_MODE=$execution_mode" \ + "SHELLBENCH_REASONING_EFFORT=$reasoning_effort" \ + "SHELLBENCH_JUDGE_REASONING_EFFORT=$judge_reasoning_effort" \ "$root/runner/scripts/native_eval/remote_run.sh" "$@" \ >"$stdout" 2>"$stderr" None: run.provider, run.proxy_model_name, ] + entry = self._store.get(run.run_label) + reasoning_effort = str(entry.get("reasoning_effort") or "") + judge_reasoning_effort = str( + entry.get("judge_reasoning_effort") or reasoning_effort + ) self._checked( self._ssh_command( lease, @@ -823,6 +848,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: self.config.harbor_reference_commit, self.config.judge_model_id, self.config.execution_mode, + reasoning_effort, + judge_reasoning_effort, *args, ], ), @@ -1162,6 +1189,8 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: rerun = { **run.to_dict(), "run_label": label, + "reasoning_effort": entry.get("reasoning_effort"), + "judge_reasoning_effort": entry.get("judge_reasoning_effort"), "attempt": next_attempt, "status": "planned", "leaderboard_eligible": None, diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 519487f..4de7025 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -99,11 +99,24 @@ def _openclaw( ) run_command = ( f"export PATH={_base_path()}; export HOME={home}; " + "log=/logs/agent/openclaw.txt; " "openclaw agent --local --json --agent main --thinking off " f"--model {shlex.quote(model)} " "--message \"$(cat /tmp/shellbench-instruction.md)\" " - ">/logs/agent/openclaw.txt 2>&1 \"$log\" 2>&1 /dev/null; do " + "if grep -Eq \"\\[agents/agent-command\\] \\[agent\\] run .* " + "ended with stopReason=\" \"$log\"; then " + "sleep 2; kill \"$pid\" 2>/dev/null || true; sleep 1; " + "kill -KILL \"$pid\" 2>/dev/null || true; " + "for f in /proc/[0-9]*/comm; do " + "name=$(cat \"$f\" 2>/dev/null || true); " + "case \"$name\" in openclaw-agent|\"npm exec chrome\"|chrome-devtools) " + "child=${f#/proc/}; child=${child%/comm}; " + "kill -KILL \"$child\" 2>/dev/null || true;; esac; done; " + "wait \"$pid\" 2>/dev/null || true; cat \"$log\"; exit 0; " + "fi; sleep 1; done; " + "wait \"$pid\"; status=$?; cat \"$log\"; exit \"$status\"" ) cleanup = ( "python3 - <<'PY'\n" @@ -189,8 +202,12 @@ def _hermes( ) cleanup = ( f"export PATH={_base_path()}; export HERMES_HOME={home}; " + "session_id=$(sed -n 's/^session_id: //p' " + "/logs/agent/hermes.txt | tail -1); " + "if [ -n \"$session_id\" ]; then " "hermes sessions export /logs/agent/hermes-session.jsonl " - "--source cli 2>/dev/null || true" + "--session-id \"$session_id\" --yes --redact; " + "fi" ) return HarnessCommand( setup_command=setup, diff --git a/scripts/native_eval/plan.py b/scripts/native_eval/plan.py index 1fdc1ad..d5544f2 100644 --- a/scripts/native_eval/plan.py +++ b/scripts/native_eval/plan.py @@ -14,12 +14,16 @@ def write_run_index( output: Path, public_tasks_commit: str, run_date: str, + reasoning_effort: str, + judge_reasoning_effort: str, ) -> list[dict[str, object]]: tasks = validate_suite(tasks_root) runs = build_matrix_plan(len(tasks), run_date=run_date) entries = [ { **run.to_dict(), + "reasoning_effort": reasoning_effort, + "judge_reasoning_effort": judge_reasoning_effort, "attempt": 0, "status": "planned", "leaderboard_eligible": None, @@ -49,12 +53,25 @@ def main() -> None: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--public-tasks-commit", required=True) parser.add_argument("--run-date", required=True) + parser.add_argument( + "--reasoning-effort", + choices=("low", "medium", "high"), + required=True, + ) + parser.add_argument( + "--judge-reasoning-effort", + choices=("low", "medium", "high"), + ) args = parser.parse_args() entries = write_run_index( tasks_root=args.tasks_root, output=args.output, public_tasks_commit=args.public_tasks_commit, run_date=args.run_date, + reasoning_effort=args.reasoning_effort, + judge_reasoning_effort=( + args.judge_reasoning_effort or args.reasoning_effort + ), ) print(f"wrote {len(entries)} planned runs to {args.output}") diff --git a/scripts/native_eval/proxy.py b/scripts/native_eval/proxy.py index 5f26b74..96b19c0 100644 --- a/scripts/native_eval/proxy.py +++ b/scripts/native_eval/proxy.py @@ -1,12 +1,18 @@ from __future__ import annotations import json +import os from pathlib import Path from scripts.native_eval.models import LITELLM_VERSION, MODELS def write_proxy_config(path: Path) -> None: + reasoning_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip() + if reasoning_effort not in {"low", "medium", "high"}: + raise ValueError( + "SHELLBENCH_REASONING_EFFORT must be low, medium, or high" + ) model_list = [] for model in MODELS: litellm_params = { @@ -15,6 +21,7 @@ def write_proxy_config(path: Path) -> None: } if model.slug in {"gpt55", "gpt56-sol"}: litellm_params["additional_drop_params"] = ["temperature"] + litellm_params["reasoning_effort"] = reasoning_effort model_list.append( { "model_name": model.proxy_model_name, @@ -44,6 +51,7 @@ def write_proxy_config(path: Path) -> None: }, "shellbench_native": { "litellm_version": LITELLM_VERSION, + "reasoning_effort": reasoning_effort, "models": [ { "slug": model.slug, diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index 0b1f8e0..3b64fa8 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -244,6 +244,10 @@ def _run_manifest( "SHELLBENCH_HARBOR_REFERENCE_COMMIT" ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), + "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), + "judge_reasoning_effort": os.environ.get( + "SHELLBENCH_JUDGE_REASONING_EFFORT" + ), "runner_commit": _git_commit(), "runner_patch_hash": _runner_patch_hash(), "public_tasks_commit": public_tasks_commit, diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index be2c475..088e786 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -31,6 +31,10 @@ class AgentSetupTimeoutError(NativeEvalError): pass +class AgentSetupError(NativeEvalError): + pass + + class NonZeroAgentExitCodeError(NativeEvalError): pass @@ -215,7 +219,26 @@ async def _discover_workdir(self) -> None: self.container_id, ] ) - self.workdir = output.strip() or "/app" + configured = output.strip() + if configured: + self.workdir = configured + return + self.workdir = ( + await capture_process( + [ + "docker", + "exec", + self.container_id, + "sh", + "-lc", + ( + "if [ -d /app ]; then printf /app; " + "elif [ -d /workspace ]; then printf /workspace; " + "else pwd; fi" + ), + ] + ) + ).strip() or "/" async def copy_instruction(self, instruction: str) -> None: local = self.trial_dir / "instruction.md" @@ -444,7 +467,7 @@ async def run_trial( stderr_path=trial_dir / "agent" / "setup-stderr.txt", ) if setup.returncode: - raise NativeEvalError(f"Agent setup exited {setup.returncode}") + raise AgentSetupError(f"Agent setup exited {setup.returncode}") except TimeoutError as exc: raise AgentSetupTimeoutError("Agent setup timed out after 300s") from exc finally: diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index c8d17d0..2623fb7 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -20,6 +20,8 @@ def _write_run( model_slug: str = "model", native: bool = False, parity_validated: bool = False, + leaderboard_eligible: bool | None = None, + exclusion_reason: str = "", ) -> None: run_dir = jobs_root / "jobs" / run_label run_dir.mkdir(parents=True) @@ -41,6 +43,9 @@ def _write_run( ) if pair_label is not None: manifest["pair_label"] = pair_label + if leaderboard_eligible is not None: + manifest["leaderboard_eligible"] = leaderboard_eligible + manifest["exclusion_reason"] = exclusion_reason if tasks is not None: manifest["tasks"] = tasks (run_dir / "run_manifest.json").write_text(json.dumps(manifest)) @@ -287,6 +292,40 @@ def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path): assert report["runs"][0]["exclusion_reason"] == "parity_not_validated" +def test_manifest_can_explicitly_exclude_complete_run(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-rerun1", + expected_task_count=1, + results=[_result("a", reward=1.0)], + leaderboard_eligible=False, + exclusion_reason="manual_intervention", + ) + + report = aggregate(jobs_root, summaries_dir) + + assert report["runs"][0]["eligible"] is False + assert report["runs"][0]["exclusion_reason"] == "manual_intervention" + + +def test_agent_setup_error_is_classified_as_infrastructure(tmp_path: Path): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-setup-failure", + expected_task_count=1, + results=[_result("a", exception_type="AgentSetupError")], + ) + + report = aggregate(jobs_root, summaries_dir) + + assert report["runs"][0]["infra"] == 1 + assert report["runs"][0]["agent_exits"] == 0 + + def test_unknown_exception_is_agent_exit_and_gateway_signature_is_excluded( tmp_path: Path, ): diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 9ecba5d..212a1a4 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -59,6 +59,8 @@ def _write_index( def _planned(run: RunSpec) -> dict[str, object]: return { **run.to_dict(), + "reasoning_effort": "high", + "judge_reasoning_effort": "high", "attempt": 0, "status": "planned", "leaderboard_eligible": None, @@ -249,7 +251,7 @@ def run( ) remote_command = shlex.split(argv[-1]) dispatch_marker = remote_command.index("fleet-dispatch") - remote_run_args = remote_command[dispatch_marker + 13 :] + remote_run_args = remote_command[dispatch_marker + 15 :] with self._dispatch_condition: self.dispatches.append(label) self.dispatch_concurrency[label] = int(remote_run_args[10]) @@ -774,6 +776,8 @@ def test_failed_run_is_preserved_and_suffixed_rerun_completes( assert runs[0]["final_result_count"] == 1 assert runs[1]["status"] == "completed" assert runs[1]["rerun_of"] == base + assert runs[1]["reasoning_effort"] == "high" + assert runs[1]["judge_reasoning_effort"] == "high" assert executor.dispatches == [base, rerun] assert (config.local_root / "raw" / f"{base}-final-artifacts.tar.gz").is_file() assert (config.local_root / "raw" / f"{rerun}-final-artifacts.tar.gz").is_file() diff --git a/tests/test_native_eval_proxy.py b/tests/test_native_eval_proxy.py index f2866d4..ddb86c9 100644 --- a/tests/test_native_eval_proxy.py +++ b/tests/test_native_eval_proxy.py @@ -4,7 +4,11 @@ from scripts.native_eval.proxy import write_proxy_config -def test_openai_proxy_models_drop_unsupported_temperature(tmp_path: Path): +def test_openai_proxy_models_enforce_reasoning_effort( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high") config_path = tmp_path / "proxy.json" write_proxy_config(config_path) @@ -17,3 +21,6 @@ def test_openai_proxy_models_drop_unsupported_temperature(tmp_path: Path): assert models["gpt-5.6-sol"]["litellm_params"]["additional_drop_params"] == [ "temperature" ] + assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == "high" + assert models["gpt-5.6-sol"]["litellm_params"]["reasoning_effort"] == "high" + assert config["shellbench_native"]["reasoning_effort"] == "high" diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 49be82c..acc9a9e 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import json import tarfile from argparse import Namespace @@ -16,13 +17,16 @@ RunSpec, build_matrix_plan, ) +from scripts.native_eval import plan as native_plan from scripts.native_eval.proxy import write_proxy_config from scripts.native_eval.run_job import _git_commit, _run_manifest, build_run_spec from scripts.native_eval.runtime import ( + DockerTaskEnvironment, collect_agent_metrics, read_reward, write_agent_trajectory, ) +from scripts.native_eval import runtime as native_runtime from scripts.native_eval.tasks import TaskSpec, validate_suite @@ -36,6 +40,26 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: assert {run.repetition for run in plan} == {1, 2, 3} +def test_run_index_records_agent_and_judge_reasoning( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setattr(native_plan, "validate_suite", lambda _root: [object()]) + + entries = native_plan.write_run_index( + tasks_root=tmp_path, + output=tmp_path / "run-index.json", + public_tasks_commit="tasks-commit", + run_date="20260728", + reasoning_effort="high", + judge_reasoning_effort="high", + ) + + assert len(entries) == 72 + assert {entry["reasoning_effort"] for entry in entries} == {"high"} + assert {entry["judge_reasoning_effort"] for entry in entries} == {"high"} + + def test_task_loader_accepts_rich_manifest_and_compose(tmp_path: Path) -> None: task_dir = tmp_path / "browser-task" (task_dir / "environment").mkdir(parents=True) @@ -102,7 +126,11 @@ def test_validate_suite_reports_missing_required_file(tmp_path: Path) -> None: raise AssertionError("expected suite validation to fail") -def test_proxy_config_pins_only_requested_upstreams(tmp_path: Path) -> None: +def test_proxy_config_pins_only_requested_upstreams( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high") path = tmp_path / "config.json" write_proxy_config(path) config = json.loads(path.read_text(encoding="utf-8")) @@ -117,6 +145,11 @@ def test_proxy_config_pins_only_requested_upstreams(tmp_path: Path) -> None: assert "OPENAI_API_KEY" in serialized assert "ANTHROPIC_API_KEY" in serialized assert "sk-" not in serialized + assert all( + item["litellm_params"].get("reasoning_effort") == "high" + for item in config["model_list"] + if item["model_info"]["provider"] == "openai" + ) def test_run_manifest_records_native_audit_metadata( @@ -126,6 +159,8 @@ def test_run_manifest_records_native_audit_metadata( monkeypatch.setenv("SHELLBENCH_EXECUTION_MODE", "native") monkeypatch.setenv("SHELLBENCH_HARBOR_REFERENCE_COMMIT", "harbor-commit") monkeypatch.setenv("SHELLBENCH_JUDGE_MODEL_ID", "gpt-5.5") + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high") + monkeypatch.setenv("SHELLBENCH_JUDGE_REASONING_EFFORT", "high") run = RunSpec( run_label="openclaw-gpt55-full-2-r1-20260727", harness="openclaw", @@ -155,6 +190,8 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["trajectory_mode"] == "unsupported" assert manifest["canonical_model_identity"] is None assert manifest["provider_model_id"] == "gpt-5.5" + assert manifest["reasoning_effort"] == "high" + assert manifest["judge_reasoning_effort"] == "high" def test_run_spec_preserves_explicit_planned_identity() -> None: @@ -207,6 +244,9 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "sb-opus5" not in command.run_command assert "OPENROUTER_API_KEY" not in command.env assert 'exit "$status"' in command.run_command + if harness.name == "openclaw": + assert "ended with stopReason=" in command.run_command + assert "--thinking off" in command.run_command def test_hermes_uses_named_local_proxy_provider() -> None: @@ -236,6 +276,36 @@ def test_hermes_uses_named_local_proxy_provider() -> None: assert command.env == {"SHELLBENCH_PROXY_KEY": "local-proxy-key"} assert "custom:shellbench" in command.setup_command assert "http://host.docker.internal:4000/v1" in command.setup_command + assert '--session-id "$session_id" --yes --redact' in command.cleanup_command + + +def test_workdir_falls_back_to_existing_container_directory( + tmp_path: Path, + monkeypatch, +) -> None: + commands: list[list[str]] = [] + + async def fake_capture(command: list[str]) -> str: + commands.append(command) + return "" if command[1] == "inspect" else "/workspace" + + monkeypatch.setattr(native_runtime, "capture_process", fake_capture) + environment = DockerTaskEnvironment( + task=object(), # type: ignore[arg-type] + trial_dir=tmp_path, + container_name="trial", + project_name="trial", + toolchain_root=tmp_path, + container_id="container-id", + ) + + asyncio.run(environment._discover_workdir()) + + assert environment.workdir == "/workspace" + assert commands[1][-1] == ( + "if [ -d /app ]; then printf /app; " + "elif [ -d /workspace ]; then printf /workspace; else pwd; fi" + ) def test_claude_code_selects_canonical_model_explicitly() -> None: