fix(eval): stabilize GPT-5.5 native harness runs

CHANGELOG.md

@@ -4,4 +4,6 @@
 
 ### Fixed
 
+- Record and enforce GPT reasoning effort in native runs, preserve it across
+  reruns, and stabilize OpenClaw and Hermes trace completion.
 - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali).

scripts/native_eval/aggregate.py

@@ -34,6 +34,7 @@
 }
 
 INFRA_EXCEPTION_TYPES = {
+    "AgentSetupError",
     "AgentSetupTimeoutError",
     "DockerStartupError",
     "EnvironmentStartTimeoutError",
@@ -805,6 +806,11 @@ def _summarize_run(
         exclusion_reason = "parity_not_validated"
     else:
         exclusion_reason = ""
+    if manifest.get("leaderboard_eligible") is False:
+        eligible = False
+        exclusion_reason = (
+            str(manifest.get("exclusion_reason") or "") or "explicitly_excluded"
+        )
 
     return {
         "run_label": run_label,

scripts/native_eval/fleet.py

@@ -47,6 +47,7 @@
 RERUN_STATUSES = {"failed", "lease_lost"}
 ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"}
 CLEANUP_STATUSES = {"exported", "stop_pending"}
+REASONING_EFFORTS = {"low", "medium", "high"}
 
 
 class CommandExecutor(Protocol):
@@ -324,6 +325,21 @@ def _validate_plan(self) -> None:
                 raise FleetError(
                     f"{run.run_label} expects {run.expected_task_count}, index expects {expected}"
                 )
+            if run.provider == "openai":
+                reasoning_effort = str(entry.get("reasoning_effort") or "")
+                if reasoning_effort not in REASONING_EFFORTS:
+                    raise FleetError(
+                        f"{run.run_label} must set reasoning_effort to "
+                        "low, medium, or high"
+                    )
+            judge_reasoning_effort = str(
+                entry.get("judge_reasoning_effort") or ""
+            )
+            if judge_reasoning_effort not in REASONING_EFFORTS:
+                raise FleetError(
+                    f"{run.run_label} must set judge_reasoning_effort to "
+                    "low, medium, or high"
+                )
 
     def _prepare_inputs(self) -> None:
         for path, description in (
@@ -764,7 +780,9 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None:
 harbor_reference_commit=$1
 judge_model_id=$2
 execution_mode=$3
-shift 3
+reasoning_effort=$4
+judge_reasoning_effort=$5
+shift 5
 mkdir -p "$root/run-logs"
 stdout="$root/run-logs/$label.stdout.log"
 stderr="$root/run-logs/$label.stderr.log"
@@ -779,6 +797,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None:
   "SHELLBENCH_HARBOR_REFERENCE_COMMIT=$harbor_reference_commit" \
   "SHELLBENCH_JUDGE_MODEL_ID=$judge_model_id" \
   "SHELLBENCH_EXECUTION_MODE=$execution_mode" \
+  "SHELLBENCH_REASONING_EFFORT=$reasoning_effort" \
+  "SHELLBENCH_JUDGE_REASONING_EFFORT=$judge_reasoning_effort" \
   "$root/runner/scripts/native_eval/remote_run.sh" "$@" \
   >"$stdout" 2>"$stderr" </dev/null &
 pid=$!
@@ -803,6 +823,11 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None:
             run.provider,
             run.proxy_model_name,
         ]
+        entry = self._store.get(run.run_label)
+        reasoning_effort = str(entry.get("reasoning_effort") or "")
+        judge_reasoning_effort = str(
+            entry.get("judge_reasoning_effort") or reasoning_effort
+        )
         self._checked(
             self._ssh_command(
                 lease,
@@ -823,6 +848,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None:
                     self.config.harbor_reference_commit,
                     self.config.judge_model_id,
                     self.config.execution_mode,
+                    reasoning_effort,
+                    judge_reasoning_effort,
                     *args,
                 ],
             ),
@@ -1162,6 +1189,8 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None:
         rerun = {
             **run.to_dict(),
             "run_label": label,
+            "reasoning_effort": entry.get("reasoning_effort"),
+            "judge_reasoning_effort": entry.get("judge_reasoning_effort"),
             "attempt": next_attempt,
             "status": "planned",
             "leaderboard_eligible": None,

scripts/native_eval/harnesses.py

@@ -99,11 +99,24 @@ def _openclaw(
     )
     run_command = (
         f"export PATH={_base_path()}; export HOME={home}; "
+        "log=/logs/agent/openclaw.txt; "
         "openclaw agent --local --json --agent main --thinking off "
         f"--model {shlex.quote(model)} "
         "--message \"$(cat /tmp/shellbench-instruction.md)\" "
-        ">/logs/agent/openclaw.txt 2>&1 </dev/null; status=$?; "
-        "cat /logs/agent/openclaw.txt; exit \"$status\""
+        ">\"$log\" 2>&1 </dev/null & pid=$!; "
+        "while kill -0 \"$pid\" 2>/dev/null; do "
+        "if grep -Eq \"\\[agents/agent-command\\] \\[agent\\] run .* "
+        "ended with stopReason=\" \"$log\"; then "
+        "sleep 2; kill \"$pid\" 2>/dev/null || true; sleep 1; "
+        "kill -KILL \"$pid\" 2>/dev/null || true; "
+        "for f in /proc/[0-9]*/comm; do "
+        "name=$(cat \"$f\" 2>/dev/null || true); "
+        "case \"$name\" in openclaw-agent|\"npm exec chrome\"|chrome-devtools) "
+        "child=${f#/proc/}; child=${child%/comm}; "
+        "kill -KILL \"$child\" 2>/dev/null || true;; esac; done; "
+        "wait \"$pid\" 2>/dev/null || true; cat \"$log\"; exit 0; "
+        "fi; sleep 1; done; "
+        "wait \"$pid\"; status=$?; cat \"$log\"; exit \"$status\""
     )
     cleanup = (
         "python3 - <<'PY'\n"
@@ -189,8 +202,12 @@ def _hermes(
     )
     cleanup = (
         f"export PATH={_base_path()}; export HERMES_HOME={home}; "
+        "session_id=$(sed -n 's/^session_id: //p' "
+        "/logs/agent/hermes.txt | tail -1); "
+        "if [ -n \"$session_id\" ]; then "
         "hermes sessions export /logs/agent/hermes-session.jsonl "
-        "--source cli 2>/dev/null || true"
+        "--session-id \"$session_id\" --yes --redact; "
+        "fi"
     )
     return HarnessCommand(
         setup_command=setup,

scripts/native_eval/plan.py

@@ -14,12 +14,16 @@ def write_run_index(
     output: Path,
     public_tasks_commit: str,
     run_date: str,
+    reasoning_effort: str,
+    judge_reasoning_effort: str,
 ) -> list[dict[str, object]]:
     tasks = validate_suite(tasks_root)
     runs = build_matrix_plan(len(tasks), run_date=run_date)
     entries = [
         {
             **run.to_dict(),
+            "reasoning_effort": reasoning_effort,
+            "judge_reasoning_effort": judge_reasoning_effort,
             "attempt": 0,
             "status": "planned",
             "leaderboard_eligible": None,
@@ -49,12 +53,25 @@ def main() -> None:
     parser.add_argument("--output", type=Path, required=True)
     parser.add_argument("--public-tasks-commit", required=True)
     parser.add_argument("--run-date", required=True)
+    parser.add_argument(
+        "--reasoning-effort",
+        choices=("low", "medium", "high"),
+        required=True,
+    )
+    parser.add_argument(
+        "--judge-reasoning-effort",
+        choices=("low", "medium", "high"),
+    )
     args = parser.parse_args()
     entries = write_run_index(
         tasks_root=args.tasks_root,
         output=args.output,
         public_tasks_commit=args.public_tasks_commit,
         run_date=args.run_date,
+        reasoning_effort=args.reasoning_effort,
+        judge_reasoning_effort=(
+            args.judge_reasoning_effort or args.reasoning_effort
+        ),
     )
     print(f"wrote {len(entries)} planned runs to {args.output}")
 

scripts/native_eval/proxy.py

@@ -1,12 +1,18 @@
 from __future__ import annotations
 
 import json
+import os
 from pathlib import Path
 
 from scripts.native_eval.models import LITELLM_VERSION, MODELS
 
 
 def write_proxy_config(path: Path) -> None:
+    reasoning_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip()
+    if reasoning_effort not in {"low", "medium", "high"}:
+        raise ValueError(
+            "SHELLBENCH_REASONING_EFFORT must be low, medium, or high"
+        )
     model_list = []
     for model in MODELS:
         litellm_params = {
@@ -15,6 +21,7 @@ def write_proxy_config(path: Path) -> None:
         }
         if model.slug in {"gpt55", "gpt56-sol"}:
             litellm_params["additional_drop_params"] = ["temperature"]
+            litellm_params["reasoning_effort"] = reasoning_effort
         model_list.append(
             {
                 "model_name": model.proxy_model_name,
@@ -44,6 +51,7 @@ def write_proxy_config(path: Path) -> None:
         },
         "shellbench_native": {
             "litellm_version": LITELLM_VERSION,
+            "reasoning_effort": reasoning_effort,
             "models": [
                 {
                     "slug": model.slug,

scripts/native_eval/run_job.py

@@ -244,6 +244,10 @@ def _run_manifest(
             "SHELLBENCH_HARBOR_REFERENCE_COMMIT"
         ),
         "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"),
+        "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"),
+        "judge_reasoning_effort": os.environ.get(
+            "SHELLBENCH_JUDGE_REASONING_EFFORT"
+        ),
         "runner_commit": _git_commit(),
         "runner_patch_hash": _runner_patch_hash(),
         "public_tasks_commit": public_tasks_commit,

scripts/native_eval/runtime.py

@@ -31,6 +31,10 @@ class AgentSetupTimeoutError(NativeEvalError):
     pass
 
 
+class AgentSetupError(NativeEvalError):
+    pass
+
+
 class NonZeroAgentExitCodeError(NativeEvalError):
     pass
 
@@ -215,7 +219,26 @@ async def _discover_workdir(self) -> None:
                 self.container_id,
             ]
         )
-        self.workdir = output.strip() or "/app"
+        configured = output.strip()
+        if configured:
+            self.workdir = configured
+            return
+        self.workdir = (
+            await capture_process(
+                [
+                    "docker",
+                    "exec",
+                    self.container_id,
+                    "sh",
+                    "-lc",
+                    (
+                        "if [ -d /app ]; then printf /app; "
+                        "elif [ -d /workspace ]; then printf /workspace; "
+                        "else pwd; fi"
+                    ),
+                ]
+            )
+        ).strip() or "/"
 
     async def copy_instruction(self, instruction: str) -> None:
         local = self.trial_dir / "instruction.md"
@@ -444,7 +467,7 @@ async def run_trial(
                 stderr_path=trial_dir / "agent" / "setup-stderr.txt",
             )
             if setup.returncode:
-                raise NativeEvalError(f"Agent setup exited {setup.returncode}")
+                raise AgentSetupError(f"Agent setup exited {setup.returncode}")
         except TimeoutError as exc:
             raise AgentSetupTimeoutError("Agent setup timed out after 300s") from exc
         finally:

tests/test_native_eval_aggregate.py

@@ -20,6 +20,8 @@ def _write_run(
     model_slug: str = "model",
     native: bool = False,
     parity_validated: bool = False,
+    leaderboard_eligible: bool | None = None,
+    exclusion_reason: str = "",
 ) -> None:
     run_dir = jobs_root / "jobs" / run_label
     run_dir.mkdir(parents=True)
@@ -41,6 +43,9 @@ def _write_run(
         )
     if pair_label is not None:
         manifest["pair_label"] = pair_label
+    if leaderboard_eligible is not None:
+        manifest["leaderboard_eligible"] = leaderboard_eligible
+        manifest["exclusion_reason"] = exclusion_reason
     if tasks is not None:
         manifest["tasks"] = tasks
     (run_dir / "run_manifest.json").write_text(json.dumps(manifest))
@@ -287,6 +292,40 @@ def test_native_run_requires_parity_validation_for_leaderboard(tmp_path: Path):
     assert report["runs"][0]["exclusion_reason"] == "parity_not_validated"
 
 
+def test_manifest_can_explicitly_exclude_complete_run(tmp_path: Path):
+    jobs_root = tmp_path / "native"
+    summaries_dir = tmp_path / "summaries"
+    _write_run(
+        jobs_root,
+        "openclaw-gpt55-rerun1",
+        expected_task_count=1,
+        results=[_result("a", reward=1.0)],
+        leaderboard_eligible=False,
+        exclusion_reason="manual_intervention",
+    )
+
+    report = aggregate(jobs_root, summaries_dir)
+
+    assert report["runs"][0]["eligible"] is False
+    assert report["runs"][0]["exclusion_reason"] == "manual_intervention"
+
+
+def test_agent_setup_error_is_classified_as_infrastructure(tmp_path: Path):
+    jobs_root = tmp_path / "native"
+    summaries_dir = tmp_path / "summaries"
+    _write_run(
+        jobs_root,
+        "openclaw-gpt55-setup-failure",
+        expected_task_count=1,
+        results=[_result("a", exception_type="AgentSetupError")],
+    )
+
+    report = aggregate(jobs_root, summaries_dir)
+
+    assert report["runs"][0]["infra"] == 1
+    assert report["runs"][0]["agent_exits"] == 0
+
+
 def test_unknown_exception_is_agent_exit_and_gateway_signature_is_excluded(
     tmp_path: Path,
 ):

tests/test_native_eval_fleet.py

@@ -59,6 +59,8 @@ def _write_index(
 def _planned(run: RunSpec) -> dict[str, object]:
     return {
         **run.to_dict(),
+        "reasoning_effort": "high",
+        "judge_reasoning_effort": "high",
         "attempt": 0,
         "status": "planned",
         "leaderboard_eligible": None,
@@ -249,7 +251,7 @@ def run(
             )
             remote_command = shlex.split(argv[-1])
             dispatch_marker = remote_command.index("fleet-dispatch")
-            remote_run_args = remote_command[dispatch_marker + 13 :]
+            remote_run_args = remote_command[dispatch_marker + 15 :]
             with self._dispatch_condition:
                 self.dispatches.append(label)
                 self.dispatch_concurrency[label] = int(remote_run_args[10])
@@ -774,6 +776,8 @@ def test_failed_run_is_preserved_and_suffixed_rerun_completes(
     assert runs[0]["final_result_count"] == 1
     assert runs[1]["status"] == "completed"
     assert runs[1]["rerun_of"] == base
+    assert runs[1]["reasoning_effort"] == "high"
+    assert runs[1]["judge_reasoning_effort"] == "high"
     assert executor.dispatches == [base, rerun]
     assert (config.local_root / "raw" / f"{base}-final-artifacts.tar.gz").is_file()
     assert (config.local_root / "raw" / f"{rerun}-final-artifacts.tar.gz").is_file()

tests/test_native_eval_proxy.py

@@ -4,7 +4,11 @@
 from scripts.native_eval.proxy import write_proxy_config
 
 
-def test_openai_proxy_models_drop_unsupported_temperature(tmp_path: Path):
+def test_openai_proxy_models_enforce_reasoning_effort(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high")
     config_path = tmp_path / "proxy.json"
 
     write_proxy_config(config_path)
@@ -17,3 +21,6 @@ def test_openai_proxy_models_drop_unsupported_temperature(tmp_path: Path):
     assert models["gpt-5.6-sol"]["litellm_params"]["additional_drop_params"] == [
         "temperature"
     ]
+    assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == "high"
+    assert models["gpt-5.6-sol"]["litellm_params"]["reasoning_effort"] == "high"
+    assert config["shellbench_native"]["reasoning_effort"] == "high"