fix: apply native reasoning effort to harnesses

scripts/native_eval/harnesses.py

@@ -532,6 +532,7 @@ def _openclaw(
 ) -> HarnessCommand:
     provider = "openai"
     model = f"{provider}/{run.model_id}"
+    thinking = run.reasoning_effort or "off"
     home = "/tmp/shellbench-openclaw"
     audit_plugin_root = f"{home}/.openclaw/shellbench-audit"
     gateway_token = secrets.token_urlsafe(32)
@@ -616,7 +617,8 @@ def _openclaw(
         'kill "$gateway_pid" 2>/dev/null || true; '
         'wait "$gateway_pid" 2>/dev/null || true; '
         'cat "$gateway_log" >&2; exit 70; fi; '
-        "openclaw agent --json --agent main --thinking off "
+        "openclaw agent --json --agent main "
+        f"--thinking {shlex.quote(thinking)} "
         f"--model {shlex.quote(model)} "
         '--message "$(cat /tmp/shellbench-instruction.md)" '
         '>"$log" 2>&1 </dev/null & pid=$!; '
@@ -725,6 +727,9 @@ def _hermes(
 ) -> HarnessCommand:
     home = "/tmp/shellbench-hermes"
     provider_name = "custom:shellbench"
+    agent_config: dict[str, object] = {"max_turns": 90}
+    if run.reasoning_effort:
+        agent_config["reasoning_effort"] = run.reasoning_effort
     config: dict[str, object] = {
         "model": {
             "default": run.model_id,
@@ -741,7 +746,7 @@ def _hermes(
             }
         },
         "toolsets": ["hermes-cli"],
-        "agent": {"max_turns": 90},
+        "agent": agent_config,
         "memory": {"memory_enabled": False, "user_profile_enabled": False},
         "compression": {"enabled": True, "threshold": 0.85},
         "terminal": {"backend": "local", "timeout": 180},
@@ -817,11 +822,17 @@ def _codex(
         f'printf %s {auth_json} > "$CODEX_HOME/auth.json"; '
         f'printf %s {config_text} > "$CODEX_HOME/config.toml"'
     )
+    reasoning_override = (
+        f"-c {shlex.quote(f'model_reasoning_effort={json.dumps(run.reasoning_effort)}')} "
+        if run.reasoning_effort
+        else ""
+    )
     run_command = (
         f"export PATH={_base_path()}; export CODEX_HOME={home}; "
         "codex exec --dangerously-bypass-approvals-and-sandbox "
         "--skip-git-repo-check "
         f"--model {shlex.quote(run.model_id)} "
+        f"{reasoning_override}"
         "--json --enable unified_exec -- "
         '"$(cat /tmp/shellbench-instruction.md)" '
         ">/logs/agent/codex.txt 2>/logs/agent/codex-stderr.txt "
@@ -871,11 +882,16 @@ def _claude_code(
         'mkdir -p "$CLAUDE_CONFIG_DIR/debug" "$CLAUDE_CONFIG_DIR/projects/-app"; '
         f'printf %s {mcp_json} > "$CLAUDE_CONFIG_DIR/.claude.json"'
     )
+    # Claude Code names its top effort level "max"; the other native harnesses
+    # use ShellBench's canonical "xhigh" spelling.
+    claude_effort = "max" if run.reasoning_effort == "xhigh" else run.reasoning_effort
+    effort_option = f"--effort {shlex.quote(claude_effort)} " if claude_effort else ""
     run_command = (
         f"export PATH={_base_path()}; export CLAUDE_CONFIG_DIR={home}; "
         "claude --verbose --output-format=stream-json "
         "--permission-mode=bypassPermissions --print "
         f"--model {shlex.quote(run.model_id)} "
+        f"{effort_option}"
         '"$(cat /tmp/shellbench-instruction.md)" '
         ">/logs/agent/claude-code.txt 2>&1 </dev/null; status=$?; "
         'cat /logs/agent/claude-code.txt; exit "$status"'

scripts/native_eval/models.py

@@ -32,6 +32,7 @@ class RunSpec:
     repetition: int
     expected_task_count: int
     run_date: str
+    reasoning_effort: str | None = None
 
     def to_dict(self) -> dict[str, object]:
         return asdict(self)
@@ -162,6 +163,7 @@ def build_matrix_plan(
                         repetition=repetition,
                         expected_task_count=expected_task_count,
                         run_date=stamp,
+                        reasoning_effort=reasoning_effort,
                     )
                 )
     return plan

scripts/native_eval/run_job.py

@@ -270,7 +270,7 @@ def _run_manifest(
             "SHELLBENCH_HARBOR_REFERENCE_COMMIT"
         ),
         "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"),
-        "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"),
+        "reasoning_effort": run.reasoning_effort or os.environ.get("SHELLBENCH_REASONING_EFFORT"),
         "judge_reasoning_effort": os.environ.get(
             "SHELLBENCH_JUDGE_REASONING_EFFORT"
         ),
@@ -358,6 +358,9 @@ def _runner_patch_hash() -> str:
 def build_run_spec(args: argparse.Namespace) -> RunSpec:
     harness = harness_by_name(args.harness)
     model = model_by_slug(args.model_slug)
+    reasoning_effort = getattr(args, "reasoning_effort", None) or os.environ.get(
+        "SHELLBENCH_REASONING_EFFORT"
+    )
     return RunSpec(
         run_label=args.run_label,
         harness=harness.name,
@@ -369,6 +372,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec:
         repetition=args.repetition,
         expected_task_count=args.expected_task_count,
         run_date=args.run_date,
+        reasoning_effort=reasoning_effort,
     )
 
 
@@ -395,6 +399,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument("--public-tasks-commit", required=True)
     parser.add_argument("--task-suite-path", required=True)
     parser.add_argument("--run-date", required=True)
+    parser.add_argument(
+        "--reasoning-effort",
+        choices=("low", "medium", "high", "xhigh"),
+    )
     parser.add_argument(
         "--toolchain-root",
         type=Path,

tests/test_native_eval_runner.py

@@ -9,6 +9,8 @@
 from pathlib import Path
 from types import SimpleNamespace
 
+import pytest
+
 from scripts.native_eval.checkpoint_loop import (
     count_result_json,
     next_checkpoint_sequence,
@@ -679,6 +681,96 @@ def test_harness_commands_preserve_canonical_model_identity() -> None:
             assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command
 
 
+@pytest.mark.parametrize("effort", ("low", "medium", "high", "xhigh"))
+@pytest.mark.parametrize("harness", ("openclaw", "hermes", "codex", "claude-code"))
+def test_harness_commands_apply_native_reasoning_effort(
+    harness: str,
+    effort: str,
+) -> None:
+    run = RunSpec(
+        run_label=f"{harness}-{effort}",
+        harness=harness,
+        harness_version="test",
+        model_slug="gpt56-sol",
+        model_id="gpt-5.6-sol",
+        provider="openai",
+        proxy_model_name="gpt-5.6-sol",
+        repetition=1,
+        expected_task_count=116,
+        run_date="20260729",
+        reasoning_effort=effort,
+    )
+
+    command = build_harness_command(
+        run,
+        proxy_url="http://host.docker.internal:4000",
+        proxy_key="local-proxy-key",
+        mcp_servers=(),
+    )
+
+    if harness == "openclaw":
+        assert f"--thinking {effort}" in command.run_command
+    elif harness == "hermes":
+        assert f'"reasoning_effort":"{effort}"' in command.setup_command
+    elif harness == "codex":
+        assert f'model_reasoning_effort="{effort}"' in command.run_command
+    else:
+        expected_effort = "max" if effort == "xhigh" else effort
+        assert f"--effort {expected_effort}" in command.run_command
+
+
+def test_harness_commands_preserve_defaults_without_reasoning_effort() -> None:
+    commands = {}
+    for harness in ("openclaw", "hermes", "codex", "claude-code"):
+        run = RunSpec(
+            run_label=f"{harness}-default",
+            harness=harness,
+            harness_version="test",
+            model_slug="gpt55",
+            model_id="gpt-5.5",
+            provider="openai",
+            proxy_model_name="gpt-5.5",
+            repetition=1,
+            expected_task_count=116,
+            run_date="20260729",
+        )
+        commands[harness] = build_harness_command(
+            run,
+            proxy_url="http://host.docker.internal:4000",
+            proxy_key="local-proxy-key",
+            mcp_servers=(),
+        )
+
+    assert "--thinking off" in commands["openclaw"].run_command
+    assert '"reasoning_effort"' not in commands["hermes"].setup_command
+    assert "model_reasoning_effort" not in commands["codex"].run_command
+    assert "--effort" not in commands["claude-code"].run_command
+
+
+def test_build_run_spec_records_reasoning_effort_from_environment(
+    monkeypatch,
+) -> None:
+    monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "xhigh")
+
+    run = build_run_spec(
+        Namespace(
+            run_label="openclaw-gpt56-sol-xhigh",
+            harness="openclaw",
+            harness_version="test",
+            model_slug="gpt56-sol",
+            model_id=None,
+            model_provider=None,
+            proxy_model_name=None,
+            repetition=1,
+            expected_task_count=116,
+            run_date="20260729",
+            reasoning_effort=None,
+        )
+    )
+
+    assert run.reasoning_effort == "xhigh"
+
+
 def test_openclaw_completion_probe_accepts_markerless_final_envelope(
     tmp_path: Path,
 ) -> None: