diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index b3c3aa0..03624da 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -532,6 +532,7 @@ def _openclaw( ) -> HarnessCommand: provider = "openai" model = f"{provider}/{run.model_id}" + thinking = run.reasoning_effort or "off" home = "/tmp/shellbench-openclaw" audit_plugin_root = f"{home}/.openclaw/shellbench-audit" gateway_token = secrets.token_urlsafe(32) @@ -616,7 +617,8 @@ def _openclaw( 'kill "$gateway_pid" 2>/dev/null || true; ' 'wait "$gateway_pid" 2>/dev/null || true; ' 'cat "$gateway_log" >&2; exit 70; fi; ' - "openclaw agent --json --agent main --thinking off " + "openclaw agent --json --agent main " + f"--thinking {shlex.quote(thinking)} " f"--model {shlex.quote(model)} " '--message "$(cat /tmp/shellbench-instruction.md)" ' '>"$log" 2>&1 HarnessCommand: home = "/tmp/shellbench-hermes" provider_name = "custom:shellbench" + agent_config: dict[str, object] = {"max_turns": 90} + if run.reasoning_effort: + agent_config["reasoning_effort"] = run.reasoning_effort config: dict[str, object] = { "model": { "default": run.model_id, @@ -741,7 +746,7 @@ def _hermes( } }, "toolsets": ["hermes-cli"], - "agent": {"max_turns": 90}, + "agent": agent_config, "memory": {"memory_enabled": False, "user_profile_enabled": False}, "compression": {"enabled": True, "threshold": 0.85}, "terminal": {"backend": "local", "timeout": 180}, @@ -817,11 +822,17 @@ def _codex( f'printf %s {auth_json} > "$CODEX_HOME/auth.json"; ' f'printf %s {config_text} > "$CODEX_HOME/config.toml"' ) + reasoning_override = ( + f"-c {shlex.quote(f'model_reasoning_effort={json.dumps(run.reasoning_effort)}')} " + if run.reasoning_effort + else "" + ) run_command = ( f"export PATH={_base_path()}; export CODEX_HOME={home}; " "codex exec --dangerously-bypass-approvals-and-sandbox " "--skip-git-repo-check " f"--model {shlex.quote(run.model_id)} " + f"{reasoning_override}" "--json --enable unified_exec -- " '"$(cat /tmp/shellbench-instruction.md)" ' ">/logs/agent/codex.txt 2>/logs/agent/codex-stderr.txt " @@ -871,11 +882,16 @@ def _claude_code( 'mkdir -p "$CLAUDE_CONFIG_DIR/debug" "$CLAUDE_CONFIG_DIR/projects/-app"; ' f'printf %s {mcp_json} > "$CLAUDE_CONFIG_DIR/.claude.json"' ) + # Claude Code names its top effort level "max"; the other native harnesses + # use ShellBench's canonical "xhigh" spelling. + claude_effort = "max" if run.reasoning_effort == "xhigh" else run.reasoning_effort + effort_option = f"--effort {shlex.quote(claude_effort)} " if claude_effort else "" run_command = ( f"export PATH={_base_path()}; export CLAUDE_CONFIG_DIR={home}; " "claude --verbose --output-format=stream-json " "--permission-mode=bypassPermissions --print " f"--model {shlex.quote(run.model_id)} " + f"{effort_option}" '"$(cat /tmp/shellbench-instruction.md)" ' ">/logs/agent/claude-code.txt 2>&1 dict[str, object]: return asdict(self) @@ -162,6 +163,7 @@ def build_matrix_plan( repetition=repetition, expected_task_count=expected_task_count, run_date=stamp, + reasoning_effort=reasoning_effort, ) ) return plan diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index ccafe5e..fbebec2 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -270,7 +270,7 @@ def _run_manifest( "SHELLBENCH_HARBOR_REFERENCE_COMMIT" ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), - "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), + "reasoning_effort": run.reasoning_effort or os.environ.get("SHELLBENCH_REASONING_EFFORT"), "judge_reasoning_effort": os.environ.get( "SHELLBENCH_JUDGE_REASONING_EFFORT" ), @@ -358,6 +358,9 @@ def _runner_patch_hash() -> str: def build_run_spec(args: argparse.Namespace) -> RunSpec: harness = harness_by_name(args.harness) model = model_by_slug(args.model_slug) + reasoning_effort = getattr(args, "reasoning_effort", None) or os.environ.get( + "SHELLBENCH_REASONING_EFFORT" + ) return RunSpec( run_label=args.run_label, harness=harness.name, @@ -369,6 +372,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: repetition=args.repetition, expected_task_count=args.expected_task_count, run_date=args.run_date, + reasoning_effort=reasoning_effort, ) @@ -395,6 +399,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--public-tasks-commit", required=True) parser.add_argument("--task-suite-path", required=True) parser.add_argument("--run-date", required=True) + parser.add_argument( + "--reasoning-effort", + choices=("low", "medium", "high", "xhigh"), + ) parser.add_argument( "--toolchain-root", type=Path, diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6aef78f..d82f2e5 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -9,6 +9,8 @@ from pathlib import Path from types import SimpleNamespace +import pytest + from scripts.native_eval.checkpoint_loop import ( count_result_json, next_checkpoint_sequence, @@ -679,6 +681,96 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command +@pytest.mark.parametrize("effort", ("low", "medium", "high", "xhigh")) +@pytest.mark.parametrize("harness", ("openclaw", "hermes", "codex", "claude-code")) +def test_harness_commands_apply_native_reasoning_effort( + harness: str, + effort: str, +) -> None: + run = RunSpec( + run_label=f"{harness}-{effort}", + harness=harness, + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=1, + expected_task_count=116, + run_date="20260729", + reasoning_effort=effort, + ) + + command = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + if harness == "openclaw": + assert f"--thinking {effort}" in command.run_command + elif harness == "hermes": + assert f'"reasoning_effort":"{effort}"' in command.setup_command + elif harness == "codex": + assert f'model_reasoning_effort="{effort}"' in command.run_command + else: + expected_effort = "max" if effort == "xhigh" else effort + assert f"--effort {expected_effort}" in command.run_command + + +def test_harness_commands_preserve_defaults_without_reasoning_effort() -> None: + commands = {} + for harness in ("openclaw", "hermes", "codex", "claude-code"): + run = RunSpec( + run_label=f"{harness}-default", + harness=harness, + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=116, + run_date="20260729", + ) + commands[harness] = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + assert "--thinking off" in commands["openclaw"].run_command + assert '"reasoning_effort"' not in commands["hermes"].setup_command + assert "model_reasoning_effort" not in commands["codex"].run_command + assert "--effort" not in commands["claude-code"].run_command + + +def test_build_run_spec_records_reasoning_effort_from_environment( + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "xhigh") + + run = build_run_spec( + Namespace( + run_label="openclaw-gpt56-sol-xhigh", + harness="openclaw", + harness_version="test", + model_slug="gpt56-sol", + model_id=None, + model_provider=None, + proxy_model_name=None, + repetition=1, + expected_task_count=116, + run_date="20260729", + reasoning_effort=None, + ) + ) + + assert run.reasoning_effort == "xhigh" + + def test_openclaw_completion_probe_accepts_markerless_final_envelope( tmp_path: Path, ) -> None: