diff --git a/CHANGELOG.md b/CHANGELOG.md index b08dd37..5a2971f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,3 +7,7 @@ - Record and enforce GPT reasoning effort in native runs, preserve it across reruns, and stabilize OpenClaw and Hermes trace completion. - Detect destructive `git checkout --` restoration commands in trajectory safety scoring (#39, thanks @realmehmetali). + +### Added + +- Add `gpt-5.6-luna` and `gpt-5.6-terra` to the native evaluation catalog. diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index dbc29ab..dbbc424 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -46,6 +46,20 @@ def to_dict(self) -> dict[str, object]: "gpt-5.6-sol", "gpt-5.6-sol", ), + ModelSpec( + "gpt56-luna", + "gpt-5.6-luna", + "openai", + "gpt-5.6-luna", + "gpt-5.6-luna", + ), + ModelSpec( + "gpt56-terra", + "gpt-5.6-terra", + "openai", + "gpt-5.6-terra", + "gpt-5.6-terra", + ), ModelSpec( "fable5", "fable-5", diff --git a/scripts/native_eval/proxy.py b/scripts/native_eval/proxy.py index 96b19c0..3532d11 100644 --- a/scripts/native_eval/proxy.py +++ b/scripts/native_eval/proxy.py @@ -19,7 +19,7 @@ def write_proxy_config(path: Path) -> None: "model": f"{model.provider}/{model.provider_model_id}", "api_key": f"os.environ/{model.provider.upper()}_API_KEY", } - if model.slug in {"gpt55", "gpt56-sol"}: + if model.provider == "openai": litellm_params["additional_drop_params"] = ["temperature"] litellm_params["reasoning_effort"] = reasoning_effort model_list.append( diff --git a/tests/test_native_eval_proxy.py b/tests/test_native_eval_proxy.py index ddb86c9..8eb6f85 100644 --- a/tests/test_native_eval_proxy.py +++ b/tests/test_native_eval_proxy.py @@ -23,4 +23,6 @@ def test_openai_proxy_models_enforce_reasoning_effort( ] assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == "high" assert models["gpt-5.6-sol"]["litellm_params"]["reasoning_effort"] == "high" + assert models["gpt-5.6-luna"]["litellm_params"]["reasoning_effort"] == "high" + assert models["gpt-5.6-terra"]["litellm_params"]["reasoning_effort"] == "high" assert config["shellbench_native"]["reasoning_effort"] == "high" diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index acc9a9e..e0fb54e 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -33,8 +33,8 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: plan = build_matrix_plan(116, run_date="20260727") - assert len(plan) == 72 - assert len({run.run_label for run in plan}) == 72 + assert len(plan) == 96 + assert len({run.run_label for run in plan}) == 96 assert {run.harness for run in plan} == {harness.name for harness in HARNESSES} assert {run.model_slug for run in plan} == {model.slug for model in MODELS} assert {run.repetition for run in plan} == {1, 2, 3} @@ -55,7 +55,7 @@ def test_run_index_records_agent_and_judge_reasoning( judge_reasoning_effort="high", ) - assert len(entries) == 72 + assert len(entries) == 96 assert {entry["reasoning_effort"] for entry in entries} == {"high"} assert {entry["judge_reasoning_effort"] for entry in entries} == {"high"}