diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml
index a3dc16d1b9..f38720a186 100644
--- a/.github/workflows/benchmark-multinode-tmpl.yml
+++ b/.github/workflows/benchmark-multinode-tmpl.yml
@@ -142,6 +142,16 @@ on:
type: boolean
required: false
default: false
+ eval-framework:
+ description: "Eval runner (lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
+ type: string
+ required: false
+ default: "lm-eval"
+ eval-suite:
+ description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+ type: string
+ required: false
+ default: ""
eval-conc:
description: "Concurrency value or space-separated list for eval requests (overrides default max-of-conc-list)"
type: string
@@ -242,6 +252,8 @@ env:
DECODE_HARDWARE: ${{ inputs.decode-hardware }}
RUN_EVAL: ${{ inputs.run-eval }}
EVAL_ONLY: ${{ inputs.eval-only }}
+ EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
+ EVAL_SUITE: ${{ inputs.eval-suite }}
EVAL_CONC: ${{ inputs.eval-conc }}
EVAL_LIMIT: ${{ inputs.eval-limit }}
SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }}
@@ -403,6 +415,9 @@ jobs:
fi
# Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV"
+ echo "EVAL_ARTIFACT_RECIPE=${RECIPE_FINGERPRINT:0:16}" >> "$GITHUB_ENV"
+ eval_artifact_conc="$(python3 -c 'import hashlib,os; print(hashlib.sha256(os.environ["CONC_LIST"].encode()).hexdigest()[:12])')"
+ echo "EVAL_ARTIFACT_CONC=${eval_artifact_conc}" >> "$GITHUB_ENV"
export ${{ join(fromJson(inputs.prefill-additional-settings), ' ') }} ${{ join(fromJson(inputs.decode-additional-settings), ' ') }}
export IS_MULTINODE=true
@@ -528,14 +543,17 @@ jobs:
if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: eval_${{ env.RESULT_FILENAME }}
+ name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
path: |
meta_env.json
results*.json
+ *_report.json
+ *_results.jsonl
sample*.jsonl
agent_preds.json
predictions.jsonl
swebench_report_*.json
+ *_artifacts.tar.gz
*.traj*
if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}
@@ -553,6 +571,9 @@ jobs:
run: |
rm -f meta_env.json || true
rm -f results*.json || true
+ rm -f -- ./*_report.json || true
+ rm -f -- ./*_results.jsonl || true
+ rm -f -- ./*_artifacts.tar.gz || true
rm -f sample*.jsonl || true
rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml
index 11ea880236..2bcbbaeb4d 100644
--- a/.github/workflows/benchmark-tmpl.yml
+++ b/.github/workflows/benchmark-tmpl.yml
@@ -90,6 +90,16 @@ on:
type: boolean
required: false
default: false
+ eval-framework:
+ description: "Eval runner (lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
+ type: string
+ required: false
+ default: "lm-eval"
+ eval-suite:
+ description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+ type: string
+ required: false
+ default: ""
random-range-ratio:
required: false
type: string
@@ -179,6 +189,8 @@ env:
DISAGG: ${{ inputs.disagg }}
RUN_EVAL: ${{ inputs.run-eval }}
EVAL_ONLY: ${{ inputs.eval-only }}
+ EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
+ EVAL_SUITE: ${{ inputs.eval-suite }}
# Agentic-coding env. Fixed-seq-len jobs leave these empty.
SCENARIO_TYPE: ${{ inputs.scenario-type }}
SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }}
@@ -203,7 +215,7 @@ env:
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
# These b300 nodes are currently broken.
- SALLOC_EXCLUDE: 'b300-005,b300-006'
+ SALLOC_EXCLUDE: 'b300-005,b300-006,b300-017'
permissions:
contents: read
@@ -263,6 +275,13 @@ jobs:
steps:
- name: Resource cleanup (pre-run)
run: &resource-cleanup |
+ # Containerized AMD Slurm jobs can leave root-owned artifacts when
+ # interrupted before their launcher EXIT trap runs. Repair ownership
+ # before checkout cleanup and again after the job via this shared step.
+ if [[ "${{ inputs.runner }}" == "cluster:mi355x-amds" && -d "$GITHUB_WORKSPACE" ]]; then
+ sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
+ fi
+
# Cleanup Docker resources
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
echo "[Docker] Cleaning up resources ..."
@@ -443,10 +462,13 @@ jobs:
if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }}
+ name: eval_${{ env.RESULT_FILENAME }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ github.run_attempt }}
path: |
meta_env.json
results*.json
+ *_report.json
+ *_results.jsonl
+ *_artifacts.tar.gz
sample*.jsonl
agent_preds.json
predictions.jsonl
@@ -464,7 +486,10 @@ jobs:
rm -f meta_env.json || true
# Remove any eval results JSONs that were moved into workspace
rm -f results*.json || true
+ rm -f -- ./*_report.json || true
+ rm -f -- ./*_results.jsonl || true
rm -f sample*.jsonl || true
+ rm -f -- ./*_artifacts.tar.gz || true
rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
- name: Resource cleanup (post-run)
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
index 22d4304e7c..5167e19f33 100644
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@@ -45,6 +45,16 @@ on:
required: false
type: string
default: ""
+ eval-framework:
+ description: "Eval runner override (auto uses model-aware selection; lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
+ required: false
+ type: string
+ default: "auto"
+ eval-suite:
+ description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+ required: false
+ type: string
+ default: ""
swebench-gen-mode:
description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
required: false
@@ -125,6 +135,16 @@ on:
required: false
type: string
default: ""
+ eval-framework:
+ description: "Eval runner override (auto uses model-aware selection; lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
+ required: false
+ type: string
+ default: "auto"
+ eval-suite:
+ description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+ required: false
+ type: string
+ default: ""
swebench-gen-mode:
description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
required: false
@@ -236,16 +256,26 @@ jobs:
CMD+=(--evals-only)
fi
RAW_CONFIG_JSON=$("${CMD[@]}")
- CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON")
+ CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals","multinode_agentic_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON")
else
GENERATE_COMMAND="${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}"
if [ -z "$GENERATE_COMMAND" ]; then
echo "generate-cli-command is required outside trusted changelog dispatch mode" >&2
exit 1
fi
+ read -r -a GENERATE_ARGS <<< "$GENERATE_COMMAND"
+ if [ "$TRIM_CONC" = "true" ]; then
+ GENERATE_ARGS+=(--trim-conc)
+ fi
+ if [ "$ALL_EVALS" = "true" ]; then
+ GENERATE_ARGS+=(--all-evals)
+ fi
+ if [ "$EVALS_ONLY" = "true" ]; then
+ GENERATE_ARGS+=(--evals-only)
+ fi
CONFIG_JSON=$(uv run --no-project --with pydantic --with pyyaml --python 3.12 \
"${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py" \
- $GENERATE_COMMAND)
+ "${GENERATE_ARGS[@]}")
fi
score_matrix() {
local family="$1"
@@ -255,9 +285,9 @@ jobs:
--queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}:${family}" \
--labels-json "$PR_LABELS"
}
- AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))" | score_matrix agentic)
+ AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('eval-only', False)]))" | score_matrix agentic)
AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval)
- MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('run-eval', False)]))" | score_matrix multi-agentic)
+ MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('eval-only', False)]))" | score_matrix multi-agentic)
MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval)
SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single)
MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi)
@@ -381,7 +411,11 @@ jobs:
decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }}
run-eval: true
eval-only: true
- eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }}
+ eval-conc: ${{ (inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework) == 'lm-eval' && matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }}
+ eval-limit: ${{ inputs.eval-limit }}
+ eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
+ eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
+ swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
ref: ${{ inputs.ref }}
test-sweep-agentic:
@@ -451,12 +485,18 @@ jobs:
model-prefix: ${{ matrix.config.model-prefix }}
framework: ${{ matrix.config.framework }}
precision: ${{ matrix.config.precision }}
+ router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }}
+ kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }}
tp: ${{ matrix.config.tp }}
+ pp: ${{ matrix.config.pp }}
+ dcp-size: ${{ matrix.config.dcp-size }}
+ pcp-size: ${{ matrix.config.pcp-size }}
ep: ${{ matrix.config.ep }}
dp-attn: ${{ matrix.config.dp-attn }}
conc: ${{ matrix.config.conc }}
kv-offloading: ${{ matrix.config.kv-offloading }}
kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }}
+ kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }}
total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }}
duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }}
agentx-fast: ${{ inputs.agentx-fast }}
@@ -469,6 +509,8 @@ jobs:
eval-only: true
eval-limit: ${{ inputs.eval-limit }}
swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
+ eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
+ eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
scenario-type: agentic-coding
ref: ${{ inputs.ref }}
@@ -559,7 +601,7 @@ jobs:
precision: ${{ matrix.config.precision }}
router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }}
kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }}
- conc-list: ${{ toJson(matrix.config.conc) }}
+ conc-list: ${{ format('[{0}]', matrix.config['eval-conc']) }}
spec-decoding: ${{ matrix.config.spec-decoding }}
disagg: ${{ matrix.config.disagg }}
prefill-hardware: ${{ matrix.config.prefill.hardware }}
@@ -590,6 +632,8 @@ jobs:
eval-conc: ${{ matrix.config['eval-conc'] }}
eval-limit: ${{ inputs.eval-limit }}
swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
+ eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
+ eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
scenario-type: agentic-coding
ref: ${{ inputs.ref }}
@@ -670,6 +714,9 @@ jobs:
run-eval: true
eval-only: true
eval-limit: ${{ inputs.eval-limit }}
+ eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
+ eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
+ swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
ref: ${{ inputs.ref }}
collect-results:
diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml
index 04b7a639b4..9603dec883 100644
--- a/.github/workflows/run-sweep.yml
+++ b/.github/workflows/run-sweep.yml
@@ -864,12 +864,12 @@ jobs:
eval-only: true
eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }}
- # Multi-node agentic (SWE-bench) eval rows carry the agentic input shape,
- # so they are dispatched with sweep-multi-node-agentic's inputs rather
- # than sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len,
- # which agentic rows don't have). SWE-bench doesn't support batched
- # concurrencies (unlike lm-eval), so eval-conc is always a single value,
- # never the joined-list form sweep-multi-node-evals uses.
+ # Multi-node agentic GSM8K eval rows carry the agentic input shape, so
+ # they are dispatched with sweep-multi-node-agentic's inputs rather than
+ # sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len,
+ # which agentic rows don't have). Agentic selection uses one highest
+ # eval-conc per topology; fixed-sequence --all-evals rows may instead pass
+ # a joined concurrency list to lm-eval.
sweep-multi-node-agentic-evals:
needs: [setup, canary-select, canary-sweep]
if: >-
@@ -906,7 +906,7 @@ jobs:
precision: ${{ matrix.config.precision }}
router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }}
kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }}
- conc-list: ${{ toJson(matrix.config.conc) }}
+ conc-list: ${{ format('[{0}]', matrix.config['eval-conc']) }}
spec-decoding: ${{ matrix.config.spec-decoding }}
disagg: ${{ matrix.config.disagg }}
prefill-hardware: ${{ matrix.config.prefill.hardware }}
diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh
index 4819f50599..e7131952c2 100644
--- a/benchmarks/benchmark_lib.sh
+++ b/benchmarks/benchmark_lib.sh
@@ -8,6 +8,12 @@
export PYTHONDONTWRITEBYTECODE=1
export PYTHONPYCACHEPREFIX="${PYTHONPYCACHEPREFIX:-/tmp/inferencex-pycache}"
mkdir -p "$PYTHONPYCACHEPREFIX" 2>/dev/null || true
+INFERENCEX_BENCHMARK_LIB_DIR="$(
+ cd "$(dirname "${BASH_SOURCE[0]}")" && pwd
+)"
+INFERENCEX_REPO_ROOT="$(
+ cd "$INFERENCEX_BENCHMARK_LIB_DIR/.." && pwd
+)"
# Inference server port shared by every benchmark recipe. Launchers that need
# a non-default value (e.g. launch_mi355x-amds.sh derives PORT from RUNNER_NAME
@@ -886,8 +892,1083 @@ _install_lm_eval_deps() {
fi
}
+_prepare_vendor_verifier_python() {
+ local verifier_name="$1"
+ local runtime_prefix="$2"
+ local use_system_site_packages="${3:-false}"
+ local minimum_python_minor="${4:-12}"
+
+ VENDOR_VERIFIER_PYTHON=python3
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR=""
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+
+ local system_python_is_compatible=false
+ if python3 -c \
+ 'import sys; raise SystemExit(sys.version_info < (3, int(sys.argv[1])))' \
+ "$minimum_python_minor"; then
+ system_python_is_compatible=true
+ if [ "$use_system_site_packages" != "true" ]; then
+ return 0
+ fi
+ fi
+
+ local python_dir uv_prefix uv_bin venv_dir prepare_rc=0
+ python_dir="$(mktemp -d "/tmp/${runtime_prefix}-XXXXXX")" || {
+ echo "ERROR: could not create a temporary Python directory for ${verifier_name}" >&2
+ return 1
+ }
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$python_dir"
+ export VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+
+ venv_dir="${python_dir}/venv"
+ if [ "$system_python_is_compatible" = "true" ]; then
+ python3 -m venv --system-site-packages "$venv_dir" || prepare_rc=$?
+ else
+ uv_prefix="${python_dir}/uv"
+ uv_bin="${uv_prefix}/bin/uv"
+ python3 -m pip install -q --no-cache-dir --break-system-packages \
+ --prefix "$uv_prefix" "uv==0.11.33" || prepare_rc=$?
+ if [ "$prepare_rc" -eq 0 ] && [ ! -x "$uv_bin" ]; then
+ echo "ERROR: pinned uv installation did not create ${uv_bin}" >&2
+ prepare_rc=1
+ fi
+ if [ "$prepare_rc" -eq 0 ]; then
+ local system_site_packages_args=()
+ if [ "$use_system_site_packages" = "true" ]; then
+ system_site_packages_args+=(--system-site-packages)
+ fi
+ UV_CACHE_DIR="${python_dir}/uv-cache" \
+ UV_PYTHON_INSTALL_DIR="${python_dir}/python" \
+ "$uv_bin" venv --python "3.${minimum_python_minor}" --seed \
+ "${system_site_packages_args[@]}" "$venv_dir" \
+ || prepare_rc=$?
+ fi
+ fi
+ if [ "$prepare_rc" -eq 0 ] && [ ! -x "${venv_dir}/bin/python" ]; then
+ echo "ERROR: pinned Python setup did not create the ${verifier_name} interpreter" >&2
+ prepare_rc=1
+ fi
+ if [ "$prepare_rc" -ne 0 ]; then
+ rm -rf "$python_dir" || true
+ VENDOR_VERIFIER_PYTHON=python3
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR=""
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+ return "$prepare_rc"
+ fi
+
+ VENDOR_VERIFIER_PYTHON="${venv_dir}/bin/python"
+ export VENDOR_VERIFIER_PYTHON
+}
+
+_install_kimi_vendor_eval_deps() {
+ local target_dir="$1"
+ local eval_suite="${2:-kimi_tool_call_schema}"
+ local -a packages=(
+ "httpx[http2]==0.28.1"
+ "openai==2.14.0"
+ "jsonschema==4.25.1"
+ "pytest==8.4.2"
+ )
+ if [ "$eval_suite" = "kimi_tool_call_schema_full" ]; then
+ packages+=("pytest-xdist==3.8.0")
+ fi
+ "${VENDOR_VERIFIER_PYTHON:-python3}" -m pip install -q --no-cache-dir \
+ --target "$target_dir" "${packages[@]}"
+}
+
+_prepare_kimi_vendor_runtime() {
+ local eval_suite="${1:-kimi_tool_call_schema}"
+ local runtime_dir install_rc=0
+ runtime_dir="$(mktemp -d /tmp/kimi-vendor-runtime-XXXXXX)" || return $?
+ _install_kimi_vendor_eval_deps "$runtime_dir" "$eval_suite" >&2 || install_rc=$?
+ if [ "$install_rc" -ne 0 ]; then
+ rm -rf "$runtime_dir"
+ return "$install_rc"
+ fi
+ printf '%s\n' "$runtime_dir"
+}
+
+_prepare_kimi_vendor_verifier() {
+ local repo_url="$1"
+ local verifier_ref="$2"
+ local expected_archive_sha256="$3"
+ local checkout_dir prepare_rc=0
+
+ checkout_dir="$(mktemp -d /tmp/kimi-vendor-verifier-XXXXXX)" || {
+ echo "ERROR: could not create a temporary directory for Kimi-Vendor-Verifier" >&2
+ return 1
+ }
+
+ "${VENDOR_VERIFIER_PYTHON:-python3}" - \
+ "$repo_url" "$verifier_ref" "$expected_archive_sha256" "$checkout_dir" <<'PY' || prepare_rc=$?
+from hashlib import sha256
+from pathlib import Path
+import re
+import socket
+import sys
+import tarfile
+import tempfile
+import time
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote, urlsplit, urlunsplit
+from urllib.request import Request, urlopen
+
+
+repo_url, verifier_ref, expected_archive_sha256, checkout_dir_arg = sys.argv[1:]
+checkout_dir = Path(checkout_dir_arg)
+stage = "derive the pinned archive URL"
+
+
+def archive_member_parts(name):
+ if not name or "\x00" in name or "\\" in name or name.startswith("/"):
+ raise ValueError(f"unsafe archive member path: {name!r}")
+ normalized = name.rstrip("/")
+ parts = normalized.split("/")
+ if not normalized or any(part in ("", ".", "..") for part in parts):
+ raise ValueError(f"unsafe archive member path: {name!r}")
+ return tuple(parts)
+
+
+try:
+ if not re.fullmatch(r"[0-9a-fA-F]{40}", verifier_ref):
+ raise ValueError(f"expected a 40-character commit SHA, got {verifier_ref!r}")
+ if not re.fullmatch(r"[0-9a-fA-F]{64}", expected_archive_sha256):
+ raise ValueError(
+ "expected a 64-character archive SHA256, got "
+ f"{expected_archive_sha256!r}"
+ )
+
+ parsed_repo_url = urlsplit(repo_url)
+ if parsed_repo_url.scheme not in ("http", "https") or not parsed_repo_url.netloc:
+ raise ValueError(f"unsupported repository URL: {repo_url!r}")
+ if parsed_repo_url.query or parsed_repo_url.fragment:
+ raise ValueError(f"repository URL must not contain a query or fragment: {repo_url!r}")
+ repo_path = parsed_repo_url.path.rstrip("/")
+ if repo_path.endswith(".git"):
+ repo_path = repo_path[:-4]
+ if not repo_path:
+ raise ValueError(f"repository URL has no repository path: {repo_url!r}")
+ archive_path = f"{repo_path}/archive/{quote(verifier_ref, safe='')}.tar.gz"
+ archive_url = urlunsplit(
+ (parsed_repo_url.scheme, parsed_repo_url.netloc, archive_path, "", "")
+ )
+
+ stage = f"download {archive_url}"
+ request = Request(
+ archive_url,
+ headers={"User-Agent": "InferenceX-Kimi-Vendor-Verifier"},
+ )
+ with tempfile.TemporaryFile() as archive_file:
+ downloaded = 0
+ for attempt in range(1, 4):
+ archive_file.seek(0)
+ archive_file.truncate()
+ downloaded = 0
+ digest = sha256()
+ deadline = time.monotonic() + 60
+ try:
+ with urlopen(request, timeout=60) as response:
+ while True:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError(
+ "archive download exceeded the 60-second deadline"
+ )
+ sock = getattr(getattr(response, "fp", None), "raw", None)
+ sock = getattr(sock, "_sock", None)
+ if sock is not None:
+ sock.settimeout(max(0.001, remaining))
+ try:
+ chunk = response.read(1024 * 1024)
+ except socket.timeout as error:
+ raise TimeoutError(
+ "archive download exceeded the 60-second deadline"
+ ) from error
+ if not chunk:
+ break
+ downloaded += len(chunk)
+ if downloaded > 128 * 1024 * 1024:
+ raise ValueError(
+ "archive download exceeds the 128 MiB safety limit"
+ )
+ archive_file.write(chunk)
+ digest.update(chunk)
+ break
+ except HTTPError as error:
+ if error.code not in (408, 429) and not 500 <= error.code < 600:
+ raise
+ if attempt == 3:
+ raise
+ print(
+ f"WARN: Kimi-Vendor-Verifier archive download attempt "
+ f"{attempt}/3 failed: {error}; retrying",
+ file=sys.stderr,
+ )
+ time.sleep(attempt)
+ except (TimeoutError, URLError, ConnectionError) as error:
+ if attempt == 3:
+ raise
+ print(
+ f"WARN: Kimi-Vendor-Verifier archive download attempt "
+ f"{attempt}/3 failed: {error}; retrying",
+ file=sys.stderr,
+ )
+ time.sleep(attempt)
+ if downloaded == 0:
+ raise ValueError("downloaded archive is empty")
+ actual_archive_sha256 = digest.hexdigest()
+ if actual_archive_sha256 != expected_archive_sha256:
+ raise ValueError(
+ "Kimi-Vendor-Verifier archive SHA256 mismatch: expected "
+ f"{expected_archive_sha256}, got {actual_archive_sha256}"
+ )
+ archive_file.seek(0)
+
+ stage = "validate the downloaded archive"
+ required_files = {
+ "pyproject.toml",
+ "tests/conftest.py",
+ "tests/__init__.py",
+ "tests/tool_call_json_schema/conftest.py",
+ "tests/tool_call_json_schema/__init__.py",
+ "tests/tool_call_json_schema/test_tool_call_json_schema.py",
+ "tests/tool_call_json_schema/validator.py",
+ "testdata/walle_validator_cases/validator_cases/TestAdditionalProperties/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestAnyOf/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestBasicTypes/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestDefs/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestDescription/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestEnforcerCases/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestID/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestKeywordsValidation/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestNestedDefsDepth/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestNumberFormat/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestRangeConstraints/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestRefInProperties/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestReferences/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestRequired/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestSingleTypeInArray/valid.jsonl",
+ "testdata/walle_validator_cases/validator_cases/TestTypeLocation/valid.jsonl",
+ }
+ selected_files = {}
+ archive_roots = set()
+ member_count = 0
+ archive_size = 0
+ selected_size = 0
+
+ with tarfile.open(fileobj=archive_file, mode="r|gz") as archive:
+ for member in archive:
+ member_count += 1
+ if member_count > 100_000:
+ raise ValueError("archive contains more than 100000 members")
+ if member.size < 0:
+ raise ValueError(
+ f"archive member has a negative size: {member.name!r}"
+ )
+ archive_size += member.size
+ if archive_size > 512 * 1024 * 1024:
+ raise ValueError("expanded archive exceeds the 512 MiB safety limit")
+
+ parts = archive_member_parts(member.name)
+ archive_roots.add(parts[0])
+ if len(archive_roots) > 1:
+ roots = ", ".join(sorted(archive_roots))
+ raise ValueError(f"archive has multiple roots: {roots}")
+ if not (member.isdir() or member.isfile()):
+ raise ValueError(
+ f"archive member has unsafe type: {member.name!r}"
+ )
+ if len(parts) == 1:
+ continue
+
+ relative_path = "/".join(parts[1:])
+ if relative_path not in required_files:
+ continue
+ if relative_path in selected_files:
+ raise ValueError(
+ f"archive contains duplicate selected path: {relative_path!r}"
+ )
+ if not member.isfile():
+ raise ValueError(
+ f"required path is not a regular file: {relative_path}"
+ )
+ selected_size += member.size
+ if selected_size > 256 * 1024 * 1024:
+ raise ValueError(
+ "selected archive subset exceeds the 256 MiB safety limit"
+ )
+ source = archive.extractfile(member)
+ if source is None:
+ raise ValueError(f"could not read archive member: {member.name!r}")
+ with source:
+ content = source.read(member.size + 1)
+ if len(content) != member.size:
+ raise ValueError(
+ f"archive member size mismatch: {member.name!r}"
+ )
+ selected_files[relative_path] = content
+
+ if member_count == 0:
+ raise ValueError("archive contains no members")
+ if len(archive_roots) != 1:
+ raise ValueError("archive does not have exactly one root")
+ missing_files = sorted(required_files - selected_files.keys())
+ if missing_files:
+ raise ValueError(
+ "archive is missing required files: " + ", ".join(missing_files)
+ )
+
+ stage = "extract the verified archive subset"
+ if any(checkout_dir.iterdir()):
+ raise ValueError(f"checkout directory is not empty: {checkout_dir}")
+ for relative_path, content in selected_files.items():
+ destination = checkout_dir.joinpath(*relative_path.split("/"))
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ with destination.open("xb") as output:
+ output.write(content)
+except Exception as error:
+ print(
+ f"ERROR: failed to {stage} for Kimi-Vendor-Verifier "
+ f"at {verifier_ref}: {error}",
+ file=sys.stderr,
+ )
+ raise SystemExit(1)
+PY
+
+ if [ "$prepare_rc" -ne 0 ]; then
+ if ! rm -rf "$checkout_dir"; then
+ echo "ERROR: failed to remove partial Kimi-Vendor-Verifier directory ${checkout_dir}" >&2
+ fi
+ return "$prepare_rc"
+ fi
+
+ printf '%s\n' "$checkout_dir"
+}
+
+_cleanup_vendor_eval() {
+ local path
+ for path in "$@"; do
+ [ -z "$path" ] || rm -rf "$path" || true
+ done
+}
+
+_has_eval_result() {
+ local results_dir="$1"
+ local filename_prefix="$2"
+ local matches=("${results_dir}/${filename_prefix}"*.json)
+ [ -f "${matches[0]}" ]
+}
+
+_prepare_eval_artifact_family() {
+ local results_dir="$1"
+ local family="$2"
+ local artifact rm_rc=0
+ local artifacts=()
+
+ export EVAL_RESULT_DIR=""
+ case "$family" in
+ kimi)
+ artifacts=(
+ "${results_dir}"/results_kimi_vendor_*.json
+ "${results_dir}/kimi_vendor_report.json"
+ )
+ ;;
+ minimax)
+ artifacts=(
+ "${results_dir}"/results_minimax_vendor_*.json
+ "${results_dir}/minimax_vendor_report.json"
+ "${results_dir}/minimax_vendor_results.jsonl"
+ )
+ ;;
+ bfcl)
+ artifacts=(
+ "${results_dir}"/results_bfcl*.json
+ "${results_dir}/bfcl_report.json"
+ "${results_dir}/bfcl_upstream_artifacts.tar.gz"
+ )
+ ;;
+ *)
+ echo "ERROR: unsupported eval artifact family '${family}'" >&2
+ return 2
+ ;;
+ esac
+
+ for artifact in "${artifacts[@]}"; do
+ if [ -e "$artifact" ] || [ -L "$artifact" ]; then
+ rm -f -- "$artifact" || rm_rc=$?
+ if [ "$rm_rc" -ne 0 ]; then
+ echo "ERROR: failed to remove stale eval artifact ${artifact}" >&2
+ return "$rm_rc"
+ fi
+ fi
+ done
+ export EVAL_RESULT_DIR="$results_dir"
+}
+
+_write_kimi_vendor_integration_error() {
+ local adapter_path="$1"
+ local model_name="$2"
+ local results_dir="$3"
+ local task_name="$4"
+ local message="$5"
+
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" \
+ --model "$model_name" \
+ --output-dir "$results_dir" \
+ --task-name "$task_name" \
+ --integration-error "$message"
+}
+
+_run_kimi_tool_call_schema_eval() {
+ local port="${PORT:-8888}"
+ local results_dir="${EVAL_RESULT_DIR:-$(mktemp -d /tmp/eval_out-XXXXXX)}"
+ local verifier_repo="https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git"
+ local verifier_ref="3dad65a760a8867cda72f6dd8848d876a4e851b4"
+ local verifier_archive_sha256="ede9ea300c72ccfde9d8975ea4b1b54e423c7625690f6631ab1e65a715821e01"
+ local eval_suite="${EVAL_SUITE:-kimi_tool_call_schema}"
+ local timeout_seconds=900
+ if [ "$eval_suite" = "kimi_tool_call_schema_full" ]; then
+ timeout_seconds=7200
+ fi
+
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --port|--results-dir)
+ if [[ $# -lt 2 || -z "${2:-}" || "${2:-}" == --* ]]; then
+ echo "ERROR: $1 requires a value" >&2
+ return 2
+ fi
+ case "$1" in
+ --port) port="$2" ;;
+ --results-dir) results_dir="$2" ;;
+ esac
+ shift 2
+ ;;
+ *)
+ echo "Unknown parameter: $1" >&2
+ return 2
+ ;;
+ esac
+ done
+
+
+ local model_name="${MODEL_NAME:-${MODEL:-}}"
+ local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/kimi_vendor_eval.py"
+ local runtime_dir=""
+ local checkout_dir=""
+
+ mkdir -p "$results_dir" || return $?
+ results_dir="$(cd "$results_dir" && pwd)" || return $?
+ _prepare_eval_artifact_family "$results_dir" kimi || return $?
+
+ local setup_rc=0 integration_error=""
+ _prepare_vendor_verifier_python "Kimi-Vendor-Verifier" "kimi-vendor-python" || {
+ setup_rc=$?
+ integration_error="Kimi Vendor Verifier Python runtime preparation failed with exit code ${setup_rc}"
+ }
+ if [ "$setup_rc" -eq 0 ]; then
+ runtime_dir=$(_prepare_kimi_vendor_runtime "$eval_suite") || {
+ setup_rc=$?
+ integration_error="Kimi Vendor Verifier dependency installation failed with exit code ${setup_rc}"
+ }
+ fi
+ if [ "$setup_rc" -eq 0 ]; then
+ checkout_dir=$(
+ _prepare_kimi_vendor_verifier \
+ "$verifier_repo" "$verifier_ref" "$verifier_archive_sha256"
+ ) || {
+ setup_rc=$?
+ integration_error="Kimi Vendor Verifier checkout failed with exit code ${setup_rc}"
+ }
+ fi
+ if [ "$setup_rc" -ne 0 ]; then
+ echo "ERROR: ${integration_error}" >&2
+ local artifact_rc=0
+ _write_kimi_vendor_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$eval_suite" \
+ "$integration_error" || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write Kimi verifier failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "$checkout_dir" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ return "$setup_rc"
+ fi
+
+ local eval_rc=0
+ PYTHONPATH="${runtime_dir}${PYTHONPATH:+:${PYTHONPATH}}" \
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" \
+ --verifier-dir "$checkout_dir" \
+ --base-url "http://127.0.0.1:${port}/v1" \
+ --api-key EMPTY \
+ --model "$model_name" \
+ --model-prefix "${MODEL_PREFIX:-}" \
+ --output-dir "$results_dir" \
+ --task-name "$eval_suite" \
+ --timeout-seconds "$timeout_seconds" \
+ || eval_rc=$?
+ if [ "$eval_rc" -ne 0 ] \
+ && ! _has_eval_result "$results_dir" "results_kimi_vendor_"; then
+ integration_error="Kimi Vendor Verifier failed with exit code ${eval_rc}"
+ local artifact_rc=0
+ _write_kimi_vendor_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$eval_suite" \
+ "$integration_error" || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write Kimi verifier failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "$checkout_dir" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ return "$eval_rc"
+}
+
+run_kimi_vendor_eval() {
+ local eval_suite="${EVAL_SUITE:-kimi_tool_call_schema}"
+ export EVAL_SUITE="$eval_suite"
+
+ case "$eval_suite" in
+ kimi_tool_call_schema|kimi_tool_call_schema_full)
+ _run_kimi_tool_call_schema_eval "$@"
+ ;;
+ *)
+ echo "ERROR: unsupported Kimi Vendor Verifier suite '${eval_suite}'" >&2
+ export EVAL_RESULT_DIR=""
+ return 2
+ ;;
+ esac
+}
+
+_install_bfcl_eval_deps() {
+ local download_dir="$1"
+ local wheel_url="https://files.pythonhosted.org/packages/ba/41/ed458527c770c50225b60bae3b0c3444b26804ee455fa2d8f187018d2cb2/bfcl_eval-2026.3.23-py3-none-any.whl"
+ local wheel_sha256="3bb6dfa5f0c68ad403c9ec50b00db2bb3b4cc9b38ab1ff33f48fe30d853d3a0a"
+ local wheel_path="${download_dir}/bfcl_eval-2026.3.23-py3-none-any.whl"
+
+ "${VENDOR_VERIFIER_PYTHON:-python3}" - \
+ "$wheel_url" "$wheel_sha256" "$wheel_path" <<'PY' || return $?
+from hashlib import sha256
+from pathlib import Path
+import sys
+from urllib.request import Request, urlopen
+
+wheel_url, expected_sha256, wheel_path_arg = sys.argv[1:]
+wheel_path = Path(wheel_path_arg)
+digest = sha256()
+downloaded = 0
+request = Request(
+ wheel_url,
+ headers={"User-Agent": "InferenceX-BFCL-Smoke"},
+)
+
+try:
+ with urlopen(request, timeout=180) as response, wheel_path.open("xb") as output:
+ while chunk := response.read(1024 * 1024):
+ downloaded += len(chunk)
+ if downloaded > 512 * 1024 * 1024:
+ raise ValueError("BFCL wheel exceeds the 512 MiB safety limit")
+ digest.update(chunk)
+ output.write(chunk)
+ if downloaded == 0:
+ raise ValueError("downloaded BFCL wheel is empty")
+ actual_sha256 = digest.hexdigest()
+ if actual_sha256 != expected_sha256:
+ raise ValueError(
+ f"BFCL wheel SHA256 mismatch: expected {expected_sha256}, "
+ f"got {actual_sha256}"
+ )
+except Exception as error:
+ wheel_path.unlink(missing_ok=True)
+ print(f"ERROR: failed to download and verify the pinned BFCL wheel: {error}", file=sys.stderr)
+ raise SystemExit(1)
+PY
+
+ timeout 600 "${VENDOR_VERIFIER_PYTHON:-python3}" -m pip install \
+ -q --no-cache-dir "$wheel_path" "soundfile==0.13.1"
+}
+
+_prepare_bfcl_runtime() {
+ local runtime_dir install_rc=0
+ runtime_dir="$(mktemp -d /tmp/bfcl-runtime-XXXXXX)" || return $?
+ _install_bfcl_eval_deps "$runtime_dir" >&2 || install_rc=$?
+ if [ "$install_rc" -ne 0 ]; then
+ rm -rf "$runtime_dir"
+ return "$install_rc"
+ fi
+ printf '%s\n' "$runtime_dir"
+}
+
+_archive_bfcl_upstream_artifacts() {
+ local project_root="$1"
+ local archive_path="$2"
+
+ "${VENDOR_VERIFIER_PYTHON:-python3}" - "$project_root" "$archive_path" <<'PY'
+import gzip
+import os
+from pathlib import Path
+import tarfile
+import sys
+
+project_root = Path(sys.argv[1])
+archive_path = Path(sys.argv[2])
+temporary_path = archive_path.with_name(f".{archive_path.name}.tmp")
+temporary_path.unlink(missing_ok=True)
+
+try:
+ with (
+ temporary_path.open("xb") as raw_archive,
+ gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as compressed,
+ tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive,
+ ):
+ for path in sorted(
+ project_root.rglob("*"),
+ key=lambda candidate: candidate.relative_to(project_root).as_posix(),
+ ):
+ relative_path = path.relative_to(project_root).as_posix()
+ if path.is_symlink():
+ raise ValueError(f"refusing to archive symbolic link: {relative_path}")
+ info = archive.gettarinfo(str(path), arcname=relative_path)
+ info.uid = 0
+ info.gid = 0
+ info.uname = ""
+ info.gname = ""
+ info.mtime = 0
+ if info.isdir():
+ archive.addfile(info)
+ elif info.isfile():
+ with path.open("rb") as source:
+ archive.addfile(info, source)
+ else:
+ raise ValueError(f"refusing to archive special file: {relative_path}")
+ os.replace(temporary_path, archive_path)
+except BaseException:
+ temporary_path.unlink(missing_ok=True)
+ raise
+PY
+}
+
+_write_bfcl_integration_error() {
+ local adapter_path="$1"
+ local model_name="$2"
+ local results_dir="$3"
+ local message="$4"
+ local suite="$5"
+ local adapter_rc=0
+
+ # Integration errors deliberately make the adapter exit nonzero after
+ # publishing both score artifacts. Treat those artifacts, not that expected
+ # status, as proof that failure reporting succeeded.
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" \
+ --model "$model_name" \
+ --output-dir "$results_dir" \
+ --suite "$suite" \
+ --integration-error "$message" \
+ || adapter_rc=$?
+ if [ -f "${results_dir}/bfcl_report.json" ] \
+ && [ -f "${results_dir}/results_bfcl.json" ]; then
+ return 0
+ fi
+ if [ "$adapter_rc" -eq 0 ]; then
+ return 1
+ fi
+ return "$adapter_rc"
+}
+
+_run_bfcl_suite_eval() {
+ local eval_suite="$1"
+ local num_threads="$2"
+ local process_timeout_seconds="$3"
+ local archive_upstream="$4"
+ shift 4
+
+ local port="${PORT:-8888}"
+ local results_dir="${EVAL_RESULT_DIR:-}"
+
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --port|--results-dir)
+ if [[ $# -lt 2 || -z "${2:-}" || "${2:-}" == --* ]]; then
+ echo "ERROR: $1 requires a value" >&2
+ return 2
+ fi
+ case "$1" in
+ --port) port="$2" ;;
+ --results-dir) results_dir="$2" ;;
+ esac
+ shift 2
+ ;;
+ *)
+ echo "Unknown parameter: $1" >&2
+ return 2
+ ;;
+ esac
+ done
+
+ if [ -z "$results_dir" ]; then
+ results_dir="$(mktemp -d /tmp/eval_out-XXXXXX)" || return $?
+ fi
+
+ local model_name="${MODEL_NAME:-${MODEL:-}}"
+ local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/bfcl_adapter.py"
+ local runtime_dir=""
+ local project_root=""
+
+ mkdir -p "$results_dir" || return $?
+ results_dir="$(cd "$results_dir" && pwd)" || return $?
+ _prepare_eval_artifact_family "$results_dir" bfcl || return $?
+
+ local setup_rc=0 integration_error=""
+ _prepare_vendor_verifier_python "BFCL" "bfcl-python" true 10 || {
+ setup_rc=$?
+ integration_error="BFCL Python runtime preparation failed with exit code ${setup_rc}"
+ }
+ if [ "$setup_rc" -eq 0 ]; then
+ runtime_dir=$(_prepare_bfcl_runtime) || {
+ setup_rc=$?
+ integration_error="BFCL dependency installation failed with exit code ${setup_rc}"
+ }
+ fi
+ if [ "$setup_rc" -eq 0 ]; then
+ project_root="$(mktemp -d /tmp/bfcl-project-root-XXXXXX)" || {
+ setup_rc=$?
+ integration_error="BFCL project root preparation failed with exit code ${setup_rc}"
+ }
+ fi
+ if [ "$setup_rc" -ne 0 ]; then
+ echo "ERROR: ${integration_error}" >&2
+ local artifact_rc=0
+ _write_bfcl_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$integration_error" \
+ "$eval_suite" || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write BFCL failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "$project_root" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ return "$setup_rc"
+ fi
+
+ local eval_rc=0
+ local -a suite_args=()
+ if [ "$eval_suite" != "bfcl_smoke" ]; then
+ suite_args=(--suite "$eval_suite")
+ fi
+ timeout "$process_timeout_seconds" \
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" \
+ --base-url "http://127.0.0.1:${port}/v1" \
+ --api-key EMPTY \
+ --model "$model_name" \
+ --output-dir "$results_dir" \
+ --bfcl-project-root "$project_root" \
+ "${suite_args[@]}" \
+ --num-threads "$num_threads" \
+ || eval_rc=$?
+ local archive_rc=0
+ if [ "$archive_upstream" = true ]; then
+ _archive_bfcl_upstream_artifacts \
+ "$project_root" "${results_dir}/bfcl_upstream_artifacts.tar.gz" \
+ || archive_rc=$?
+ if [ "$archive_rc" -ne 0 ]; then
+ echo "ERROR: failed to archive BFCL upstream artifacts (exit code ${archive_rc})" >&2
+ fi
+ fi
+ if [ "$eval_rc" -ne 0 ] \
+ && { [ ! -f "${results_dir}/bfcl_report.json" ] \
+ || [ ! -f "${results_dir}/results_bfcl.json" ]; }; then
+ local integration_error="BFCL evaluation failed with exit code ${eval_rc}"
+ local artifact_rc=0
+ _write_bfcl_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$integration_error" \
+ "$eval_suite" || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write BFCL failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "$project_root" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ if [ "$eval_rc" -ne 0 ]; then
+ return "$eval_rc"
+ fi
+ return "$archive_rc"
+}
+
+
+_run_bfcl_smoke_eval() {
+ _run_bfcl_suite_eval bfcl_smoke 4 900 false "$@"
+}
+
+run_bfcl_eval() {
+ local eval_suite="${EVAL_SUITE:-bfcl_smoke}"
+ export EVAL_SUITE="$eval_suite"
+
+ case "$eval_suite" in
+ bfcl_smoke)
+ _run_bfcl_smoke_eval "$@"
+ ;;
+ bfcl_vllm_minimax_m3)
+ _run_bfcl_suite_eval "$eval_suite" 8 7200 true "$@"
+ ;;
+ bfcl_vllm_kimi)
+ _run_bfcl_suite_eval "$eval_suite" 16 14400 true "$@"
+ ;;
+ *)
+ echo "ERROR: unsupported BFCL suite '${eval_suite}'" >&2
+ export EVAL_RESULT_DIR=""
+ return 2
+ ;;
+ esac
+}
+
+
+_write_minimax_vendor_integration_error() {
+ local adapter_path="$1"
+ local model_name="$2"
+ local results_dir="$3"
+ local message="$4"
+
+ # The failure path is stdlib-only, so it remains usable when runtime
+ # provisioning or dependency installation is what failed.
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" failure \
+ --model "$model_name" \
+ --output-dir "$results_dir" \
+ --message "$message"
+}
+
+_run_minimax_m3_smoke_eval() {
+ local port="${PORT:-8888}"
+ local results_dir="${EVAL_RESULT_DIR:-}"
+
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --port|--results-dir)
+ if [[ $# -lt 2 || -z "${2:-}" || "${2:-}" == --* ]]; then
+ echo "ERROR: $1 requires a value" >&2
+ return 2
+ fi
+ case "$1" in
+ --port) port="$2" ;;
+ --results-dir) results_dir="$2" ;;
+ esac
+ shift 2
+ ;;
+ *)
+ echo "Unknown parameter: $1" >&2
+ return 2
+ ;;
+ esac
+ done
+
+ if [ -z "$results_dir" ]; then
+ results_dir="$(mktemp -d /tmp/eval_out-XXXXXX)" || return $?
+ fi
+
+ local model_name="${MODEL_NAME:-${MODEL:-}}"
+ local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/minimax_provider_eval.py"
+ local fixture_path="${INFERENCEX_REPO_ROOT}/utils/evals/minimax_m3_smoke.json"
+ local runtime_dir=""
+
+ mkdir -p "$results_dir" || return $?
+ results_dir="$(cd "$results_dir" && pwd)" || return $?
+ _prepare_eval_artifact_family "$results_dir" minimax || return $?
+
+ local setup_rc=0 integration_error=""
+ _prepare_vendor_verifier_python "MiniMax Provider Verifier" "minimax-vendor-python" || {
+ setup_rc=$?
+ integration_error="MiniMax Provider Verifier Python runtime preparation failed with exit code ${setup_rc}"
+ }
+ if [ "$setup_rc" -eq 0 ]; then
+ runtime_dir=$(_prepare_minimax_m3_full_runtime) || {
+ setup_rc=$?
+ integration_error="MiniMax Provider Verifier pinned runtime preparation failed with exit code ${setup_rc}"
+ }
+ fi
+ if [ "$setup_rc" -ne 0 ]; then
+ echo "ERROR: ${integration_error}" >&2
+ local artifact_rc=0
+ _write_minimax_vendor_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$integration_error" \
+ || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write MiniMax verifier failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ return "$setup_rc"
+ fi
+
+ local eval_rc=0
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" run \
+ --python "${VENDOR_VERIFIER_PYTHON:-python3}" \
+ --source-dir "${runtime_dir}/source" \
+ --dependency-dir "${runtime_dir}/deps" \
+ --base-url "http://127.0.0.1:${port}/v1" \
+ --model "$model_name" \
+ --output-dir "$results_dir" \
+ --fixture "$fixture_path" \
+ || eval_rc=$?
+ if [ "$eval_rc" -ne 0 ] \
+ && ! _has_eval_result "$results_dir" "results_minimax_vendor_"; then
+ integration_error="MiniMax Provider Verifier failed with exit code ${eval_rc}"
+ local artifact_rc=0
+ _write_minimax_vendor_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$integration_error" \
+ || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write MiniMax verifier failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ return "$eval_rc"
+}
+
+_install_minimax_m3_full_deps() {
+ local target_dir="$1"
+ "${VENDOR_VERIFIER_PYTHON:-python3}" -m pip install -q --no-cache-dir --target "$target_dir" \
+ "jsonschema==4.25.1" \
+ "loguru==0.7.3" \
+ "megfile==4.2.5" \
+ "numpy==2.3.4" \
+ "openai==2.7.1" \
+ "tqdm==4.67.1"
+}
+
+_prepare_minimax_m3_full_runtime() {
+ local source_adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/minimax_m3_full_eval.py"
+ local runtime_dir prepare_rc=0
+ runtime_dir="$(mktemp -d /tmp/minimax-m3-full-runtime-XXXXXX)" || return $?
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$source_adapter_path" prepare-source \
+ --source-dir "${runtime_dir}/source" >&2 || prepare_rc=$?
+ if [ "$prepare_rc" -eq 0 ]; then
+ _install_minimax_m3_full_deps "${runtime_dir}/deps" >&2 || prepare_rc=$?
+ fi
+ if [ "$prepare_rc" -ne 0 ]; then
+ rm -rf "$runtime_dir"
+ return "$prepare_rc"
+ fi
+ printf '%s\n' "$runtime_dir"
+}
+
+_write_minimax_m3_full_integration_error() {
+ local adapter_path="$1"
+ local model_name="$2"
+ local results_dir="$3"
+ local message="$4"
+
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" failure \
+ --model "$model_name" \
+ --output-dir "$results_dir" \
+ --message "$message"
+}
+
+_run_minimax_m3_full_eval() {
+ local port="${PORT:-8888}"
+ local results_dir="${EVAL_RESULT_DIR:-}"
+
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --port|--results-dir)
+ if [[ $# -lt 2 || -z "${2:-}" || "${2:-}" == --* ]]; then
+ echo "ERROR: $1 requires a value" >&2
+ return 2
+ fi
+ case "$1" in
+ --port) port="$2" ;;
+ --results-dir) results_dir="$2" ;;
+ esac
+ shift 2
+ ;;
+ *)
+ echo "Unknown parameter: $1" >&2
+ return 2
+ ;;
+ esac
+ done
+
+ if [ -z "$results_dir" ]; then
+ results_dir="$(mktemp -d /tmp/eval_out-XXXXXX)" || return $?
+ fi
+
+ local model_name="${MODEL_NAME:-${MODEL:-}}"
+ local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/minimax_m3_full_eval.py"
+ local runtime_dir=""
+
+ mkdir -p "$results_dir" || return $?
+ results_dir="$(cd "$results_dir" && pwd)" || return $?
+ _prepare_eval_artifact_family "$results_dir" minimax || return $?
+
+ local setup_rc=0 integration_error=""
+ _prepare_vendor_verifier_python "MiniMax M3 full verifier" "minimax-m3-full-python" || {
+ setup_rc=$?
+ integration_error="MiniMax M3 full Python runtime preparation failed with exit code ${setup_rc}"
+ }
+ if [ "$setup_rc" -eq 0 ]; then
+ runtime_dir=$(_prepare_minimax_m3_full_runtime) || {
+ setup_rc=$?
+ integration_error="MiniMax M3 full pinned runtime preparation failed with exit code ${setup_rc}"
+ }
+ fi
+ if [ "$setup_rc" -ne 0 ]; then
+ echo "ERROR: ${integration_error}" >&2
+ local artifact_rc=0
+ _write_minimax_m3_full_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$integration_error" \
+ || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write MiniMax full verifier failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ return "$setup_rc"
+ fi
+
+ local eval_rc=0
+ "${VENDOR_VERIFIER_PYTHON:-python3}" "$adapter_path" run \
+ --python "${VENDOR_VERIFIER_PYTHON:-python3}" \
+ --source-dir "${runtime_dir}/source" \
+ --dependency-dir "${runtime_dir}/deps" \
+ --base-url "http://127.0.0.1:${port}/v1" \
+ --model "$model_name" \
+ --output-dir "$results_dir" \
+ || eval_rc=$?
+ if [ "$eval_rc" -ne 0 ] \
+ && ! _has_eval_result "$results_dir" "results_minimax_vendor_full_"; then
+ integration_error="MiniMax M3 full verifier failed with exit code ${eval_rc}"
+ local artifact_rc=0
+ _write_minimax_m3_full_integration_error \
+ "$adapter_path" "$model_name" "$results_dir" "$integration_error" \
+ || artifact_rc=$?
+ if [ "$artifact_rc" -ne 0 ]; then
+ echo "ERROR: failed to write MiniMax full verifier failure artifact (exit code ${artifact_rc})" >&2
+ fi
+ fi
+ _cleanup_vendor_eval \
+ "$runtime_dir" "${VENDOR_VERIFIER_PYTHON_CLEANUP_DIR:-}"
+ return "$eval_rc"
+}
+
+
+run_minimax_vendor_eval() {
+ local eval_suite="${EVAL_SUITE:-minimax_m3_smoke}"
+ export EVAL_SUITE="$eval_suite"
+
+ case "$eval_suite" in
+ minimax_m3_smoke)
+ _run_minimax_m3_smoke_eval "$@"
+ ;;
+ minimax_m3_full)
+ _run_minimax_m3_full_eval "$@"
+ ;;
+ *)
+ echo "ERROR: unsupported MiniMax Provider Verifier suite '${eval_suite}'" >&2
+ export EVAL_RESULT_DIR=""
+ return 2
+ ;;
+ esac
+}
+
_eval_patches_dir() {
- cd "$(dirname "${BASH_SOURCE[0]}")/../utils/evals/patches" && pwd
+ printf '%s\n' "${INFERENCEX_REPO_ROOT}/utils/evals/patches"
}
_patch_lm_eval() {
@@ -996,14 +2077,15 @@ run_lm_eval() {
done
# Serving images may use a different WORKDIR.
- local _repo_root
- _repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+ local _repo_root="$INFERENCEX_REPO_ROOT"
if [[ "$tasks_dir" == *.yaml && "$tasks_dir" != /* \
&& ! -f "$tasks_dir" && -f "$_repo_root/$tasks_dir" ]]; then
echo "run_lm_eval: anchoring relative task '$tasks_dir' to repo root -> $_repo_root/$tasks_dir"
tasks_dir="$_repo_root/$tasks_dir"
fi
+ export EVAL_TASKS_DIR="$tasks_dir"
+
if [ "${INFERENCEX_LM_EVAL_RUNTIME_READY:-false}" != "true" ]; then
_install_lm_eval_deps
_patch_lm_eval
@@ -1119,8 +2201,8 @@ _eval_concs_to_json() {
}
_env_is_true() {
- case "${1,,}" in
- 1|true|yes|on) return 0 ;;
+ case "${1:-}" in
+ 1|[Tt][Rr][Uu][Ee]|[Yy][Ee][Ss]|[Oo][Nn]) return 0 ;;
*) return 1 ;;
esac
}
@@ -1150,11 +2232,11 @@ _normalize_bool_json() {
bridge_disagg_eval_metadata() {
export TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-${TP:-1}}}"
export PREFILL_TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-${TP:-1}}}"
- export PREFILL_EP="$(_resolve_disagg_ep "${PREFILL_EP:-1}" "${PREFILL_ENABLE_EP:-false}" "${PREFILL_TP_SIZE:-${PREFILL_TP:-1}}")"
+ export PREFILL_EP="$(_resolve_disagg_ep "${PREFILL_EP:-${EP_SIZE:-${EP:-1}}}" "${PREFILL_ENABLE_EP:-false}" "${PREFILL_TP_SIZE:-${PREFILL_TP:-1}}")"
export EP_SIZE="${PREFILL_EP}"
export PREFILL_NUM_WORKERS="${PREFILL_NUM_WORKERS:-${xP:-1}}"
export DECODE_TP="${DECODE_TP:-${DECODE_TP_SIZE:-${TP:-1}}}"
- export DECODE_EP="$(_resolve_disagg_ep "${DECODE_EP:-1}" "${DECODE_ENABLE_EP:-false}" "${DECODE_TP_SIZE:-${DECODE_TP:-1}}")"
+ export DECODE_EP="$(_resolve_disagg_ep "${DECODE_EP:-${EP_SIZE:-${EP:-1}}}" "${DECODE_ENABLE_EP:-false}" "${DECODE_TP_SIZE:-${DECODE_TP:-1}}")"
export DECODE_NUM_WORKERS="${DECODE_NUM_WORKERS:-${yD:-1}}"
local prefill_dp="${PREFILL_DP_ATTN:-${PREFILL_DP_ATTENTION:-${PREFILL_ENABLE_DP:-false}}}"
@@ -1213,6 +2295,13 @@ _write_lm_eval_meta_json() {
fi
fi
fi
+ local eval_suite="${EVAL_COMPLETED_SUITE:-${EVAL_SUITE:-}}"
+ if [ -z "$eval_suite" ] && [ -n "${EVAL_TASKS_DIR:-}" ]; then
+ eval_suite="$(basename "${EVAL_TASKS_DIR}")"
+ eval_suite="${eval_suite%.yaml}"
+ eval_suite="${eval_suite%.yml}"
+ fi
+ eval_suite="${eval_suite:-gsm8k}"
cat > "${meta_json}" <&2
- fi
- if [ -d "${out_dir}" ]; then
- while IFS= read -r -d '' jf; do
- base=$(basename "$jf")
- if [ "$base" != "meta_env.json" ]; then
- mv -f "$jf" ./ || echo "WARN: failed to move ${jf}" >&2
- fi
- done < <(find "${out_dir}" -type f -name "*.json*" -print0 2>/dev/null)
- fi
+ # Copy the complete allowlisted eval artifact set before removing its temp dir.
+ stage_eval_artifacts "$(pwd)" "$out_dir" || return $?
# Best-effort cleanup of the temp directory
if [ -n "${out_dir}" ] && [ -d "${out_dir}" ]; then
rm -rf --one-file-system "${out_dir}" || rm -rf "${out_dir}" || true
fi
- echo "Moved eval artifacts to: $(pwd)"
+ echo "Staged eval artifacts in: $(pwd)"
+}
+
+stage_eval_artifacts() {
+ local destination="$1"
+ shift
+
+ mkdir -p "$destination" || return $?
+ local source_dir artifact
+ local copied=0
+ local artifacts=()
+ for source_dir in "$@"; do
+ [ -d "$source_dir" ] || continue
+ artifacts=(
+ "$source_dir"/meta_env.json
+ "$source_dir"/results*.json
+ "$source_dir"/*_report.json
+ "$source_dir"/*_results.jsonl
+ "$source_dir"/*_artifacts.tar.gz
+ "$source_dir"/sample*.jsonl
+ "$source_dir"/agent_preds.json
+ "$source_dir"/swebench_report_*.json
+ "$source_dir"/predictions.jsonl
+ "$source_dir"/*.traj*
+ )
+ for artifact in "${artifacts[@]}"; do
+ [ -f "$artifact" ] || continue
+ cp -f "$artifact" "$destination/" || return $?
+ copied=$((copied + 1))
+ done
+ done
+ if [ "$copied" -eq 0 ]; then
+ echo "ERROR: no eval artifacts found to stage" >&2
+ return 1
+ fi
}
@@ -1545,6 +2663,7 @@ PYSWEEP
run_swebench_eval() {
local out_dir="${EVAL_RESULT_DIR:-$(mktemp -d /tmp/eval_out-XXXXXX)}"
local task_name="${SWEBENCH_TASK_NAME:-swebench_lite}"
+ export EVAL_SUITE="${EVAL_SUITE:-$task_name}"
local gen_dir
gen_dir=$(mktemp -d /tmp/swebench_gen-XXXXXX)
@@ -1653,6 +2772,101 @@ run_swebench_eval() {
fi
}
+_wait_for_openai_chat_route() {
+ local port="${PORT:-8888}"
+ local timeout_seconds="${EVAL_ENDPOINT_READY_TIMEOUT_SECONDS:-1800}"
+ local poll_seconds=5
+ local stabilization_seconds="${EVAL_MODEL_STABILIZATION_SECONDS:-30}"
+ local start_seconds=$SECONDS
+ local server_ready_since=-1
+ local next_report=0
+ local elapsed percent chat_status
+ local served_model="${SERVED_MODEL_NAME:-${MODEL:-}}"
+ local health_url models_url chat_url
+
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --port)
+ if [[ $# -lt 2 || -z "${2:-}" || "${2:-}" == --* ]]; then
+ echo "ERROR: --port requires a value" >&2
+ return 2
+ fi
+ port="$2"
+ shift 2
+ ;;
+ *) shift ;;
+ esac
+ done
+ if ! [[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then
+ echo "ERROR: EVAL_ENDPOINT_READY_TIMEOUT_SECONDS must be a positive integer" >&2
+ return 2
+ fi
+ if ! [[ "$stabilization_seconds" =~ ^[0-9]+$ ]]; then
+ echo "ERROR: EVAL_MODEL_STABILIZATION_SECONDS must be a non-negative integer" >&2
+ return 2
+ fi
+ if [ -z "$served_model" ]; then
+ echo "ERROR: MODEL or SERVED_MODEL_NAME is required for chat endpoint readiness" >&2
+ return 2
+ fi
+ health_url="http://localhost:${port}/health"
+ models_url="http://localhost:${port}/v1/models"
+ chat_url="http://localhost:${port}/v1/chat/completions"
+
+ while true; do
+ local model_ready=false
+ local server_ready=false
+ if curl -fsS --max-time 10 "$health_url" >/dev/null 2>&1; then
+ server_ready=true
+ fi
+ if curl -fsS --max-time 10 "$models_url" 2>/dev/null \
+ | python3 -c '
+import json
+import sys
+
+expected = sys.argv[1]
+payload = json.load(sys.stdin)
+models = payload.get("data", [])
+raise SystemExit(0 if any(model.get("id") == expected for model in models) else 1)
+' "$served_model" >/dev/null 2>&1; then
+ model_ready=true
+ fi
+
+ chat_status=""
+ if [ "$model_ready" = true ]; then
+ chat_status="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
+ "$chat_url" 2>/dev/null)" || true
+ case "$chat_status" in
+ 401|403|405) break ;;
+ esac
+ fi
+ if [ "$server_ready" = true ]; then
+ if [ "$server_ready_since" -lt 0 ]; then
+ server_ready_since=$SECONDS
+ fi
+ if [ $((SECONDS - server_ready_since)) -ge "$stabilization_seconds" ]; then
+ break
+ fi
+ else
+ server_ready_since=-1
+ fi
+
+ elapsed=$((SECONDS - start_seconds))
+ if [ "$elapsed" -ge "$timeout_seconds" ]; then
+ echo "ERROR: chat endpoint for model '$served_model' did not become ready within ${timeout_seconds}s: $chat_url" >&2
+ return 1
+ fi
+ if [ "$elapsed" -ge "$next_report" ]; then
+ percent=$((elapsed * 100 / timeout_seconds))
+ echo "Waiting for chat endpoint for model '$served_model': ${elapsed}/${timeout_seconds}s (${percent}%)"
+ next_report=$((next_report + 60))
+ fi
+ sleep "$poll_seconds"
+ done
+ echo "OpenAI chat endpoint ready for model '$served_model': $chat_url"
+}
+
+
# ------------------------------
# Unified eval entrypoint
# ------------------------------
@@ -1660,6 +2874,9 @@ run_swebench_eval() {
run_eval() {
local cli_framework=""
local forwarded=()
+ # Keep runner-selected suite identity scoped to this invocation.
+ local EVAL_SUITE="${EVAL_SUITE:-}"
+ unset EVAL_COMPLETED_SUITE
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -1685,9 +2902,48 @@ run_eval() {
fi
local framework="${EVAL_FRAMEWORK:-${cli_framework:-$scenario_default}}"
+ case "$framework" in
+ kimi-vendor)
+ [ -n "${EVAL_SUITE:-}" ] || EVAL_SUITE="kimi_tool_call_schema"
+ ;;
+ minimax-vendor)
+ [ -n "${EVAL_SUITE:-}" ] || EVAL_SUITE="minimax_m3_smoke"
+ ;;
+ bfcl)
+ [ -n "${EVAL_SUITE:-}" ] || EVAL_SUITE="bfcl_smoke"
+ ;;
+ esac
+
+ case "${EVAL_SUITE:-}" in
+ "") ;;
+ *[!A-Za-z0-9_.-]*)
+ echo "ERROR: EVAL_SUITE may contain only letters, digits, '.', '_', and '-'" >&2
+ return 2
+ ;;
+ esac
+
+ if [ -n "${EVAL_SUITE:-}" ] \
+ && [ "$framework" != "kimi-vendor" ] \
+ && [ "$framework" != "minimax-vendor" ] \
+ && [ "$framework" != "bfcl" ]; then
+ echo "ERROR: EVAL_SUITE is only supported with kimi-vendor, minimax-vendor, or bfcl" >&2
+ return 2
+ fi
- # Compute EVAL_MAX_MODEL_LEN if not already set by the calling script
- if [ -z "${EVAL_MAX_MODEL_LEN:-}" ]; then
+ if [ "${EVAL_ONLY:-false}" = "true" ]; then
+ case "$framework" in
+ kimi-vendor|minimax-vendor|bfcl)
+ _wait_for_openai_chat_route "${forwarded[@]}" || return $?
+ ;;
+ esac
+ fi
+
+ # Explicit verifier suites use fixed request budgets and do not consume
+ # EVAL_MAX_MODEL_LEN, so avoid loading model configuration for those paths.
+ if [ "$framework" != "kimi-vendor" ] \
+ && [ "$framework" != "minimax-vendor" ] \
+ && [ "$framework" != "bfcl" ] \
+ && [ -z "${EVAL_MAX_MODEL_LEN:-}" ]; then
compute_eval_context_length "$MODEL" "${MAX_MODEL_LEN:-0}" > /dev/null
fi
@@ -1754,26 +3010,46 @@ run_eval() {
return 0
fi
+ if [ -n "${EVAL_CONCURRENT_REQUESTS:-}" ]; then
+ export CONC="$EVAL_CONCURRENT_REQUESTS"
+ fi
+
local eval_rc=0
case "$framework" in
lm-eval|lm_eval) run_lm_eval "${forwarded[@]}" || eval_rc=$? ;;
swebench) run_swebench_eval "${forwarded[@]}" || eval_rc=$? ;;
+ kimi-vendor) run_kimi_vendor_eval "${forwarded[@]}" || eval_rc=$? ;;
+ minimax-vendor) run_minimax_vendor_eval "${forwarded[@]}" || eval_rc=$? ;;
+ bfcl) run_bfcl_eval "${forwarded[@]}" || eval_rc=$? ;;
*) echo "Unknown framework '${framework}'"; eval_rc=1 ;;
esac
- # Agentic eval-only recipes have no separate staging step.
- if [ "${EVAL_ONLY:-false}" = "true" ] && [ "$scenario_is_agentic" = "1" ]; then
- append_lm_eval_summary || true
+ if [ -n "${EVAL_SUITE:-}" ]; then
+ export EVAL_COMPLETED_SUITE="$EVAL_SUITE"
fi
+ local stage_rc=0
+ # Agentic eval-only recipes have no separate staging step. Provider
+ # failures are staged before returning so diagnostic artifacts survive.
+ if { [ "${EVAL_ONLY:-false}" = "true" ] && [ "$scenario_is_agentic" = "1" ]; } \
+ || { { [ "$framework" = "kimi-vendor" ] \
+ || [ "$framework" = "minimax-vendor" ] \
+ || [ "$framework" = "bfcl" ]; } \
+ && [ "$eval_rc" -ne 0 ]; }; then
+ append_lm_eval_summary || stage_rc=$?
+ fi
if [ "$eval_rc" -ne 0 ]; then
echo "ERROR: run_eval failed with exit code $eval_rc" >&2
- if [ "${EVAL_ONLY}" = "true" ]; then
+ if [ "${EVAL_ONLY:-false}" = "true" ]; then
echo "Eval-only mode: failing after artifact collection" >&2
- return "$eval_rc"
fi
+ return "$eval_rc"
+ fi
+ if [ "$stage_rc" -ne 0 ]; then
+ echo "ERROR: eval artifact staging failed with exit code $stage_rc" >&2
+ return "$stage_rc"
fi
- return $eval_rc
+ return 0
}
@@ -1829,13 +3105,8 @@ install_agentic_deps() {
return
fi
- # AIPerf must not share site-packages with the inference server. Installing
- # it into vLLM/SGLang's system Python can upgrade FastAPI, Starlette,
- # transformers, or other packages while the server imports from that same
- # environment.
- if ! command -v git >/dev/null 2>&1; then
- apt-get update && apt-get install -y git
- fi
+ # Install from the checked-out aiperf source with uv. This path does not
+ # require git, and rootless Enroot containers cannot mutate dpkg.
ensure_agentic_uv
rm -rf "$AIPERF_VENV"
diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh
index 0ea94e0bde..e62123a99a 100644
--- a/benchmarks/multi_node/agentic_srt.sh
+++ b/benchmarks/multi_node/agentic_srt.sh
@@ -39,8 +39,12 @@ for concurrency in "${CONCURRENCIES[@]}"; do
fi
done
+
resolve_trace_source
install_agentic_deps
+if [[ "${EVAL_ONLY:-false}" == "true" ]]; then
+ _wait_for_openai_chat_route --port "$PORT"
+fi
wait_for_agentic_servers_idle() {
local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}"
diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index edf83015a9..cb6f4ad4d1 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -323,6 +323,9 @@ export ENGINE=$ENGINE
export RUN_EVAL="${RUN_EVAL:-false}"
export EVAL_ONLY="${EVAL_ONLY:-false}"
export EVAL_CONC="${EVAL_CONC:-}"
+export EVAL_FRAMEWORK="${EVAL_FRAMEWORK:-lm-eval}"
+export EVAL_SUITE="${EVAL_SUITE:-}"
+export SWEBENCH_GEN_MODE="${SWEBENCH_GEN_MODE:-}"
export FRAMEWORK="${FRAMEWORK:-}"
export PRECISION="${PRECISION:-}"
export MODEL_PREFIX="${MODEL_PREFIX:-}"
@@ -430,6 +433,10 @@ DOCKER_ENV_COMMON=(
-e RUN_EVAL=\$RUN_EVAL
-e EVAL_ONLY=\$EVAL_ONLY
-e \"EVAL_CONC=\$EVAL_CONC\"
+ -e EVAL_FRAMEWORK=\$EVAL_FRAMEWORK
+ -e EVAL_LIMIT=\$EVAL_LIMIT
+ -e EVAL_SUITE=\$EVAL_SUITE
+ -e SWEBENCH_GEN_MODE=\$SWEBENCH_GEN_MODE
-e FRAMEWORK=\$FRAMEWORK
-e PRECISION=\$PRECISION
-e MODEL_PREFIX=\$MODEL_PREFIX
diff --git a/benchmarks/multi_node/amd_utils/server_atom.sh b/benchmarks/multi_node/amd_utils/server_atom.sh
index 2fee5679ca..5bad8fd39b 100755
--- a/benchmarks/multi_node/amd_utils/server_atom.sh
+++ b/benchmarks/multi_node/amd_utils/server_atom.sh
@@ -411,13 +411,13 @@ if [ "$NODE_RANK" -eq 0 ]; then
fi
if [[ "$DRY_RUN" -eq 1 ]]; then
- echo "DRY RUN: run_eval --framework lm-eval --port ${ROUTER_PORT} (conc=${EVAL_CONCURRENT_REQUESTS})"
+ echo "DRY RUN: run_eval --port ${ROUTER_PORT} (framework=${EVAL_FRAMEWORK:-lm-eval}, conc=${EVAL_CONCURRENT_REQUESTS})"
else
- MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" run_eval --framework lm-eval --port ${ROUTER_PORT}
+ MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" run_eval --port "${ROUTER_PORT}"
eval_rc=$?
if [[ $eval_rc -ne 0 ]]; then
- echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2
+ echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2
EVAL_FAILED=1
else
export TP="${PREFILL_TP_SIZE}"
@@ -433,15 +433,15 @@ if [ "$NODE_RANK" -eq 0 ]; then
MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" append_lm_eval_summary
- EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results"
- mkdir -p "$EVAL_COPY_DIR"
- for f in meta_env.json; do
- [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/"
- done
- find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \;
- find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \;
+ fi
- echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR"
+ EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results"
+ if stage_eval_artifacts \
+ "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then
+ echo "Eval artifacts staged in $EVAL_COPY_DIR"
+ else
+ echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2
+ EVAL_FAILED=1
fi
fi
diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index aaaca61ef5..7815e5a911 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -1113,13 +1113,13 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || {
# RESULT_FILENAME are already set via Docker -e flags from job.slurm
if [[ "$DRY_RUN" -eq 1 ]]; then
- echo "DRY RUN: run_eval --framework lm-eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})"
+ echo "DRY RUN: run_eval --port 30000 (framework=${EVAL_FRAMEWORK:-lm-eval}, conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})"
else
- run_eval --framework lm-eval --port 30000
+ run_eval --port 30000
eval_rc=$?
if [[ $eval_rc -ne 0 ]]; then
- echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2
+ echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2
EVAL_FAILED=1
else
# Always rewrite meta_env.json so EP/DPA match the workflow
@@ -1132,15 +1132,15 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || {
append_lm_eval_summary
fi
- EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results"
- mkdir -p "$EVAL_COPY_DIR"
- for f in meta_env.json; do
- [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/"
- done
- find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \;
- find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \;
+ fi
- echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR"
+ EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results"
+ if stage_eval_artifacts \
+ "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then
+ echo "Eval artifacts staged in $EVAL_COPY_DIR"
+ else
+ echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2
+ EVAL_FAILED=1
fi
fi
diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh
index 55154cd015..5a6ce23577 100755
--- a/benchmarks/multi_node/amd_utils/server_vllm.sh
+++ b/benchmarks/multi_node/amd_utils/server_vllm.sh
@@ -355,13 +355,13 @@ if [ "$NODE_RANK" -eq 0 ]; then
fi
if [[ "$DRY_RUN" -eq 1 ]]; then
- echo "DRY RUN: run_eval --framework lm-eval --port $ROUTER_PORT (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})"
+ echo "DRY RUN: run_eval --port $ROUTER_PORT (framework=${EVAL_FRAMEWORK:-lm-eval}, conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})"
else
- run_eval --framework lm-eval --port "$ROUTER_PORT"
+ run_eval --port "$ROUTER_PORT"
eval_rc=$?
if [[ $eval_rc -ne 0 ]]; then
- echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2
+ echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2
EVAL_FAILED=1
else
export TP="${PREFILL_TP_SIZE}"
@@ -384,15 +384,15 @@ if [ "$NODE_RANK" -eq 0 ]; then
append_lm_eval_summary
- EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results"
- mkdir -p "$EVAL_COPY_DIR"
- for f in meta_env.json; do
- [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/"
- done
- find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \;
- find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \;
+ fi
- echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR"
+ EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results"
+ if stage_eval_artifacts \
+ "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then
+ echo "Eval artifacts staged in $EVAL_COPY_DIR"
+ else
+ echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2
+ EVAL_FAILED=1
fi
fi
diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh
index e7ecc8d913..3b7552492f 100755
--- a/benchmarks/multi_node/amd_utils/submit.sh
+++ b/benchmarks/multi_node/amd_utils/submit.sh
@@ -152,6 +152,9 @@ export DRY_RUN="${DRY_RUN:-0}"
export RUN_EVAL="${RUN_EVAL:-false}"
export EVAL_ONLY="${EVAL_ONLY:-false}"
export EVAL_CONC="${EVAL_CONC:-}"
+export EVAL_FRAMEWORK="${EVAL_FRAMEWORK:-lm-eval}"
+export EVAL_SUITE="${EVAL_SUITE:-}"
+export SWEBENCH_GEN_MODE="${SWEBENCH_GEN_MODE:-}"
export FRAMEWORK="${FRAMEWORK:-}"
export PRECISION="${PRECISION:-}"
export MODEL_PREFIX="${MODEL_PREFIX:-}"
diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm
index f260c1a6fb..536507bee5 100644
--- a/benchmarks/multi_node/llm-d/job.slurm
+++ b/benchmarks/multi_node/llm-d/job.slurm
@@ -150,6 +150,15 @@ exec docker run --rm \
-e RUN_EVAL=$RUN_EVAL \
-e EVAL_ONLY=$EVAL_ONLY \
-e EVAL_CONC=$EVAL_CONC \
+ -e EVAL_FRAMEWORK=$EVAL_FRAMEWORK \
+ -e EVAL_LIMIT=$EVAL_LIMIT \
+ -e EVAL_SUITE=$EVAL_SUITE \
+ -e SWEBENCH_GEN_MODE=$SWEBENCH_GEN_MODE \
+ -e SWEBENCH_USE_MODAL=$SWEBENCH_USE_MODAL \
+ -e MODAL_TOKEN_ID \
+ -e MODAL_TOKEN_SECRET \
+ -e IS_AGENTIC=$IS_AGENTIC \
+ -e SCENARIO_TYPE=$SCENARIO_TYPE \
-e FRAMEWORK=$FRAMEWORK \
-e PRECISION=$PRECISION \
-e MODEL_PREFIX=$MODEL_PREFIX \
@@ -186,7 +195,9 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then
export PREFILL_DP_SIZE DECODE_DP_SIZE
export BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY
export BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER
- export RUN_EVAL EVAL_ONLY EVAL_CONC FRAMEWORK PRECISION MODEL_PREFIX
+ export RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE
+ export SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET
+ export IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX
export RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE
PYXIS_ENV_LIST="NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP"
@@ -195,7 +206,9 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then
PYXIS_ENV_LIST+=",PREFILL_DP_SIZE,DECODE_DP_SIZE"
PYXIS_ENV_LIST+=",BENCH_INPUT_LEN,BENCH_OUTPUT_LEN,BENCH_MAX_CONCURRENCY"
PYXIS_ENV_LIST+=",BENCH_REQUEST_RATE,BENCH_RANDOM_RANGE_RATIO,BENCH_NUM_PROMPTS_MULTIPLIER"
- PYXIS_ENV_LIST+=",RUN_EVAL,EVAL_ONLY,EVAL_CONC,FRAMEWORK,PRECISION,MODEL_PREFIX"
+ PYXIS_ENV_LIST+=",RUN_EVAL,EVAL_ONLY,EVAL_CONC,EVAL_FRAMEWORK,EVAL_LIMIT,EVAL_SUITE"
+ PYXIS_ENV_LIST+=",SWEBENCH_GEN_MODE,SWEBENCH_USE_MODAL,MODAL_TOKEN_ID,MODAL_TOKEN_SECRET"
+ PYXIS_ENV_LIST+=",IS_AGENTIC,SCENARIO_TYPE,FRAMEWORK,PRECISION,MODEL_PREFIX"
PYXIS_ENV_LIST+=",RUNNER_TYPE,RESULT_FILENAME,SPEC_DECODING,IS_MULTINODE,CONFIG_FILE"
PYXIS_MOUNTS="${MODEL_DIR}:/models:ro"
diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh
index 7e189767bc..89894b210c 100755
--- a/benchmarks/multi_node/llm-d/server.sh
+++ b/benchmarks/multi_node/llm-d/server.sh
@@ -562,6 +562,7 @@ PY
_bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE ))
_bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE ))
_bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus ))
+ if [[ "${EVAL_ONLY:-false}" != "true" ]]; then
for max_concurrency in "${CONCURRENCIES[@]}"; do
num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER ))
[[ "$num_prompts" -lt 16 ]] && num_prompts=16
@@ -601,6 +602,7 @@ PY
"${bench_extra_args[@]}" \
|| echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)"
done
+ fi
# ---- Eval (optional) ----
if [[ "${RUN_EVAL:-false}" == "true" ]]; then
@@ -624,7 +626,7 @@ PY
# the host-side workflow checks look; the subshell keeps the cd local.
(
cd /workspace
- run_eval --framework lm-eval --port "$ENVOY_PORT"
+ run_eval --port "$ENVOY_PORT"
append_lm_eval_summary
)
fi
diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh
index 11c34736f8..9d00125c1c 100755
--- a/benchmarks/multi_node/llm-d/submit.sh
+++ b/benchmarks/multi_node/llm-d/submit.sh
@@ -79,6 +79,15 @@ export BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}"
export RUN_EVAL="${RUN_EVAL:-false}"
export EVAL_ONLY="${EVAL_ONLY:-false}"
export EVAL_CONC="${EVAL_CONC:-}"
+export EVAL_FRAMEWORK="${EVAL_FRAMEWORK:-lm-eval}"
+export EVAL_LIMIT="${EVAL_LIMIT:-}"
+export EVAL_SUITE="${EVAL_SUITE:-}"
+export SWEBENCH_GEN_MODE="${SWEBENCH_GEN_MODE:-}"
+export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}"
+export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}"
+export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}"
+export IS_AGENTIC="${IS_AGENTIC:-0}"
+export SCENARIO_TYPE="${SCENARIO_TYPE:-}"
export FRAMEWORK="${FRAMEWORK:-llmd-vllm}"
export PRECISION="${PRECISION:-}"
export MODEL_PREFIX="${MODEL_PREFIX:-}"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml
index c9b73e39fa..6878c47b00 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml
@@ -40,6 +40,11 @@ frontend:
type: dynamo
enable_multiple_frontends: false
args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ enable-auto-tool-choice: true
router-mode: "random"
router-session-affinity-ttl-secs: 900
env:
@@ -94,6 +99,9 @@ backend:
max-num-batched-tokens: 8192
trust-remote-code: true
language-model-only: true
+ dyn-tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ dyn-reasoning-parser: "kimi_k3"
load-format: "safetensors"
safetensors-load-strategy: "lazy"
moe-backend: "auto"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml
index 242e21c53e..e10ec63b24 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml
@@ -40,6 +40,11 @@ frontend:
type: dynamo
enable_multiple_frontends: false
args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ enable-auto-tool-choice: true
router-mode: "random"
router-session-affinity-ttl-secs: 900
env:
@@ -93,6 +98,9 @@ backend:
max-num-batched-tokens: 16384
trust-remote-code: true
language-model-only: true
+ dyn-tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ dyn-reasoning-parser: "kimi_k3"
load-format: "safetensors"
safetensors-load-strategy: "lazy"
moe-backend: "auto"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml
index e7de6389a1..f1fde0a104 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml
@@ -48,7 +48,13 @@ frontend:
type: dynamo
enable_multiple_frontends: false
args:
- dyn-chat-processor: "dynamo"
+ # Use vLLM's Kimi K3 parser so OpenAI responses expose structured
+ # tool_calls instead of raw XTML in message.content.
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ enable-auto-tool-choice: true
router-mode: "kv"
router-kv-events: true
router-temperature: "0"
@@ -108,7 +114,7 @@ backend:
# runtime workspace for FlashInfer's MXFP4 MoE kernel.
max-num-seqs: 64
max-num-batched-tokens: 16384
- speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
+ speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}'
block-size: 64
language-model-only: true
@@ -129,7 +135,6 @@ srun_options:
benchmark:
type: custom
- aiperf_server_metrics: true
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml
index 703262fc3b..b748d10f21 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml
@@ -49,7 +49,13 @@ frontend:
type: dynamo
enable_multiple_frontends: false
args:
- dyn-chat-processor: "dynamo"
+ # Use vLLM's Kimi K3 parser so OpenAI responses expose structured
+ # tool_calls instead of raw XTML in message.content.
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ enable-auto-tool-choice: true
router-mode: "kv"
router-kv-events: true
router-temperature: "0"
@@ -110,7 +116,7 @@ backend:
# sizes are exact hits, while odd loads pad by at most one sequence.
max-num-seqs: 96
max-num-batched-tokens: 16384
- speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
+ speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}'
block-size: 64
language-model-only: true
@@ -132,7 +138,6 @@ srun_options:
benchmark:
type: custom
- aiperf_server_metrics: true
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml
index bf135c0e33..be797ce972 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml
@@ -49,7 +49,13 @@ frontend:
type: dynamo
enable_multiple_frontends: false
args:
- dyn-chat-processor: "dynamo"
+ # Use vLLM's Kimi K3 parser so OpenAI responses expose structured
+ # tool_calls instead of raw XTML in message.content.
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ enable-auto-tool-choice: true
router-mode: "kv"
router-kv-events: true
router-temperature: "0"
@@ -100,8 +106,8 @@ backend:
gpu-memory-utilization: 0.92
max-num-seqs: 32
max-num-batched-tokens: 8192
- speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
- compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96]}'
+ speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
+ compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}'
block-size: 64
language-model-only: true
disable-custom-all-reduce: true
@@ -111,7 +117,6 @@ backend:
reasoning-parser: "kimi_k3"
dyn-reasoning-parser: "kimi_k3"
no-enable-flashinfer-autotune: true
- compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}'
sbatch_directives:
cpus-per-task: "144"
@@ -122,7 +127,6 @@ srun_options:
benchmark:
type: custom
- aiperf_server_metrics: true
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml
index baed3f19e7..221ab1693e 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml
@@ -48,7 +48,13 @@ frontend:
type: dynamo
enable_multiple_frontends: false
args:
- dyn-chat-processor: "dynamo"
+ # Use vLLM's Kimi K3 parser so OpenAI responses expose structured
+ # tool_calls instead of raw XTML in message.content.
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "kimi_k3"
+ reasoning-parser: "kimi_k3"
+ enable-auto-tool-choice: true
router-mode: "kv"
router-kv-events: true
router-temperature: "0"
@@ -97,8 +103,8 @@ backend:
gpu-memory-utilization: 0.92
max-num-seqs: 8
max-num-batched-tokens: 8192
- speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
- compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24]}'
+ speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
+ compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}'
block-size: 64
language-model-only: true
disable-custom-all-reduce: true
@@ -108,7 +114,6 @@ backend:
reasoning-parser: "kimi_k3"
dyn-reasoning-parser: "kimi_k3"
no-enable-flashinfer-autotune: true
- compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}'
sbatch_directives:
cpus-per-task: "144"
@@ -119,7 +124,6 @@ srun_options:
benchmark:
type: custom
- aiperf_server_metrics: true
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml
index 4686d003fc..faf3d780be 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml
@@ -15,7 +15,18 @@ frontend:
type: dynamo
enable_multiple_frontends: false
env: { DYN_TCP_REQUEST_TIMEOUT: "60" }
- args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 }
+ args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "minimax_m3"
+ reasoning-parser: "minimax_m3"
+ enable-auto-tool-choice: true
+ router-mode: "kv"
+ router-kv-events: true
+ router-reset-states: true
+ router-temperature: "0"
+ router-session-affinity-ttl-secs: 14400
+ kv-cache-block-size: 128
backend:
type: vllm
connector: null
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml
index 528e08b835..3d81cbacf0 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml
@@ -15,7 +15,18 @@ frontend:
type: dynamo
enable_multiple_frontends: false
env: { DYN_TCP_REQUEST_TIMEOUT: "60" }
- args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 }
+ args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "minimax_m3"
+ reasoning-parser: "minimax_m3"
+ enable-auto-tool-choice: true
+ router-mode: "kv"
+ router-kv-events: true
+ router-reset-states: true
+ router-temperature: "0"
+ router-session-affinity-ttl-secs: 14400
+ kv-cache-block-size: 128
backend:
type: vllm
connector: null
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml
index b8b4c33411..c056c101dd 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml
@@ -15,7 +15,18 @@ frontend:
type: dynamo
enable_multiple_frontends: false
env: { DYN_TCP_REQUEST_TIMEOUT: "60" }
- args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 }
+ args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "minimax_m3"
+ reasoning-parser: "minimax_m3"
+ enable-auto-tool-choice: true
+ router-mode: "kv"
+ router-kv-events: true
+ router-reset-states: true
+ router-temperature: "0"
+ router-session-affinity-ttl-secs: 14400
+ kv-cache-block-size: 128
backend:
type: vllm
connector: null
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml
index b74dd665b4..6403d69e5b 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml
@@ -30,6 +30,11 @@ frontend:
enable_multiple_frontends: false
env: { DYN_TCP_REQUEST_TIMEOUT: "60" }
args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "minimax_m3"
+ reasoning-parser: "minimax_m3"
+ enable-auto-tool-choice: true
router-mode: "kv"
router-kv-events: true
router-reset-states: true
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml
index 1d62599edd..41a1835083 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml
@@ -21,7 +21,18 @@ frontend:
type: dynamo
enable_multiple_frontends: false
env: { DYN_TCP_REQUEST_TIMEOUT: "60" }
- args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 }
+ args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "minimax_m3"
+ reasoning-parser: "minimax_m3"
+ enable-auto-tool-choice: true
+ router-mode: "kv"
+ router-kv-events: true
+ router-reset-states: true
+ router-temperature: "0"
+ router-session-affinity-ttl-secs: 14400
+ kv-cache-block-size: 128
backend:
type: vllm
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml
index 7c2343f404..1a798085b9 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml
@@ -23,7 +23,18 @@ frontend:
type: dynamo
enable_multiple_frontends: false
env: { DYN_TCP_REQUEST_TIMEOUT: "60" }
- args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 }
+ args:
+ dyn-chat-processor: "vllm"
+ trust-remote-code: true
+ tool-call-parser: "minimax_m3"
+ reasoning-parser: "minimax_m3"
+ enable-auto-tool-choice: true
+ router-mode: "kv"
+ router-kv-events: true
+ router-reset-states: true
+ router-temperature: "0"
+ router-session-affinity-ttl-secs: 14400
+ kv-cache-block-size: 128
backend:
type: vllm
connector: null
diff --git a/benchmarks/multi_node/tilert_utils/run_node.sh b/benchmarks/multi_node/tilert_utils/run_node.sh
index 7a1e9cea3e..8927624150 100755
--- a/benchmarks/multi_node/tilert_utils/run_node.sh
+++ b/benchmarks/multi_node/tilert_utils/run_node.sh
@@ -211,6 +211,7 @@ run_bench_and_eval() {
wait_for_server_ready --port "$ROUTER_PORT" \
--server-log "$BENCHMARK_LOGS_DIR/tilert_router.log" --server-pid "$ROUTER_PID"
local rc=0 conc np
+ if [[ "${EVAL_ONLY:-false}" != "true" ]]; then
for conc in $CONC_LIST; do
np=$(( conc * 10 ))
[[ "$np" -lt 16 ]] && np=16
@@ -226,6 +227,7 @@ run_bench_and_eval() {
--result-filename "$(bench_result_stem "$conc")" --result-dir "$RESULT_DIR" \
|| { rc=$?; echo "[bench] WARNING: conc=$conc failed/timed out (rc=$rc)"; }
done
+ fi
run_lm_eval
return $rc
}
@@ -238,7 +240,7 @@ run_lm_eval() {
export EVAL_CONCURRENT_REQUESTS="$(tr ' ' '\n' <<< "$CONC_LIST" | sort -n | tail -1)"
fi
export CONC="$EVAL_CONCURRENT_REQUESTS"
- run_eval --framework lm-eval --port "$ROUTER_PORT"
+ run_eval --port "$ROUTER_PORT"
append_lm_eval_summary
}
diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index 1212309a21..86b07e5f2d 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -269,6 +269,9 @@ VLLM_CMD=(
--max-num-batched-tokens 16384
--trust-remote-code
--language-model-only
+ --enable-auto-tool-choice
+ --tool-call-parser kimi_k3
+ --reasoning-parser kimi_k3
--load-format fastsafetensors
--moe-backend auto
--no-enable-flashinfer-autotune
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh
index 5f32bc6dd8..8dd5807d23 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh
@@ -21,8 +21,6 @@ set -x
source "$(dirname "$0")/../../benchmark_lib.sh"
-export EVAL_FRAMEWORK="lm-eval"
-
check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA"
@@ -128,6 +126,7 @@ install_agentic_deps
OFFLOAD_ARGS=()
if require_agentic_kv_offload_backend vllm-simple; then
+ python3 "$(dirname "$0")/../../../runners/patch_vllm_simple_kv_offload.py"
CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024))
export VLLM_USE_SIMPLE_KV_OFFLOAD=1
OFFLOAD_CONFIG=$(printf \
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh
index 686145460b..81e0349089 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh
@@ -11,7 +11,7 @@ set -x
source "$(dirname "$0")/../../benchmark_lib.sh"
-export EVAL_FRAMEWORK="lm-eval"
+export EVAL_FRAMEWORK="${EVAL_FRAMEWORK:-lm-eval}"
check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EVAL_ONLY
@@ -44,6 +44,10 @@ nvidia-smi
resolve_trace_source
install_agentic_deps
+# BFCL's stock OpenAI client sends the standard `store=false` field. TRT-LLM
+# 1.3 rejects that field even though this server never persists responses.
+python3 "$(dirname "$0")/../../../runners/patch_trtllm_chat_store.py"
+
# kv_cache_config.host_cache_size is pinned per topology in ser.yaml below
# (200 GiB at TP8, 250 GiB at TP4 -- see $mem_off), NOT derived from
# TOTAL_CPU_DRAM_GB.
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh
index 3593cac4ce..91accfee80 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh
@@ -7,8 +7,6 @@ set -x
source "$(dirname "$0")/../../benchmark_lib.sh"
-export EVAL_FRAMEWORK="lm-eval"
-
check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA"
@@ -43,6 +41,7 @@ install_agentic_deps
OFFLOAD_ARGS=()
if require_agentic_kv_offload_backend vllm-simple; then
+ python3 "$(dirname "$0")/../../../runners/patch_vllm_simple_kv_offload.py"
CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024))
export VLLM_USE_SIMPLE_KV_OFFLOAD=1
OFFLOAD_CONFIG=$(printf \
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh
index df11cae903..0ce9d0abb1 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh
@@ -11,7 +11,7 @@ set -x
source "$(dirname "$0")/../../benchmark_lib.sh"
-export EVAL_FRAMEWORK="lm-eval"
+export EVAL_FRAMEWORK="${EVAL_FRAMEWORK:-lm-eval}"
check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EVAL_ONLY
@@ -43,6 +43,10 @@ nvidia-smi
resolve_trace_source
install_agentic_deps
+# BFCL's stock OpenAI client sends the standard `store=false` field. TRT-LLM
+# 1.3 rejects that field even though this server never persists responses.
+python3 "$(dirname "$0")/../../../runners/patch_trtllm_chat_store.py"
+
SERVER_LOG="$RESULT_DIR/server.log"
mkdir -p "$RESULT_DIR"
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh
index 8895b6ae46..0a47115e94 100644
--- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh
@@ -11,7 +11,6 @@ set -x
source "$(dirname "$0")/../../benchmark_lib.sh"
-export EVAL_FRAMEWORK="lm-eval"
check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
@@ -216,6 +215,7 @@ VLLM_CMD=(
--moe-backend aiter
--kv-cache-dtype fp8
--tool-call-parser minimax_m3
+ --reasoning-parser minimax_m3
--enable-auto-tool-choice
--default-chat-template-kwargs '{"thinking_mode":"enabled"}'
--max-num-seqs "$((2 * CONC))"
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh
index 88d9078819..4234a66490 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh
@@ -6,7 +6,6 @@ set -x
source "$(dirname "$0")/../../benchmark_lib.sh"
-export EVAL_FRAMEWORK="lm-eval"
check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh
index 1851139dab..28d901417b 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh
@@ -4,7 +4,6 @@ set -x
source "$(dirname "$0")/../../benchmark_lib.sh"
-export EVAL_FRAMEWORK="lm-eval"
check_env_vars MODEL TP CONC KV_OFFLOADING RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 72adfa2fa7..481db4fcef 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7451,7 +7451,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp:
- "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache.yaml"
minimaxm3-fp4-b300-vllm-agentic-mtp:
- image: vllm/vllm-openai:nightly-ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9
+ image: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45
model: nvidia/MiniMax-M3-NVFP4
model-prefix: minimaxm3
runner: cluster:b300-nv
@@ -7605,7 +7605,7 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp:
# the same 3 TB AgentX ceiling before the proportional-GPU rule is applied.
# GPU-resident points receive a zero budget.
minimaxm3-fp4-b200-vllm-agentic-mtp:
- image: vllm/vllm-openai:nightly-ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9
+ image: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45
model: nvidia/MiniMax-M3-NVFP4
model-prefix: minimaxm3
runner: cluster:b200-nscale
diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md
index 120fff7fa4..6db5d351e8 100644
--- a/docs/eval-agentx-procedures.md
+++ b/docs/eval-agentx-procedures.md
@@ -18,11 +18,11 @@ There are two distinct layers: the matrix generator decides **which jobs exist**
| Normal sweep | no eval option | Throughput jobs plus the selected 8k/1k eval subset |
| Throughput only | `--no-evals` | No eval jobs |
| Selected eval subset only | `--evals-only` | Jobs have `RUN_EVAL=true`, `EVAL_ONLY=true` |
-| Every eligible eval only | `--all-evals` | Equivalent to `--evals-only --all-evals` and includes all fixed-sequence 8k/1k rows, plus single-node agentic SWE-bench rows |
+| Every eligible eval only | `--all-evals` | Equivalent to `--evals-only --all-evals` and includes all fixed-sequence 8k/1k rows plus single-node and multi-node agentic GSM8K rows |
| Throughput then eval in one recipe | `RUN_EVAL=true`, `EVAL_ONLY=false` | Server starts, throughput runs, then `run_eval` runs |
| Eval against a freshly started server | `RUN_EVAL=true`, `EVAL_ONLY=true` | Launcher expands eval context, skips throughput, and runs the eval |
-Default selection is scenario-aware. Single-node fixed-sequence evals use the median and highest eligible concurrency for each 8k/1k model/runner/framework/precision/parallelism group. Multi-node evals use the highest eligible concurrency per topology. Concurrency below 16 is not selected. Agentic evals are opt-in, and multi-node agentic eval is unsupported. See [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L238-L339) and [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L342-L398).
+Default selection is scenario-aware. Single-node fixed-sequence evals use the median and highest eligible concurrency for each 8k/1k model/runner/framework/precision/parallelism group. Multi-node evals use the highest eligible concurrency per topology. Concurrency below 16 is not selected. Agentic evals are opt-in; single-node and multi-node agentic rows select their highest eligible concurrency. See [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L276-L396) and [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L399-L482).
On a PR, combine one primary sweep label (normally `full-sweep-fail-fast`) with eval modifiers. `all-evals` expands coverage without suppressing throughput. `evals-only` suppresses throughput. Together they run all eligible evals only. Runs with `evals-only` are not reusable, while normal full sweeps and `all-evals` full sweeps are reusable. Adding or removing a modifier restarts the active sweep ([label policy](../.github/workflows/README.md#pr-eval-modifiers)).
@@ -48,14 +48,14 @@ uv run --no-project --with pydantic --with pyyaml --python 3.12 \
--config-files configs/nvidia-master.yaml | jq .
```
-A correct AgentX eval row contains `"scenario-type": "agentic-coding"`, `"run-eval": true`, and `"eval-only": true`. The workflow splits generated rows into throughput, fixed-sequence eval, and agentic eval jobs in [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L257-L271).
+A correct AgentX eval row contains `"scenario-type": "agentic-coding"`, `"run-eval": true`, and `"eval-only": true`. The workflow splits generated rows into throughput, fixed-sequence eval, and agentic eval jobs in [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L278-L293).
## 2. Add a graded eval
1. Add `utils/evals/.yaml` using the lm-evaluation-harness task format. Pin the dataset/split, deterministic generation settings, prompt contract, filters, and primary metric. Use [`gsm8k.yaml`](../utils/evals/gsm8k.yaml) or [`gpqa_diamond.yaml`](../utils/evals/gpqa_diamond.yaml) as an in-tree pattern.
2. Give `task:` a stable name. That exact name is the key used by score thresholds and appears in collected rows.
3. Add the minimum accepted score to [`utils/evals/thresholds.yaml`](../utils/evals/thresholds.yaml). Put a general floor under `default`. Add `models..` only when a justified model-specific floor is required.
-4. If the task's primary result is not compatible with the collector's strict/extract/accuracy rules, extend [`extract_lm_metrics()`](../utils/collect_eval_results.py#L114-L181). Do not publish a row whose `score` is null.
+4. If the task's primary result is not compatible with the collector's strict/extract/accuracy rules, extend [`extract_lm_metrics()`](../utils/collect_eval_results.py#L115-L197). Do not publish a row whose `score` is null.
5. Run a small explicit slice, inspect samples, then run the full split. `EVAL_LIMIT` is a smoke-test control, not a publishable score setting.
Against an already healthy OpenAI-compatible server:
@@ -85,7 +85,7 @@ append_lm_eval_summary
python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX"
```
-`run_lm_eval` passes concurrency through `num_concurrent` in `--model_args`. It is deliberately an environment variable, not a `run_eval` CLI option. The exact invocation is in [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L890-L970).
+`run_lm_eval` passes concurrency through `num_concurrent` in `--model_args`. It is deliberately an environment variable, not a `run_eval` CLI option. The exact invocation is in [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L1080-L1162).
## 3. `EVAL_ONLY` is a launcher contract
@@ -97,9 +97,9 @@ Set `EVAL_ONLY=true` **before server launch**. It is not merely a switch inside
4. Throughput returns immediately or is skipped.
5. `run_eval` and artifact staging run.
-Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L853-L888), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1537-L1654), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L162-L185).
+Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L1049-L1078), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1789-L1923), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L79-L97).
-Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L387-L404), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)).
+Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L399-L417), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)).
## 4. Batched eval concurrency
@@ -121,9 +121,9 @@ The batch runner creates a fresh temporary output directory per point, stages fi
- `completed_eval_concs`: eval and staging both succeeded.
- `failed_eval_concs`: either eval or staging failed.
-A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1537-L1631), [artifact suffixing](../benchmarks/benchmark_lib.sh#L972-L1030), and [manifest validation](../utils/evals/validate_scores.py#L72-L171).
+A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1839-L1900), [artifact suffixing](../benchmarks/benchmark_lib.sh#L1163-L1222), and [manifest validation](../utils/evals/validate_scores.py#L72-L171).
-For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L375-L378)). Never compare a point if its `_conc` result or completed-manifest entry is missing.
+For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L397-L400)). Never compare a point if its `_conc` result or completed-manifest entry is missing.
## 5. Validate scores, not file existence
@@ -173,7 +173,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench
## 7. Run AgentX: fast feedback versus canonical evidence
-AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L1824-L1848)).
+AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)).
Keep non-index engine or router wheels reproducible and immutable: check in the source patch and builder beside the launcher, verify the upstream wheel's digest before patching, assign an explicit local version, and install the published artifact through an exact URL with a SHA256 fragment. A local backport must not use an unreleased upstream version number.
@@ -201,17 +201,18 @@ Targeted AgentX SWE-bench smoke eval (first ten instances, real agentic generati
gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \
-f generate-cli-command='test-config --config-keys qwen3.5-fp8-b200-sglang-agentic --conc 1 --evals-only --config-files configs/nvidia-master.yaml' \
-f test-name='swebench-smoke-qwen35-c1' \
+ -f eval-framework=swebench \
-f eval-limit='10' \
-f swebench-gen-mode='agentic'
```
For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`, which is only a debugging escape hatch. SWE-bench generation/scoring controls and its `0.50` full-split threshold are documented next to the implementation in [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench).
-Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L1982-L1989)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success.
+Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2266-L2268)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success.
## 8. Preserve trace and run provenance
-AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L1743-L1822), [replay semantics](../benchmarks/benchmark_lib.sh#L1824-L1850)).
+AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L2023-L2102), [replay semantics](../benchmarks/benchmark_lib.sh#L2104-L2270)).
Capture orchestration provenance immediately:
@@ -243,7 +244,7 @@ For each concurrency retain:
- server/frontend logs and every metrics endpoint represented.
- run URL/ID, attempt, head SHA, recipe/config identity, image, topology, fast flag, and any override.
-The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2040-L2079)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L337-L346), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)).
+The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2320-L2360)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L349-L358), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)).
## 9. Debug long AgentX runs from live evidence
@@ -292,7 +293,7 @@ curl -fsS '' | \
rg -i 'request|queue|cache|token|prefill|decode|error|fail'
```
-Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L1963-L1980)).
+Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L2236-L2260)).
Use phase markers, not total Slurm age:
diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md
index ecc7f06212..6e8c8e03c8 100644
--- a/docs/eval-agentx-procedures_zh.md
+++ b/docs/eval-agentx-procedures_zh.md
@@ -18,11 +18,11 @@
| 常规 sweep | 不加 eval 选项 | 吞吐量作业，加上选定的 8k/1k eval 子集 |
| 仅吞吐量 | `--no-evals` | 不生成 eval 作业 |
| 仅选定的 eval 子集 | `--evals-only` | 作业带有 `RUN_EVAL=true`、`EVAL_ONLY=true` |
-| 仅运行所有符合条件的 eval | `--all-evals` | 等价于 `--evals-only --all-evals`；包含全部定长序列 8k/1k 行，以及单节点 agentic SWE-bench 行 |
+| 仅运行所有符合条件的 eval | `--all-evals` | 等价于 `--evals-only --all-evals`；包含全部定长序列 8k/1k 行，以及单节点和多节点 agentic GSM8K 行 |
| 在一个 recipe 中先跑吞吐量再跑 eval | `RUN_EVAL=true`、`EVAL_ONLY=false` | 启动服务，运行吞吐量，然后执行 `run_eval` |
| 对新启动的服务仅运行 eval | `RUN_EVAL=true`、`EVAL_ONLY=true` | launcher 扩大 eval context，跳过吞吐量并运行 eval |
-默认选择会区分场景。单节点定长序列 eval 对每个 8k/1k 的模型/runner/framework/precision/并行配置分组选取符合条件的中位和最高并发；多节点 eval 对每种拓扑选取符合条件的最高并发。低于 16 的并发不会被选中。Agentic eval 需要显式启用；不支持多节点 agentic eval。参见 [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L238-L339) 与 [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L342-L398)。
+默认选择会区分场景。单节点定长序列 eval 对每个 8k/1k 的模型/runner/framework/precision/并行配置分组选取符合条件的中位和最高并发；多节点 eval 对每种拓扑选取符合条件的最高并发。低于 16 的并发不会被选中。Agentic eval 需要显式启用；单节点和多节点 agentic 行均选择符合条件的最高并发。参见 [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L276-L396) 与 [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L399-L482)。
在 PR 上，应将一个主要 sweep label（通常为 `full-sweep-fail-fast`）与 eval modifier 组合使用。`all-evals` 在不抑制吞吐量的情况下扩大覆盖范围；`evals-only` 会抑制吞吐量；两者一起使用时只运行所有符合条件的 eval。带有 `evals-only` 的运行不可复用，而常规 full sweep 和 `all-evals` full sweep 可以复用。添加或移除 modifier 会重启当前 sweep（[label 策略](../.github/workflows/README.md#pr-eval-modifiers)）。
@@ -48,14 +48,14 @@ uv run --no-project --with pydantic --with pyyaml --python 3.12 \
--config-files configs/nvidia-master.yaml | jq .
```
-正确的 AgentX eval 行包含 `"scenario-type": "agentic-coding"`、`"run-eval": true` 和 `"eval-only": true`。工作流会在 [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L257-L271) 中将生成的行拆分到吞吐量、定长序列 eval 和 agentic eval 作业。
+正确的 AgentX eval 行包含 `"scenario-type": "agentic-coding"`、`"run-eval": true` 和 `"eval-only": true`。工作流会在 [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L278-L293) 中将生成的行拆分到吞吐量、定长序列 eval 和 agentic eval 作业。
## 2. 添加评分 eval
1. 按照 lm-evaluation-harness task 格式添加 `utils/evals/.yaml`。固定 dataset/split、确定性生成设置、prompt 约定、filter 和主指标。可参考仓库内的 [`gsm8k.yaml`](../utils/evals/gsm8k.yaml) 或 [`gpqa_diamond.yaml`](../utils/evals/gpqa_diamond.yaml)。
2. 为 `task:` 指定稳定名称。分数阈值以该精确名称为键，收集后的行中也会出现该名称。
3. 在 [`utils/evals/thresholds.yaml`](../utils/evals/thresholds.yaml) 中添加最低可接受分数。通用下限放在 `default`；只有在确有依据需要模型专用下限时，才添加 `models..`。
-4. 如果 task 的主结果与 collector 的 strict/extract/accuracy 规则不兼容，请扩展 [`extract_lm_metrics()`](../utils/collect_eval_results.py#L114-L181)。不要发布 `score` 为 null 的行。
+4. 如果 task 的主结果与 collector 的 strict/extract/accuracy 规则不兼容，请扩展 [`extract_lm_metrics()`](../utils/collect_eval_results.py#L115-L197)。不要发布 `score` 为 null 的行。
5. 先运行一个显式的小切片并检查样本，再运行完整 split。`EVAL_LIMIT` 是 smoke test 控制项，不是可发布分数的运行设置。
对已经健康的 OpenAI-compatible 服务执行：
@@ -85,7 +85,7 @@ append_lm_eval_summary
python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX"
```
-`run_lm_eval` 通过 `--model_args` 中的 `num_concurrent` 传递并发；它刻意采用环境变量，而不是 `run_eval` CLI 选项。准确调用见 [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L890-L970)。
+`run_lm_eval` 通过 `--model_args` 中的 `num_concurrent` 传递并发；它刻意采用环境变量，而不是 `run_eval` CLI 选项。准确调用见 [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L1080-L1162)。
## 3. `EVAL_ONLY` 是 launcher 约定
@@ -97,9 +97,9 @@ python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX"
4. 吞吐量路径立即返回或被跳过。
5. 运行 `run_eval` 和 artifact staging。
-相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L853-L888)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1537-L1654) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L162-L185)。
+相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L1049-L1078)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1789-L1923) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L79-L97)。
-不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L387-L404)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)）。
+不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L399-L417)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)）。
## 4. 批量 eval 并发
@@ -121,9 +121,9 @@ python3 utils/evals/validate_scores.py --expected-concs '16 32 64'
- `completed_eval_concs`：eval 与 staging 均成功的点；
- `failed_eval_concs`：eval 或 staging 失败的点。
-失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1537-L1631)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L972-L1030) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。
+失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1839-L1900)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L1163-L1222) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。
-对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L375-L378)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。
+对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L397-L400)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。
## 5. 校验分数，而不只是检查文件存在
@@ -173,7 +173,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \
## 7. 运行 AgentX：快速反馈与 canonical 证据
-AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[Fast replay 设置](../benchmarks/benchmark_lib.sh#L1824-L1848)）。
+AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128)）。
对于未发布到 package index 的 engine 或 router wheel，必须保证构建可复现且 artifact 不可变：在 launcher 旁签入源码 patch 与构建器，打 patch 前校验上游 wheel 的 digest，分配明确的 local version，并通过带 SHA256 fragment 的精确 URL 安装已发布 artifact。本地 backport 不得冒用尚未发布的上游版本号。
@@ -201,17 +201,18 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \
gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \
-f generate-cli-command='test-config --config-keys qwen3.5-fp8-b200-sglang-agentic --conc 1 --evals-only --config-files configs/nvidia-master.yaml' \
-f test-name='swebench-smoke-qwen35-c1' \
+ -f eval-framework=swebench \
-f eval-limit='10' \
-f swebench-gen-mode='agentic'
```
要得到可发布的 SWE-bench 分数，省略 `eval-limit`；不要使用 `single-shot`，它只是诊断逃生选项。SWE-bench generation/scoring 控制项以及完整 split 的 `0.50` 阈值在实现旁的 [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench) 中说明。
-Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L1982-L1989)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。
+Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2266-L2268)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。
## 8. 保留 trace 与运行 provenance
-AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L1743-L1822)、[replay 语义](../benchmarks/benchmark_lib.sh#L1824-L1850)）。
+AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L2023-L2102)、[replay 语义](../benchmarks/benchmark_lib.sh#L2104-L2270)）。
立即记录 orchestration provenance：
@@ -243,7 +244,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \
- server/frontend 日志以及所代表的每个 metrics endpoint；
- run URL/ID、attempt、head SHA、recipe/config 标识、image、topology、fast 标志和所有 override。
-Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2040-L2079)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L337-L346)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)）。
+Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2320-L2360)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L349-L358)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)）。
## 9. 用实时证据调试长时间 AgentX 运行
@@ -292,7 +293,7 @@ curl -fsS '' | \
rg -i 'request|queue|cache|token|prefill|decode|error|fail'
```
-通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L1963-L1980)）。
+通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L2236-L2260)）。
应使用 phase marker，而不是 Slurm 总运行时间：
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index a619ab7c7d..2f58d1b442 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6674,3 +6674,59 @@
- "Bump image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828"
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2763
+
+- config-keys:
+ - minimaxm3-fp4-b300-vllm-agentic-mtp
+ - minimaxm3-fp4-b200-vllm-agentic-mtp
+ scenario-type:
+ - agentic-coding
+ description:
+ - "Replace the expired vLLM nightly image with the current pinned upstream nightly so MiniMax-M3 AgentX jobs can import their container instead of failing with Docker Hub HTTP 404."
+ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634
+
+- config-keys:
+ - kimik3-fp4-gb200-dynamo-vllm-agentic
+ scenario-type:
+ - agentic-coding
+ description:
+ - "Correct srt-slurm's external data-parallel process allocation so Kimi-K3 TP4/DP4 workers receive four GPUs per DP rank instead of one invalid rank per GPU."
+ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634
+
+- config-keys:
+ - minimaxm3-fp4-b300-trtllm-agentic-mtp
+ - minimaxm3-fp4-b200-trtllm-agentic-mtp
+ scenario-type:
+ - agentic-coding
+ description:
+ - "Accept the stock BFCL client's standard store=false chat-completion field instead of rejecting every TensorRT-LLM tool-use request with HTTP 400."
+ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634
+
+- config-keys:
+ - minimaxm3-fp4-b300-vllm-agentic-mtp
+ - minimaxm3-fp4-b200-vllm-agentic-mtp
+ scenario-type:
+ - agentic-coding
+ description:
+ - "Keep AgentX dependency setup rootless so minimal vLLM images without git do not fail while attempting a privileged apt install."
+ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634
+
+- config-keys:
+ - minimaxm3-fp4-b200-vllm-agentic-mtp
+ - minimaxm3-fp4-b300-vllm-agentic-mtp
+ scenario-type:
+ - agentic-coding
+ description:
+ - "Split heterogeneous vLLM layer-compact KV backing storage into per-layer offload regions so MiniMax-M3 cache layouts initialize without invalid tensor views."
+ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634
+
+- config-keys:
+ - kimik3-fp4-b300-vllm-agentic-dspark
+ - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp
+ - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp
+ - minimaxm3-fp4-mi355x-vllm-agentic-mtp
+ scenario-type:
+ - agentic-coding
+ description:
+ - "Configure model-specific tool and reasoning parsers for the Kimi-K3 and MiniMax-M3 serving paths so opt-in verifier requests produce structured OpenAI tool calls."
+ - "Limit shared AgentX readiness and synthetic-acceptance rewrites to eval-only runs so default throughput recipe rendering remains unchanged."
+ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634
diff --git a/runners/inject_synthetic_acceptance.py b/runners/inject_synthetic_acceptance.py
index bafda7a043..ab10bb1126 100644
--- a/runners/inject_synthetic_acceptance.py
+++ b/runners/inject_synthetic_acceptance.py
@@ -1,18 +1,13 @@
#!/usr/bin/env python3
-"""Inject synthetic acceptance parameters into an srt-slurm recipe (generic driver).
+"""Configure speculative acceptance in an srt-slurm recipe.
-This is the framework-agnostic half of the synthetic-acceptance mechanism. It
-decides *whether* to inject (the ``SYNTHETIC_ACCEPTANCE`` flag) and *what* mean
-acceptance length to inject, then delegates the actual recipe rewrite to a
-framework-specific backend (see ``runners/synthetic_injectors/``).
-
-The script is a no-op (exit 0, file untouched) when:
- - SYNTHETIC_ACCEPTANCE is unset/false,
-so existing callers that do not opt in get exactly the previous behavior. When
-enabled it requires a backend registered for the given framework; the vLLM
-backend is added in a follow-up framework-support change.
+Eval-only runs remove synthetic acceptance so generated text is checked against
+the target model. Throughput runs inject a configured synthetic acceptance
+length only when ``SYNTHETIC_ACCEPTANCE=true``. Framework-specific rewriting
+lives under ``runners/synthetic_injectors/``.
Environment variables:
+ EVAL_ONLY "true" to restore real target verification
SYNTHETIC_ACCEPTANCE "true" to enable (default: "false")
SYNTHETIC_ACCEPTANCE_LENGTH target mean acceptance length; if unset, it is
auto-resolved from the reference AL YAML using
@@ -76,7 +71,9 @@ def _lookup_al(model_block, num_spec_tokens):
if isinstance(model_block, dict):
# Thinking matrix form: pick the requested mode, then index by level.
if any(str(k).startswith("thinking") for k in model_block):
- mode = os.environ.get("THINKING_MODE", "thinking_on").strip() or "thinking_on"
+ mode = (
+ os.environ.get("THINKING_MODE", "thinking_on").strip() or "thinking_on"
+ )
mode_block = model_block.get(mode)
if mode_block is None:
sys.exit(
@@ -119,7 +116,9 @@ def _resolve_al(config_text, injector, ref_yaml):
al = _lookup_al(model_block, num_spec_tokens)
if al is None:
- sys.exit(f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}")
+ sys.exit(
+ f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}"
+ )
_log(
f"Auto-resolved AL={al} from {ref_yaml} "
@@ -129,14 +128,30 @@ def _resolve_al(config_text, injector, ref_yaml):
def inject(config_file, framework):
+ injector = get_injector(framework)
+
if _enabled("EVAL_ONLY"):
- print("[Synthetic AL] EVAL_ONLY=true: keeping real MTP recipe")
+ if injector is None or not hasattr(injector, "rewrite_real"):
+ print(
+ f"[Synthetic AL] EVAL_ONLY=true: no real-acceptance rewriter "
+ f"for FRAMEWORK='{framework}'"
+ )
+ return 0
+
+ with open(config_file) as f:
+ content = f.read()
+ new_content, count = injector.rewrite_real(content, _log)
+ if count:
+ with open(config_file, "w") as f:
+ f.write(new_content)
+ _log(f"Restored real acceptance in {count} speculative-config entries")
+ else:
+ _log("EVAL_ONLY=true: recipe already uses real acceptance")
return 0
if not _enabled("SYNTHETIC_ACCEPTANCE"):
return 0
- injector = get_injector(framework)
if injector is None:
sys.exit(
"ERROR: SYNTHETIC_ACCEPTANCE=true but no synthetic-acceptance "
@@ -149,7 +164,12 @@ def inject(config_file, framework):
al = _resolve_al(
content,
injector,
- os.path.join(os.path.dirname(__file__), "..", "benchmarks", "speedbench-reference-al.yaml"),
+ os.path.join(
+ os.path.dirname(__file__),
+ "..",
+ "benchmarks",
+ "speedbench-reference-al.yaml",
+ ),
)
_log(f"Injecting synthetic acceptance (length={al}) into {config_file}")
diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh
index 6621bc8a83..a93f0e5ccb 100644
--- a/runners/launch_b200-nscale-compat.sh
+++ b/runners/launch_b200-nscale-compat.sh
@@ -227,6 +227,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
cd "$SRT_REPO_DIR" || exit 1
git checkout sa-submission-q2-2026
fi
+ if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1
+ fi
echo "Installing srtctl..."
export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin"
@@ -391,6 +394,10 @@ EOF
# so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time.
# Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix.
sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}"
+ if [[ "${EVAL_ONLY:-false}" == "true" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \
+ "${CONFIG_FILE%%:*}" "$FRAMEWORK" || exit 1
+ fi
SRTCTL_PREFLIGHT_ARGS=()
# Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node.
diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh
index 1e613d7ed1..e3dd684a57 100755
--- a/runners/launch_b200-nscale-slurm.sh
+++ b/runners/launch_b200-nscale-slurm.sh
@@ -148,6 +148,11 @@ else
mkdir -p recipes/vllm/kimi-k2.6
cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6" recipes/vllm/kimi-k2.6
fi
+if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \
+ || exit 1
+fi
+
echo "Installing srtctl..."
export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin"
diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh
index 37d35c7b3d..b56c03a56b 100644
--- a/runners/launch_b300-nv.sh
+++ b/runners/launch_b300-nv.sh
@@ -152,6 +152,11 @@ else
cd "$SRT_REPO_DIR" || exit 1
git checkout sa-submission-q2-2026
fi
+if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \
+ || exit 1
+fi
+
echo "Installing srtctl..."
export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin"
@@ -266,6 +271,10 @@ sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH"
if [[ "$MODEL_PREFIX" == "minimaxm3" && -n "$MINIMAX_M3_SLURM_EXCLUDED_NODELIST" ]]; then
sed -i "/^name:.*/a sbatch_directives:\n exclude: \"${MINIMAX_M3_SLURM_EXCLUDED_NODELIST}\"" "$CONFIG_PATH"
fi
+if [[ "${EVAL_ONLY:-false}" == "true" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \
+ "$CONFIG_PATH" "$FRAMEWORK" || exit 1
+fi
SRTCTL_APPLY_ARGS=(
-f "$CONFIG_FILE"
--tags "b300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)"
@@ -551,8 +560,21 @@ else
export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
- SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}"
- salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT -N 1 --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME"
+ SALLOC_ARGS=(
+ --partition="$SLURM_PARTITION"
+ --account="$SLURM_ACCOUNT"
+ -N 1
+ --gres="gpu:$GPU_COUNT"
+ --exclusive
+ --mem=0
+ --time="${SALLOC_TIME_LIMIT:-480}"
+ --no-shell
+ --job-name="$RUNNER_NAME"
+ )
+ if [[ -n "${SALLOC_EXCLUDE:-}" ]]; then
+ SALLOC_ARGS+=(--exclude="$SALLOC_EXCLUDE")
+ fi
+ salloc "${SALLOC_ARGS[@]}"
JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
srun --jobid=$JOB_ID \
diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh
index 4d55473ce6..9799aa3b71 100755
--- a/runners/launch_gb200-nv.sh
+++ b/runners/launch_gb200-nv.sh
@@ -442,6 +442,11 @@ elif [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION"
elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "kimik3" ]]; then
git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1
cd "$SRT_REPO_DIR" || exit 1
+ test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || {
+ echo "Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2
+ exit 1
+ }
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_vllm_dp_ranks.py" "$(pwd)" || exit 1
mkdir -p recipes/vllm/kimi-k3/agentic || exit 1
cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \
recipes/vllm/kimi-k3/agentic || exit 1
@@ -549,6 +554,9 @@ else
git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR"
cd "$SRT_REPO_DIR"
fi
+if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1
+fi
echo "Installing srtctl..."
curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -703,10 +711,11 @@ if command -v squeue >/dev/null 2>&1; then
fi
sed -i "s/^name:.*/name: \"${SRT_SLURM_JOB_NAME}\"/" "$CONFIG_PATH"
-# Optionally inject synthetic acceptance into the recipe's speculative-config
-# when SYNTHETIC_ACCEPTANCE=true (no-op otherwise). Must run after the name
-# override and before srtctl apply so the rendered job picks it up.
-python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" "$CONFIG_PATH" "$FRAMEWORK"
+# The driver preserves both contracts: real verification for EVAL_ONLY and
+# synthetic acceptance for throughput when SYNTHETIC_ACCEPTANCE is enabled.
+# It is otherwise a no-op.
+python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \
+ "$CONFIG_PATH" "$FRAMEWORK" || exit 1
# Don't leak the login-node venv to the compute-node orchestrator. sbatch's
# default --export=ALL propagates VIRTUAL_ENV (set by `source
diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh
index 97c7a14505..91fb3b4918 100644
--- a/runners/launch_gb300-nv.sh
+++ b/runners/launch_gb300-nv.sh
@@ -421,6 +421,11 @@ else
git checkout sa-submission-q2-2026
fi
+if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \
+ || exit 1
+fi
+
echo "Installing srtctl..."
export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin"
curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -521,8 +526,8 @@ CONFIG_PATH="${CONFIG_FILE%%:*}"
sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH"
# Throughput recipes opt into synthetic acceptance through the master config.
-# Eval-only jobs leave the checked-in real-MTP recipe unchanged so generated
-# tokens still pass target-model verification.
+# Eval-only jobs remove those settings so generated tokens use real target-model
+# verification.
inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1
# --no-preflight skips srtctl's pre-submit model-path stat, which runs on
diff --git a/runners/launch_h100-cr.sh b/runners/launch_h100-cr.sh
index e7bd48dfa9..07bfa80dda 100644
--- a/runners/launch_h100-cr.sh
+++ b/runners/launch_h100-cr.sh
@@ -18,7 +18,7 @@ docker run --rm --network=host --name=$server_name \
--runtime=nvidia --gpus="$GPU_COUNT" --ipc=host --privileged --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 \
-v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
-v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \
--e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e PP_SIZE -e DCP_SIZE -e PCP_SIZE -e GPU_COUNT -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e RUN_EVAL -e EVAL_ONLY -e RUNNER_TYPE -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e PORT=$PORT \
+-e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e PP_SIZE -e DCP_SIZE -e PCP_SIZE -e GPU_COUNT -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e RUN_EVAL -e EVAL_ONLY -e EVAL_FRAMEWORK -e EVAL_LIMIT -e EVAL_SUITE -e IS_AGENTIC -e SCENARIO_TYPE -e SWEBENCH_GEN_MODE -e SWEBENCH_USE_MODAL -e MODAL_TOKEN_ID -e MODAL_TOKEN_SECRET -e RUNNER_TYPE -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e PORT=$PORT \
-e PROFILE -e SGLANG_TORCH_PROFILER_DIR -e VLLM_TORCH_PROFILER_DIR -e VLLM_RPC_TIMEOUT \
-e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e TORCH_CUDA_ARCH_LIST="9.0" -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e CUDA_VISIBLE_DEVICES \
--entrypoint=/bin/bash \
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 1334c95542..50c6a8ad1f 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -54,6 +54,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
cd "$SRT_REPO_DIR"
git checkout sa-submission-q2-2026
fi
+ if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \
+ || exit 1
+ fi
+
echo "Installing srtctl..."
export UV_INSTALL_DIR="/mnt/nfs/sa-shared/.uv/bin"
@@ -145,6 +150,10 @@ EOF
sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE"
# Raise sglang's torch-distributed TCPStore timeout from the 600s gloo default
sed -i '/^ watchdog-timeout:/a\ dist-timeout: 1800' "${CONFIG_FILE%%:*}"
+ if [[ "${EVAL_ONLY:-false}" == "true" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \
+ "${CONFIG_FILE%%:*}" "$FRAMEWORK" || exit 1
+ fi
SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h100,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1)
echo "$SRTCTL_OUTPUT"
@@ -297,21 +306,22 @@ else
fi
trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
- # flock-serialize the enroot import so concurrent sweep jobs on the same
- # shared NFS path don't race each other into 'File already exists' (race
- # observed on PR #1509: 13/30 jobs failed, all on the dgxc-slurm runners
- # hitting the same /mnt/nfs/lustre/containers/.sqsh path). Matches
- # the canonical pattern already used in launch_h100-cw.sh + the mi3xx
- # launchers. The skip-if-valid check avoids re-downloading when the file
- # was successfully created by an earlier job.
+ # Check the shared cache before opening its lock. A valid squash file is
+ # immutable, so readers do not need to touch a lock owned by another user.
srun --jobid=$JOB_ID bash -c "
- exec 9>\"$LOCK_FILE\"
- flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; }
if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then
echo 'Squash file already exists and is valid, skipping import'
else
- rm -f \"$SQUASH_FILE\"
- enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
+ if ! { exec 9>\"$LOCK_FILE\"; } 2>/dev/null; then
+ exec 9<\"$LOCK_FILE\" || { echo 'Failed to open lock for $SQUASH_FILE'; exit 1; }
+ fi
+ flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; }
+ if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then
+ echo 'Squash file was imported by another job'
+ else
+ rm -f \"$SQUASH_FILE\"
+ enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
+ fi
fi
"
diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh
index da510b8177..8e3d8235b1 100755
--- a/runners/launch_h200-dgxc-slurm.sh
+++ b/runners/launch_h200-dgxc-slurm.sh
@@ -149,6 +149,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
cd "$SRT_REPO_DIR"
git checkout sa-submission-q2-2026
fi
+ if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \
+ || exit 1
+ fi
echo "Installing srtctl..."
curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -313,6 +317,10 @@ EOF
sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH"
sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "$CONFIG_PATH"
printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "$CONFIG_PATH"
+ if [[ "${EVAL_ONLY:-false}" == "true" ]]; then
+ python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \
+ "$CONFIG_PATH" "$FRAMEWORK" || exit 1
+ fi
WORKLOAD_TAG="${ISL}x${OSL}"
if [[ "$IS_AGENTIC" == "1" ]]; then
WORKLOAD_TAG="agentic"
diff --git a/runners/launch_mi300x-amd.sh b/runners/launch_mi300x-amd.sh
old mode 100755
new mode 100644
diff --git a/runners/launch_mi325x-tw.sh b/runners/launch_mi325x-tw.sh
index 0ed4be196c..7d0bc63734 100644
--- a/runners/launch_mi325x-tw.sh
+++ b/runners/launch_mi325x-tw.sh
@@ -26,7 +26,7 @@ docker run --rm --network=host --name="$server_name" \
--security-opt seccomp=unconfined --cap-add=SYS_PTRACE \
-v "$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE" \
-v "$GITHUB_WORKSPACE:/workspace/" -w /workspace/ \
--e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e PP_SIZE -e DCP_SIZE -e PCP_SIZE -e GPU_COUNT -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e RUN_EVAL -e EVAL_ONLY -e RUNNER_TYPE -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e PORT="$PORT" \
+-e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e PP_SIZE -e DCP_SIZE -e PCP_SIZE -e GPU_COUNT -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e RUN_EVAL -e EVAL_ONLY -e EVAL_FRAMEWORK -e EVAL_LIMIT -e EVAL_SUITE -e IS_AGENTIC -e SCENARIO_TYPE -e SWEBENCH_GEN_MODE -e SWEBENCH_USE_MODAL -e MODAL_TOKEN_ID -e MODAL_TOKEN_SECRET -e RUNNER_TYPE -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e PORT="$PORT" \
-e DP_ATTENTION -e EP_SIZE -e DP_SIZE -e EVAL_MAX_MODEL_LEN -e SPEC_DECODING -e NUM_SPEC_TOKENS \
-e PROFILE -e SGLANG_TORCH_PROFILER_DIR -e VLLM_TORCH_PROFILER_DIR -e VLLM_RPC_TIMEOUT \
-e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e CUDA_DEVICE_ORDER=PCI_BUS_ID \
diff --git a/runners/launch_rtx6000pro-lat.sh b/runners/launch_rtx6000pro-lat.sh
index 6ad56e7092..4a63a446d7 100755
--- a/runners/launch_rtx6000pro-lat.sh
+++ b/runners/launch_rtx6000pro-lat.sh
@@ -106,7 +106,9 @@ docker run \
--env NUM_SPEC_TOKENS \
--env RUN_EVAL \
--env EVAL_ONLY \
+ --env EVAL_FRAMEWORK \
--env EVAL_LIMIT \
+ --env EVAL_SUITE \
--env EVAL_MAX_MODEL_LEN \
--env RUNNER_TYPE \
--env RUNNER_NAME \
diff --git a/runners/patch_srt_eval_dispatch.py b/runners/patch_srt_eval_dispatch.py
new file mode 100755
index 0000000000..206e9c0133
--- /dev/null
+++ b/runners/patch_srt_eval_dispatch.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Enable InferenceX-selected eval dispatch in an srt-slurm checkout."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+DO_SWEEP_ENV_BLOCK = """ "EVAL_ONLY",
+ "IS_MULTINODE","""
+DO_SWEEP_ENV_REPLACEMENT = """ "EVAL_ONLY",
+ "EVAL_FRAMEWORK",
+ "EVAL_CONC",
+ "EVAL_LIMIT",
+ "EVAL_SUITE",
+ "SWEBENCH_GEN_MODE",
+ "SWEBENCH_USE_MODAL",
+ "MODAL_TOKEN_ID",
+ "MODAL_TOKEN_SECRET",
+ "IS_AGENTIC",
+ "SCENARIO_TYPE",
+ "IS_MULTINODE","""
+LM_EVAL_COMMAND = 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?'
+GENERIC_EVAL_COMMAND = 'run_eval --port "$PORT" || eval_rc=$?'
+EVAL_ARTIFACT_COPY = """cp -v results*.json /logs/eval_results/ 2>/dev/null || true
+cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true"""
+VERIFIER_ARTIFACT_COPY = 'stage_eval_artifacts /logs/eval_results "$PWD" || true'
+
+
+def prepare_replacements(
+ path: Path,
+ replacements: tuple[tuple[str, str], ...],
+) -> tuple[str, str, bool]:
+ """Validate source replacements without mutating the checkout."""
+ original = path.read_text()
+ content = original
+ changed = False
+ for old, new in replacements:
+ old_count = content.count(old)
+ new_count = content.count(new)
+ if old_count == 1 and new_count == 0:
+ anchor = old.splitlines()[0]
+ anchor_count = content.count(anchor)
+ if anchor_count != 1:
+ raise RuntimeError(
+ f"invalid patch state in {path}: "
+ f"anchor {anchor!r} count={anchor_count}"
+ )
+ content = content.replace(old, new, 1)
+ changed = True
+ elif old_count != 0 or new_count != 1:
+ raise RuntimeError(
+ f"invalid patch state in {path}: old anchor count={old_count}, "
+ f"replacement count={new_count}"
+ )
+ return original, content, changed
+
+
+def patch_checkout(root: Path) -> list[Path]:
+ """Patch both post-eval sources after validating the complete checkout."""
+ patches = (
+ (
+ root / "src/srtctl/cli/do_sweep.py",
+ ((DO_SWEEP_ENV_BLOCK, DO_SWEEP_ENV_REPLACEMENT),),
+ ),
+ (
+ root / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh",
+ (
+ (LM_EVAL_COMMAND, GENERIC_EVAL_COMMAND),
+ (EVAL_ARTIFACT_COPY, VERIFIER_ARTIFACT_COPY),
+ ),
+ ),
+ )
+ staged = [
+ (path, *prepare_replacements(path, replacements))
+ for path, replacements in patches
+ ]
+ changed = []
+ written = []
+ try:
+ for path, original, replacement, needs_write in staged:
+ if needs_write:
+ path.write_text(replacement)
+ written.append((path, original))
+ changed.append(path)
+ except OSError:
+ for path, original in reversed(written):
+ path.write_text(original)
+ raise
+ return changed
+
+
+def main(argv: list[str]) -> int:
+ """Patch the checkout named on the command line."""
+ if len(argv) != 2:
+ print(f"Usage: {argv[0]} SRT_SLURM_CHECKOUT", file=sys.stderr)
+ return 2
+ root = Path(argv[1]).resolve()
+ try:
+ changed = patch_checkout(root)
+ except (OSError, RuntimeError) as error:
+ print(
+ f"ERROR: failed to patch srt-slurm eval dispatch: {error}", file=sys.stderr
+ )
+ return 1
+ if changed:
+ for path in changed:
+ print(f"Patched srt-slurm eval dispatch: {path}")
+ else:
+ print("srt-slurm eval dispatch is already patched")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/runners/patch_srt_vllm_dp_ranks.py b/runners/patch_srt_vllm_dp_ranks.py
new file mode 100755
index 0000000000..9decf0f06e
--- /dev/null
+++ b/runners/patch_srt_vllm_dp_ranks.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Patch srt-slurm v1.0.53 to allocate one process per vLLM DP rank."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+OLD_BLOCK = ''' else:
+ # DP+EP mode: one process per GPU
+ # Each process gets a single GPU and a unique dp_rank
+ dp_rank = 0
+ # Allocate a unique DP RPC port for this endpoint's leader node
+ dp_rpc_port = port_allocator.next_dp_rpc_port(endpoint.leader_node)
+ # Allocate a single NIXL base port for this endpoint.
+ # vLLM internally computes: actual_port = base + data_parallel_rank
+ # so all DP ranks in the endpoint share the same base port.
+ dp_size = self._get_dp_size(endpoint.mode) or len(endpoint.gpu_indices)
+ nixl_base_port = port_allocator.next_nixl_port_block(dp_size)
+ for _node_rank, node in enumerate(endpoint.nodes):
+ for gpu_idx in sorted(endpoint.gpu_indices):
+ is_leader = dp_rank == 0
+ http_port = port_allocator.next_http_port(node) if is_leader else 0
+ bootstrap_port = (
+ port_allocator.next_bootstrap_port(node)
+ if endpoint.mode == "prefill" and is_leader
+ else None
+ )
+ kv_events_port = port_allocator.next_kv_events_port()
+ nixl_port = nixl_base_port
+
+ processes.append(
+ Process(
+ node=node,
+ gpu_indices=frozenset([gpu_idx]), # Single GPU per process
+ sys_port=current_sys_port,
+ http_port=http_port,
+ endpoint_mode=endpoint.mode,
+ endpoint_index=endpoint.index,
+ node_rank=dp_rank, # dp_rank stored in node_rank for now
+ bootstrap_port=bootstrap_port,
+ kv_events_port=kv_events_port,
+ nixl_port=nixl_port,
+ dp_rpc_port=dp_rpc_port,
+ )
+ )
+ current_sys_port += 1
+ dp_rank += 1
+'''
+
+NEW_BLOCK = ''' else:
+ # External DP mode: one process per DP rank. A rank may own
+ # multiple GPUs when tensor or pipeline parallelism is enabled.
+ dp_rank = 0
+ dp_rpc_port = port_allocator.next_dp_rpc_port(endpoint.leader_node)
+ config = self.get_config_for_mode(endpoint.mode)
+ dp_size = self._get_dp_size(endpoint.mode) or endpoint.total_gpus
+ tp_size = config.get("tensor-parallel-size") or config.get("tensor_parallel_size") or 1
+ pp_size = config.get("pipeline-parallel-size") or config.get("pipeline_parallel_size") or 1
+ gpus_per_dp_rank = tp_size * pp_size
+ expected_gpus = dp_size * gpus_per_dp_rank
+ if endpoint.total_gpus != expected_gpus:
+ raise ValueError(
+ f"{endpoint.mode} DP={dp_size}, TP={tp_size}, PP={pp_size} requires "
+ f"{expected_gpus} GPUs, but the endpoint allocated {endpoint.total_gpus}"
+ )
+
+ nixl_base_port = port_allocator.next_nixl_port_block(dp_size)
+ for node in endpoint.nodes:
+ local_gpus = sorted(endpoint.gpu_indices)
+ if len(local_gpus) % gpus_per_dp_rank != 0:
+ raise ValueError(
+ f"{endpoint.mode} TP={tp_size}, PP={pp_size} requires "
+ f"{gpus_per_dp_rank} GPUs per DP rank, but node {node} has "
+ f"{len(local_gpus)} allocated GPUs"
+ )
+ for offset in range(0, len(local_gpus), gpus_per_dp_rank):
+ rank_gpus = frozenset(local_gpus[offset : offset + gpus_per_dp_rank])
+ is_leader = dp_rank == 0
+ http_port = port_allocator.next_http_port(node) if is_leader else 0
+ bootstrap_port = (
+ port_allocator.next_bootstrap_port(node)
+ if endpoint.mode == "prefill" and is_leader
+ else None
+ )
+
+ processes.append(
+ Process(
+ node=node,
+ gpu_indices=rank_gpus,
+ sys_port=current_sys_port,
+ http_port=http_port,
+ endpoint_mode=endpoint.mode,
+ endpoint_index=endpoint.index,
+ node_rank=dp_rank,
+ bootstrap_port=bootstrap_port,
+ kv_events_port=port_allocator.next_kv_events_port(),
+ nixl_port=nixl_base_port,
+ dp_rpc_port=dp_rpc_port,
+ )
+ )
+ current_sys_port += 1
+ dp_rank += 1
+
+ if dp_rank != dp_size:
+ raise ValueError(
+ f"{endpoint.mode} allocated {dp_rank} DP ranks, expected {dp_size}"
+ )
+'''
+
+
+def patch_backend(root: Path) -> bool:
+ """Apply the rank allocator patch and return whether the file changed."""
+ backend = root / "src/srtctl/backends/vllm.py"
+ source = backend.read_text()
+
+ if NEW_BLOCK in source:
+ return False
+ if source.count(OLD_BLOCK) != 1:
+ raise RuntimeError(
+ f"unsupported srt-slurm vLLM backend at {backend}: expected allocation block not found exactly once"
+ )
+
+ backend.write_text(source.replace(OLD_BLOCK, NEW_BLOCK))
+ return True
+
+
+def main(argv: list[str]) -> int:
+ if len(argv) != 2:
+ print(f"Usage: {argv[0]} SRT_SLURM_CHECKOUT", file=sys.stderr)
+ return 2
+
+ try:
+ changed = patch_backend(Path(argv[1]).resolve())
+ except (OSError, RuntimeError) as error:
+ print(f"ERROR: failed to patch srt-slurm vLLM DP ranks: {error}", file=sys.stderr)
+ return 1
+
+ state = "Patched" if changed else "Already patched"
+ print(f"{state} srt-slurm vLLM DP rank allocation")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv))
diff --git a/runners/patch_trtllm_chat_store.py b/runners/patch_trtllm_chat_store.py
new file mode 100755
index 0000000000..09fa4e5160
--- /dev/null
+++ b/runners/patch_trtllm_chat_store.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Accept OpenAI's non-persistent chat request field in TensorRT-LLM."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+CLASS_HEADER = "class ChatCompletionRequest(OpenAIBaseModel):\n"
+NEXT_CLASS = "\nclass "
+FIELD_ANCHOR = " stream: Optional[bool] = False\n"
+STORE_FIELD = " store: Optional[Literal[False]] = False\n"
+
+
+def installed_protocol_path() -> Path:
+ """Return the protocol module path from the installed TensorRT-LLM package."""
+ spec = importlib.util.find_spec("tensorrt_llm")
+ if spec is None or not spec.submodule_search_locations:
+ raise RuntimeError("tensorrt_llm package is not installed")
+ package_root = Path(next(iter(spec.submodule_search_locations)))
+ return package_root / "serve/openai_protocol.py"
+
+
+def patch_protocol(protocol_path: Path) -> bool:
+ """Accept only ``store=false`` and return whether the source changed."""
+ source = protocol_path.read_text()
+ if source.count(CLASS_HEADER) != 1:
+ raise RuntimeError(
+ f"unsupported TensorRT-LLM protocol at {protocol_path}: "
+ "ChatCompletionRequest not found exactly once"
+ )
+
+ class_start = source.index(CLASS_HEADER)
+ class_end = source.find(NEXT_CLASS, class_start + len(CLASS_HEADER))
+ if class_end == -1:
+ raise RuntimeError(
+ f"unsupported TensorRT-LLM protocol at {protocol_path}: "
+ "ChatCompletionRequest boundary not found"
+ )
+
+ class_source = source[class_start:class_end]
+ if STORE_FIELD in class_source:
+ return False
+ if " store:" in class_source:
+ raise RuntimeError(
+ f"unsupported TensorRT-LLM store field at {protocol_path}"
+ )
+ if class_source.count(FIELD_ANCHOR) != 1:
+ raise RuntimeError(
+ f"unsupported TensorRT-LLM protocol at {protocol_path}: "
+ "ChatCompletionRequest stream field not found exactly once"
+ )
+
+ patched_class = class_source.replace(
+ FIELD_ANCHOR,
+ STORE_FIELD + FIELD_ANCHOR,
+ )
+ protocol_path.write_text(source[:class_start] + patched_class + source[class_end:])
+ return True
+
+
+def main(argv: list[str]) -> int:
+ if len(argv) > 2:
+ print(f"Usage: {argv[0]} [OPENAI_PROTOCOL_PATH]", file=sys.stderr)
+ return 2
+
+ try:
+ protocol_path = (
+ Path(argv[1]).resolve() if len(argv) == 2 else installed_protocol_path()
+ )
+ changed = patch_protocol(protocol_path)
+ except (OSError, RuntimeError) as error:
+ print(f"ERROR: failed to patch TensorRT-LLM chat requests: {error}", file=sys.stderr)
+ return 1
+
+ state = "Patched" if changed else "Already patched"
+ print(f"{state} TensorRT-LLM ChatCompletionRequest store=false support")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv))
diff --git a/runners/patch_vllm_simple_kv_offload.py b/runners/patch_vllm_simple_kv_offload.py
new file mode 100755
index 0000000000..872bd9960a
--- /dev/null
+++ b/runners/patch_vllm_simple_kv_offload.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Split heterogeneous vLLM KV backing storage into valid CPU offload regions."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+OLD_SETUP = """ logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size
+
+ # The DMA backend copies whole blocks as base + block_id * stride(0),
+"""
+NEW_SETUP = """ logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size
+ split_storage_by_layer = any(
+ logical_storage_bytes
+ % (num_blocks * cache_tensor.block_stride)
+ != 0
+ for cache_tensor in self.kv_cache_config.kv_cache_tensors
+ )
+
+ # The DMA backend copies whole blocks as base + block_id * stride(0),
+"""
+OLD_LOOP = """ unique_gpu_caches: dict[str, torch.Tensor] = {}
+ seen: set[tuple[torch.device, int]] = set()
+ for name, tensor in kv_caches.items():
+ storage = tensor.untyped_storage()
+ key = (tensor.device, storage.data_ptr())
+ if key in seen:
+ continue
+ seen.add(key)
+
+ physical_per_block, remainder = divmod(tensor.shape[0], num_blocks)
+ assert remainder == 0, (
+ f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which "
+ f"is not divisible by {num_blocks} scheduler blocks"
+ )
+ block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block
+ raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage)
+ assert raw.numel() >= logical_storage_bytes, (
+ f"KV cache {name!r} storage has {raw.numel()} bytes, smaller "
+ f"than the configured {logical_storage_bytes}-byte allocation"
+ )
+ regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes)
+ for idx, region in enumerate(regions):
+ key_name = name if len(regions) == 1 else f"{name}.{idx}"
+ unique_gpu_caches[key_name] = region
+"""
+NEW_LOOP = """ unique_gpu_caches: dict[str, torch.Tensor] = {}
+ seen: set[tuple[torch.device, int, int, int]] = set()
+ for name, tensor in kv_caches.items():
+ physical_per_block, remainder = divmod(tensor.shape[0], num_blocks)
+ assert remainder == 0, (
+ f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which "
+ f"is not divisible by {num_blocks} scheduler blocks"
+ )
+ block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block
+ storage = tensor.untyped_storage()
+ raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage)
+
+ if split_storage_by_layer:
+ region_offset = tensor.storage_offset() * tensor.element_size()
+ region_bytes = num_blocks * block_bytes
+ else:
+ region_offset = 0
+ region_bytes = logical_storage_bytes
+
+ key = (tensor.device, storage.data_ptr(), region_offset, region_bytes)
+ if key in seen:
+ continue
+ seen.add(key)
+
+ region_end = region_offset + region_bytes
+ assert raw.numel() >= region_end, (
+ f"KV cache {name!r} storage has {raw.numel()} bytes, smaller "
+ f"than the required {region_end}-byte region"
+ )
+ regions = raw[region_offset:region_end].view(
+ -1, num_blocks, block_bytes
+ )
+ for idx, region in enumerate(regions):
+ key_name = name if len(regions) == 1 else f"{name}.{idx}"
+ unique_gpu_caches[key_name] = region
+"""
+
+
+def installed_worker_path() -> Path:
+ """Return the SimpleCPUOffload worker module from the installed vLLM."""
+ spec = importlib.util.find_spec("vllm")
+ if spec is None or not spec.submodule_search_locations:
+ raise RuntimeError("vllm package is not installed")
+ package_root = Path(next(iter(spec.submodule_search_locations)))
+ return package_root / "v1/simple_kv_offload/worker.py"
+
+
+def patch_worker(worker_path: Path) -> bool:
+ """Patch heterogeneous layer-region sizing and return whether source changed."""
+ source = worker_path.read_text()
+ if NEW_SETUP in source and NEW_LOOP in source:
+ return False
+ if NEW_SETUP in source or NEW_LOOP in source:
+ raise RuntimeError(f"partially patched vLLM worker at {worker_path}")
+ if source.count(OLD_SETUP) != 1 or source.count(OLD_LOOP) != 1:
+ raise RuntimeError(
+ f"unsupported vLLM SimpleCPUOffload worker at {worker_path}"
+ )
+
+ patched = source.replace(OLD_SETUP, NEW_SETUP).replace(OLD_LOOP, NEW_LOOP)
+ worker_path.write_text(patched)
+ return True
+
+
+def main(argv: list[str]) -> int:
+ if len(argv) > 2:
+ print(f"Usage: {argv[0]} [WORKER_PATH]", file=sys.stderr)
+ return 2
+
+ try:
+ worker_path = (
+ Path(argv[1]).resolve() if len(argv) == 2 else installed_worker_path()
+ )
+ changed = patch_worker(worker_path)
+ except (OSError, RuntimeError) as error:
+ print(f"ERROR: failed to patch vLLM CPU offload: {error}", file=sys.stderr)
+ return 1
+
+ state = "Patched" if changed else "Already patched"
+ print(f"{state} vLLM SimpleCPUOffload heterogeneous layer regions")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv))
diff --git a/runners/synthetic_injectors/sglang.py b/runners/synthetic_injectors/sglang.py
index 48fb1706aa..d5cd6e0b16 100644
--- a/runners/synthetic_injectors/sglang.py
+++ b/runners/synthetic_injectors/sglang.py
@@ -7,6 +7,9 @@
_SPEC_STEPS_RE = re.compile(r"(?m)^\s+speculative-num-steps:\s*([0-9]+)\s*$")
_ENV_BLOCK_RE = re.compile(r"(?m)^( (?:aggregated|prefill|decode)_environment:\s*)$")
+_SIMULATED_ACCEPTANCE_ENV_RE = re.compile(
+ r"(?m)^[ \t]+SGLANG_SIMULATE_ACC_(?:LEN|METHOD|TOKEN_MODE):[^\n]*(?:\n|$)"
+)
def spec_tokens_from_recipe(text):
@@ -25,10 +28,21 @@ def rewrite(content, al, log):
'\n SGLANG_SIMULATE_ACC_METHOD: "match-expected"'
'\n SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"'
)
- rewritten, count = _ENV_BLOCK_RE.subn(lambda match: match.group(1) + variables, content)
+ rewritten, count = _ENV_BLOCK_RE.subn(
+ lambda match: match.group(1) + variables,
+ content,
+ )
if count:
log(f"Added SGLANG_SIMULATE_ACC_* to {count} worker environment block(s)")
return rewritten, count
+def rewrite_real(content, log):
+ """Remove throughput-only simulated-acceptance variables for evals."""
+ rewritten, count = _SIMULATED_ACCEPTANCE_ENV_RE.subn("", content)
+ if count:
+ log(f"Removed {count} SGLANG_SIMULATE_ACC_* environment variable(s)")
+ return rewritten, count
+
+
register("dynamo-sglang", sys.modules[__name__])
diff --git a/runners/synthetic_injectors/vllm.py b/runners/synthetic_injectors/vllm.py
index e71a5df2b4..d36740195c 100644
--- a/runners/synthetic_injectors/vllm.py
+++ b/runners/synthetic_injectors/vllm.py
@@ -1,13 +1,8 @@
-"""vLLM synthetic-acceptance backend (FRAMEWORK=dynamo-vllm).
+"""vLLM speculative-acceptance recipe rewriting.
-Rewrites every ``speculative-config: ''`` entry in an srt-slurm recipe to
-use synthetic rejection sampling: it adds ``rejection_sample_method=synthetic``
-and ``synthetic_acceptance_length=`` to the JSON so the engine emits a
-controlled mean acceptance length instead of running the real draft model.
-
-Registered under the "dynamo-vllm" framework key at import time, so importing
-the ``synthetic_injectors`` package is enough for the generic driver to resolve
-this backend.
+Throughput opt-ins can inject synthetic acceptance. Eval-only runs restore real
+block verification so model outputs remain valid for accuracy checks. The
+backend is registered for both direct vLLM and Dynamo-vLLM recipes.
"""
import json
@@ -57,7 +52,9 @@ def _replace(match):
new_content, count = _SPEC_CONFIG_RE.subn(_replace, content)
if count:
- after = [ln.strip() for ln in new_content.splitlines() if _SPEC_CONFIG_RE.search(ln)]
+ after = [
+ ln.strip() for ln in new_content.splitlines() if _SPEC_CONFIG_RE.search(ln)
+ ]
if after:
log("After:")
for ln in after:
@@ -66,4 +63,28 @@ def _replace(match):
return new_content, count
+def rewrite_real(content, log):
+ """Restore real block verification in every synthetic config entry."""
+ modified = 0
+
+ def _replace(match):
+ nonlocal modified
+ spec = json.loads(match.group(1))
+ if (
+ spec.get("rejection_sample_method") != "synthetic"
+ and "synthetic_acceptance_length" not in spec
+ ):
+ return match.group(0)
+ spec["rejection_sample_method"] = "block"
+ spec.pop("synthetic_acceptance_length", None)
+ modified += 1
+ return "speculative-config: '" + json.dumps(spec, separators=(",", ":")) + "'"
+
+ new_content = _SPEC_CONFIG_RE.sub(_replace, content)
+ if modified:
+ log("Restored real block verification for eval-only mode")
+ return new_content, modified
+
+
register("dynamo-vllm", sys.modules[__name__])
+register("vllm", sys.modules[__name__])
diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py
index c027cc9b2f..162acab380 100644
--- a/runners/test_slurm_utils.py
+++ b/runners/test_slurm_utils.py
@@ -1,9 +1,18 @@
+import json
+import os
+import runpy
import subprocess
from pathlib import Path
+import yaml
REPO_ROOT = Path(__file__).resolve().parents[1]
SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh"
+PATCH_SRT_EVAL = REPO_ROOT / "runners" / "patch_srt_eval_dispatch.py"
+PATCH_SRT_DP_RANKS = REPO_ROOT / "runners" / "patch_srt_vllm_dp_ranks.py"
+PATCH_TRTLLM_CHAT_STORE = REPO_ROOT / "runners" / "patch_trtllm_chat_store.py"
+PATCH_VLLM_SIMPLE_KV = REPO_ROOT / "runners" / "patch_vllm_simple_kv_offload.py"
+INJECT_ACCEPTANCE = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py"
def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str]:
@@ -55,3 +64,516 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing(
assert result.returncode != 0
assert "no run_conc*.json results found" in result.stderr
+
+
+def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent(
+ tmp_path: Path,
+) -> None:
+ do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py"
+ eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh"
+ do_sweep.parent.mkdir(parents=True)
+ eval_script.parent.mkdir(parents=True)
+ do_sweep.write_text(
+ " for var in [\n"
+ ' "RUN_EVAL",\n'
+ ' "EVAL_ONLY",\n'
+ ' "IS_MULTINODE",\n'
+ " ]:\n"
+ )
+ eval_script.write_text(
+ 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n'
+ "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n"
+ "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n"
+ )
+
+ first = subprocess.run(
+ ["python3", str(PATCH_SRT_EVAL), str(tmp_path)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ second = subprocess.run(
+ ["python3", str(PATCH_SRT_EVAL), str(tmp_path)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert first.returncode == 0, first.stderr
+ assert second.returncode == 0, second.stderr
+ assert do_sweep.read_text().count('"EVAL_FRAMEWORK"') == 1
+ assert do_sweep.read_text().count('"EVAL_SUITE"') == 1
+ assert do_sweep.read_text().count('"EVAL_CONC"') == 1
+ assert do_sweep.read_text().count('"EVAL_LIMIT"') == 1
+ assert do_sweep.read_text().count('"SWEBENCH_GEN_MODE"') == 1
+ assert do_sweep.read_text().count('"SWEBENCH_USE_MODAL"') == 1
+ assert do_sweep.read_text().count('"MODAL_TOKEN_ID"') == 1
+ assert do_sweep.read_text().count('"MODAL_TOKEN_SECRET"') == 1
+ assert do_sweep.read_text().count('"IS_AGENTIC"') == 1
+ assert do_sweep.read_text().count('"SCENARIO_TYPE"') == 1
+ assert 'run_eval --port "$PORT"' in eval_script.read_text()
+ assert "--framework lm-eval" not in eval_script.read_text()
+ assert 'stage_eval_artifacts /logs/eval_results "$PWD" || true' in eval_script.read_text()
+ assert "cp -v" not in eval_script.read_text()
+ assert "already patched" in second.stdout
+
+
+def test_patch_srt_vllm_dp_ranks_groups_tensor_parallel_devices(
+ tmp_path: Path,
+) -> None:
+ symbols = runpy.run_path(str(PATCH_SRT_DP_RANKS))
+ backend = tmp_path / "src/srtctl/backends/vllm.py"
+ backend.parent.mkdir(parents=True)
+ backend.write_text(f"prefix\n{symbols['OLD_BLOCK']}suffix\n")
+
+ first = subprocess.run(
+ ["python3", str(PATCH_SRT_DP_RANKS), str(tmp_path)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ second = subprocess.run(
+ ["python3", str(PATCH_SRT_DP_RANKS), str(tmp_path)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ patched = backend.read_text()
+ assert first.returncode == 0, first.stderr
+ assert second.returncode == 0, second.stderr
+ assert symbols["OLD_BLOCK"] not in patched
+ assert patched.count(symbols["NEW_BLOCK"]) == 1
+ assert "gpus_per_dp_rank = tp_size * pp_size" in patched
+ assert "gpu_indices=rank_gpus" in patched
+ assert "already patched" in second.stdout.lower()
+
+
+def test_patch_srt_vllm_dp_ranks_rejects_unknown_source(tmp_path: Path) -> None:
+ backend = tmp_path / "src/srtctl/backends/vllm.py"
+ backend.parent.mkdir(parents=True)
+ backend.write_text("unsupported backend\n")
+
+ result = subprocess.run(
+ ["python3", str(PATCH_SRT_DP_RANKS), str(tmp_path)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 1
+ assert backend.read_text() == "unsupported backend\n"
+
+
+
+def test_patch_trtllm_chat_store_accepts_false_and_is_idempotent(
+ tmp_path: Path,
+) -> None:
+ protocol = tmp_path / "openai_protocol.py"
+ protocol.write_text(
+ "from typing import Literal, Optional\n\n"
+ "class ChatCompletionRequest(OpenAIBaseModel):\n"
+ " messages: list\n"
+ " stream: Optional[bool] = False\n"
+ " user: Optional[str] = None\n\n"
+ "class ResponsesRequest(OpenAIBaseModel):\n"
+ " store: Optional[bool] = True\n"
+ )
+
+ first = subprocess.run(
+ ["python3", str(PATCH_TRTLLM_CHAT_STORE), str(protocol)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ second = subprocess.run(
+ ["python3", str(PATCH_TRTLLM_CHAT_STORE), str(protocol)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ patched = protocol.read_text()
+ assert first.returncode == 0, first.stderr
+ assert second.returncode == 0, second.stderr
+ assert patched.count("store: Optional[Literal[False]] = False") == 1
+ assert patched.count("store: Optional[bool] = True") == 1
+ assert "already patched" in second.stdout.lower()
+
+
+def test_patch_trtllm_chat_store_rejects_unknown_source(tmp_path: Path) -> None:
+ protocol = tmp_path / "openai_protocol.py"
+ protocol.write_text("unsupported protocol\n")
+
+ result = subprocess.run(
+ ["python3", str(PATCH_TRTLLM_CHAT_STORE), str(protocol)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 1
+ assert protocol.read_text() == "unsupported protocol\n"
+
+def test_patch_vllm_simple_kv_offload_splits_heterogeneous_layers(
+ tmp_path: Path,
+) -> None:
+ symbols = runpy.run_path(str(PATCH_VLLM_SIMPLE_KV))
+ worker = tmp_path / "worker.py"
+ worker.write_text(
+ f"prefix\n{symbols['OLD_SETUP']}{symbols['OLD_LOOP']}suffix\n"
+ )
+
+ first = subprocess.run(
+ ["python3", str(PATCH_VLLM_SIMPLE_KV), str(worker)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ second = subprocess.run(
+ ["python3", str(PATCH_VLLM_SIMPLE_KV), str(worker)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ patched = worker.read_text()
+ assert first.returncode == 0, first.stderr
+ assert second.returncode == 0, second.stderr
+ assert symbols["OLD_SETUP"] not in patched
+ assert symbols["OLD_LOOP"] not in patched
+ assert patched.count(symbols["NEW_SETUP"]) == 1
+ assert patched.count(symbols["NEW_LOOP"]) == 1
+ assert "split_storage_by_layer" in patched
+ assert "tensor.storage_offset() * tensor.element_size()" in patched
+ assert "already patched" in second.stdout.lower()
+
+
+def test_patch_vllm_simple_kv_offload_rejects_unknown_source(
+ tmp_path: Path,
+) -> None:
+ worker = tmp_path / "worker.py"
+ worker.write_text("unsupported worker\n")
+
+ result = subprocess.run(
+ ["python3", str(PATCH_VLLM_SIMPLE_KV), str(worker)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 1
+ assert worker.read_text() == "unsupported worker\n"
+
+
+def test_minimax_vllm_launchers_patch_simple_kv_offload() -> None:
+ launchers = (
+ REPO_ROOT / "benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh",
+ REPO_ROOT / "benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh",
+ )
+
+ for launcher in launchers:
+ assert "patch_vllm_simple_kv_offload.py" in launcher.read_text(), launcher
+
+
+def test_minimax_trt_launchers_patch_chat_store_request() -> None:
+ launchers = (
+ REPO_ROOT / "benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh",
+ REPO_ROOT / "benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh",
+ )
+
+ for launcher in launchers:
+ assert "patch_trtllm_chat_store.py" in launcher.read_text(), launcher
+
+def test_patch_srt_eval_dispatch_preflights_before_writing(tmp_path: Path) -> None:
+ do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py"
+ eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh"
+ do_sweep.parent.mkdir(parents=True)
+ eval_script.parent.mkdir(parents=True)
+ original_do_sweep = ' "EVAL_ONLY",\n "IS_MULTINODE",\n'
+ original_eval_script = "unsupported eval hook\n"
+ do_sweep.write_text(original_do_sweep)
+ eval_script.write_text(original_eval_script)
+
+ result = subprocess.run(
+ ["python3", str(PATCH_SRT_EVAL), str(tmp_path)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 1
+ assert do_sweep.read_text() == original_do_sweep
+ assert eval_script.read_text() == original_eval_script
+
+
+def test_patch_srt_eval_dispatch_rejects_mixed_patch_state(tmp_path: Path) -> None:
+ do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py"
+ eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh"
+ do_sweep.parent.mkdir(parents=True)
+ eval_script.parent.mkdir(parents=True)
+ original_do_sweep = (
+ ' "EVAL_ONLY",\n'
+ ' "IS_MULTINODE",\n'
+ ' "EVAL_ONLY",\n'
+ ' "EVAL_FRAMEWORK",\n'
+ ' "EVAL_SUITE",\n'
+ ' "IS_MULTINODE",\n'
+ )
+ original_eval_script = (
+ 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n'
+ "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n"
+ "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n"
+ )
+ do_sweep.write_text(original_do_sweep)
+ eval_script.write_text(original_eval_script)
+
+ result = subprocess.run(
+ ["python3", str(PATCH_SRT_EVAL), str(tmp_path)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 1
+ assert "invalid patch state" in result.stderr
+ assert do_sweep.read_text() == original_do_sweep
+ assert eval_script.read_text() == original_eval_script
+
+
+def test_eval_only_restores_real_vllm_acceptance(tmp_path: Path) -> None:
+ recipe = tmp_path / "recipe.yaml"
+ recipe.write_text(
+ "speculative-config: "
+ """'{\"method\":\"dspark\",\"num_speculative_tokens\":2,"""
+ """\"rejection_sample_method\":\"synthetic\","""
+ """\"synthetic_acceptance_length\":2.51}'\n"""
+ )
+ env = {
+ **os.environ,
+ "EVAL_ONLY": "true",
+ "SYNTHETIC_ACCEPTANCE": "true",
+ }
+
+ result = subprocess.run(
+ ["python3", str(INJECT_ACCEPTANCE), str(recipe), "vllm"],
+ env=env,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 0, result.stderr
+ rewritten = recipe.read_text()
+ assert '"rejection_sample_method":"block"' in rewritten
+ assert "synthetic_acceptance_length" not in rewritten
+
+
+
+def test_eval_only_removes_sglang_simulated_acceptance(tmp_path: Path) -> None:
+ recipe = tmp_path / "recipe.yaml"
+ recipe.write_text(
+ "backend:\n"
+ " sglang_config:\n"
+ " decode_environment:\n"
+ ' SGLANG_SIMULATE_ACC_LEN: "2.99"\n'
+ ' SGLANG_SIMULATE_ACC_METHOD: "match-expected"\n'
+ ' SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"\n'
+ " KEEP_ME: unchanged\n"
+ )
+
+ result = subprocess.run(
+ ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"],
+ env={**os.environ, "EVAL_ONLY": "true"},
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 0, result.stderr
+ rewritten = recipe.read_text()
+ assert "SGLANG_SIMULATE_ACC_" not in rewritten
+ assert "KEEP_ME: unchanged" in rewritten
+
+
+def test_sglang_throughput_rejects_existing_simulated_acceptance(
+ tmp_path: Path,
+) -> None:
+ recipe = tmp_path / "recipe.yaml"
+ original = (
+ "backend:\n"
+ " aggregated_environment:\n"
+ ' SGLANG_SIMULATE_ACC_LEN: "2.99"\n'
+ ' SGLANG_SIMULATE_ACC_METHOD: "match-expected"\n'
+ ' SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"\n'
+ " KEEP_ME: unchanged\n"
+ )
+ recipe.write_text(original)
+
+ result = subprocess.run(
+ ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"],
+ env={
+ **os.environ,
+ "SYNTHETIC_ACCEPTANCE": "true",
+ "SYNTHETIC_ACCEPTANCE_LENGTH": "3.39",
+ },
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode != 0
+ assert "already contains SGLANG_SIMULATE_ACC_" in result.stderr
+ assert recipe.read_text() == original
+
+
+def test_eval_only_acceptance_rewrite_allows_non_speculative_recipe(
+ tmp_path: Path,
+) -> None:
+ recipe = tmp_path / "recipe.yaml"
+ original = "backend:\n type: vllm\n"
+ recipe.write_text(original)
+
+ result = subprocess.run(
+ ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-vllm"],
+ env={**os.environ, "EVAL_ONLY": "true"},
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert recipe.read_text() == original
+
+
+def test_nvidia_srt_launchers_prepare_kimi_eval_dispatch() -> None:
+ launchers = (
+ REPO_ROOT / "runners/launch_h100-dgxc-slurm.sh",
+ REPO_ROOT / "runners/launch_h200-dgxc-slurm.sh",
+ REPO_ROOT / "runners/launch_b200-nscale-slurm.sh",
+ REPO_ROOT / "runners/launch_b300-nv.sh",
+ REPO_ROOT / "runners/launch_gb200-nv.sh",
+ REPO_ROOT / "runners/launch_gb300-nv.sh",
+ )
+
+ for launcher in launchers:
+ content = launcher.read_text()
+ assert "patch_srt_eval_dispatch.py" in content
+ patch_command = content.index("patch_srt_eval_dispatch.py")
+ assert "|| exit 1" in content[patch_command : patch_command + 200]
+ assert 'EVAL_FRAMEWORK:-lm-eval}" != "lm-eval"' in content
+ assert "inject_synthetic_acceptance" in content
+
+def test_gb200_acceptance_driver_is_unconditional_and_fail_closed() -> None:
+ content = (REPO_ROOT / "runners/launch_gb200-nv.sh").read_text()
+ command = (
+ 'python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py"'
+ )
+ command_index = content.index(command)
+
+ assert content.rfind("\nfi", 0, command_index) > content.rfind(
+ "\nif ", 0, command_index
+ )
+ assert "|| exit 1" in content[command_index : command_index + 180]
+
+
+def test_gb200_kimi_compilation_config_preserves_all_settings() -> None:
+ recipes = {
+ "agg-gb200-tep16-balanced-agentic.yaml": 96,
+ "agg-gb200-tp16-latency-agentic.yaml": 24,
+ }
+ recipe_dir = (
+ REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic"
+ )
+
+ for filename, largest_capture in recipes.items():
+ recipe = yaml.safe_load((recipe_dir / filename).read_text())
+ raw_config = recipe["backend"]["vllm_config"]["aggregated"][
+ "compilation-config"
+ ]
+ compilation_config = json.loads(raw_config)
+
+ assert compilation_config["cudagraph_capture_sizes"][-1] == largest_capture
+ assert compilation_config["pass_config"]["fuse_allreduce_rms"] is False
+
+
+def test_gb200_kimi_recipes_configure_tool_parser() -> None:
+ recipe_dir = (
+ REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic"
+ )
+ recipe_paths = sorted(recipe_dir.glob("agg-gb200-*-agentic.yaml"))
+ frontend_counts = {"dynamo": 0, "vllm": 0}
+
+ assert len(recipe_paths) == 11
+ for recipe_path in recipe_paths:
+ recipe = yaml.safe_load(recipe_path.read_text())
+ frontend = recipe["frontend"]
+ frontend_type = frontend["type"]
+ config = recipe["backend"]["vllm_config"]["aggregated"]
+ assert frontend_type in frontend_counts, recipe_path
+ frontend_counts[frontend_type] += 1
+ if frontend_type == "dynamo":
+ args = frontend["args"]
+ assert args["dyn-chat-processor"] == "vllm", recipe_path
+ assert args["tool-call-parser"] == "kimi_k3", recipe_path
+ assert args["reasoning-parser"] == "kimi_k3", recipe_path
+ assert args["enable-auto-tool-choice"] is True, recipe_path
+ assert config["dyn-tool-call-parser"] == "kimi_k3", recipe_path
+ assert config["dyn-reasoning-parser"] == "kimi_k3", recipe_path
+ else:
+ assert config["enable-auto-tool-choice"] is True, recipe_path
+ assert config["tool-call-parser"] == "kimi_k3", recipe_path
+ assert config["reasoning-parser"] == "kimi_k3", recipe_path
+
+ assert frontend_counts == {"dynamo": 6, "vllm": 5}
+
+
+def test_gb200_dynamo_minimax_recipes_configure_frontend_tool_parser() -> None:
+ recipe_dir = (
+ REPO_ROOT
+ / "benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3"
+ / "gb200-fp4/agentic"
+ )
+ recipe_paths = sorted(recipe_dir.glob("*.yaml"))
+
+ assert len(recipe_paths) == 6
+ for recipe_path in recipe_paths:
+ recipe = yaml.safe_load(recipe_path.read_text())
+ args = recipe["frontend"]["args"]
+ assert args["dyn-chat-processor"] == "vllm", recipe_path
+ assert args["tool-call-parser"] == "minimax_m3", recipe_path
+ assert args["reasoning-parser"] == "minimax_m3", recipe_path
+ assert args["enable-auto-tool-choice"] is True, recipe_path
+
+
+def test_mi355_minimax_launcher_configures_reasoning_parser() -> None:
+ launcher = (
+ REPO_ROOT
+ / "benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh"
+ ).read_text()
+
+ assert "--tool-call-parser minimax_m3" in launcher
+ assert "--reasoning-parser minimax_m3" in launcher
+ assert "--enable-auto-tool-choice" in launcher
+
+
+def test_swebench_container_paths_forward_modal_credentials() -> None:
+ paths = (
+ REPO_ROOT / "benchmarks/multi_node/llm-d/submit.sh",
+ REPO_ROOT / "benchmarks/multi_node/llm-d/job.slurm",
+ REPO_ROOT / "runners/launch_h100-cr.sh",
+ REPO_ROOT / "runners/launch_mi325x-tw.sh",
+ )
+
+ for path in paths:
+ content = path.read_text()
+ assert "SWEBENCH_USE_MODAL" in content, path
+ assert "MODAL_TOKEN_ID" in content, path
+ assert "MODAL_TOKEN_SECRET" in content, path
+ assert "IS_AGENTIC" in content, path
+ assert "SCENARIO_TYPE" in content, path
+ if path.name == "job.slurm" and "llm-d" in path.parts:
+ assert "-e MODAL_TOKEN_ID \\" in content
+ assert "-e MODAL_TOKEN_SECRET \\" in content
+ assert "-e MODAL_TOKEN_ID=" not in content
+ assert "-e MODAL_TOKEN_SECRET=" not in content
diff --git a/utils/collect_eval_results.py b/utils/collect_eval_results.py
index 667e60bc6f..7792a983c1 100644
--- a/utils/collect_eval_results.py
+++ b/utils/collect_eval_results.py
@@ -1,9 +1,12 @@
#!/usr/bin/env python3
-import sys
import json
+import math
import re
+import sys
+from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
+
from tabulate import tabulate
MODEL = "Model"
@@ -32,6 +35,7 @@
SPEC_DECODING = "Spec Decode"
CONC_SUFFIX_RE = re.compile(r"_conc(\d+)(?:_\d+)?\.json$")
+EVAL_RESULT_FORMAT = "inferencex-eval-v1"
def load_json(path: Path) -> Optional[Dict[str, Any]]:
@@ -71,10 +75,10 @@ def result_concurrency(path: Path) -> Optional[int]:
def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]:
- """Return lm-eval result JSONs from one artifact directory.
+ """Return the latest collector-compatible eval result JSONs.
- Legacy artifacts contribute their latest result file. Batched artifacts
- contribute the latest result file for each `_concN` suffix.
+ Result filenames contain sortable timestamps. Mtime remains a fallback for
+ legacy names, with the filename as a deterministic tie-breaker.
"""
immediate_jsons = set(d.glob('results*.json'))
immediate_jsons.update(
@@ -82,17 +86,45 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]:
)
lm_paths = []
+ def recency_key(path: Path) -> Tuple[int, str]:
+ match = re.search(
+ r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:\.\d+)?",
+ path.name,
+ )
+ if match:
+ try:
+ timestamp = match.group(0)
+ base, separator, fraction = timestamp.partition(".")
+ parsed = datetime.strptime(
+ base,
+ "%Y-%m-%dT%H-%M-%S",
+ ).replace(tzinfo=timezone.utc)
+ fractional_ns = (
+ int((fraction + "000000000")[:9])
+ if separator
+ else 0
+ )
+ order_ns = (
+ int(parsed.timestamp()) * 1_000_000_000
+ + fractional_ns
+ )
+ except ValueError:
+ order_ns = path.stat().st_mtime_ns
+ else:
+ order_ns = path.stat().st_mtime_ns
+ return order_ns, path.name
+
for p in immediate_jsons:
data = load_json(p)
if not isinstance(data, dict):
continue
- if 'lm_eval_version' in data:
+ if data.get('result_format') == EVAL_RESULT_FORMAT or 'lm_eval_version' in data:
lm_paths.append(p)
if not lm_paths:
return []
if not batched:
- return [max(lm_paths, key=lambda path: path.stat().st_mtime)]
+ return [max(lm_paths, key=recency_key)]
latest_by_conc: Dict[int, Path] = {}
for path in lm_paths:
@@ -100,15 +132,28 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]:
if conc is None:
continue
current = latest_by_conc.get(conc)
- if current is None or path.stat().st_mtime > current.stat().st_mtime:
+ if current is None or recency_key(path) > recency_key(current):
latest_by_conc[conc] = path
return [latest_by_conc[conc] for conc in sorted(latest_by_conc)]
-def detect_eval_jsons(d: Path) -> Tuple[Optional[Path], Optional[Path]]:
- """Return the latest legacy lm-eval JSON and deprecated second slot."""
- lm_paths = detect_lm_eval_jsons(d)
- return (lm_paths[0] if lm_paths else None), None
+def has_invalid_effective_count(data: Dict[str, Any], task: str) -> bool:
+ """Return whether a task has an explicitly invalid effective count."""
+ if 'n-samples' not in data:
+ return False
+ sample_counts = data['n-samples']
+ if not isinstance(sample_counts, dict) or task not in sample_counts:
+ return True
+ task_samples = sample_counts[task]
+ if not isinstance(task_samples, dict) or 'effective' not in task_samples:
+ return True
+ effective = task_samples['effective']
+ return (
+ isinstance(effective, bool)
+ or not isinstance(effective, (int, float))
+ or not math.isfinite(effective)
+ or effective <= 0
+ )
def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]:
@@ -124,16 +169,68 @@ def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]:
"""
data = load_json(json_path) or {}
results = data.get('results', {})
- configs = data.get('configs', {})
+ raw_configs = data.get('configs', {})
+ configs = raw_configs if isinstance(raw_configs, dict) else {}
- if not results:
+ if not isinstance(results, dict) or not results:
return []
extracted = []
for task in results.keys():
task_results = results[task]
- task_config = configs.get(task, {})
+ raw_task_config = configs.get(task, {})
+ task_config = (
+ raw_task_config if isinstance(raw_task_config, dict) else {}
+ )
+ raw_metadata = task_config.get('metadata', {})
+ metadata = raw_metadata if isinstance(raw_metadata, dict) else {}
+ model = data.get('model_name') or metadata.get('model')
+
+ sample_counts = data.get('n-samples')
+ raw_task_samples = (
+ sample_counts.get(task)
+ if isinstance(sample_counts, dict)
+ else None
+ )
+ n_eff = (
+ raw_task_samples.get('effective')
+ if isinstance(raw_task_samples, dict)
+ else None
+ )
+ invalid_effective_count = has_invalid_effective_count(data, task)
+ integration_error = data.get('integration_error')
+ if integration_error is None and invalid_effective_count:
+ integration_error = {
+ 'type': 'InvalidEffectiveSampleCount',
+ 'message': f'invalid effective sample count: {n_eff!r}',
+ }
+ if integration_error is None and not isinstance(task_results, dict):
+ integration_error = {
+ 'type': 'InvalidTaskResults',
+ 'message': f'invalid task results for {task!r}',
+ }
+ if integration_error is not None:
+ if not isinstance(integration_error, dict):
+ integration_error = {
+ 'type': 'IntegrationError',
+ 'message': str(integration_error),
+ }
+ extracted.append({
+ 'task': task,
+ 'strict': None,
+ 'strict_se': None,
+ 'flex': None,
+ 'flex_se': None,
+ 'accuracy': None,
+ 'accuracy_se': None,
+ 'n_eff': 0,
+ 'model': model,
+ 'source': str(json_path),
+ 'infrastructure_success': False,
+ 'integration_error': integration_error,
+ })
+ continue
# Base metric: from config's metric_list
metric_list = task_config.get('metric_list', [])
@@ -168,18 +265,14 @@ def get_val_se(filter_name: str) -> Tuple[Optional[float], Optional[float]]:
# SWE-bench uses resolved rate as its primary score.
if 'strict' in fname or 'resolved' in fname:
strict_val, strict_se = get_val_se(fname)
+ elif base_metric == 'acc' and fname == 'none':
+ accuracy_val, accuracy_se = get_val_se(fname)
elif 'flex' in fname or 'extract' in fname:
flex_val, flex_se = get_val_se(fname)
# N-samples (effective count)
n_eff = data.get('n-samples', {}).get(task, {}).get('effective')
- # Model name
- model = (
- data.get('model_name')
- or task_config.get('metadata', {}).get('model')
- )
-
extracted.append({
'task': task,
'strict': strict_val,
@@ -190,7 +283,9 @@ def get_val_se(filter_name: str) -> Tuple[Optional[float], Optional[float]]:
'accuracy_se': accuracy_se,
'n_eff': n_eff,
'model': model,
- 'source': str(json_path)
+ 'source': str(json_path),
+ 'infrastructure_success': True,
+ 'integration_error': None,
})
return extracted
@@ -282,8 +377,13 @@ def build_row(meta: Dict[str, Any], m: Dict[str, Any]) -> Dict[str, Any]:
'em_flexible_se': m.get('flex_se'),
'n_eff': m.get('n_eff'),
'source': m.get('source'),
+ 'infrastructure_success': m.get('infrastructure_success', True),
+ 'integration_error': m.get('integration_error'),
}
+ if 'eval_suite' in meta:
+ row['eval_suite'] = meta['eval_suite']
+
# Add universal score field (primary metric for unified comparison)
if m.get('strict') is not None:
row['score'] = m.get('strict')
@@ -293,6 +393,10 @@ def build_row(meta: Dict[str, Any], m: Dict[str, Any]) -> Dict[str, Any]:
row['score'] = m.get('accuracy')
row['score_name'] = 'accuracy'
row['score_se'] = m.get('accuracy_se')
+ elif m.get('flex') is not None:
+ row['score'] = m.get('flex')
+ row['score_name'] = 'em_flexible'
+ row['score_se'] = m.get('flex_se')
else:
row['score'] = None
row['score_name'] = None
@@ -326,6 +430,42 @@ def collect_eval_rows(root: Path) -> List[Dict[str, Any]]:
metrics_list = extract_lm_metrics(lm_path)
for metrics in metrics_list:
+ if metrics.get('infrastructure_success') is False:
+ rows.append(build_row(row_meta, metrics))
+ continue
+ primary_score = next(
+ (
+ metrics.get(name)
+ for name in ('strict', 'accuracy', 'flex')
+ if metrics.get(name) is not None
+ ),
+ None,
+ )
+ invalid_primary_score = (
+ isinstance(primary_score, bool)
+ or not isinstance(primary_score, (int, float))
+ or not math.isfinite(primary_score)
+ or not 0.0 <= primary_score <= 1.0
+ )
+ if invalid_primary_score:
+ failed_metrics = {
+ **metrics,
+ 'strict': None,
+ 'strict_se': None,
+ 'accuracy': None,
+ 'accuracy_se': None,
+ 'flex': None,
+ 'flex_se': None,
+ 'infrastructure_success': False,
+ 'integration_error': {
+ 'type': 'InvalidPrimaryScore',
+ 'message': (
+ f'invalid primary score: {primary_score!r}'
+ ),
+ },
+ }
+ rows.append(build_row(row_meta, failed_metrics))
+ continue
rows.append(build_row(row_meta, metrics))
return rows
@@ -404,7 +544,7 @@ def main():
f"{pct(r['score'])}{se(r['score_se'])}",
f"{pct(r['em_strict'])}{se(r['em_strict_se'])}",
f"{pct(r['em_flexible'])}{se(r['em_flexible_se'])}",
- r['n_eff'] or '',
+ r['n_eff'] if r['n_eff'] is not None else '',
r['model'],
]
for r in single_node_rows
@@ -442,7 +582,7 @@ def main():
f"{pct(r['score'])}{se(r['score_se'])}",
f"{pct(r['em_strict'])}{se(r['em_strict_se'])}",
f"{pct(r['em_flexible'])}{se(r['em_flexible_se'])}",
- r['n_eff'] or '',
+ r['n_eff'] if r['n_eff'] is not None else '',
r['model'],
]
for r in multinode_rows
diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md
index 0a7bf213df..1bd2a69344 100644
--- a/utils/evals/EVALS.md
+++ b/utils/evals/EVALS.md
@@ -7,25 +7,32 @@ from throughput. Selection lives in `mark_eval_entries()` in
## Selection
-- **Single-node:** 8k1k only, at the highest and median concurrency for every model,
- runner, framework, precision, TP, and decoding configuration.
-- **Multi-node:** 8k1k only, with one job per parallelism topology at its highest
- eligible concurrency. Rows differing only by concurrency share a topology.
-- **Agentic (SWE-bench), single-node:** highest-conc entry per (model,
- runner, framework, precision) group.
-- **Agentic (SWE-bench), multi-node:** same policy as multi-node fixed-seq-len
- above (highest eligible conc per parallelism topology), since SWE-bench
- doesn't support batched concurrencies the way lm-eval does.
+- **Fixed-sequence, single-node:** 8k1k only, at the highest and median
+ concurrency for every model, runner, framework, precision, TP, and decoding
+ configuration.
+- **Fixed-sequence, multi-node:** 8k1k only, with one job per parallelism
+ topology at its highest eligible concurrency. Rows differing only by
+ concurrency share a topology.
+- **Kimi K3 agentic:** every generated point automatically runs
+ `kimi-vendor` with `kimi_tool_call_schema`.
+- **MiniMax M3 agentic:** every generated point automatically runs
+ `minimax-vendor` with `minimax_m3_smoke`.
+- **Other agentic models (GSM8K):** opt-in through `--evals-only` or
+ `--all-evals`, at the highest concurrency per deployment group.
+- **BFCL:** explicit only. No automatic model mapping selects BFCL.
Generator eval modes:
-- Default: throughput plus the selected eval subset.
-- `--no-evals`: throughput only.
+- Default: throughput plus the fixed-sequence subset and every automatically
+ selected Kimi K3 or MiniMax M3 vendor eval.
+- `--no-evals`: throughput only, including no automatic vendor evals.
- `--evals-only`: selected evals only.
-- `--all-evals`: every fixed-sequence eval only. This is equivalent to
- `--evals-only --all-evals`. Multi-node topologies run all `conc-list` values
- sequentially on one engine. Agentic-coding configs are included and run
- GSM8K (they are excluded only from the default, non-eval sweep).
+- `--all-evals`: every eligible fixed-sequence and agentic eval. This is
+ equivalent to `--evals-only --all-evals`. Multi-node fixed-sequence
+ topologies run all `conc-list` values sequentially on one engine.
+- `--trim-conc`: after eval selection, retain the minimum concurrency for each
+ single-node or multi-node deployment shape and move that shape's selected eval
+ to the retained row. This is the deployment smoke mode, not a throughput sweep.
Changelog entries use `evals-only: true` and `all-evals: true`. The `all-evals`
setting implies eval-only there. On PRs, the same names are modifier labels:
@@ -35,6 +42,57 @@ suppresses it. Modifier runs cannot be reused.
Deduplication is scenario-aware: fixed-sequence coverage does not suppress
agentic coverage, and `all-evals` wins over default eval coverage.
+### Tool-use support contract
+
+The tool-use adapters are backend-independent clients of the local
+OpenAI-compatible endpoint. The deployment-smoke target set contains every
+generated Kimi K3 and MiniMax M3 agentic configuration in the NVIDIA and AMD
+master configs, including their single-node and multi-node vLLM and
+Dynamo-vLLM recipes. A configuration is verified only when the current PR head
+launches its tool-aware endpoint, the matching vendor smoke and `bfcl_smoke`
+complete their expected sample counts, and the native and
+`inferencex-eval-v1` artifacts are collected without `integration_error`.
+
+Infrastructure support does not mean every model must pass every quality
+threshold. A completed result with a positive effective sample count can score
+below its threshold and fail the quality gate without being a deployment
+failure. Missing parser support, transport errors, timeouts, malformed output,
+missing samples, and missing artifacts are infrastructure failures.
+
+Generator coverage and static parser checks do not prove a live backend. Before
+claiming complete deployment support, run both smoke suites on every row from
+the matrices below at the current PR head. Run each full vendor or BFCL
+model-quality suite on at least one matching deployment; these longer suites do
+not need to repeat on every equivalent parser topology.
+
+Generate the complete deployment-smoke matrices with:
+
+```bash
+uv run --no-project --with pydantic --with pyyaml --python 3.12 \
+ python utils/matrix_logic/generate_sweep_configs.py full-sweep \
+ --config-files configs/nvidia-master.yaml configs/amd-master.yaml \
+ --model-prefix kimik3 \
+ --scenario-type agentic-coding \
+ --evals-only --all-evals --trim-conc
+
+uv run --no-project --with pydantic --with pyyaml --python 3.12 \
+ python utils/matrix_logic/generate_sweep_configs.py full-sweep \
+ --config-files configs/nvidia-master.yaml configs/amd-master.yaml \
+ --model-prefix minimaxm3 \
+ --scenario-type agentic-coding \
+ --evals-only --all-evals --trim-conc
+```
+
+Capacity-limited campaigns can split a `test-config` result with `--conc` and
+`--exp-names`. Each requested experiment name must match exactly one generated
+row, so a shard cannot silently include another deployment that shares the same
+configuration key and concurrency.
+
+Run each generated matrix with the matching vendor smoke and `bfcl_smoke`.
+The full Kimi, MiniMax, and BFCL suites use the same endpoint and artifact
+paths, but are diagnostic model-quality campaigns rather than a replacement
+for the per-topology deployment smoke.
+
### Artifact reuse
Default full sweeps may reuse their eval subset. Source coverage is
@@ -44,9 +102,335 @@ malformed metadata, duplicates, and raw/aggregate mismatches are not. See
[workflow reuse](../../.github/workflows/README.md#reusing-an-approved-pr-full-sweep).
## How?
-`run_eval` in `benchmarks/benchmark_lib.sh` runs EleutherAI/lm-evaluation-harness against the server's OpenAI-compatible endpoint. Concurrency is set via `EVAL_CONCURRENT_REQUESTS` env var (not a CLI flag). Results are collected by `utils/collect_eval_results.py` and published as a summary table.
-The default eval framework is [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) (`lm-eval`). Agentic eval-only matrix jobs inherit this default and therefore run the same GSM8K task as 8k1k. Explicit agentic runs can still select SWE-bench.
+`run_eval` in `benchmarks/benchmark_lib.sh` dispatches to the selected eval
+runner. `e2e-tests.yml` defaults `eval-framework` to `auto`, then reads the
+concrete framework and suite from each eval matrix row. Fixed-sequence and
+opted-in generic agentic evals use
+[lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness)
+(`lm-eval`) with GSM8K. Workflow inputs can explicitly override the
+matrix-selected framework or suite for manual diagnostics.
+
+The Kimi smoke runs automatically for every generated `kimik3` agentic point.
+The matrix selects `eval-framework: kimi-vendor` and
+`eval-suite: kimi_tool_call_schema`. To invoke the same smoke manually from the
+repository root after a server is ready:
+
+```bash
+source benchmarks/benchmark_lib.sh
+export EVAL_FRAMEWORK=kimi-vendor
+export EVAL_SUITE=kimi_tool_call_schema
+export EVAL_RESULT_DIR="$(mktemp -d /tmp/eval_out-XXXXXX)"
+run_eval --port "$PORT"
+append_lm_eval_summary
+python3 utils/evals/validate_scores.py
+```
+
+The framework selects a suite-specific subprocess adapter, while the suite
+selects a case set understood by that adapter. Each adapter owns its endpoint
+format, dependencies, native report, metrics, and integration-failure policy.
+Kimi, MiniMax, and BFCL use separate explicit `run_eval` cases rather than a
+shared request or report abstraction. Automatic selection chooses only the Kimi
+and MiniMax vendor cases; BFCL remains an explicit workflow override.
+Agentic eval jobs forward the matrix `spec-decoding` value, so MTP entries
+launch their existing `*_mtp.sh` server instead of silently falling back to STP.
+
+### Stock Kimi tool-call schema smoke
+
+The smoke runs the unmodified
+[MoonshotAI/Kimi-Vendor-Verifier](https://github.com/MoonshotAI/Kimi-Vendor-Verifier)
+at commit `3dad65a760a8867cda72f6dd8848d876a4e851b4`. Each run downloads and
+SHA256-verifies the fresh pinned GitHub source archive, then safely extracts
+only the upstream pytest configuration, tool-call schema tests, and bundled
+Walle cases. InferenceX does not install the verifier package or reimplement
+its request, streaming, or validation logic.
+
+System Python 3.12 or newer is preferred and used directly. On older images,
+the runner uses the existing system `pip` to install pinned `uv==0.11.33` under
+a temporary prefix, then provisions an isolated Python 3.12 virtual environment.
+The selected interpreter installs the minimal pinned verifier runtime
+(`httpx[http2]`, `openai`, `jsonschema`, and `pytest`) into a separate
+temporary package directory, then runs upstream
+`tests/tool_call_json_schema/test_tool_call_json_schema.py` with:
+
+- the local OpenAI-compatible endpoint, `EMPTY` API key, and served model name;
+- `--think-mode none` for other models, or `--think-mode opensource --thinking`
+ for `dsv4`, plus `--selection object --max-cases 1 --max-tokens 2048`;
+- the bundled Walle case directory and `--tool-json-report`.
+
+The temporary Python runtime, package directory, and verifier checkout are
+removed after both successful and failed runs.
+
+The selection is `TestAdditionalProperties:1`, parametrized upstream in
+non-streaming and streaming modes. Each mode runs once through the unchanged
+upstream pytest harness. The unchanged native report remains one final outcome
+per mode. It is uploaded as `kimi_vendor_report.json`, and
+`utils/evals/kimi_vendor_eval.py` projects those two outcomes into the existing
+eval result shape. Both outcomes are recorded with a `0.0`
+`kimi_tool_call_schema` threshold, so model quality remains diagnostic. Setup,
+timeout, and collection failures emit a zero-score result with error metadata.
+The adapter's 900-second global timeout bounds the entire upstream pytest
+process.
+
+This smoke validates one object-schema tool call. It does not cover tool choice,
+parallel calls, multi-turn execution, or general agent quality. Multi-value
+batched concurrency is unsupported. Multi-node aggregate jobs run the same
+two-case smoke against their OpenAI-compatible frontend. Eval-only launchers
+restore real block verification before submitting recipes that otherwise use
+synthetic acceptance for throughput.
+
+### Kimi full tool-call schema diagnostic
+
+`kimi_tool_call_schema_full` runs the same pinned upstream test module with
+`--selection all`. It evaluates all 204 selected Walle schema cases in
+non-streaming and streaming modes, for 408 reported outcomes. Eight pytest
+workers share a two-hour whole-suite timeout. The native report must declare
+the exact selected suite and line identities, contain both modes for every
+case, and reconcile all outcome counts before projection.
+
+Select it explicitly with `eval-framework: kimi-vendor` and
+`eval-suite: kimi_tool_call_schema_full`. Its threshold is `0.0`, so model
+quality is diagnostic while setup, timeout, malformed-report, and integration
+failures still fail through the standard zero-effective-sample error path. The
+full suite reuses the smoke's pinned checkout, stock invocation, result
+envelope, artifact staging, collector, and dashboard path.
+
+### MiniMax provider compatibility smoke
+
+The Phase 1 MiniMax smoke runs automatically for every generated `minimaxm3`
+agentic point. The matrix selects `eval-framework: minimax-vendor` and
+`eval-suite: minimax_m3_smoke`. To invoke the same smoke manually from the
+repository root against an already-ready server:
+
+```bash
+source benchmarks/benchmark_lib.sh
+export EVAL_FRAMEWORK=minimax-vendor
+export MODEL_NAME=""
+export EVAL_SUITE=minimax_m3_smoke
+export EVAL_RESULT_DIR="$(mktemp -d /tmp/eval_out-XXXXXX)"
+run_eval --port "$PORT"
+append_lm_eval_summary
+python3 utils/evals/validate_scores.py
+```
+
+`utils/evals/minimax_m3_smoke.json` is derived from
+[MiniMax-AI/MiniMax-Provider-Verifier](https://github.com/MiniMax-AI/MiniMax-Provider-Verifier)
+`sample.jsonl` at commit
+`c899f95e17bfc4a338ddd4cb1638279125885e55`. The vendored fixture retains
+the full upstream MIT copyright, permission, and warranty notice. It contains
+only upstream zero-based row 71, an `expected_tool_call: true` request
+exercising tool-call trigger and argument-schema validation.
+
+Each run downloads and hash-verifies the pinned upstream `verify.py`, complete
+`sample.jsonl`, and validator modules. InferenceX writes row 71 unchanged to a
+temporary JSONL input and invokes the stock verifier with its documented CLI.
+The invocation uses concurrency one, the stock 600-second request timeout and
+three-retry setting, and the documented `--extra-body` override
+`{\"temperature\":0,\"top_p\":1,\"max_tokens\":40960}`. A one-hour outer
+process deadline bounds the stock harness without changing its request,
+response, retry, or scoring code.
+
+`minimax_vendor_report.json` and `minimax_vendor_results.jsonl` are the
+unchanged stock summary and detailed result artifacts. The adapter additionally
+writes exactly one timestamped `results_minimax_vendor_*.json` compatibility
+artifact. Its `result_format` is `inferencex-eval-v1`, `eval_adapter` is
+`minimax-provider-verifier`, task is `minimax_m3_smoke`, and primary metric is
+`exact_match,strict-match`. A completed run records original and effective
+sample counts of one. Its score is the minimum of the stock verifier's
+tool-call match rate, tool-call schema accuracy, and one minus its
+error-only-reasoning rate. A successful request that emits no tool calls has
+zero schema accuracy, so it remains an effective model-quality result rather
+than an integration failure. The `minimax_m3_smoke` threshold is `0.0`, so its
+model quality remains diagnostic.
+
+Setup, transport, timeout, malformed native output, and collection failures
+emit a zero-effective-sample compatibility artifact with integration-error
+metadata. A complete stock result remains a model-quality outcome rather than
+an integration failure.
+
+This is a fixed single-case provider compatibility smoke, not the full
+102-case MiniMax Provider Verifier, BFCL, or a cross-model quality comparison.
+It does not estimate the upstream dataset's aggregate rates, stochastic
+pass-at-k behavior, streaming behavior, parallel-call behavior, multi-turn tool
+execution, language following, scenario key-order recall, or general agent
+quality.
+
+### MiniMax M3 full provider diagnostic
+
+`minimax_m3_full` is an explicit, non-gating expansion of the smoke to all 102
+rows in the pinned MiniMax Provider Verifier dataset:
+
+```bash
+source benchmarks/benchmark_lib.sh
+export EVAL_FRAMEWORK=minimax-vendor
+export EVAL_SUITE=minimax_m3_full
+export MODEL_NAME=''
+run_eval --port "$PORT"
+append_lm_eval_summary
+python3 utils/evals/validate_scores.py
+```
+
+The runner downloads only the eight source and validator files allowlisted in
+`utils/evals/minimax_m3_full_eval.py` at commit
+`c899f95e17bfc4a338ddd4cb1638279125885e55`, verifies each SHA256, and executes
+the pinned `verify.py` once. It uses five workers, a 600-second request timeout,
+three upstream retries, and a seven-hour whole-suite timeout. The workflow
+retains at least one hour for artifact staging, score validation, and cleanup.
+
+The native files are `minimax_vendor_report.json` and
+`minimax_vendor_results.jsonl`. The compatibility result publishes task
+`minimax_m3_full` with the native `tool_calls_match_rate`, requires exactly 102
+successful result rows, and rejects transport failures or inconsistent
+summaries. Its threshold is `0.0`, so this suite is diagnostic during the first
+rollout. Setup, transport, timeout, malformed-output, and integration failures
+still fail through the standard zero-effective-sample error path.
+
+### BFCL V4 deterministic tool-use smoke
+
+The BFCL smoke is opt-in for models served through an OpenAI-compatible
+chat-completions endpoint. Select `eval-framework: bfcl` and
+`eval-suite: bfcl_smoke` in `e2e-tests.yml`, or run it from the repository root
+against an already-ready server:
+
+```bash
+source benchmarks/benchmark_lib.sh
+export EVAL_FRAMEWORK=bfcl
+export MODEL_NAME=""
+export EVAL_SUITE=bfcl_smoke
+export EVAL_RESULT_DIR="$(mktemp -d /tmp/eval_out-XXXXXX)"
+run_eval --port "$PORT"
+append_lm_eval_summary
+python3 utils/evals/validate_scores.py
+```
+
+The validator reads BFCL's declared `acc` metric from the compatibility result,
+so workflows do not need a framework-specific metric override.
+
+The runtime pins
+[`bfcl-eval==2026.3.23`](https://pypi.org/project/bfcl-eval/2026.3.23/), built
+from Gorilla commit
+[`6ea57973c7a6097fd7c5915698c54c17c5b1b6c8`](https://github.com/ShishirPatil/gorilla/commit/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8).
+It downloads the exact
+[`bfcl_eval-2026.3.23-py3-none-any.whl`](https://files.pythonhosted.org/packages/ba/41/ed458527c770c50225b60bae3b0c3444b26804ee455fa2d8f187018d2cb2/bfcl_eval-2026.3.23-py3-none-any.whl)
+and verifies SHA256
+`3bb6dfa5f0c68ad403c9ec50b00db2bb3b4cc9b38ab1ff33f48fe30d853d3a0a`
+before installation. The integration follows the pinned
+[vLLM perf-eval BFCL runner](https://github.com/vllm-project/perf-eval/blob/7ecb11405df86b202f4c5cca322bd133052fee82/lib/run_bfcl.py),
+but uses a fixed four-case V4 partial evaluation:
+
+| BFCL category | Exact upstream case ID | Projected task |
+|---------------|------------------------|----------------|
+| `simple_python` | `simple_python_141` | `bfcl_simple_python` |
+| `multiple` | `multiple_38` | `bfcl_multiple` |
+| `parallel` | `parallel_1` | `bfcl_parallel` |
+| `irrelevance` | `irrelevance_0` | `bfcl_irrelevance` |
+
+The verified wheel and its undeclared `soundfile==0.13.1` import dependency are
+installed into a temporary Python 3.10-or-newer virtual environment with system
+site packages enabled so the image's existing Torch/Transformers stack can be
+reused; it never mutates the global Python environment. The temporary
+environment and BFCL project root are removed after
+the run. Once package installation finishes, evaluation is local-only: BFCL
+skips its server setup and uses only the already-running local API root,
+typically `http://127.0.0.1:$PORT/v1`. The OpenAI SDK appends
+`/chat/completions`; the adapter base URL is not the full endpoint. BFCL does
+not download a model or call a remote inference API.
+
+The smoke fixes temperature to `0` and uses four BFCL worker threads. Request
+construction, response interpretation, and retry behavior remain those of the
+pinned stock BFCL OpenAI-completions handler and OpenAI SDK. The adapter only
+registers the served model against that stock handler. A 900-second external
+process deadline bounds the smoke; each full suite uses its declared deadline.
+Dependency installation is separately bounded at 600 seconds. Dependency,
+setup, transport, timeout, and collection failures write
+zero-score artifacts with integration-error metadata and fail the runner
+nonzero. A completed evaluation exits independently of model quality; the
+workflow score-validation step applies the threshold afterward.
+
+The endpoint must implement OpenAI chat completions at `/v1/chat/completions`,
+accept `tools` and the tool-selection fields emitted by BFCL, and return the
+served model's OpenAI tool-call shape. In particular, assistant tool calls need
+function names and JSON-encoded `function.arguments`; the response must also
+support a normal no-tool answer for the irrelevance case. Starting a nominally
+OpenAI-compatible server is not sufficient if it cannot parse that model's
+native tool-call syntax.
+
+Configure the server's model-specific function-calling parser and, when the
+model's default template does not render tools correctly, its tool-aware chat
+template. For vLLM, automatic calls require `--enable-auto-tool-choice` plus
+`--tool-call-parser`, with `--chat-template` when needed. SGLang uses its
+corresponding `--tool-call-parser`; TensorRT-LLM uses `--tool_parser`, plus the
+matching reasoning-parser option when the model requires one. Parser names are
+engine-specific. Current common mappings are Kimi K3 (`kimi_k3`), MiniMax M3
+(`minimax_m3` in vLLM/TRT-LLM and `minimax-m3` in SGLang), and DeepSeek V4
+(`deepseek_v4` in vLLM/TRT-LLM and `deepseekv4` in SGLang). GLM-4.5 uses
+`glm45` in vLLM/SGLang, while GLM-4.7 uses `glm47`; Qwen3-Coder uses
+`qwen3_coder` in vLLM/SGLang, with `qwen3_xml` for vLLM's XML variant and
+`qwen3` for the corresponding TensorRT-LLM parser. The model recipe and
+installed engine version are authoritative; BFCL does not replace a missing or
+mismatched parser/chat template.
+
+`bfcl_report.json` is the native report. `results_bfcl.json` is the
+`inferencex-eval-v1` compatibility result consumed by the existing artifact
+upload, `append_lm_eval_summary`, collector, and score validator. It projects
+the four-case aggregate as task `bfcl_smoke` and the four one-case diagnostic
+tasks shown above. Every row uses lm-eval-compatible `acc,none` (plus
+`acc_stderr,none`); BFCL workflows therefore validate with metric prefix
+`acc,` rather than the default exact-match prefix.
+
+The `bfcl_smoke` and four `bfcl_` thresholds are `0.0`, so all five
+scores remain diagnostic. Dependency, endpoint, timeout, malformed-output,
+missing-sample, and integration failures still fail through the standard
+zero-effective-sample path. BFCL reuses the existing eval job, upload paths,
+aggregation, and validation instead of adding a parallel workflow or artifact
+route.
+
+#### BFCL V4 model-quality suites
+
+Two explicit BFCL suites extend the four-case endpoint smoke into broader
+model-quality diagnostics:
+
+| Suite | Selected BFCL V4 categories | Requests |
+|-------|-----------------------------|----------|
+| `bfcl_vllm_minimax_m3` | `simple_python` (400), `multiple` (200), `parallel` (200), `parallel_multiple` (200) | 1000 |
+| `bfcl_vllm_kimi` | The same 1000 single-turn cases plus 60 each from `multi_turn_base`, `multi_turn_miss_func`, `multi_turn_miss_param`, and `multi_turn_long_context` | 1240 |
+
+These are the model-specific non-live and multi-turn slices used by the pinned
+BFCL vLLM integration, not every BFCL V4 leaderboard category. They exclude
+the V4 agentic web-search and memory evaluations.
+
+Select these suites explicitly with `eval-framework: bfcl`; `bfcl_smoke`
+remains the framework default. Both suites use BFCL's OpenAI completions
+handler against the local endpoint rather than a hosted-provider handler. They
+fix temperature to `0.001` and retain the stock handler's request construction,
+response interpretation, and retry behavior. A transport-only subclass pins
+the OpenAI SDK to two retries and a 180-second per-attempt timeout. MiniMax uses
+eight worker threads and a two-hour whole-suite timeout. Kimi uses 16 threads,
+caps multi-turn cases at ten steps, and uses a four-hour whole-suite timeout.
+
+The adapter builds a deterministic run-ID map from the pinned BFCL dataset.
+Single-turn suites select every case in their named categories. The Kimi
+multi-turn selection sorts each leaf category and takes its first 60 cases.
+Although upstream BFCL evaluates these subsets with `partial_eval`, the
+adapter rejects missing, unexpected, or duplicate result IDs and score headers
+whose counts or accuracy do not reconcile with the selected corpus.
+
+The compatibility result publishes `bfcl_vllm_minimax_m3` or
+`bfcl_vllm_kimi` as the aggregate task and preserves per-category
+`bfcl_` tasks. Kimi also publishes a combined `bfcl_multi_turn`
+task. Full-suite thresholds are `0.0`; they are diagnostic until repeated
+hardware runs establish model, precision, and backend baselines. A completed
+zero-score run therefore passes threshold validation, while dependency,
+transport, timeout, malformed-output, and integration failures still fail.
+
+`bfcl_upstream_artifacts.tar.gz` preserves the pinned upstream result and
+failure-only score JSONL files, exact selected-ID map, file locks, provenance
+manifest, and Apache 2.0 license copy for debugging and attribution.
+The native `bfcl_report.json` includes the package version, wheel hash, source
+revision, integration revision, per-category score headers,
+case IDs, failure records, and sampling settings. The compatibility
+`results_bfcl.json` remains the only input to the normal InferenceX eval
+collector and dashboard path.
### Benchmark script flow
@@ -60,7 +444,7 @@ All benchmark scripts in `benchmarks/` follow one of two flows:
# 4. Run evals:
if [ "${RUN_EVAL}" = "true" ]; then
run_eval --framework lm-eval --port "$PORT"
- append_lm_eval_summary # Writes meta_env.json and moves artifacts
+ append_lm_eval_summary # Writes meta_env.json and stages artifacts
fi
# Eval-only mode (EVAL_ONLY=true):
@@ -77,14 +461,28 @@ Key eval functions in `benchmarks/benchmark_lib.sh`:
|----------|-------------|
| `run_eval` | Unified entrypoint - dispatches to framework-specific runner |
| `run_lm_eval` | Runs lm-eval harness against the OpenAI-compatible endpoint |
-| `append_lm_eval_summary` | Writes `meta_env.json` and moves eval artifacts to workspace |
+| `run_kimi_vendor_eval` | Selects and runs a pinned Kimi Vendor Verifier suite |
+| `run_minimax_vendor_eval` | Selects the pinned MiniMax smoke or full diagnostic |
+| `run_bfcl_eval` | Selects a pinned BFCL V4 smoke or model-quality suite |
+| `append_lm_eval_summary` | Writes `meta_env.json` and stages eval artifacts in the workspace |
| `_install_lm_eval_deps` | Installs lm-eval dependencies |
+| `_prepare_vendor_verifier_python` | Uses system Python 3.12+ or provisions an isolated pinned Python 3.12 runtime for provider verifiers |
+| `_prepare_kimi_vendor_runtime` | Installs the pinned verifier dependencies in an isolated temp path |
+| `_prepare_minimax_m3_full_runtime` | Downloads hash-verified stock MiniMax sources and installs their pinned dependencies for smoke and full suites |
+| `_prepare_bfcl_runtime` | Installs the verified BFCL wheel in a temporary virtual environment |
+| `_install_bfcl_eval_deps` | Downloads, verifies, and installs the pinned BFCL wheel |
+| `_prepare_kimi_vendor_verifier` | Downloads, hash-verifies, and safely extracts a fresh subset of the pinned source archive |
| `_patch_lm_eval` | Patches lm-eval for reasoning tokens and TRT compatibility |
| `compute_eval_context_length` | Computes eval context length (requested benchmark context, capped at model native max) |
| `get_native_max_context_length` | Extracts model's native max context length from HF config |
+`EVAL_FRAMEWORK` is the orchestration-level selection and takes precedence over
+legacy `--framework lm-eval` arguments embedded in fixed-sequence recipes.
+Without that environment variable, an explicit `--framework` argument takes
+precedence over the scenario default.
+
### Single-node
-In eval-only mode (`EVAL_ONLY=true`), the benchmark script computes `EVAL_MAX_MODEL_LEN` via `compute_eval_context_length`, starts the server with that context length, skips throughput, and runs lm-eval directly. Each framework wires that context differently (`--context-length` for SGLang, `--max_seq_len` for TRT-LLM).
+For default lm-eval jobs in eval-only mode (`EVAL_ONLY=true`), the benchmark script computes `EVAL_MAX_MODEL_LEN` via `compute_eval_context_length`, starts the server with that context length, skips throughput, and runs lm-eval. Each framework wires that context differently (`--context-length` for SGLang, `--max_seq_len` for TRT-LLM).
### Multi-node
Multi-node evals support two hardware paths:
@@ -92,13 +490,13 @@ Multi-node evals support two hardware paths:
**MI355X (AMD)** — `benchmarks/multi_node/amd_utils/server_sglang.sh`
- Skips throughput when `EVAL_ONLY=true`
- Fixed-seq-len: runs lm-eval via `run_eval --framework lm-eval` against the router on port 30000
-- Agentic-coding (disaggregated, `IS_AGENTIC=1`): runs SWE-bench via `run_eval --port 30000` (no
- `--framework` override, same auto-selection as single-node agentic eval-only). Since there's no
- single "TP" for a disaggregated topology, and the workflow spells a couple of metadata fields
- differently (`PREFILL_DP_ATTN`/`DECODE_DP_ATTN`) than `append_lm_eval_summary` expects
- (`PREFILL_DP_ATTENTION`/`DECODE_DP_ATTENTION`), the agentic branch bridges those before calling
- `run_eval`; `append_lm_eval_summary` itself runs automatically inside `run_eval()` (same
- `EVAL_ONLY=true && IS_AGENTIC` auto-staging as single-node), not as a separate call.
+- Agentic-coding (disaggregated, `IS_AGENTIC=1`): follows the same GSM8K/lm-eval path via
+ `run_eval --framework lm-eval`. Since there's no single "TP" for a disaggregated topology,
+ and the workflow spells a couple of metadata fields differently
+ (`PREFILL_DP_ATTN`/`DECODE_DP_ATTN`) than `append_lm_eval_summary` expects
+ (`PREFILL_DP_ATTENTION`/`DECODE_DP_ATTENTION`), the agentic branch bridges those before
+ calling `run_eval`; `append_lm_eval_summary` itself runs automatically inside `run_eval()`
+ (same `EVAL_ONLY=true && IS_AGENTIC` auto-staging as single-node), not as a separate call.
- Concurrency uses workflow-provided `EVAL_CONC` when set, otherwise falls back to max of `BENCH_MAX_CONCURRENCY` (x-separated values)
- Eval artifacts copied to `/run_logs/slurm_job-*/eval_results/`
- `runners/launch_mi355x-amds.sh` skips benchmark result collection when `EVAL_ONLY=true` and uses `find` to locate eval results
@@ -106,10 +504,10 @@ Multi-node evals support two hardware paths:
**NVIDIA Slurm multi-node (GB200, GB300, B200, B300, H100, H200)** runs through [srt-slurm](https://github.com/NVIDIA/srt-slurm) on the `sa-submission-q2-2026` branch.
- `do_sweep.py` skips the benchmark stage when `EVAL_ONLY=true`, runs `_run_post_eval()` directly
- In eval-only mode, uses the full `wait_for_model()` health check (same as benchmark stage) since the benchmark health check was skipped
-- `lm-eval` runner (`benchmarks/lm_eval.py`) is invoked by `do_sweep.py` as a post/eval-only step and sources InferenceX's `benchmark_lib.sh` from the mounted workspace (`/infmax-workspace`)
+- The registered srt-slurm `lm-eval` post-runner sources InferenceX's `benchmark_lib.sh` from the mounted workspace (`/infmax-workspace`). Kimi-selected launches patch that hook to use generic `run_eval` dispatch while preserving lm-eval as the default.
- Eval artifacts written to `/logs/eval_results/` inside the container, collected by launch scripts
- NVIDIA Slurm launch scripts always collect server logs for debugging but skip benchmark result collection when `EVAL_ONLY=true`
-- Env vars threaded: `RUN_EVAL`, `EVAL_ONLY`, `IS_MULTINODE`, `FRAMEWORK`, `PRECISION`, `MODEL_PREFIX`, `RUNNER_TYPE`, `RESULT_FILENAME`, `SPEC_DECODING`, `ISL`, `OSL`, `PREFILL_TP/EP/NUM_WORKERS/DP_ATTN`, `DECODE_TP/EP/NUM_WORKERS/DP_ATTN`, `MODEL_NAME`, `EVAL_CONC`
+- Env vars threaded: `RUN_EVAL`, `EVAL_ONLY`, `EVAL_FRAMEWORK`, `EVAL_SUITE`, `IS_MULTINODE`, `FRAMEWORK`, `PRECISION`, `MODEL_PREFIX`, `RUNNER_TYPE`, `RESULT_FILENAME`, `SPEC_DECODING`, `ISL`, `OSL`, `PREFILL_TP/EP/NUM_WORKERS/DP_ATTN`, `DECODE_TP/EP/NUM_WORKERS/DP_ATTN`, `MODEL_NAME`, `EVAL_CONC`
For multi-node `all-evals`, `EVAL_CONC` is a space-separated list. When it contains multiple values, `run_eval` runs those concurrency points sequentially against the same live engine, stages each result with a `_concN` filename suffix, and records expected/completed/failed points in `meta_env.json`.
@@ -143,7 +541,9 @@ gh run download --repo SemiAnalysisAI/InferenceX -n eval_results_all -D
# View eval summary
cat ./evals/agg_eval_all.json | jq -r '
- .[] | [.hw, .framework, .precision, .tp, .conc, .task, (.score * 100 | round | . / 100)]
+ .[] | [.hw, .framework, .precision, .tp, .conc, .task,
+ (if .infrastructure_success then ((.score * 100 | round) / 100)
+ else .integration_error.type end)]
| @tsv' | column -t
# Filter to specific hardware
@@ -159,6 +559,16 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]'
| `em_flexible` | Flexible extraction (looser number matching) |
| `n_eff` | Number of samples evaluated |
| `task` | Eval task name (e.g., `gsm8k`) |
+| `eval_suite` | Explicit suite identity used for collection and artifact reuse |
+| `infrastructure_success` | `false` when setup, transport, timeout, sample-count, or score validation failed |
+| `integration_error` | Structured infrastructure failure type and message, otherwise `null` |
+
+Collection retains the latest attempt for each artifact or batched concurrency.
+Raw compatibility artifacts encode infrastructure failures with `score: 0`,
+`n_eff: 0`, and `integration_error`. Aggregation preserves the failure row but
+sets `score: null` and `infrastructure_success: false`, so dashboards cannot
+mistake an endpoint failure for measured model quality. An older successful
+attempt cannot replace a newer failed retry.
### Environment variables
@@ -166,7 +576,8 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]'
|----------|---------|-------------|
| `RUN_EVAL` | `false` | Enable eval after throughput benchmark |
| `EVAL_ONLY` | `false` | Skip throughput, only run evals (set by workflow) |
-| `EVAL_FRAMEWORK` | `lm-eval` | Eval framework to use |
+| `EVAL_FRAMEWORK` | Workflow: `auto`; benchmark runner: `lm-eval` | Eval runner (`lm-eval`, `swebench`, `kimi-vendor`, `minimax-vendor`, or `bfcl`). `auto` resolves from matrix metadata before reusable workflow dispatch |
+| `EVAL_SUITE` | Matrix-selected for automatic vendor evals; otherwise basename of `EVAL_TASKS_DIR` or `gsm8k` | Provider suite selector and artifact identity. Explicit workflow overrides remain supported |
| `EVAL_TASKS_DIR` | `utils/evals/gsm8k.yaml` | Path to lm-eval task YAML |
| `EVAL_RESULT_DIR` | `/tmp/eval_out-*` | Output directory for eval results |
| `EVAL_MAX_MODEL_LEN` | `16384` | Max context for eval (set by `compute_eval_context_length`) |
@@ -182,6 +593,15 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]'
2. Set `EVAL_TASKS_DIR=utils/evals/.yaml` when running benchmarks.
3. Update `utils/collect_eval_results.py` if new metrics need extraction.
+### Adding a provider verifier
+
+1. Add a provider-specific adapter under `utils/evals/`.
+2. Add an explicit framework case in `run_eval`; keep suite-specific policy in
+ that adapter's shell runner.
+3. Install dependencies in a provider-specific isolated runtime.
+4. Emit `result_format: inferencex-eval-v1`, preserve the native report in an
+ explicitly uploaded suite-specific path, set `EVAL_SUITE`, and add a threshold.
+
### Runtime patches (`utils/evals/patches/`)
The benchmark helpers invoke these standalone scripts against pinned dependencies.
diff --git a/utils/evals/bfcl_adapter.py b/utils/evals/bfcl_adapter.py
new file mode 100644
index 0000000000..e27574ce24
--- /dev/null
+++ b/utils/evals/bfcl_adapter.py
@@ -0,0 +1,1027 @@
+#!/usr/bin/env python3
+"""Run pinned BFCL V4 OpenAI chat-completions suites."""
+
+from __future__ import annotations
+
+import argparse
+import inspect
+import json
+import math
+import os
+import sys
+import urllib.parse
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+from typing import Any, Protocol
+
+TASK_NAME = "bfcl_smoke"
+NATIVE_REPORT_FILENAME = "bfcl_report.json"
+COMPATIBILITY_FILENAME = "results_bfcl.json"
+COMPATIBILITY_GLOB = "results_bfcl*.json"
+RESULT_FORMAT = "inferencex-eval-v1"
+ADAPTER_NAME = "bfcl-v4-openai-completions"
+DEFAULT_NUM_THREADS = 4
+REQUIRED_SCORE = 0.0
+FULL_SUITE_REQUEST_TIMEOUT_SECONDS = 180
+FULL_SUITE_REQUEST_MAX_RETRIES = 2
+KIMI_MAXIMUM_STEP_LIMIT = 10
+
+BFCL_PACKAGE = "bfcl-eval"
+BFCL_PACKAGE_VERSION = "2026.3.23"
+BFCL_WHEEL_SHA256 = "3bb6dfa5f0c68ad403c9ec50b00db2bb3b4cc9b38ab1ff33f48fe30d853d3a0a"
+UPSTREAM_REPOSITORY = "https://github.com/ShishirPatil/gorilla"
+UPSTREAM_SOURCE = "https://pypi.org/project/bfcl-eval/2026.3.23/"
+UPSTREAM_REF = f"{BFCL_PACKAGE}=={BFCL_PACKAGE_VERSION}"
+SOURCE_REVISION = "6ea57973c7a6097fd7c5915698c54c17c5b1b6c8"
+VLLM_INTEGRATION_REF = "7ecb11405df86b202f4c5cca322bd133052fee82"
+UPSTREAM_LICENSE = "Apache-2.0"
+UPSTREAM_LICENSE_URL = f"{UPSTREAM_REPOSITORY}/blob/{SOURCE_REVISION}/LICENSE"
+UPSTREAM_LICENSE_FILENAME = "BFCL_LICENSE.apache-2.0.txt"
+UPSTREAM_ATTRIBUTION_FILENAME = "BFCL_ATTRIBUTION.json"
+
+# Dict insertion order is intentional: smoke reports and the upstream run-ID
+# file remain byte-for-byte stable.
+SMOKE_CASE_IDS: Mapping[str, tuple[str, ...]] = MappingProxyType(
+ {
+ "simple_python": ("simple_python_141",),
+ "multiple": ("multiple_38",),
+ "parallel": ("parallel_1",),
+ "irrelevance": ("irrelevance_0",),
+ }
+)
+EXPECTED_SAMPLE_COUNT = sum(len(case_ids) for case_ids in SMOKE_CASE_IDS.values())
+
+
+@dataclass(frozen=True)
+class SuiteSpec:
+ name: str
+ generation_categories: tuple[str, ...]
+ expected_leaf_counts: tuple[tuple[str, int], ...]
+ temperature: float
+ default_num_threads: int
+ threshold: float
+ category_limits: tuple[tuple[str, int], ...] = ()
+
+ @property
+ def leaf_categories(self) -> tuple[str, ...]:
+ return tuple(category for category, _ in self.expected_leaf_counts)
+
+ @property
+ def expected_sample_count(self) -> int:
+ return sum(count for _, count in self.expected_leaf_counts)
+
+ def projected_task(self, category: str) -> str:
+ if self.name == TASK_NAME:
+ return f"bfcl_{category}"
+ return f"{self.name}_{category}"
+
+
+SMOKE_SUITE = SuiteSpec(
+ name=TASK_NAME,
+ generation_categories=tuple(SMOKE_CASE_IDS),
+ expected_leaf_counts=tuple(
+ (category, len(case_ids)) for category, case_ids in SMOKE_CASE_IDS.items()
+ ),
+ temperature=0.0,
+ default_num_threads=DEFAULT_NUM_THREADS,
+ threshold=REQUIRED_SCORE,
+)
+MINIMAX_SUITE = SuiteSpec(
+ name="bfcl_vllm_minimax_m3",
+ generation_categories=(
+ "simple_python",
+ "multiple",
+ "parallel",
+ "parallel_multiple",
+ ),
+ expected_leaf_counts=(
+ ("simple_python", 400),
+ ("multiple", 200),
+ ("parallel", 200),
+ ("parallel_multiple", 200),
+ ),
+ temperature=0.001,
+ default_num_threads=8,
+ threshold=0.0,
+)
+KIMI_SUITE = SuiteSpec(
+ name="bfcl_vllm_kimi",
+ generation_categories=(
+ "simple_python",
+ "multiple",
+ "parallel",
+ "parallel_multiple",
+ "multi_turn",
+ ),
+ expected_leaf_counts=(
+ ("simple_python", 400),
+ ("multiple", 200),
+ ("parallel", 200),
+ ("parallel_multiple", 200),
+ ("multi_turn_base", 60),
+ ("multi_turn_miss_func", 60),
+ ("multi_turn_miss_param", 60),
+ ("multi_turn_long_context", 60),
+ ),
+ temperature=0.001,
+ default_num_threads=16,
+ threshold=0.0,
+ category_limits=(("multi_turn", 240),),
+)
+SUITE_SPECS: Mapping[str, SuiteSpec] = MappingProxyType(
+ {
+ suite.name: suite
+ for suite in (
+ SMOKE_SUITE,
+ MINIMAX_SUITE,
+ KIMI_SUITE,
+ )
+ }
+)
+
+
+class UpstreamRunner(Protocol):
+ """Injectable boundary around the optional BFCL installation."""
+
+ def __call__(
+ self,
+ *,
+ model: str,
+ project_root: Path,
+ base_url: str,
+ api_key: str,
+ num_threads: int,
+ ) -> None: ...
+
+
+@dataclass(frozen=True)
+class CategoryScore:
+ category: str
+ case_ids: tuple[str, ...]
+ score_file: str
+ header: Mapping[str, Any]
+ records: tuple[Mapping[str, Any], ...]
+ accuracy: float
+ correct_count: int
+ total_count: int
+
+ def as_dict(self) -> dict[str, Any]:
+ failure_ids = {record["id"] for record in self.records}
+ return {
+ "category": self.category,
+ "case_ids": list(self.case_ids),
+ "score_file": self.score_file,
+ "score_header": dict(self.header),
+ "score_records": [dict(record) for record in self.records],
+ "case_scores": [
+ {
+ "id": case_id,
+ "score": 0.0 if case_id in failure_ids else 1.0,
+ "correct": case_id not in failure_ids,
+ }
+ for case_id in self.case_ids
+ ],
+ }
+
+
+def _nonempty_string(value: str) -> str:
+ if not value.strip():
+ raise argparse.ArgumentTypeError("must be a non-empty string")
+ return value.strip()
+
+
+def _absolute_http_url(value: str) -> str:
+ normalized = _nonempty_string(value).rstrip("/")
+ parsed = urllib.parse.urlsplit(normalized)
+ if (
+ parsed.scheme not in {"http", "https"}
+ or not parsed.netloc
+ or parsed.query
+ or parsed.fragment
+ ):
+ raise argparse.ArgumentTypeError(
+ "must be an absolute HTTP(S) URL without query or fragment"
+ )
+ if parsed.path.rstrip("/").endswith("/chat/completions"):
+ raise argparse.ArgumentTypeError(
+ "must be an API root URL; the OpenAI client appends /chat/completions"
+ )
+ return normalized
+
+
+def _positive_int(value: str) -> int:
+ try:
+ parsed = int(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("must be a positive integer") from exc
+ if parsed <= 0:
+ raise argparse.ArgumentTypeError("must be a positive integer")
+ return parsed
+
+
+def _source_details(
+ suite: SuiteSpec, case_ids_by_category: Mapping[str, tuple[str, ...]]
+) -> dict[str, Any]:
+ return {
+ "url": UPSTREAM_SOURCE,
+ "repository": UPSTREAM_REPOSITORY,
+ "ref": UPSTREAM_REF,
+ "package": BFCL_PACKAGE,
+ "package_version": BFCL_PACKAGE_VERSION,
+ "wheel_sha256": BFCL_WHEEL_SHA256,
+ "source_revision": SOURCE_REVISION,
+ "vllm_integration_ref": VLLM_INTEGRATION_REF,
+ "case_ids": {
+ category: list(case_ids)
+ for category, case_ids in case_ids_by_category.items()
+ },
+ }
+
+
+def _error_dict(error: BaseException) -> dict[str, str]:
+ return {"type": type(error).__name__, "message": str(error)}
+
+
+def _expected_category_details(
+ case_ids_by_category: Mapping[str, tuple[str, ...]],
+) -> list[dict[str, Any]]:
+ return [
+ {
+ "category": category,
+ "case_ids": list(case_ids),
+ "score_file": None,
+ "score_header": None,
+ "score_records": [],
+ "case_scores": [
+ {"id": case_id, "score": 0.0, "correct": False} for case_id in case_ids
+ ],
+ }
+ for category, case_ids in case_ids_by_category.items()
+ ]
+
+
+def _diagnostics(
+ suite: SuiteSpec,
+ case_ids_by_category: Mapping[str, tuple[str, ...]],
+ scores: Sequence[CategoryScore] | None = None,
+) -> dict[str, Any]:
+ return {
+ "source": _source_details(suite, case_ids_by_category),
+ "categories": (
+ [score.as_dict() for score in scores]
+ if scores is not None
+ else _expected_category_details(case_ids_by_category)
+ ),
+ }
+
+
+def _native_report(
+ *,
+ suite: SuiteSpec,
+ case_ids_by_category: Mapping[str, tuple[str, ...]],
+ model: str,
+ base_url: str | None,
+ num_threads: int,
+ scores: Sequence[CategoryScore] | None,
+ integration_error: BaseException | None = None,
+) -> dict[str, Any]:
+ correct_count = sum(score.correct_count for score in scores or ())
+ total_count = sum(score.total_count for score in scores or ())
+ accuracy = correct_count / total_count if total_count else 0.0
+ report: dict[str, Any] = {
+ "verifier": ADAPTER_NAME,
+ "task": suite.name,
+ "model": model,
+ "endpoint": base_url,
+ "completed": integration_error is None,
+ "passed": integration_error is None and accuracy >= suite.threshold,
+ "threshold": suite.threshold,
+ "sampling": {
+ "temperature": suite.temperature,
+ "num_threads": num_threads,
+ },
+ "summary": {
+ "accuracy": accuracy,
+ "correct_count": correct_count,
+ "total_count": total_count,
+ "expected_count": suite.expected_sample_count,
+ },
+ "bfcl": _diagnostics(suite, case_ids_by_category, scores),
+ }
+ if integration_error is not None:
+ report["integration_error"] = _error_dict(integration_error)
+ return report
+
+
+def _compatibility_result(
+ *,
+ suite: SuiteSpec,
+ case_ids_by_category: Mapping[str, tuple[str, ...]],
+ model: str,
+ scores: Sequence[CategoryScore] | None,
+ integration_error: BaseException | None = None,
+) -> dict[str, Any]:
+ score_by_category = {score.category: score for score in scores or ()}
+ total_count = sum(score.total_count for score in scores or ())
+ correct_count = sum(score.correct_count for score in scores or ())
+ accuracy = correct_count / total_count if total_count else 0.0
+ task_scores = {suite.name: accuracy}
+ task_samples = {
+ suite.name: {
+ "original": suite.expected_sample_count,
+ "effective": total_count,
+ }
+ }
+ for category, expected_count in suite.expected_leaf_counts:
+ category_score = score_by_category.get(category)
+ task_name = suite.projected_task(category)
+ task_scores[task_name] = (
+ category_score.accuracy if category_score is not None else 0.0
+ )
+ task_samples[task_name] = {
+ "original": expected_count,
+ "effective": (
+ category_score.total_count if category_score is not None else 0
+ ),
+ }
+
+ if suite is KIMI_SUITE:
+ multi_turn_categories = suite.leaf_categories[-4:]
+ multi_turn_scores = [
+ score_by_category[category]
+ for category in multi_turn_categories
+ if category in score_by_category
+ ]
+ multi_turn_total = sum(score.total_count for score in multi_turn_scores)
+ multi_turn_correct = sum(score.correct_count for score in multi_turn_scores)
+ multi_turn_accuracy = (
+ multi_turn_correct / multi_turn_total if multi_turn_total else 0.0
+ )
+ aggregate_task = suite.projected_task("multi_turn")
+ task_scores[aggregate_task] = multi_turn_accuracy
+ task_samples[aggregate_task] = {
+ "original": 240,
+ "effective": multi_turn_total,
+ }
+
+ results = {
+ task_name: {
+ "acc,none": task_score,
+ "acc_stderr,none": 0.0,
+ }
+ for task_name, task_score in task_scores.items()
+ }
+ configs = {
+ task_name: {
+ "metric_list": [{"metric": "acc"}],
+ "filter_list": [{"name": "none"}],
+ }
+ for task_name in task_scores
+ }
+ result: dict[str, Any] = {
+ "result_format": RESULT_FORMAT,
+ "eval_adapter": ADAPTER_NAME,
+ "model_name": model,
+ "results": results,
+ "configs": configs,
+ "n-samples": task_samples,
+ "bfcl": _diagnostics(suite, case_ids_by_category, scores),
+ }
+ if integration_error is not None:
+ result["integration_error"] = _error_dict(integration_error)
+ return result
+
+
+def _write_json(path: Path, value: Mapping[str, Any]) -> None:
+ path.write_text(
+ json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+ )
+
+
+def _write_upstream_attribution(project_root: Path) -> None:
+ """Keep BFCL provenance and its Apache license with archived outputs."""
+ project_root.mkdir(parents=True, exist_ok=True)
+ repository_license = Path(__file__).resolve().parents[2] / "LICENSE"
+ if not repository_license.is_file():
+ raise FileNotFoundError(f"Apache license file not found: {repository_license}")
+ (project_root / UPSTREAM_LICENSE_FILENAME).write_bytes(
+ repository_license.read_bytes()
+ )
+ _write_json(
+ project_root / UPSTREAM_ATTRIBUTION_FILENAME,
+ {
+ "artifact": "BFCL-generated evaluation results",
+ "upstream": {
+ "package": BFCL_PACKAGE,
+ "package_version": BFCL_PACKAGE_VERSION,
+ "wheel_sha256": BFCL_WHEEL_SHA256,
+ "repository": UPSTREAM_REPOSITORY,
+ "source_revision": SOURCE_REVISION,
+ "vllm_integration_revision": VLLM_INTEGRATION_REF,
+ "license": UPSTREAM_LICENSE,
+ "license_url": UPSTREAM_LICENSE_URL,
+ "license_file": UPSTREAM_LICENSE_FILENAME,
+ },
+ "modifications": (
+ "InferenceX selected deterministic case subsets and projected "
+ "upstream scores; this archive does not modify upstream BFCL source."
+ ),
+ },
+ )
+
+
+def _prepare_output_paths(output_dir: Path) -> tuple[Path, Path]:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ native_path = output_dir / NATIVE_REPORT_FILENAME
+ native_path.unlink(missing_ok=True)
+ for stale_path in output_dir.glob(COMPATIBILITY_GLOB):
+ if stale_path.is_file() or stale_path.is_symlink():
+ stale_path.unlink()
+ compatibility_path = output_dir / COMPATIBILITY_FILENAME
+ return native_path, compatibility_path
+
+
+def _clear_upstream_modules() -> None:
+ """Reload BFCL's import-time paths and limits for each adapter invocation."""
+ for module_name in tuple(sys.modules):
+ if module_name == "bfcl_eval" or module_name.startswith("bfcl_eval."):
+ sys.modules.pop(module_name, None)
+
+
+def _apply_suite_runtime_limits(suite: SuiteSpec) -> None:
+ """Apply pinned BFCL limits before importing its model handlers."""
+ if suite is KIMI_SUITE:
+ from bfcl_eval.constants import default_prompts as bfcl_prompts
+
+ bfcl_prompts.MAXIMUM_STEP_LIMIT = KIMI_MAXIMUM_STEP_LIMIT
+
+
+def _bounded_openai_handler(stock_handler: type[Any]) -> type[Any]:
+ """Retain BFCL's handler while bounding its OpenAI transport."""
+
+ class BoundedOpenAICompletionsHandler(stock_handler):
+ def _build_client_kwargs(self) -> dict[str, Any]:
+ kwargs = super()._build_client_kwargs()
+ kwargs.update(
+ timeout=FULL_SUITE_REQUEST_TIMEOUT_SECONDS,
+ max_retries=FULL_SUITE_REQUEST_MAX_RETRIES,
+ )
+ return kwargs
+
+ return BoundedOpenAICompletionsHandler
+
+
+
+
+def _write_id_map(
+ project_root: Path, case_ids_by_category: Mapping[str, tuple[str, ...]]
+) -> None:
+ project_root.mkdir(parents=True, exist_ok=True)
+ _write_json(
+ project_root / "test_case_ids_to_generate.json",
+ {
+ category: list(case_ids)
+ for category, case_ids in case_ids_by_category.items()
+ },
+ )
+
+
+def _function_defaults(function: Callable[..., Any]) -> dict[str, Any]:
+ """Resolve Typer OptionInfo defaults before directly calling a command function."""
+ import typer # Lazy: only the real BFCL path needs this third-party dependency.
+
+ defaults: dict[str, Any] = {}
+ for name, parameter in inspect.signature(function).parameters.items():
+ if parameter.default is inspect.Parameter.empty:
+ continue
+ default = parameter.default
+ if isinstance(default, typer.models.OptionInfo):
+ default = default.default
+ defaults[name] = default
+ return defaults
+
+
+def _load_dataset_helpers() -> tuple[
+ Callable[[str], Any],
+ Callable[[list[str]], Any],
+ Callable[[Any], Any],
+]:
+ """Import BFCL's pinned dataset helpers only when a full suite is selected."""
+ from bfcl_eval.utils import (
+ load_dataset_entry,
+ parse_test_category_argument,
+ sort_key,
+ )
+
+ return load_dataset_entry, parse_test_category_argument, sort_key
+
+
+def _build_suite_case_ids(
+ suite: SuiteSpec,
+) -> dict[str, tuple[str, ...]]:
+ if suite is SMOKE_SUITE:
+ return dict(SMOKE_CASE_IDS)
+
+ load_dataset_entry, parse_test_category_argument, sort_key = _load_dataset_helpers()
+ category_limits = dict(suite.category_limits)
+ selected_by_leaf: dict[str, tuple[str, ...]] = {}
+ seen_ids: set[str] = set()
+ for category in suite.generation_categories:
+ leaf_categories = list(parse_test_category_argument([category]))
+ by_leaf = {
+ leaf: sorted(load_dataset_entry(leaf), key=sort_key)
+ for leaf in leaf_categories
+ }
+ limit = category_limits.get(category)
+ if limit is None:
+ quotas = [len(by_leaf[leaf]) for leaf in leaf_categories]
+ else:
+ base, extra = divmod(limit, len(leaf_categories))
+ quotas = [base + (index < extra) for index in range(len(leaf_categories))]
+ for leaf, quota in zip(leaf_categories, quotas, strict=True):
+ entries = by_leaf[leaf][:quota]
+ ids: list[str] = []
+ for entry in entries:
+ if not isinstance(entry, Mapping):
+ raise ValueError(f"{leaf} dataset entry must be a mapping")
+ case_id = entry.get("id")
+ if not isinstance(case_id, str) or not case_id:
+ raise ValueError(
+ f"{leaf} dataset entry must contain a non-empty string id"
+ )
+ if case_id in seen_ids:
+ raise ValueError(f"BFCL dataset contains duplicate id {case_id}")
+ seen_ids.add(case_id)
+ ids.append(case_id)
+ selected_by_leaf[leaf] = tuple(ids)
+
+ actual_counts = {
+ category: len(case_ids) for category, case_ids in selected_by_leaf.items()
+ }
+ expected_counts = dict(suite.expected_leaf_counts)
+ if actual_counts != expected_counts:
+ raise ValueError(
+ f"{suite.name} selected leaf counts {actual_counts!r}; "
+ f"expected {expected_counts!r}"
+ )
+ return {
+ category: selected_by_leaf[category]
+ for category, _ in suite.expected_leaf_counts
+ }
+
+
+def _read_selected_suite(
+ project_root: Path,
+) -> tuple[SuiteSpec, dict[str, tuple[str, ...]]]:
+ raw = json.loads(
+ (project_root / "test_case_ids_to_generate.json").read_text(encoding="utf-8")
+ )
+ if not isinstance(raw, Mapping):
+ raise ValueError("BFCL test-case ID map must be a JSON object")
+ case_ids_by_category: dict[str, tuple[str, ...]] = {}
+ for category, case_ids in raw.items():
+ if not isinstance(category, str) or not isinstance(case_ids, list):
+ raise ValueError("BFCL test-case ID map has an invalid category entry")
+ if not all(isinstance(case_id, str) and case_id for case_id in case_ids):
+ raise ValueError(f"{category} test-case IDs must be non-empty strings")
+ case_ids_by_category[category] = tuple(case_ids)
+
+ shape = tuple(
+ (category, len(case_ids)) for category, case_ids in case_ids_by_category.items()
+ )
+ for suite in SUITE_SPECS.values():
+ if shape != suite.expected_leaf_counts:
+ continue
+ if suite is SMOKE_SUITE and case_ids_by_category != dict(SMOKE_CASE_IDS):
+ continue
+ return suite, case_ids_by_category
+ raise ValueError(f"test-case ID map does not match a supported suite: {shape!r}")
+
+
+def _run_upstream(
+ *,
+ model: str,
+ project_root: Path,
+ base_url: str,
+ api_key: str,
+ num_threads: int,
+) -> None:
+ """Lazily load and invoke the pinned BFCL API against an existing server."""
+ suite, case_ids_by_category = _read_selected_suite(project_root)
+ _apply_suite_runtime_limits(suite)
+ os.environ["BFCL_PROJECT_ROOT"] = str(project_root)
+ os.environ["OPENAI_BASE_URL"] = base_url
+ os.environ["OPENAI_API_KEY"] = api_key
+
+ import bfcl_eval.constants.model_config as bfcl_model_config
+ from bfcl_eval.__main__ import evaluate, generate
+ from bfcl_eval.constants.model_config import ModelConfig
+ from bfcl_eval.model_handler.api_inference.openai_completion import (
+ OpenAICompletionsHandler,
+ )
+ handler = (
+ OpenAICompletionsHandler
+ if suite is SMOKE_SUITE
+ else _bounded_openai_handler(OpenAICompletionsHandler)
+ )
+
+ bfcl_model_config.MODEL_CONFIG_MAPPING[model] = ModelConfig(
+ model_name=model,
+ display_name=f"{model} (FC) (InferenceX)",
+ url="",
+ org="",
+ license="unknown",
+ model_handler=handler,
+ input_price=None,
+ output_price=None,
+ is_fc_model=True,
+ underscore_to_dot=True,
+ )
+
+ categories = list(suite.generation_categories)
+ generation_kwargs = _function_defaults(generate)
+ generation_kwargs.update(
+ model=[model],
+ test_category=categories,
+ temperature=suite.temperature,
+ num_threads=num_threads,
+ skip_server_setup=True,
+ run_ids=True,
+ allow_overwrite=True,
+ )
+ generate(**generation_kwargs)
+ _validate_generated_results(project_root, case_ids_by_category)
+
+ evaluation_kwargs = _function_defaults(evaluate)
+ evaluation_kwargs.update(
+ model=[model],
+ test_category=categories,
+ partial_eval=True,
+ )
+ evaluate(**evaluation_kwargs)
+
+
+def _validate_generated_results(
+ project_root: Path,
+ case_ids_by_category: Mapping[str, tuple[str, ...]] = SMOKE_CASE_IDS,
+) -> None:
+ for category, case_ids in case_ids_by_category.items():
+ matches = sorted(project_root.glob(f"result/**/BFCL_v4_{category}_result.json"))
+ if len(matches) != 1:
+ raise ValueError(
+ f"expected exactly one {category} result file, found {len(matches)}"
+ )
+ result_ids: list[str] = []
+ with matches[0].open(encoding="utf-8") as result_file:
+ for line_number, result_line in enumerate(result_file, start=1):
+ try:
+ record = json.loads(result_line)
+ except json.JSONDecodeError as exc:
+ raise ValueError(
+ f"{category} result record on line {line_number} "
+ "is malformed JSON"
+ ) from exc
+ if not isinstance(record, Mapping):
+ raise ValueError(
+ f"{category} result record on line {line_number} "
+ "must be a JSON object"
+ )
+ case_id = record.get("id")
+ if not isinstance(case_id, str) or not case_id:
+ raise ValueError(
+ f"{category} result record on line {line_number} "
+ "must contain a non-empty string id"
+ )
+ if case_id not in case_ids:
+ raise ValueError(
+ f"{category} result file contains unexpected id {case_id}"
+ )
+ if case_id in result_ids:
+ raise ValueError(
+ f"{category} result file contains duplicate id {case_id}"
+ )
+ if "traceback" in record or (
+ isinstance(record.get("result"), str)
+ and record["result"].startswith("Error during inference:")
+ ):
+ raise RuntimeError(
+ f"{category} result {case_id} contains an inference error"
+ )
+ result_ids.append(case_id)
+ if result_ids != list(case_ids):
+ raise ValueError(
+ f"{category} result ids {result_ids!r} "
+ f"do not match expected ids {list(case_ids)!r}"
+ )
+
+
+def _validate_header(
+ *, category: str, header: Any, expected_count: int
+) -> tuple[float, int, int]:
+ if not isinstance(header, Mapping):
+ raise ValueError(f"{category} score header must be a JSON object")
+ missing = {"accuracy", "correct_count", "total_count"} - header.keys()
+ if missing:
+ raise ValueError(
+ f"{category} score header missing: {', '.join(sorted(missing))}"
+ )
+
+ correct_count = header["correct_count"]
+ total_count = header["total_count"]
+ accuracy = header["accuracy"]
+ if isinstance(correct_count, bool) or not isinstance(correct_count, int):
+ raise ValueError(f"{category} correct_count must be an integer")
+ if isinstance(total_count, bool) or not isinstance(total_count, int):
+ raise ValueError(f"{category} total_count must be an integer")
+ if (
+ isinstance(accuracy, bool)
+ or not isinstance(accuracy, (int, float))
+ or not math.isfinite(float(accuracy))
+ ):
+ raise ValueError(f"{category} accuracy must be a finite number")
+ if total_count != expected_count:
+ raise ValueError(
+ f"{category} evaluated {total_count} cases; expected {expected_count}"
+ )
+ if correct_count < 0 or correct_count > total_count:
+ raise ValueError(f"{category} correct_count is outside [0, total_count]")
+
+ computed_accuracy = correct_count / total_count
+ if not math.isclose(float(accuracy), computed_accuracy, rel_tol=0.0, abs_tol=1e-12):
+ raise ValueError(f"{category} accuracy is inconsistent with its count fields")
+ return float(accuracy), correct_count, total_count
+
+
+def _collect_scores(
+ project_root: Path,
+ case_ids_by_category: Mapping[str, tuple[str, ...]] = SMOKE_CASE_IDS,
+) -> list[CategoryScore]:
+ scores: list[CategoryScore] = []
+ for category, case_ids in case_ids_by_category.items():
+ matches = sorted(project_root.glob(f"score/**/BFCL_v4_{category}_score.json"))
+ if len(matches) != 1:
+ raise ValueError(
+ f"expected exactly one {category} score file, found {len(matches)}"
+ )
+ score_path = matches[0]
+ with score_path.open(encoding="utf-8") as score_file:
+ first_line = score_file.readline()
+ record_lines = list(score_file)
+ if not first_line:
+ raise ValueError(f"{category} score file has no header")
+ try:
+ header = json.loads(first_line)
+ except json.JSONDecodeError as exc:
+ raise ValueError(f"{category} score header is malformed JSON") from exc
+ accuracy, correct_count, total_count = _validate_header(
+ category=category,
+ header=header,
+ expected_count=len(case_ids),
+ )
+ records: list[dict[str, Any]] = []
+ record_ids: list[str] = []
+ for line_number, record_line in enumerate(record_lines, start=2):
+ try:
+ record = json.loads(record_line)
+ except json.JSONDecodeError as exc:
+ raise ValueError(
+ f"{category} score record on line {line_number} is malformed JSON"
+ ) from exc
+ if not isinstance(record, Mapping):
+ raise ValueError(
+ f"{category} score record on line {line_number} "
+ "must be a JSON object"
+ )
+ case_id = record.get("id")
+ if not isinstance(case_id, str) or not case_id:
+ raise ValueError(
+ f"{category} score record on line {line_number} "
+ "must contain a non-empty string id"
+ )
+ if case_id not in case_ids:
+ raise ValueError(
+ f"{category} score file contains unexpected id {case_id}"
+ )
+ if case_id in record_ids:
+ raise ValueError(
+ f"{category} score file contains duplicate id {case_id}"
+ )
+ record_ids.append(case_id)
+ records.append(dict(record))
+ expected_failure_count = total_count - correct_count
+ if len(records) != expected_failure_count:
+ raise ValueError(
+ f"{category} score file contains {len(records)} failure records; "
+ f"expected {expected_failure_count}"
+ )
+ scores.append(
+ CategoryScore(
+ category=category,
+ case_ids=case_ids,
+ score_file=score_path.relative_to(project_root).as_posix(),
+ header=dict(header),
+ records=tuple(records),
+ accuracy=accuracy,
+ correct_count=correct_count,
+ total_count=total_count,
+ )
+ )
+ return scores
+
+
+def publish_integration_error(
+ *,
+ output_dir: Path,
+ model: str,
+ error: BaseException,
+ suite: SuiteSpec = SMOKE_SUITE,
+) -> None:
+ """Publish required zero-score artifacts without importing BFCL or Typer."""
+ native_path, compatibility_path = _prepare_output_paths(output_dir)
+ case_ids_by_category = (
+ dict(SMOKE_CASE_IDS)
+ if suite is SMOKE_SUITE
+ else {category: () for category in suite.leaf_categories}
+ )
+ _write_json(
+ native_path,
+ _native_report(
+ suite=suite,
+ case_ids_by_category=case_ids_by_category,
+ model=model,
+ base_url=None,
+ num_threads=suite.default_num_threads,
+ scores=None,
+ integration_error=error,
+ ),
+ )
+ _write_json(
+ compatibility_path,
+ _compatibility_result(
+ suite=suite,
+ case_ids_by_category=case_ids_by_category,
+ model=model,
+ scores=None,
+ integration_error=error,
+ ),
+ )
+
+
+def run_evaluation(
+ *,
+ base_url: str,
+ api_key: str,
+ model: str,
+ output_dir: Path,
+ bfcl_project_root: Path,
+ suite: SuiteSpec = SMOKE_SUITE,
+ num_threads: int | None = None,
+ upstream_runner: UpstreamRunner = _run_upstream,
+) -> bool:
+ """Run one immutable BFCL suite and always publish both report formats."""
+ native_path, compatibility_path = _prepare_output_paths(output_dir)
+ selected_case_ids: dict[str, tuple[str, ...]] = (
+ dict(SMOKE_CASE_IDS)
+ if suite is SMOKE_SUITE
+ else {category: () for category in suite.leaf_categories}
+ )
+ resolved_num_threads = (
+ suite.default_num_threads if num_threads is None else num_threads
+ )
+ try:
+ if suite.name not in SUITE_SPECS or SUITE_SPECS[suite.name] is not suite:
+ raise ValueError(f"unsupported BFCL suite: {suite.name}")
+ normalized_url = _absolute_http_url(base_url)
+ normalized_model = _nonempty_string(model)
+ normalized_key = _nonempty_string(api_key)
+ if isinstance(resolved_num_threads, bool) or not isinstance(
+ resolved_num_threads, int
+ ):
+ raise ValueError("num_threads must be a positive integer")
+ if resolved_num_threads <= 0:
+ raise ValueError("num_threads must be a positive integer")
+ if not callable(upstream_runner):
+ raise TypeError("upstream_runner must be callable")
+
+ os.environ["BFCL_PROJECT_ROOT"] = str(bfcl_project_root)
+ _clear_upstream_modules()
+ _write_upstream_attribution(bfcl_project_root)
+ selected_case_ids = _build_suite_case_ids(suite)
+ _write_id_map(bfcl_project_root, selected_case_ids)
+ upstream_runner(
+ model=normalized_model,
+ project_root=bfcl_project_root,
+ base_url=normalized_url,
+ api_key=normalized_key,
+ num_threads=resolved_num_threads,
+ )
+ _validate_generated_results(bfcl_project_root, selected_case_ids)
+ scores = _collect_scores(bfcl_project_root, selected_case_ids)
+ if sum(score.total_count for score in scores) != suite.expected_sample_count:
+ raise ValueError(
+ f"{suite.name} evaluated an unexpected total number of cases"
+ )
+ except Exception as exc: # noqa: BLE001 - artifact publication is the boundary
+ _write_json(
+ native_path,
+ _native_report(
+ suite=suite,
+ case_ids_by_category=selected_case_ids,
+ model=model,
+ base_url=base_url,
+ num_threads=resolved_num_threads,
+ scores=None,
+ integration_error=exc,
+ ),
+ )
+ _write_json(
+ compatibility_path,
+ _compatibility_result(
+ suite=suite,
+ case_ids_by_category=selected_case_ids,
+ model=model,
+ scores=None,
+ integration_error=exc,
+ ),
+ )
+ return False
+
+ native = _native_report(
+ suite=suite,
+ case_ids_by_category=selected_case_ids,
+ model=normalized_model,
+ base_url=normalized_url,
+ num_threads=resolved_num_threads,
+ scores=scores,
+ )
+ compatibility = _compatibility_result(
+ suite=suite,
+ case_ids_by_category=selected_case_ids,
+ model=normalized_model,
+ scores=scores,
+ )
+ _write_json(native_path, native)
+ _write_json(compatibility_path, compatibility)
+ return True
+
+
+def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Run a pinned BFCL V4 OpenAI completions suite."
+ )
+ parser.add_argument("--base-url", type=_absolute_http_url)
+ parser.add_argument("--api-key", type=_nonempty_string, default="EMPTY")
+ parser.add_argument("--model", type=_nonempty_string, required=True)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ parser.add_argument("--bfcl-project-root", type=Path)
+ parser.add_argument(
+ "--suite",
+ choices=tuple(SUITE_SPECS),
+ default=TASK_NAME,
+ )
+ parser.add_argument("--num-threads", type=_positive_int)
+ parser.add_argument("--integration-error")
+ args = parser.parse_args(argv)
+ args.num_threads = (
+ SUITE_SPECS[args.suite].default_num_threads
+ if args.num_threads is None
+ else args.num_threads
+ )
+ if args.integration_error is None:
+ if args.base_url is None:
+ parser.error("--base-url required unless --integration-error is provided")
+ if args.bfcl_project_root is None:
+ parser.error(
+ "--bfcl-project-root required unless --integration-error is provided"
+ )
+ return args
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = parse_args(argv)
+ suite = SUITE_SPECS[args.suite]
+ if args.integration_error is not None:
+ publish_integration_error(
+ output_dir=args.output_dir,
+ model=args.model,
+ error=RuntimeError(args.integration_error),
+ suite=suite,
+ )
+ return 1
+
+ passed = run_evaluation(
+ base_url=args.base_url,
+ api_key=args.api_key,
+ model=args.model,
+ output_dir=args.output_dir,
+ bfcl_project_root=args.bfcl_project_root,
+ suite=suite,
+ num_threads=args.num_threads,
+ )
+ return 0 if passed else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py
new file mode 100755
index 0000000000..ab09059910
--- /dev/null
+++ b/utils/evals/kimi_vendor_eval.py
@@ -0,0 +1,522 @@
+#!/usr/bin/env python3
+"""Run the stock Kimi Vendor Verifier and project its native report."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from collections.abc import Mapping, Sequence
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+TASK_NAME = "kimi_tool_call_schema"
+FULL_TASK_NAME = "kimi_tool_call_schema_full"
+SUPPORTED_TASK_NAMES = (TASK_NAME, FULL_TASK_NAME)
+NATIVE_REPORT_FILENAME = "kimi_vendor_report.json"
+COMPATIBILITY_GLOB = "results_kimi_vendor_*.json"
+EXPECTED_MODES = {"non-stream", "stream"}
+EXPECTED_TOTALS = {TASK_NAME: 2, FULL_TASK_NAME: 408}
+FULL_SELECTED_CASES = 204
+DEFAULT_TIMEOUT_SECONDS = 900
+FULL_TIMEOUT_SECONDS = 7200
+FULL_WORKERS = 8
+RESULT_FORMAT = "inferencex-eval-v1"
+ADAPTER_NAME = "kimi-vendor-verifier"
+
+ENDPOINT_REJECTION_RE = re.compile(
+ r"(?im)^(?:E\s+)?AssertionError:.*tool schema rejected:"
+)
+
+
+def prepare_compatibility_path(output_dir: Path) -> Path:
+ """Remove stale projections and return a timestamped collector artifact path."""
+ for stale_path in output_dir.glob(COMPATIBILITY_GLOB):
+ stale_path.unlink()
+ timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f")
+ return output_dir / f"results_kimi_vendor_{timestamp}.json"
+
+
+def build_pytest_command(
+ *,
+ base_url: str,
+ api_key: str,
+ model: str,
+ model_prefix: str = "",
+ report_path: Path,
+ task_name: str = TASK_NAME,
+) -> list[str]:
+ """Build the fixed invocation of the pinned upstream verifier."""
+ _expected_total(task_name)
+ thinking_args = (
+ ["--think-mode", "opensource", "--thinking"]
+ if model_prefix == "dsv4"
+ else ["--think-mode", "none"]
+ )
+ parallel_args = ["-n", str(FULL_WORKERS)] if task_name == FULL_TASK_NAME else []
+ selection_args = (
+ ["--selection", "all"]
+ if task_name == FULL_TASK_NAME
+ else ["--selection", "object", "--max-cases", "1"]
+ )
+ return [
+ sys.executable,
+ "-m",
+ "pytest",
+ "tests/tool_call_json_schema/test_tool_call_json_schema.py",
+ *parallel_args,
+ "--base-url",
+ base_url,
+ "--api-key",
+ api_key,
+ "--smoke-model",
+ model,
+ *thinking_args,
+ *selection_args,
+ "--case-dir",
+ "testdata/walle_validator_cases/validator_cases",
+ "--max-tokens",
+ "2048",
+ "--tool-json-report",
+ str(report_path),
+ ]
+
+
+def _mapping(value: Any, name: str) -> Mapping[str, Any]:
+ if not isinstance(value, Mapping):
+ raise ValueError(f"{name} must be an object")
+ return value
+
+
+def _expected_total(task_name: str) -> int:
+ try:
+ return EXPECTED_TOTALS[task_name]
+ except KeyError as exc:
+ raise ValueError(f"unsupported Kimi task: {task_name}") from exc
+
+
+def _endpoint_rejection_messages(report: Any) -> list[str]:
+ """Return upstream failures rejected before argument-schema validation."""
+ root = _mapping(report, "report")
+ results = root.get("results")
+ if not isinstance(results, list):
+ raise ValueError("report.results must be an array")
+ messages: list[str] = []
+ for index, result in enumerate(results):
+ record = _mapping(result, f"report.results[{index}]")
+ message = record.get("message")
+ if (
+ record.get("status") == "failed"
+ and isinstance(message, str)
+ and ENDPOINT_REJECTION_RE.search(message)
+ ):
+ messages.append(message)
+ return messages
+
+
+def _project_report(
+ model: str,
+ report: Any,
+ *,
+ task_name: str = TASK_NAME,
+) -> tuple[dict[str, Any], bool]:
+ expected_total = _expected_total(task_name)
+ root = _mapping(report, "report")
+ summary = _mapping(root.get("summary"), "report.summary")
+ results = root.get("results")
+ if not isinstance(results, list):
+ raise ValueError("report.results must be an array")
+
+ total = summary.get("total")
+ by_status = _mapping(summary.get("by_status"), "report.summary.by_status")
+ if not isinstance(total, int) or isinstance(total, bool):
+ raise ValueError("report summary contains invalid counts")
+ for status, count in by_status.items():
+ if (
+ status not in {"passed", "failed"}
+ or not isinstance(count, int)
+ or isinstance(count, bool)
+ or count < 0
+ ):
+ raise ValueError("report summary contains invalid counts")
+ if sum(by_status.values()) != total:
+ raise ValueError("report summary does not match total")
+ passed = by_status.get("passed", 0)
+ selected_identities: set[tuple[str, int]] = set()
+ if task_name == FULL_TASK_NAME:
+ selected_cases = root.get("selected_cases")
+ if (
+ not isinstance(selected_cases, list)
+ or len(selected_cases) != FULL_SELECTED_CASES
+ ):
+ raise ValueError(
+ f"report.selected_cases must contain {FULL_SELECTED_CASES} cases"
+ )
+ selected_keys: set[tuple[str, int, str]] = set()
+ for index, selected_case in enumerate(selected_cases):
+ record = _mapping(
+ selected_case,
+ f"report.selected_cases[{index}]",
+ )
+ suite = record.get("suite")
+ line = record.get("line")
+ selection_reason = record.get("selection_reason")
+ if (
+ not isinstance(suite, str)
+ or not suite
+ or not isinstance(line, int)
+ or isinstance(line, bool)
+ or line < 1
+ or not isinstance(selection_reason, str)
+ ):
+ raise ValueError(f"report.selected_cases[{index}] has invalid identity")
+ selected_key = (suite, line, selection_reason)
+ if selected_key in selected_keys:
+ raise ValueError("report.selected_cases contains a duplicate case")
+ selected_keys.add(selected_key)
+ selected_identities.add((suite, line))
+
+ modes: list[str] = []
+ case_modes: dict[tuple[str, int], set[str]] = {}
+ result_passes = 0
+ for index, result in enumerate(results):
+ record = _mapping(result, f"report.results[{index}]")
+ mode = record.get("mode")
+ status = record.get("status")
+ if (
+ not isinstance(mode, str)
+ or not isinstance(status, str)
+ or status not in {"passed", "failed"}
+ ):
+ raise ValueError(f"report.results[{index}] has invalid mode or status")
+ modes.append(mode)
+ result_passes += status == "passed"
+
+ if task_name == FULL_TASK_NAME:
+ suite = record.get("suite")
+ line = record.get("line")
+ if (
+ not isinstance(suite, str)
+ or not suite
+ or not isinstance(line, int)
+ or isinstance(line, bool)
+ or line < 1
+ ):
+ raise ValueError(
+ f"report.results[{index}] has invalid suite or line identity"
+ )
+ identity = (suite, line)
+ identity_modes = case_modes.setdefault(identity, set())
+ if mode in identity_modes:
+ raise ValueError(
+ "report contains a duplicate mode for a selected suite and line"
+ )
+ identity_modes.add(mode)
+
+ if total != len(results) or passed != result_passes:
+ raise ValueError("report summary does not match result records")
+ if total != expected_total:
+ raise ValueError(
+ f"report contains {total} records; expected {expected_total} for {task_name}"
+ )
+
+ if task_name == TASK_NAME:
+ if set(modes) != EXPECTED_MODES or len(modes) != len(set(modes)):
+ raise ValueError("report does not contain the expected stream modes")
+ elif any(
+ modes_for_case != EXPECTED_MODES for modes_for_case in case_modes.values()
+ ):
+ raise ValueError(
+ "report does not contain exactly one of each stream mode "
+ "for every selected suite and line"
+ )
+ if task_name == FULL_TASK_NAME and set(case_modes) != selected_identities:
+ raise ValueError(
+ "report results do not match the selected suite and line identities"
+ )
+
+ score = passed / total
+ return (
+ _compatibility_result(
+ model,
+ score,
+ task_name=task_name,
+ n_samples=total,
+ ),
+ passed == total,
+ )
+
+
+def _compatibility_result(
+ model: str,
+ score: float,
+ *,
+ task_name: str = TASK_NAME,
+ n_samples: int,
+ integration_error: BaseException | None = None,
+) -> dict[str, Any]:
+ expected_total = _expected_total(task_name)
+ result: dict[str, Any] = {
+ "result_format": RESULT_FORMAT,
+ "eval_adapter": ADAPTER_NAME,
+ "model_name": model,
+ "results": {
+ task_name: {
+ "exact_match,strict-match": score,
+ "exact_match_stderr,strict-match": 0.0,
+ }
+ },
+ "configs": {
+ task_name: {
+ "metric_list": [{"metric": "exact_match"}],
+ "filter_list": [{"name": "strict-match"}],
+ }
+ },
+ "n-samples": {
+ task_name: {
+ "original": expected_total,
+ "effective": n_samples,
+ }
+ },
+ }
+ if integration_error is not None:
+ result["integration_error"] = {
+ "type": type(integration_error).__name__,
+ "message": str(integration_error),
+ }
+ return result
+
+
+def _write_native_failure(
+ path: Path,
+ *,
+ model: str,
+ task_name: str,
+ error: BaseException,
+) -> None:
+ """Write a native diagnostic envelope when upstream cannot produce one."""
+ path.write_text(
+ json.dumps(
+ {
+ "generated_at": datetime.now(timezone.utc).isoformat(),
+ "model": model,
+ "task": task_name,
+ "completed": False,
+ "summary": {
+ "total": 0,
+ "expected_total": _expected_total(task_name),
+ "by_status": {},
+ },
+ "results": [],
+ "integration_error": {
+ "type": type(error).__name__,
+ "message": str(error),
+ },
+ },
+ indent=2,
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+
+
+def _write_compatibility(path: Path, result: Mapping[str, Any]) -> None:
+ path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+
+
+def run_evaluation(
+ *,
+ verifier_dir: Path,
+ base_url: str,
+ api_key: str,
+ model: str,
+ model_prefix: str = "",
+ output_dir: Path,
+ task_name: str = TASK_NAME,
+ timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+) -> bool:
+ """Run upstream pytest and always attempt to publish a compatibility result."""
+ output_dir.mkdir(parents=True, exist_ok=True)
+ native_report = output_dir / NATIVE_REPORT_FILENAME
+ compatibility_path = prepare_compatibility_path(output_dir)
+ subprocess_rc: int | None = None
+ integration_error: BaseException | None = None
+ compatibility = _compatibility_result(
+ model,
+ 0.0,
+ task_name=task_name,
+ n_samples=0,
+ )
+ all_passed = False
+ completed_successfully = False
+ try:
+ native_report.unlink(missing_ok=True)
+ completed = subprocess.run(
+ build_pytest_command(
+ base_url=base_url,
+ api_key=api_key,
+ model=model,
+ model_prefix=model_prefix,
+ report_path=native_report.resolve(),
+ task_name=task_name,
+ ),
+ cwd=verifier_dir,
+ check=False,
+ timeout=timeout_seconds,
+ )
+ subprocess_rc = completed.returncode
+ report = json.loads(native_report.read_text(encoding="utf-8"))
+ compatibility, all_passed = _project_report(
+ model,
+ report,
+ task_name=task_name,
+ )
+ endpoint_rejections = _endpoint_rejection_messages(report)
+ valid_outcome = (subprocess_rc == 0 and all_passed) or (
+ subprocess_rc == 1 and not all_passed
+ )
+ completed_successfully = subprocess_rc == 0 and all_passed
+ if endpoint_rejections:
+ integration_error = RuntimeError(
+ "upstream verifier reported "
+ f"{len(endpoint_rejections)} endpoint request or response failure(s)"
+ )
+ compatibility = _compatibility_result(
+ model,
+ 0.0,
+ task_name=task_name,
+ n_samples=0,
+ integration_error=integration_error,
+ )
+ completed_successfully = False
+ elif valid_outcome:
+ completed_successfully = True
+ elif not valid_outcome:
+ integration_error = RuntimeError(
+ f"upstream verifier exited with code {subprocess_rc}"
+ )
+ compatibility = _compatibility_result(
+ model,
+ 0.0,
+ task_name=task_name,
+ n_samples=0,
+ integration_error=integration_error,
+ )
+ completed_successfully = False
+ except (OSError, ValueError, subprocess.TimeoutExpired) as exc:
+ integration_error = exc
+ compatibility = _compatibility_result(
+ model,
+ 0.0,
+ task_name=task_name,
+ n_samples=0,
+ integration_error=exc,
+ )
+ if not native_report.exists():
+ _write_native_failure(
+ native_report,
+ model=model,
+ task_name=task_name,
+ error=exc,
+ )
+ finally:
+ try:
+ _write_compatibility(compatibility_path, compatibility)
+ except OSError as exc:
+ if integration_error is not None:
+ exc.add_note(f"Earlier integration error: {integration_error}")
+ raise
+
+ return completed_successfully and integration_error is None
+
+
+def _positive_int(value: str) -> int:
+ parsed = int(value)
+ if parsed < 1:
+ raise argparse.ArgumentTypeError("must be a positive integer")
+ return parsed
+
+
+def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Run the pinned stock Kimi Vendor Verifier tool-schema evaluation."
+ )
+ parser.add_argument("--verifier-dir", type=Path)
+ parser.add_argument("--base-url")
+ parser.add_argument("--api-key", default="EMPTY")
+ parser.add_argument("--model", required=True)
+ parser.add_argument("--model-prefix", default="")
+ parser.add_argument("--output-dir", required=True, type=Path)
+ parser.add_argument(
+ "--task-name",
+ choices=SUPPORTED_TASK_NAMES,
+ default=TASK_NAME,
+ )
+ parser.add_argument("--timeout-seconds", type=_positive_int)
+ parser.add_argument("--integration-error")
+ args = parser.parse_args(argv)
+ if args.integration_error is None:
+ missing = [
+ option
+ for option, value in (
+ ("--verifier-dir", args.verifier_dir),
+ ("--base-url", args.base_url),
+ )
+ if value is None
+ ]
+ if missing:
+ parser.error(
+ f"{', '.join(missing)} required unless --integration-error is provided"
+ )
+ return args
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = parse_args(argv)
+ if args.integration_error is not None:
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ error = RuntimeError(args.integration_error)
+ _write_native_failure(
+ args.output_dir / NATIVE_REPORT_FILENAME,
+ model=args.model,
+ task_name=args.task_name,
+ error=error,
+ )
+ _write_compatibility(
+ prepare_compatibility_path(args.output_dir),
+ _compatibility_result(
+ args.model,
+ 0.0,
+ task_name=args.task_name,
+ n_samples=0,
+ integration_error=error,
+ ),
+ )
+ return 0
+ timeout_seconds = (
+ args.timeout_seconds
+ if args.timeout_seconds is not None
+ else (
+ FULL_TIMEOUT_SECONDS
+ if args.task_name == FULL_TASK_NAME
+ else DEFAULT_TIMEOUT_SECONDS
+ )
+ )
+ passed = run_evaluation(
+ verifier_dir=args.verifier_dir,
+ base_url=args.base_url,
+ api_key=args.api_key,
+ model=args.model,
+ model_prefix=args.model_prefix,
+ output_dir=args.output_dir,
+ task_name=args.task_name,
+ timeout_seconds=timeout_seconds,
+ )
+ return 0 if passed else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/utils/evals/minimax_m3_full_eval.py b/utils/evals/minimax_m3_full_eval.py
new file mode 100755
index 0000000000..832c43f173
--- /dev/null
+++ b/utils/evals/minimax_m3_full_eval.py
@@ -0,0 +1,555 @@
+#!/usr/bin/env python3
+"""Run and project the pinned full MiniMax M3 provider verifier."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import subprocess
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Callable, Mapping, Sequence
+from datetime import datetime, timezone
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+TASK_NAME = "minimax_m3_full"
+RESULT_FORMAT = "inferencex-eval-v1"
+ADAPTER_NAME = "minimax-provider-verifier"
+NATIVE_REPORT_FILENAME = "minimax_vendor_report.json"
+NATIVE_RESULTS_FILENAME = "minimax_vendor_results.jsonl"
+COMPATIBILITY_GLOB = "results_minimax_vendor_*.json"
+EXPECTED_RESULT_COUNT = 102
+UPSTREAM_REF = "c899f95e17bfc4a338ddd4cb1638279125885e55"
+UPSTREAM_BASE_URL = (
+ "https://raw.githubusercontent.com/MiniMax-AI/MiniMax-Provider-Verifier/"
+ f"{UPSTREAM_REF}"
+)
+EXPECTED_SAMPLE_SHA256 = (
+ "3ead102af0f888acc95867b3a9916942524b02f4f64931f020a1bfb4fee9aae2"
+)
+REQUIRED_SOURCE_SHA256 = {
+ "verify.py": "6bc00948d9be06189f31c5a53bb7929b15555402f0b3495609d26b468090ee4a",
+ "sample.jsonl": EXPECTED_SAMPLE_SHA256,
+ "validator/__init__.py": "955a5ee77b72fb1d128f5d1ab6c65072e54b8cf527916e70a69e251811087e7b",
+ "validator/base.py": "00f1776d4b4d4200e4ce865f044ef9cdd7f375ca3b752a8eefba4ab375f7e03d",
+ "validator/tool_calls.py": "eb6a91a704a3e1706a1fc0f0f4233f4dd08ecb38b7f41fd19abaf5e981d0406b",
+ "validator/russian_characters.py": "09b429f45b43b34c241d5b54401aab6c4c55de228d87c882bb7f8858c9856c0c",
+ "validator/repeat_ngram.py": "72948cd9501e8daeb49a2c9fac4421a182191fc4251846713021c27d1a5fa315",
+ "validator/scenario_check.py": "51d691a1595f6fa3193f4f624a98581a9795323e77439df7a2fd5d83950c5ba1",
+}
+MAX_SOURCE_BYTES = 16 * 1024 * 1024
+DOWNLOAD_TIMEOUT_SECONDS = 60
+DOWNLOAD_ATTEMPTS = 3
+DOWNLOAD_RETRY_DELAY_SECONDS = 3
+UPSTREAM_TIMEOUT_SECONDS = 7 * 60 * 60
+
+Runner = Callable[..., subprocess.CompletedProcess[Any]]
+Fetcher = Callable[[str], bytes]
+
+
+class FullSuiteError(RuntimeError):
+ """The full verifier could not produce one complete diagnostic run."""
+
+
+class _RejectRedirects(urllib.request.HTTPRedirectHandler):
+ def redirect_request(self, *args: Any, **kwargs: Any) -> None:
+ return None
+
+
+_NO_REDIRECT_OPENER = urllib.request.build_opener(_RejectRedirects())
+
+
+def source_url(relative_path: str) -> str:
+ """Return the immutable URL for one allowlisted required source file."""
+ if relative_path not in REQUIRED_SOURCE_SHA256:
+ raise ValueError(f"unapproved upstream source path: {relative_path!r}")
+ path = PurePosixPath(relative_path)
+ if path.is_absolute() or ".." in path.parts:
+ raise ValueError(f"unsafe upstream source path: {relative_path!r}")
+ return f"{UPSTREAM_BASE_URL}/{relative_path}"
+
+
+def verify_source_content(relative_path: str, content: bytes) -> None:
+ """Verify one allowlisted file against its pinned SHA256 identity."""
+ if relative_path not in REQUIRED_SOURCE_SHA256:
+ raise ValueError(f"unapproved upstream source path: {relative_path!r}")
+ if not isinstance(content, bytes):
+ raise TypeError("source content must be bytes")
+ if len(content) > MAX_SOURCE_BYTES:
+ raise ValueError(f"pinned source {relative_path} exceeds the size limit")
+ actual = hashlib.sha256(content).hexdigest()
+ expected = REQUIRED_SOURCE_SHA256[relative_path]
+ if actual != expected:
+ raise ValueError(
+ f"pinned source {relative_path} SHA256 mismatch: "
+ f"expected {expected}, got {actual}"
+ )
+
+
+def _fetch_source(relative_path: str) -> bytes:
+ url = source_url(relative_path)
+ request = urllib.request.Request(
+ url,
+ headers={"Accept": "application/octet-stream"},
+ method="GET",
+ )
+ last_error: BaseException | None = None
+ for attempt in range(1, DOWNLOAD_ATTEMPTS + 1):
+ try:
+ with _NO_REDIRECT_OPENER.open(
+ request, timeout=DOWNLOAD_TIMEOUT_SECONDS
+ ) as response:
+ status = getattr(response, "status", None)
+ if status != 200 or response.geturl() != url:
+ raise FullSuiteError(
+ f"unexpected response for pinned source {relative_path}: "
+ f"status={status!r}, url={response.geturl()!r}"
+ )
+ declared_size = response.headers.get("Content-Length")
+ if (
+ declared_size is not None
+ and int(declared_size) > MAX_SOURCE_BYTES
+ ):
+ raise FullSuiteError(
+ f"pinned source {relative_path} exceeds the size limit"
+ )
+ content = response.read(MAX_SOURCE_BYTES + 1)
+ except (
+ FullSuiteError,
+ OSError,
+ ValueError,
+ urllib.error.URLError,
+ ) as exc:
+ last_error = exc
+ if attempt < DOWNLOAD_ATTEMPTS:
+ time.sleep(DOWNLOAD_RETRY_DELAY_SECONDS)
+ continue
+ break
+ verify_source_content(relative_path, content)
+ return content
+ raise FullSuiteError(
+ f"failed to download pinned source {relative_path} after "
+ f"{DOWNLOAD_ATTEMPTS} attempts: {last_error}"
+ ) from last_error
+
+
+def _validate_sample(content: bytes) -> None:
+ try:
+ text = content.decode("utf-8")
+ except UnicodeDecodeError as exc:
+ raise ValueError("pinned sample.jsonl is not UTF-8") from exc
+ lines = text.splitlines()
+ if len(lines) != EXPECTED_RESULT_COUNT or any(not line.strip() for line in lines):
+ raise ValueError(
+ f"pinned sample.jsonl must contain exactly {EXPECTED_RESULT_COUNT} rows"
+ )
+ for line_number, line in enumerate(lines, 1):
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError as exc:
+ raise ValueError(
+ f"pinned sample.jsonl row {line_number} is invalid JSON"
+ ) from exc
+ if not isinstance(row, dict):
+ raise ValueError(
+ f"pinned sample.jsonl row {line_number} must be an object"
+ )
+
+
+def verify_source_tree(source_dir: Path) -> None:
+ """Verify every required file in an already prepared source directory."""
+ root = source_dir.resolve()
+ for relative_path in REQUIRED_SOURCE_SHA256:
+ path = source_dir / relative_path
+ if path.is_symlink() or not path.is_file():
+ raise ValueError(f"required pinned source is missing: {relative_path}")
+ if not path.resolve().is_relative_to(root):
+ raise ValueError(f"required pinned source escapes runtime: {relative_path}")
+ content = path.read_bytes()
+ verify_source_content(relative_path, content)
+ if relative_path == "sample.jsonl":
+ _validate_sample(content)
+
+
+def prepare_source_tree(source_dir: Path, fetcher: Fetcher = _fetch_source) -> None:
+ """Download only the allowlisted verifier files and verify the complete tree."""
+ source_dir.mkdir(parents=True, exist_ok=False)
+ for relative_path in REQUIRED_SOURCE_SHA256:
+ content = fetcher(relative_path)
+ verify_source_content(relative_path, content)
+ destination = source_dir / relative_path
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ temporary = destination.with_name(f".{destination.name}.tmp")
+ temporary.write_bytes(content)
+ temporary.replace(destination)
+ verify_source_tree(source_dir)
+
+
+def build_verifier_command(
+ *,
+ python: Path,
+ source_dir: Path,
+ base_url: str,
+ model: str,
+ output_dir: Path,
+) -> list[str]:
+ """Build the single pinned-upstream invocation for all 102 rows."""
+ if not isinstance(model, str) or not model.strip():
+ raise ValueError("model must be a non-empty string")
+ normalized_base_url = base_url.strip().rstrip("/")
+ parsed_base_url = urllib.parse.urlsplit(normalized_base_url)
+ if (
+ parsed_base_url.scheme not in {"http", "https"}
+ or not parsed_base_url.netloc
+ or parsed_base_url.query
+ or parsed_base_url.fragment
+ ):
+ raise ValueError(
+ "base_url must be an absolute HTTP(S) URL without query or fragment"
+ )
+ extra_body = json.dumps(
+ {"temperature": 0, "top_p": 1, "max_tokens": 40960},
+ separators=(",", ":"),
+ )
+ return [
+ str(python),
+ str(source_dir / "verify.py"),
+ str(source_dir / "sample.jsonl"),
+ "--model",
+ model,
+ "--base-url",
+ normalized_base_url,
+ "--api-key",
+ "EMPTY",
+ "--concurrency",
+ "5",
+ "--output",
+ str(output_dir / NATIVE_RESULTS_FILENAME),
+ "--summary",
+ str(output_dir / NATIVE_REPORT_FILENAME),
+ "--timeout",
+ "600",
+ "--retries",
+ "3",
+ "--extra-body",
+ extra_body,
+ ]
+
+
+def _error_dict(error: BaseException) -> dict[str, str]:
+ return {"type": type(error).__name__, "message": str(error)}
+
+
+def _compatibility_result(
+ *,
+ model: str,
+ score: float,
+ effective: int,
+ integration_error: BaseException | None = None,
+) -> dict[str, Any]:
+ result: dict[str, Any] = {
+ "result_format": RESULT_FORMAT,
+ "eval_adapter": ADAPTER_NAME,
+ "model_name": model,
+ "results": {
+ TASK_NAME: {
+ "exact_match,strict-match": score,
+ "exact_match_stderr,strict-match": 0.0,
+ }
+ },
+ "configs": {
+ TASK_NAME: {
+ "metric_list": [{"metric": "exact_match"}],
+ "filter_list": [{"name": "strict-match"}],
+ "native_metric": "tool_calls_match_rate",
+ "diagnostic_threshold": 0.0,
+ }
+ },
+ "n-samples": {
+ TASK_NAME: {
+ "original": EXPECTED_RESULT_COUNT,
+ "effective": effective,
+ }
+ },
+ "source": {
+ "repository": "MiniMax-AI/MiniMax-Provider-Verifier",
+ "ref": UPSTREAM_REF,
+ "sample_sha256": EXPECTED_SAMPLE_SHA256,
+ },
+ }
+ if integration_error is not None:
+ result["integration_error"] = _error_dict(integration_error)
+ return result
+
+
+def _compatibility_path(output_dir: Path) -> Path:
+ for stale_path in output_dir.glob(COMPATIBILITY_GLOB):
+ stale_path.unlink()
+ timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f")
+ return output_dir / f"results_minimax_vendor_full_{timestamp}.json"
+
+
+def _write_json(path: Path, value: Mapping[str, Any]) -> None:
+ path.write_text(
+ json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+ )
+
+
+def _read_native_report(path: Path) -> dict[str, Any]:
+ try:
+ value = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as exc:
+ raise FullSuiteError(f"native summary is unavailable or invalid: {exc}") from exc
+ if not isinstance(value, dict):
+ raise FullSuiteError("native summary must be a JSON object")
+ return value
+
+
+def _read_native_results(path: Path) -> list[dict[str, Any]]:
+ try:
+ lines = path.read_text(encoding="utf-8").splitlines()
+ except OSError as exc:
+ raise FullSuiteError(f"native results are unavailable: {exc}") from exc
+ if len(lines) != EXPECTED_RESULT_COUNT or any(not line.strip() for line in lines):
+ raise FullSuiteError(
+ f"native results must contain exactly {EXPECTED_RESULT_COUNT} rows, "
+ f"found {len(lines)}"
+ )
+ results: list[dict[str, Any]] = []
+ for line_number, line in enumerate(lines, 1):
+ try:
+ value = json.loads(line)
+ except json.JSONDecodeError as exc:
+ raise FullSuiteError(
+ f"native result row {line_number} is invalid JSON"
+ ) from exc
+ if not isinstance(value, dict):
+ raise FullSuiteError(
+ f"native result row {line_number} must be a JSON object"
+ )
+ results.append(value)
+ return results
+
+
+def _count(value: Any, name: str) -> int:
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+ raise FullSuiteError(f"native summary {name} must be a non-negative integer")
+ return value
+
+
+def _rate(value: Any, name: str) -> float:
+ if (
+ isinstance(value, bool)
+ or not isinstance(value, (int, float))
+ or not math.isfinite(value)
+ or not 0.0 <= value <= 1.0
+ ):
+ raise FullSuiteError(f"native summary {name} must be a finite rate")
+ return float(value)
+
+
+def project_native_artifacts(*, output_dir: Path, model: str) -> Path:
+ """Validate a complete native run and emit its compatibility projection."""
+ report = _read_native_report(output_dir / NATIVE_REPORT_FILENAME)
+ results = _read_native_results(output_dir / NATIVE_RESULTS_FILENAME)
+ indices = [row.get("data_index") for row in results]
+ if indices != list(range(1, EXPECTED_RESULT_COUNT + 1)):
+ raise FullSuiteError("native results must retain ordered data_index values 1..102")
+ failed_indices = [
+ row["data_index"] for row in results if row.get("status") != "success"
+ ]
+ if failed_indices:
+ raise FullSuiteError(
+ "native verifier has transport failures at data_index "
+ + ", ".join(str(index) for index in failed_indices)
+ )
+ success_count = _count(report.get("success_count"), "success_count")
+ failure_count = _count(report.get("failure_count"), "failure_count")
+ if success_count != EXPECTED_RESULT_COUNT or failure_count != 0:
+ raise FullSuiteError(
+ "native summary is incomplete: "
+ f"success_count={success_count}, failure_count={failure_count}"
+ )
+ if report.get("model") != model:
+ raise FullSuiteError("native summary model does not match the requested model")
+ score = _rate(report.get("tool_calls_match_rate"), "tool_calls_match_rate")
+ compatibility_path = _compatibility_path(output_dir)
+ _write_json(
+ compatibility_path,
+ _compatibility_result(
+ model=model,
+ score=score,
+ effective=EXPECTED_RESULT_COUNT,
+ ),
+ )
+ return compatibility_path
+
+
+def publish_failure(*, output_dir: Path, model: str, error: BaseException) -> Path:
+ """Publish canonical failure metadata while retaining partial result rows."""
+ output_dir.mkdir(parents=True, exist_ok=True)
+ native_report_path = output_dir / NATIVE_REPORT_FILENAME
+ native_results_path = output_dir / NATIVE_RESULTS_FILENAME
+ _write_json(
+ native_report_path,
+ {
+ "verifier": ADAPTER_NAME,
+ "task": TASK_NAME,
+ "model": model,
+ "completed": False,
+ "threshold": 0.0,
+ "source": {
+ "ref": UPSTREAM_REF,
+ "sample_sha256": EXPECTED_SAMPLE_SHA256,
+ },
+ "success_count": 0,
+ "failure_count": EXPECTED_RESULT_COUNT,
+ "tool_calls_match_rate": 0.0,
+ "integration_error": _error_dict(error),
+ },
+ )
+ if not native_results_path.exists():
+ native_results_path.write_text(
+ json.dumps(
+ {
+ "status": "integration_error",
+ "model": model,
+ "error": _error_dict(error),
+ },
+ ensure_ascii=False,
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ compatibility_path = _compatibility_path(output_dir)
+ _write_json(
+ compatibility_path,
+ _compatibility_result(
+ model=model,
+ score=0.0,
+ effective=0,
+ integration_error=error,
+ ),
+ )
+ return compatibility_path
+
+
+def run_full_suite(
+ *,
+ python: Path,
+ source_dir: Path,
+ dependency_dir: Path,
+ base_url: str,
+ model: str,
+ output_dir: Path,
+ runner: Runner = subprocess.run,
+) -> bool:
+ """Run upstream once, then classify transport versus diagnostic failures."""
+ output_dir.mkdir(parents=True, exist_ok=True)
+ for filename in (NATIVE_REPORT_FILENAME, NATIVE_RESULTS_FILENAME):
+ (output_dir / filename).unlink(missing_ok=True)
+ for stale_path in output_dir.glob(COMPATIBILITY_GLOB):
+ stale_path.unlink()
+ try:
+ verify_source_tree(source_dir)
+ command = build_verifier_command(
+ python=python,
+ source_dir=source_dir,
+ base_url=base_url,
+ model=model,
+ output_dir=output_dir,
+ )
+ environment = os.environ.copy()
+ environment["PYTHONPATH"] = os.pathsep.join(
+ (str(source_dir), str(dependency_dir))
+ )
+ environment["PYTHONNOUSERSITE"] = "1"
+ completed = runner(
+ command,
+ env=environment,
+ timeout=UPSTREAM_TIMEOUT_SECONDS,
+ check=False,
+ )
+ if completed.returncode != 0:
+ raise FullSuiteError(
+ f"pinned upstream verifier exited with code {completed.returncode}"
+ )
+ project_native_artifacts(output_dir=output_dir, model=model)
+ except (OSError, ValueError, FullSuiteError, subprocess.TimeoutExpired) as exc:
+ publish_failure(output_dir=output_dir, model=model, error=exc)
+ return False
+ return True
+
+
+def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Run or project the pinned full MiniMax M3 verifier."
+ )
+ subparsers = parser.add_subparsers(dest="command", required=True)
+
+ prepare = subparsers.add_parser("prepare-source")
+ prepare.add_argument("--source-dir", required=True, type=Path)
+
+ run = subparsers.add_parser("run")
+ run.add_argument("--python", required=True, type=Path)
+ run.add_argument("--source-dir", required=True, type=Path)
+ run.add_argument("--dependency-dir", required=True, type=Path)
+ run.add_argument("--base-url", required=True)
+ run.add_argument("--model", required=True)
+ run.add_argument("--output-dir", required=True, type=Path)
+
+ project = subparsers.add_parser("project")
+ project.add_argument("--model", required=True)
+ project.add_argument("--output-dir", required=True, type=Path)
+
+ failure = subparsers.add_parser("failure")
+ failure.add_argument("--model", required=True)
+ failure.add_argument("--output-dir", required=True, type=Path)
+ failure.add_argument("--message", required=True)
+ return parser.parse_args(argv)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = parse_args(argv)
+ if args.command == "prepare-source":
+ try:
+ prepare_source_tree(args.source_dir)
+ except (OSError, ValueError, FullSuiteError) as exc:
+ print(f"ERROR: {exc}", file=os.sys.stderr)
+ return 1
+ return 0
+ if args.command == "failure":
+ (args.output_dir / NATIVE_RESULTS_FILENAME).unlink(missing_ok=True)
+ publish_failure(
+ output_dir=args.output_dir,
+ model=args.model,
+ error=FullSuiteError(args.message),
+ )
+ return 0
+ if args.command == "project":
+ try:
+ project_native_artifacts(output_dir=args.output_dir, model=args.model)
+ except (OSError, ValueError, FullSuiteError) as exc:
+ publish_failure(output_dir=args.output_dir, model=args.model, error=exc)
+ print(f"ERROR: {exc}", file=os.sys.stderr)
+ return 1
+ return 0
+ completed = run_full_suite(
+ python=args.python,
+ source_dir=args.source_dir,
+ dependency_dir=args.dependency_dir,
+ base_url=args.base_url,
+ model=args.model,
+ output_dir=args.output_dir,
+ )
+ return 0 if completed else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/utils/evals/minimax_m3_smoke.json b/utils/evals/minimax_m3_smoke.json
new file mode 100644
index 0000000000..b5a3eeb239
--- /dev/null
+++ b/utils/evals/minimax_m3_smoke.json
@@ -0,0 +1,99 @@
+{
+ "source": "https://raw.githubusercontent.com/MiniMax-AI/MiniMax-Provider-Verifier/c899f95e17bfc4a338ddd4cb1638279125885e55/sample.jsonl",
+ "ref": "c899f95e17bfc4a338ddd4cb1638279125885e55",
+ "indices": [
+ 71
+ ],
+ "license": "MIT License\n\nCopyright (c) 2025 MiniMax Provider Verifier Contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.",
+ "rows": [
+ {
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are MiniMax M2, an AI assistant developed by Minimax. You are a helpful assistant."
+ },
+ {
+ "role": "user",
+ "content": "Run a duplicate therapy check for patient P-009417 using the medication list at /mnt/clinical/emr/patients/P-009417/med_list_v3.json. Flag duplicates at the RxNorm ingredient level, consider only active meds (ignore discontinued and on_hold), include OTC items, require the same route to count as a duplicate, and return a detailed JSON report."
+ }
+ ],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "flag_duplicate_therapies",
+ "description": "Analyze a patient's medication list to identify potential duplicate therapies by ingredient or class. Supports RxNorm and ATC classification, status filtering, route matching, and OTC inclusion. Returns structured results.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "patient_id": {
+ "type": "string",
+ "description": "Patient identifier"
+ },
+ "med_list_path": {
+ "type": "string",
+ "description": "Absolute file path to the medication list (JSON or CSV)"
+ },
+ "classification_scheme": {
+ "type": "string",
+ "description": "Classification used for duplicate detection",
+ "enum": [
+ "rxnorm_ingredient",
+ "rxnorm_clinical_drug",
+ "atc_class"
+ ]
+ },
+ "overlap_policy": {
+ "type": "string",
+ "description": "How to determine concurrent therapy",
+ "enum": [
+ "current_only",
+ "date_overlap"
+ ]
+ },
+ "concurrency_window_days": {
+ "type": "integer",
+ "description": "If overlap_policy is date_overlap, the window in days to consider therapies overlapping (0 means exact date overlap)",
+ "minimum": 0
+ },
+ "strict_route_matching": {
+ "type": "boolean",
+ "description": "Flag as duplicate only when routes match (e.g., oral vs inhalation)"
+ },
+ "include_otc": {
+ "type": "boolean",
+ "description": "Include over-the-counter medications in the analysis"
+ },
+ "ignore_statuses": {
+ "type": "array",
+ "description": "Medication statuses to exclude from analysis",
+ "items": {
+ "type": "string",
+ "enum": [
+ "active",
+ "on_hold",
+ "discontinued"
+ ]
+ }
+ },
+ "output_format": {
+ "type": "string",
+ "description": "Output report format",
+ "enum": [
+ "summary",
+ "detailed_json"
+ ]
+ }
+ },
+ "required": [
+ "patient_id",
+ "med_list_path"
+ ]
+ }
+ }
+ }
+ ],
+ "expected_tool_call": true
+ }
+ ]
+}
diff --git a/utils/evals/minimax_provider_eval.py b/utils/evals/minimax_provider_eval.py
new file mode 100755
index 0000000000..077258e4e0
--- /dev/null
+++ b/utils/evals/minimax_provider_eval.py
@@ -0,0 +1,408 @@
+#!/usr/bin/env python3
+"""Run a pinned MiniMax M3 smoke subset through the stock provider verifier."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import subprocess
+import urllib.parse
+from collections.abc import Callable, Mapping, Sequence
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+if __package__:
+ from .minimax_m3_full_eval import UPSTREAM_REF, verify_source_tree
+else:
+ from minimax_m3_full_eval import UPSTREAM_REF, verify_source_tree
+
+TASK_NAME = "minimax_m3_smoke"
+NATIVE_REPORT_FILENAME = "minimax_vendor_report.json"
+NATIVE_RESULTS_FILENAME = "minimax_vendor_results.jsonl"
+COMPATIBILITY_GLOB = "results_minimax_vendor_*.json"
+DEFAULT_FIXTURE_PATH = Path(__file__).with_name("minimax_m3_smoke.json")
+RESULT_FORMAT = "inferencex-eval-v1"
+ADAPTER_NAME = "minimax-provider-verifier"
+EXPECTED_INDICES = (71,)
+EXPECTED_LICENSE_SHA256 = (
+ "aa7cec386fcb5e555aba0e8b1c31307940af41967708c9bc0f78b4e02e235dd5"
+)
+EXPECTED_CASE_SHA256 = {
+ 71: "3d51571a1ed7d0bb644c3ae978ef5822b3150479b1e34bbbae7276f671657870",
+}
+UPSTREAM_SOURCE = (
+ "https://raw.githubusercontent.com/MiniMax-AI/MiniMax-Provider-Verifier/"
+ f"{UPSTREAM_REF}/sample.jsonl"
+)
+UPSTREAM_TIMEOUT_SECONDS = 60 * 60
+
+Runner = Callable[..., subprocess.CompletedProcess[Any]]
+
+
+class SmokeSuiteError(RuntimeError):
+ """The stock verifier could not produce one complete smoke result."""
+
+
+def _mapping(value: Any, name: str) -> Mapping[str, Any]:
+ if not isinstance(value, Mapping):
+ raise ValueError(f"{name} must be an object")
+ return value
+
+
+def load_fixture(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+ """Load the exact pinned row without modifying its request fields."""
+ root = _mapping(json.loads(path.read_text(encoding="utf-8")), "fixture")
+ if root.get("source") != UPSTREAM_SOURCE or root.get("ref") != UPSTREAM_REF:
+ raise ValueError("fixture source or ref does not match the pinned upstream")
+ if root.get("indices") != list(EXPECTED_INDICES):
+ raise ValueError("fixture indices must be exactly [71]")
+ license_text = root.get("license")
+ if (
+ not isinstance(license_text, str)
+ or hashlib.sha256(license_text.encode()).hexdigest() != EXPECTED_LICENSE_SHA256
+ ):
+ raise ValueError("fixture must preserve the complete upstream MIT notice")
+
+ raw_rows = root.get("rows")
+ if not isinstance(raw_rows, list) or len(raw_rows) != 1:
+ raise ValueError("fixture must contain exactly one row")
+ row = dict(_mapping(raw_rows[0], "fixture.rows[0]"))
+ if "data_index" in row:
+ raise ValueError("fixture request must not contain adapter-only data_index")
+ digest = hashlib.sha256(
+ json.dumps(row, ensure_ascii=False, separators=(",", ":")).encode()
+ ).hexdigest()
+ if digest != EXPECTED_CASE_SHA256[EXPECTED_INDICES[0]]:
+ raise ValueError("fixture row 71 differs from pinned upstream")
+ return dict(root), [row]
+
+
+def _normalized_base_url(value: str) -> str:
+ if not isinstance(value, str) or not value.strip():
+ raise ValueError("base_url must be a non-empty string")
+ normalized = value.strip().rstrip("/")
+ parsed = urllib.parse.urlsplit(normalized)
+ if (
+ parsed.scheme not in {"http", "https"}
+ or not parsed.netloc
+ or parsed.query
+ or parsed.fragment
+ ):
+ raise ValueError(
+ "base_url must be an absolute HTTP(S) URL without query or fragment"
+ )
+ return normalized
+
+
+def prepare_smoke_input(*, fixture_path: Path, destination: Path) -> None:
+ """Write the pinned row as stock verifier JSONL input."""
+ _, rows = load_fixture(fixture_path)
+ destination.write_text(
+ "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
+ encoding="utf-8",
+ )
+
+
+def build_verifier_command(
+ *,
+ python: Path,
+ source_dir: Path,
+ sample_path: Path,
+ base_url: str,
+ model: str,
+ output_dir: Path,
+) -> list[str]:
+ """Build one stock verify.py invocation over the pinned smoke row."""
+ if not isinstance(model, str) or not model.strip():
+ raise ValueError("model must be a non-empty string")
+ extra_body = json.dumps(
+ {"temperature": 0, "top_p": 1, "max_tokens": 40960},
+ separators=(",", ":"),
+ )
+ return [
+ str(python),
+ str(source_dir / "verify.py"),
+ str(sample_path),
+ "--model",
+ model,
+ "--base-url",
+ _normalized_base_url(base_url),
+ "--api-key",
+ "EMPTY",
+ "--concurrency",
+ "1",
+ "--output",
+ str(output_dir / NATIVE_RESULTS_FILENAME),
+ "--summary",
+ str(output_dir / NATIVE_REPORT_FILENAME),
+ "--timeout",
+ "600",
+ "--retries",
+ "3",
+ "--extra-body",
+ extra_body,
+ ]
+
+
+def _error_dict(error: BaseException) -> dict[str, str]:
+ return {"type": type(error).__name__, "message": str(error)}
+
+
+def _rate(value: Any, name: str) -> float:
+ if (
+ isinstance(value, bool)
+ or not isinstance(value, (int, float))
+ or not math.isfinite(value)
+ or not 0.0 <= value <= 1.0
+ ):
+ raise SmokeSuiteError(f"native summary {name} must be a finite rate")
+ return float(value)
+
+
+def _nonnegative_count(value: Any, name: str) -> int:
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+ raise SmokeSuiteError(f"native summary {name} must be a non-negative integer")
+ return value
+
+
+def _compatibility_path(output_dir: Path) -> Path:
+ for stale_path in output_dir.glob(COMPATIBILITY_GLOB):
+ stale_path.unlink()
+ timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f")
+ return output_dir / f"results_minimax_vendor_{timestamp}.json"
+
+
+def _compatibility_result(
+ *,
+ model: str,
+ score: float,
+ effective: int,
+ integration_error: BaseException | None = None,
+) -> dict[str, Any]:
+ result: dict[str, Any] = {
+ "result_format": RESULT_FORMAT,
+ "eval_adapter": ADAPTER_NAME,
+ "model_name": model,
+ "results": {
+ TASK_NAME: {
+ "exact_match,strict-match": score,
+ "exact_match_stderr,strict-match": 0.0,
+ }
+ },
+ "configs": {
+ TASK_NAME: {
+ "metric_list": [{"metric": "exact_match"}],
+ "filter_list": [{"name": "strict-match"}],
+ "native_metrics": [
+ "tool_calls_match_rate",
+ "tool_calls_schema_validation_error_count",
+ "tool_calls_total_count",
+ "error_only_reasoning_rate",
+ ],
+ }
+ },
+ "n-samples": {
+ TASK_NAME: {"original": 1, "effective": effective},
+ },
+ "source": {
+ "repository": "MiniMax-AI/MiniMax-Provider-Verifier",
+ "ref": UPSTREAM_REF,
+ "indices": list(EXPECTED_INDICES),
+ },
+ }
+ if integration_error is not None:
+ result["integration_error"] = _error_dict(integration_error)
+ return result
+
+
+def _write_json(path: Path, value: Mapping[str, Any]) -> None:
+ path.write_text(
+ json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+ )
+
+
+def project_native_artifacts(*, output_dir: Path, model: str) -> Path:
+ """Validate stock outputs and project only their published metrics."""
+ report = _mapping(
+ json.loads((output_dir / NATIVE_REPORT_FILENAME).read_text(encoding="utf-8")),
+ "native summary",
+ )
+ result_lines = (
+ (output_dir / NATIVE_RESULTS_FILENAME).read_text(encoding="utf-8").splitlines()
+ )
+ if len(result_lines) != 1 or not result_lines[0].strip():
+ raise SmokeSuiteError("native results must contain exactly one row")
+ result = _mapping(json.loads(result_lines[0]), "native result")
+ if result.get("data_index") != 1:
+ raise SmokeSuiteError("native result must identify the one-row smoke input")
+ if result.get("status") != "success":
+ raise SmokeSuiteError("native verifier reported a request failure")
+ if report.get("model") != model:
+ raise SmokeSuiteError("native summary model does not match the requested model")
+ if report.get("success_count") != 1 or report.get("failure_count") != 0:
+ raise SmokeSuiteError("native summary does not describe one successful request")
+
+ match_rate = _rate(report.get("tool_calls_match_rate"), "tool_calls_match_rate")
+ schema_errors = _nonnegative_count(
+ report.get("tool_calls_schema_validation_error_count"),
+ "tool_calls_schema_validation_error_count",
+ )
+ tool_call_total = _nonnegative_count(
+ report.get("tool_calls_total_count"), "tool_calls_total_count"
+ )
+ if schema_errors > tool_call_total:
+ raise SmokeSuiteError(
+ "native summary tool-call schema counts are inconsistent"
+ )
+ schema_rate = (
+ 0.0 if tool_call_total == 0 else 1.0 - (schema_errors / tool_call_total)
+ )
+ reasoning_error_rate = _rate(
+ report.get("error_only_reasoning_rate"), "error_only_reasoning_rate"
+ )
+ score = min(match_rate, schema_rate, 1.0 - reasoning_error_rate)
+ compatibility_path = _compatibility_path(output_dir)
+ _write_json(
+ compatibility_path,
+ _compatibility_result(model=model, score=score, effective=1),
+ )
+ return compatibility_path
+
+
+def publish_failure(*, output_dir: Path, model: str, error: BaseException) -> Path:
+ """Publish integration metadata without rewriting stock artifacts."""
+ output_dir.mkdir(parents=True, exist_ok=True)
+ native_report_path = output_dir / NATIVE_REPORT_FILENAME
+ if not native_report_path.exists():
+ _write_json(
+ native_report_path,
+ {
+ "verifier": ADAPTER_NAME,
+ "task": TASK_NAME,
+ "model": model,
+ "completed": False,
+ "source": {"ref": UPSTREAM_REF, "indices": list(EXPECTED_INDICES)},
+ "integration_error": _error_dict(error),
+ },
+ )
+ compatibility_path = _compatibility_path(output_dir)
+ _write_json(
+ compatibility_path,
+ _compatibility_result(
+ model=model,
+ score=0.0,
+ effective=0,
+ integration_error=error,
+ ),
+ )
+ return compatibility_path
+
+
+def run_evaluation(
+ *,
+ python: Path,
+ source_dir: Path,
+ dependency_dir: Path,
+ base_url: str,
+ model: str,
+ output_dir: Path,
+ fixture_path: Path = DEFAULT_FIXTURE_PATH,
+ runner: Runner = subprocess.run,
+) -> bool:
+ """Run the stock upstream verifier once, then project its native metrics."""
+ output_dir.mkdir(parents=True, exist_ok=True)
+ for filename in (NATIVE_REPORT_FILENAME, NATIVE_RESULTS_FILENAME):
+ (output_dir / filename).unlink(missing_ok=True)
+ for stale_path in output_dir.glob(COMPATIBILITY_GLOB):
+ stale_path.unlink()
+ smoke_input = output_dir / "minimax_vendor_smoke_input.jsonl"
+ smoke_input.unlink(missing_ok=True)
+ try:
+ verify_source_tree(source_dir)
+ prepare_smoke_input(fixture_path=fixture_path, destination=smoke_input)
+ command = build_verifier_command(
+ python=python,
+ source_dir=source_dir,
+ sample_path=smoke_input,
+ base_url=base_url,
+ model=model,
+ output_dir=output_dir,
+ )
+ environment = os.environ.copy()
+ environment["PYTHONPATH"] = os.pathsep.join(
+ (str(source_dir), str(dependency_dir))
+ )
+ environment["PYTHONNOUSERSITE"] = "1"
+ completed = runner(
+ command,
+ env=environment,
+ timeout=UPSTREAM_TIMEOUT_SECONDS,
+ check=False,
+ )
+ if completed.returncode != 0:
+ raise SmokeSuiteError(
+ f"pinned upstream verifier exited with code {completed.returncode}"
+ )
+ project_native_artifacts(output_dir=output_dir, model=model)
+ except (
+ OSError,
+ ValueError,
+ SmokeSuiteError,
+ subprocess.TimeoutExpired,
+ ) as exc:
+ publish_failure(output_dir=output_dir, model=model, error=exc)
+ return False
+ finally:
+ smoke_input.unlink(missing_ok=True)
+ return True
+
+
+def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Run the pinned MiniMax M3 smoke through stock verify.py."
+ )
+ subparsers = parser.add_subparsers(dest="command", required=True)
+
+ run = subparsers.add_parser("run")
+ run.add_argument("--python", required=True, type=Path)
+ run.add_argument("--source-dir", required=True, type=Path)
+ run.add_argument("--dependency-dir", required=True, type=Path)
+ run.add_argument("--base-url", required=True)
+ run.add_argument("--model", required=True)
+ run.add_argument("--output-dir", required=True, type=Path)
+ run.add_argument("--fixture", type=Path, default=DEFAULT_FIXTURE_PATH)
+
+ failure = subparsers.add_parser("failure")
+ failure.add_argument("--model", required=True)
+ failure.add_argument("--output-dir", required=True, type=Path)
+ failure.add_argument("--message", required=True)
+ return parser.parse_args(argv)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = parse_args(argv)
+ if args.command == "failure":
+ publish_failure(
+ output_dir=args.output_dir,
+ model=args.model,
+ error=SmokeSuiteError(args.message),
+ )
+ return 0
+ completed = run_evaluation(
+ python=args.python,
+ source_dir=args.source_dir,
+ dependency_dir=args.dependency_dir,
+ base_url=args.base_url,
+ model=args.model,
+ output_dir=args.output_dir,
+ fixture_path=args.fixture,
+ )
+ return 0 if completed else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/utils/evals/test_batched_eval.py b/utils/evals/test_batched_eval.py
index a5d6df0085..a4aace9389 100644
--- a/utils/evals/test_batched_eval.py
+++ b/utils/evals/test_batched_eval.py
@@ -227,6 +227,106 @@ def test_validate_scores_checks_threshold_for_every_concurrency(
assert "FAIL: [conc=4] gsm8k exact_match,strict-match" in captured.err
+def test_validate_scores_reports_integration_failure_without_thresholding(
+ tmp_path: Path,
+ monkeypatch,
+ capsys,
+) -> None:
+ result_path = tmp_path / "results_test.json"
+ result_path.write_text(json.dumps({
+ "integration_error": {
+ "type": "RuntimeError",
+ "message": "vendor verifier checkout failed",
+ },
+ "results": {
+ "gsm8k": {
+ "exact_match,strict-match": 0.0,
+ },
+ },
+ "n-samples": {"gsm8k": {"effective": 0}},
+ }))
+ monkeypatch.setattr(sys, "argv", [
+ "validate_scores.py",
+ "--meta-env",
+ str(tmp_path / "meta_env.json"),
+ "--results-glob",
+ str(result_path),
+ ])
+
+ assert validate_scores_main() == 1
+ captured = capsys.readouterr()
+ assert "integration failure: RuntimeError: vendor verifier checkout failed" in captured.err
+ assert "gsm8k exact_match,strict-match" not in captured.err
+
+
+def test_validate_scores_rejects_invalid_effective_count_without_thresholding(
+ tmp_path: Path,
+ monkeypatch,
+ capsys,
+) -> None:
+ result_path = tmp_path / "results_test.json"
+ result_path.write_text(json.dumps({
+ "results": {
+ "gsm8k": {
+ "exact_match,strict-match": 0.0,
+ },
+ "other": {
+ "exact_match,strict-match": 0.0,
+ },
+ "nonfinite": {
+ "exact_match,strict-match": 1.0,
+ },
+ },
+ "n-samples": {
+ "gsm8k": {"effective": "unknown"},
+ "other": {"effective": 0},
+ "nonfinite": {"effective": float("inf")},
+ },
+ }))
+ monkeypatch.setattr(sys, "argv", [
+ "validate_scores.py",
+ "--meta-env",
+ str(tmp_path / "meta_env.json"),
+ "--results-glob",
+ str(result_path),
+ ])
+
+ assert validate_scores_main() == 1
+ captured = capsys.readouterr()
+ assert "gsm8k invalid effective sample count: 'unknown'" in captured.err
+ assert "gsm8k exact_match,strict-match" not in captured.err
+ assert "other invalid effective sample count: 0" in captured.err
+ assert "other exact_match,strict-match" not in captured.err
+
+ assert "nonfinite invalid effective sample count: inf" in captured.err
+ assert "nonfinite exact_match,strict-match" not in captured.err
+
+def test_validate_scores_accepts_legacy_result_without_effective_count(
+ tmp_path: Path,
+ monkeypatch,
+ capsys,
+) -> None:
+ result_path = tmp_path / "results_test.json"
+ result_path.write_text(json.dumps({
+ "results": {
+ "gsm8k": {
+ "exact_match,strict-match": 1.0,
+ },
+ },
+ }))
+ monkeypatch.setattr(sys, "argv", [
+ "validate_scores.py",
+ "--meta-env",
+ str(tmp_path / "meta_env.json"),
+ "--results-glob",
+ str(result_path),
+ ])
+
+ assert validate_scores_main() == 0
+ captured = capsys.readouterr()
+ assert "PASS: gsm8k exact_match,strict-match" in captured.out
+
+
def test_amd_multinode_container_forwards_eval_concurrency_list() -> None:
job_slurm = (
Path(__file__).resolve().parents[2]
@@ -248,3 +348,4 @@ def test_amd_multinode_container_forwards_eval_concurrency_list() -> None:
).read_text()
assert 'expected_concs="${EVAL_CONC}"' in workflow
assert 'validate_scores.py --expected-concs "${expected_concs}"' in workflow
+ assert "--metric-prefix" not in workflow
diff --git a/utils/evals/test_bfcl_eval.py b/utils/evals/test_bfcl_eval.py
new file mode 100644
index 0000000000..dac3ffa17d
--- /dev/null
+++ b/utils/evals/test_bfcl_eval.py
@@ -0,0 +1,1029 @@
+import builtins
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from types import ModuleType
+from typing import Any
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import bfcl_adapter as be
+import validate_scores as vs
+
+
+def _compatibility_path(output_dir: Path) -> Path:
+ path = output_dir / be.COMPATIBILITY_FILENAME
+ assert path.exists()
+ return path
+
+
+def _compatibility(output_dir: Path) -> dict[str, Any]:
+ return json.loads(_compatibility_path(output_dir).read_text(encoding="utf-8"))
+
+
+def _native(output_dir: Path) -> dict[str, Any]:
+ return json.loads(
+ (output_dir / be.NATIVE_REPORT_FILENAME).read_text(encoding="utf-8")
+ )
+
+
+def _score(output_dir: Path) -> float:
+ return _compatibility(output_dir)["results"][be.TASK_NAME]["acc,none"]
+
+
+def _category_score(
+ category: str,
+ total_count: int,
+ correct_count: int,
+) -> be.CategoryScore:
+ case_ids = tuple(f"{category}_{index}" for index in range(total_count))
+ return be.CategoryScore(
+ category=category,
+ case_ids=case_ids,
+ score_file=f"score/model-a/BFCL_v4_{category}_score.json",
+ header={
+ "accuracy": correct_count / total_count,
+ "correct_count": correct_count,
+ "total_count": total_count,
+ },
+ records=tuple({"id": case_id} for case_id in case_ids[correct_count:]),
+ accuracy=correct_count / total_count,
+ correct_count=correct_count,
+ total_count=total_count,
+ )
+
+
+def test_thresholds_are_stdlib_readable_without_pyyaml(monkeypatch) -> None:
+ real_import = builtins.__import__
+
+ def import_without_yaml(name, *args, **kwargs):
+ if name == "yaml":
+ raise ModuleNotFoundError("No module named 'yaml'")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", import_without_yaml)
+ thresholds = vs.load_config(str(Path(vs.__file__).with_name("thresholds.yaml")))
+
+ assert thresholds["default"]["bfcl_smoke"] == 0.0
+ assert thresholds["default"]["bfcl_parallel"] == 0.0
+
+
+def test_score_validator_uses_declared_bfcl_metric(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ result = {
+ "results": {"bfcl_smoke": {"acc,none": 0.8, "acc_stderr,none": 0.1}},
+ "configs": {
+ "bfcl_smoke": {"metric_list": [{"metric": "acc", "aggregation": "mean"}]}
+ },
+ "n-samples": {"bfcl_smoke": {"original": 4, "effective": 4}},
+ }
+ (tmp_path / "results_bfcl.json").write_text(json.dumps(result))
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(sys, "argv", ["validate_scores.py"])
+
+ assert vs.main() == 0
+
+
+def test_adapter_module_does_not_collide_with_upstream_package(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ upstream = object()
+ child = object()
+ monkeypatch.setitem(sys.modules, "bfcl_eval", upstream)
+ monkeypatch.setitem(sys.modules, "bfcl_eval.constants", child)
+
+ be._clear_upstream_modules()
+
+ assert be.__name__ == "bfcl_adapter"
+ assert "bfcl_eval" not in sys.modules
+ assert "bfcl_eval.constants" not in sys.modules
+
+
+def _write_result(
+ project_root: Path,
+ category: str,
+ rows: list[dict[str, Any]] | None = None,
+) -> Path:
+ result_path = (
+ project_root
+ / "result"
+ / "model-a"
+ / "nested"
+ / f"BFCL_v4_{category}_result.json"
+ )
+ result_path.parent.mkdir(parents=True, exist_ok=True)
+ if rows is None:
+ rows = [{"id": be.SMOKE_CASE_IDS[category][0], "result": []}]
+ result_path.write_text(
+ "".join(json.dumps(row) + "\n" for row in rows),
+ encoding="utf-8",
+ )
+ return result_path
+
+
+def _write_score(
+ project_root: Path,
+ category: str,
+ correct_count: int,
+ *,
+ header_override: str | None = None,
+ record_id: str | None = None,
+) -> None:
+ score_path = (
+ project_root / "score" / "model-a" / "nested" / f"BFCL_v4_{category}_score.json"
+ )
+ score_path.parent.mkdir(parents=True, exist_ok=True)
+ if header_override is not None:
+ first_line = header_override
+ else:
+ first_line = json.dumps(
+ {
+ "accuracy": float(correct_count),
+ "correct_count": correct_count,
+ "total_count": 1,
+ }
+ )
+ record_text = ""
+ if correct_count == 0 or record_id is not None:
+ case_id = be.SMOKE_CASE_IDS[category][0] if record_id is None else record_id
+ record_text = json.dumps({"id": case_id}) + "\n"
+ score_path.write_text(first_line + "\n" + record_text, encoding="utf-8")
+ _write_result(project_root, category)
+
+
+def _score_runner(
+ correct_by_category: dict[str, int] | None = None,
+ *,
+ missing: str | None = None,
+ malformed: str | None = None,
+ invocation: dict[str, Any] | None = None,
+):
+ expected = correct_by_category or {category: 1 for category in be.SMOKE_CASE_IDS}
+
+ def run(**kwargs: Any) -> None:
+ if invocation is not None:
+ invocation.update(kwargs)
+ project_root = kwargs["project_root"]
+ for category in be.SMOKE_CASE_IDS:
+ if category == missing:
+ _write_result(project_root, category)
+ continue
+ _write_score(
+ project_root,
+ category,
+ expected[category],
+ header_override="{not-json" if category == malformed else None,
+ )
+
+ return run
+
+
+def _run(
+ tmp_path: Path,
+ runner,
+ *,
+ output_dir: Path | None = None,
+) -> tuple[bool, Path, Path]:
+ output = output_dir or tmp_path / "output"
+ project_root = tmp_path / "bfcl-project"
+ passed = be.run_evaluation(
+ base_url="http://127.0.0.1:8000/v1/",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output,
+ bfcl_project_root=project_root,
+ upstream_runner=runner,
+ )
+ return passed, output, project_root
+
+
+def test_command_defaults_and_required_runtime_inputs(tmp_path: Path) -> None:
+ args = be.parse_args(
+ [
+ "--base-url",
+ "http://localhost:8000/v1/",
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(tmp_path / "output"),
+ "--bfcl-project-root",
+ str(tmp_path / "bfcl"),
+ ]
+ )
+
+ assert args.base_url == "http://localhost:8000/v1"
+ assert args.api_key == "EMPTY"
+ assert args.num_threads == 4
+ assert args.integration_error is None
+ assert args.suite == be.TASK_NAME
+ with pytest.raises(SystemExit):
+ be.parse_args(["--model", "model-a", "--output-dir", str(tmp_path / "missing")])
+
+
+def test_cli_selects_suite_defaults_and_rejects_unknown_suite(tmp_path: Path) -> None:
+ args = be.parse_args(
+ [
+ "--base-url",
+ "http://localhost:8000/v1",
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(tmp_path / "output"),
+ "--bfcl-project-root",
+ str(tmp_path / "bfcl"),
+ "--suite",
+ "bfcl_vllm_kimi",
+ ]
+ )
+
+ assert args.suite == "bfcl_vllm_kimi"
+ assert args.num_threads == 16
+
+ with pytest.raises(SystemExit):
+ be.parse_args(
+ [
+ "--base-url",
+ "http://localhost:8000/v1",
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(tmp_path / "output"),
+ "--bfcl-project-root",
+ str(tmp_path / "bfcl"),
+ "--suite",
+ "bfcl_unknown",
+ ]
+ )
+
+
+def test_cli_forwards_selected_suite_on_normal_path(
+ tmp_path: Path, monkeypatch
+) -> None:
+ invocation: dict[str, Any] = {}
+
+ def run_evaluation(**kwargs: Any) -> bool:
+ invocation.update(kwargs)
+ return True
+
+ monkeypatch.setattr(be, "run_evaluation", run_evaluation)
+
+ return_code = be.main(
+ [
+ "--base-url",
+ "http://localhost:8000/v1",
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(tmp_path / "output"),
+ "--bfcl-project-root",
+ str(tmp_path / "bfcl"),
+ "--suite",
+ "bfcl_vllm_kimi",
+ ]
+ )
+
+ assert return_code == 0
+ assert invocation["suite"] is be.KIMI_SUITE
+ assert invocation["num_threads"] == 16
+
+
+@pytest.mark.parametrize(("flag", "value"), (("--num-threads", "0"),))
+def test_cli_rejects_invalid_positive_values(
+ tmp_path: Path, flag: str, value: str
+) -> None:
+ with pytest.raises(SystemExit):
+ be.parse_args(
+ [
+ "--base-url",
+ "http://localhost/v1",
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(tmp_path / "output"),
+ "--bfcl-project-root",
+ str(tmp_path / "bfcl"),
+ flag,
+ value,
+ ]
+ )
+
+
+@pytest.mark.parametrize(
+ "base_url",
+ (
+ "http://localhost/v1/chat/completions",
+ "http://localhost/v1?mode=test",
+ "http://localhost/v1#fragment",
+ ),
+)
+def test_cli_rejects_invalid_api_root(tmp_path: Path, base_url: str) -> None:
+ with pytest.raises(SystemExit):
+ be.parse_args(
+ [
+ "--base-url",
+ base_url,
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(tmp_path / "output"),
+ "--bfcl-project-root",
+ str(tmp_path / "bfcl"),
+ ]
+ )
+
+
+def test_perfect_score_projects_pinned_ids_and_upstream_headers(
+ tmp_path: Path,
+) -> None:
+ invocation: dict[str, Any] = {}
+ passed, output_dir, project_root = _run(
+ tmp_path, _score_runner(invocation=invocation)
+ )
+
+ assert passed
+ assert invocation == {
+ "model": "model-a",
+ "project_root": project_root,
+ "base_url": "http://127.0.0.1:8000/v1",
+ "api_key": "EMPTY",
+ "num_threads": 4,
+ }
+ assert json.loads(
+ (project_root / "test_case_ids_to_generate.json").read_text(encoding="utf-8")
+ ) == {
+ "simple_python": ["simple_python_141"],
+ "multiple": ["multiple_38"],
+ "parallel": ["parallel_1"],
+ "irrelevance": ["irrelevance_0"],
+ }
+ assert (
+ (project_root / be.UPSTREAM_LICENSE_FILENAME)
+ .read_text(encoding="utf-8")
+ .startswith(" Apache License")
+ )
+ attribution = json.loads(
+ (project_root / be.UPSTREAM_ATTRIBUTION_FILENAME).read_text(encoding="utf-8")
+ )
+ assert attribution == {
+ "artifact": "BFCL-generated evaluation results",
+ "upstream": {
+ "package": "bfcl-eval",
+ "package_version": "2026.3.23",
+ "wheel_sha256": be.BFCL_WHEEL_SHA256,
+ "repository": "https://github.com/ShishirPatil/gorilla",
+ "source_revision": be.SOURCE_REVISION,
+ "vllm_integration_revision": be.VLLM_INTEGRATION_REF,
+ "license": "Apache-2.0",
+ "license_url": (
+ "https://github.com/ShishirPatil/gorilla/blob/"
+ f"{be.SOURCE_REVISION}/LICENSE"
+ ),
+ "license_file": be.UPSTREAM_LICENSE_FILENAME,
+ },
+ "modifications": (
+ "InferenceX selected deterministic case subsets and projected upstream "
+ "scores; this archive does not modify upstream BFCL source."
+ ),
+ }
+
+ compatibility = _compatibility(output_dir)
+ native = _native(output_dir)
+ assert compatibility["result_format"] == "inferencex-eval-v1"
+ expected_tasks = [
+ "bfcl_smoke",
+ "bfcl_simple_python",
+ "bfcl_multiple",
+ "bfcl_parallel",
+ "bfcl_irrelevance",
+ ]
+ assert list(compatibility["results"]) == expected_tasks
+ assert all(
+ compatibility["results"][task_name] == {"acc,none": 1.0, "acc_stderr,none": 0.0}
+ for task_name in expected_tasks
+ )
+ assert all(
+ compatibility["configs"][task_name]
+ == {
+ "metric_list": [{"metric": "acc"}],
+ "filter_list": [{"name": "none"}],
+ }
+ for task_name in expected_tasks
+ )
+ assert compatibility["n-samples"][be.TASK_NAME] == {
+ "original": 4,
+ "effective": 4,
+ }
+ assert all(
+ compatibility["n-samples"][f"bfcl_{category}"]
+ == {"original": 1, "effective": 1}
+ for category in be.SMOKE_CASE_IDS
+ )
+ assert compatibility["bfcl"]["source"]["package_version"] == "2026.3.23"
+ assert compatibility["bfcl"]["source"]["wheel_sha256"] == (
+ "3bb6dfa5f0c68ad403c9ec50b00db2bb3b4cc9b38ab1ff33f48fe30d853d3a0a"
+ )
+ assert compatibility["bfcl"]["source"]["source_revision"] == (
+ "6ea57973c7a6097fd7c5915698c54c17c5b1b6c8"
+ )
+ assert [entry["category"] for entry in compatibility["bfcl"]["categories"]] == [
+ "simple_python",
+ "multiple",
+ "parallel",
+ "irrelevance",
+ ]
+ assert all(
+ entry["score_header"] == {"accuracy": 1.0, "correct_count": 1, "total_count": 1}
+ for entry in compatibility["bfcl"]["categories"]
+ )
+ assert all(
+ entry["score_records"] == [] for entry in compatibility["bfcl"]["categories"]
+ )
+ assert native["completed"] is True
+ assert native["passed"] is True
+ assert native["summary"] == {
+ "accuracy": 1.0,
+ "correct_count": 4,
+ "total_count": 4,
+ "expected_count": 4,
+ }
+ assert native["bfcl"] == compatibility["bfcl"]
+ assert sorted(path.name for path in output_dir.iterdir()) == [
+ "bfcl_report.json",
+ "results_bfcl.json",
+ ]
+
+
+def test_weighted_score_is_complete_and_diagnostic(
+ tmp_path: Path,
+) -> None:
+ completed, output_dir, _ = _run(
+ tmp_path,
+ _score_runner(
+ {
+ "simple_python": 1,
+ "multiple": 0,
+ "parallel": 0,
+ "irrelevance": 1,
+ }
+ ),
+ )
+
+ assert completed
+ assert _score(output_dir) == 0.5
+ compatibility = _compatibility(output_dir)
+ assert compatibility["results"]["bfcl_multiple"]["acc,none"] == 0.0
+ assert compatibility["results"]["bfcl_parallel"]["acc,none"] == 0.0
+ assert compatibility["n-samples"][be.TASK_NAME]["effective"] == 4
+ assert "integration_error" not in compatibility
+ native = _native(output_dir)
+ assert native["completed"] is True
+ assert native["passed"] is True
+ assert native["threshold"] == 0.0
+ assert native["summary"]["correct_count"] == 2
+
+
+def test_stale_compatibility_outputs_are_removed_without_touching_foreign_results(
+ tmp_path: Path,
+) -> None:
+ output_dir = tmp_path / "output"
+ output_dir.mkdir()
+ stale = output_dir / "results_bfcl_2000-01-01T00-00-00.000000.json"
+ stale.write_text("{}", encoding="utf-8")
+ stale_native = output_dir / be.NATIVE_REPORT_FILENAME
+ stale_native.write_text("{}", encoding="utf-8")
+ foreign = output_dir / "results_other_eval.json"
+ foreign.write_text("{}", encoding="utf-8")
+
+ passed, _, _ = _run(tmp_path, _score_runner(), output_dir=output_dir)
+
+ assert passed
+ assert not stale.exists()
+ assert foreign.exists()
+ assert len(list(output_dir.glob(be.COMPATIBILITY_GLOB))) == 1
+ assert _native(output_dir)["summary"]["accuracy"] == 1.0
+
+
+def test_swallowed_inference_error_is_an_integration_failure(
+ tmp_path: Path,
+) -> None:
+ base_runner = _score_runner()
+
+ def runner(**kwargs: Any) -> None:
+ base_runner(**kwargs)
+ _write_result(
+ kwargs["project_root"],
+ "parallel",
+ [
+ {
+ "id": "parallel_1",
+ "result": "Error during inference: request timed out",
+ "traceback": "TimeoutError: request timed out",
+ }
+ ],
+ )
+
+ completed, output_dir, _ = _run(tmp_path, runner)
+
+ assert not completed
+ assert _score(output_dir) == 0.0
+ assert _compatibility(output_dir)["integration_error"] == {
+ "type": "RuntimeError",
+ "message": "parallel result parallel_1 contains an inference error",
+ }
+
+
+@pytest.mark.parametrize(
+ ("mode", "expected_message"),
+ (
+ (
+ "missing",
+ "parallel result ids [] do not match expected ids ['parallel_1']",
+ ),
+ ("duplicate", "parallel result file contains duplicate id parallel_1"),
+ ),
+)
+def test_missing_or_duplicate_generated_id_is_an_integration_failure(
+ tmp_path: Path,
+ mode: str,
+ expected_message: str,
+) -> None:
+ base_runner = _score_runner()
+
+ def runner(**kwargs: Any) -> None:
+ base_runner(**kwargs)
+ row = {"id": "parallel_1", "result": []}
+ _write_result(
+ kwargs["project_root"],
+ "parallel",
+ [] if mode == "missing" else [row, row],
+ )
+
+ completed, output_dir, _ = _run(tmp_path, runner)
+
+ assert not completed
+ assert _score(output_dir) == 0.0
+ assert _compatibility(output_dir)["integration_error"] == {
+ "type": "ValueError",
+ "message": expected_message,
+ }
+
+
+def test_missing_category_is_an_integration_failure(tmp_path: Path) -> None:
+ passed, output_dir, _ = _run(tmp_path, _score_runner(missing="parallel"))
+
+ assert not passed
+ compatibility = _compatibility(output_dir)
+ assert _score(output_dir) == 0.0
+ assert compatibility["n-samples"][be.TASK_NAME]["effective"] == 0
+ assert compatibility["integration_error"]["type"] == "ValueError"
+ assert "parallel score file" in compatibility["integration_error"]["message"]
+ assert _native(output_dir)["completed"] is False
+
+
+def test_malformed_score_header_is_an_integration_failure(tmp_path: Path) -> None:
+ passed, output_dir, _ = _run(tmp_path, _score_runner(malformed="multiple"))
+
+ assert not passed
+ compatibility = _compatibility(output_dir)
+ assert _score(output_dir) == 0.0
+ assert compatibility["integration_error"] == {
+ "type": "ValueError",
+ "message": "multiple score header is malformed JSON",
+ }
+
+
+def test_unexpected_score_record_id_is_an_integration_failure(
+ tmp_path: Path,
+) -> None:
+ def runner(**kwargs: Any) -> None:
+ project_root = kwargs["project_root"]
+ for category in be.SMOKE_CASE_IDS:
+ _write_score(
+ project_root,
+ category,
+ 0 if category == "parallel" else 1,
+ record_id="parallel_0" if category == "parallel" else None,
+ )
+
+ completed, output_dir, _ = _run(tmp_path, runner)
+
+ assert not completed
+ compatibility = _compatibility(output_dir)
+ assert _score(output_dir) == 0.0
+ assert compatibility["integration_error"] == {
+ "type": "ValueError",
+ "message": "parallel score file contains unexpected id parallel_0",
+ }
+
+
+def test_incomplete_score_header_is_an_integration_failure(tmp_path: Path) -> None:
+ def runner(**kwargs: Any) -> None:
+ project_root = kwargs["project_root"]
+ for category in be.SMOKE_CASE_IDS:
+ if category == "irrelevance":
+ _write_score(
+ project_root,
+ category,
+ 1,
+ header_override=json.dumps({"accuracy": 1.0, "correct_count": 1}),
+ )
+ else:
+ _write_score(project_root, category, 1)
+
+ passed, output_dir, _ = _run(tmp_path, runner)
+
+ assert not passed
+ assert _compatibility(output_dir)["integration_error"]["message"] == (
+ "irrelevance score header missing: total_count"
+ )
+
+
+def test_upstream_exception_publishes_zero_score_reports(tmp_path: Path) -> None:
+ def fail(**kwargs: Any) -> None:
+ raise RuntimeError("BFCL generation failed")
+
+ passed, output_dir, project_root = _run(tmp_path, fail)
+
+ assert not passed
+ assert (project_root / "test_case_ids_to_generate.json").exists()
+ assert _score(output_dir) == 0.0
+ compatibility = _compatibility(output_dir)
+ assert compatibility["integration_error"] == {
+ "type": "RuntimeError",
+ "message": "BFCL generation failed",
+ }
+ assert compatibility["bfcl"]["source"]["case_ids"] == {
+ "simple_python": ["simple_python_141"],
+ "multiple": ["multiple_38"],
+ "parallel": ["parallel_1"],
+ "irrelevance": ["irrelevance_0"],
+ }
+ assert _native(output_dir)["summary"]["total_count"] == 0
+
+
+def test_integration_error_cli_is_stdlib_only_and_returns_nonzero(
+ tmp_path: Path,
+) -> None:
+ output_dir = tmp_path / "output"
+ script = Path(be.__file__).resolve()
+
+ completed = subprocess.run(
+ [
+ sys.executable,
+ "-I",
+ str(script),
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(output_dir),
+ "--integration-error",
+ "pinned wheel installation failed",
+ ],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert completed.returncode == 1
+ assert completed.stderr == ""
+ assert _score(output_dir) == 0.0
+ compatibility = _compatibility(output_dir)
+ assert compatibility["n-samples"][be.TASK_NAME] == {
+ "original": 4,
+ "effective": 0,
+ }
+ assert compatibility["integration_error"] == {
+ "type": "RuntimeError",
+ "message": "pinned wheel installation failed",
+ }
+ native = _native(output_dir)
+ assert native["completed"] is False
+ assert native["integration_error"] == compatibility["integration_error"]
+
+
+def test_full_suite_sets_project_root_before_dataset_import(
+ monkeypatch, tmp_path: Path
+) -> None:
+ output_dir = tmp_path / "output"
+ project_root = tmp_path / "project"
+ observed_roots: list[str | None] = []
+ monkeypatch.setenv("BFCL_PROJECT_ROOT", "stale-root")
+
+ def stop_after_observing_root(suite: be.SuiteSpec):
+ observed_roots.append(os.environ.get("BFCL_PROJECT_ROOT"))
+ raise RuntimeError("stop after project-root check")
+
+ monkeypatch.setattr(be, "_build_suite_case_ids", stop_after_observing_root)
+
+ passed = be.run_evaluation(
+ base_url="http://localhost:8000/v1",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output_dir,
+ bfcl_project_root=project_root,
+ suite=be.MINIMAX_SUITE,
+ )
+
+ assert passed is False
+ assert observed_roots == [str(project_root)]
+
+
+@pytest.mark.parametrize("suite", (be.MINIMAX_SUITE, be.KIMI_SUITE))
+def test_full_suite_ids_use_exact_sorted_leaf_allocations(
+ monkeypatch,
+ suite: be.SuiteSpec,
+) -> None:
+ multi_turn_leaves = (
+ "multi_turn_base",
+ "multi_turn_miss_func",
+ "multi_turn_miss_param",
+ "multi_turn_long_context",
+ )
+ upstream_multi_turn_leaves = (
+ "multi_turn_base",
+ "multi_turn_long_context",
+ "multi_turn_miss_func",
+ "multi_turn_miss_param",
+ )
+ available_counts = {
+ "simple_python": 400,
+ "multiple": 200,
+ "parallel": 200,
+ "parallel_multiple": 200,
+ **{leaf: 75 for leaf in multi_turn_leaves},
+ }
+
+ def load_dataset_entry(category: str) -> list[dict[str, str]]:
+ return [
+ {"id": f"{category}_{index:03d}"}
+ for index in reversed(range(available_counts[category]))
+ ]
+
+ def parse_test_category_argument(categories: list[str]) -> list[str]:
+ assert len(categories) == 1
+ return (
+ list(upstream_multi_turn_leaves)
+ if categories[0] == "multi_turn"
+ else categories
+ )
+
+ def load_helpers():
+ return (
+ load_dataset_entry,
+ parse_test_category_argument,
+ lambda entry: entry["id"],
+ )
+
+ monkeypatch.setattr(be, "_load_dataset_helpers", load_helpers)
+
+ selected = be._build_suite_case_ids(suite)
+
+ assert (
+ tuple((category, len(case_ids)) for category, case_ids in selected.items())
+ == suite.expected_leaf_counts
+ )
+ assert sum(map(len, selected.values())) == suite.expected_sample_count
+ assert all(list(case_ids) == sorted(case_ids) for case_ids in selected.values())
+ if suite is be.KIMI_SUITE:
+ assert {leaf: len(selected[leaf]) for leaf in multi_turn_leaves} == {
+ leaf: 60 for leaf in multi_turn_leaves
+ }
+ assert all(selected[leaf][-1].endswith("_059") for leaf in multi_turn_leaves)
+
+
+def test_mixed_category_diagnostics_project_each_failure_id() -> None:
+ score = be.CategoryScore(
+ category="parallel",
+ case_ids=("parallel_0", "parallel_1", "parallel_2"),
+ score_file="score/model-a/BFCL_v4_parallel_score.json",
+ header={"accuracy": 2 / 3, "correct_count": 2, "total_count": 3},
+ records=({"id": "parallel_1", "error": "wrong call"},),
+ accuracy=2 / 3,
+ correct_count=2,
+ total_count=3,
+ )
+
+ assert score.as_dict()["case_scores"] == [
+ {"id": "parallel_0", "score": 1.0, "correct": True},
+ {"id": "parallel_1", "score": 0.0, "correct": False},
+ {"id": "parallel_2", "score": 1.0, "correct": True},
+ ]
+
+
+def test_kimi_projects_namespaced_leaf_and_weighted_aggregate_scores() -> None:
+ assert be.KIMI_SUITE.expected_sample_count == 1240
+ selected = {
+ category: tuple(f"{category}_{index}" for index in range(total_count))
+ for category, total_count in be.KIMI_SUITE.expected_leaf_counts
+ }
+ correct_counts = {
+ "simple_python": 200,
+ "multiple": 100,
+ "parallel": 50,
+ "parallel_multiple": 200,
+ "multi_turn_base": 60,
+ "multi_turn_miss_func": 30,
+ "multi_turn_miss_param": 0,
+ "multi_turn_long_context": 15,
+ }
+ scores = [
+ _category_score(category, total_count, correct_counts[category])
+ for category, total_count in be.KIMI_SUITE.expected_leaf_counts
+ ]
+
+ compatibility = be._compatibility_result(
+ suite=be.KIMI_SUITE,
+ case_ids_by_category=selected,
+ model="model-a",
+ scores=scores,
+ )
+
+ assert compatibility["results"]["bfcl_vllm_kimi"]["acc,none"] == 655 / 1240
+ assert (
+ compatibility["results"]["bfcl_vllm_kimi_multi_turn"]["acc,none"] == 105 / 240
+ )
+ assert compatibility["n-samples"]["bfcl_vllm_kimi_multi_turn"] == {
+ "original": 240,
+ "effective": 240,
+ }
+ assert all(
+ f"bfcl_vllm_kimi_{category}" in compatibility["results"]
+ for category in selected
+ )
+ assert "bfcl_simple_python" not in compatibility["results"]
+ assert "bfcl_multi_turn" not in compatibility["results"]
+
+
+def test_selected_suite_integration_error_preserves_suite_identity(
+ tmp_path: Path,
+) -> None:
+ output_dir = tmp_path / "output"
+
+ return_code = be.main(
+ [
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(output_dir),
+ "--suite",
+ "bfcl_vllm_minimax_m3",
+ "--integration-error",
+ "pinned wheel installation failed",
+ ]
+ )
+
+ assert return_code == 1
+ compatibility = _compatibility(output_dir)
+ native = _native(output_dir)
+ assert native["task"] == "bfcl_vllm_minimax_m3"
+ assert native["summary"]["expected_count"] == 1000
+ assert native["sampling"] == {
+ "temperature": 0.001,
+ "num_threads": 8,
+ }
+ assert list(compatibility["results"]) == [
+ "bfcl_vllm_minimax_m3",
+ "bfcl_vllm_minimax_m3_simple_python",
+ "bfcl_vllm_minimax_m3_multiple",
+ "bfcl_vllm_minimax_m3_parallel",
+ "bfcl_vllm_minimax_m3_parallel_multiple",
+ ]
+ assert compatibility["n-samples"]["bfcl_vllm_minimax_m3"] == {
+ "original": 1000,
+ "effective": 0,
+ }
+ assert native["integration_error"] == compatibility["integration_error"]
+
+
+def test_score_total_must_match_every_selected_id(tmp_path: Path) -> None:
+ project_root = tmp_path / "bfcl"
+ score_path = project_root / "score" / "model-a" / "BFCL_v4_simple_python_score.json"
+ score_path.parent.mkdir(parents=True)
+ score_path.write_text(
+ json.dumps({"accuracy": 1.0, "correct_count": 1, "total_count": 1}) + "\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(
+ ValueError,
+ match="simple_python evaluated 1 cases; expected 2",
+ ):
+ be._collect_scores(
+ project_root,
+ {"simple_python": ("simple_python_0", "simple_python_1")},
+ )
+
+
+def test_full_suite_handler_bounds_openai_requests() -> None:
+ class StockOpenAICompletionsHandler:
+ def _build_client_kwargs(self) -> dict[str, Any]:
+ return {"api_key": "stock-key"}
+
+ handler = be._bounded_openai_handler(StockOpenAICompletionsHandler)
+
+ assert issubclass(handler, StockOpenAICompletionsHandler)
+ assert handler()._build_client_kwargs() == {
+ "api_key": "stock-key",
+ "timeout": 180,
+ "max_retries": 2,
+ }
+
+
+def test_kimi_suite_caps_multi_turn_steps(monkeypatch: pytest.MonkeyPatch) -> None:
+ constants = ModuleType("bfcl_eval.constants")
+ constants.__path__ = []
+ prompts = ModuleType("bfcl_eval.constants.default_prompts")
+ prompts.MAXIMUM_STEP_LIMIT = 20
+ constants.default_prompts = prompts
+ monkeypatch.setitem(sys.modules, "bfcl_eval.constants", constants)
+ monkeypatch.setitem(
+ sys.modules,
+ "bfcl_eval.constants.default_prompts",
+ prompts,
+ )
+
+ be._apply_suite_runtime_limits(be.MINIMAX_SUITE)
+ assert prompts.MAXIMUM_STEP_LIMIT == 20
+
+ be._apply_suite_runtime_limits(be.KIMI_SUITE)
+ assert prompts.MAXIMUM_STEP_LIMIT == 10
+
+
+def test_upstream_registration_uses_exact_stock_openai_handler(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ project_root = tmp_path / "bfcl"
+ be._write_id_map(project_root, be.SMOKE_CASE_IDS)
+ model_config_mapping: dict[str, Any] = {}
+
+ class ModelConfig:
+ def __init__(self, **kwargs: Any) -> None:
+ self.__dict__.update(kwargs)
+
+ class OpenAICompletionsHandler:
+ pass
+
+ def generate(**_: Any) -> None:
+ result_dir = project_root / "result" / "model-a"
+ result_dir.mkdir(parents=True)
+ for category, case_ids in be.SMOKE_CASE_IDS.items():
+ (result_dir / f"BFCL_v4_{category}_result.json").write_text(
+ "".join(
+ json.dumps({"id": case_id, "result": []}) + "\n"
+ for case_id in case_ids
+ ),
+ encoding="utf-8",
+ )
+
+ def evaluate(**_: Any) -> None:
+ pass
+
+ modules = {
+ "bfcl_eval": ModuleType("bfcl_eval"),
+ "bfcl_eval.constants": ModuleType("bfcl_eval.constants"),
+ "bfcl_eval.constants.model_config": ModuleType(
+ "bfcl_eval.constants.model_config"
+ ),
+ "bfcl_eval.model_handler": ModuleType("bfcl_eval.model_handler"),
+ "bfcl_eval.model_handler.api_inference": ModuleType(
+ "bfcl_eval.model_handler.api_inference"
+ ),
+ "bfcl_eval.model_handler.api_inference.openai_completion": ModuleType(
+ "bfcl_eval.model_handler.api_inference.openai_completion"
+ ),
+ "bfcl_eval.__main__": ModuleType("bfcl_eval.__main__"),
+ }
+ modules[
+ "bfcl_eval.constants.model_config"
+ ].MODEL_CONFIG_MAPPING = model_config_mapping
+ modules["bfcl_eval.constants.model_config"].ModelConfig = ModelConfig
+ modules[
+ "bfcl_eval.model_handler.api_inference.openai_completion"
+ ].OpenAICompletionsHandler = OpenAICompletionsHandler
+ modules["bfcl_eval.__main__"].generate = generate
+ modules["bfcl_eval.__main__"].evaluate = evaluate
+ for name, module in modules.items():
+ if name in {
+ "bfcl_eval",
+ "bfcl_eval.constants",
+ "bfcl_eval.model_handler",
+ "bfcl_eval.model_handler.api_inference",
+ }:
+ module.__path__ = []
+ monkeypatch.setitem(sys.modules, name, module)
+ monkeypatch.setattr(be, "_function_defaults", lambda _: {})
+
+ be._run_upstream(
+ model="model-a",
+ project_root=project_root,
+ base_url="http://127.0.0.1:8000/v1",
+ api_key="EMPTY",
+ num_threads=4,
+ )
+
+ assert model_config_mapping["model-a"].model_handler is OpenAICompletionsHandler
diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py
new file mode 100644
index 0000000000..9587483ab3
--- /dev/null
+++ b/utils/evals/test_kimi_vendor_eval.py
@@ -0,0 +1,460 @@
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import kimi_vendor_eval as kve
+
+
+def _report(stream_status: str = "passed") -> dict[str, Any]:
+ statuses = ["passed", stream_status]
+ by_status = {status: statuses.count(status) for status in set(statuses)}
+ return {
+ "summary": {"total": 2, "by_status": by_status},
+ "results": [
+ {"mode": "non-stream", "status": "passed"},
+ {"mode": "stream", "status": stream_status},
+ ],
+ }
+
+
+def _full_report(*, failed_records: int = 0) -> dict[str, Any]:
+ selected_cases: list[dict[str, Any]] = []
+ results: list[dict[str, Any]] = []
+ for line in range(1, kve.FULL_SELECTED_CASES + 1):
+ selected_cases.append(
+ {
+ "suite": "TestSchema",
+ "line": line,
+ "selection_reason": "all",
+ "schema": {},
+ }
+ )
+ for mode in ("non-stream", "stream"):
+ results.append(
+ {
+ "suite": "TestSchema",
+ "line": line,
+ "mode": mode,
+ "status": ("failed" if len(results) < failed_records else "passed"),
+ }
+ )
+ return {
+ "summary": {
+ "total": len(results),
+ "by_status": {
+ "passed": len(results) - failed_records,
+ "failed": failed_records,
+ },
+ },
+ "selected_cases": selected_cases,
+ "results": results,
+ }
+
+
+def _report_with_inconsistent_counts() -> dict[str, Any]:
+ report = _report("failed")
+ report["summary"]["by_status"]["failed"] = 2
+ return report
+
+
+def _result(output_dir: Path) -> dict[str, Any]:
+ paths = list(output_dir.glob(kve.COMPATIBILITY_GLOB))
+ assert len(paths) == 1
+ assert re.fullmatch(
+ r"results_kimi_vendor_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}\.json",
+ paths[0].name,
+ )
+ return json.loads(paths[0].read_text())
+
+
+def _score(output_dir: Path, task_name: str = kve.TASK_NAME) -> float:
+ return _result(output_dir)["results"][task_name]["exact_match,strict-match"]
+
+
+def _n_eff(output_dir: Path, task_name: str = kve.TASK_NAME) -> int:
+ return _result(output_dir)["n-samples"][task_name]["effective"]
+
+
+def test_builds_fixed_upstream_pytest_command(tmp_path: Path) -> None:
+ report = tmp_path / kve.NATIVE_REPORT_FILENAME
+
+ assert kve.build_pytest_command(
+ base_url="http://127.0.0.1:8000/v1",
+ api_key="EMPTY",
+ model="test-model",
+ report_path=report,
+ ) == [
+ sys.executable,
+ "-m",
+ "pytest",
+ "tests/tool_call_json_schema/test_tool_call_json_schema.py",
+ "--base-url",
+ "http://127.0.0.1:8000/v1",
+ "--api-key",
+ "EMPTY",
+ "--smoke-model",
+ "test-model",
+ "--think-mode",
+ "none",
+ "--selection",
+ "object",
+ "--max-cases",
+ "1",
+ "--case-dir",
+ "testdata/walle_validator_cases/validator_cases",
+ "--max-tokens",
+ "2048",
+ "--tool-json-report",
+ str(report),
+ ]
+
+
+def test_builds_full_upstream_pytest_command(tmp_path: Path) -> None:
+ report = tmp_path / kve.NATIVE_REPORT_FILENAME
+
+ assert kve.build_pytest_command(
+ base_url="http://127.0.0.1:8000/v1",
+ api_key="EMPTY",
+ model="test-model",
+ report_path=report,
+ task_name=kve.FULL_TASK_NAME,
+ ) == [
+ sys.executable,
+ "-m",
+ "pytest",
+ "tests/tool_call_json_schema/test_tool_call_json_schema.py",
+ "-n",
+ "8",
+ "--base-url",
+ "http://127.0.0.1:8000/v1",
+ "--api-key",
+ "EMPTY",
+ "--smoke-model",
+ "test-model",
+ "--think-mode",
+ "none",
+ "--selection",
+ "all",
+ "--case-dir",
+ "testdata/walle_validator_cases/validator_cases",
+ "--max-tokens",
+ "2048",
+ "--tool-json-report",
+ str(report),
+ ]
+
+
+def test_builds_dsv4_thinking_command(tmp_path: Path) -> None:
+ command = kve.build_pytest_command(
+ base_url="http://127.0.0.1:8000/v1",
+ api_key="EMPTY",
+ model="deepseek-ai/DeepSeek-V4-Pro",
+ model_prefix="dsv4",
+ report_path=tmp_path / kve.NATIVE_REPORT_FILENAME,
+ )
+
+ think_mode_index = command.index("--think-mode")
+ selection_index = command.index("--selection")
+ assert command[think_mode_index:selection_index] == [
+ "--think-mode",
+ "opensource",
+ "--thinking",
+ ]
+
+
+@pytest.mark.parametrize(
+ (
+ "stream_status",
+ "return_code",
+ "expected_pass",
+ "expected_score",
+ "expected_n_eff",
+ "expected_error",
+ ),
+ (
+ ("passed", 0, True, 1.0, 2, None),
+ ("passed", 1, False, 0.0, 0, "RuntimeError"),
+ ("failed", 0, False, 0.0, 0, "RuntimeError"),
+ ("failed", 1, True, 0.5, 2, None),
+ ("failed", 2, False, 0.0, 0, "RuntimeError"),
+ ),
+)
+def test_projects_upstream_outcomes(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ stream_status: str,
+ return_code: int,
+ expected_pass: bool,
+ expected_score: float,
+ expected_n_eff: int,
+ expected_error: str | None,
+) -> None:
+ output_dir = tmp_path / "output"
+ native_bytes = json.dumps(_report(stream_status)).encode()
+ invocation: dict[str, Any] = {}
+
+ def fake_run(
+ command: list[str], *, cwd: Path, check: bool, timeout: int
+ ) -> SimpleNamespace:
+ invocation.update(command=command, cwd=cwd, check=check, timeout=timeout)
+ Path(command[command.index("--tool-json-report") + 1]).write_bytes(native_bytes)
+ return SimpleNamespace(returncode=return_code)
+
+ monkeypatch.setattr(kve.subprocess, "run", fake_run)
+
+ assert (
+ kve.run_evaluation(
+ verifier_dir=tmp_path,
+ base_url="http://localhost/v1",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output_dir,
+ )
+ is expected_pass
+ )
+ assert invocation["cwd"] == tmp_path
+ assert invocation["check"] is False
+ assert invocation["timeout"] == kve.DEFAULT_TIMEOUT_SECONDS
+ assert _score(output_dir) == expected_score
+ assert _n_eff(output_dir) == expected_n_eff
+ projected = _result(output_dir)
+ assert projected["result_format"] == kve.RESULT_FORMAT
+ assert projected["eval_adapter"] == kve.ADAPTER_NAME
+ if expected_error is None:
+ assert "integration_error" not in projected
+ else:
+ assert projected["integration_error"]["type"] == expected_error
+ assert "lm_eval_version" not in projected
+ assert (output_dir / kve.NATIVE_REPORT_FILENAME).read_bytes() == native_bytes
+
+
+def test_full_report_projects_all_mode_records_and_defers_quality_gating(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ output_dir = tmp_path / "output"
+ report = _full_report(failed_records=1)
+ report["results"][0]["message"] = (
+ ' f"tool schema rejected: {response.message}"\n'
+ "E AssertionError: TestSchema:1 [non-stream] (all) "
+ "arguments validation failed: 'bad' is not valid"
+ )
+ native_bytes = json.dumps(report).encode()
+ invocation: dict[str, Any] = {}
+
+ def fake_run(
+ command: list[str], *, cwd: Path, check: bool, timeout: int
+ ) -> SimpleNamespace:
+ invocation.update(command=command, cwd=cwd, check=check, timeout=timeout)
+ Path(command[command.index("--tool-json-report") + 1]).write_bytes(native_bytes)
+ return SimpleNamespace(returncode=1)
+
+ monkeypatch.setattr(kve.subprocess, "run", fake_run)
+
+ assert kve.run_evaluation(
+ verifier_dir=tmp_path,
+ base_url="http://localhost/v1",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output_dir,
+ task_name=kve.FULL_TASK_NAME,
+ timeout_seconds=kve.FULL_TIMEOUT_SECONDS,
+ )
+ projected = _result(output_dir)
+ assert invocation["timeout"] == 7200
+ assert set(projected["results"]) == {kve.FULL_TASK_NAME}
+ assert _score(output_dir, kve.FULL_TASK_NAME) == 407 / 408
+ assert projected["n-samples"][kve.FULL_TASK_NAME] == {
+ "original": 408,
+ "effective": 408,
+ }
+ assert "integration_error" not in projected
+ assert (output_dir / kve.NATIVE_REPORT_FILENAME).read_bytes() == native_bytes
+
+
+def test_full_report_classifies_endpoint_failures_as_integration_errors(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ report = _full_report(failed_records=1)
+ report["results"][0]["message"] = (
+ "AssertionError: TestSchema:1 [non-stream] (all) tool schema rejected: "
+ "Data returned by API invalid for expected schema."
+ )
+ native_bytes = json.dumps(report).encode()
+
+ def fake_run(
+ command: list[str], *, cwd: Path, check: bool, timeout: int
+ ) -> SimpleNamespace:
+ Path(command[command.index("--tool-json-report") + 1]).write_bytes(native_bytes)
+ return SimpleNamespace(returncode=1)
+
+ monkeypatch.setattr(kve.subprocess, "run", fake_run)
+ output_dir = tmp_path / "output"
+
+ assert not kve.run_evaluation(
+ verifier_dir=tmp_path,
+ base_url="http://localhost/v1",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output_dir,
+ task_name=kve.FULL_TASK_NAME,
+ timeout_seconds=kve.FULL_TIMEOUT_SECONDS,
+ )
+ projected = _result(output_dir)
+ assert _score(output_dir, kve.FULL_TASK_NAME) == 0.0
+ assert _n_eff(output_dir, kve.FULL_TASK_NAME) == 0
+ assert projected["integration_error"]["type"] == "RuntimeError"
+ assert (
+ "endpoint request or response failure"
+ in projected["integration_error"]["message"]
+ )
+ assert (output_dir / kve.NATIVE_REPORT_FILENAME).read_bytes() == native_bytes
+
+
+def test_full_report_rejects_incomplete_modes(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ report = _full_report()
+ report["results"][-1]["mode"] = "non-stream"
+
+ def fake_run(
+ command: list[str], *, cwd: Path, check: bool, timeout: int
+ ) -> SimpleNamespace:
+ Path(command[command.index("--tool-json-report") + 1]).write_text(
+ json.dumps(report)
+ )
+ return SimpleNamespace(returncode=1)
+
+ monkeypatch.setattr(kve.subprocess, "run", fake_run)
+ output_dir = tmp_path / "output"
+
+ assert not kve.run_evaluation(
+ verifier_dir=tmp_path,
+ base_url="http://localhost/v1",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output_dir,
+ task_name=kve.FULL_TASK_NAME,
+ timeout_seconds=kve.FULL_TIMEOUT_SECONDS,
+ )
+ projected = _result(output_dir)
+ assert _score(output_dir, kve.FULL_TASK_NAME) == 0.0
+ assert _n_eff(output_dir, kve.FULL_TASK_NAME) == 0
+ assert projected["integration_error"]["type"] == "ValueError"
+
+
+@pytest.mark.parametrize(
+ ("failure", "error_type"),
+ (
+ (None, "FileNotFoundError"),
+ ("{bad-json", "JSONDecodeError"),
+ (OSError("boom"), "OSError"),
+ (json.dumps(_report("skipped")), "ValueError"),
+ (json.dumps(_report_with_inconsistent_counts()), "ValueError"),
+ (
+ subprocess.TimeoutExpired("pytest", kve.DEFAULT_TIMEOUT_SECONDS),
+ "TimeoutExpired",
+ ),
+ ),
+)
+def test_collection_failures_write_zero_score(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ failure: str | BaseException | None,
+ error_type: str,
+) -> None:
+ def fake_run(
+ command: list[str], *, cwd: Path, check: bool, timeout: int
+ ) -> SimpleNamespace:
+ if isinstance(failure, BaseException):
+ raise failure
+ if failure is not None:
+ Path(command[command.index("--tool-json-report") + 1]).write_text(failure)
+ return SimpleNamespace(returncode=1)
+
+ monkeypatch.setattr(kve.subprocess, "run", fake_run)
+ output_dir = tmp_path / "output"
+
+ assert not kve.run_evaluation(
+ verifier_dir=tmp_path,
+ base_url="http://localhost/v1",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output_dir,
+ )
+ projected = _result(output_dir)
+ assert _score(output_dir) == 0.0
+ assert projected["integration_error"]["type"] == error_type
+ assert _n_eff(output_dir) == 0
+
+
+def test_failure_cannot_reuse_stale_outputs(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ output_dir = tmp_path / "output"
+ output_dir.mkdir()
+ native_report = output_dir / kve.NATIVE_REPORT_FILENAME
+ native_report.write_text(json.dumps(_report()))
+ (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text(
+ "{}"
+ )
+ foreign_result = output_dir / "results_other_eval.json"
+ foreign_result.write_text("{}")
+
+ def fail_collection(*args: Any, **kwargs: Any) -> SimpleNamespace:
+ assert not native_report.exists()
+ return SimpleNamespace(returncode=2)
+
+ monkeypatch.setattr(kve.subprocess, "run", fail_collection)
+
+ assert not kve.run_evaluation(
+ verifier_dir=tmp_path,
+ base_url="http://localhost/v1",
+ api_key="EMPTY",
+ model="model-a",
+ output_dir=output_dir,
+ )
+ assert _score(output_dir) == 0.0
+ assert json.loads(native_report.read_text())["completed"] is False
+ assert foreign_result.exists()
+
+
+def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None:
+ output_dir = tmp_path / "output"
+ output_dir.mkdir()
+ (output_dir / kve.NATIVE_REPORT_FILENAME).write_text(json.dumps(_report()))
+ (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text(
+ "{}"
+ )
+
+ assert (
+ kve.main(
+ [
+ "--model",
+ "model-a",
+ "--output-dir",
+ str(output_dir),
+ "--integration-error",
+ "checkout failed",
+ ]
+ )
+ == 0
+ )
+ projected = _result(output_dir)
+ native = json.loads((output_dir / kve.NATIVE_REPORT_FILENAME).read_text())
+ assert native["completed"] is False
+ assert native["summary"]["expected_total"] == 2
+ assert native["integration_error"]["message"] == "checkout failed"
+ assert _score(output_dir) == 0.0
+ assert projected["integration_error"]["message"] == "checkout failed"
+ assert _n_eff(output_dir) == 0
diff --git a/utils/evals/test_minimax_m3_full_eval.py b/utils/evals/test_minimax_m3_full_eval.py
new file mode 100644
index 0000000000..e3ef5c485a
--- /dev/null
+++ b/utils/evals/test_minimax_m3_full_eval.py
@@ -0,0 +1,311 @@
+import hashlib
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import minimax_m3_full_eval as full
+
+
+def _write_native_run(
+ output_dir: Path,
+ *,
+ model: str = "MiniMax-M3",
+ score: float = 0.625,
+ result_count: int = full.EXPECTED_RESULT_COUNT,
+ failed_index: int | None = None,
+) -> bytes:
+ output_dir.mkdir(exist_ok=True)
+ report = {
+ "model": model,
+ "success_count": result_count - (failed_index is not None),
+ "failure_count": int(failed_index is not None),
+ "success_rate": 1.0 if failed_index is None else 0.99,
+ "tool_calls_match_rate": score,
+ "tool_calls_successful_count": 60,
+ "error_only_reasoning_rate": 0.0,
+ "language_following_valid_count": 2,
+ "scenario_check_pass_rate": 1.0,
+ }
+ report_bytes = (json.dumps(report, indent=4) + "\n").encode()
+ (output_dir / full.NATIVE_REPORT_FILENAME).write_bytes(report_bytes)
+ rows = [
+ {
+ "data_index": index,
+ "status": "failed" if index == failed_index else "success",
+ }
+ for index in range(1, result_count + 1)
+ ]
+ (output_dir / full.NATIVE_RESULTS_FILENAME).write_text(
+ "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8"
+ )
+ return report_bytes
+
+
+def test_prepared_source_tree_requires_every_pinned_byte(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ sample = b"".join(b"{}\n" for _ in range(full.EXPECTED_RESULT_COUNT))
+ contents = {
+ "verify.py": b"print('pinned verifier')\n",
+ "sample.jsonl": sample,
+ "validator/__init__.py": b"",
+ }
+ monkeypatch.setattr(
+ full,
+ "REQUIRED_SOURCE_SHA256",
+ {name: hashlib.sha256(content).hexdigest() for name, content in contents.items()},
+ )
+
+ source_dir = tmp_path / "source"
+ full.prepare_source_tree(source_dir, fetcher=contents.__getitem__)
+ full.verify_source_tree(source_dir)
+
+ (source_dir / "verify.py").write_text("mutated\n", encoding="utf-8")
+ with pytest.raises(ValueError, match="SHA256 mismatch"):
+ full.verify_source_tree(source_dir)
+ assert full.EXPECTED_SAMPLE_SHA256 == (
+ "3ead102af0f888acc95867b3a9916942524b02f4f64931f020a1bfb4fee9aae2"
+ )
+
+def test_fetch_source_retries_transient_network_failures(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ content = b"pinned source"
+ monkeypatch.setattr(
+ full,
+ "REQUIRED_SOURCE_SHA256",
+ {"verify.py": hashlib.sha256(content).hexdigest()},
+ )
+ sleeps: list[int] = []
+
+ class Response:
+ status = 200
+ headers: dict[str, str] = {}
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ return None
+
+ def geturl(self) -> str:
+ return full.source_url("verify.py")
+
+ def read(self, _: int) -> bytes:
+ return content
+
+ class Opener:
+ calls = 0
+
+ def open(self, request, *, timeout):
+ self.calls += 1
+ if self.calls < full.DOWNLOAD_ATTEMPTS:
+ raise full.urllib.error.URLError("transient")
+ return Response()
+
+ opener = Opener()
+ monkeypatch.setattr(full, "_NO_REDIRECT_OPENER", opener)
+ monkeypatch.setattr(full.time, "sleep", sleeps.append)
+
+ assert full._fetch_source("verify.py") == content
+ assert opener.calls == full.DOWNLOAD_ATTEMPTS
+ assert sleeps == [full.DOWNLOAD_RETRY_DELAY_SECONDS] * 2
+
+
+def test_fetch_source_reports_exhausted_retries(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class Opener:
+ calls = 0
+
+ def open(self, request, *, timeout):
+ self.calls += 1
+ raise full.urllib.error.URLError("offline")
+
+ opener = Opener()
+ monkeypatch.setattr(full, "_NO_REDIRECT_OPENER", opener)
+ monkeypatch.setattr(full.time, "sleep", lambda _: None)
+
+ with pytest.raises(full.FullSuiteError, match="after 3 attempts"):
+ full._fetch_source("verify.py")
+ assert opener.calls == full.DOWNLOAD_ATTEMPTS
+
+
+def test_verifier_command_is_one_102_row_run_with_fixed_m3_settings(
+ tmp_path: Path,
+) -> None:
+ command = full.build_verifier_command(
+ python=Path("/runtime/python"),
+ source_dir=Path("/runtime/source"),
+ base_url="http://127.0.0.1:8000/v1/",
+ model="MiniMax-M3",
+ output_dir=tmp_path,
+ )
+
+ assert command == [
+ "/runtime/python",
+ "/runtime/source/verify.py",
+ "/runtime/source/sample.jsonl",
+ "--model",
+ "MiniMax-M3",
+ "--base-url",
+ "http://127.0.0.1:8000/v1",
+ "--api-key",
+ "EMPTY",
+ "--concurrency",
+ "5",
+ "--output",
+ str(tmp_path / full.NATIVE_RESULTS_FILENAME),
+ "--summary",
+ str(tmp_path / full.NATIVE_REPORT_FILENAME),
+ "--timeout",
+ "600",
+ "--retries",
+ "3",
+ "--extra-body",
+ '{"temperature":0,"top_p":1,"max_tokens":40960}',
+ ]
+ assert "pass" not in " ".join(command).lower()
+ assert command.count("/runtime/source/sample.jsonl") == 1
+
+
+def test_projects_exactly_102_results_from_native_match_rate_without_rewriting_report(
+ tmp_path: Path,
+) -> None:
+ report_bytes = _write_native_run(tmp_path)
+
+ compatibility_path = full.project_native_artifacts(
+ output_dir=tmp_path, model="MiniMax-M3"
+ )
+
+ compatibility = json.loads(compatibility_path.read_text(encoding="utf-8"))
+ assert compatibility["result_format"] == "inferencex-eval-v1"
+ assert compatibility["eval_adapter"] == "minimax-provider-verifier"
+ assert compatibility["results"] == {
+ "minimax_m3_full": {
+ "exact_match,strict-match": 0.625,
+ "exact_match_stderr,strict-match": 0.0,
+ }
+ }
+ assert compatibility["configs"]["minimax_m3_full"]["native_metric"] == (
+ "tool_calls_match_rate"
+ )
+ assert compatibility["configs"]["minimax_m3_full"][
+ "diagnostic_threshold"
+ ] == 0.0
+ assert compatibility["n-samples"]["minimax_m3_full"] == {
+ "original": 102,
+ "effective": 102,
+ }
+ assert (tmp_path / full.NATIVE_REPORT_FILENAME).read_bytes() == report_bytes
+ assert len(
+ (tmp_path / full.NATIVE_RESULTS_FILENAME)
+ .read_text(encoding="utf-8")
+ .splitlines()
+ ) == 102
+
+def test_full_projection_removes_stale_smoke_result(tmp_path: Path) -> None:
+ stale_smoke = tmp_path / "results_minimax_vendor_2026-01-01.json"
+ stale_smoke.write_text("{}")
+
+ compatibility_path = full._compatibility_path(tmp_path)
+
+ assert not stale_smoke.exists()
+ assert compatibility_path.name.startswith("results_minimax_vendor_full_")
+
+
+def test_failure_command_replaces_stale_native_artifacts(tmp_path: Path) -> None:
+ native_report = tmp_path / full.NATIVE_REPORT_FILENAME
+ native_results = tmp_path / full.NATIVE_RESULTS_FILENAME
+ native_report.write_text('{"stale": true}\n')
+ native_results.write_text('{"stale": true}\n')
+
+ assert (
+ full.main(
+ [
+ "failure",
+ "--model",
+ "MiniMax-M3",
+ "--output-dir",
+ str(tmp_path),
+ "--message",
+ "runtime setup failed",
+ ]
+ )
+ == 0
+ )
+
+ report = json.loads(native_report.read_text(encoding="utf-8"))
+ assert report["completed"] is False
+ assert report["integration_error"]["message"] == "runtime setup failed"
+ result_rows = native_results.read_text(encoding="utf-8").splitlines()
+ assert len(result_rows) == 1
+ assert json.loads(result_rows[0])["status"] == "integration_error"
+
+
+def test_full_suite_timeout_leaves_workflow_cleanup_margin() -> None:
+ shortest_workflow_timeout_seconds = 480 * 60
+ cleanup_margin_seconds = 60 * 60
+
+ assert (
+ full.UPSTREAM_TIMEOUT_SECONDS
+ <= shortest_workflow_timeout_seconds - cleanup_margin_seconds
+ )
+
+
+@pytest.mark.parametrize(
+ ("result_count", "failed_index", "message"),
+ [
+ (101, None, "exactly 102 rows"),
+ (102, 7, "transport failures"),
+ ],
+)
+def test_projection_rejects_incomplete_or_transport_failed_native_results(
+ tmp_path: Path,
+ result_count: int,
+ failed_index: int | None,
+ message: str,
+) -> None:
+ _write_native_run(
+ tmp_path,
+ result_count=result_count,
+ failed_index=failed_index,
+ )
+
+ with pytest.raises(full.FullSuiteError, match=message):
+ full.project_native_artifacts(output_dir=tmp_path, model="MiniMax-M3")
+
+
+def test_upstream_process_failure_is_nonzero_and_publishes_all_failure_artifacts(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ source_dir = tmp_path / "source"
+ dependency_dir = tmp_path / "deps"
+ output_dir = tmp_path / "output"
+ source_dir.mkdir()
+ dependency_dir.mkdir()
+ monkeypatch.setattr(full, "verify_source_tree", lambda _path: None)
+
+ def failed_runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess:
+ return subprocess.CompletedProcess([], 9)
+
+ assert not full.run_full_suite(
+ python=Path("/runtime/python"),
+ source_dir=source_dir,
+ dependency_dir=dependency_dir,
+ base_url="http://127.0.0.1:8000/v1",
+ model="MiniMax-M3",
+ output_dir=output_dir,
+ runner=failed_runner,
+ )
+ compatibility_path = next(output_dir.glob(full.COMPATIBILITY_GLOB))
+ compatibility = json.loads(compatibility_path.read_text(encoding="utf-8"))
+ assert compatibility["integration_error"]["type"] == "FullSuiteError"
+ assert compatibility["n-samples"]["minimax_m3_full"]["effective"] == 0
+ assert (output_dir / full.NATIVE_REPORT_FILENAME).is_file()
+ assert (output_dir / full.NATIVE_RESULTS_FILENAME).is_file()
diff --git a/utils/evals/test_minimax_provider_eval.py b/utils/evals/test_minimax_provider_eval.py
new file mode 100644
index 0000000000..7e88f49db9
--- /dev/null
+++ b/utils/evals/test_minimax_provider_eval.py
@@ -0,0 +1,285 @@
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+from typing import Any
+
+import minimax_provider_eval as mpe
+
+
+def _native_outputs(
+ output_dir: Path,
+ *,
+ model: str = "MiniMax-M3",
+ status: str = "success",
+ match_rate: float = 1.0,
+ schema_rate: float = 1.0,
+ reasoning_error_rate: float = 0.0,
+ tool_call_total: int = 10,
+) -> None:
+ (output_dir / mpe.NATIVE_RESULTS_FILENAME).write_text(
+ json.dumps({"data_index": 1, "status": status}) + "\n",
+ encoding="utf-8",
+ )
+ schema_errors = round((1.0 - schema_rate) * tool_call_total)
+ (output_dir / mpe.NATIVE_REPORT_FILENAME).write_text(
+ json.dumps(
+ {
+ "model": model,
+ "success_count": 1 if status == "success" else 0,
+ "failure_count": 0 if status == "success" else 1,
+ "tool_calls_match_rate": match_rate,
+ "tool_calls_schema_validation_error_count": schema_errors,
+ "tool_calls_total_count": tool_call_total,
+ "error_only_reasoning_rate": reasoning_error_rate,
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+
+
+def _compatibility(output_dir: Path) -> dict[str, Any]:
+ paths = list(output_dir.glob(mpe.COMPATIBILITY_GLOB))
+ assert len(paths) == 1
+ return json.loads(paths[0].read_text(encoding="utf-8"))
+
+
+def test_fixture_is_exact_pinned_upstream_row() -> None:
+ metadata, rows = mpe.load_fixture(mpe.DEFAULT_FIXTURE_PATH)
+
+ assert metadata["ref"] == mpe.UPSTREAM_REF
+ assert metadata["indices"] == [71]
+ assert set(rows[0]) == {"messages", "tools", "expected_tool_call"}
+ assert "data_index" not in rows[0]
+
+
+def test_prepare_smoke_input_preserves_fixture_row(tmp_path: Path) -> None:
+ destination = tmp_path / "smoke.jsonl"
+
+ mpe.prepare_smoke_input(
+ fixture_path=mpe.DEFAULT_FIXTURE_PATH,
+ destination=destination,
+ )
+
+ _, rows = mpe.load_fixture(mpe.DEFAULT_FIXTURE_PATH)
+ assert [json.loads(line) for line in destination.read_text().splitlines()] == rows
+
+
+def test_build_command_invokes_stock_verifier_without_source_changes(
+ tmp_path: Path,
+) -> None:
+ source_dir = tmp_path / "source"
+ output_dir = tmp_path / "output"
+ sample_path = tmp_path / "smoke.jsonl"
+
+ command = mpe.build_verifier_command(
+ python=Path("/venv/bin/python"),
+ source_dir=source_dir,
+ sample_path=sample_path,
+ base_url="http://127.0.0.1:8000/v1/",
+ model="MiniMax-M3",
+ output_dir=output_dir,
+ )
+
+ assert command[:3] == [
+ "/venv/bin/python",
+ str(source_dir / "verify.py"),
+ str(sample_path),
+ ]
+ assert command[command.index("--base-url") + 1] == "http://127.0.0.1:8000/v1"
+ assert command[command.index("--concurrency") + 1] == "1"
+ assert command[command.index("--timeout") + 1] == "600"
+ assert command[command.index("--retries") + 1] == "3"
+ assert json.loads(command[command.index("--extra-body") + 1]) == {
+ "temperature": 0,
+ "top_p": 1,
+ "max_tokens": 40960,
+ }
+
+
+def test_run_uses_verified_stock_source_and_projects_native_metrics(
+ tmp_path: Path, monkeypatch
+) -> None:
+ output_dir = tmp_path / "output"
+ source_dir = tmp_path / "source"
+ dependency_dir = tmp_path / "deps"
+ source_dir.mkdir()
+ dependency_dir.mkdir()
+ verified: list[Path] = []
+ invocation: dict[str, Any] = {}
+
+ monkeypatch.setattr(mpe, "verify_source_tree", verified.append)
+
+ def runner(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
+ invocation["command"] = command
+ invocation.update(kwargs)
+ _native_outputs(
+ output_dir,
+ match_rate=0.9,
+ schema_rate=0.8,
+ reasoning_error_rate=0.1,
+ )
+ return subprocess.CompletedProcess(command, 0)
+
+ passed = mpe.run_evaluation(
+ python=Path("/venv/bin/python"),
+ source_dir=source_dir,
+ dependency_dir=dependency_dir,
+ base_url="http://127.0.0.1:8000/v1",
+ model="MiniMax-M3",
+ output_dir=output_dir,
+ runner=runner,
+ )
+
+ assert passed is True
+ assert verified == [source_dir]
+ assert invocation["check"] is False
+ assert invocation["timeout"] == mpe.UPSTREAM_TIMEOUT_SECONDS
+ assert invocation["env"]["PYTHONPATH"] == f"{source_dir}:{dependency_dir}"
+ assert invocation["env"]["PYTHONNOUSERSITE"] == "1"
+ assert not (output_dir / "minimax_vendor_smoke_input.jsonl").exists()
+ compatibility = _compatibility(output_dir)
+ assert compatibility["results"][mpe.TASK_NAME]["exact_match,strict-match"] == 0.8
+ assert compatibility["n-samples"][mpe.TASK_NAME]["effective"] == 1
+ assert "integration_error" not in compatibility
+
+
+def test_completed_model_failure_is_not_reclassified_as_integration_error(
+ tmp_path: Path, monkeypatch
+) -> None:
+ output_dir = tmp_path / "output"
+ source_dir = tmp_path / "source"
+ dependency_dir = tmp_path / "deps"
+ source_dir.mkdir()
+ dependency_dir.mkdir()
+ monkeypatch.setattr(mpe, "verify_source_tree", lambda _: None)
+
+ def runner(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]:
+ _native_outputs(output_dir, match_rate=0.0, schema_rate=0.0)
+ return subprocess.CompletedProcess(command, 0)
+
+ passed = mpe.run_evaluation(
+ python=Path("python"),
+ source_dir=source_dir,
+ dependency_dir=dependency_dir,
+ base_url="http://127.0.0.1:8000/v1",
+ model="MiniMax-M3",
+ output_dir=output_dir,
+ runner=runner,
+ )
+
+ assert passed is True
+ compatibility = _compatibility(output_dir)
+ assert compatibility["results"][mpe.TASK_NAME]["exact_match,strict-match"] == 0.0
+ assert compatibility["n-samples"][mpe.TASK_NAME]["effective"] == 1
+ assert "integration_error" not in compatibility
+
+
+def test_completed_zero_tool_call_result_is_model_failure(
+ tmp_path: Path, monkeypatch
+) -> None:
+ output_dir = tmp_path / "output"
+ source_dir = tmp_path / "source"
+ dependency_dir = tmp_path / "deps"
+ source_dir.mkdir()
+ dependency_dir.mkdir()
+ monkeypatch.setattr(mpe, "verify_source_tree", lambda _: None)
+
+ def runner(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]:
+ _native_outputs(output_dir, match_rate=0.0, tool_call_total=0)
+ return subprocess.CompletedProcess(command, 0)
+
+ passed = mpe.run_evaluation(
+ python=Path("python"),
+ source_dir=source_dir,
+ dependency_dir=dependency_dir,
+ base_url="http://127.0.0.1:8000/v1",
+ model="MiniMax-M3",
+ output_dir=output_dir,
+ runner=runner,
+ )
+
+ assert passed is True
+ compatibility = _compatibility(output_dir)
+ assert compatibility["results"][mpe.TASK_NAME]["exact_match,strict-match"] == 0.0
+ assert compatibility["n-samples"][mpe.TASK_NAME]["effective"] == 1
+ assert "integration_error" not in compatibility
+
+
+def test_request_failure_is_reported_as_integration_error(
+ tmp_path: Path, monkeypatch
+) -> None:
+ output_dir = tmp_path / "output"
+ source_dir = tmp_path / "source"
+ dependency_dir = tmp_path / "deps"
+ source_dir.mkdir()
+ dependency_dir.mkdir()
+ monkeypatch.setattr(mpe, "verify_source_tree", lambda _: None)
+
+ def runner(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]:
+ _native_outputs(output_dir, status="failed")
+ return subprocess.CompletedProcess(command, 0)
+
+ passed = mpe.run_evaluation(
+ python=Path("python"),
+ source_dir=source_dir,
+ dependency_dir=dependency_dir,
+ base_url="http://127.0.0.1:8000/v1",
+ model="MiniMax-M3",
+ output_dir=output_dir,
+ runner=runner,
+ )
+
+ assert passed is False
+ compatibility = _compatibility(output_dir)
+ assert compatibility["n-samples"][mpe.TASK_NAME]["effective"] == 0
+ assert compatibility["integration_error"]["type"] == "SmokeSuiteError"
+ assert "request failure" in compatibility["integration_error"]["message"]
+ native = json.loads((output_dir / mpe.NATIVE_REPORT_FILENAME).read_text())
+ assert native["failure_count"] == 1
+ assert "integration_error" not in native
+
+
+def test_publish_failure_does_not_replace_existing_native_report(
+ tmp_path: Path,
+) -> None:
+ output_dir = tmp_path / "output"
+ output_dir.mkdir()
+ native_path = output_dir / mpe.NATIVE_REPORT_FILENAME
+ native_path.write_text('{"stock": true}\n', encoding="utf-8")
+
+ mpe.publish_failure(
+ output_dir=output_dir,
+ model="MiniMax-M3",
+ error=RuntimeError("transport failed"),
+ )
+
+ assert json.loads(native_path.read_text()) == {"stock": True}
+ compatibility = _compatibility(output_dir)
+ assert compatibility["integration_error"] == {
+ "type": "RuntimeError",
+ "message": "transport failed",
+ }
+
+
+def test_failure_cli_is_stdlib_only(tmp_path: Path) -> None:
+ output_dir = tmp_path / "output"
+
+ assert (
+ mpe.main(
+ [
+ "failure",
+ "--model",
+ "MiniMax-M3",
+ "--output-dir",
+ str(output_dir),
+ "--message",
+ "setup failed",
+ ]
+ )
+ == 0
+ )
+ compatibility = _compatibility(output_dir)
+ assert compatibility["integration_error"]["message"] == "setup failed"
diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py
index 45be3a6e4d..af762d5d92 100644
--- a/utils/evals/test_run_eval_dispatch.py
+++ b/utils/evals/test_run_eval_dispatch.py
@@ -1,25 +1,59 @@
-
from __future__ import annotations
+import hashlib
+import io
+import json
import os
import stat
import subprocess
+import sys
+import tarfile
+import threading
+from contextlib import contextmanager
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
-BENCHMARK_LIB = Path(__file__).resolve().parents[2] / "benchmarks" / "benchmark_lib.sh"
-
-_SCRIPT = r'''
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh"
+MULTINODE_AGENTIC_SCRIPT = REPO_ROOT / "benchmarks/multi_node/agentic_srt.sh"
+SINGLE_NODE_WORKFLOW = REPO_ROOT / ".github/workflows/benchmark-tmpl.yml"
+MULTINODE_WORKFLOW = REPO_ROOT / ".github/workflows/benchmark-multinode-tmpl.yml"
+E2E_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "e2e-tests.yml"
+AUTO_EVAL_FRAMEWORK_EXPR = (
+ "${{ inputs.eval-framework == 'auto' && "
+ "(matrix.config['eval-framework'] || 'lm-eval') || "
+ "inputs.eval-framework }}"
+)
+AUTO_EVAL_SUITE_EXPR = (
+ "${{ inputs.eval-suite != '' && inputs.eval-suite || "
+ "(inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}"
+)
+
+
+_SCRIPT = r"""
source "$BENCHMARK_LIB"
+_wait_for_openai_chat_route() { echo "READY=$*"; }
run_lm_eval() { echo "DISPATCH=lm-eval"; }
run_swebench_eval() { echo "DISPATCH=swebench"; }
+run_kimi_vendor_eval() { echo "DISPATCH=kimi-vendor"; }
+run_minimax_vendor_eval() { echo "DISPATCH=minimax-vendor"; }
+run_bfcl_eval() { echo "DISPATCH=bfcl"; }
append_lm_eval_summary() { echo "STAGED=summary"; }
export EVAL_MAX_MODEL_LEN=16384
-unset EVAL_CONCURRENT_REQUESTS
+export EVAL_CONCURRENT_REQUESTS=""
run_eval ${CLI_FW:+--framework "$CLI_FW"} --port 8888
-'''
+"""
-def _dispatch(*, is_agentic: str = "0", eval_only: str = "false", cli_fw=None, env_fw=None) -> str:
+def _dispatch(
+ *,
+ is_agentic: str = "0",
+ eval_only: str = "false",
+ cli_fw=None,
+ env_fw=None,
+) -> str:
env = {
**os.environ,
"BENCHMARK_LIB": str(BENCHMARK_LIB),
@@ -30,6 +64,7 @@ def _dispatch(*, is_agentic: str = "0", eval_only: str = "false", cli_fw=None, e
env.pop("EVAL_FRAMEWORK", None)
env.pop("CLI_FW", None)
env.pop("KV_OFFLOAD_BACKEND", None)
+ env.pop("EVAL_SUITE", None)
if cli_fw is not None:
env["CLI_FW"] = cli_fw
if env_fw is not None:
@@ -40,6 +75,53 @@ def _dispatch(*, is_agentic: str = "0", eval_only: str = "false", cli_fw=None, e
return res.stdout
+def test_agentic_dependency_install_is_rootless_without_git(tmp_path: Path) -> None:
+ fake_bin = tmp_path / "bin"
+ fake_bin.mkdir()
+ fake_uv = fake_bin / "uv"
+ fake_uv.write_text(
+ "#!/bin/bash\n"
+ "if [[ \"$1\" == \"venv\" ]]; then\n"
+ " target=\"${@: -1}\"\n"
+ " mkdir -p \"$target/bin\"\n"
+ " printf '#!/bin/sh\\nexit 0\\n' > \"$target/bin/aiperf\"\n"
+ " printf '#!/bin/sh\\nexit 0\\n' > \"$target/bin/hf\"\n"
+ " chmod +x \"$target/bin/aiperf\" \"$target/bin/hf\"\n"
+ "fi\n"
+ )
+ fake_uv.chmod(0o755)
+ fake_apt = fake_bin / "apt-get"
+ fake_apt.write_text("#!/bin/sh\nexit 97\n")
+ fake_apt.chmod(0o755)
+
+ env = {
+ **os.environ,
+ "AGENTIC_DIR": str(tmp_path / "agentic"),
+ "AIPERF_DIR": str(tmp_path / "aiperf"),
+ "AIPERF_RUNTIME_DIR": str(tmp_path / "runtime"),
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "FAKE_UV": str(fake_uv),
+ "PATH": f"{fake_bin}:/bin",
+ "PYTHONPYCACHEPREFIX": str(tmp_path / "pycache"),
+ }
+ result = subprocess.run(
+ [
+ "/bin/bash",
+ "-c",
+ 'source "$BENCHMARK_LIB"; '
+ 'ensure_agentic_uv() { AIPERF_UV_BIN="$FAKE_UV"; }; '
+ "install_agentic_deps",
+ ],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert (tmp_path / "runtime/venv/bin/aiperf").is_file()
+ assert (tmp_path / "runtime/venv/bin/hf").is_file()
+
def test_agentic_scenario_defaults_to_gsm8k_lm_eval():
assert "DISPATCH=lm-eval" in _dispatch(is_agentic="1")
@@ -48,6 +130,7 @@ def test_agentic_scenario_defaults_to_gsm8k_lm_eval():
def test_fixed_seqlen_scenario_defaults_to_lm_eval():
assert "DISPATCH=lm-eval" in _dispatch(is_agentic="0")
+
def test_agentic_eval_only_stages_summary():
output = _dispatch(is_agentic="1", eval_only="true")
assert "DISPATCH=lm-eval" in output
@@ -58,6 +141,15 @@ def test_fixed_seqlen_eval_only_leaves_staging_to_recipe():
assert "STAGED=summary" not in _dispatch(is_agentic="0", eval_only="true")
+def test_fixed_seqlen_provider_leaves_staging_to_recipe() -> None:
+ output = _dispatch(
+ is_agentic="0",
+ eval_only="true",
+ env_fw="minimax-vendor",
+ )
+ assert "DISPATCH=minimax-vendor" in output
+ assert "STAGED=summary" not in output
+
def test_explicit_framework_arg_overrides_scenario():
assert "DISPATCH=lm-eval" in _dispatch(is_agentic="1", cli_fw="lm-eval")
@@ -67,10 +159,75 @@ def test_env_framework_overrides_scenario():
assert "DISPATCH=lm-eval" in _dispatch(is_agentic="1", env_fw="lm-eval")
+def test_environment_framework_overrides_legacy_recipe_argument() -> None:
+ assert "DISPATCH=kimi-vendor" in _dispatch(
+ is_agentic="1",
+ cli_fw="bfcl",
+ env_fw="kimi-vendor",
+ )
+
+
def test_env_can_force_swebench_on_fixed_seqlen():
assert "DISPATCH=swebench" in _dispatch(is_agentic="0", env_fw="swebench")
+def test_env_can_force_kimi_vendor_on_agentic_eval() -> None:
+ assert "DISPATCH=kimi-vendor" in _dispatch(
+ is_agentic="1",
+ eval_only="true",
+ env_fw="kimi-vendor",
+ )
+
+
+def test_kimi_vendor_skips_unused_model_context_loading() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+unset EVAL_MAX_MODEL_LEN
+compute_eval_context_length() { echo "UNEXPECTED_CONTEXT_LOAD"; return 99; }
+run_kimi_vendor_eval() { echo "DISPATCH=kimi-vendor"; }
+export EVAL_FRAMEWORK=kimi-vendor
+export EVAL_CONCURRENT_REQUESTS=""
+export EVAL_ONLY=false
+export IS_AGENTIC=0
+run_eval --port 8888
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "DISPATCH=kimi-vendor" in result.stdout
+ assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout
+
+
+def test_kimi_failure_preserves_rc_without_eval_only() -> None:
+ script = r"""
+set -u
+source "$BENCHMARK_LIB"
+run_kimi_vendor_eval() { return 7; }
+export EVAL_FRAMEWORK=kimi-vendor
+export EVAL_CONCURRENT_REQUESTS=""
+export EVAL_MAX_MODEL_LEN=16384
+export IS_AGENTIC=0
+unset EVAL_ONLY
+run_eval --port 8888
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 7
+ assert "unbound variable" not in result.stderr
+
+
def test_recipe_lm_eval_arg_still_lm_eval_on_fixed_seqlen():
assert "DISPATCH=lm-eval" in _dispatch(is_agentic="0", cli_fw="lm-eval")
@@ -95,6 +252,1186 @@ def test_run_eval_rejects_missing_framework_value():
assert "--framework requires a value" in result.stderr
+def test_run_eval_rejects_unsafe_suite_name() -> None:
+ result = _run_invalid_call(
+ "EVAL_SUITE='kimi\"suite' run_eval --framework kimi-vendor"
+ )
+
+ assert result.returncode == 2
+ assert "EVAL_SUITE may contain only" in result.stderr
+
+
+def test_run_eval_rejects_suite_override_for_lm_eval() -> None:
+ result = _run_invalid_call("EVAL_SUITE=gpqa_diamond run_eval --framework lm-eval")
+
+ assert result.returncode == 2
+ assert "only supported with kimi-vendor, minimax-vendor, or bfcl" in result.stderr
+
+
+def test_run_eval_scopes_runner_selected_suite_to_one_call() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+run_kimi_vendor_eval() {
+ export EVAL_SUITE=kimi_tool_call_schema
+ echo "DISPATCH=kimi-vendor SUITE=$EVAL_SUITE"
+}
+run_lm_eval() {
+ echo "DISPATCH=lm-eval SUITE=${EVAL_SUITE:-unset} COMPLETED=${EVAL_COMPLETED_SUITE:-unset}"
+}
+append_lm_eval_summary() {
+ echo "METADATA=${EVAL_COMPLETED_SUITE:-gsm8k}"
+}
+export EVAL_MAX_MODEL_LEN=16384
+export EVAL_CONCURRENT_REQUESTS=""
+export EVAL_ONLY=false
+export IS_AGENTIC=0
+unset EVAL_SUITE
+export EVAL_FRAMEWORK=kimi-vendor
+run_eval --port 8888
+printf 'KIMI_COMPLETED=%s\n' "${EVAL_COMPLETED_SUITE:-unset}"
+append_lm_eval_summary
+export EVAL_FRAMEWORK=lm-eval
+run_eval --port 8888
+printf 'LM_COMPLETED=%s\n' "${EVAL_COMPLETED_SUITE:-unset}"
+append_lm_eval_summary
+printf 'FINAL_SUITE=%s\n' "${EVAL_SUITE-unset}"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "DISPATCH=kimi-vendor SUITE=kimi_tool_call_schema" in result.stdout
+ assert "KIMI_COMPLETED=kimi_tool_call_schema" in result.stdout
+ assert "METADATA=kimi_tool_call_schema" in result.stdout
+ assert "DISPATCH=lm-eval SUITE=unset COMPLETED=unset" in result.stdout
+ assert "LM_COMPLETED=unset" in result.stdout
+ assert "METADATA=gsm8k" in result.stdout
+ assert "FINAL_SUITE=unset" in result.stdout
+
+
+def test_kimi_default_suite_reaches_eval_only_metadata() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+_wait_for_openai_chat_route() { :; }
+run_kimi_vendor_eval() { echo "DISPATCH=$EVAL_SUITE"; }
+append_lm_eval_summary() { echo "METADATA=$EVAL_COMPLETED_SUITE"; }
+export EVAL_FRAMEWORK=kimi-vendor
+export EVAL_ONLY=true
+export IS_AGENTIC=1
+export EVAL_CONCURRENT_REQUESTS=""
+unset EVAL_SUITE
+run_eval --port 8888
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "DISPATCH=kimi_tool_call_schema" in result.stdout
+ assert "METADATA=kimi_tool_call_schema" in result.stdout
+
+
+def test_agentic_eval_propagates_artifact_staging_failure() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+_wait_for_openai_chat_route() { :; }
+run_kimi_vendor_eval() { :; }
+append_lm_eval_summary() { return 73; }
+export EVAL_FRAMEWORK=kimi-vendor
+export EVAL_ONLY=true
+export IS_AGENTIC=1
+export EVAL_CONCURRENT_REQUESTS=""
+unset EVAL_SUITE
+run_eval --port 8888
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 73
+ assert "eval artifact staging failed with exit code 73" in result.stderr
+
+
+def test_kimi_full_suite_dispatches_to_schema_runner() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+_run_kimi_tool_call_schema_eval() {
+ printf 'DISPATCH=%s ARGS=<%s>\n' "$EVAL_SUITE" "$*"
+}
+EVAL_SUITE=kimi_tool_call_schema_full run_kimi_vendor_eval --port 9999
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "DISPATCH=kimi_tool_call_schema_full ARGS=<--port 9999>" in result.stdout
+
+
+def test_minimax_full_suite_dispatches_to_full_runner() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+_run_minimax_m3_full_eval() {
+ printf 'DISPATCH=%s ARGS=<%s>\n' "$EVAL_SUITE" "$*"
+}
+EVAL_SUITE=minimax_m3_full run_minimax_vendor_eval --port 9999
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "DISPATCH=minimax_m3_full ARGS=<--port 9999>" in result.stdout
+
+
+def test_kimi_vendor_rejects_batched_concurrency() -> None:
+ result = _run_invalid_call(
+ "EVAL_MAX_MODEL_LEN=16384 "
+ "EVAL_CONCURRENT_REQUESTS='1 4' "
+ "run_eval --framework kimi-vendor"
+ )
+ assert result.returncode == 1
+ assert "batched eval concurrency is only supported for lm-eval" in result.stderr
+
+
+def test_kimi_vendor_rejects_unsupported_suite() -> None:
+ result = _run_invalid_call("EVAL_SUITE=gsm8k run_kimi_vendor_eval")
+ assert result.returncode == 2
+ assert "unsupported Kimi Vendor Verifier suite 'gsm8k'" in result.stderr
+
+
+def _run_minimax_dispatch(*, suite: str | None = None, concurrency: str = "") -> str:
+ script = r"""
+source "$BENCHMARK_LIB"
+unset EVAL_MAX_MODEL_LEN
+compute_eval_context_length() { echo "UNEXPECTED_CONTEXT_LOAD"; return 99; }
+MINIMAX_DISPATCH_COUNT=0
+run_minimax_vendor_eval() {
+ MINIMAX_DISPATCH_COUNT=$((MINIMAX_DISPATCH_COUNT + 1))
+ printf 'DISPATCH=minimax-vendor SUITE=%s ARGS=<%s>\n' "$EVAL_SUITE" "$*"
+}
+export EVAL_CONCURRENT_REQUESTS="$TEST_EVAL_CONCURRENCY"
+export EVAL_ONLY=false
+export IS_AGENTIC=0
+run_eval --framework minimax-vendor --port 9999
+printf 'DISPATCH_COUNT=%s\n' "$MINIMAX_DISPATCH_COUNT"
+printf 'COMPLETED_SUITE=%s\n' "$EVAL_COMPLETED_SUITE"
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "MODEL": "served-model",
+ "MODEL_PREFIX": "minimaxm3",
+ "TEST_EVAL_CONCURRENCY": concurrency,
+ }
+ for key in ("EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_COMPLETED_SUITE"):
+ env.pop(key, None)
+ if suite is not None:
+ env["EVAL_SUITE"] = suite
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout
+ return result.stdout
+
+
+def test_minimax_vendor_defaults_suite_dispatches_once_and_records_completion() -> None:
+ output = _run_minimax_dispatch()
+
+ assert "DISPATCH=minimax-vendor SUITE=minimax_m3_smoke" in output
+ assert "ARGS=<--port 9999>" in output
+ assert "DISPATCH_COUNT=1" in output
+ assert "COMPLETED_SUITE=minimax_m3_smoke" in output
+
+
+def test_minimax_vendor_accepts_explicit_supported_suite() -> None:
+ output = _run_minimax_dispatch(suite="minimax_m3_smoke")
+
+ assert "DISPATCH=minimax-vendor SUITE=minimax_m3_smoke" in output
+ assert "DISPATCH_COUNT=1" in output
+ assert "COMPLETED_SUITE=minimax_m3_smoke" in output
+
+
+def test_minimax_vendor_rejects_unsupported_suite() -> None:
+ result = _run_invalid_call(
+ "MODEL_PREFIX=minimaxm3 EVAL_SUITE=gsm8k run_eval --framework minimax-vendor"
+ )
+
+ assert result.returncode == 2
+ assert "unsupported MiniMax Provider Verifier suite 'gsm8k'" in result.stderr
+
+
+def test_run_eval_rejects_unknown_framework() -> None:
+ result = _run_invalid_call(
+ "EVAL_MAX_MODEL_LEN=16384 run_eval --framework not-a-framework"
+ )
+
+ assert result.returncode == 1
+ assert "Unknown framework 'not-a-framework'" in result.stdout
+
+
+def test_minimax_vendor_rejects_concurrency_sweep_for_sequential_smoke() -> None:
+ result = _run_invalid_call(
+ "MODEL_PREFIX=minimaxm3 "
+ "EVAL_CONCURRENT_REQUESTS='1 4' "
+ "run_eval --framework minimax-vendor"
+ )
+
+ assert result.returncode == 1
+ assert "batched eval concurrency is only supported for lm-eval" in result.stderr
+
+
+def test_minimax_vendor_ignores_single_launcher_concurrency_value() -> None:
+ output = _run_minimax_dispatch(concurrency="128")
+
+ assert "DISPATCH=minimax-vendor SUITE=minimax_m3_smoke" in output
+ assert "DISPATCH_COUNT=1" in output
+
+
+def test_minimax_vendor_accepts_non_m3_model() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+_run_minimax_m3_smoke_eval() { echo "DISPATCH=$EVAL_SUITE"; }
+unset EVAL_SUITE EVAL_RESULT_DIR
+MODEL=moonshotai/Kimi-K3 MODEL_PREFIX=kimik3 run_minimax_vendor_eval
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "DISPATCH=minimax_m3_smoke" in result.stdout
+
+
+def test_minimax_vendor_accepts_case_insensitive_m3_model_name() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+_run_minimax_m3_smoke_eval() { echo "DISPATCH=$EVAL_SUITE"; }
+unset MODEL_PREFIX EVAL_SUITE EVAL_RESULT_DIR
+MODEL_NAME=vendor/MINIMAX-M3-custom run_minimax_vendor_eval
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "DISPATCH=minimax_m3_smoke" in result.stdout
+
+
+def test_minimax_vendor_setup_failure_uses_integration_error_and_stages(
+ tmp_path: Path,
+) -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+unset EVAL_SUITE EVAL_RESULT_DIR EVAL_COMPLETED_SUITE
+unset VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+_prepare_vendor_verifier_python() {
+ if compgen -G "$RESULTS_DIR/results_minimax_vendor_*.json" >/dev/null \
+ || [ -e "$RESULTS_DIR/minimax_vendor_report.json" ]; then
+ echo "STALE_MINIMAX_ARTIFACT"
+ return 99
+ fi
+ mkdir -p "$PYTHON_DIR"
+ cat >"$PYTHON_DIR/python3" <<'PY'
+#!/bin/bash
+printf 'ADAPTER_ARG=<%s>\n' "$@"
+PY
+ chmod +x "$PYTHON_DIR/python3"
+ export VENDOR_VERIFIER_PYTHON="$PYTHON_DIR/python3"
+ export VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$PYTHON_DIR"
+}
+_prepare_minimax_m3_full_runtime() { return 12; }
+append_lm_eval_summary() {
+ printf 'STAGED=<%s>\n' "$EVAL_RESULT_DIR"
+ printf 'STAGED_CONC=<%s>\n' "$CONC"
+}
+export MODEL_PREFIX=minimaxm3
+export MODEL=test-model
+export EVAL_CONCURRENT_REQUESTS=7
+export EVAL_ONLY=false
+export IS_AGENTIC=0
+run_eval --framework minimax-vendor --results-dir "$RESULTS_DIR"
+eval_rc=$?
+printf 'EVAL_RC=%s\n' "$eval_rc"
+"""
+ results_dir = tmp_path / "results"
+ results_dir.mkdir()
+ (results_dir / "results_minimax_vendor_stale.json").write_text("{}")
+ (results_dir / "minimax_vendor_report.json").write_text("{}")
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(results_dir),
+ "PYTHON_DIR": str(tmp_path / "python"),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ output = result.stdout + result.stderr
+
+ assert "EVAL_RC=12" in output
+ assert "STALE_MINIMAX_ARTIFACT" not in output
+ assert not (tmp_path / "python").exists()
+ assert (
+ f"ADAPTER_ARG=<{REPO_ROOT / 'utils/evals/minimax_provider_eval.py'}>" in output
+ )
+ assert "ADAPTER_ARG=" in output
+ assert f"ADAPTER_ARG=<{results_dir}>" in output
+ assert "ADAPTER_ARG=" in output
+ assert "ADAPTER_ARG=<--message>" in output
+ assert (
+ "ADAPTER_ARG="
+ ) in output
+ assert f"STAGED=<{results_dir}>" in output
+ assert output.count("STAGED=<") == 1
+ assert "STAGED_CONC=<7>" in output
+
+
+def test_minimax_full_dependency_install_matches_pinned_upstream_requirements(
+ tmp_path: Path,
+) -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+selected_python() { printf 'PYTHON_ARG=<%s>\n' "$@"; }
+VENDOR_VERIFIER_PYTHON=selected_python
+_install_minimax_m3_full_deps "$RUNTIME_DIR"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RUNTIME_DIR": str(tmp_path / "runtime"),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ for requirement in (
+ "jsonschema==4.25.1",
+ "loguru==0.7.3",
+ "megfile==4.2.5",
+ "numpy==2.3.4",
+ "openai==2.7.1",
+ "tqdm==4.67.1",
+ ):
+ assert f"PYTHON_ARG=<{requirement}>" in result.stdout
+ assert "--break-system-packages" not in result.stdout
+
+
+def test_minimax_runtime_prepares_source_with_full_adapter(tmp_path: Path) -> None:
+ runtime_dir = tmp_path / "runtime"
+ calls_path = tmp_path / "calls"
+ script = r"""
+source "$BENCHMARK_LIB"
+mktemp() {
+ mkdir -p "$RUNTIME_DIR"
+ printf '%s\n' "$RUNTIME_DIR"
+}
+selected_python() {
+ printf 'PYTHON_ARG=<%s>\n' "$@" >> "$CALLS_PATH"
+ mkdir -p "$RUNTIME_DIR/source"
+}
+_install_minimax_m3_full_deps() { mkdir -p "$1"; }
+VENDOR_VERIFIER_PYTHON=selected_python
+prepared_runtime=$(_prepare_minimax_m3_full_runtime)
+printf 'RUNTIME=<%s>\n' "$prepared_runtime"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RUNTIME_DIR": str(runtime_dir),
+ "CALLS_PATH": str(calls_path),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ calls = calls_path.read_text()
+
+ assert f"PYTHON_ARG=<{REPO_ROOT / 'utils/evals/minimax_m3_full_eval.py'}>" in calls
+ assert "PYTHON_ARG=" in calls
+ assert f"PYTHON_ARG=<{runtime_dir / 'source'}>" in calls
+ assert "minimax_provider_eval.py" not in calls
+ assert f"RUNTIME=<{runtime_dir}>" in result.stdout
+
+
+def test_minimax_vendor_runner_uses_fixed_adapter_contract(tmp_path: Path) -> None:
+ results_dir = tmp_path / "results"
+ runtime_dir = tmp_path / "runtime"
+ python_dir = tmp_path / "python"
+ script = r"""
+source "$BENCHMARK_LIB"
+selected_python() {
+ printf 'PYTHONPATH=<%s>\n' "$PYTHONPATH" >&2
+ printf 'PYTHON_ARG=<%s>\n' "$@" >&2
+}
+_prepare_vendor_verifier_python() {
+ mkdir "$PYTHON_DIR"
+ VENDOR_VERIFIER_PYTHON=selected_python
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$PYTHON_DIR"
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+}
+_prepare_minimax_m3_full_runtime() {
+ mkdir -p "$RUNTIME_DIR/source" "$RUNTIME_DIR/deps"
+ printf '%s\n' "$RUNTIME_DIR"
+}
+mktemp() { echo "UNEXPECTED_DEFAULT_RESULTS_DIR" >&2; return 99; }
+run_minimax_vendor_eval --port 9999 --results-dir "$RESULTS_DIR"
+printf 'EVAL_SUITE=%s\n' "$EVAL_SUITE"
+printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR"
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(results_dir),
+ "RUNTIME_DIR": str(runtime_dir),
+ "PYTHON_DIR": str(python_dir),
+ "MODEL": "test-model",
+ "MODEL_PREFIX": "minimaxm3",
+ "OPENAI_API_KEY": "must-not-be-forwarded",
+ }
+ for key in ("EVAL_SUITE", "EVAL_RESULT_DIR", "MODEL_NAME"):
+ env.pop(key, None)
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ output = result.stdout + result.stderr
+ adapter = REPO_ROOT / "utils/evals/minimax_provider_eval.py"
+ fixture = REPO_ROOT / "utils/evals/minimax_m3_smoke.json"
+
+ for value in (
+ adapter,
+ "run",
+ "selected_python",
+ runtime_dir / "source",
+ runtime_dir / "deps",
+ "http://127.0.0.1:9999/v1",
+ "test-model",
+ results_dir,
+ fixture,
+ ):
+ assert f"PYTHON_ARG=<{value}>" in output
+ for option in (
+ "--python",
+ "--source-dir",
+ "--dependency-dir",
+ "--base-url",
+ "--model",
+ "--output-dir",
+ "--fixture",
+ ):
+ assert f"PYTHON_ARG=<{option}>" in output
+ assert "must-not-be-forwarded" not in output
+ assert "UNEXPECTED_DEFAULT_RESULTS_DIR" not in output
+ assert "EVAL_SUITE=minimax_m3_smoke" in output
+ assert f"EVAL_RESULT_DIR={results_dir}" in output
+ assert not runtime_dir.exists()
+ assert not python_dir.exists()
+
+
+def test_kimi_vendor_setup_failure_writes_compatibility_result(
+ tmp_path: Path,
+) -> None:
+ results_dir = tmp_path / "results"
+ results_dir.mkdir()
+ (results_dir / "results_kimi_vendor_stale.json").write_text("{}")
+ (results_dir / "kimi_vendor_report.json").write_text("{}")
+ python_dir = tmp_path / "python"
+ script = r"""
+source "$BENCHMARK_LIB"
+_prepare_vendor_verifier_python() {
+ if compgen -G "$RESULTS_DIR/results_kimi_vendor_*.json" >/dev/null \
+ || [ -e "$RESULTS_DIR/kimi_vendor_report.json" ]; then
+ echo "STALE_KIMI_ARTIFACT"
+ return 99
+ fi
+ mkdir "$PYTHON_DIR"
+ cat >"$PYTHON_DIR/python3" <<'PY'
+#!/bin/bash
+exec /usr/bin/env python3 "$@"
+PY
+ chmod +x "$PYTHON_DIR/python3"
+ VENDOR_VERIFIER_PYTHON="$PYTHON_DIR/python3"
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$PYTHON_DIR"
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+}
+_prepare_kimi_vendor_runtime() { return 12; }
+run_kimi_vendor_eval --results-dir "$RESULTS_DIR"
+printf 'SETUP_RC=%s\n' "$?"
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(results_dir),
+ "PYTHON_DIR": str(python_dir),
+ "MODEL": "test-model",
+ "IS_MULTINODE": "false",
+ "KV_OFFLOADING": "none",
+ }
+ for key in (
+ "EVAL_SUITE",
+ "EVAL_RESULT_DIR",
+ "MODEL_NAME",
+ ):
+ env.pop(key, None)
+
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ message = "Kimi Vendor Verifier dependency installation failed with exit code 12"
+ score_files = list(results_dir.glob("results*.json"))
+
+ assert "SETUP_RC=12" in result.stdout
+ assert "STALE_KIMI_ARTIFACT" not in result.stdout + result.stderr
+ assert message in result.stderr
+ assert "failed to write Kimi verifier failure artifact" not in result.stderr
+ assert len(score_files) == 1
+ score_result = json.loads(score_files[0].read_text())
+ assert (
+ score_result["results"]["kimi_tool_call_schema"]["exact_match,strict-match"]
+ == 0.0
+ )
+ assert score_result["integration_error"]["message"] == message
+ native_result = json.loads((results_dir / "kimi_vendor_report.json").read_text())
+ assert native_result["completed"] is False
+ assert native_result["integration_error"]["message"] == message
+ assert not python_dir.exists()
+
+
+def test_preclear_failure_cannot_stage_stale_provider_result(tmp_path: Path) -> None:
+ results_dir = tmp_path / "results"
+ results_dir.mkdir()
+ stale_result = results_dir / "results_kimi_vendor_stale.json"
+ stale_result.write_text('{"stale": true}\n')
+ work_dir = tmp_path / "work"
+ work_dir.mkdir()
+ script = r"""
+source "$BENCHMARK_LIB"
+rm() { return 73; }
+append_lm_eval_summary() {
+ if [ -n "${EVAL_RESULT_DIR:-}" ]; then
+ echo "UNEXPECTED_STAGING"
+ fi
+ return 1
+}
+unset EVAL_FRAMEWORK EVAL_SUITE EVAL_RESULT_DIR EVAL_COMPLETED_SUITE
+export MODEL=test-model
+cd "$WORK_DIR"
+run_eval --framework kimi-vendor --results-dir "$RESULTS_DIR"
+printf 'EVAL_RC=%s\n' "$?"
+printf 'EVAL_RESULT_DIR=<%s>\n' "${EVAL_RESULT_DIR:-}"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(results_dir),
+ "WORK_DIR": str(work_dir),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "EVAL_RC=73" in result.stdout
+ assert "EVAL_RESULT_DIR=<>" in result.stdout
+ assert "UNEXPECTED_STAGING" not in result.stdout
+ assert "failed to remove stale eval artifact" in result.stderr
+ assert stale_result.is_file()
+ assert list(work_dir.iterdir()) == []
+
+
+_KIMI_VERIFIER_REQUIRED_FILES = {
+ "pyproject.toml",
+ "tests/conftest.py",
+ "tests/__init__.py",
+ "tests/tool_call_json_schema/conftest.py",
+ "tests/tool_call_json_schema/__init__.py",
+ "tests/tool_call_json_schema/test_tool_call_json_schema.py",
+ "tests/tool_call_json_schema/validator.py",
+ *{
+ f"testdata/walle_validator_cases/validator_cases/{case}/valid.jsonl"
+ for case in (
+ "TestAdditionalProperties",
+ "TestAnyOf",
+ "TestBasicTypes",
+ "TestDefs",
+ "TestDescription",
+ "TestEnforcerCases",
+ "TestID",
+ "TestKeywordsValidation",
+ "TestNestedDefsDepth",
+ "TestNumberFormat",
+ "TestRangeConstraints",
+ "TestRefInProperties",
+ "TestReferences",
+ "TestRequired",
+ "TestSingleTypeInArray",
+ "TestTypeLocation",
+ )
+ },
+}
+
+
+def _kimi_verifier_archive(
+ *,
+ missing: str | None = None,
+ unsafe_member: tarfile.TarInfo | None = None,
+) -> bytes:
+ output = io.BytesIO()
+ with tarfile.open(fileobj=output, mode="w:gz") as archive:
+ for relative_path in sorted(_KIMI_VERIFIER_REQUIRED_FILES - {missing}):
+ payload = relative_path.encode()
+ member = tarfile.TarInfo(f"verifier-pinned/{relative_path}")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ extra = b"must not be extracted"
+ member = tarfile.TarInfo("verifier-pinned/README.md")
+ member.size = len(extra)
+ archive.addfile(member, io.BytesIO(extra))
+ if unsafe_member is not None:
+ archive.addfile(
+ unsafe_member,
+ io.BytesIO(b"unsafe") if unsafe_member.isfile() else None,
+ )
+ return output.getvalue()
+
+
+@contextmanager
+def _serve_archive(payload: bytes, *, transient_failures: int = 0):
+ request_paths = []
+ request_count = 0
+
+ class ArchiveHandler(BaseHTTPRequestHandler):
+ def do_GET(self):
+ nonlocal request_count
+ request_paths.append(self.path)
+ request_count += 1
+ if request_count <= transient_failures:
+ self.send_response(503)
+ self.end_headers()
+ return
+ self.send_response(200)
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+ def log_message(self, *args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), ArchiveHandler)
+ thread = threading.Thread(target=server.serve_forever)
+ thread.start()
+ try:
+ yield (
+ f"http://127.0.0.1:{server.server_port}/owner/verifier.git",
+ request_paths,
+ )
+ finally:
+ server.shutdown()
+ thread.join()
+ server.server_close()
+
+
+def _prepare_local_kimi_verifier(
+ tmp_path: Path,
+ payload: bytes,
+ verifier_ref: str = "1" * 40,
+ transient_failures: int = 0,
+ archive_sha256: str | None = None,
+) -> tuple[subprocess.CompletedProcess[str], Path, list[str]]:
+ checkout = tmp_path / "checkout"
+ script = r"""
+source "$BENCHMARK_LIB"
+git() { echo "git must not be invoked" >&2; return 127; }
+mktemp() { mkdir "$CHECKOUT"; printf '%s\n' "$CHECKOUT"; }
+_prepare_kimi_vendor_verifier "$REPO_URL" "$VERIFIER_REF" "$ARCHIVE_SHA256"
+"""
+ with _serve_archive(
+ payload,
+ transient_failures=transient_failures,
+ ) as (repo_url, request_paths):
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "CHECKOUT": str(checkout),
+ "REPO_URL": repo_url,
+ "VERIFIER_REF": verifier_ref,
+ "ARCHIVE_SHA256": archive_sha256 or hashlib.sha256(payload).hexdigest(),
+ },
+ text=True,
+ capture_output=True,
+ )
+ return result, checkout, request_paths
+
+
+def test_kimi_vendor_verifier_fetches_expected_subset_without_git(
+ tmp_path: Path,
+) -> None:
+ result, checkout, request_paths = _prepare_local_kimi_verifier(
+ tmp_path,
+ _kimi_verifier_archive(),
+ )
+ verifier_ref = "1" * 40
+
+ assert result.returncode == 0, result.stderr
+ assert result.stdout.strip() == str(checkout)
+ assert request_paths == [
+ f"/owner/verifier/archive/{verifier_ref}.tar.gz",
+ ]
+ assert {
+ path.relative_to(checkout).as_posix()
+ for path in checkout.rglob("*")
+ if path.is_file()
+ } == _KIMI_VERIFIER_REQUIRED_FILES
+ assert "git must not be invoked" not in result.stderr
+
+
+def test_kimi_vendor_verifier_retries_transient_archive_failure(
+ tmp_path: Path,
+) -> None:
+ result, checkout, request_paths = _prepare_local_kimi_verifier(
+ tmp_path,
+ _kimi_verifier_archive(),
+ transient_failures=1,
+ )
+ verifier_ref = "1" * 40
+
+ assert result.returncode == 0, result.stderr
+ assert result.stdout.strip() == str(checkout)
+ assert request_paths == [
+ f"/owner/verifier/archive/{verifier_ref}.tar.gz",
+ f"/owner/verifier/archive/{verifier_ref}.tar.gz",
+ ]
+ assert "archive download attempt 1/3 failed" in result.stderr
+
+
+def test_kimi_vendor_verifier_rejects_archive_hash_mismatch(
+ tmp_path: Path,
+) -> None:
+ result, checkout, _ = _prepare_local_kimi_verifier(
+ tmp_path,
+ _kimi_verifier_archive(),
+ archive_sha256="0" * 64,
+ )
+
+ assert result.returncode == 1
+ assert "archive SHA256 mismatch" in result.stderr
+ assert not checkout.exists()
+
+
+def test_kimi_vendor_verifier_removes_partial_checkout_when_member_missing(
+ tmp_path: Path,
+) -> None:
+ missing = "tests/tool_call_json_schema/validator.py"
+ result, checkout, _ = _prepare_local_kimi_verifier(
+ tmp_path,
+ _kimi_verifier_archive(missing=missing),
+ )
+
+ assert result.returncode == 1
+ assert missing in result.stderr
+ assert not checkout.exists()
+
+
+def test_kimi_vendor_verifier_rejects_unsafe_archive_members(tmp_path: Path) -> None:
+ unsafe = tarfile.TarInfo("verifier-pinned/../../escaped")
+ unsafe.size = len(b"unsafe")
+ result, checkout, _ = _prepare_local_kimi_verifier(
+ tmp_path,
+ _kimi_verifier_archive(unsafe_member=unsafe),
+ )
+
+ assert result.returncode == 1
+ assert "unsafe archive member path" in result.stderr
+ assert not checkout.exists()
+ assert not (tmp_path / "escaped").exists()
+
+
+def test_kimi_vendor_uses_system_python_fast_path() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+python3() {
+ printf 'SYSTEM_PYTHON_ARG=<%s>\n' "$@"
+ [ "$1" = "-c" ]
+}
+mktemp() { echo "UNEXPECTED_MKTEMP"; return 99; }
+VENDOR_VERIFIER_PYTHON=/previous/python
+VENDOR_VERIFIER_PYTHON_CLEANUP_DIR=/previous/runtime
+_prepare_vendor_verifier_python "Kimi Vendor Verifier" "kimi-vendor-python"
+printf 'SELECTED_PYTHON=<%s>\n' "$VENDOR_VERIFIER_PYTHON"
+printf 'PYTHON_CLEANUP=<%s>\n' "$VENDOR_VERIFIER_PYTHON_CLEANUP_DIR"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "SYSTEM_PYTHON_ARG=<-c>" in result.stdout
+ assert "SELECTED_PYTHON=" in result.stdout
+ assert "PYTHON_CLEANUP=<>" in result.stdout
+ assert "UNEXPECTED_MKTEMP" not in result.stdout
+
+
+def test_kimi_vendor_bootstraps_pinned_python_and_cleans_it(
+ tmp_path: Path,
+) -> None:
+ log_path = tmp_path / "bootstrap.log"
+ fake_uv = tmp_path / "fake-uv"
+ fake_uv.write_text(
+ r"""#!/usr/bin/env bash
+printf 'UV_CACHE_DIR=<%s>\n' "$UV_CACHE_DIR" >> "$KIMI_LOG"
+printf 'UV_PYTHON_INSTALL_DIR=<%s>\n' "$UV_PYTHON_INSTALL_DIR" >> "$KIMI_LOG"
+printf 'UV_ARG=<%s>\n' "$@" >> "$KIMI_LOG"
+venv_dir="${!#}"
+mkdir -p "$venv_dir/bin"
+cat > "$venv_dir/bin/python" <<'PYTHON'
+#!/usr/bin/env bash
+printf 'SELECTED_PYTHON_ARG=<%s>\n' "$@" >> "$KIMI_LOG"
+PYTHON
+chmod +x "$venv_dir/bin/python"
+"""
+ )
+ fake_uv.chmod(0o755)
+ script = r"""
+source "$BENCHMARK_LIB"
+python3() {
+ if [ "$1" = "-c" ]; then
+ printf 'VERSION_CHECK\n' >> "$KIMI_LOG"
+ return 1
+ fi
+ printf 'SYSTEM_PYTHON_ARG=<%s>\n' "$@" >> "$KIMI_LOG"
+ local prefix=""
+ while [[ $# -gt 0 ]]; do
+ if [ "$1" = "--prefix" ]; then
+ prefix="$2"
+ break
+ fi
+ shift
+ done
+ mkdir -p "$prefix/bin"
+ cp "$FAKE_UV" "$prefix/bin/uv"
+ chmod +x "$prefix/bin/uv"
+}
+_prepare_vendor_verifier_python "Kimi Vendor Verifier" "kimi-vendor-python"
+cleanup_dir="$VENDOR_VERIFIER_PYTHON_CLEANUP_DIR"
+printf 'SELECTED_PYTHON=<%s>\n' "$VENDOR_VERIFIER_PYTHON"
+printf 'PYTHON_CLEANUP=<%s>\n' "$cleanup_dir"
+runtime_dir="$TEST_ROOT/runtime"
+mkdir "$runtime_dir"
+_install_kimi_vendor_eval_deps "$runtime_dir"
+_cleanup_vendor_eval "$runtime_dir" "$cleanup_dir"
+[ ! -e "$runtime_dir" ] && [ ! -e "$cleanup_dir" ] && printf 'CLEANED\n'
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "FAKE_UV": str(fake_uv),
+ "KIMI_LOG": str(log_path),
+ "TEST_ROOT": str(tmp_path),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ log = log_path.read_text()
+
+ assert "VERSION_CHECK" in log
+ assert "SYSTEM_PYTHON_ARG=<--prefix>" in log
+ assert "SYSTEM_PYTHON_ARG=<--break-system-packages>" in log
+ assert "SYSTEM_PYTHON_ARG=" in log
+ assert "UV_ARG=" in log
+ assert "UV_ARG=<--python>" in log
+ assert "UV_ARG=<3.12>" in log
+ assert "UV_ARG=<--seed>" in log
+ assert "UV_CACHE_DIR=" in log
+ assert "SELECTED_PYTHON_ARG= None:
+ runtime_dir = tmp_path / "runtime"
+ script = r"""
+source "$BENCHMARK_LIB"
+selected_python() { printf 'PYTHON_ARG=<%s>\n' "$@"; }
+VENDOR_VERIFIER_PYTHON=selected_python
+_install_kimi_vendor_eval_deps "$RUNTIME_DIR"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RUNTIME_DIR": str(runtime_dir),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "PYTHON_ARG=<--target>" in result.stdout
+ assert f"PYTHON_ARG=<{runtime_dir}>" in result.stdout
+ assert "PYTHON_ARG= None:
+ script = r"""
+source "$BENCHMARK_LIB"
+_prepare_vendor_verifier_python() { return 12; }
+_write_kimi_vendor_integration_error() { return 23; }
+run_kimi_vendor_eval --results-dir "$RESULTS_DIR"
+printf 'EVAL_RC=%s\n' "$?"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(tmp_path / "results"),
+ "MODEL": "test-model",
+ "IS_MULTINODE": "false",
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "EVAL_RC=12" in result.stdout
+ assert "failed to write Kimi verifier failure artifact" in result.stderr
+
+
+def test_kimi_vendor_multinode_runner_uses_fixed_upstream_contract(
+ tmp_path: Path,
+) -> None:
+ results_dir = tmp_path / "results"
+ verifier_dir = tmp_path / "verifier"
+ runtime_dir = tmp_path / "runtime"
+ python_dir = tmp_path / "python"
+ verifier_dir.mkdir()
+ script = r"""
+source "$BENCHMARK_LIB"
+_prepare_vendor_verifier_python() {
+ mkdir "$PYTHON_DIR"
+ VENDOR_VERIFIER_PYTHON=selected_python
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$PYTHON_DIR"
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+}
+_prepare_kimi_vendor_runtime() {
+ mkdir "$RUNTIME_DIR"
+ _install_kimi_vendor_eval_deps "$RUNTIME_DIR" >&2
+ printf '%s\n' "$RUNTIME_DIR"
+}
+_prepare_kimi_vendor_verifier() {
+ printf 'CHECKOUT=%s@%s\n' "$1" "$2" >&2
+ printf 'CHECKOUT_SHA=%s\n' "$3" >&2
+ "$VENDOR_VERIFIER_PYTHON" - "$1" "$2" "$3" "$VERIFIER_DIR" <<'PY' >&2
+archive extraction
+PY
+ printf '%s\n' "$VERIFIER_DIR"
+}
+selected_python() {
+ printf 'PYTHONPATH=<%s>\n' "$PYTHONPATH" >&2
+ printf 'PYTHON_ARG=<%s>\n' "$@" >&2
+}
+python3() { echo "SYSTEM_PYTHON_UNEXPECTED" >&2; return 99; }
+run_kimi_vendor_eval --port 9999 --results-dir "$RESULTS_DIR"
+printf 'EVAL_SUITE=%s\n' "$EVAL_SUITE"
+printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR"
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(results_dir),
+ "VERIFIER_DIR": str(verifier_dir),
+ "MODEL": "test-model",
+ "MODEL_PREFIX": "dsv4",
+ "RUNTIME_DIR": str(runtime_dir),
+ "PYTHON_DIR": str(python_dir),
+ "OPENAI_API_KEY": "must-not-be-forwarded",
+ "KV_OFFLOADING": "none",
+ "IS_MULTINODE": "true",
+ }
+ for key in (
+ "EVAL_SUITE",
+ "EVAL_RESULT_DIR",
+ "MODEL_NAME",
+ ):
+ env.pop(key, None)
+
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ output = result.stdout + result.stderr
+ adapter = BENCHMARK_LIB.parents[1] / "utils/evals/kimi_vendor_eval.py"
+
+ assert f"PYTHONPATH=<{tmp_path / 'runtime'}" in output
+ assert (
+ "CHECKOUT=https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git"
+ "@3dad65a760a8867cda72f6dd8848d876a4e851b4"
+ ) in output
+ assert (
+ "CHECKOUT_SHA=ede9ea300c72ccfde9d8975ea4b1b54e423c7625690f6631ab1e65a715821e01"
+ in output
+ )
+ assert "PYTHON_ARG=<->" in output
+ assert "PYTHON_ARG=<3dad65a760a8867cda72f6dd8848d876a4e851b4>" in output
+ for value in (
+ adapter,
+ verifier_dir,
+ "http://127.0.0.1:9999/v1",
+ "EMPTY",
+ "test-model",
+ results_dir,
+ ):
+ assert f"PYTHON_ARG=<{value}>" in output
+ assert "PYTHON_ARG=<--model-prefix>" in output
+ assert "PYTHON_ARG=" in output
+ assert "PYTHON_ARG=<--task-name>" in output
+ assert "PYTHON_ARG=" in output
+ assert "PYTHON_ARG=<--timeout-seconds>" in output
+ assert "PYTHON_ARG=<900>" in output
+ assert "must-not-be-forwarded" not in output
+ assert "SYSTEM_PYTHON_UNEXPECTED" not in output
+ assert "EVAL_SUITE=kimi_tool_call_schema" in output
+ assert f"EVAL_RESULT_DIR={results_dir}" in output
+ assert not (tmp_path / "runtime").exists()
+ assert not verifier_dir.exists()
+ assert not python_dir.exists()
+
+
+def test_kimi_full_runner_installs_xdist_sets_timeout_and_cleans_runtimes(
+ tmp_path: Path,
+) -> None:
+ results_dir = tmp_path / "results"
+ verifier_dir = tmp_path / "verifier"
+ runtime_dir = tmp_path / "runtime"
+ python_dir = tmp_path / "python"
+ script = r"""
+source "$BENCHMARK_LIB"
+_prepare_vendor_verifier_python() {
+ mkdir "$PYTHON_DIR"
+ VENDOR_VERIFIER_PYTHON=selected_python
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$PYTHON_DIR"
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+}
+_prepare_kimi_vendor_runtime() {
+ printf 'RUNTIME_SUITE=<%s>\n' "$1" >&2
+ mkdir "$RUNTIME_DIR"
+ _install_kimi_vendor_eval_deps "$RUNTIME_DIR" "$1" >&2
+ printf '%s\n' "$RUNTIME_DIR"
+}
+_prepare_kimi_vendor_verifier() {
+ mkdir "$VERIFIER_DIR"
+ printf '%s\n' "$VERIFIER_DIR"
+}
+selected_python() {
+ printf 'PYTHONPATH=<%s>\n' "$PYTHONPATH" >&2
+ printf 'PYTHON_ARG=<%s>\n' "$@" >&2
+}
+run_kimi_vendor_eval --port 9999 --results-dir "$RESULTS_DIR"
+printf 'EVAL_SUITE=%s\n' "$EVAL_SUITE"
+printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR"
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "EVAL_SUITE": "kimi_tool_call_schema_full",
+ "RESULTS_DIR": str(results_dir),
+ "VERIFIER_DIR": str(verifier_dir),
+ "MODEL": "test-model",
+ "RUNTIME_DIR": str(runtime_dir),
+ "PYTHON_DIR": str(python_dir),
+ }
+ for key in ("EVAL_RESULT_DIR", "MODEL_NAME"):
+ env.pop(key, None)
+
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ output = result.stdout + result.stderr
+
+ assert "RUNTIME_SUITE=" in output
+ assert "PYTHON_ARG=" in output
+ assert "PYTHON_ARG=<--task-name>" in output
+ assert "PYTHON_ARG=" in output
+ assert "PYTHON_ARG=<--timeout-seconds>" in output
+ assert "PYTHON_ARG=<7200>" in output
+ assert "EVAL_SUITE=kimi_tool_call_schema_full" in output
+ assert f"EVAL_RESULT_DIR={results_dir}" in output
+ assert not runtime_dir.exists()
+ assert not verifier_dir.exists()
+ assert not python_dir.exists()
+
+
def test_run_lm_eval_rejects_missing_option_value():
result = _run_invalid_call("run_lm_eval --port")
assert result.returncode == 2
@@ -102,13 +1439,13 @@ def test_run_lm_eval_rejects_missing_option_value():
def test_lm_patch_copy_resolves_outside_repo(tmp_path):
- script = r'''
+ script = r"""
source "$BENCHMARK_LIB"
cd "$OTHER_CWD"
_patch_lm_eval
patch_dir=${PYTHONPATH%%:*}
cmp "$(_eval_patches_dir)/lm_eval_sitecustomize.py" "$patch_dir/sitecustomize.py"
-'''
+"""
env = {
**os.environ,
"BENCHMARK_LIB": str(BENCHMARK_LIB),
@@ -118,8 +1455,7 @@ def test_lm_patch_copy_resolves_outside_repo(tmp_path):
subprocess.run(["bash", "-c", script], env=env, check=True)
-
-_EVAL_LIMIT_SCRIPT = r'''
+_EVAL_LIMIT_SCRIPT = r"""
set -e
SHIM_DIR=$(mktemp -d)
cat > "$SHIM_DIR/python3" <<'PY'
@@ -129,56 +1465,352 @@ def test_lm_patch_copy_resolves_outside_repo(tmp_path):
PY
chmod +x "$SHIM_DIR/python3"
-source "$BENCHMARK_LIB"
+source "$BENCHMARK_LIB"
+
+export EVAL_MAX_MODEL_LEN=16384
+export MODEL_NAME=test-model
+export OPENAI_API_KEY=EMPTY
+export INFERENCEX_LM_EVAL_RUNTIME_READY=true
+
+_install_lm_eval_deps() { :; }
+_patch_lm_eval() { :; }
+
+PATH="$SHIM_DIR:$PATH" run_lm_eval --port 9999 2>&1
+"""
+
+
+def _run_lm_eval_cmdline(*, eval_limit=None) -> str:
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "KV_OFFLOADING": "none",
+ }
+ env.pop("EVAL_LIMIT", None)
+ if eval_limit is not None:
+ env["EVAL_LIMIT"] = str(eval_limit)
+ res = subprocess.run(
+ ["bash", "-c", _EVAL_LIMIT_SCRIPT],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ return res.stdout + res.stderr
+
+
+def test_eval_limit_appended_when_set():
+ out = _run_lm_eval_cmdline(eval_limit=10)
+ assert "--limit 10" in out, f"Expected '--limit 10' in output:\n{out}"
+
+
+def test_eval_limit_absent_when_unset():
+ out = _run_lm_eval_cmdline(eval_limit=None)
+ assert "--limit" not in out, f"Expected no '--limit' in output:\n{out}"
+
+
+def test_lm_eval_defaults_to_gsm8k():
+ out = _run_lm_eval_cmdline()
+ assert "utils/evals/gsm8k.yaml" in out
+
+
+def _summary_metadata(tmp_path: Path, **overrides: str) -> dict:
+ work_dir = tmp_path / "work"
+ results_dir = tmp_path / "results"
+ work_dir.mkdir(parents=True)
+ results_dir.mkdir()
+ script = r"""
+source "$BENCHMARK_LIB"
+cd "$WORK_DIR"
+append_lm_eval_summary >/dev/null
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "WORK_DIR": str(work_dir),
+ "EVAL_RESULT_DIR": str(results_dir),
+ "MODEL": "test-model",
+ "CONC": "7",
+ "KV_OFFLOADING": "none",
+ }
+ for key in ("EVAL_COMPLETED_SUITE", "EVAL_SUITE", "EVAL_TASKS_DIR"):
+ env.pop(key, None)
+ env.update(overrides)
+ subprocess.run(["bash", "-c", script], env=env, check=True)
+ return json.loads((work_dir / "meta_env.json").read_text())
+
+
+def test_summary_stages_bfcl_upstream_archive_before_cleanup(tmp_path: Path) -> None:
+ work_dir = tmp_path / "work"
+ results_dir = tmp_path / "results"
+ work_dir.mkdir()
+ results_dir.mkdir()
+ archive = results_dir / "bfcl_upstream_artifacts.tar.gz"
+ archive.write_bytes(b"bfcl-archive")
+ script = r"""
+source "$BENCHMARK_LIB"
+cd "$WORK_DIR"
+append_lm_eval_summary >/dev/null
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "WORK_DIR": str(work_dir),
+ "EVAL_RESULT_DIR": str(results_dir),
+ "MODEL": "test-model",
+ "CONC": "7",
+ "KV_OFFLOADING": "none",
+ }
+
+ subprocess.run(["bash", "-c", script], env=env, check=True)
+
+ assert (work_dir / archive.name).read_bytes() == b"bfcl-archive"
+ assert not results_dir.exists()
+
+
+def test_stage_eval_artifacts_copies_eval_outputs_only(tmp_path: Path) -> None:
+ source_one = tmp_path / "source-one"
+ source_two = tmp_path / "source-two"
+ destination = tmp_path / "destination"
+ source_one.mkdir()
+ source_two.mkdir()
+ expected = {
+ "meta_env.json",
+ "results_bfcl.json",
+ "kimi_vendor_report.json",
+ "kimi_vendor_results.jsonl",
+ "bfcl_report.json",
+ "bfcl_upstream_artifacts.tar.gz",
+ "sample_eval.jsonl",
+ "agent_preds.json",
+ "predictions.jsonl",
+ "swebench_report_eval.json",
+ "trace.traj.json",
+ }
+ for filename in expected:
+ source = source_one if filename.endswith(".json") else source_two
+ (source / filename).write_text(filename)
+ (source_one / "unrelated.log").write_text("skip")
+ script = r"""
+source "$BENCHMARK_LIB"
+stage_eval_artifacts "$DESTINATION" "$SOURCE_ONE" "$SOURCE_TWO"
+"""
+ subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "DESTINATION": str(destination),
+ "SOURCE_ONE": str(source_one),
+ "SOURCE_TWO": str(source_two),
+ "KV_OFFLOADING": "none",
+ },
+ check=True,
+ )
+
+ assert {path.name for path in destination.iterdir()} == expected
+
+
+def test_stage_eval_artifacts_propagates_copy_failure(tmp_path: Path) -> None:
+ source = tmp_path / "source"
+ source.mkdir()
+ (source / "bfcl_report.json").write_text("{}")
+ script = r"""
+source "$BENCHMARK_LIB"
+cp() { return 73; }
+stage_eval_artifacts "$DESTINATION" "$SOURCE"
+"""
+
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "DESTINATION": str(tmp_path / "destination"),
+ "SOURCE": str(source),
+ "KV_OFFLOADING": "none",
+ },
+ check=False,
+ )
+
+ assert result.returncode == 73
+
+
+def test_stage_eval_artifacts_fails_when_no_artifacts_exist(tmp_path: Path) -> None:
+ source = tmp_path / "source"
+ source.mkdir()
+ script = r"""
+source "$BENCHMARK_LIB"
+stage_eval_artifacts "$DESTINATION" "$SOURCE"
+"""
+
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "DESTINATION": str(tmp_path / "destination"),
+ "SOURCE": str(source),
+ "KV_OFFLOADING": "none",
+ },
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode != 0
+ assert "no eval artifacts found to stage" in result.stderr
+
+
+def test_summary_propagates_artifact_staging_failure(tmp_path: Path) -> None:
+ work_dir = tmp_path / "work"
+ results_dir = tmp_path / "results"
+ work_dir.mkdir()
+ results_dir.mkdir()
+ (results_dir / "results_eval.json").write_text("{}")
+ script = r"""
+source "$BENCHMARK_LIB"
+cp() { return 73; }
+cd "$WORK_DIR"
+append_lm_eval_summary
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "WORK_DIR": str(work_dir),
+ "EVAL_RESULT_DIR": str(results_dir),
+ "MODEL": "test-model",
+ "CONC": "7",
+ "KV_OFFLOADING": "none",
+ },
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 73
-export EVAL_MAX_MODEL_LEN=16384
-export MODEL_NAME=test-model
-export OPENAI_API_KEY=EMPTY
-export INFERENCEX_LM_EVAL_RUNTIME_READY=true
-_install_lm_eval_deps() { :; }
-_patch_lm_eval() { :; }
+def test_summary_metadata_preserves_lm_eval_gsm8k_defaults(tmp_path: Path) -> None:
+ meta = _summary_metadata(tmp_path)
-PATH="$SHIM_DIR:$PATH" run_lm_eval --port 9999 2>&1
-'''
+ assert meta["eval_suite"] == "gsm8k"
+ assert meta["conc"] == 7
-def _run_lm_eval_cmdline(*, eval_limit=None) -> str:
+def test_summary_metadata_preserves_single_node_expert_parallelism(
+ tmp_path: Path,
+) -> None:
+ meta = _summary_metadata(tmp_path, TP="8", EP_SIZE="8")
+
+ assert meta["ep"] == 8
+ assert meta["prefill_ep"] == 8
+ assert meta["decode_ep"] == 8
+
+
+def test_run_lm_eval_exports_cli_task_path(tmp_path: Path) -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+python3() { :; }
+export EVAL_MAX_MODEL_LEN=16384
+export INFERENCEX_LM_EVAL_RUNTIME_READY=true
+run_lm_eval --task custom.yaml --results-dir "$RESULTS_DIR"
+printf 'EVAL_TASKS_DIR=%s\n' "$EVAL_TASKS_DIR"
+"""
env = {
**os.environ,
"BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(tmp_path / "results"),
+ "MODEL_NAME": "test-model",
+ "OPENAI_API_KEY": "EMPTY",
"KV_OFFLOADING": "none",
}
- env.pop("EVAL_LIMIT", None)
- if eval_limit is not None:
- env["EVAL_LIMIT"] = str(eval_limit)
- res = subprocess.run(
- ["bash", "-c", _EVAL_LIMIT_SCRIPT],
+ env.pop("EVAL_TASKS_DIR", None)
+ result = subprocess.run(
+ ["bash", "-c", script],
env=env,
text=True,
capture_output=True,
check=True,
)
- return res.stdout + res.stderr
+ assert "EVAL_TASKS_DIR=custom.yaml" in result.stdout
-def test_eval_limit_appended_when_set():
- out = _run_lm_eval_cmdline(eval_limit=10)
- assert "--limit 10" in out, f"Expected '--limit 10' in output:\n{out}"
+def test_summary_metadata_prefers_explicit_suite_then_task_basename(
+ tmp_path: Path,
+) -> None:
+ from_task = _summary_metadata(
+ tmp_path / "task",
+ EVAL_TASKS_DIR="/tmp/custom_reasoning.yaml",
+ )
+ explicit = _summary_metadata(
+ tmp_path / "explicit",
+ EVAL_SUITE="kimi_tool_call_schema",
+ EVAL_TASKS_DIR="/tmp/ignored.yaml",
+ )
+
+ assert from_task["eval_suite"] == "custom_reasoning"
+ assert explicit["eval_suite"] == "kimi_tool_call_schema"
-def test_eval_limit_absent_when_unset():
- out = _run_lm_eval_cmdline(eval_limit=None)
- assert "--limit" not in out, f"Expected no '--limit' in output:\n{out}"
+def test_summary_metadata_prefers_completed_eval_identity(tmp_path: Path) -> None:
+ meta = _summary_metadata(
+ tmp_path,
+ EVAL_COMPLETED_SUITE="kimi_tool_call_schema",
+ EVAL_SUITE="stale_input_selector",
+ EVAL_TASKS_DIR="/tmp/ignored.yaml",
+ )
-def test_lm_eval_defaults_to_gsm8k():
- out = _run_lm_eval_cmdline()
- assert "utils/evals/gsm8k.yaml" in out
+ assert meta["eval_suite"] == "kimi_tool_call_schema"
+
+
+def test_env_is_true_is_case_insensitive_and_unset_safe() -> None:
+ script = r"""
+set -u
+source "$BENCHMARK_LIB"
+for value in TrUe yEs oN 1 false 0; do
+ if _env_is_true "$value"; then
+ echo true
+ else
+ echo false
+ fi
+done
+for empty_call in with-argument without-argument; do
+ if [ "$empty_call" = "with-argument" ]; then
+ _env_is_true ""
+ else
+ _env_is_true
+ fi
+ if [ "$?" -eq 0 ]; then
+ echo true
+ else
+ echo false
+ fi
+done
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)},
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ assert result.stdout.splitlines() == [
+ "true",
+ "true",
+ "true",
+ "true",
+ "false",
+ "false",
+ "false",
+ "false",
+ ]
-_MODAL_CREDS_SCRIPT = r'''
+_MODAL_CREDS_SCRIPT = r"""
source "$BENCHMARK_LIB"
_ensure_modal_credentials
echo "HOME_AFTER=$HOME"
@@ -187,10 +1819,12 @@ def test_lm_eval_defaults_to_gsm8k():
PERMS=$(stat -c '%a' "$HOME/.modal.toml" 2>/dev/null || stat -f '%A' "$HOME/.modal.toml" 2>/dev/null)
echo "TOML_PERMS=$PERMS"
fi
-'''
+"""
-def _run_modal_creds(tmp_path: Path, *, home: str, token_id="tok-id", token_secret="tok-secret") -> str:
+def _run_modal_creds(
+ tmp_path: Path, *, home: str, token_id="tok-id", token_secret="tok-secret"
+) -> str:
env = {
**os.environ,
"BENCHMARK_LIB": str(BENCHMARK_LIB),
@@ -228,7 +1862,9 @@ def test_modal_creds_remaps_home_when_not_writable_parent(tmp_path):
nested_home = str(readonly_parent / "nested_home")
try:
out = _run_modal_creds(tmp_path, home=nested_home)
- assert "HOME_AFTER=/tmp/inferencex-modal-home" in out, f"Expected HOME remap:\n{out}"
+ assert "HOME_AFTER=/tmp/inferencex-modal-home" in out, (
+ f"Expected HOME remap:\n{out}"
+ )
assert "remapped" in out.lower() or "HOME remapped" in out
assert "TOML_EXISTS=true" in out
toml_path = Path("/tmp/inferencex-modal-home/.modal.toml")
@@ -244,7 +1880,9 @@ def test_modal_creds_remaps_home_when_not_writable(tmp_path):
readonly_home.mkdir(mode=0o555)
try:
out = _run_modal_creds(tmp_path, home=str(readonly_home))
- assert "HOME_AFTER=/tmp/inferencex-modal-home" in out, f"Expected HOME remap:\n{out}"
+ assert "HOME_AFTER=/tmp/inferencex-modal-home" in out, (
+ f"Expected HOME remap:\n{out}"
+ )
assert "TOML_EXISTS=true" in out
finally:
readonly_home.chmod(0o755)
@@ -272,8 +1910,7 @@ def test_modal_creds_no_remap_when_disabled(tmp_path):
assert "TOML_EXISTS" not in out
-
-_INCLUDE_PATH_SCRIPT = r'''
+_INCLUDE_PATH_SCRIPT = r"""
set -e
SHIM_DIR=$(mktemp -d)
cat > "$SHIM_DIR/python3" <<'PY'
@@ -294,7 +1931,7 @@ def test_modal_creds_no_remap_when_disabled(tmp_path):
_patch_lm_eval() { :; }
PATH="$SHIM_DIR:$PATH" run_lm_eval --port 9999 2>&1
-'''
+"""
def _run_lm_eval_with_include_path(
@@ -350,7 +1987,7 @@ def test_include_path_absent_when_eval_include_path_unset():
def test_swebench_single_shot_registers_task_yaml():
- script = r'''
+ script = r"""
source "$BENCHMARK_LIB"
run_lm_eval() {
echo "TASK=$EVAL_TASKS_DIR"
@@ -361,7 +1998,7 @@ def test_swebench_single_shot_registers_task_yaml():
export EVAL_TASKS_DIR="$TASK_YAML"
export MODEL=test-model
run_swebench_eval
-'''
+"""
env = {
**os.environ,
"BENCHMARK_LIB": str(BENCHMARK_LIB),
@@ -393,7 +2030,9 @@ def test_modal_credentials_sanitizes_whitespace_contaminated_tokens(tmp_path):
echo SANITIZED_OK
"""
env = {**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "HOME": str(home)}
- res = subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True)
+ res = subprocess.run(
+ ["bash", "-c", script], env=env, text=True, capture_output=True
+ )
assert res.returncode == 0, res.stdout + res.stderr
assert "SANITIZED_OK" in res.stdout
@@ -407,7 +2046,7 @@ def test_agentic_generation_invokes_mini_swe_agent(tmp_path):
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_name_or_path\": \"m\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_name_or_path": "m", "model_patch": "d"}}\' > "$out/preds.json"\n'
)
(shim / "mini-extra").chmod(0o755)
default_yaml = shim / "default.yaml"
@@ -434,11 +2073,15 @@ def test_agentic_generation_invokes_mini_swe_agent(tmp_path):
grep -q 'runtime_timeout: 3600' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo NO_RUNTIME_TIMEOUT; exit 1; }
echo AGENTIC_GEN_OK
"""
- env = {**os.environ,
- "BENCHMARK_LIB": str(BENCHMARK_LIB),
- "GEN_DIR": str(gen_dir),
- "PATH": f"{shim}:{os.environ['PATH']}"}
- res = subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True)
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "GEN_DIR": str(gen_dir),
+ "PATH": f"{shim}:{os.environ['PATH']}",
+ }
+ res = subprocess.run(
+ ["bash", "-c", script], env=env, text=True, capture_output=True
+ )
assert res.returncode == 0, res.stdout + res.stderr
assert "AGENTIC_GEN_OK" in res.stdout
argv = (shim / "argv.log").read_text()
@@ -472,37 +2115,45 @@ def _run_agentic(shim, gen_dir, extra_env=None):
_run_swebench_agentic_generation "$GEN_DIR" --port 8899
echo "GEN_RC=$?"
"""
- env = {**os.environ,
- "BENCHMARK_LIB": str(BENCHMARK_LIB),
- "GEN_DIR": str(gen_dir),
- "MODEL_NAME": "test-model",
- "SWEBENCH_SANDBOX_SWEEP": "0",
- "SWEBENCH_WATCHDOG_POLL": "1",
- "PATH": f"{shim}:{os.environ['PATH']}",
- **(extra_env or {})}
- return subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True)
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "GEN_DIR": str(gen_dir),
+ "MODEL_NAME": "test-model",
+ "SWEBENCH_SANDBOX_SWEEP": "0",
+ "SWEBENCH_WATCHDOG_POLL": "1",
+ "PATH": f"{shim}:{os.environ['PATH']}",
+ **(extra_env or {}),
+ }
+ return subprocess.run(
+ ["bash", "-c", script], env=env, text=True, capture_output=True
+ )
def test_agentic_watchdog_kills_hung_mini(tmp_path):
- shim, gen_dir = _agentic_shim(tmp_path,
+ shim, gen_dir = _agentic_shim(
+ tmp_path,
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
- "exec sleep 600 /dev/null 2>&1\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_patch": "d"}}\' > "$out/preds.json"\n'
+ "exec sleep 600 /dev/null 2>&1\n",
+ )
+ res = _run_agentic(
+ shim, gen_dir, {"EVAL_LIMIT": "1", "SWEBENCH_AGENT_EXIT_GRACE": "2"}
)
- res = _run_agentic(shim, gen_dir, {"EVAL_LIMIT": "1", "SWEBENCH_AGENT_EXIT_GRACE": "2"})
assert "GEN_RC=0" in res.stdout, res.stdout + res.stderr
assert "hung after completing all instances" in res.stdout + res.stderr
def test_agentic_salvage_partial_preds_on_failure(tmp_path):
- shim, gen_dir = _agentic_shim(tmp_path,
+ shim, gen_dir = _agentic_shim(
+ tmp_path,
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
- "exit 7\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_patch": "d"}}\' > "$out/preds.json"\n'
+ "exit 7\n",
)
res = _run_agentic(shim, gen_dir, {"EVAL_LIMIT": "2"})
assert "GEN_RC=0" in res.stdout, res.stdout + res.stderr
@@ -516,12 +2167,13 @@ def test_agentic_no_preds_still_fails(tmp_path):
def test_agentic_eval_limit_defaults_to_full_split(tmp_path):
- shim, gen_dir = _agentic_shim(tmp_path,
- 'echo "MINI_ARGV: $*" >> ' + "ARGVLOG" + '\n'
+ shim, gen_dir = _agentic_shim(
+ tmp_path,
+ 'echo "MINI_ARGV: $*" >> ' + "ARGVLOG" + "\n"
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_patch": "d"}}\' > "$out/preds.json"\n',
)
body = (shim / "mini-extra").read_text().replace("ARGVLOG", str(shim / "argv.log"))
(shim / "mini-extra").write_text(body)
@@ -532,12 +2184,13 @@ def test_agentic_eval_limit_defaults_to_full_split(tmp_path):
def test_agentic_eval_limit_full_runs_whole_split(tmp_path):
- shim, gen_dir = _agentic_shim(tmp_path,
- 'echo "MINI_ARGV: $*" >> ' + "ARGVLOG" + '\n'
+ shim, gen_dir = _agentic_shim(
+ tmp_path,
+ 'echo "MINI_ARGV: $*" >> ' + "ARGVLOG" + "\n"
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_patch": "d"}}\' > "$out/preds.json"\n',
)
body = (shim / "mini-extra").read_text().replace("ARGVLOG", str(shim / "argv.log"))
(shim / "mini-extra").write_text(body)
@@ -547,37 +2200,272 @@ def test_agentic_eval_limit_full_runs_whole_split(tmp_path):
assert "GEN_RC=0" in res.stdout, res.stdout + res.stderr
+def test_multinode_eval_artifact_names_are_bounded_and_distinct() -> None:
+ workflow = yaml.safe_load(MULTINODE_WORKFLOW.read_text())
+ upload = next(
+ step
+ for step in workflow["jobs"]["benchmark"]["steps"]
+ if step.get("name") == "Upload eval results (if any)"
+ )
+ expression = upload["with"]["name"]
+ assert expression.startswith("eval_")
+ assert "RESULT_FILENAME" not in expression
+
+ targets = [
+ {
+ "EXP_NAME": "kimik3_p2x16ep32dpa_d0x16ep32dpa_conc12",
+ "PRECISION": "fp4",
+ "FRAMEWORK": "vllm",
+ "PREFILL_NUM_WORKERS": "2",
+ "PREFILL_TP": "16",
+ "PREFILL_PP_SIZE": "1",
+ "SPEC_DECODING": "mtp",
+ "PREFILL_DCP_SIZE": "1",
+ "PREFILL_PCP_SIZE": "1",
+ "PREFILL_EP": "32",
+ "PREFILL_DP_ATTN": "true",
+ "DECODE_NUM_WORKERS": "0",
+ "DECODE_TP": "16",
+ "DECODE_PP_SIZE": "1",
+ "DECODE_DCP_SIZE": "1",
+ "DECODE_PCP_SIZE": "1",
+ "DECODE_EP": "32",
+ "DECODE_DP_ATTN": "true",
+ "KV_OFFLOADING": "none",
+ "KV_OFFLOAD_BACKEND": "",
+ "conc-list": ["1", "12", "16"],
+ "runner.name": "h200-dgxc-slurm_00",
+ },
+ {
+ "EXP_NAME": "kimik3_p4x8ep32dpa_d0x8ep32dpa_conc12",
+ "PRECISION": "fp4",
+ "FRAMEWORK": "vllm",
+ "PREFILL_NUM_WORKERS": "4",
+ "PREFILL_TP": "8",
+ "PREFILL_PP_SIZE": "1",
+ "PREFILL_DCP_SIZE": "1",
+ "PREFILL_PCP_SIZE": "1",
+ "PREFILL_EP": "32",
+ "PREFILL_DP_ATTN": "true",
+ "DECODE_NUM_WORKERS": "0",
+ "SPEC_DECODING": "mtp",
+ "DECODE_TP": "8",
+ "DECODE_PP_SIZE": "1",
+ "DECODE_DCP_SIZE": "1",
+ "DECODE_PCP_SIZE": "1",
+ "DECODE_EP": "32",
+ "DECODE_DP_ATTN": "true",
+ "KV_OFFLOADING": "none",
+ "KV_OFFLOAD_BACKEND": "",
+ "conc-list": ["1", "12", "16"],
+ "runner.name": "h200-dgxc-slurm_01",
+ },
+ {
+ "EXP_NAME": "kimik3_p4x8ep32dpa_d0x8ep32dpa_conc12_kvdram-vllm-simple",
+ "PRECISION": "fp4",
+ "FRAMEWORK": "vllm",
+ "PREFILL_NUM_WORKERS": "4",
+ "PREFILL_TP": "8",
+ "PREFILL_PP_SIZE": "1",
+ "PREFILL_DCP_SIZE": "1",
+ "PREFILL_PCP_SIZE": "1",
+ "PREFILL_EP": "32",
+ "PREFILL_DP_ATTN": "true",
+ "DECODE_NUM_WORKERS": "0",
+ "DECODE_TP": "8",
+ "DECODE_PP_SIZE": "1",
+ "DECODE_DCP_SIZE": "1",
+ "DECODE_PCP_SIZE": "1",
+ "DECODE_EP": "32",
+ "SPEC_DECODING": "mtp",
+ "DECODE_DP_ATTN": "true",
+ "KV_OFFLOADING": "dram",
+ "KV_OFFLOAD_BACKEND": "vllm-simple",
+ "conc-list": ["1", "12", "16"],
+ "runner.name": "h200-dgxc-slurm_02",
+ },
+ {
+ "EXP_NAME": "kimik3_p1x8_d0x8_conc16",
+ "PRECISION": "fp4",
+ "FRAMEWORK": "dynamo-vllm",
+ "PREFILL_NUM_WORKERS": "1",
+ "PREFILL_TP": "8",
+ "PREFILL_PP_SIZE": "2",
+ "PREFILL_DCP_SIZE": "1",
+ "PREFILL_PCP_SIZE": "1",
+ "PREFILL_EP": "1",
+ "PREFILL_DP_ATTN": "false",
+ "DECODE_NUM_WORKERS": "0",
+ "DECODE_TP": "8",
+ "DECODE_PP_SIZE": "2",
+ "DECODE_DCP_SIZE": "1",
+ "DECODE_PCP_SIZE": "1",
+ "DECODE_EP": "1",
+ "DECODE_DP_ATTN": "false",
+ "SPEC_DECODING": "mtp",
+ "KV_OFFLOADING": "none",
+ "KV_OFFLOAD_BACKEND": "",
+ "conc-list": ["1", "16"],
+ "runner.name": "b200-dgxc_00",
+ },
+ {
+ "EXP_NAME": "kimik3_p1x16ep16_d0x16ep16_conc16",
+ "PRECISION": "fp4",
+ "FRAMEWORK": "dynamo-vllm",
+ "PREFILL_NUM_WORKERS": "1",
+ "PREFILL_TP": "16",
+ "PREFILL_PP_SIZE": "1",
+ "PREFILL_DCP_SIZE": "1",
+ "PREFILL_PCP_SIZE": "1",
+ "PREFILL_EP": "16",
+ "PREFILL_DP_ATTN": "false",
+ "DECODE_NUM_WORKERS": "0",
+ "DECODE_TP": "16",
+ "DECODE_PP_SIZE": "1",
+ "DECODE_DCP_SIZE": "1",
+ "DECODE_PCP_SIZE": "1",
+ "DECODE_EP": "16",
+ "DECODE_DP_ATTN": "false",
+ "KV_OFFLOADING": "none",
+ "SPEC_DECODING": "mtp",
+ "KV_OFFLOAD_BACKEND": "",
+ "conc-list": ["16"],
+ "runner.name": "gb200-nv_00",
+ },
+ {
+ "EXP_NAME": "kimik3_p1x16_d0x16_conc1",
+ "PRECISION": "fp4",
+ "FRAMEWORK": "dynamo-vllm",
+ "PREFILL_NUM_WORKERS": "1",
+ "PREFILL_TP": "16",
+ "PREFILL_PP_SIZE": "1",
+ "PREFILL_DCP_SIZE": "1",
+ "PREFILL_PCP_SIZE": "1",
+ "PREFILL_EP": "1",
+ "PREFILL_DP_ATTN": "false",
+ "DECODE_NUM_WORKERS": "0",
+ "DECODE_TP": "16",
+ "DECODE_PP_SIZE": "1",
+ "DECODE_DCP_SIZE": "1",
+ "DECODE_PCP_SIZE": "1",
+ "DECODE_EP": "1",
+ "DECODE_DP_ATTN": "false",
+ "KV_OFFLOADING": "none",
+ "KV_OFFLOAD_BACKEND": "",
+ "SPEC_DECODING": "mtp",
+ "conc-list": ["1"],
+ "runner.name": "gb200-nv_01",
+ },
+ ]
+ non_mtp_twin = {**targets[3], "SPEC_DECODING": "none"}
+ disagg_twin = {**targets[3], "DISAGG": "true"}
+ recipe_twin = {
+ **targets[3],
+ "EVAL_ARTIFACT_RECIPE": "0123456789abcdef",
+ }
+ suite_twin = {**targets[3], "EVAL_SUITE": "bfcl_vllm_kimi"}
+ conc_twin = {**targets[3], "conc-list": ["2", "16"]}
+
+ def render(values: dict[str, object]) -> str:
+ name = expression
+ defaults: dict[str, object] = {
+ "DISAGG": "false",
+ "EVAL_ARTIFACT_RECIPE": "",
+ "EVAL_FRAMEWORK": "bfcl",
+ "EVAL_SUITE": "bfcl_smoke",
+ }
+ defaults["EVAL_ARTIFACT_CONC"] = hashlib.sha256(
+ " ".join(values["conc-list"]).encode()
+ ).hexdigest()[:12]
+ for key, value in {**defaults, **values}.items():
+ if key != "conc-list":
+ name = name.replace(f"${{{{ env.{key} }}}}", str(value))
+ name = name.replace("${{ runner.name }}", str(values["runner.name"]))
+ name = name.replace("${{ github.run_attempt }}", "1")
+ assert "${{" not in name
+ return name
+
+ variants = [
+ *targets,
+ non_mtp_twin,
+ disagg_twin,
+ recipe_twin,
+ suite_twin,
+ conc_twin,
+ ]
+ names = [render(target) for target in variants]
+ assert len(names) == len(set(names)) == len(variants)
+ assert all(name.startswith("eval_") and len(name.encode()) <= 256 for name in names)
+
+
+def test_single_node_eval_artifact_name_includes_suite_identity() -> None:
+ workflow = yaml.safe_load(SINGLE_NODE_WORKFLOW.read_text())
+ upload = next(
+ step
+ for step in workflow["jobs"]["benchmark"]["steps"]
+ if step.get("name") == "Upload eval results (if any)"
+ )
+ expression = upload["with"]["name"]
+ assert "EVAL_FRAMEWORK" in expression
+ assert "EVAL_SUITE" in expression
+ assert "github.run_attempt" in expression
+
-_GENMODE_SCRIPT = r'''
+_GENMODE_SCRIPT = r"""
source "$BENCHMARK_LIB" 2>/dev/null
_install_swebench_agent_deps() { :; }
_ensure_modal_credentials() { :; }
-_run_swebench_agentic_generation() { echo "GEN=agentic"; return 42; }
-run_lm_eval() { echo "GEN=single-shot"; return 42; }
+_run_swebench_agentic_generation() {
+ echo "GEN=agentic"
+ echo "SUITE=$EVAL_SUITE"
+ return 42
+}
+run_lm_eval() {
+ echo "GEN=single-shot"
+ echo "SUITE=$EVAL_SUITE"
+ return 42
+}
run_swebench_eval --port 8888
echo "RC=$?"
-'''
+"""
-def _gen_mode(tmp_path, *, is_agentic, gen_mode=None) -> str:
- env = {**os.environ,
- "BENCHMARK_LIB": str(BENCHMARK_LIB),
- "KV_OFFLOADING": "none",
- "IS_AGENTIC": is_agentic,
- "EVAL_RESULT_DIR": str(tmp_path / "out")}
+def _gen_mode(
+ tmp_path: Path,
+ *,
+ is_agentic,
+ gen_mode=None,
+ eval_suite=None,
+) -> str:
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "KV_OFFLOADING": "none",
+ "IS_AGENTIC": is_agentic,
+ "EVAL_RESULT_DIR": str(tmp_path / "out"),
+ }
env.pop("SWEBENCH_GEN_MODE", None)
env.pop("SCENARIO_TYPE", None)
+ env.pop("EVAL_SUITE", None)
if gen_mode is not None:
env["SWEBENCH_GEN_MODE"] = gen_mode
- res = subprocess.run(["bash", "-c", _GENMODE_SCRIPT], env=env,
- text=True, capture_output=True,
- cwd=BENCHMARK_LIB.parents[1])
+ if eval_suite is not None:
+ env["EVAL_SUITE"] = eval_suite
+ res = subprocess.run(
+ ["bash", "-c", _GENMODE_SCRIPT],
+ env=env,
+ text=True,
+ capture_output=True,
+ cwd=BENCHMARK_LIB.parents[1],
+ )
assert "RC=42" in res.stdout, res.stdout + res.stderr
return res.stdout
def test_gen_mode_defaults_to_agentic(tmp_path):
- assert "GEN=agentic" in _gen_mode(tmp_path, is_agentic="1")
+ output = _gen_mode(tmp_path, is_agentic="1")
+ assert "GEN=agentic" in output
+ assert "SUITE=swebench_lite" in output
def test_gen_mode_agentic_even_without_agentic_scenario(tmp_path):
@@ -585,17 +2473,33 @@ def test_gen_mode_agentic_even_without_agentic_scenario(tmp_path):
def test_explicit_single_shot_escape_hatch(tmp_path):
- assert "GEN=single-shot" in _gen_mode(tmp_path, is_agentic="1", gen_mode="single-shot")
+ output = _gen_mode(tmp_path, is_agentic="1", gen_mode="single-shot")
+ assert "GEN=single-shot" in output
+ assert "SUITE=swebench_lite" in output
+
+
+def test_swebench_generation_modes_preserve_explicit_suite(tmp_path):
+ for gen_mode in ("agentic", "single-shot"):
+ output = _gen_mode(
+ tmp_path / gen_mode,
+ is_agentic="1",
+ gen_mode=gen_mode,
+ eval_suite="explicit_swebench",
+ )
+ assert "SUITE=explicit_swebench" in output
def test_agent_sandbox_cpu_knob(tmp_path):
- shim, gen_dir = _agentic_shim(tmp_path,
+ shim, gen_dir = _agentic_shim(
+ tmp_path,
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_patch": "d"}}\' > "$out/preds.json"\n',
+ )
+ res = _run_agentic(
+ shim, gen_dir, {"EVAL_LIMIT": "1", "SWEBENCH_AGENT_SANDBOX_CPU": "1"}
)
- res = _run_agentic(shim, gen_dir, {"EVAL_LIMIT": "1", "SWEBENCH_AGENT_SANDBOX_CPU": "1"})
assert "GEN_RC=0" in res.stdout, res.stdout + res.stderr
cfg = (gen_dir / "mini_swebench_overrides.yaml").read_text()
assert "modal_sandbox_kwargs" in cfg and "cpu: 1" in cfg, cfg
@@ -609,27 +2513,31 @@ def test_agent_sandbox_cpu_knob(tmp_path):
def test_eval_limit_rejects_non_positive_integer(tmp_path):
- shim, gen_dir = _agentic_shim(tmp_path,
+ shim, gen_dir = _agentic_shim(
+ tmp_path,
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_patch": "d"}}\' > "$out/preds.json"\n',
)
for bad in ("-5", "abc", "3.5"):
- gd = tmp_path / f"gen_{bad.replace('-','neg').replace('.','_')}"
+ gd = tmp_path / f"gen_{bad.replace('-', 'neg').replace('.', '_')}"
gd.mkdir()
res = _run_agentic(shim, gd, {"EVAL_LIMIT": bad})
- assert "GEN_RC=1" in res.stdout, f"EVAL_LIMIT={bad!r} should fail: {res.stdout}{res.stderr}"
+ assert "GEN_RC=1" in res.stdout, (
+ f"EVAL_LIMIT={bad!r} should fail: {res.stdout}{res.stderr}"
+ )
assert "must be a positive integer" in res.stdout + res.stderr
def test_eval_limit_full_and_zero_accepted(tmp_path):
- shim, gen_dir = _agentic_shim(tmp_path,
- 'echo "MINI_ARGV: $*" >> ' + "ARGVLOG" + '\n'
+ shim, gen_dir = _agentic_shim(
+ tmp_path,
+ 'echo "MINI_ARGV: $*" >> ' + "ARGVLOG" + "\n"
'out=""; prev=""\n'
'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n'
'mkdir -p "$out"\n'
- "printf '{\"i1\": {\"instance_id\": \"i1\", \"model_patch\": \"d\"}}' > \"$out/preds.json\"\n"
+ 'printf \'{"i1": {"instance_id": "i1", "model_patch": "d"}}\' > "$out/preds.json"\n',
)
body = (shim / "mini-extra").read_text().replace("ARGVLOG", str(shim / "argv.log"))
(shim / "mini-extra").write_text(body)
@@ -637,6 +2545,836 @@ def test_eval_limit_full_and_zero_accepted(tmp_path):
gd = tmp_path / f"gen_{sentinel}"
gd.mkdir()
res = _run_agentic(shim, gd, {"EVAL_LIMIT": sentinel})
- assert "GEN_RC=0" in res.stdout, f"EVAL_LIMIT={sentinel!r}: {res.stdout}{res.stderr}"
+ assert "GEN_RC=0" in res.stdout, (
+ f"EVAL_LIMIT={sentinel!r}: {res.stdout}{res.stderr}"
+ )
argv = (shim / "argv.log").read_text()
assert "--slice" not in argv
+
+
+
+
+def test_chat_route_readiness_requires_model_and_active_route(tmp_path: Path) -> None:
+ bin_dir = tmp_path / "bin"
+ events_path = tmp_path / "events"
+ bin_dir.mkdir()
+ curl = bin_dir / "curl"
+ curl.write_text(
+ """#!/usr/bin/env bash
+printf 'curl %s\n' "$*" >> "$EVENTS"
+case "$*" in
+ */v1/models*) printf '{"data":[{"id":"test-model"}]}\n' ;;
+ */v1/chat/completions*) printf '405' ;;
+esac
+""",
+ encoding="utf-8",
+ )
+ curl.chmod(curl.stat().st_mode | stat.S_IXUSR)
+ script = r"""
+source "$BENCHMARK_LIB"
+MODEL=test-model
+_wait_for_openai_chat_route --port 8765
+"""
+
+ subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "EVENTS": str(events_path),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ events = events_path.read_text().splitlines()
+ assert events[0].endswith("http://localhost:8765/health")
+ assert events[1].endswith("http://localhost:8765/v1/models")
+ assert events[2].endswith("http://localhost:8765/v1/chat/completions")
+ assert "--data" not in events[2]
+
+
+def test_chat_route_readiness_accepts_stable_server_with_different_model_id(
+ tmp_path: Path,
+) -> None:
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ curl = bin_dir / "curl"
+ curl.write_text(
+ """#!/usr/bin/env bash
+case "$*" in
+ */v1/models*) printf '{"data":[{"id":"/models/different-model"}]}\n' ;;
+ */v1/chat/completions*) printf '404' ;;
+esac
+""",
+ encoding="utf-8",
+ )
+ curl.chmod(curl.stat().st_mode | stat.S_IXUSR)
+
+ subprocess.run(
+ [
+ "bash",
+ "-c",
+ 'source "$BENCHMARK_LIB"; MODEL=test-model; '
+ "_wait_for_openai_chat_route --port 8765",
+ ],
+ env={
+ **os.environ,
+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "EVAL_MODEL_STABILIZATION_SECONDS": "0",
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+
+def test_multinode_agentic_waits_only_for_eval_openai_endpoint(
+ tmp_path: Path,
+) -> None:
+ workspace = tmp_path / "workspace"
+ events_path = tmp_path / "events"
+ (workspace / "benchmarks").mkdir(parents=True)
+ (workspace / "benchmarks/benchmark_lib.sh").write_text(
+ """
+PORT=8765
+check_env_vars() { :; }
+resolve_trace_source() { echo resolve >> "$EVENTS"; }
+AIPERF_PYTHON=python3
+install_agentic_deps() { echo deps >> "$EVENTS"; }
+_wait_for_openai_chat_route() { echo "ready $*" >> "$EVENTS"; }
+build_replay_cmd() { echo build >> "$EVENTS"; }
+run_agentic_replay_and_write_outputs() { echo replay >> "$EVENTS"; }
+""",
+ encoding="utf-8",
+ )
+
+ base_env = {
+ **os.environ,
+ "INFMAX_CONTAINER_WORKSPACE": str(workspace),
+ "EVENTS": str(events_path),
+ "MODEL": "test-model",
+ "MODEL_PREFIX": "test-prefix",
+ "FRAMEWORK": "dynamo-vllm",
+ "PRECISION": "fp4",
+ "CONC": "1",
+ "RESULT_FILENAME": "result",
+ "RESULT_DIR": str(tmp_path / "results"),
+ "DURATION": "1",
+ }
+ expected_without_readiness = ["resolve", "deps", "build", "replay"]
+
+ for eval_only, expected in (
+ ("false", expected_without_readiness),
+ ("true", [*expected_without_readiness[:2], "ready --port 8765", *expected_without_readiness[2:]]),
+ ):
+ events_path.unlink(missing_ok=True)
+ subprocess.run(
+ ["bash", str(MULTINODE_AGENTIC_SCRIPT)],
+ env={**base_env, "EVAL_ONLY": eval_only},
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ assert events_path.read_text().splitlines() == expected
+
+
+def test_agentic_eval_workflow_forwards_runner_contract() -> None:
+ workflow = yaml.safe_load(E2E_WORKFLOW.read_text())
+ forwarded = workflow["jobs"]["test-sweep-agentic-evals"]["with"]
+ throughput = workflow["jobs"]["test-sweep-agentic"]["with"]
+
+ for field in (
+ "router",
+ "kv-p2p-transfer",
+ "tp",
+ "pp",
+ "dcp-size",
+ "pcp-size",
+ "ep",
+ "dp-attn",
+ ):
+ assert forwarded[field] == throughput[field]
+
+
+ assert forwarded["spec-decoding"] == "${{ matrix.config.spec-decoding }}"
+ assert forwarded["eval-framework"] == AUTO_EVAL_FRAMEWORK_EXPR
+ assert forwarded["eval-suite"] == AUTO_EVAL_SUITE_EXPR
+ assert forwarded["kv-offload-backend"] == (
+ "${{ matrix.config['kv-offload-backend'].name }}"
+ )
+ assert forwarded["kv-offload-backend-metadata"] == (
+ "${{ matrix.config['kv-offload-backend'] && "
+ "toJson(matrix.config['kv-offload-backend']) || '' }}"
+ )
+
+
+def test_fixed_eval_workflows_forward_provider_contract() -> None:
+ workflow = yaml.safe_load(E2E_WORKFLOW.read_text())
+ for job_name in ("test-sweep-evals", "test-sweep-multi-node-evals"):
+ forwarded = workflow["jobs"][job_name]["with"]
+ assert forwarded["eval-framework"] == AUTO_EVAL_FRAMEWORK_EXPR
+ assert forwarded["eval-suite"] == AUTO_EVAL_SUITE_EXPR
+
+ reusable_workflow = yaml.safe_load(SINGLE_NODE_WORKFLOW.read_text())
+ assert reusable_workflow["env"]["EVAL_FRAMEWORK"] == "${{ inputs.eval-framework }}"
+ assert reusable_workflow["env"]["EVAL_SUITE"] == "${{ inputs.eval-suite }}"
+ assert "*_report.json" in SINGLE_NODE_WORKFLOW.read_text()
+ assert "*_results.jsonl" in SINGLE_NODE_WORKFLOW.read_text()
+ assert "*_artifacts.tar.gz" in SINGLE_NODE_WORKFLOW.read_text()
+ assert "bfcl_vllm_minimax_m3" in SINGLE_NODE_WORKFLOW.read_text()
+ assert "bfcl_vllm_kimi" in SINGLE_NODE_WORKFLOW.read_text()
+
+
+def test_multinode_agentic_eval_workflow_forwards_runner_contract() -> None:
+ workflow = yaml.safe_load(E2E_WORKFLOW.read_text())
+ forwarded = workflow["jobs"]["test-sweep-multi-node-agentic-evals"]["with"]
+ reusable_workflow = yaml.safe_load(MULTINODE_WORKFLOW.read_text())
+
+ assert forwarded["eval-framework"] == AUTO_EVAL_FRAMEWORK_EXPR
+ assert forwarded["eval-suite"] == AUTO_EVAL_SUITE_EXPR
+ assert reusable_workflow["env"]["EVAL_FRAMEWORK"] == "${{ inputs.eval-framework }}"
+ assert reusable_workflow["env"]["EVAL_SUITE"] == "${{ inputs.eval-suite }}"
+ assert "*_report.json" in MULTINODE_WORKFLOW.read_text()
+ assert "*_results.jsonl" in MULTINODE_WORKFLOW.read_text()
+ assert "*_artifacts.tar.gz" in MULTINODE_WORKFLOW.read_text()
+
+ assert "bfcl_vllm_minimax_m3" in MULTINODE_WORKFLOW.read_text()
+ assert "bfcl_vllm_kimi" in MULTINODE_WORKFLOW.read_text()
+
+
+def test_e2e_eval_workflow_defaults_to_model_aware_selection() -> None:
+ workflow = yaml.safe_load(E2E_WORKFLOW.read_text())
+ triggers = workflow[True]
+
+ for trigger_name in ("workflow_dispatch", "workflow_call"):
+ eval_input = triggers[trigger_name]["inputs"]["eval-framework"]
+ assert eval_input["default"] == "auto"
+
+ forwarded = workflow["jobs"]["test-sweep-multi-node-evals"]["with"]
+ assert forwarded["eval-conc"] == (
+ "${{ (inputs.eval-framework == 'auto' && "
+ "(matrix.config['eval-framework'] || 'lm-eval') || "
+ "inputs.eval-framework) == 'lm-eval' && "
+ "matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || "
+ "matrix.config['eval-conc'] }}"
+ )
+
+
+def test_agentic_throughput_split_keeps_eval_marked_rows() -> None:
+ workflow = yaml.safe_load(E2E_WORKFLOW.read_text())
+ get_jobs = next(
+ step
+ for step in workflow["jobs"]["get-jobs"]["steps"]
+ if step.get("id") == "get-jobs"
+ )
+ split_lines = get_jobs["run"].splitlines()
+
+ for variable in ("AGENTIC", "MULTI_AGENTIC"):
+ line = next(line for line in split_lines if line.strip().startswith(
+ f"{variable}=$("
+ ))
+ assert "not x.get('eval-only', False)" in line
+ assert "not x.get('run-eval', False)" not in line
+
+
+def test_trusted_changelog_matrix_keeps_multinode_agentic_evals() -> None:
+ workflow = yaml.safe_load(E2E_WORKFLOW.read_text())
+ get_jobs = next(
+ step
+ for step in workflow["jobs"]["get-jobs"]["steps"]
+ if step.get("id") == "get-jobs"
+ )
+ flatten_command = next(
+ line for line in get_jobs["run"].splitlines() if "rows.extend" in line
+ )
+
+ assert '"multinode_agentic_evals"' in flatten_command
+ get_jobs_command = get_jobs["run"]
+ assert get_jobs_command.count("EVALS=$(") == 1
+ assert "score_matrix eval" in get_jobs_command
+
+
+def test_env_can_force_bfcl_on_agentic_eval() -> None:
+ output = _dispatch(is_agentic="1", eval_only="true", env_fw="bfcl")
+
+ assert "DISPATCH=bfcl" in output
+ assert "STAGED=summary" in output
+
+
+def test_cli_can_force_bfcl_on_fixed_seqlen_eval() -> None:
+ output = _dispatch(is_agentic="0", cli_fw="bfcl")
+
+ assert "DISPATCH=bfcl" in output
+ assert "STAGED=summary" not in output
+
+
+def test_bfcl_defaults_suite_dispatches_once_without_context_loading() -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+unset EVAL_MAX_MODEL_LEN
+compute_eval_context_length() { echo "UNEXPECTED_CONTEXT_LOAD"; return 99; }
+BFCL_DISPATCH_COUNT=0
+run_bfcl_eval() {
+ BFCL_DISPATCH_COUNT=$((BFCL_DISPATCH_COUNT + 1))
+ printf 'DISPATCH=bfcl SUITE=%s ARGS=<%s>\n' "$EVAL_SUITE" "$*"
+}
+append_lm_eval_summary() { printf 'STAGED=%s\n' "$EVAL_COMPLETED_SUITE"; }
+export EVAL_CONCURRENT_REQUESTS=""
+export EVAL_ONLY=false
+export IS_AGENTIC=0
+run_eval --framework bfcl --port 9999
+printf 'DISPATCH_COUNT=%s\n' "$BFCL_DISPATCH_COUNT"
+printf 'COMPLETED_SUITE=%s\n' "$EVAL_COMPLETED_SUITE"
+"""
+ env = {**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "MODEL": "served-model"}
+ for key in ("EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_COMPLETED_SUITE"):
+ env.pop(key, None)
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "DISPATCH=bfcl SUITE=bfcl_smoke ARGS=<--port 9999>" in result.stdout
+ assert "DISPATCH_COUNT=1" in result.stdout
+ assert "COMPLETED_SUITE=bfcl_smoke" in result.stdout
+ assert "STAGED=bfcl_smoke" not in result.stdout
+ assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout
+
+
+def test_bfcl_rejects_suite_from_another_provider() -> None:
+ result = _run_invalid_call(
+ "EVAL_CONCURRENT_REQUESTS='' "
+ "EVAL_SUITE=minimax_m3_smoke "
+ "run_eval --framework bfcl"
+ )
+
+ assert result.returncode == 2
+ assert "unsupported BFCL suite 'minimax_m3_smoke'" in result.stderr
+
+
+def test_bfcl_suite_is_rejected_by_mismatched_framework() -> None:
+ result = _run_invalid_call(
+ "EVAL_CONCURRENT_REQUESTS='' "
+ "EVAL_SUITE=bfcl_smoke "
+ "run_eval --framework minimax-vendor"
+ )
+
+ assert result.returncode == 2
+ assert "unsupported MiniMax Provider Verifier suite 'bfcl_smoke'" in result.stderr
+
+
+def test_bfcl_rejects_unknown_suite() -> None:
+ result = _run_invalid_call("EVAL_SUITE=not_a_bfcl_suite run_bfcl_eval")
+
+ assert result.returncode == 2
+ assert "unsupported BFCL suite 'not_a_bfcl_suite'" in result.stderr
+
+
+def test_bfcl_suite_thresholds_are_diagnostic_and_namespaced() -> None:
+ thresholds = yaml.safe_load(
+ (REPO_ROOT / "utils/evals/thresholds.yaml").read_text()
+ )["default"]
+ full_suite_tasks = (
+ "bfcl_vllm_minimax_m3",
+ "bfcl_vllm_minimax_m3_simple_python",
+ "bfcl_vllm_minimax_m3_multiple",
+ "bfcl_vllm_minimax_m3_parallel",
+ "bfcl_vllm_minimax_m3_parallel_multiple",
+ "bfcl_vllm_kimi",
+ "bfcl_vllm_kimi_simple_python",
+ "bfcl_vllm_kimi_multiple",
+ "bfcl_vllm_kimi_parallel",
+ "bfcl_vllm_kimi_parallel_multiple",
+ "bfcl_vllm_kimi_multi_turn",
+ "bfcl_vllm_kimi_multi_turn_base",
+ "bfcl_vllm_kimi_multi_turn_miss_func",
+ "bfcl_vllm_kimi_multi_turn_miss_param",
+ "bfcl_vllm_kimi_multi_turn_long_context",
+ )
+
+ assert thresholds["bfcl_smoke"] == 0.0
+ assert all(thresholds[task] == 0.0 for task in full_suite_tasks)
+
+
+def test_bfcl_dependency_timeout_uses_integration_error_and_stages(
+ tmp_path: Path,
+) -> None:
+ results_dir = tmp_path / "results"
+ python_dir = tmp_path / "python"
+ results_dir.mkdir()
+ stale_result = results_dir / "results_bfcl_previous.json"
+ stale_result.write_text('{"stale": true}\n')
+ script = r"""
+source "$BENCHMARK_LIB"
+export EVAL_SUITE=bfcl_vllm_kimi
+unset VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+selected_python() {
+ printf 'ADAPTER_ARG=<%s>\n' "$@"
+ touch "$RESULTS_DIR/bfcl_report.json" "$RESULTS_DIR/results_bfcl.json"
+ return 1
+}
+_prepare_vendor_verifier_python() {
+ mkdir "$PYTHON_DIR"
+ VENDOR_VERIFIER_PYTHON=selected_python
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$PYTHON_DIR"
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+}
+_prepare_bfcl_runtime() { return 124; }
+python3() { echo "UNEXPECTED_SYSTEM_PYTHON"; return 99; }
+append_lm_eval_summary() { printf 'STAGED=<%s>\n' "$EVAL_RESULT_DIR"; }
+export EVAL_CONCURRENT_REQUESTS=""
+export EVAL_ONLY=false
+export IS_AGENTIC=0
+eval_rc=0
+run_eval --framework bfcl --results-dir "$RESULTS_DIR" || eval_rc=$?
+printf 'EVAL_RC=%s\n' "$eval_rc"
+exit "$eval_rc"
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(results_dir),
+ "PYTHON_DIR": str(python_dir),
+ "MODEL": "test-model",
+ }
+ env.pop("EVAL_FRAMEWORK", None)
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ output = result.stdout + result.stderr
+
+ assert result.returncode == 124
+ assert "EVAL_RC=124" in output
+ assert f"ADAPTER_ARG=<{REPO_ROOT / 'utils/evals/bfcl_adapter.py'}>" in output
+ assert "ADAPTER_ARG=" in output
+ assert f"ADAPTER_ARG=<{results_dir}>" in output
+ assert "ADAPTER_ARG=<--integration-error>" in output
+ assert "ADAPTER_ARG=<--suite>" in output
+ assert "ADAPTER_ARG=" in output
+ assert (
+ "ADAPTER_ARG=" in output
+ )
+ assert output.count(f"STAGED=<{results_dir}>") == 1
+ assert (results_dir / "bfcl_report.json").exists()
+ assert (results_dir / "results_bfcl.json").exists()
+ assert not stale_result.exists()
+ assert "failed to write BFCL failure artifact" not in output
+ assert "UNEXPECTED_SYSTEM_PYTHON" not in output
+ assert not (results_dir / "bfcl_upstream_artifacts.tar.gz").exists()
+ assert not python_dir.exists()
+
+
+def _run_bfcl_adapter_command(
+ tmp_path: Path,
+ *,
+ adapter_rc: int = 0,
+ suite: str = "",
+ archive_rc: int = 0,
+) -> tuple[subprocess.CompletedProcess[str], tuple[Path, Path, Path, Path]]:
+ results_dir = tmp_path / "results"
+ runtime_dir = tmp_path / "runtime"
+ python_dir = tmp_path / "python"
+ project_root = tmp_path / "bfcl-project"
+ script = r"""
+source "$BENCHMARK_LIB"
+selected_python() {
+ printf 'ADAPTER_ARG=<%s>\n' "$@"
+ local arg
+ for arg in "$@"; do
+ if [ "$arg" = "--integration-error" ]; then
+ touch "$RESULTS_DIR/bfcl_report.json" "$RESULTS_DIR/results_bfcl.json"
+ return 1
+ fi
+ done
+ if [ "$TEST_ADAPTER_RC" -eq 0 ]; then
+ touch "$RESULTS_DIR/bfcl_report.json" "$RESULTS_DIR/results_bfcl.json"
+ fi
+ return "$TEST_ADAPTER_RC"
+}
+_prepare_vendor_verifier_python() {
+ printf 'PREPARE_ARG=<%s>\n' "$@"
+ mkdir "$PYTHON_DIR"
+ VENDOR_VERIFIER_PYTHON=selected_python
+ VENDOR_VERIFIER_PYTHON_CLEANUP_DIR="$PYTHON_DIR"
+ export VENDOR_VERIFIER_PYTHON VENDOR_VERIFIER_PYTHON_CLEANUP_DIR
+}
+_prepare_bfcl_runtime() {
+ mkdir "$RUNTIME_DIR"
+ printf '%s\n' "$RUNTIME_DIR"
+}
+mktemp() {
+ printf 'MKTEMP_ARG=<%s>\n' "$@" >&2
+ mkdir "$PROJECT_ROOT"
+ printf '%s\n' "$PROJECT_ROOT"
+}
+_archive_bfcl_upstream_artifacts() {
+ printf 'ARCHIVE_PROJECT_ROOT=<%s>\n' "$1"
+ printf 'ARCHIVE_PATH=<%s>\n' "$2"
+ if [ "$TEST_ARCHIVE_RC" -eq 0 ]; then
+ touch "$2"
+ fi
+ return "$TEST_ARCHIVE_RC"
+}
+timeout() {
+ printf 'TIMEOUT_ARG=<%s>\n' "$1"
+ shift
+ "$@"
+}
+append_lm_eval_summary() { printf 'STAGED=<%s>\n' "$EVAL_RESULT_DIR"; }
+if [ -n "$TEST_SUITE" ]; then
+ export EVAL_SUITE="$TEST_SUITE"
+else
+ unset EVAL_SUITE
+fi
+unset EVAL_RESULT_DIR EVAL_COMPLETED_SUITE EVAL_MAX_MODEL_LEN
+compute_eval_context_length() { echo "UNEXPECTED_CONTEXT_LOAD"; return 99; }
+export EVAL_CONCURRENT_REQUESTS=""
+export EVAL_ONLY=false
+export IS_AGENTIC=0
+eval_rc=0
+run_eval --framework bfcl --port 9999 --results-dir "$RESULTS_DIR" || eval_rc=$?
+printf 'EVAL_RC=%s\n' "$eval_rc"
+printf 'EVAL_COMPLETED_SUITE=%s\n' "$EVAL_COMPLETED_SUITE"
+printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR"
+exit "$eval_rc"
+"""
+ env = {
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "RESULTS_DIR": str(results_dir),
+ "RUNTIME_DIR": str(runtime_dir),
+ "PYTHON_DIR": str(python_dir),
+ "PROJECT_ROOT": str(project_root),
+ "MODEL": "repository/model",
+ "MODEL_NAME": "served-model",
+ "OPENAI_API_KEY": "must-not-be-forwarded",
+ "TEST_ADAPTER_RC": str(adapter_rc),
+ "TEST_SUITE": suite,
+ "TEST_ARCHIVE_RC": str(archive_rc),
+ }
+ for key in (
+ "EVAL_FRAMEWORK",
+ "EVAL_SUITE",
+ "EVAL_RESULT_DIR",
+ "EVAL_COMPLETED_SUITE",
+ "VENDOR_VERIFIER_PYTHON",
+ "VENDOR_VERIFIER_PYTHON_CLEANUP_DIR",
+ ):
+ env.pop(key, None)
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env=env,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ return result, (results_dir, runtime_dir, python_dir, project_root)
+
+
+def test_bfcl_runner_uses_fixed_adapter_contract_and_cleans_runtime(
+ tmp_path: Path,
+) -> None:
+ result, paths = _run_bfcl_adapter_command(tmp_path)
+ results_dir, runtime_dir, python_dir, project_root = paths
+ output = result.stdout + result.stderr
+
+ assert result.returncode == 0, result.stderr
+ for value in (
+ str(REPO_ROOT / "utils/evals/bfcl_adapter.py"),
+ "--base-url",
+ "http://127.0.0.1:9999/v1",
+ "--api-key",
+ "EMPTY",
+ "--model",
+ "served-model",
+ "--output-dir",
+ str(results_dir),
+ "--bfcl-project-root",
+ str(project_root),
+ "--num-threads",
+ "4",
+ ):
+ assert f"ADAPTER_ARG=<{value}>" in output
+ assert "PREPARE_ARG=" in output
+ assert "PREPARE_ARG=" in output
+ assert "PREPARE_ARG=" in output
+ assert "PREPARE_ARG=<10>" in output
+ assert "TIMEOUT_ARG=<900>" in output
+ assert "ADAPTER_ARG=<--suite>" not in output
+ assert "ARCHIVE_PROJECT_ROOT=" not in output
+ assert not (results_dir / "bfcl_upstream_artifacts.tar.gz").exists()
+ assert "ADAPTER_ARG=" not in output
+ assert "UNEXPECTED_CONTEXT_LOAD" not in output
+ assert f"STAGED=<{results_dir}>" not in output
+ assert "EVAL_RC=0" in output
+ assert "EVAL_COMPLETED_SUITE=bfcl_smoke" in output
+ assert f"EVAL_RESULT_DIR={results_dir}" in output
+ assert results_dir.exists()
+ assert not runtime_dir.exists()
+ assert not python_dir.exists()
+ assert not project_root.exists()
+
+
+def test_bfcl_full_suites_use_suite_specific_runtime_and_archive_before_cleanup(
+ tmp_path: Path,
+) -> None:
+ suite_contracts = (
+ ("bfcl_vllm_minimax_m3", "8", "7200"),
+ ("bfcl_vllm_kimi", "16", "14400"),
+ )
+
+ for suite, expected_threads, expected_timeout in suite_contracts:
+ suite_tmp_path = tmp_path / suite
+ suite_tmp_path.mkdir()
+ result, paths = _run_bfcl_adapter_command(
+ suite_tmp_path,
+ suite=suite,
+ )
+ results_dir, runtime_dir, python_dir, project_root = paths
+ output = result.stdout + result.stderr
+
+ assert result.returncode == 0, result.stderr
+ assert f"TIMEOUT_ARG=<{expected_timeout}>" in output
+ assert "ADAPTER_ARG=<--suite>" in output
+ assert f"ADAPTER_ARG=<{suite}>" in output
+ assert f"EVAL_COMPLETED_SUITE={suite}" in output
+ assert "ADAPTER_ARG=<--num-threads>" in output
+ assert f"ADAPTER_ARG=<{expected_threads}>" in output
+ assert f"ARCHIVE_PROJECT_ROOT=<{project_root}>" in output
+ assert (
+ f"ARCHIVE_PATH=<{results_dir / 'bfcl_upstream_artifacts.tar.gz'}>" in output
+ )
+ assert (results_dir / "bfcl_upstream_artifacts.tar.gz").exists()
+ assert not runtime_dir.exists()
+ assert not python_dir.exists()
+ assert not project_root.exists()
+
+
+def test_bfcl_full_suite_archive_failure_preserves_scores_and_cleans_runtime(
+ tmp_path: Path,
+) -> None:
+ result, paths = _run_bfcl_adapter_command(
+ tmp_path,
+ suite="bfcl_vllm_kimi",
+ archive_rc=73,
+ )
+ results_dir, runtime_dir, python_dir, project_root = paths
+ output = result.stdout + result.stderr
+
+ assert result.returncode == 73
+ assert "failed to archive BFCL upstream artifacts (exit code 73)" in output
+ assert (results_dir / "bfcl_report.json").exists()
+ assert (results_dir / "results_bfcl.json").exists()
+ assert not (results_dir / "bfcl_upstream_artifacts.tar.gz").exists()
+ assert not runtime_dir.exists()
+ assert not python_dir.exists()
+ assert not project_root.exists()
+
+
+def test_bfcl_adapter_timeout_writes_reports_stages_and_propagates(
+ tmp_path: Path,
+) -> None:
+ result, paths = _run_bfcl_adapter_command(tmp_path, adapter_rc=124)
+ results_dir, runtime_dir, python_dir, project_root = paths
+ output = result.stdout + result.stderr
+
+ assert result.returncode == 124
+ assert "EVAL_RC=124" in output
+ assert output.count(f"STAGED=<{results_dir}>") == 1
+ assert "ADAPTER_ARG=<--integration-error>" in output
+ assert "ADAPTER_ARG=<--suite>" in output
+ assert "ADAPTER_ARG=" in output
+ assert "ADAPTER_ARG=" in output
+ assert (results_dir / "bfcl_report.json").exists()
+ assert (results_dir / "results_bfcl.json").exists()
+ assert "failed to write BFCL failure artifact" not in output
+ assert "run_eval failed with exit code 124" in result.stderr
+ assert not runtime_dir.exists()
+ assert not python_dir.exists()
+ assert not project_root.exists()
+
+
+def test_bfcl_upstream_archive_is_deterministic_and_survives_cleanup(
+ tmp_path: Path,
+) -> None:
+ project_root = tmp_path / "project"
+ result_path = project_root / "result/run/BFCL_v4_simple_python_result.json"
+ score_path = project_root / "score/run/BFCL_v4_simple_python_score.json"
+ id_path = project_root / "test_case_ids_to_generate.json"
+ for path, content in (
+ (result_path, '{"id":"simple_python_0"}\n'),
+ (score_path, '{"accuracy":1.0}\n'),
+ (id_path, '{"simple_python":["simple_python_0"]}\n'),
+ ):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content)
+ first_archive = tmp_path / "first.tar.gz"
+ second_archive = tmp_path / "second.tar.gz"
+ script = r"""
+source "$BENCHMARK_LIB"
+VENDOR_VERIFIER_PYTHON="$PYTHON"
+_archive_bfcl_upstream_artifacts "$PROJECT_ROOT" "$FIRST_ARCHIVE"
+_archive_bfcl_upstream_artifacts "$PROJECT_ROOT" "$SECOND_ARCHIVE"
+_cleanup_vendor_eval "$PROJECT_ROOT"
+"""
+ subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "PYTHON": sys.executable,
+ "PROJECT_ROOT": str(project_root),
+ "FIRST_ARCHIVE": str(first_archive),
+ "SECOND_ARCHIVE": str(second_archive),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert first_archive.read_bytes() == second_archive.read_bytes()
+ with tarfile.open(first_archive, "r:gz") as archive:
+ assert archive.getnames() == [
+ "result",
+ "result/run",
+ "result/run/BFCL_v4_simple_python_result.json",
+ "score",
+ "score/run",
+ "score/run/BFCL_v4_simple_python_score.json",
+ "test_case_ids_to_generate.json",
+ ]
+ assert not project_root.exists()
+
+
+def test_bfcl_upstream_archive_rejects_symbolic_links(tmp_path: Path) -> None:
+ project_root = tmp_path / "project"
+ project_root.mkdir()
+ outside = tmp_path / "outside.json"
+ outside.write_text('{"secret":true}\n')
+ (project_root / "escape.json").symlink_to(outside)
+ archive = tmp_path / "unsafe.tar.gz"
+ script = r"""
+source "$BENCHMARK_LIB"
+VENDOR_VERIFIER_PYTHON="$PYTHON"
+_archive_bfcl_upstream_artifacts "$PROJECT_ROOT" "$ARCHIVE"
+"""
+
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "PYTHON": sys.executable,
+ "PROJECT_ROOT": str(project_root),
+ "ARCHIVE": str(archive),
+ },
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode != 0
+ assert "refusing to archive symbolic link: escape.json" in result.stderr
+ assert not archive.exists()
+ assert not (tmp_path / ".unsafe.tar.gz.tmp").exists()
+
+
+def test_bfcl_installer_uses_verified_wheel_in_selected_venv(tmp_path: Path) -> None:
+ script = r"""
+source "$BENCHMARK_LIB"
+selected_python() { printf 'PYTHON_ARG=<%s>\n' "$@"; }
+timeout() {
+ printf 'TIMEOUT_ARG=<%s>\n' "$1"
+ shift
+ "$@"
+}
+VENDOR_VERIFIER_PYTHON=selected_python
+_install_bfcl_eval_deps "$DOWNLOAD_DIR"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "DOWNLOAD_DIR": str(tmp_path),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ wheel_path = tmp_path / "bfcl_eval-2026.3.23-py3-none-any.whl"
+
+ for value in (
+ "https://files.pythonhosted.org/packages/ba/41/"
+ "ed458527c770c50225b60bae3b0c3444b26804ee455fa2d8f187018d2cb2/"
+ "bfcl_eval-2026.3.23-py3-none-any.whl",
+ "3bb6dfa5f0c68ad403c9ec50b00db2bb3b4cc9b38ab1ff33f48fe30d853d3a0a",
+ str(wheel_path),
+ "-m",
+ "pip",
+ "install",
+ "--no-cache-dir",
+ "soundfile==0.13.1",
+ ):
+ assert f"PYTHON_ARG=<{value}>" in result.stdout
+ assert "TIMEOUT_ARG=<600>" in result.stdout
+ assert "--break-system-packages" not in result.stdout
+ assert "--target" not in result.stdout
+
+
+def test_bfcl_python_preparation_exposes_system_site_packages(
+ tmp_path: Path,
+) -> None:
+ python_root = tmp_path / "bfcl-python"
+ script = r"""
+source "$BENCHMARK_LIB"
+python3() {
+ if [ "$1" = "-c" ]; then
+ printf 'VERSION_CHECK_ARG=<%s>\n' "$@"
+ return 0
+ fi
+ printf 'SYSTEM_PYTHON_ARG=<%s>\n' "$@"
+ venv_dir="${!#}"
+ mkdir -p "$venv_dir/bin"
+ printf '#!/usr/bin/env bash\n' > "$venv_dir/bin/python"
+ chmod +x "$venv_dir/bin/python"
+}
+mktemp() {
+ mkdir "$PYTHON_ROOT"
+ printf '%s\n' "$PYTHON_ROOT"
+}
+_prepare_vendor_verifier_python "BFCL" "bfcl-python" true 10
+cleanup_dir="$VENDOR_VERIFIER_PYTHON_CLEANUP_DIR"
+printf 'SELECTED_PYTHON=<%s>\n' "$VENDOR_VERIFIER_PYTHON"
+_cleanup_vendor_eval "$cleanup_dir"
+"""
+ result = subprocess.run(
+ ["bash", "-c", script],
+ env={
+ **os.environ,
+ "BENCHMARK_LIB": str(BENCHMARK_LIB),
+ "PYTHON_ROOT": str(python_root),
+ },
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+
+ assert "SYSTEM_PYTHON_ARG=<-m>" in result.stdout
+ assert "SYSTEM_PYTHON_ARG=" in result.stdout
+ assert "SYSTEM_PYTHON_ARG=<--system-site-packages>" in result.stdout
+ assert "VERSION_CHECK_ARG=<10>" in result.stdout
+ assert f"SELECTED_PYTHON=<{python_root / 'venv/bin/python'}>" in result.stdout
+ assert "--prefix" not in result.stdout
+ assert not python_root.exists()
diff --git a/utils/evals/thresholds.yaml b/utils/evals/thresholds.yaml
index 6ff9731c45..d9b41eba40 100644
--- a/utils/evals/thresholds.yaml
+++ b/utils/evals/thresholds.yaml
@@ -1,22 +1,57 @@
-# Model thresholds override task defaults.
-default:
- gsm8k: 0.90
- gpqa_diamond_cot_n_shot: 0.30
- swebench_lite: 0.50
-models:
- dsr1:
- gsm8k: 0.91
- dsv4:
- gsm8k: 0.91
- glm5:
- gsm8k: 0.94
- glm5.1:
- gsm8k: 0.93
- gptoss:
- gsm8k: 0.91
- kimik2.5:
- gsm8k: 0.90
- minimaxm2.5:
- gsm8k: 0.92
- qwen3.5:
- gsm8k: 0.94
+{
+ "default": {
+ "gsm8k": 0.90,
+ "kimi_tool_call_schema": 0.0,
+ "kimi_tool_call_schema_full": 0.0,
+ "minimax_m3_smoke": 0.0,
+ "minimax_m3_full": 0.0,
+ "bfcl_smoke": 0.0,
+ "bfcl_simple_python": 0.0,
+ "bfcl_multiple": 0.0,
+ "bfcl_parallel": 0.0,
+ "bfcl_irrelevance": 0.0,
+ "bfcl_vllm_minimax_m3": 0.0,
+ "bfcl_vllm_minimax_m3_simple_python": 0.0,
+ "bfcl_vllm_minimax_m3_multiple": 0.0,
+ "bfcl_vllm_minimax_m3_parallel": 0.0,
+ "bfcl_vllm_minimax_m3_parallel_multiple": 0.0,
+ "bfcl_vllm_kimi": 0.0,
+ "bfcl_vllm_kimi_simple_python": 0.0,
+ "bfcl_vllm_kimi_multiple": 0.0,
+ "bfcl_vllm_kimi_parallel": 0.0,
+ "bfcl_vllm_kimi_parallel_multiple": 0.0,
+ "bfcl_vllm_kimi_multi_turn": 0.0,
+ "bfcl_vllm_kimi_multi_turn_base": 0.0,
+ "bfcl_vllm_kimi_multi_turn_miss_func": 0.0,
+ "bfcl_vllm_kimi_multi_turn_miss_param": 0.0,
+ "bfcl_vllm_kimi_multi_turn_long_context": 0.0,
+ "gpqa_diamond_cot_n_shot": 0.30,
+ "swebench_lite": 0.50
+ },
+ "models": {
+ "dsr1": {
+ "gsm8k": 0.91
+ },
+ "dsv4": {
+ "gsm8k": 0.91
+ },
+ "glm5": {
+ "gsm8k": 0.94
+ },
+ "glm5.1": {
+ "gsm8k": 0.93
+ },
+ "gptoss": {
+ "gsm8k": 0.91
+ },
+ "kimik2.5": {
+ "gsm8k": 0.90
+ },
+ "minimaxm2.5": {
+ "gsm8k": 0.92
+ },
+ "qwen3.5": {
+ "gsm8k": 0.94
+ }
+ }
+}
diff --git a/utils/evals/validate_scores.py b/utils/evals/validate_scores.py
index ba7fc13962..ebea50fb5e 100644
--- a/utils/evals/validate_scores.py
+++ b/utils/evals/validate_scores.py
@@ -2,9 +2,11 @@
"""Validate eval scores against per-task and per-model thresholds."""
from __future__ import annotations
+
import argparse
import glob
import json
+import math
import os
import re
import sys
@@ -68,6 +70,49 @@ def resolve_threshold(config: dict, prefix: str | None, task: str, fallback: flo
return default[task], "default"
return fallback, "min-score"
+def invalid_effective_count(data: dict, task: str) -> tuple[bool, object]:
+ """Return whether an explicitly present effective count is invalid."""
+ if "n-samples" not in data:
+ return False, None
+ sample_counts = data["n-samples"]
+ if not isinstance(sample_counts, dict) or task not in sample_counts:
+ return True, sample_counts
+ task_samples = sample_counts[task]
+ if not isinstance(task_samples, dict) or "effective" not in task_samples:
+ return True, task_samples
+ effective = task_samples["effective"]
+ invalid = (
+ isinstance(effective, bool)
+ or not isinstance(effective, (int, float))
+ or not math.isfinite(effective)
+ or effective <= 0
+ )
+ return invalid, effective
+
+def metric_prefixes(data: dict, task: str, override: str | None) -> tuple[str, ...]:
+ """Resolve score metric prefixes from an explicit override or task config."""
+ if override is not None:
+ return (override,)
+ task_config = data.get("configs", {}).get(task, {})
+ metric_list = task_config.get("metric_list", [])
+ declared = tuple(
+ f"{item['metric']},"
+ for item in metric_list
+ if isinstance(item, dict)
+ and isinstance(item.get("metric"), str)
+ and item["metric"]
+ )
+ return declared or ("exact_match,",)
+
+
+def integration_error_message(error: object) -> str:
+ """Render the structured integration error fields for a direct failure."""
+ if isinstance(error, dict):
+ error_type = error.get("type", "unknown")
+ message = error.get("message", "")
+ return f"{error_type}: {message}"
+ return f"unknown: {error}"
+
def validate_batch_manifest(
meta_env_path: str,
@@ -197,8 +242,9 @@ def main() -> int:
help="Override the detected model prefix (default: read from meta_env.json / $MODEL_PREFIX)",
)
parser.add_argument(
- "--metric-prefix", default="exact_match,",
- help="Only check metrics whose name starts with this prefix (default: 'exact_match,')",
+ "--metric-prefix",
+ default=None,
+ help="Override task-config metric selection with one metric prefix",
)
parser.add_argument(
"--results-glob", default="results*.json",
@@ -277,10 +323,28 @@ def main() -> int:
conc_label = f"[conc={match.group(1)}] " if match else ""
with open(f) as fh:
data = json.load(fh)
+ if "integration_error" in data:
+ print(
+ f"FAIL: {conc_label}integration failure: "
+ f"{integration_error_message(data['integration_error'])}",
+ file=sys.stderr,
+ )
+ failed = True
+ continue
for task, metrics in data.get("results", {}).items():
+ invalid_effective, effective = invalid_effective_count(data, task)
+ if invalid_effective:
+ print(
+ f"FAIL: {conc_label}{task} invalid effective sample count: "
+ f"{effective!r}",
+ file=sys.stderr,
+ )
+ failed = True
+ continue
min_score, source = resolve_threshold(config, prefix, task, args.min_score)
+ prefixes = metric_prefixes(data, task, args.metric_prefix)
for name, val in metrics.items():
- if not name.startswith(args.metric_prefix) or "stderr" in name:
+ if not name.startswith(prefixes) or "stderr" in name:
continue
if not isinstance(val, (int, float)):
continue
@@ -297,7 +361,8 @@ def main() -> int:
)
if checked == 0:
- print("WARN: no metrics matched prefix '{}'".format(args.metric_prefix), file=sys.stderr)
+ selector = args.metric_prefix or "declared task metrics"
+ print(f"WARN: no metrics matched {selector!r}", file=sys.stderr)
return 1 if (failed or checked == 0) else 0
diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py
index e3872c3d77..ecd5bd7bad 100644
--- a/utils/matrix_logic/generate_sweep_configs.py
+++ b/utils/matrix_logic/generate_sweep_configs.py
@@ -27,6 +27,11 @@
}
MIN_EVAL_CONC = 16
+DEFAULT_EVAL_FRAMEWORK = "lm-eval"
+AUTOMATIC_AGENTIC_VENDOR_EVALS = {
+ "kimik3": ("kimi-vendor", "kimi_tool_call_schema"),
+ "minimaxm3": ("minimax-vendor", "minimax_m3_smoke"),
+}
# Bound how many multinode agentic conc points share one server allocation.
# 1 = one task/SLURM allocation per concurrency (matches single-node agentic).
MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1
@@ -47,6 +52,78 @@ def seq_len_to_str(isl: int, osl: int) -> str:
"""
return seq_len_itos.get((isl, osl), f"{isl}_{osl}")
+def freeze_config_value(value):
+ """Convert JSON-shaped config values into deterministic hashable values."""
+ if isinstance(value, dict):
+ return tuple(
+ sorted((key, freeze_config_value(item)) for key, item in value.items())
+ )
+ if isinstance(value, list):
+ return tuple(freeze_config_value(item) for item in value)
+ return value
+
+
+def trim_conc(entries: list[dict]) -> list[dict]:
+ """Retain the lowest concurrency for each generated deployment shape.
+
+ Entries are grouped by every non-eval field except ``conc`` and the
+ generated ``exp-name``. Multi-node rows may encode concurrency either as a
+ list within one row or as one-row list chunks; both representations collapse
+ to one row whose ``conc`` and dispatch-facing ``eval-conc`` use the minimum.
+ """
+ ignored_fields = {
+ "conc",
+ "exp-name",
+ "run-eval",
+ "eval-only",
+ "eval-conc",
+ "eval-all-concs",
+ Fields.EVAL_FRAMEWORK.value,
+ Fields.EVAL_SUITE.value,
+ }
+ groups: dict[tuple, list[int]] = {}
+ out: list[dict] = []
+
+ def minimum_concurrency(entry: dict):
+ conc = entry["conc"]
+ return min(conc) if isinstance(conc, list) else conc
+
+ for source_entry in entries:
+ entry = source_entry
+ conc = entry.get("conc")
+ if entry.get("prefill") is not None and isinstance(conc, list) and conc:
+ minimum_conc = min(conc)
+ if len(conc) > 1 or entry.get("eval-conc") != minimum_conc:
+ entry = {**entry, "conc": [minimum_conc]}
+ if "eval-conc" in entry:
+ entry["eval-conc"] = minimum_conc
+
+ key = tuple(
+ sorted(
+ (key, freeze_config_value(value))
+ for key, value in entry.items()
+ if key not in ignored_fields
+ )
+ )
+ groups.setdefault(key, []).append(len(out))
+ out.append(entry)
+
+ drop: set[int] = set()
+ for indices in groups.values():
+ keep = min(indices, key=lambda index: minimum_concurrency(out[index]))
+ kept_entry = out[keep]
+ if any(out[index].get("run-eval") is True for index in indices):
+ kept_entry = {**kept_entry, "run-eval": True}
+ if kept_entry.get("prefill") is not None:
+ kept_entry["eval-conc"] = minimum_concurrency(kept_entry)
+ if any(
+ out[index].get("eval-all-concs") is True for index in indices
+ ):
+ kept_entry["eval-all-concs"] = True
+ out[keep] = kept_entry
+ drop.update(index for index in indices if index != keep)
+ return [entry for index, entry in enumerate(out) if index not in drop]
+
def runner_labels(runner_data: dict) -> dict:
"""Return runner scheduling labels."""
@@ -439,6 +516,8 @@ def _multinode_parallelism_key(entry: dict) -> tuple:
Fields.EVAL_CONC.value,
Fields.EVAL_ALL_CONCS.value,
Fields.EXP_NAME.value,
+ Fields.EVAL_FRAMEWORK.value,
+ Fields.EVAL_SUITE.value,
}
return tuple(sorted(
(key, _freeze_matrix_value(value))
@@ -447,26 +526,25 @@ def _multinode_parallelism_key(entry: dict) -> tuple:
))
+def automatic_agentic_vendor_eval(entry: dict) -> tuple[str, str] | None:
+ """Return the default vendor evaluator for supported agentic models."""
+ if entry.get(Fields.SCENARIO_TYPE.value) != "agentic-coding":
+ return None
+ return AUTOMATIC_AGENTIC_VENDOR_EVALS.get(
+ entry.get(Fields.MODEL_PREFIX.value)
+ )
+
+
def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) -> list[dict]:
- """Eval selection policy:
- - Single-node: only consider 8k1k (isl=8192, osl=1024).
- For each unique (model, runner, framework, precision, isl, osl, spec-decoding, dp-attn):
- - Ignore entries with conc < MIN_EVAL_CONC
- - Mark all entries at the highest CONC (all TPs)
- - Mark all entries at the median CONC (all TPs)
- - Multi-node: only consider 8k1k entries. For every distinct parallelism
- configuration:
- - Ignore entries with all conc values < MIN_EVAL_CONC
- - Mark the entry containing its highest eligible concurrency
- - Set eval-conc to that highest eligible concurrency
- - Agentic evals are opt-in to preserve default throughput coverage.
- - Single-node: run GSM8K through the same lm-eval path as fixed-sequence
- 8k1k evals, marking the highest-conc entry per (model, runner,
- framework, precision) group.
- - Multi-node: same policy as the fixed-seq-len multi-node case above
- (highest eligible conc per distinct parallelism config, via
- eval-conc), using SWE-bench since it doesn't support batched
- concurrencies.
+ """Apply the default eval selection policy.
+
+ Kimi K3 and MiniMax M3 agentic rows use their vendor validators at every
+ generated concurrency. Other agentic rows remain opt-in and use GSM8K at
+ the highest concurrency in each deployment group.
+
+ Fixed-sequence selection is unchanged: single-node 8k1k rows use the
+ highest and median concurrency per model/runtime group, while multi-node
+ 8k1k rows use the highest eligible concurrency per parallelism topology.
"""
from collections import defaultdict
@@ -479,6 +557,18 @@ def _eligible_eval_concs(entry):
conc_values = conc if isinstance(conc, list) else [conc]
return sorted(c for c in conc_values if c >= MIN_EVAL_CONC)
+ automatic_eval_specs: dict[int, tuple[str, str]] = {}
+ for i, entry in enumerate(matrix_values):
+ eval_spec = automatic_agentic_vendor_eval(entry)
+ if eval_spec is None:
+ continue
+ automatic_eval_specs[i] = eval_spec
+ eval_indices.add(i)
+ if Fields.PREFILL.value in entry:
+ conc = entry[Fields.CONC.value]
+ conc_values = conc if isinstance(conc, list) else [conc]
+ mn_eval_conc[i] = max(conc_values)
+
# Single-node: group by (model, runner, framework, precision, isl, osl, spec-decoding, dp-attn).
# Only 8k1k entries with a top-level TP (single-node schema).
sn_groups = defaultdict(list)
@@ -534,10 +624,11 @@ def _eligible_eval_concs(entry):
ag_sn_groups = defaultdict(list)
# Multi-node agentic: same "highest eligible conc per distinct
# parallelism config" policy as the fixed-seq-len mn_groups above.
- # SWE-bench doesn't support batched concurrencies (unlike lm-eval),
- # so exactly one conc is picked per group, never the full list.
+ # The selected eval subset uses exactly one conc per group.
ag_mn_groups = defaultdict(list)
for i, entry in enumerate(matrix_values):
+ if i in automatic_eval_specs:
+ continue
if entry.get(Fields.SCENARIO_TYPE.value) != 'agentic-coding':
continue
if Fields.PREFILL.value in entry:
@@ -563,7 +654,14 @@ def _eligible_eval_concs(entry):
mn_eval_conc[best_idx] = best_eval_conc
for i, entry in enumerate(matrix_values):
- entry[Fields.RUN_EVAL.value] = i in eval_indices
+ run_eval = i in eval_indices
+ entry[Fields.RUN_EVAL.value] = run_eval
+ if run_eval:
+ eval_framework, eval_suite = automatic_eval_specs.get(
+ i, (DEFAULT_EVAL_FRAMEWORK, "")
+ )
+ entry[Fields.EVAL_FRAMEWORK.value] = eval_framework
+ entry[Fields.EVAL_SUITE.value] = eval_suite
if i in mn_eval_conc:
entry[Fields.EVAL_CONC.value] = mn_eval_conc[i]
@@ -571,20 +669,16 @@ def _eligible_eval_concs(entry):
def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]:
- """Expand eval selection to every 8k1k fixed-sequence entry.
-
- Evals only run at 8k1k (matching mark_eval_entries), so entries at other
- sequence lengths (e.g. 1k1k) are passed through untouched rather than
- expanded into eval rows.
- Single-node agentic entries use GSM8K through the same lm-eval path as
- fixed-sequence 8k1k evals. Multi-node agentic entries use SWE-bench,
- which doesn't support batched concurrencies (unlike lm-eval): multi-node
- agentic rows with the same topology are merged (to recombine any chunking
- split), but only the highest resulting conc is marked for eval via
- eval-conc, not the full list.
- Multi-node fixed-seq-len rows with the same engine topology are merged
- into one eval row whose full concurrency list is run sequentially
- against the same engine.
+ """Expand eval selection across all eligible entries.
+
+ Kimi K3 and MiniMax M3 agentic rows remain one eval job per generated
+ concurrency, using the model's vendor validator. Other agentic entries use
+ GSM8K through lm-eval. Their multi-node rows are merged by topology and
+ select the highest resulting concurrency.
+
+ Fixed-sequence evals only run at 8k1k. Multi-node rows with the same engine
+ topology are merged into one eval row that runs every concurrency
+ sequentially against the live engine.
"""
expanded_entries: list[dict] = []
multinode_indices: dict[tuple, int] = {}
@@ -593,6 +687,23 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]:
target_isl, target_osl = seq_len_stoi["8k1k"]
for entry in matrix_values:
+ automatic_eval = automatic_agentic_vendor_eval(entry)
+ if automatic_eval is not None:
+ eval_framework, eval_suite = automatic_eval
+ eval_entry = {
+ **entry,
+ Fields.RUN_EVAL.value: True,
+ Fields.EVAL_FRAMEWORK.value: eval_framework,
+ Fields.EVAL_SUITE.value: eval_suite,
+ }
+ if Fields.PREFILL.value in entry:
+ conc = entry[Fields.CONC.value]
+ conc_values = conc if isinstance(conc, list) else [conc]
+ eval_entry[Fields.CONC.value] = sorted(set(conc_values))
+ eval_entry[Fields.EVAL_CONC.value] = max(conc_values)
+ expanded_entries.append(eval_entry)
+ continue
+
if entry.get(Fields.SCENARIO_TYPE.value) == 'agentic-coding':
if Fields.PREFILL.value not in entry:
entry[Fields.RUN_EVAL.value] = True
@@ -653,6 +764,14 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]:
entry[Fields.RUN_EVAL.value] = True
expanded_entries.append(entry)
+ for entry in expanded_entries:
+ if not entry.get(Fields.RUN_EVAL.value):
+ continue
+ if not entry.get(Fields.EVAL_FRAMEWORK.value):
+ entry[Fields.EVAL_FRAMEWORK.value] = DEFAULT_EVAL_FRAMEWORK
+ if entry.get(Fields.EVAL_SUITE.value) is None:
+ entry[Fields.EVAL_SUITE.value] = ""
+
return expanded_entries
@@ -1413,6 +1532,33 @@ def expand_config_keys(config_keys, available_keys):
return list(seen)
+def filter_exp_names(entries: list[dict], exp_names: list[str]) -> list[dict]:
+ """Select exact generated experiment identities and reject ambiguity."""
+ requested = set(exp_names)
+ if len(requested) != len(exp_names):
+ raise ValueError("--exp-names contains duplicate values")
+
+ matches: dict[str, int] = {name: 0 for name in exp_names}
+ for entry in entries:
+ exp_name = entry.get(Fields.EXP_NAME.value)
+ if exp_name in matches:
+ matches[exp_name] += 1
+
+ missing = sorted(name for name, count in matches.items() if count == 0)
+ ambiguous = sorted(name for name, count in matches.items() if count > 1)
+ if missing:
+ raise ValueError("Experiment name(s) not found: " + ", ".join(missing))
+ if ambiguous:
+ raise ValueError(
+ "Experiment name(s) matched multiple rows: " + ", ".join(ambiguous)
+ )
+ return [
+ entry
+ for entry in entries
+ if entry.get(Fields.EXP_NAME.value) in requested
+ ]
+
+
def apply_node_type_defaults(args):
"""Default both single_node and multi_node to True when neither is specified."""
if hasattr(args, 'single_node') and hasattr(args, 'multi_node'):
@@ -1455,6 +1601,14 @@ def main():
'Can be combined with --evals-only; used alone, it also emits eval-only jobs.'
)
)
+ parent_parser.add_argument(
+ '--trim-conc',
+ action='store_true',
+ help=(
+ 'Trim each generated deployment shape to its minimum concurrency '
+ 'after applying eval selection.'
+ )
+ )
parent_parser.add_argument(
'--runner-node-filter',
required=False,
@@ -1584,6 +1738,15 @@ def main():
required=False,
help='Only include these concurrency values. Values must exist in the config conc-range/list.'
)
+ test_config_keys_parser.add_argument(
+ '--exp-names',
+ nargs='+',
+ required=False,
+ help=(
+ 'Only include exact generated experiment names. Each name must '
+ 'match exactly one row after config and concurrency filtering.'
+ )
+ )
test_config_keys_parser.add_argument(
'--seq-lens',
nargs='+',
@@ -1622,13 +1785,23 @@ def main():
matrix_values = generate_test_config_sweep(args, all_config_data, runner_data)
else:
parser.error(f"Unknown command: {args.command}")
+
+ if args.command == 'test-config' and args.exp_names:
+ try:
+ matrix_values = filter_exp_names(matrix_values, args.exp_names)
+ except ValueError as error:
+ parser.error(str(error))
+
# Apply the existing eval policy first, then expand it when requested.
if not args.no_evals:
matrix_values = mark_eval_entries(matrix_values, include_agentic=args.evals_only or args.all_evals)
if args.all_evals:
matrix_values = mark_all_eval_entries(matrix_values)
+ if args.trim_conc:
+ matrix_values = trim_conc(matrix_values)
+
if args.evals_only or args.all_evals:
matrix_values = [e for e in matrix_values if e.get(Fields.RUN_EVAL.value, False)]
for entry in matrix_values:
diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py
index 1b4de07d48..520e4105e0 100644
--- a/utils/matrix_logic/test_generate_sweep_configs.py
+++ b/utils/matrix_logic/test_generate_sweep_configs.py
@@ -1,7 +1,8 @@
"""Comprehensive tests for generate_sweep_configs.py"""
-
import argparse
import copy
+import hashlib
+import json
from pathlib import Path
import pytest
@@ -10,6 +11,7 @@
add_multinode_node_count,
apply_node_type_defaults,
expand_config_keys,
+ filter_exp_names,
generate_full_sweep,
generate_test_config_sweep,
mark_all_eval_entries,
@@ -19,6 +21,7 @@
seq_len_itos,
seq_len_stoi,
seq_len_to_str,
+ trim_conc,
)
from validation import load_config_files, load_runner_file
@@ -469,6 +472,100 @@ def test_default_mode_does_not_mark_agentic(self):
f"Expected 0 agentic entries marked run-eval in default mode, got {len(marked)}"
)
+ def test_default_marks_every_supported_vendor_point(self):
+ matrix_values = [
+ {
+ "scenario-type": "agentic-coding",
+ "model-prefix": model_prefix,
+ "model": model_prefix,
+ "runner": "b300",
+ "framework": "vllm",
+ "precision": "fp4",
+ "tp": 8,
+ "conc": conc,
+ }
+ for model_prefix in ("kimik3", "minimaxm3")
+ for conc in (1, 64)
+ ]
+ matrix_values.append({
+ "scenario-type": "agentic-coding",
+ "model-prefix": "minimaxm3-bfcl",
+ "model": "unsupported",
+ "runner": "b300",
+ "framework": "vllm",
+ "precision": "fp4",
+ "tp": 8,
+ "conc": 64,
+ })
+
+ result = mark_eval_entries(matrix_values)
+
+ expected = {
+ "kimik3": ("kimi-vendor", "kimi_tool_call_schema"),
+ "minimaxm3": ("minimax-vendor", "minimax_m3_smoke"),
+ }
+ for model_prefix, eval_spec in expected.items():
+ rows = [row for row in result if row["model-prefix"] == model_prefix]
+ assert {row["conc"] for row in rows} == {1, 64}
+ assert all(row["run-eval"] is True for row in rows)
+ assert {
+ (row["eval-framework"], row["eval-suite"]) for row in rows
+ } == {eval_spec}
+
+ unsupported = result[-1]
+ assert unsupported["run-eval"] is False
+ assert "eval-framework" not in unsupported
+ assert all(row.get("eval-framework") != "bfcl" for row in result)
+
+ def test_default_marks_every_multinode_vendor_point(self):
+ common = {
+ "scenario-type": "agentic-coding",
+ "model-prefix": "kimik3",
+ "model": "kimi",
+ "runner": "gb200",
+ "framework": "sglang-disagg",
+ "precision": "fp4",
+ "spec-decoding": "none",
+ "disagg": True,
+ "prefill": {"num-worker": 1, "tp": 8},
+ "decode": {"num-worker": 1, "tp": 8},
+ }
+ matrix_values = [
+ {**common, "conc": [2], "exp-name": "kimi-conc2"},
+ {**common, "conc": [32], "exp-name": "kimi-conc32"},
+ ]
+
+ result = mark_eval_entries(matrix_values)
+
+ assert len(result) == 2
+ assert all(row["run-eval"] is True for row in result)
+ assert [row["eval-conc"] for row in result] == [2, 32]
+ assert all(row["eval-framework"] == "kimi-vendor" for row in result)
+ assert all(
+ row["eval-suite"] == "kimi_tool_call_schema" for row in result
+ )
+
+ def test_fixed_sequence_eval_uses_lm_eval_metadata(self):
+ matrix_values = [{
+ "model": "m",
+ "runner": "b200",
+ "framework": "vllm",
+ "precision": "fp8",
+ "isl": 8192,
+ "osl": 1024,
+ "spec-decoding": "none",
+ "dp-attn": False,
+ "tp": 8,
+ "conc": MIN_EVAL_CONC,
+ }]
+
+ result = mark_eval_entries(matrix_values)
+
+ assert result[0]["run-eval"] is True
+ assert result[0]["eval-framework"] == "lm-eval"
+ assert result[0]["eval-suite"] == ""
+
+
def test_single_node_skips_eval_entries_below_min_conc(self):
"""Single-node eval selection should ignore conc values below MIN_EVAL_CONC."""
matrix_values = [
@@ -893,11 +990,11 @@ def test_marks_agentic_entries_for_gsm8k(self):
assert result[0]['run-eval'] is True
assert 'eval-conc' not in result[0]
- def test_marks_multinode_agentic_entries_for_swebench(self):
- """Unlike fixed-seq-len multi-node (which batches every concurrency
- into one lm-eval row via eval-all-concs), multi-node agentic rows for
- the same topology are merged but only their highest conc is marked
- via eval-conc, since SWE-bench doesn't support batched concurrencies."""
+ def test_marks_multinode_agentic_entries_for_gsm8k(self):
+ """Unlike fixed-seq-len multi-node evals, generic agentic rows with the
+ same topology merge but select only their highest concurrency through
+ eval-conc.
+ """
common = {
'scenario-type': 'agentic-coding',
'model': 'm', 'runner': 'r', 'framework': 'sglang-disagg',
@@ -919,6 +1016,33 @@ def test_marks_multinode_agentic_entries_for_swebench(self):
assert result[0]['eval-conc'] == 32
assert 'eval-all-concs' not in result[0]
+ def test_keeps_every_multinode_vendor_point_separate(self):
+ common = {
+ "scenario-type": "agentic-coding",
+ "model-prefix": "minimaxm3",
+ "model": "minimax",
+ "runner": "gb200",
+ "framework": "sglang-disagg",
+ "precision": "fp4",
+ "spec-decoding": "none",
+ "disagg": True,
+ "prefill": {"num-worker": 1, "tp": 8},
+ "decode": {"num-worker": 1, "tp": 8},
+ }
+ entries = [
+ {**common, "conc": [2], "exp-name": "minimax-conc2"},
+ {**common, "conc": [32], "exp-name": "minimax-conc32"},
+ ]
+
+ result = mark_all_eval_entries(mark_eval_entries(entries))
+
+ assert len(result) == 2
+ assert [row["conc"] for row in result] == [[2], [32]]
+ assert [row["eval-conc"] for row in result] == [2, 32]
+ assert all(row["eval-framework"] == "minimax-vendor" for row in result)
+ assert all(row["eval-suite"] == "minimax_m3_smoke" for row in result)
+
+
# =============================================================================
# Test generate_full_sweep for single-node
@@ -1875,7 +1999,7 @@ class TestArgumentDefaults:
def test_runner_config_default_value(self):
"""Verify --runner-config defaults to configs/runners.yaml."""
import sys
- from generate_sweep_configs import main
+
# Save original sys.argv
original_argv = sys.argv
@@ -1892,7 +2016,7 @@ def test_runner_config_default_value(self):
# Parse args using the ArgumentParser from main
# We need to access the parser directly
import argparse
- from generate_sweep_configs import main
+
# Create the same parent parser as in main()
parent_parser = argparse.ArgumentParser(add_help=False)
@@ -1999,6 +2123,7 @@ def test_all_evals_cli_marks_every_fixed_sequence_entry(
"""--all-evals bypasses the default min-conc/highest-median policy but
still only evaluates 8k1k (1k1k entries are excluded)."""
import sys
+
import generate_sweep_configs
monkeypatch.setattr(
@@ -2038,6 +2163,7 @@ def test_all_evals_composes_with_evals_only(
sample_runner_config,
):
import sys
+
import generate_sweep_configs
monkeypatch.setattr(
@@ -2068,6 +2194,196 @@ def test_all_evals_composes_with_evals_only(
assert all(entry['run-eval'] is True for entry in result)
assert all(entry['eval-only'] is True for entry in result)
+ def test_trim_conc_reduces_generated_eval_matrix(
+ self,
+ monkeypatch,
+ sample_single_node_config,
+ sample_runner_config,
+ ):
+ import sys
+
+ import generate_sweep_configs
+
+ monkeypatch.setattr(
+ generate_sweep_configs,
+ 'load_config_files',
+ lambda _: sample_single_node_config,
+ )
+ monkeypatch.setattr(
+ generate_sweep_configs,
+ 'load_runner_file',
+ lambda _: sample_runner_config,
+ )
+ monkeypatch.setattr(sys, 'argv', [
+ 'generate_sweep_configs.py',
+ 'test-config',
+ '--config-files', 'dummy.yaml',
+ '--config-keys', 'dsr1-fp8-mi300x-sglang',
+ '--evals-only',
+ '--all-evals',
+ '--trim-conc',
+ ])
+
+ result = generate_sweep_configs.main()
+
+ assert len(result) == 1
+ assert result[0]['conc'] == 4
+ assert result[0]['run-eval'] is True
+ assert result[0]['eval-only'] is True
+
+ def test_trim_conc_updates_multinode_dispatch_concurrency(self):
+ low_entry = {
+ 'prefill': {'num-worker': 1, 'tp': 8},
+ 'decode': {'num-worker': 0, 'tp': 8},
+ 'conc': [4],
+ }
+ high_entry = {
+ **low_entry,
+ 'conc': [64],
+ 'run-eval': True,
+ 'eval-conc': 64,
+ }
+
+ result = trim_conc([high_entry, low_entry])
+
+ assert len(result) == 1
+ assert result[0]['conc'] == [4]
+ assert result[0]['eval-conc'] == 4
+ assert result[0]['run-eval'] is True
+
+ def test_kimi_minimax_default_matrix_marks_every_current_point(
+ self,
+ monkeypatch,
+ ):
+ import sys
+
+ import generate_sweep_configs
+
+ repo_root = Path(__file__).resolve().parents[2]
+ monkeypatch.setattr(sys, 'argv', [
+ 'generate_sweep_configs.py',
+ 'full-sweep',
+ '--config-files',
+ str(repo_root / 'configs/nvidia-master.yaml'),
+ str(repo_root / 'configs/amd-master.yaml'),
+ '--runner-config',
+ str(repo_root / 'configs/runners.yaml'),
+ '--model-prefix',
+ 'kimik3',
+ 'minimaxm3',
+ '--scenario-type',
+ 'agentic-coding',
+ ])
+
+ rows = generate_sweep_configs.main()
+
+ expected_eval_specs = {
+ 'kimik3': ('kimi-vendor', 'kimi_tool_call_schema'),
+ 'minimaxm3': ('minimax-vendor', 'minimax_m3_smoke'),
+ }
+ assert {row['model-prefix'] for row in rows} == set(expected_eval_specs)
+ assert all(row['run-eval'] is True for row in rows)
+ assert all(row.get('eval-only') is not True for row in rows)
+ assert {
+ (
+ row['model-prefix'],
+ row['eval-framework'],
+ row['eval-suite'],
+ )
+ for row in rows
+ } == {
+ (model_prefix, *eval_spec)
+ for model_prefix, eval_spec in expected_eval_specs.items()
+ }
+ assert all(row['eval-framework'] != 'bfcl' for row in rows)
+
+
+ def test_kimi_minimax_trimmed_eval_matrix_covers_current_configs(
+ self,
+ monkeypatch,
+ ):
+ import sys
+
+ import generate_sweep_configs
+
+ repo_root = Path(__file__).resolve().parents[2]
+ monkeypatch.setattr(sys, 'argv', [
+ 'generate_sweep_configs.py',
+ 'full-sweep',
+ '--config-files',
+ str(repo_root / 'configs/nvidia-master.yaml'),
+ str(repo_root / 'configs/amd-master.yaml'),
+ '--runner-config',
+ str(repo_root / 'configs/runners.yaml'),
+ '--model-prefix',
+ 'kimik3',
+ 'minimaxm3',
+ '--scenario-type',
+ 'agentic-coding',
+ '--evals-only',
+ '--all-evals',
+ '--trim-conc',
+ ])
+
+ rows = generate_sweep_configs.main()
+ manifest_fields = (
+ 'model-prefix',
+ 'runner',
+ 'framework',
+ 'precision',
+ 'tp',
+ 'pp',
+ 'dcp-size',
+ 'pcp-size',
+ 'ep',
+ 'dp-attn',
+ 'prefill',
+ 'decode',
+ 'disagg',
+ 'kv-offloading',
+ 'kv-offload-backend',
+ 'spec-decoding',
+ 'exp-name',
+ )
+ manifest = sorted(
+ tuple(
+ (
+ field,
+ generate_sweep_configs.freeze_config_value(row.get(field)),
+ )
+ for field in manifest_fields
+ )
+ for row in rows
+ )
+ manifest_digest = hashlib.sha256(
+ json.dumps(manifest, separators=(',', ':')).encode()
+ ).hexdigest()
+
+ assert len(manifest) == 63
+ assert manifest_digest == (
+ '1630cdd6fbf77302ee3286b118710576aa090e1502b3e5ec482b7f537a3f1132'
+ ), manifest_digest
+ for row in rows:
+ if isinstance(row['conc'], list):
+ assert row['conc'] == [row['eval-conc']]
+ assert all(row['run-eval'] is True for row in rows)
+ assert all(row['eval-only'] is True for row in rows)
+ expected_eval_specs = {
+ 'kimik3': ('kimi-vendor', 'kimi_tool_call_schema'),
+ 'minimaxm3': ('minimax-vendor', 'minimax_m3_smoke'),
+ }
+ assert {
+ (
+ row['model-prefix'],
+ row['eval-framework'],
+ row['eval-suite'],
+ )
+ for row in rows
+ } == {
+ (model_prefix, *eval_spec)
+ for model_prefix, eval_spec in expected_eval_specs.items()
+ }
+
def test_all_evals_batches_each_multinode_concurrency(
self,
monkeypatch,
@@ -2075,6 +2391,7 @@ def test_all_evals_batches_each_multinode_concurrency(
sample_runner_config,
):
import sys
+
import generate_sweep_configs
config = sample_multinode_config
@@ -2117,6 +2434,7 @@ def test_all_evals_batches_each_multinode_concurrency(
def test_all_evals_cannot_combine_with_no_evals(self, monkeypatch):
import sys
+
import generate_sweep_configs
monkeypatch.setattr(sys, 'argv', [
@@ -2821,6 +3139,46 @@ def test_qwen_b300_fp4_fp8_memory_tier_matrix_is_balanced(self):
)
+# =============================================================================
+# Test filter_exp_names
+# =============================================================================
+
+
+class TestFilterExpNames:
+ def test_selects_exact_names_in_matrix_order(self):
+ entries = [
+ {"exp-name": "deployment-a", "conc": 1},
+ {"exp-name": "deployment-b", "conc": 1},
+ {"exp-name": "deployment-c", "conc": 2},
+ ]
+
+ result = filter_exp_names(entries, ["deployment-b", "deployment-a"])
+
+ assert result == entries[:2]
+
+ @pytest.mark.parametrize(
+ ("entries", "names", "message"),
+ (
+ ([{"exp-name": "deployment-a"}], ["missing"], "not found"),
+ (
+ [{"exp-name": "deployment-a"}, {"exp-name": "deployment-a"}],
+ ["deployment-a"],
+ "multiple rows",
+ ),
+ (
+ [{"exp-name": "deployment-a"}],
+ ["deployment-a", "deployment-a"],
+ "duplicate values",
+ ),
+ ),
+ )
+ def test_rejects_missing_ambiguous_or_duplicate_names(
+ self, entries, names, message
+ ):
+ with pytest.raises(ValueError, match=message):
+ filter_exp_names(entries, names)
+
+
# =============================================================================
# Test expand_config_keys
# =============================================================================
diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
index 436ea97b39..7014e54298 100644
--- a/utils/matrix_logic/validation.py
+++ b/utils/matrix_logic/validation.py
@@ -90,6 +90,8 @@ class Fields(Enum):
EVAL_ONLY = 'eval-only'
EVAL_CONC = 'eval-conc'
EVAL_ALL_CONCS = 'eval-all-concs'
+ EVAL_FRAMEWORK = 'eval-framework'
+ EVAL_SUITE = 'eval-suite'
"""
@@ -178,6 +180,10 @@ class SingleNodeMatrixEntry(BaseModel):
disagg: Literal[False]
run_eval: bool = Field(alias=Fields.RUN_EVAL.value)
eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False)
+ eval_framework: Optional[str] = Field(
+ default=None, alias=Fields.EVAL_FRAMEWORK.value
+ )
+ eval_suite: Optional[str] = Field(default=None, alias=Fields.EVAL_SUITE.value)
router: Optional[ComponentMetadata] = None
recipe_fingerprint: Optional[str] = Field(
default=None,
@@ -274,6 +280,10 @@ class MultiNodeMatrixEntry(BaseModel):
eval_all_concs: bool = Field(
default=False, alias=Fields.EVAL_ALL_CONCS.value
)
+ eval_framework: Optional[str] = Field(
+ default=None, alias=Fields.EVAL_FRAMEWORK.value
+ )
+ eval_suite: Optional[str] = Field(default=None, alias=Fields.EVAL_SUITE.value)
router: Optional[ComponentMetadata] = None
kv_p2p_transfer: Optional[str] = Field(
default=None, alias=Fields.KV_P2P_TRANSFER.value, min_length=1
@@ -329,10 +339,14 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
duration: int = Field(alias=Fields.DURATION.value)
exp_name: str = Field(alias=Fields.EXP_NAME.value)
scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value)
- # Agentic GSM8K eval rows carry run-eval/eval-only; benchmark rows
- # omit them, and exclude_none keeps them out of dumped benchmark output.
+ # Agentic eval rows carry selection and evaluator metadata. Benchmark-only
+ # rows omit them, and exclude_none keeps them out of dumped matrix output.
run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value)
eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value)
+ eval_framework: Optional[str] = Field(
+ default=None, alias=Fields.EVAL_FRAMEWORK.value
+ )
+ eval_suite: Optional[str] = Field(default=None, alias=Fields.EVAL_SUITE.value)
recipe_fingerprint: Optional[str] = Field(
default=None,
alias=Fields.RECIPE_FINGERPRINT.value,
@@ -378,13 +392,17 @@ class MultiNodeAgenticMatrixEntry(BaseModel):
exp_name: str = Field(alias=Fields.EXP_NAME.value)
disagg: bool
scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value)
- # Agentic eval rows (SWE-bench) carry run-eval/eval-only/eval-conc;
- # benchmark rows omit them, and exclude_none keeps them out of dumped
- # throughput output. SWE-bench doesn't support batched concurrencies
- # (unlike lm-eval), so there is no eval-all-concs field here.
+ # Agentic eval rows carry selection, concurrency, and evaluator metadata.
+ # Benchmark-only rows omit them, and exclude_none keeps them out of dumped
+ # matrix output. Multi-node agentic evals run one selected concurrency per
+ # job, so they do not use eval-all-concs.
run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value)
eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value)
eval_conc: Optional[int] = Field(default=None, alias=Fields.EVAL_CONC.value)
+ eval_framework: Optional[str] = Field(
+ default=None, alias=Fields.EVAL_FRAMEWORK.value
+ )
+ eval_suite: Optional[str] = Field(default=None, alias=Fields.EVAL_SUITE.value)
recipe_fingerprint: Optional[str] = Field(
default=None,
alias=Fields.RECIPE_FINGERPRINT.value,
diff --git a/utils/process_changelog.py b/utils/process_changelog.py
index 6149d8fba7..e246ce788f 100644
--- a/utils/process_changelog.py
+++ b/utils/process_changelog.py
@@ -12,7 +12,11 @@
import yaml
from constants import GENERATE_SWEEPS_PY_SCRIPT, MASTER_CONFIGS
-from matrix_logic.generate_sweep_configs import seq_len_to_str
+from matrix_logic.generate_sweep_configs import (
+ freeze_config_value,
+ seq_len_to_str,
+ trim_conc,
+)
from matrix_logic.validation import (
ChangelogEntry,
ChangelogMatrixEntry,
@@ -29,15 +33,6 @@ class GenerationInputs:
runner_config: str
-def _freeze_config_value(value):
- """Convert JSON-shaped config values into deterministic hashable values."""
- if isinstance(value, dict):
- return tuple(
- sorted((key, _freeze_config_value(item)) for key, item in value.items())
- )
- if isinstance(value, list):
- return tuple(_freeze_config_value(item) for item in value)
- return value
def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str:
@@ -67,54 +62,6 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str:
return "\n".join(added_lines)
-def trim_conc(entries: list[dict]) -> list[dict]:
- """Trim each parallelism config's concurrency sweep to its lowest point.
-
- Non-full-sweep PRs only need a single concurrency point per parallelism
- config to validate a change runs end-to-end, so the shared cluster stays
- clear. Push-to-main and ``full-sweep-enabled`` PRs skip this reduction.
-
- The retained value is the minimum configured concurrency — independent of
- the source ordering of ``conc-list`` / ``conc-start``.
-
- Input comes from ``json.loads(subprocess.stdout)`` so ``conc`` is always
- ``int`` (single-node) or ``list`` (multi-node). Other fields may contain
- nested dictionaries or lists, such as KV-offload backend metadata.
-
- - Single-node entries: group by every configuration field other than
- ``conc`` and the generated ``exp-name``, then keep only the entry with
- the lowest ``conc`` per group.
- - Multi-node entries: trim the ``conc`` list in place to ``[min(conc)]``.
- """
- groups: dict[tuple, list[int]] = {}
- out: list[dict] = []
-
- for entry in entries:
- if entry.get("prefill") is not None:
- conc = entry.get("conc")
- if isinstance(conc, list) and len(conc) > 1:
- entry = {**entry, "conc": [min(conc)]}
- out.append(entry)
- continue
-
- key = tuple(
- sorted(
- (k, _freeze_config_value(v))
- for k, v in entry.items()
- if k not in {"conc", "exp-name"}
- )
- )
- groups.setdefault(key, []).append(len(out))
- out.append(entry)
-
- drop: set[int] = set()
- for idxs in groups.values():
- if len(idxs) > 1:
- keep = min(idxs, key=lambda i: out[i]["conc"])
- drop.update(i for i in idxs if i != keep)
- return [e for i, e in enumerate(out) if i not in drop]
-
-
def filter_eval_rows_by_prefill_ep(
eval_rows: list[dict], min_prefill_ep: int | None
) -> list[dict]:
@@ -210,7 +157,7 @@ def _matrix_curve_key(entry: dict) -> tuple:
"""Identify one curve while deliberately excluding point-level fields."""
return tuple(
sorted(
- (key, _freeze_config_value(value))
+ (key, freeze_config_value(value))
for key, value in entry.items()
if key not in {"conc", "exp-name", "recipe-fingerprint"}
)
diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py
index 019bbdf123..6faad14111 100644
--- a/utils/test_collect_eval_results.py
+++ b/utils/test_collect_eval_results.py
@@ -1,9 +1,24 @@
"""Tests for eval result aggregation."""
import json
+import os
from pathlib import Path
-from collect_eval_results import build_row, collect_eval_rows
+from collect_eval_results import (
+ EVAL_RESULT_FORMAT,
+ build_row,
+ collect_eval_rows,
+)
+from evals.kimi_vendor_eval import RESULT_FORMAT as KIMI_VENDOR_RESULT_FORMAT
+from evals.minimax_provider_eval import RESULT_FORMAT as MINIMAX_RESULT_FORMAT
+
+
+def test_kimi_vendor_result_format_matches_collector_contract() -> None:
+ assert KIMI_VENDOR_RESULT_FORMAT == EVAL_RESULT_FORMAT
+
+
+def test_minimax_result_format_matches_collector_contract() -> None:
+ assert MINIMAX_RESULT_FORMAT == EVAL_RESULT_FORMAT
def test_build_row_preserves_sequence_lengths() -> None:
@@ -21,25 +36,39 @@ def test_build_row_preserves_sequence_lengths() -> None:
assert row["isl"] == 1024
assert row["osl"] == 1024
+ assert "eval_suite" not in row
+
+
+def test_build_row_preserves_explicit_eval_suite() -> None:
+ row = build_row(
+ {"eval_suite": "kimi_tool_call_schema"},
+ {"task": "kimi_tool_call_schema"},
+ )
+
+ assert row["eval_suite"] == "kimi_tool_call_schema"
-def _write_lm_eval_result(path: Path, score: float) -> None:
+def _write_lm_eval_result(
+ path: Path,
+ score: float,
+ task: str = "gsm8k",
+) -> None:
path.write_text(json.dumps({
"lm_eval_version": "0.4.0",
"model_name": "test-model",
"results": {
- "gsm8k": {
+ task: {
"exact_match,strict-match": score,
"exact_match_stderr,strict-match": 0.01,
},
},
"configs": {
- "gsm8k": {
+ task: {
"metric_list": [{"metric": "exact_match"}],
"filter_list": [{"name": "strict-match"}],
},
},
- "n-samples": {"gsm8k": {"effective": 10}},
+ "n-samples": {task: {"effective": 10}},
}))
@@ -67,6 +96,7 @@ def test_collect_eval_rows_expands_batched_concurrencies(
"completed_eval_concs": [4, 16],
"failed_eval_concs": [],
"conc": 4,
+ "eval_suite": "gsm8k",
}))
_write_lm_eval_result(
artifact_dir / "results_test_conc4.json",
@@ -81,6 +111,7 @@ def test_collect_eval_rows_expands_batched_concurrencies(
assert [row["conc"] for row in rows] == [4, 16]
assert [row["score"] for row in rows] == [0.90, 0.91]
+ assert {row["eval_suite"] for row in rows} == {"gsm8k"}
def test_collect_eval_rows_ignores_failed_batch_points(
@@ -107,3 +138,314 @@ def test_collect_eval_rows_ignores_failed_batch_points(
rows = collect_eval_rows(tmp_path)
assert [row["conc"] for row in rows] == [4]
+
+
+
+def test_collect_eval_rows_accepts_minimax_compatibility_result(
+ tmp_path: Path,
+) -> None:
+ artifact_dir = tmp_path / "eval_minimax"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({"eval_suite": "minimax_m3_smoke"})
+ )
+ result_path = artifact_dir / "results_minimax_vendor.json"
+ _write_lm_eval_result(result_path, 1.0, task="minimax_m3_smoke")
+ result = json.loads(result_path.read_text())
+ result.pop("lm_eval_version")
+ result["result_format"] = EVAL_RESULT_FORMAT
+ result["eval_adapter"] = "minimax-provider-verifier"
+ result_path.write_text(json.dumps(result))
+
+ rows = collect_eval_rows(tmp_path)
+
+ assert len(rows) == 1
+ assert rows[0]["task"] == "minimax_m3_smoke"
+ assert rows[0]["score"] == 1.0
+ assert rows[0]["eval_suite"] == "minimax_m3_smoke"
+
+
+def test_collect_eval_rows_retains_integration_and_sample_failures(
+ tmp_path: Path,
+) -> None:
+ for name, invalid in (
+ ("integration", "integration"),
+ ("zero", 0),
+ ("nonnumeric", "unknown"),
+ ("nonfinite", float("nan")),
+ ("malformed", []),
+ ):
+ artifact_dir = tmp_path / f"eval_{name}"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(json.dumps({
+ "eval_suite": "gsm8k",
+ }))
+ result_path = artifact_dir / f"results_{name}.json"
+ _write_lm_eval_result(result_path, 0.0)
+ result = json.loads(result_path.read_text())
+ if invalid == "integration":
+ result["integration_error"] = {
+ "type": "RuntimeError",
+ "message": "vendor verifier checkout failed",
+ }
+ else:
+ result["n-samples"]["gsm8k"]["effective"] = invalid
+ result_path.write_text(json.dumps(result))
+
+ rows = collect_eval_rows(tmp_path)
+ assert len(rows) == 5
+ assert all(row["infrastructure_success"] is False for row in rows)
+ assert all(row["score"] is None for row in rows)
+ assert all(row["n_eff"] == 0 for row in rows)
+ assert {
+ row["integration_error"]["type"]
+ for row in rows
+ } == {"RuntimeError", "InvalidEffectiveSampleCount"}
+
+
+def test_collect_eval_rows_handles_malformed_failure_metadata(
+ tmp_path: Path,
+) -> None:
+ for index, (configs, sample_counts) in enumerate(
+ (
+ (None, None),
+ ({"gsm8k": None}, {"gsm8k": None}),
+ )
+ ):
+ artifact_dir = tmp_path / f"eval_malformed_metadata_{index}"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({"eval_suite": "gsm8k"})
+ )
+ result_path = artifact_dir / f"results_{index}.json"
+ _write_lm_eval_result(result_path, 0.0)
+ result = json.loads(result_path.read_text())
+ result["configs"] = configs
+ result["n-samples"] = sample_counts
+ result["integration_error"] = {
+ "type": "RuntimeError",
+ "message": "setup failed",
+ }
+ result_path.write_text(json.dumps(result))
+
+ rows = collect_eval_rows(tmp_path)
+ assert len(rows) == 2
+ assert all(row["score"] is None for row in rows)
+ assert all(row["n_eff"] == 0 for row in rows)
+ assert all(row["infrastructure_success"] is False for row in rows)
+
+
+def test_collect_eval_rows_accepts_legacy_missing_effective_count(
+ tmp_path: Path,
+) -> None:
+ artifact_dir = tmp_path / "eval_legacy"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(json.dumps({
+ "eval_suite": "gsm8k",
+ }))
+ result_path = artifact_dir / "results_legacy.json"
+ _write_lm_eval_result(result_path, 0.9)
+ result = json.loads(result_path.read_text())
+ result.pop("n-samples")
+ result_path.write_text(json.dumps(result))
+
+ rows = collect_eval_rows(tmp_path)
+
+ assert len(rows) == 1
+ assert rows[0]["score"] == 0.9
+ assert rows[0]["n_eff"] is None
+
+
+def test_collect_eval_rows_does_not_resurrect_stale_valid_result(
+ tmp_path: Path,
+) -> None:
+ artifact_dir = tmp_path / "eval_retry"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(json.dumps({
+ "eval_suite": "kimi_tool_call_schema",
+ }))
+ stale_path = (
+ artifact_dir / "results_kimi_vendor_2026-08-12T01-00-00.000000.json"
+ )
+ _write_lm_eval_result(stale_path, 1.0, task="kimi_tool_call_schema")
+ current_path = (
+ artifact_dir / "results_kimi_vendor_2026-08-12T02-00-00.000000.json"
+ )
+ _write_lm_eval_result(current_path, 0.0, task="kimi_tool_call_schema")
+ result = json.loads(current_path.read_text())
+ result["integration_error"] = {
+ "type": "RuntimeError",
+ "message": "vendor verifier checkout failed",
+ }
+ current_path.write_text(json.dumps(result))
+ current_path.touch()
+ stale_path.touch()
+
+ rows = collect_eval_rows(tmp_path)
+ assert len(rows) == 1
+ assert rows[0]["infrastructure_success"] is False
+ assert rows[0]["integration_error"]["message"] == (
+ "vendor verifier checkout failed"
+ )
+
+
+def test_collect_eval_rows_uses_mtime_for_newer_legacy_name(
+ tmp_path: Path,
+) -> None:
+ artifact_dir = tmp_path / "eval_retry"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({"eval_suite": "kimi_tool_call_schema"})
+ )
+ stale_path = (
+ artifact_dir / "results_kimi_vendor_2026-08-12T01-00-00.000000.json"
+ )
+ _write_lm_eval_result(stale_path, 1.0, task="kimi_tool_call_schema")
+ current_path = artifact_dir / "results.json"
+ _write_lm_eval_result(current_path, 0.0, task="kimi_tool_call_schema")
+ current = json.loads(current_path.read_text())
+ current["integration_error"] = {
+ "type": "RuntimeError",
+ "message": "latest attempt failed",
+ }
+ current_path.write_text(json.dumps(current))
+ os.utime(current_path, (2_000_000_000, 2_000_000_000))
+
+ rows = collect_eval_rows(tmp_path)
+ assert len(rows) == 1
+ assert rows[0]["infrastructure_success"] is False
+ assert rows[0]["integration_error"]["message"] == "latest attempt failed"
+
+
+def test_collect_eval_rows_retains_missing_or_out_of_range_scores(
+ tmp_path: Path,
+) -> None:
+ for index, score in enumerate(
+ (None, True, float("nan"), float("inf"), -0.1, 1.1)
+ ):
+ artifact_dir = tmp_path / f"eval_invalid_{index}"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({"eval_suite": "kimi_tool_call_schema"})
+ )
+ _write_lm_eval_result(
+ artifact_dir / f"results_{index}.json",
+ score,
+ task="kimi_tool_call_schema",
+ )
+
+ rows = collect_eval_rows(tmp_path)
+ assert len(rows) == 6
+ assert all(row["score"] is None for row in rows)
+ assert all(row["infrastructure_success"] is False for row in rows)
+ assert {
+ row["integration_error"]["type"] for row in rows
+ } == {"InvalidPrimaryScore"}
+
+
+def test_collect_eval_rows_falls_back_for_invalid_filename_timestamp(
+ tmp_path: Path,
+) -> None:
+ artifact_dir = tmp_path / "eval_invalid_timestamp"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({"eval_suite": "kimi_tool_call_schema"})
+ )
+ _write_lm_eval_result(
+ artifact_dir / "results_2026-99-99T99-99-99.json",
+ 1.0,
+ task="kimi_tool_call_schema",
+ )
+
+ rows = collect_eval_rows(tmp_path)
+
+ assert len(rows) == 1
+ assert rows[0]["score"] == 1.0
+
+
+def test_collect_eval_rows_uses_extract_filter_as_primary_score(
+ tmp_path: Path,
+) -> None:
+ artifact_dir = tmp_path / "eval_gpqa"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({"eval_suite": "gpqa_diamond_cot_n_shot"})
+ )
+ (artifact_dir / "results_gpqa.json").write_text(json.dumps({
+ "lm_eval_version": "0.4.0",
+ "results": {
+ "gpqa_diamond_cot_n_shot": {
+ "exact_match,extract_abcd": 0.75,
+ "exact_match_stderr,extract_abcd": 0.02,
+ },
+ },
+ "configs": {
+ "gpqa_diamond_cot_n_shot": {
+ "metric_list": [{"metric": "exact_match"}],
+ "filter_list": [{"name": "extract_abcd"}],
+ },
+ },
+ "n-samples": {
+ "gpqa_diamond_cot_n_shot": {"effective": 8},
+ },
+ }))
+
+ rows = collect_eval_rows(tmp_path)
+
+ assert len(rows) == 1
+ assert rows[0]["score"] == 0.75
+ assert rows[0]["score_name"] == "em_flexible"
+
+
+def test_collect_eval_rows_accepts_bfcl_compatibility_and_ignores_native_report(
+ tmp_path: Path,
+) -> None:
+ artifact_dir = tmp_path / "eval_bfcl"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({"eval_suite": "bfcl_smoke"})
+ )
+ (artifact_dir / "bfcl_report.json").write_text(
+ json.dumps(
+ {
+ "results": {"native_only": {"acc,none": 1.0}},
+ }
+ )
+ )
+ compatibility_path = artifact_dir / "results_bfcl.json"
+ tasks = {
+ "bfcl_smoke": 0.75,
+ "bfcl_simple_python": 1.0,
+ }
+ compatibility_path.write_text(
+ json.dumps(
+ {
+ "result_format": EVAL_RESULT_FORMAT,
+ "model_name": "test-model",
+ "results": {
+ task: {
+ "acc,none": score,
+ "acc_stderr,none": 0.0,
+ }
+ for task, score in tasks.items()
+ },
+ "configs": {
+ task: {
+ "metric_list": [{"metric": "acc"}],
+ "filter_list": [{"name": "none"}],
+ }
+ for task in tasks
+ },
+ "n-samples": {
+ "bfcl_smoke": {"effective": 4},
+ "bfcl_simple_python": {"effective": 1},
+ },
+ }
+ )
+ )
+
+ rows = collect_eval_rows(tmp_path)
+
+ assert {row["task"]: row["score"] for row in rows} == tasks
+ assert {row["score_name"] for row in rows} == {"accuracy"}
+ assert {row["source"] for row in rows} == {str(compatibility_path)}
diff --git a/utils/test_validate_reusable_sweep_artifacts.py b/utils/test_validate_reusable_sweep_artifacts.py
index 8b9867fa3e..049ed83ba3 100644
--- a/utils/test_validate_reusable_sweep_artifacts.py
+++ b/utils/test_validate_reusable_sweep_artifacts.py
@@ -1,14 +1,16 @@
from __future__ import annotations
import json
+import os
import sys
from pathlib import Path
from validate_reusable_sweep_artifacts import (
agentic_key,
benchmark_key,
- eval_key,
dedupe_reran_evals,
+ eval_key,
+ eval_result_key,
main,
validate_agentic_artifacts,
validate_eval_artifacts,
@@ -32,8 +34,9 @@ def single_eval_result(
runner: str = "h100-dgxc-slurm",
isl: int = 8192,
osl: int = 1024,
+ eval_suite: str | None = None,
) -> dict:
- return {
+ row = {
"is_multinode": False,
"hw": runner.upper(),
"model_prefix": "gptoss",
@@ -51,6 +54,9 @@ def single_eval_result(
"conc": conc,
"task": "gsm8k",
}
+ if eval_suite is not None:
+ row["eval_suite"] = eval_suite
+ return row
def single_eval_meta(
@@ -58,12 +64,37 @@ def single_eval_meta(
runner: str = "h100-dgxc-slurm",
isl: int = 8192,
osl: int = 1024,
+ eval_suite: str | None = None,
) -> dict:
- row = single_eval_result(conc, runner, isl, osl)
+ row = single_eval_result(conc, runner, isl, osl, eval_suite)
row["infmax_model_prefix"] = row.pop("model_prefix")
return row
+def raw_eval_result(
+ score: float = 0.9,
+ *,
+ effective: object = 10,
+ task: str = "gsm8k",
+) -> dict:
+ return {
+ "lm_eval_version": "0.4.0",
+ "results": {
+ task: {
+ "exact_match,strict-match": score,
+ "exact_match_stderr,strict-match": 0.01,
+ },
+ },
+ "configs": {
+ task: {
+ "metric_list": [{"metric": "exact_match"}],
+ "filter_list": [{"name": "strict-match"}],
+ },
+ },
+ "n-samples": {task: {"effective": effective}},
+ }
+
+
def write_raw_eval_artifact(
root: Path,
conc: int,
@@ -72,11 +103,23 @@ def write_raw_eval_artifact(
physical_runner: str = "h100-dgxc-slurm_00",
isl: int = 8192,
osl: int = 1024,
+ eval_suite: str | None = None,
) -> None:
artifact_dir = root / f"eval_result_conc{conc}_{physical_runner}"
artifact_dir.mkdir()
(artifact_dir / "meta_env.json").write_text(
- json.dumps(single_eval_meta(conc, logical_runner, isl, osl))
+ json.dumps(
+ single_eval_meta(
+ conc,
+ logical_runner,
+ isl,
+ osl,
+ eval_suite,
+ )
+ )
+ )
+ (artifact_dir / "results_test.json").write_text(
+ json.dumps(raw_eval_result())
)
@@ -112,16 +155,19 @@ def write_raw_batched_eval_artifact(
) -> None:
artifact_dir = root / "eval_gptoss_8k1k_batch"
artifact_dir.mkdir()
+ completed = concs if completed_concs is None else completed_concs
meta = multinode_eval_result(concs[0])
meta["infmax_model_prefix"] = meta.pop("model_prefix")
meta["eval_concs"] = concs
- meta["completed_eval_concs"] = (
- concs if completed_concs is None else completed_concs
- )
+ meta["completed_eval_concs"] = completed
meta["failed_eval_concs"] = (
[] if failed_concs is None else failed_concs
)
(artifact_dir / "meta_env.json").write_text(json.dumps(meta))
+ for conc in completed:
+ (artifact_dir / f"results_test_conc{conc}.json").write_text(
+ json.dumps(raw_eval_result())
+ )
def fixed_result(conc: int) -> dict:
@@ -330,7 +376,7 @@ def test_eval_validation_requires_raw_result_dirs_not_eval_debug_dirs(
assert any("unexpected" in error for error in errors)
-def test_eval_validation_accepts_matching_raw_and_aggregate(
+def test_eval_validation_accepts_matching_legacy_artifacts_without_suite(
tmp_path: Path,
) -> None:
write_eval_aggregate(
@@ -347,6 +393,38 @@ def test_eval_validation_accepts_matching_raw_and_aggregate(
assert validate_eval_artifacts(tmp_path) == []
+def test_eval_validation_separates_explicit_suite_identities(
+ tmp_path: Path,
+) -> None:
+ gsm8k = single_eval_result(32, eval_suite="gsm8k")
+ tool_use = single_eval_result(
+ 32,
+ eval_suite="kimi_tool_call_schema",
+ )
+ write_eval_aggregate(tmp_path, [gsm8k, tool_use])
+ write_raw_eval_artifact(
+ tmp_path,
+ 32,
+ eval_suite="gsm8k",
+ )
+ write_raw_eval_artifact(
+ tmp_path,
+ 32,
+ physical_runner="h100-dgxc-slurm_01",
+ eval_suite="kimi_tool_call_schema",
+ )
+
+ assert eval_key(gsm8k) != eval_key(tool_use)
+ assert validate_eval_artifacts(tmp_path) == []
+
+
+def test_eval_result_key_includes_task_identity() -> None:
+ gsm8k = single_eval_result(32, eval_suite="tool_use")
+ bfcl = {**gsm8k, "task": "bfcl_smoke"}
+
+ assert eval_result_key(gsm8k) != eval_result_key(bfcl)
+
+
def test_eval_validation_distinguishes_sequence_lengths(tmp_path: Path) -> None:
write_eval_aggregate(
tmp_path,
@@ -421,7 +499,24 @@ def test_eval_validation_expands_one_batched_multinode_artifact(
assert validate_eval_artifacts(tmp_path) == []
-def test_eval_validation_accepts_completed_points_from_failed_batch(
+def test_eval_validation_accepts_legacy_batch_without_failed_list(
+ tmp_path: Path,
+) -> None:
+ concs = [4, 16]
+ write_eval_aggregate(
+ tmp_path,
+ [multinode_eval_result(conc) for conc in concs],
+ )
+ write_raw_batched_eval_artifact(tmp_path, concs)
+ meta_path = tmp_path / "eval_gptoss_8k1k_batch" / "meta_env.json"
+ meta = json.loads(meta_path.read_text())
+ meta.pop("failed_eval_concs")
+ meta_path.write_text(json.dumps(meta))
+
+ assert validate_eval_artifacts(tmp_path) == []
+
+
+def test_eval_validation_rejects_failed_batch(
tmp_path: Path,
) -> None:
requested_concs = [4, 16, 64]
@@ -437,7 +532,9 @@ def test_eval_validation_accepts_completed_points_from_failed_batch(
failed_concs=[16],
)
- assert validate_eval_artifacts(tmp_path) == []
+ errors = validate_eval_artifacts(tmp_path)
+
+ assert any("reports failed eval concurrencies" in error for error in errors)
def test_eval_aggregate_validation_is_exact(tmp_path: Path) -> None:
@@ -472,6 +569,195 @@ def test_eval_aggregate_validation_rejects_duplicate_identity(
)
+def test_eval_aggregate_validation_rejects_non_list_file(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32)
+ eval_dir = tmp_path / "eval_results_all"
+ eval_dir.mkdir()
+ (eval_dir / "agg_eval_all.json").write_text(
+ json.dumps(single_eval_result(32))
+ )
+
+ errors = validate_eval_artifacts(tmp_path)
+
+ assert any("is not a list" in error for error in errors)
+
+
+def test_eval_aggregate_validation_rejects_non_object_row(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32)
+ eval_dir = tmp_path / "eval_results_all"
+ eval_dir.mkdir()
+ (eval_dir / "agg_eval_all.json").write_text(
+ json.dumps([single_eval_result(32), "not-a-row"])
+ )
+
+ errors = validate_eval_artifacts(tmp_path)
+
+ assert any("row 1 is not an object" in error for error in errors)
+
+
+def test_eval_validation_rejects_scores_outside_unit_interval(
+ tmp_path: Path,
+) -> None:
+ for index, score in enumerate((-0.01, 1.01)):
+ root = tmp_path / str(index)
+ root.mkdir()
+ write_raw_eval_artifact(root, 32)
+ result_path = next(
+ (root / "eval_result_conc32_h100-dgxc-slurm_00").glob(
+ "results*.json"
+ )
+ )
+ result_path.write_text(json.dumps(raw_eval_result(score)))
+ write_eval_aggregate(root, [single_eval_result(32)])
+
+ errors = validate_eval_artifacts(root)
+
+ assert any("invalid score" in error for error in errors)
+
+
+def test_eval_validation_rejects_directory_with_only_markerless_results(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32)
+ result_path = next(
+ (
+ tmp_path / "eval_result_conc32_h100-dgxc-slurm_00"
+ ).glob("results*.json")
+ )
+ data = json.loads(result_path.read_text())
+ data.pop("lm_eval_version")
+ result_path.write_text(json.dumps(data))
+ write_eval_aggregate(tmp_path, [single_eval_result(32)])
+
+ errors = validate_eval_artifacts(tmp_path)
+
+ assert any("has no recognized eval result" in error for error in errors)
+
+
+def test_eval_validation_accepts_neutral_result_format_marker(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32)
+ result_path = next(
+ (
+ tmp_path / "eval_result_conc32_h100-dgxc-slurm_00"
+ ).glob("results*.json")
+ )
+ data = json.loads(result_path.read_text())
+ data.pop("lm_eval_version")
+ data["result_format"] = "inferencex-eval-v1"
+ result_path.write_text(json.dumps(data))
+ write_eval_aggregate(tmp_path, [single_eval_result(32)])
+
+ assert validate_eval_artifacts(tmp_path) == []
+
+
+def test_eval_validation_accepts_neutral_filter_primary_score(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32, eval_suite="bfcl_smoke")
+ result_path = next(tmp_path.glob("eval_*/results*.json"))
+ data = json.loads(result_path.read_text())
+ data["results"]["gsm8k"] = {
+ "acc,none": 1.0,
+ "acc_stderr,none": 0.0,
+ }
+ data["configs"]["gsm8k"] = {
+ "metric_list": [{"metric": "acc"}],
+ "filter_list": [{"name": "none"}],
+ }
+ result_path.write_text(json.dumps(data))
+ write_eval_aggregate(
+ tmp_path,
+ [single_eval_result(32, eval_suite="bfcl_smoke")],
+ )
+
+ assert validate_eval_artifacts(tmp_path) == []
+
+
+def test_eval_validation_accepts_multiple_tasks_from_one_artifact(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32, eval_suite="tool_use")
+ result_path = next(tmp_path.glob("eval_*/results*.json"))
+ data = json.loads(result_path.read_text())
+ data["results"]["bfcl_smoke"] = {
+ "exact_match,strict-match": 1.0,
+ "exact_match_stderr,strict-match": 0.0,
+ }
+ data["configs"]["bfcl_smoke"] = data["configs"]["gsm8k"]
+ data["n-samples"]["bfcl_smoke"] = {"effective": 1}
+ result_path.write_text(json.dumps(data))
+ base_row = single_eval_result(32, eval_suite="tool_use")
+ write_eval_aggregate(
+ tmp_path,
+ [base_row, {**base_row, "task": "bfcl_smoke"}],
+ )
+
+ assert validate_eval_artifacts(tmp_path) == []
+
+
+def test_eval_validation_rejects_invalid_legacy_concurrency(
+ tmp_path: Path,
+) -> None:
+ for index, conc in enumerate((0, -1, True, "4")):
+ root = tmp_path / str(index)
+ root.mkdir()
+ artifact_dir = root / "eval_invalid_legacy"
+ artifact_dir.mkdir()
+ (artifact_dir / "meta_env.json").write_text(
+ json.dumps({**single_eval_meta(4), "conc": conc})
+ )
+ (artifact_dir / "results_test.json").write_text(
+ json.dumps(raw_eval_result())
+ )
+
+ errors = validate_eval_artifacts(root)
+
+ assert any("invalid legacy concurrency" in error for error in errors)
+
+
+def test_eval_validation_rejects_malformed_batch_metadata(
+ tmp_path: Path,
+) -> None:
+ cases = (
+ ({"eval_concs": [4, True]}, "invalid eval_concs"),
+ ({"eval_concs": [4, 4]}, "duplicate eval_concs"),
+ ({"completed_eval_concs": [4, 4]}, "duplicate completed_eval_concs"),
+ ({"failed_eval_concs": [16, 16]}, "duplicate failed_eval_concs"),
+ ({"completed_eval_concs": [4, 16], "failed_eval_concs": [16]}, "overlapping"),
+ ({"completed_eval_concs": [4, 32]}, "unexpected"),
+ ({"completed_eval_concs": [4], "failed_eval_concs": [32]}, "failed unexpected"),
+ ({"completed_eval_concs": "4"}, "invalid batched concurrency metadata"),
+ ({"completed_eval_concs": [4], "failed_eval_concs": [16]}, "reports failed"),
+ ({"failed_eval_concs": None}, "invalid batched concurrency metadata"),
+ ({"eval_concs": [], "completed_eval_concs": []}, "empty eval_concs"),
+ )
+ for index, (overrides, expected) in enumerate(cases):
+ root = tmp_path / str(index)
+ root.mkdir()
+ artifact_dir = root / "eval_invalid_batch"
+ artifact_dir.mkdir()
+ meta = _dd_meta(0)
+ meta.update(
+ {
+ "eval_concs": [4, 16],
+ "completed_eval_concs": [4, 16],
+ "failed_eval_concs": [],
+ }
+ )
+ meta.update(overrides)
+ (artifact_dir / "meta_env.json").write_text(json.dumps(meta))
+
+ errors = validate_eval_artifacts(root)
+
+ assert any(expected in error for error in errors), errors
+
+
def test_fixed_sequence_validation_accepts_unique_source_rows(tmp_path: Path) -> None:
results = tmp_path / "results_bmk"
results.mkdir()
@@ -640,7 +926,9 @@ def _dd_write_legacy_raw(
artifact_dir.mkdir()
(artifact_dir / "meta_env.json").write_text(json.dumps(_dd_meta(conc)))
if timestamp is not None:
- (artifact_dir / f"results_{timestamp}.json").write_text("{}")
+ (artifact_dir / f"results_{timestamp}.json").write_text(
+ json.dumps(raw_eval_result())
+ )
def test_dedupe_keeps_latest_legacy_rerun(tmp_path: Path) -> None:
@@ -676,9 +964,9 @@ def test_dedupe_keeps_latest_legacy_rerun(tmp_path: Path) -> None:
assert any("kept 1 of 3" in message for message in messages)
-def test_dedupe_leaves_ambiguous_duplicates_for_validation(tmp_path: Path) -> None:
- # Duplicate raw identities with no result timestamps cannot be ordered, so
- # dedupe must leave them and validation must still reject them.
+def test_dedupe_leaves_ambiguous_artifacts_for_validation(tmp_path: Path) -> None:
+ # Result-less raw artifacts cannot be ordered or reused. Dedupe leaves them
+ # for validation to reject as missing recognized results.
for name in ("eval_minimaxm3_conc4096_b300-nv_01", "eval_minimaxm3_conc4096_b300-nv_02"):
_dd_write_legacy_raw(tmp_path, name, 4096, None)
_dd_write_aggregate(
@@ -687,7 +975,10 @@ def test_dedupe_leaves_ambiguous_duplicates_for_validation(tmp_path: Path) -> No
)
assert dedupe_reran_evals(tmp_path) == []
- assert any("duplicate" in e for e in validate_eval_artifacts(tmp_path))
+ assert any(
+ "no recognized eval result" in error
+ for error in validate_eval_artifacts(tmp_path)
+ )
def test_dedupe_is_noop_for_clean_artifacts(tmp_path: Path) -> None:
@@ -718,9 +1009,12 @@ def test_dedupe_prunes_superseded_batched_conc(tmp_path: Path) -> None:
meta = _dd_meta(0)
meta["eval_concs"] = concs
meta["completed_eval_concs"] = list(concs)
+ meta["failed_eval_concs"] = []
(artifact_dir / "meta_env.json").write_text(json.dumps(meta))
for conc in concs:
- (artifact_dir / f"results_{stamp}_conc{conc}.json").write_text("{}")
+ (artifact_dir / f"results_{stamp}_conc{conc}.json").write_text(
+ json.dumps(raw_eval_result())
+ )
_dd_write_aggregate(
tmp_path,
[
@@ -735,7 +1029,369 @@ def test_dedupe_prunes_superseded_batched_conc(tmp_path: Path) -> None:
assert validate_eval_artifacts(tmp_path) == []
assert json.loads((older / "meta_env.json").read_text())["completed_eval_concs"] == [16]
+ assert json.loads((older / "meta_env.json").read_text())["eval_concs"] == [16]
assert not (older / "results_2026-06-26T10-00-00.000000_conc32.json").exists()
assert (older / "results_2026-06-26T10-00-00.000000_conc16.json").exists()
rows = json.loads((tmp_path / "eval_results_all" / "agg_eval_all.json").read_text())
assert [r["em_strict"] for r in rows if r["conc"] == 32] == [0.90]
+
+
+def test_dedupe_orders_timestamped_and_legacy_results_coherently(
+ tmp_path: Path,
+) -> None:
+ timestamped = "eval_minimaxm3_conc4096_b300-nv_timestamped"
+ legacy = "eval_minimaxm3_conc4096_b300-nv_legacy"
+ _dd_write_legacy_raw(
+ tmp_path,
+ timestamped,
+ 4096,
+ "2026-06-27T04-28-31.838775",
+ )
+ _dd_write_legacy_raw(tmp_path, legacy, 4096, "retry")
+ legacy_result = next((tmp_path / legacy).glob("results*.json"))
+ future_ns = 2_000_000_000_000_000_000
+ os.utime(legacy_result, ns=(future_ns, future_ns))
+ _dd_write_aggregate(
+ tmp_path,
+ [
+ _dd_agg_row(
+ 4096,
+ f"eval_results/{timestamped}/results_2026-06-27T04-28-31.838775.json",
+ 0.5,
+ ),
+ _dd_agg_row(
+ 4096,
+ f"eval_results/{legacy}/results_retry.json",
+ 0.9,
+ ),
+ ],
+ )
+
+ dedupe_reran_evals(tmp_path)
+
+ assert validate_eval_artifacts(tmp_path) == []
+ assert (tmp_path / legacy).is_dir()
+ assert not (tmp_path / timestamped).exists()
+ rows = json.loads(
+ (tmp_path / "eval_results_all" / "agg_eval_all.json").read_text()
+ )
+ assert [row["em_strict"] for row in rows] == [0.9]
+
+
+def test_dedupe_collapses_identity_across_aggregate_files(
+ tmp_path: Path,
+) -> None:
+ old = "eval_minimaxm3_conc4096_b300-nv_old"
+ new = "eval_minimaxm3_conc4096_b300-nv_new"
+ _dd_write_legacy_raw(
+ tmp_path,
+ old,
+ 4096,
+ "2026-06-26T01-00-00.000000",
+ )
+ _dd_write_legacy_raw(
+ tmp_path,
+ new,
+ 4096,
+ "2026-06-27T01-00-00.000000",
+ )
+ aggregate_dir = tmp_path / "eval_results_all"
+ aggregate_dir.mkdir()
+ old_path = aggregate_dir / "old.json"
+ new_path = aggregate_dir / "new.json"
+ old_path.write_text(
+ json.dumps(
+ [
+ _dd_agg_row(
+ 4096,
+ f"eval_results/{old}/results_2026-06-26T01-00-00.000000.json",
+ 0.4,
+ )
+ ]
+ )
+ )
+ new_path.write_text(
+ json.dumps(
+ [
+ _dd_agg_row(
+ 4096,
+ f"eval_results/{new}/results_2026-06-27T01-00-00.000000.json",
+ 0.9,
+ )
+ ]
+ )
+ )
+
+ messages = dedupe_reran_evals(tmp_path)
+
+ assert json.loads(old_path.read_text()) == []
+ assert [
+ row["em_strict"]
+ for row in json.loads(new_path.read_text())
+ ] == [0.9]
+ assert not (tmp_path / old).exists()
+ assert (tmp_path / new).is_dir()
+ assert validate_eval_artifacts(tmp_path) == []
+ assert any("old.json: kept 0 of 1" in message for message in messages)
+
+
+def test_dedupe_accepts_zero_score_as_structurally_valid(
+ tmp_path: Path,
+) -> None:
+ old = "eval_minimaxm3_conc4096_b300-nv_old"
+ new = "eval_minimaxm3_conc4096_b300-nv_new"
+ _dd_write_legacy_raw(tmp_path, old, 4096, "2026-06-26T01-00-00.000000")
+ _dd_write_legacy_raw(tmp_path, new, 4096, "2026-06-27T01-00-00.000000")
+ new_result = next((tmp_path / new).glob("results*.json"))
+ new_result.write_text(json.dumps(raw_eval_result(0.0)))
+ _dd_write_aggregate(
+ tmp_path,
+ [
+ _dd_agg_row(4096, f"eval_results/{old}/results_2026-06-26T01-00-00.000000.json", 0.8),
+ _dd_agg_row(4096, f"eval_results/{new}/results_2026-06-27T01-00-00.000000.json", 0.0),
+ ],
+ )
+
+ dedupe_reran_evals(tmp_path)
+
+ assert validate_eval_artifacts(tmp_path) == []
+ assert (tmp_path / new).is_dir()
+ assert not (tmp_path / old).exists()
+
+
+def test_dedupe_does_not_resurrect_stale_result_after_integration_error(
+ tmp_path: Path,
+) -> None:
+ name = "eval_minimaxm3_conc4096_b300-nv_retry"
+ _dd_write_legacy_raw(
+ tmp_path,
+ name,
+ 4096,
+ "2026-06-26T01-00-00.000000",
+ )
+ new_result = (
+ tmp_path
+ / name
+ / "results_2026-06-27T01-00-00.000000.json"
+ )
+ failed = raw_eval_result(0.0)
+ failed["integration_error"] = {
+ "type": "RuntimeError",
+ "message": "verifier checkout failed",
+ }
+ new_result.write_text(json.dumps(failed))
+ _dd_write_aggregate(
+ tmp_path,
+ [_dd_agg_row(4096, f"eval_results/{name}/results_old.json", 0.8)],
+ )
+
+ assert dedupe_reran_evals(tmp_path) == []
+ errors = validate_eval_artifacts(tmp_path)
+
+ assert any("integration error" in error for error in errors)
+ assert (tmp_path / name).is_dir()
+
+
+def test_newer_foreign_result_does_not_suppress_recognized_result(
+ tmp_path: Path,
+) -> None:
+ markerless = raw_eval_result()
+ markerless.pop("lm_eval_version")
+ for index, contents in enumerate(("{not-json", json.dumps(markerless))):
+ root = tmp_path / str(index)
+ root.mkdir()
+ name = "eval_minimaxm3_conc4096_b300-nv_retry"
+ _dd_write_legacy_raw(
+ root,
+ name,
+ 4096,
+ "2026-06-26T01-00-00.000000",
+ )
+ (
+ root
+ / name
+ / "results_2026-06-27T01-00-00.000000.json"
+ ).write_text(contents)
+ _dd_write_aggregate(
+ root,
+ [_dd_agg_row(4096, f"eval_results/{name}/results_old.json", 0.8)],
+ )
+
+ assert dedupe_reran_evals(root) == []
+ assert validate_eval_artifacts(root) == []
+
+
+def test_dedupe_does_not_prune_when_latest_recognized_result_is_invalid(
+ tmp_path: Path,
+) -> None:
+ cases = (
+ (json.dumps({**raw_eval_result(), "results": {}}), "empty or malformed"),
+ (json.dumps({**raw_eval_result(), "results": []}), "empty or malformed"),
+ (
+ json.dumps(
+ {
+ **raw_eval_result(),
+ "results": {
+ "gsm8k": {"exact_match,strict-match": "invalid"}
+ },
+ }
+ ),
+ "invalid score",
+ ),
+ (
+ json.dumps(
+ {
+ **raw_eval_result(),
+ "results": {"gsm8k": {"alias": "gsm8k"}},
+ }
+ ),
+ "no score",
+ ),
+ (json.dumps(raw_eval_result(effective=0)), "invalid effective sample count"),
+ (
+ json.dumps(raw_eval_result(effective="unknown")),
+ "invalid effective sample count",
+ ),
+ (json.dumps(raw_eval_result(effective=True)), "invalid effective sample count"),
+ (
+ json.dumps(
+ {
+ **raw_eval_result(),
+ "n-samples": {"gsm8k": {}},
+ }
+ ),
+ "malformed effective sample count",
+ ),
+ )
+ for index, (contents, expected) in enumerate(cases):
+ root = tmp_path / str(index)
+ root.mkdir()
+ name = "eval_minimaxm3_conc4096_b300-nv_retry"
+ _dd_write_legacy_raw(
+ root,
+ name,
+ 4096,
+ "2026-06-26T01-00-00.000000",
+ )
+ (
+ root
+ / name
+ / "results_2026-06-27T01-00-00.000000.json"
+ ).write_text(contents)
+ _dd_write_aggregate(
+ root,
+ [_dd_agg_row(4096, f"eval_results/{name}/results_old.json", 0.8)],
+ )
+
+ assert dedupe_reran_evals(root) == []
+ errors = validate_eval_artifacts(root)
+
+ assert any(expected in error for error in errors), errors
+ assert (root / name).is_dir()
+
+
+def test_dedupe_requires_aggregate_row_for_latest_raw_directory(
+ tmp_path: Path,
+) -> None:
+ old = "eval_minimaxm3_conc4096_b300-nv_old"
+ new = "eval_minimaxm3_conc4096_b300-nv_new"
+ _dd_write_legacy_raw(tmp_path, old, 4096, "2026-06-26T01-00-00.000000")
+ _dd_write_legacy_raw(tmp_path, new, 4096, "2026-06-27T01-00-00.000000")
+ agg_path = _dd_write_aggregate(
+ tmp_path,
+ [
+ _dd_agg_row(
+ 4096,
+ f"eval_results/{new}-unrelated/results_old.json",
+ 0.8,
+ )
+ ],
+ )
+ before = agg_path.read_text()
+
+ assert dedupe_reran_evals(tmp_path) == []
+ assert agg_path.read_text() == before
+ assert (tmp_path / old).is_dir()
+ assert (tmp_path / new).is_dir()
+ assert any("duplicate" in error for error in validate_eval_artifacts(tmp_path))
+
+
+
+def test_eval_validation_accepts_extract_filter_primary_score(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32, eval_suite="gpqa")
+ raw_path = next(tmp_path.glob("eval_*/results*.json"))
+ data = raw_eval_result(task="gpqa")
+ data["results"]["gpqa"] = {
+ "exact_match,extract_abcd": 0.75,
+ "exact_match_stderr,extract_abcd": 0.02,
+ "answer_token_count": 42,
+ }
+ data["configs"]["gpqa"]["filter_list"] = [{"name": "extract_abcd"}]
+ raw_path.write_text(json.dumps(data))
+ write_eval_aggregate(
+ tmp_path,
+ [
+ {
+ **single_eval_result(32, eval_suite="gpqa"),
+ "task": "gpqa",
+ }
+ ],
+ )
+
+ assert validate_eval_artifacts(tmp_path) == []
+
+
+def test_eval_dedupe_leaves_invalid_suite_for_validation(
+ tmp_path: Path,
+) -> None:
+ write_raw_eval_artifact(tmp_path, 32)
+ raw_dir = next(tmp_path.glob("eval_*"))
+ meta_path = raw_dir / "meta_env.json"
+ meta = json.loads(meta_path.read_text())
+ meta["eval_suite"] = []
+ meta_path.write_text(json.dumps(meta))
+ row = single_eval_result(32)
+ row["eval_suite"] = []
+ write_eval_aggregate(tmp_path, [row])
+
+ assert dedupe_reran_evals(tmp_path) == []
+ errors = validate_eval_artifacts(tmp_path)
+ assert any("invalid eval_suite" in error for error in errors)
+
+
+def test_dedupe_uses_winning_legacy_result_mtime_for_aggregate(
+ tmp_path: Path,
+) -> None:
+ artifact_name = "eval_minimaxm3_conc4096_b300-nv_retry"
+ _dd_write_legacy_raw(tmp_path, artifact_name, 4096, "a")
+ artifact_dir = tmp_path / artifact_name
+ older = artifact_dir / "results_b.json"
+ older.write_text(json.dumps(raw_eval_result()))
+ newer = artifact_dir / "results_a.json"
+ os.utime(older, ns=(1_000_000_000, 1_000_000_000))
+ os.utime(newer, ns=(2_000_000_000, 2_000_000_000))
+ _dd_write_aggregate(
+ tmp_path,
+ [
+ _dd_agg_row(
+ 4096,
+ f"eval_results/{artifact_name}/results_b.json",
+ 0.1,
+ ),
+ _dd_agg_row(
+ 4096,
+ f"eval_results/{artifact_name}/results_a.json",
+ 0.9,
+ ),
+ ],
+ )
+
+ dedupe_reran_evals(tmp_path)
+
+ rows = json.loads(
+ (tmp_path / "eval_results_all" / "agg_eval_all.json").read_text()
+ )
+ assert [row["em_strict"] for row in rows] == [0.9]
+ assert validate_eval_artifacts(tmp_path) == []
diff --git a/utils/validate_reusable_sweep_artifacts.py b/utils/validate_reusable_sweep_artifacts.py
index 0cfd1d2662..d83bbd8f3a 100644
--- a/utils/validate_reusable_sweep_artifacts.py
+++ b/utils/validate_reusable_sweep_artifacts.py
@@ -5,10 +5,12 @@
import argparse
import json
+import math
import re
import shutil
import sys
from collections import Counter
+from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Optional
@@ -313,6 +315,18 @@ def normalized_runner(value: Any) -> str:
return str(value or "").lower()
+LEGACY_EVAL_SUITE = ""
+def invalid_eval_suite(row: dict[str, Any]) -> bool:
+ """Return whether an explicit eval-suite identity is malformed."""
+ suite = row.get("eval_suite")
+ return "eval_suite" in row and (
+ not isinstance(suite, str) or not suite
+ )
+
+
+
+
+
def eval_key(row: dict[str, Any]) -> tuple[Any, ...]:
"""Build an eval identity from one aggregate row."""
if as_bool(row.get("is_multinode", False)):
@@ -322,6 +336,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]:
row.get("model_prefix", row.get("infmax_model_prefix")),
row.get("framework"),
row.get("precision"),
+ row.get("eval_suite", LEGACY_EVAL_SUITE),
row.get("spec_decoding", "none"),
as_int(row.get("isl", 8192), 8192),
as_int(row.get("osl", 1024), 1024),
@@ -347,6 +362,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]:
row.get("model_prefix", row.get("infmax_model_prefix")),
row.get("framework"),
row.get("precision"),
+ row.get("eval_suite", LEGACY_EVAL_SUITE),
row.get("spec_decoding", "none"),
as_int(row.get("isl", 8192), 8192),
as_int(row.get("osl", 1024), 1024),
@@ -360,6 +376,11 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]:
)
+def eval_result_key(row: dict[str, Any]) -> tuple[Any, ...]:
+ """Build a task-level eval result identity."""
+ return (*eval_key(row), row.get("task"))
+
+
def raw_eval_artifact_dirs(artifacts_dir: Path) -> list[Path]:
"""Return raw eval result artifacts, excluding aggregate and debug artifacts."""
return sorted(
@@ -373,10 +394,79 @@ def raw_eval_artifact_dirs(artifacts_dir: Path) -> list[Path]:
)
+def _positive_int(value: Any) -> bool:
+ """Return whether value is a positive JSON integer (not a boolean)."""
+ return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+
+def _raw_meta_contributions(
+ artifact_name: str,
+ meta: dict[str, Any],
+) -> tuple[
+ list[tuple[tuple[Any, ...], Optional[int]]],
+ bool,
+ list[str],
+]:
+ """Validate raw eval metadata and return its logical contributions."""
+ prefix = f"raw eval artifact {artifact_name!r}"
+ if "eval_concs" not in meta:
+ conc = meta.get("conc")
+ if not _positive_int(conc):
+ return [], False, [f"{prefix} has invalid legacy concurrency"]
+ return [(eval_key(meta), None)], False, []
+
+ expected = meta.get("eval_concs")
+ completed = meta.get("completed_eval_concs")
+ failed = meta.get("failed_eval_concs", [])
+ fields = (
+ ("eval_concs", expected),
+ ("completed_eval_concs", completed),
+ ("failed_eval_concs", failed),
+ )
+ errors: list[str] = []
+ if not all(isinstance(values, list) for _, values in fields):
+ return [], True, [f"{prefix} has invalid batched concurrency metadata"]
+
+ for field, values in fields:
+ if any(not _positive_int(value) for value in values):
+ errors.append(f"{prefix} has invalid {field}")
+ continue
+ if len(set(values)) != len(values):
+ errors.append(f"{prefix} has duplicate {field}")
+ if errors:
+ return [], True, errors
+
+ expected_set = set(expected)
+ completed_set = set(completed)
+ failed_set = set(failed)
+ if not expected_set:
+ errors.append(f"{prefix} has empty eval_concs")
+ if not completed_set:
+ errors.append(f"{prefix} has no completed eval concurrencies")
+ if not completed_set <= expected_set:
+ errors.append(f"{prefix} completed unexpected eval concurrencies")
+ if not failed_set <= expected_set:
+ errors.append(f"{prefix} failed unexpected eval concurrencies")
+ if completed_set & failed_set:
+ errors.append(f"{prefix} has overlapping completed and failed concurrencies")
+ if completed_set | failed_set != expected_set:
+ errors.append(f"{prefix} has unaccounted eval concurrencies")
+ if failed_set:
+ errors.append(f"{prefix} reports failed eval concurrencies")
+ if errors:
+ return [], True, errors
+
+ return (
+ [(eval_key({**meta, "conc": conc}), conc) for conc in completed],
+ True,
+ [],
+ )
+
+
def raw_eval_key_rows(
artifacts_dir: Path,
) -> tuple[list[tuple[Any, ...]], list[str]]:
- """Build logical eval identities from each raw artifact's metadata."""
+ """Build and validate logical identities from raw eval artifacts."""
rows: list[tuple[Any, ...]] = []
errors: list[str] = []
for artifact_dir in raw_eval_artifact_dirs(artifacts_dir):
@@ -388,7 +478,7 @@ def raw_eval_key_rows(
continue
try:
meta = load_json(meta_path)
- except (OSError, json.JSONDecodeError) as exc:
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
errors.append(
f"raw eval artifact {artifact_dir.name!r} has invalid "
f"meta_env.json: {exc}"
@@ -400,20 +490,69 @@ def raw_eval_key_rows(
"meta_env.json"
)
continue
- eval_concs = meta.get("completed_eval_concs")
- if isinstance(meta.get("eval_concs"), list):
- if not isinstance(eval_concs, list):
+ if invalid_eval_suite(meta):
+ errors.append(
+ f"raw eval artifact {artifact_dir.name!r} has invalid "
+ "eval_suite"
+ )
+ continue
+
+ contributions, batched, meta_errors = _raw_meta_contributions(
+ artifact_dir.name,
+ meta,
+ )
+ errors.extend(meta_errors)
+ if meta_errors:
+ continue
+
+ result_paths = _recognized_eval_result_paths(
+ artifact_dir.glob("results*.json")
+ )
+ if batched:
+ expected = set(meta["eval_concs"])
+ for path in result_paths:
+ conc = _result_concurrency(path.name)
+ if conc is None:
+ errors.append(
+ f"raw eval artifact {artifact_dir.name!r} has batched "
+ f"result {path.name!r} without a concurrency suffix"
+ )
+ elif conc not in expected:
+ errors.append(
+ f"raw eval artifact {artifact_dir.name!r} has result "
+ f"{path.name!r} for unexpected concurrency {conc}"
+ )
+
+ for _, conc in contributions:
+ candidates = [
+ path
+ for path in result_paths
+ if not batched or _result_concurrency(path.name) == conc
+ ]
+ conc_label = f" for concurrency {conc}" if conc is not None else ""
+ if not candidates:
errors.append(
- f"raw eval artifact {artifact_dir.name!r} has invalid "
- "batched concurrency metadata"
+ f"raw eval artifact {artifact_dir.name!r} has no "
+ f"recognized eval result{conc_label}"
)
continue
+ latest = max(candidates, key=_result_order)
+ result_error = _raw_result_error(latest)
+ if result_error is not None:
+ errors.append(
+ f"raw eval artifact {artifact_dir.name!r} latest result "
+ f"{latest.name!r}{conc_label} {result_error}"
+ )
+ continue
+ result_data = load_json(latest)
+ result_tasks = result_data["results"]
+ contribution_meta = (
+ {**meta, "conc": conc} if conc is not None else meta
+ )
rows.extend(
- eval_key({**meta, "conc": eval_conc})
- for eval_conc in eval_concs
+ eval_result_key({**contribution_meta, "task": task})
+ for task in result_tasks
)
- else:
- rows.append(eval_key(meta))
return rows, errors
@@ -433,14 +572,33 @@ def validate_eval_artifacts(
row_count = 0
aggregate_rows: list[tuple[Any, ...]] = []
for path in aggregate_files:
- data = load_json(path)
- if isinstance(data, list):
- row_count += len(data)
- aggregate_rows.extend(
- eval_key(row)
- for row in data
- if isinstance(row, dict)
+ try:
+ data = load_json(path)
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
+ errors.append(
+ f"eval aggregate {path.name!r} is invalid JSON: {exc}"
+ )
+ continue
+ if not isinstance(data, list):
+ errors.append(
+ f"eval aggregate {path.name!r} is not a list"
)
+ continue
+ row_count += len(data)
+ for index, row in enumerate(data):
+ if not isinstance(row, dict):
+ errors.append(
+ f"eval aggregate {path.name!r} row {index} "
+ "is not an object"
+ )
+ continue
+ if invalid_eval_suite(row):
+ errors.append(
+ f"eval aggregate {path.name!r} row {index} "
+ "has invalid eval_suite"
+ )
+ continue
+ aggregate_rows.append(eval_result_key(row))
if row_count == 0:
errors.append("eval_results_all contains no rows")
errors.extend(
@@ -474,16 +632,17 @@ def validate_run_stats(artifacts_dir: Path, required: bool) -> list[str]:
# A flaky eval retried several times leaves multiple raw ``eval_*`` dirs and
# multiple ``eval_results_all`` rows for one logical eval identity, which the
# checks above would otherwise reject. ``dedupe_reran_evals`` collapses those to
-# the latest result per identity (by lm-eval result timestamp) so a legitimate
+# the latest result per identity (by timestamp or legacy mtime) so a legitimate
# rerun does not fail validation. It only acts on identities that have a clear
-# latest result; genuinely ambiguous duplicates (no result timestamp to order
-# them by) are left in place for validation to reject. Eval-only; fixed-sequence
-# and agentic artifacts are untouched.
+# latest result, ordered by a filename timestamp or legacy mtime. Identities
+# with no result file are left in place for validation to reject. Eval-only;
+# fixed-sequence and agentic artifacts are untouched.
# lm-eval result files are ``results_.json`` (optionally a ``_concN`` /
-# staging suffix). The timestamp uses dashes throughout, so it is fixed-width
-# and lexicographically sortable.
+# staging suffix). Timestamped names and legacy mtimes are both converted to
+# epoch nanoseconds so mixed naming schemes have one coherent ordering.
_TIMESTAMP_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:\.\d+)?")
+_EVAL_RESULT_FORMAT = "inferencex-eval-v1"
# Batched result files carry their concurrency as a ``_concN`` suffix (kept in
# sync with ``collect_eval_results.CONC_SUFFIX_RE``).
@@ -502,107 +661,336 @@ def _result_timestamp(name: str) -> Optional[str]:
return match.group(0) if match else None
+def _timestamp_ns(stamp: str) -> int:
+ """Convert an lm-eval filename timestamp to UTC epoch nanoseconds."""
+ date, clock = stamp.split("T", 1)
+ hms, separator, fraction = clock.partition(".")
+ parsed = datetime.strptime(
+ f"{date}T{hms}",
+ "%Y-%m-%dT%H-%M-%S",
+ ).replace(tzinfo=timezone.utc)
+ epoch = datetime(1970, 1, 1, tzinfo=timezone.utc)
+ delta = parsed - epoch
+ fractional_ns = (
+ int((fraction + "000000000")[:9]) if separator else 0
+ )
+ return (
+ delta.days * 86_400_000_000_000
+ + delta.seconds * 1_000_000_000
+ + fractional_ns
+ )
+
+
+def _result_order(path: Path) -> tuple[int, str]:
+ """Return one deterministic recency key for timestamped and legacy files."""
+ stamp = _result_timestamp(path.name)
+ try:
+ recency = (
+ _timestamp_ns(stamp)
+ if stamp is not None
+ else path.stat().st_mtime_ns
+ )
+ except ValueError:
+ recency = path.stat().st_mtime_ns
+ return recency, path.name
+
+
+def _recognized_eval_result_paths(paths: Iterable[Path]) -> list[Path]:
+ """Return result JSONs carrying a collector-recognized eval marker."""
+ recognized: list[Path] = []
+ for path in paths:
+ try:
+ data = load_json(path)
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+ continue
+ if isinstance(data, dict) and (
+ "lm_eval_version" in data
+ or data.get("result_format") == _EVAL_RESULT_FORMAT
+ ):
+ recognized.append(path)
+ return recognized
+
+
+def _raw_result_error(path: Path) -> Optional[str]:
+ """Return a structural error for a raw result, or None when reusable."""
+ try:
+ data = load_json(path)
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
+ return f"is malformed JSON: {exc}"
+ if not isinstance(data, dict):
+ return "is not an object"
+ if "integration_error" in data:
+ return "reports an integration error"
+ if (
+ "lm_eval_version" not in data
+ and data.get("result_format") != _EVAL_RESULT_FORMAT
+ ):
+ return "has no recognized eval result format"
+
+ results = data.get("results")
+ if not isinstance(results, dict) or not results:
+ return "has empty or malformed results"
+ configs = data.get("configs", {})
+ if not isinstance(configs, dict):
+ return "has malformed configs"
+
+ sample_counts = data.get("n-samples")
+ if "n-samples" in data and not isinstance(sample_counts, dict):
+ return "has malformed effective sample counts"
+
+ for task, metrics in results.items():
+ if not isinstance(task, str) or not task:
+ return "has an invalid task name"
+ if not isinstance(metrics, dict) or not metrics:
+ return f"has empty or malformed results for task {task!r}"
+ task_config = configs.get(task, {})
+ if not isinstance(task_config, dict):
+ return f"has malformed config for task {task!r}"
+ metric_list = task_config.get("metric_list", [])
+ filter_list = task_config.get("filter_list", [])
+ if not isinstance(metric_list, list) or not isinstance(filter_list, list):
+ return f"has malformed config for task {task!r}"
+ if metric_list:
+ first_metric = metric_list[0]
+ if (
+ not isinstance(first_metric, dict)
+ or not isinstance(first_metric.get("metric"), str)
+ or not first_metric["metric"]
+ ):
+ return f"has malformed metric config for task {task!r}"
+ base_metric = first_metric["metric"]
+ else:
+ base_metric = "exact_match"
+ if filter_list:
+ if any(
+ not isinstance(item, dict)
+ or not isinstance(item.get("name"), str)
+ or not item["name"]
+ for item in filter_list
+ ):
+ return f"has malformed filter config for task {task!r}"
+ configured_names = [
+ f"{base_metric},{item['name']}"
+ for item in filter_list
+ ]
+ strict_names = [
+ name
+ for name in configured_names
+ if "strict" in name or "resolved" in name
+ ]
+ fallback_names = [
+ name
+ for name in configured_names
+ if "flex" in name or "extract" in name
+ ]
+ primary_names = strict_names or fallback_names or configured_names
+ else:
+ primary_names = ["acc" if "acc" in metrics else base_metric]
+ if not primary_names or any(name not in metrics for name in primary_names):
+ return f"has no score for task {task!r}"
+
+ for name in primary_names:
+ score = metrics[name]
+ if (
+ isinstance(score, bool)
+ or not isinstance(score, (int, float))
+ or not math.isfinite(score)
+ or score < 0
+ or score > 1
+ ):
+ return (
+ f"has invalid score {name!r} for task {task!r}: "
+ f"{score!r}"
+ )
+ if sample_counts is not None:
+ task_counts = sample_counts.get(task)
+ if not isinstance(task_counts, dict) or "effective" not in task_counts:
+ return f"has malformed effective sample count for task {task!r}"
+ effective = task_counts["effective"]
+ if (
+ isinstance(effective, bool)
+ or not isinstance(effective, (int, float))
+ or not math.isfinite(effective)
+ or effective <= 0
+ ):
+ return (
+ f"has invalid effective sample count for task {task!r}: "
+ f"{effective!r}"
+ )
+ return None
+
+
def _raw_dir_contributions(
artifact_dir: Path,
) -> tuple[list[tuple[tuple[Any, ...], Optional[int]]], dict[str, Any], bool]:
- """Return (identity, conc) pairs a raw dir contributes, plus its meta.
-
- Mirrors ``raw_eval_key_rows``: a batched artifact contributes one identity
- per ``completed_eval_concs`` entry; a legacy artifact contributes one from
- its meta. ``conc`` is the batched concurrency (``None`` for legacy).
- """
- meta = load_json(artifact_dir / "meta_env.json")
+ """Return validated (identity, conc) contributions and raw metadata."""
+ try:
+ meta = load_json(artifact_dir / "meta_env.json")
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+ return [], {}, False
if not isinstance(meta, dict):
return [], {}, False
- batched = isinstance(meta.get("eval_concs"), list)
- if batched:
- concs = meta.get("completed_eval_concs")
- if not isinstance(concs, list):
- return [], meta, True
- return (
- [(eval_key({**meta, "conc": conc}), as_int(conc)) for conc in concs],
- meta,
- True,
- )
- return [(eval_key(meta), None)], meta, False
+ if invalid_eval_suite(meta):
+ return [], meta, False
+ contributions, batched, errors = _raw_meta_contributions(
+ artifact_dir.name,
+ meta,
+ )
+ if errors:
+ return [], meta, batched
+ return contributions, meta, batched
+
+
+def _source_names_raw_dir(source: Any, artifact_name: str) -> bool:
+ """Return whether an aggregate source path names this exact raw directory."""
+ return artifact_name in re.split(r"[\\/]+", str(source or ""))
def _eval_winners(artifacts_dir: Path) -> dict[tuple[Any, ...], str]:
- """Pick the raw dir holding the latest result for each eval identity.
+ """Pick structurally valid, aggregate-backed latest raw results."""
+ aggregate_sources: dict[tuple[Any, ...], list[Any]] = {}
+ aggregate_dir = artifacts_dir / "eval_results_all"
+ for path in sorted(aggregate_dir.glob("*.json")):
+ try:
+ data = load_json(path)
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+ continue
+ if not isinstance(data, list):
+ continue
+ for row in data:
+ if isinstance(row, dict) and not invalid_eval_suite(row):
+ aggregate_sources.setdefault(eval_key(row), []).append(
+ row.get("source")
+ )
- Ranks only by the lm-eval result timestamp, so an identity appears here
- only when at least one of its raw dirs carries a real result. Identities
- with no timestamped result get no winner and are left untouched.
- """
- best: dict[tuple[Any, ...], tuple[str, str]] = {}
+ best: dict[
+ tuple[Any, ...],
+ tuple[tuple[int, str], str, Path],
+ ] = {}
for artifact_dir in raw_eval_artifact_dirs(artifacts_dir):
contributions, _, batched = _raw_dir_contributions(artifact_dir)
- if not contributions:
- continue
- for path in artifact_dir.glob("results*.json"):
- stamp = _result_timestamp(path.name)
- if stamp is None:
+ result_paths = _recognized_eval_result_paths(
+ artifact_dir.glob("results*.json")
+ )
+ for key, key_conc in contributions:
+ candidates = [
+ path
+ for path in result_paths
+ if not batched or _result_concurrency(path.name) == key_conc
+ ]
+ if not candidates:
continue
- conc = _result_concurrency(path.name)
- for key, key_conc in contributions:
- # A batched result file is tagged with its conc; only let it
- # compete for the matching identity.
- if batched and conc is not None and key_conc != conc:
- continue
- candidate = (stamp, artifact_dir.name)
- if best.get(key) is None or candidate > best[key]:
- best[key] = candidate
- return {key: name for key, (_, name) in best.items()}
+ latest = max(candidates, key=_result_order)
+ candidate = (_result_order(latest), artifact_dir.name, latest)
+ current = best.get(key)
+ if current is None or candidate[:2] > current[:2]:
+ best[key] = candidate
+
+ winners: dict[tuple[Any, ...], str] = {}
+ for key, (_, artifact_name, path) in best.items():
+ if _raw_result_error(path) is not None:
+ continue
+ if any(
+ _source_names_raw_dir(source, artifact_name)
+ for source in aggregate_sources.get(key, [])
+ ):
+ winners[key] = artifact_name
+ return winners
def _dedupe_eval_aggregate(
artifacts_dir: Path, winners: dict[tuple[Any, ...], str]
) -> list[str]:
- """Keep one aggregate row per winning identity (its winner dir's row)."""
+ """Keep one aggregate row per winning identity across all aggregate files."""
eval_dir = artifacts_dir / "eval_results_all"
if not eval_dir.is_dir():
return []
- messages: list[str] = []
+
+ loaded: dict[Path, list[Any]] = {}
+ groups: dict[
+ tuple[Any, ...],
+ list[tuple[Path, int, dict[str, Any]]],
+ ] = {}
for agg_path in sorted(eval_dir.glob("*.json")):
- data = load_json(agg_path)
+ try:
+ data = load_json(agg_path)
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+ continue
if not isinstance(data, list):
continue
- groups: dict[tuple[Any, ...], list[int]] = {}
- keep: set[int] = set()
- for idx, row in enumerate(data):
- if not isinstance(row, dict):
- keep.add(idx)
- continue
- groups.setdefault(eval_key(row), []).append(idx)
- for key, indices in groups.items():
- winner = winners.get(key)
- # Only collapse identities with a clear latest result; leave
- # ambiguous duplicates for validation to reject.
- if winner is None or len(indices) == 1:
- keep.update(indices)
- continue
- keep.add(
- next(
- (
- idx
- for idx in indices
- if winner in str(data[idx].get("source") or "")
- ),
- max(
- indices,
- key=lambda idx: _result_timestamp(
- str(data[idx].get("source") or "")
- )
- or "",
- ),
+ loaded[agg_path] = data
+ for index, row in enumerate(data):
+ if isinstance(row, dict) and not invalid_eval_suite(row):
+ groups.setdefault(eval_result_key(row), []).append(
+ (agg_path, index, row)
)
+
+ keep = {
+ path: set(range(len(data)))
+ for path, data in loaded.items()
+ }
+ winner_result_names: dict[tuple[Any, ...], str] = {}
+ for key, artifact_name in winners.items():
+ artifact_dir = artifacts_dir / artifact_name
+ contributions, _, batched = _raw_dir_contributions(artifact_dir)
+ conc = next(
+ (candidate_conc for candidate_key, candidate_conc in contributions
+ if candidate_key == key),
+ None,
+ )
+ candidates = [
+ path
+ for path in _recognized_eval_result_paths(
+ artifact_dir.glob("results*.json")
)
- if len(keep) != len(data):
- kept = [row for idx, row in enumerate(data) if idx in keep]
- agg_path.write_text(json.dumps(kept, indent=2))
- messages.append(
- f"{agg_path.name}: kept {len(kept)} of {len(data)} eval row(s)"
- )
+ if not batched or _result_concurrency(path.name) == conc
+ ]
+ if candidates:
+ winner_result_names[key] = max(candidates, key=_result_order).name
+
+ for key, entries in groups.items():
+ artifact_key = key[:-1]
+ winner = winners.get(artifact_key)
+ if winner is None or len(entries) == 1:
+ continue
+ matching = [
+ entry
+ for entry in entries
+ if _source_names_raw_dir(entry[2].get("source"), winner)
+ ]
+ winner_result_name = winner_result_names.get(artifact_key)
+ exact_matching = [
+ entry
+ for entry in matching
+ if re.split(
+ r"[\\/]+",
+ str(entry[2].get("source") or ""),
+ )[-1] == winner_result_name
+ ]
+ if not exact_matching:
+ continue
+ chosen = max(
+ exact_matching,
+ key=lambda entry: (entry[0].name, entry[1]),
+ )
+ chosen_location = chosen[0], chosen[1]
+ for path, index, _ in entries:
+ if (path, index) != chosen_location:
+ keep[path].discard(index)
+
+ messages: list[str] = []
+ for agg_path, data in loaded.items():
+ kept = [
+ row
+ for index, row in enumerate(data)
+ if index in keep[agg_path]
+ ]
+ if len(kept) == len(data):
+ continue
+ agg_path.write_text(json.dumps(kept, indent=2))
+ messages.append(
+ f"{agg_path.name}: kept {len(kept)} of {len(data)} eval row(s)"
+ )
return messages
@@ -636,11 +1024,12 @@ def superseded(key: tuple[Any, ...]) -> bool:
remaining = [
conc
for conc in meta.get("completed_eval_concs", [])
- if as_int(conc) not in losing
+ if conc not in losing
]
if not remaining:
shutil.rmtree(artifact_dir)
return f"removed superseded batched raw eval dir {name!r}"
+ meta["eval_concs"] = remaining
meta["completed_eval_concs"] = remaining
(artifact_dir / "meta_env.json").write_text(json.dumps(meta))
dropped = ",".join(str(conc) for conc in sorted(losing))