diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh index 0fbe6619ab..c3d38e1b48 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -14,12 +14,10 @@ if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" fi -if [ "$TP" -ne 8 ] || [ "$EP_SIZE" -ne 1 ] || [ "$DP_ATTENTION" != "false" ]; then - echo "This recipe requires TP=8, EP_SIZE=1, and DP_ATTENTION=false" >&2 - exit 1 -fi require_agentic_kv_offload_none +echo "Attention mode: $([ "$DP_ATTENTION" = "true" ] && echo dp || echo tp) (DP_ATTENTION=$DP_ATTENTION, CONC=$CONC)" + if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" fi @@ -48,14 +46,37 @@ export ATOM_DEBUG_PREFIX_HITS=1 export ATOM_PROFILER_MORE=0 export ATOM_PROFILER_TIMEOUT=1200 -# AgentX/AIPerf network, failure, warmup, and trace-gap settings from the -# validated one-hour baseline. +# DP-attention runs layer ATOM's DPA routing and two-batch-overlap knobs on top of +# the TP settings above (recipe section "Server - DP attention"); exported only for +# the DP band. ATOM_DP_SESSION_AFFINITY is not optional: without it a session's +# turns scatter across DP ranks, the prefix KV written by one turn is unreachable +# by the next, and the multi-turn agentic workload collapses to cold prefill. +# GPU_MAX_HW_QUEUES and ATOM_NUMA_BIND are prerequisites of --enable-tbo. +DP_ATTN_ARGS=() +if [ "$DP_ATTENTION" = "true" ]; then + export GPU_MAX_HW_QUEUES=5 + export ATOM_NUMA_BIND=1 + export ATOM_DP_SESSION_AFFINITY=1 + export ATOM_DP_LB_REQ_EQUIV=512 + export ATOM_ENABLE_PREFILL_DELAYER=1 + export ATOM_PREFILL_DECODE_INTERVAL=10 + # Client-side counterpart to session affinity: make AIPerf emit a stable + # session id (x-dynamo-session-id, falling back to the always-sent + # x-correlation-id) so the DPA router pins each conversation to one rank. + export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=true + DP_ATTN_ARGS=(--enable-dp-attention --enable-tbo) +fi + +# Raise the AIPerf HTTP TCP user timeout to 900000 ms (15 min), well above the +# aiperf default of 30000 ms (30 s), so long-stalling AgentX request +# connections are not torn down as dead during extended server-side pauses. export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -export AIPERF_FAILED_REQUEST_THRESHOLD=0.10 -export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0.10 -export AIPERF_TRACE_IDLE_GAP_CAP_SECONDS=300 -export AIPERF_WARMUP_REQUESTS_PER_LANE=10 -export AIPERF_BENCHMARK_GRACE_PERIOD=30 +export AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT=300 +export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0 +export AIPERF_DATASET_CONFIGURATION_TIMEOUT=1800 +export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800 +export AIPERF_UI_REALTIME_METRICS_ENABLED=true # Require ATOM Prometheus metrics in every official result. export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" @@ -83,8 +104,7 @@ trap 'exit 143' TERM MAX_NUM_SEQS=$((2 * CONC)) # golden_al_distribution/dsv4_mtp.yaml: thinking_on, 3 draft tokens -> AL 2.49 -# --spec-decode-acceptance-length 2.49. -# https://github.com/ROCm/ATOM/pull/1948 +# https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/dsv4_mtp.yaml NUM_SPEC_TOKENS=3 SPEC_DECODE_AL=2.49 SPEC_ARGS=( @@ -102,6 +122,12 @@ ATOM_CMD=( --served-model-name "$MODEL" --host 0.0.0.0 --server-port "$PORT" + # uvicorn defaults to a 5s idle keep-alive; AIPerf pools sockets for far + # longer (aiohttp ~15s) and warmup inter-turn gaps under backlog exceed 5s, + # so the server closes an idle pooled socket and the reused write hits + # 'Connection reset by peer' (errno 104). One such reset on a root AgentX + # warmup request aborts the whole run. Outlast the client idle window. + --timeout-keep-alive 900 --tensor-parallel-size "$TP" --kv-cache-dtype fp8 --index-cache-dtype fp4 @@ -113,6 +139,7 @@ ATOM_CMD=( --level 3 --cudagraph-mode FULL "${SPEC_ARGS[@]}" + "${DP_ATTN_ARGS[@]}" --max-num-seqs "$MAX_NUM_SEQS" ) write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 99f2aecfa4..ab36d1ce24 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1336,7 +1336,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # uses the thinking_on golden AL 2.49 for three draft tokens; eval uses real # MTP acceptance. max-num-seqs is set to 2x concurrency by the recipe. dsv4-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202608201032 + image: rocm/atom-dev:nightly_202608280858 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1346,7 +1346,8 @@ dsv4-fp4-mi355x-atom-agentic-mtp: scenarios: agentic-coding: - search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 48] } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16] } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48, 64, 96, 128, 256] } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e75a1ffdbf..a619ab7c7d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6577,6 +6577,18 @@ - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730 +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Split the DeepSeek-V4-Pro FP4 ATOM AgentX MTP sweep into two attention bands per the ROCm/ATOM DeepSeek-V4-Agentic-InferenceX.md recipe: tensor-parallel attention at concurrency 1, 2, 4, 8, and 16, and DP attention at concurrency 48, 64, 96, 128, and 256 (previously a single tensor-parallel band at concurrency 1, 2, 4, 8, 16, 32, and 48)." + - "The DP-attention band adds --enable-dp-attention and --enable-tbo plus ATOM's DPA routing knobs (ATOM_DP_SESSION_AFFINITY, ATOM_DP_LB_REQ_EQUIV, ATOM_ENABLE_PREFILL_DELAYER, ATOM_PREFILL_DECODE_INTERVAL), the two-batch-overlap prerequisites GPU_MAX_HW_QUEUES and ATOM_NUMA_BIND, and the AIPerf session-affinity vars that keep each agentic conversation pinned to one DP rank; the tensor-parallel band is otherwise unchanged." + - "Session affinity is mandatory for the DP band: without it a session's turns scatter across DP ranks, the prefix KV written by one turn is unreachable by the next, and the multi-turn agentic workload collapses to cold prefill." + - "The attention band is selected by the config's dp-attn, propagated to the recipe as DP_ATTENTION, so the script's former DP_ATTENTION=false hard-fail guard is removed." + - "Bump the ATOM image from rocm/atom-dev:nightly_202608201032 to nightly_202608280858; FP8 KV/index caches, prefix caching, 32K state checkpoints, 16K batching/prefill chunks, FULL cudagraph mode, three-token MTP with golden AL 2.49, and max-num-seqs at twice concurrency are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2778 + - config-keys: - dsv4-fp4-b200-sglang-agentic-hicache-mtp scenario-type: