diff --git a/.github/scripts/monitor_slurm_job.sh b/.github/scripts/monitor_slurm_job.sh index cb69e5c01..3d4c4a717 100755 --- a/.github/scripts/monitor_slurm_job.sh +++ b/.github/scripts/monitor_slurm_job.sh @@ -86,7 +86,15 @@ is_terminal_state() { # (Phoenix 'embers') a job can stay PENDING for hours, burning the CI job # timeout and holding a runner slot; fail early so it reads as queue starvation, # not a test failure. 0 = wait indefinitely. -: "${SLURM_MAX_QUEUE_SECONDS:=5400}" # 90 minutes +# +# The budget has to clear a normal bad day on a busy machine, or it converts +# routine queue pressure into red CI. Frontier's own numbers make the case: +# over one week, MFC jobs on `batch` waited p50=1m but p90=96m and p95=176m, +# with a 466m tail. A 90-minute budget cut into that distribution, tripping on +# 11% of the jobs that did eventually start -- plus the ones that never did. +# Four hours clears p95 with room to spare while staying well inside the 480m +# job-level `timeout-minutes`, which remains the real backstop. +: "${SLURM_MAX_QUEUE_SECONDS:=14400}" # 4 hours # Reject a non-integer override rather than silently skipping the budget. if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2 diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh index 6803b3766..16035419b 100755 --- a/.github/scripts/submit-slurm-job.sh +++ b/.github/scripts/submit-slurm-job.sh @@ -62,7 +62,10 @@ case "$cluster" in # CFD154; submitting under it now fails outright with "Invalid qos # specification". "normal" is the only QOS on this allocation without a # one-job-at-a-time cap, so it is the only one that can run the CI - # matrix concurrently. + # matrix concurrently. Note that the g1 partition carries its own + # partition QOS (also named "g1"), which slurmctld applies on its own + # when a job lands there. Do not add --qos=g1: CFD154 has no + # association with it and sbatch rejects the job outright. qos="normal" # Let each job's slurmstepd broker its own steps instead of routing # every srun through slurmctld. The in-job test suite launches ~1700+ @@ -105,9 +108,11 @@ if [ "$device" = "cpu" ]; then #SBATCH --mem-per-cpu=8G" ;; frontier|frontier_amd) + # g1 is a dedicated 64-node carve-out; its nodes are not in batch, + # so CI starts promptly instead of queueing behind the machine. sbatch_device_opts="\ #SBATCH -n 32 -#SBATCH -p batch" +#SBATCH -p g1" ;; esac elif [ "$device" = "gpu" ]; then @@ -135,7 +140,7 @@ elif [ "$device" = "gpu" ]; then frontier|frontier_amd) sbatch_device_opts="\ #SBATCH -n 8 -#SBATCH -p batch" +#SBATCH -p g1" ;; esac else