From 60e20ec40de5bad7a609323efde5ca33b7273472 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Wed, 15 Jul 2026 23:40:39 +0800 Subject: [PATCH 1/2] =?UTF-8?q?ci(mirror):=20=E6=AF=8F=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E8=AE=A1=E6=97=B6=20+=202m=20=E4=B8=8A=E4=BC=A0=E4=B8=8A?= =?UTF-8?q?=E9=99=90(=E8=B6=85=E6=97=B6=20warn=20=E5=B9=B6=E8=B7=B3?= =?UTF-8?q?=E8=BF=87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.0.94 发布时 publish-ecosystem 被 30min job timeout 杀掉,日志里看不出是 哪个资产在慢 —— 事后查是 gtc 跨境传 linux-x86_64(8.4MB)/linux-aarch64 (30.5MB)两个大件卡住,最后靠本地 gtc 手工补齐。 - **每资源计时**:上传返回即打印 ` <资产> (<大小>) uploaded in Ns`; 已 serving 的仍先 probe → 打印 skipping,不重传(原有行为,补上 host 标签)。 - **2m 上限**(`MIRROR_UPLOAD_TIMEOUT`,默认 120s):超时打印 WARN + **放弃该资产**, 后续轮次不再重试。 - 每条 mirror leg 结束打印耗时;失败时 hint 出手工补传命令。 **为何这个 cap 与 v0.0.90 事故不同**:那次 `timeout 300` 杀掉慢但**在推进**的上传 后**重传**,每次从字节零开始,永不收敛,把 20min job 预算拖垮。本 cap 的安全性来自 **超时即放弃、绝不重试**;完整性 gate 仍是唯一 pass/fail,漏传会响亮报错而不是 静默发半个镜像。脚本顶部的血泪注释已按此改写。 验证: - 真实 0.0.94 镜像干跑 → 16 资产两端全 200、已存在全部 skip、206s 通过。 - stub 双分支(PATH 上的真可执行文件,非 shell function —— timeout 对函数无效): 快传 → `github big.tar.gz (2.9MB) uploaded in 1s`; 慢传 → `WARN: gitcode ... exceeded the 2s cap after 2s — skipping`。 --- .github/tools/mirror_res.sh | 89 ++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/.github/tools/mirror_res.sh b/.github/tools/mirror_res.sh index c8a43c2..c04a8bb 100755 --- a/.github/tools/mirror_res.sh +++ b/.github/tools/mirror_res.sh @@ -42,8 +42,56 @@ read -r -a ASSETS <<< "${ASSETS:-$DEFAULT_ASSETS}" info() { echo "[mirror] $*"; } +# Per-asset upload cap. Exceeding it WARNs and abandons that asset (no retry, +# no delete) so one slow cross-border PUT can't eat the whole job budget — the +# v0.0.94 release lost `publish-ecosystem` to a 30min job timeout with zero +# per-asset visibility, and the two biggest linux tarballs had to be pushed by +# hand afterwards. +# +# CAUTION (v0.0.90 postmortem, still binding): a cap that kills a +# slow-but-PROGRESSING upload and then RETRIES it is strictly worse than no cap +# — every restart resumes from byte zero and the mirror never converges. This +# cap is safe only because a capped asset is SKIPPED, never re-uploaded. The +# completeness gate at the bottom is still the pass/fail, so a skipped asset +# fails the release loudly instead of silently shipping a half mirror. +# +# Raise it (or set it per-host) when the cross-border leg legitimately needs +# longer than the cap for the big archives; the gate will tell you. +: "${MIRROR_UPLOAD_TIMEOUT:=120}" + DL="$(mktemp -d)"; trap 'rm -rf "$DL"' EXIT +human_size() { # path → e.g. 31.9MB + local b; b=$(stat -c %s "$1" 2>/dev/null || stat -f %z "$1" 2>/dev/null || echo 0) + awk -v b="$b" 'BEGIN{ if (b>=1048576) printf "%.1fMB", b/1048576; else if (b>=1024) printf "%.1fKB", b/1024; else printf "%dB", b }' +} + +host_label() { [[ "$1" == gh ]] && echo github || echo gitcode; } + +# Upload one asset under the cap, timing it. 0 = the command returned within +# the cap (NOT proof it landed — gtc's exit code lies both ways, so the probe / +# gate remains the only source of truth); 1 = the cap fired, asset abandoned. +upload_asset() { # kind(gh|gtc) asset → 0 returned / 1 capped + local kind="$1" a="$2" start elapsed rc=0 sz host + sz=$(human_size "$DL/$a") + host=$(host_label "$kind") + start=$SECONDS + if [[ "$kind" == gh ]]; then + GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout "$MIRROR_UPLOAD_TIMEOUT" \ + gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber >/dev/null 2>&1 || rc=$? + else + timeout "$MIRROR_UPLOAD_TIMEOUT" gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" \ + >/dev/null 2>&1 || rc=$? + fi + elapsed=$((SECONDS - start)) + if [[ $rc == 124 || $rc == 137 ]]; then + info "WARN: $host $a ($sz) exceeded the ${MIRROR_UPLOAD_TIMEOUT}s cap after ${elapsed}s — skipping (not retried; the verify gate below decides the release)" + return 1 + fi + info "$host $a ($sz) uploaded in ${elapsed}s" + return 0 +} + info "downloading $SRC_REPO v$VER assets ($PROJ)" for a in "${ASSETS[@]}"; do gh release download "v$VER" -R "$SRC_REPO" -D "$DL" -p "$a" 2>/dev/null || { echo "[mirror] FAIL: missing $a in $SRC_REPO v$VER" >&2; exit 1; } @@ -93,36 +141,43 @@ verify_batch() { # base_url asset... → prints assets still not serving # Hard-won rules (0.0.86 / 0.0.89 / 0.0.90 postmortems): # - NEVER delete on a verify timeout — the eager 404→delete loop repeatedly # deleted GOOD uploads whose propagation was merely slow. -# - NEVER kill a slow-but-progressing upload: the outer `timeout` MUST -# exceed gtc's inner PUT timeout (600s). v0.0.90 wrapped uploads in -# `timeout 300`, so every cross-border PUT >5min was SIGKILLed at 60%% -# and restarted from byte zero — the 20min job ceiling fell to this. +# - NEVER kill a slow-but-progressing upload AND RETRY IT. v0.0.90 wrapped +# uploads in `timeout 300`, so every cross-border PUT >5min was SIGKILLed +# at 60%% and restarted from byte zero — the 20min job ceiling fell to +# this. MIRROR_UPLOAD_TIMEOUT keeps a cap but ABANDONS the asset instead of +# retrying it, which is what makes the cap safe; the gate then fails the +# release loudly rather than thrashing until the job is killed. # - gtc's exit code lies both ways (obs_callback flakiness); the download # probe is the only source of truth. mirror_host() { # kind(gh|gtc) base_url local kind="$1" base="$2" try a local pending failed + local -A capped=() + local host_start=$SECONDS + local host; host=$(host_label "$kind") for try in 1 2 3; do pending=() for a in "${ASSETS[@]}"; do + # A capped asset is never re-attempted (see MIRROR_UPLOAD_TIMEOUT). + [[ -n "${capped[$a]:-}" ]] && continue + # Step 1: already serving? then it's mirrored — never re-upload it. if probe "${base}/${a}"; then - [[ $try == 1 ]] && info "$kind $a already mirrored, skipping" + [[ $try == 1 ]] && info "$host $a already mirrored, skipping" continue fi - pending+=("$a") - if [[ "$kind" == gh ]]; then - GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout 900 \ - gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber || true + if upload_asset "$kind" "$a"; then + pending+=("$a") else - timeout 900 gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" \ - >/dev/null 2>&1 || true + capped[$a]=1 fi done - [[ ${#pending[@]} == 0 ]] && return 0 + [[ ${#pending[@]} == 0 ]] && break failed=$(verify_batch "$base" "${pending[@]}") - [[ -z "$failed" ]] && return 0 - echo "[mirror] $kind not serving after patience (try $try): $failed — re-uploading (no delete)" + [[ -z "$failed" ]] && break + info "$host not serving after patience (try $try): $failed — re-uploading (no delete)" done + ((${#capped[@]})) && info "WARN: $host abandoned ${#capped[@]} asset(s) at the ${MIRROR_UPLOAD_TIMEOUT}s cap: ${!capped[*]}" + info "$host mirror leg finished in $((SECONDS - host_start))s" return 0 # the completeness gate below is the real pass/fail } @@ -174,5 +229,9 @@ for host in "${hosts[@]}"; do [[ "$code" == 200 ]] || { rc=1; echo "[mirror] FAIL: missing/unverified: https://${host}/releases/download/${VER}/${a}" >&2; } done done -[[ $rc == 0 ]] && info "all assets mirrored + verified on ${#hosts[@]} host(s)" +if [[ $rc != 0 ]]; then + echo "[mirror] hint: if the asset above was WARNed as capped at ${MIRROR_UPLOAD_TIMEOUT}s, either raise MIRROR_UPLOAD_TIMEOUT for this run or push it by hand:" >&2 + echo "[mirror] gh release download v$VER -R $SRC_REPO -p '' && gtc release upload $GTC_DST '' --tag $VER" >&2 +fi +[[ $rc == 0 ]] && info "all assets mirrored + verified on ${#hosts[@]} host(s) in ${SECONDS}s" exit $rc From 4f78fa3c09df148e1f885b13d81dba2b5e7bd7a4 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Wed, 15 Jul 2026 23:46:27 +0800 Subject: [PATCH 2/2] =?UTF-8?q?ci(mirror):=20=E5=8D=95=E8=B5=84=E4=BA=A7?= =?UTF-8?q?=20cap=203min=20+=20=E9=95=9C=E5=83=8F=E6=AE=B5=2010min=20?= =?UTF-8?q?=E7=A1=AC=E9=A1=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 review 定档: - `MIRROR_UPLOAD_TIMEOUT` 120 → **180s**。mcpp 最大资产约 30MB(单包不超 100M), 3 分钟还没传完不是「慢」是「卡住」,不值得继续占 CI —— 跳过并由本地补传 (gate 会打印确切的补传命令)。 - Mirror 步骤加 **`timeout-minutes: 10`** 硬顶。健康的 run 远在此之内;10min 是 兜底,防止某一端病态时烧光 job 预算(v0.0.94 就是 30min 挂死且零可见性)。 --- .github/tools/mirror_res.sh | 8 +++++--- .github/workflows/release.yml | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/tools/mirror_res.sh b/.github/tools/mirror_res.sh index c04a8bb..eb055fe 100755 --- a/.github/tools/mirror_res.sh +++ b/.github/tools/mirror_res.sh @@ -55,9 +55,11 @@ info() { echo "[mirror] $*"; } # completeness gate at the bottom is still the pass/fail, so a skipped asset # fails the release loudly instead of silently shipping a half mirror. # -# Raise it (or set it per-host) when the cross-border leg legitimately needs -# longer than the cap for the big archives; the gate will tell you. -: "${MIRROR_UPLOAD_TIMEOUT:=120}" +# Sizing: mcpp's largest asset is ~30MB (no package exceeds 100MB). 180s is a +# generous ceiling for that — an upload still running at 3min is not "slow", it +# is stuck, and the right move is to stop paying CI for it and push that one +# asset by hand (the gate below prints the exact command). +: "${MIRROR_UPLOAD_TIMEOUT:=180}" DL="$(mktemp -d)"; trap 'rm -rf "$DL"' EXIT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dda5c71..0db2650 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -725,6 +725,13 @@ jobs: - name: Mirror binaries to xlings-res/mcpp (gh + gtc) if: ${{ env.XLINGS_RES_TOKEN != '' }} + # Hard ceiling for the whole mirror segment. The script caps each asset + # at MIRROR_UPLOAD_TIMEOUT (180s) and abandons anything slower, so a + # healthy run lands well inside this; 10min is the backstop that keeps a + # pathological host from burning the job's budget the way v0.0.94 did + # (30min, killed, zero per-asset visibility). Whatever gets skipped is + # reported by name + size and pushed by hand. + timeout-minutes: 10 env: GH_TOKEN: ${{ secrets.XLINGS_RES_TOKEN }} run: |