Skip to content

Commit b4c25ad

Browse files
SunrisepeakSunrisepeak
committed
fix(mirror): two-phase upload+verify, ranged-GET probes, never delete on verify timeout
The v0.0.89 publish-ecosystem job hit its 20min ceiling and was cancelled. Three compounding causes: - 18s patience < GitHub's ~20-60s CDN propagation floor, and the timeout path DELETED the (good) asset and restarted propagation from zero — the 0.0.86 eager-delete bug recurring at a milder setting (11 delete+reupload cycles across 8 assets). Deletes are gone: --clobber re-upload already replaces; the probe is the source of truth. - wait-loop probes did FULL GETs — minute-scale per poll against gitcode from a US runner. Now a ranged GET (real object read, one byte); the final completeness gate keeps full GETs (0.0.75/76 lesson). - per-asset upload→wait serialized 8 propagation windows; now batch upload then batch verify, so propagation overlaps.
1 parent f6fd39e commit b4c25ad

1 file changed

Lines changed: 69 additions & 44 deletions

File tree

‎.github/tools/mirror_res.sh‎

Lines changed: 69 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -49,71 +49,96 @@ for a in "${ASSETS[@]}"; do
4949
gh release download "v$VER" -R "$SRC_REPO" -D "$DL" -p "$a" 2>/dev/null || { echo "[mirror] FAIL: missing $a in $SRC_REPO v$VER" >&2; exit 1; }
5050
done
5151

52-
# ── GitHub (per-file timeout + verify + delete-and-reupload retry) ──
52+
# ── Probes ──────────────────────────────────────────────────────────
53+
# Wait-loop probe: a RANGED GET (first byte). It's a real object read — HEAD
54+
# lies on both hosts (gitcode returns a redirect stub; the 0.0.75/76 github
55+
# phantom assets HEAD'd fine) — but it doesn't download multi-MB assets on
56+
# every poll (the old full-GET probes against gitcode from a US runner were
57+
# minute-scale each and blew the job's 20min budget, v0.0.89 incident).
58+
# The final completeness gate below still does FULL GETs.
59+
probe() { # host_path asset → 0 iff the object serves bytes
60+
local code
61+
code=$(curl -fsSL -o /dev/null -w '%{http_code}' -r 0-0 -L "$1" 2>/dev/null)
62+
[[ "$code" == 200 || "$code" == 206 ]]
63+
}
64+
65+
# Wait for an asset to serve, with real propagation patience (fresh uploads
66+
# take ~20-60s to reach the download CDN). ~2min ceiling, 5s cadence.
67+
wait_asset() { # url → 0 iff 200/206 within the window
68+
for _ in $(seq 1 24); do
69+
probe "$1" && return 0
70+
sleep 5
71+
done
72+
return 1
73+
}
74+
75+
# ── GitHub ──────────────────────────────────────────────────────────
5376
# A4 hardening: `gh release upload` was observed both HANGING (>1h on one
54-
# asset, 2026-07-08) and leaving a phantom asset that later 404s. Treat GH
55-
# exactly like GitCode below: bounded upload, verify the actual download,
56-
# retry with a delete first (the observed bad state cleared on re-upload).
77+
# asset, 2026-07-08) and leaving a phantom asset that later 404s (0.0.75/76).
78+
# Shape (v0.0.89 lesson): upload ALL assets first, then verify with patience —
79+
# propagation overlaps instead of serializing per asset. NEVER delete on a
80+
# verify timeout: --clobber re-upload already replaces, and the eager
81+
# 404→delete loop repeatedly deleted GOOD uploads whose propagation was
82+
# merely slower than the wait window (0.0.86 incident, recurred at 18s in
83+
# v0.0.89 — 11 delete+reupload cycles across 8 assets).
5784
GH_ENABLED=0
5885
if [[ -n "${XLINGS_RES_TOKEN:-}" ]] || gh auth status >/dev/null 2>&1; then
5986
GH_ENABLED=1
6087
info "GitHub $GH_DST tag $VER"
6188
GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release view "$VER" -R "$GH_DST" >/dev/null 2>&1 \
6289
|| GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release create "$VER" -R "$GH_DST" --title "$VER" --notes "$PROJ $VER (mirror of $SRC_REPO)"
63-
for a in "${ASSETS[@]}"; do
64-
# Idempotent re-runs (incident recovery) skip assets already serving 200.
65-
if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://github.com/${GH_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
66-
info "gh $a already mirrored (200), skipping"
67-
continue
68-
fi
69-
for try in 1 2 3 4 5; do
90+
for try in 1 2 3; do
91+
# Phase 1: upload everything not yet serving (idempotent re-runs skip).
92+
pending=()
93+
for a in "${ASSETS[@]}"; do
94+
if probe "https://github.com/${GH_DST}/releases/download/${VER}/${a}"; then
95+
[[ $try == 1 ]] && info "gh $a already mirrored, skipping"
96+
continue
97+
fi
98+
pending+=("$a")
7099
GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout 300 gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber || true
71-
# Freshly uploaded assets take a few seconds to propagate to the
72-
# download CDN. Verify with patience — an eager 404→delete loop
73-
# DELETED successfully uploaded assets over and over (0.0.86
74-
# incident). Only delete + retry after propagation clearly failed.
75-
ok=""
76-
for v in 1 2 3 4 5 6; do
77-
sleep 3
78-
if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://github.com/${GH_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
79-
ok=1; break
80-
fi
81-
done
82-
[[ -n "$ok" ]] && break
83-
echo "[mirror] gh $a still not 200 ~18s after upload (try $try); delete + reupload..."
84-
GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release delete-asset "$VER" "$a" -R "$GH_DST" -y 2>/dev/null || true
85-
sleep 4
86100
done
101+
[[ ${#pending[@]} == 0 ]] && break
102+
# Phase 2: verify the batch with propagation patience.
103+
failed=()
104+
for a in "${pending[@]}"; do
105+
wait_asset "https://github.com/${GH_DST}/releases/download/${VER}/${a}" \
106+
|| failed+=("$a")
107+
done
108+
[[ ${#failed[@]} == 0 ]] && break
109+
echo "[mirror] gh not serving after patience (try $try): ${failed[*]} — re-uploading (no delete)"
87110
done
88111
else
89112
info "no github auth; skipping github mirror"
90113
fi
91114

92-
# ── GitCode (gtc, per-file retry — multi-file upload can 502 and drop files) ──
115+
# ── GitCode (gtc — multi-file upload can 502 and drop files) ─────────
116+
# Same two-phase shape. gtc can report errors on uploads that actually
117+
# succeeded (obs_callback flakiness) — the probe, not gtc's exit code, is
118+
# the source of truth.
93119
GTC_ENABLED=0
94120
if [[ -n "${GITCODE_TOKEN:-}" ]] && command -v gtc >/dev/null 2>&1; then
95121
GTC_ENABLED=1
96122
info "GitCode $GTC_DST tag $VER"
97123
gtc release create "$GTC_DST" --tag "$VER" --name "$VER" 2>/dev/null || true
98-
# Upload then verify the actual DOWNLOAD is 200 (gtc can report success yet
99-
# leave a phantom/missing asset — obs_callback flakiness), retry up to 5.
100-
for a in "${ASSETS[@]}"; do
101-
if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
102-
info "gtc $a already mirrored (200), skipping"
103-
continue
104-
fi
105-
for try in 1 2 3 4 5; do
124+
for try in 1 2 3; do
125+
pending=()
126+
for a in "${ASSETS[@]}"; do
127+
if probe "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}"; then
128+
[[ $try == 1 ]] && info "gtc $a already mirrored, skipping"
129+
continue
130+
fi
131+
pending+=("$a")
106132
timeout 300 gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" >/dev/null 2>&1 || true
107-
ok=""
108-
for v in 1 2 3 4 5 6; do
109-
sleep 3
110-
if [[ "$(curl -fsSL -o /dev/null -w '%{http_code}' -L "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}" 2>/dev/null)" == 200 ]]; then
111-
ok=1; break
112-
fi
113-
done
114-
[[ -n "$ok" ]] && break
115-
echo "[mirror] gtc $a not 200 after try $try, retrying..."; sleep 4
116133
done
134+
[[ ${#pending[@]} == 0 ]] && break
135+
failed=()
136+
for a in "${pending[@]}"; do
137+
wait_asset "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}" \
138+
|| failed+=("$a")
139+
done
140+
[[ ${#failed[@]} == 0 ]] && break
141+
echo "[mirror] gtc not serving after patience (try $try): ${failed[*]} — re-uploading"
117142
done
118143
else
119144
info "no GITCODE_TOKEN/gtc; skipping gitcode mirror"

0 commit comments

Comments
 (0)