perf(ci): cache smoke downloads across runs + dedup portable + narrow detect

Sunrisepeak · Sunrisepeak · commit f60feac263a7 · 2026-06-29T19:53:14.000+08:00
Three independent levers to cut smoke-CI time (was: every run re-downloads
xim:python/make, gcc/llvm toolchains, and every compat source archive — only
~/.mcpp/registry was cached).

1. Cross-run download cache: add actions/cache for the smoke download dir
   (${runner.temp}/mcpp-smoke-cache) in smoke-full-linux AND smoke-portable,
   keyed by MCPP_VERSION + a restore-keys prefix so a partial hit survives any
   pkg/test edit (only changed archives re-download).

2. Portable within-run dedup: smoke-portable never set MCPP_INDEX_SMOKE_CACHE_DIR
   and smoke_compat_portable.sh had only a restore half — so its 5 projects each
   re-downloaded the full llvm toolchain + archives within one run. Add the
   save_smoke_cache half (+ pbuild wrapper routing all 5 builds through it) and
   wire MCPP_INDEX_SMOKE_CACHE_DIR in the portable job.

3. Narrow detect: smoke-full-linux and smoke-portable run disjoint scripts on
   disjoint platforms, so split the single full=true into full_linux/full_portable.
   A change to one platform's harness no longer triggers the other's suite
   (a smoke_compat_portable.sh edit skips the ~12-min Linux job; a Linux smoke
   script edit skips win+mac). Common harness, unclassified test changes, and
   pkgs without an example still fall back to BOTH — no coverage is dropped.
   Classification verified against 9 representative changed-file sets.

Diagnosis: see memory mcpp-index-ci-repeated-downloads. Python is pulled only by
compat.xcb (build-dep, transitively via x11/glfw), Linux-only.
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
@@ -103,6 +103,8 @@ jobs:
     outputs:
       examples: ${{ steps.f.outputs.examples }}
       full: ${{ steps.f.outputs.full }}
+      full_linux: ${{ steps.f.outputs.full_linux }}
+      full_portable: ${{ steps.f.outputs.full_portable }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -111,30 +113,51 @@ jobs:
         run: sudo apt-get install -y --no-install-recommends jq
       - id: f
         run: |
-          full=false
+          # The two heavy suites run on disjoint platforms and from disjoint
+          # scripts: smoke-full-linux runs the smoke_compat_{core,imgui,archive,
+          # imgui_window}.sh + smoke_imgui_module.sh scripts (Linux); smoke-portable
+          # runs ONLY smoke_compat_portable.sh (windows+macos). So a change to one
+          # platform's harness needn't trigger the other's. We classify each changed
+          # file into full_linux / full_portable; unclassified test changes, the
+          # common harness, and pkgs without an example fall back to BOTH (no
+          # coverage is ever silently dropped).
+          full_linux=false
+          full_portable=false
           examples='[]'
           event="${{ github.event_name }}"
           if [ "$event" != "pull_request" ]; then
             # push to main / nightly schedule / manual dispatch → full regression
-            full=true
+            full_linux=true; full_portable=true
           else
             git fetch -q origin "${{ github.base_ref }}"
             changed=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD")
             echo "changed files:"; echo "$changed" | sed 's/^/  /'
-            # scaffolding / CI change → full regression
-            if echo "$changed" | grep -qE '^(tests/run_example\.sh|tests/smoke_.*\.sh|tests/check_mirror_urls\.lua|tests/list_cn_urls\.lua|\.github/workflows/validate\.yml)$'; then
-              full=true
-            fi
             sel=""
             while IFS= read -r file; do
+              [ -z "$file" ] && continue
               case "$file" in
+                # Common harness / the workflow itself → both platforms.
+                tests/run_example.sh|tests/check_mirror_urls.lua|tests/list_cn_urls.lua|.github/workflows/validate.yml)
+                  full_linux=true; full_portable=true ;;
+                # Portable suite script runs only in smoke-portable (win+mac).
+                tests/smoke_compat_portable.sh)
+                  full_portable=true ;;
+                # Linux-only smoke scripts run only in smoke-full-linux.
+                tests/smoke_compat_core.sh|tests/smoke_compat_imgui.sh|tests/smoke_compat_archive.sh|tests/smoke_compat_imgui_window.sh|tests/smoke_imgui_module.sh)
+                  full_linux=true ;;
+                # Example projects are handled by the smoke-examples fast path.
+                tests/examples/*)
+                  : ;;
+                # Any other tests/ change (new helper / new smoke script) → both (safe).
+                tests/*)
+                  full_linux=true; full_portable=true ;;
                 pkgs/*/*.lua)
                   b=$(basename "$file" .lua); b=${b#compat.}
                   if [ -d "tests/examples/$b" ]; then
                     sel="$sel$b"$'\n'
                   else
-                    # changed pkg with no example yet → cannot narrow, run full
-                    full=true
+                    # changed pkg with no example yet → cannot narrow, run both
+                    full_linux=true; full_portable=true
                   fi
                   ;;
               esac
@@ -143,9 +166,13 @@ jobs:
               examples=$(printf '%s' "$sel" | grep -v '^$' | sort -u | jq -R . | jq -sc .)
             fi
           fi
+          full=false
+          { [ "$full_linux" = true ] || [ "$full_portable" = true ]; } && full=true
           echo "full=$full" >> "$GITHUB_OUTPUT"
+          echo "full_linux=$full_linux" >> "$GITHUB_OUTPUT"
+          echo "full_portable=$full_portable" >> "$GITHUB_OUTPUT"
           echo "examples=$examples" >> "$GITHUB_OUTPUT"
-          echo "=> full=$full examples=$examples"
+          echo "=> full_linux=$full_linux full_portable=$full_portable examples=$examples"
 
   # ── Fast path: per-changed-package example projects (Linux) ───────────
   smoke-examples:
@@ -186,7 +213,7 @@ jobs:
   # ── Full regression (legacy whole-suite smoke) ────────────────────────
   smoke-full-linux:
     needs: detect
-    if: needs.detect.outputs.full == 'true'
+    if: needs.detect.outputs.full_linux == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -197,6 +224,18 @@ jobs:
           key: mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}-${{ hashFiles('pkgs/**/*.lua', 'tests/**', '.github/workflows/validate.yml') }}
           restore-keys: |
             mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}-
+      # Persist the smoke download cache (toolchains + compat source archives)
+      # ACROSS runs. Without this, every run re-downloads xim:python/make, the
+      # gcc/llvm toolchains, and every source tarball. restore-keys gives a
+      # partial hit even when pkgs change, so only new/changed archives download;
+      # save_smoke_cache (in the smoke scripts) repopulates the dir each run.
+      - name: Cache smoke downloads
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/mcpp-smoke-cache
+          key: smoke-dl-${{ runner.os }}-${{ env.MCPP_VERSION }}-${{ hashFiles('pkgs/**/*.lua') }}
+          restore-keys: |
+            smoke-dl-${{ runner.os }}-${{ env.MCPP_VERSION }}-
       - name: Download mcpp
         run: |
           curl -L -fsS -o mcpp.tar.gz \
@@ -230,7 +269,7 @@ jobs:
 
   smoke-portable:
     needs: detect
-    if: needs.detect.outputs.full == 'true'
+    if: needs.detect.outputs.full_portable == 'true'
     name: smoke-${{ matrix.platform }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
@@ -259,6 +298,18 @@ jobs:
           key: mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}-${{ hashFiles('pkgs/**/*.lua', 'tests/**', '.github/workflows/validate.yml') }}
           restore-keys: |
             mcpp-registry-${{ runner.os }}-${{ env.MCPP_VERSION }}-
+      # Persist the smoke download cache across runs (same rationale as
+      # smoke-full-linux). Previously the portable job never set
+      # MCPP_INDEX_SMOKE_CACHE_DIR at all, so its 5 projects each re-downloaded
+      # the full llvm toolchain + compat archives WITHIN one run; now they share
+      # one dir, persisted between runs too.
+      - name: Cache smoke downloads
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/mcpp-smoke-cache
+          key: smoke-dl-${{ runner.os }}-${{ env.MCPP_VERSION }}-${{ hashFiles('pkgs/**/*.lua') }}
+          restore-keys: |
+            smoke-dl-${{ runner.os }}-${{ env.MCPP_VERSION }}-
       - name: Download mcpp
         shell: bash
         env:
@@ -292,6 +343,9 @@ jobs:
         shell: bash
         env:
           MCPP_INDEX_MIRROR: GLOBAL
+          # Enables the within-run + cross-run download cache (copy/save_smoke_cache).
+          MCPP_INDEX_SMOKE_CACHE_DIR: ${{ runner.temp }}/mcpp-smoke-cache
         run: |
+          mkdir -p "$MCPP_INDEX_SMOKE_CACHE_DIR"
           "$MCPP" --version
           bash tests/smoke_compat_portable.sh
diff --git a/tests/smoke_compat_portable.sh b/tests/smoke_compat_portable.sh
@@ -97,6 +97,24 @@ copy_smoke_cache() {
         -exec cp -f {} .mcpp/.xlings/data/runtimedir/ \;
 }
 
+# Save half (was missing): stash this project's freshly-downloaded toolchain /
+# source archives back into SMOKE_CACHE_DIR so the NEXT project in this run (and,
+# when SMOKE_CACHE_DIR is on actions/cache, the next CI run) restores them instead
+# of re-downloading. Mirrors tests/smoke_compat_imgui.sh. `cp -n` = no-clobber.
+save_smoke_cache() {
+    [[ -n "$SMOKE_CACHE_DIR" && -d .mcpp/.xlings/data ]] || return 0
+    mkdir -p "$SMOKE_CACHE_DIR"
+    find .mcpp/.xlings/data -type f \
+        \( -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.zip' \) \
+        -exec cp -n {} "$SMOKE_CACHE_DIR"/ \; 2>/dev/null || true
+}
+
+# Build + populate the cache, so every project's downloads are reused.
+pbuild() {
+    "$MCPP_BIN_POSIX" build
+    save_smoke_cache
+}
+
 write_build_ldflags() {
     case "$platform" in
         Linux)
@@ -196,7 +214,7 @@ TEST(CompatPortableCore, UpstreamHeadersAndRuntime) {
     EXPECT_EQ(static_cast<khronos_uint32_t>(1), 1u);
 }
 EOF
-"$MCPP_BIN_POSIX" build
+pbuild
 "$MCPP_BIN_POSIX" run
 
 make_project "compat-portable-archive-smoke"
@@ -232,7 +250,7 @@ int main() {
     return 0;
 }
 EOF
-"$MCPP_BIN_POSIX" build
+pbuild
 "$MCPP_BIN_POSIX" run
 
 make_project "compat-portable-compression-smoke"
@@ -356,7 +374,7 @@ int main() {
     return 0;
 }
 EOF
-"$MCPP_BIN_POSIX" build
+pbuild
 "$MCPP_BIN_POSIX" run
 
 make_project "compat-portable-imgui-glfw-smoke"
@@ -411,7 +429,7 @@ int main() {
            IMGUI_VERSION_NUM >= 19200 ? 0 : 1;
 }
 EOF
-"$MCPP_BIN_POSIX" build
+pbuild
 "$MCPP_BIN_POSIX" run
 
 # compat.openblas — Windows-only build-AND-run. This is the only place that
@@ -447,7 +465,7 @@ int main() {
     return 0;
 }
 EOF
-    "$MCPP_BIN_POSIX" build
+    pbuild
     # `mcpp run` also prepends the source bin/ to PATH, so it alone would not
     # prove the DLL was deployed. Run the produced .exe DIRECTLY from a neutral
     # CWD: Windows searches the executable's own directory first, so a clean exit
