From fa924bdf74cee31b750189b4752b44936df2df20 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 9 Aug 2026 05:36:47 +0000 Subject: [PATCH 1/6] prebuilt: key the ROCm ccache by the resolved ROCm version The ROCm legs install a rolling nightly toolchain, but the cache key only carried the gfx target and the source tag. ccache hashes compiler identity into every entry, so a cache written by a different nightly can never hit: the job paid a 64-128 MB restore, missed all 787 objects, then re-saved. Every ROCm cache in the repo has last_accessed_at == created_at. Also hash the compiler by content on Linux. The action already does this on Windows and macOS, but Linux was left on the default mtime check, and ROCm is re-extracted from the tarball on every run. --- .github/workflows/unsloth-prebuilt-rocm.yml | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index 29fd847c2a39..8fb229b814a9 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -260,12 +260,17 @@ jobs: # save: false -- the explicit actions/cache/save step at the end of the job # runs after packaging, so a failed package step does not persist a cache # for a bundle that never shipped (same pattern as the CPU/CUDA children). + # + # The ROCm version is in the key because ccache hashes compiler identity + # into every entry: a cache written by a different nightly can never hit, + # so restoring one only costs a download and a re-save. The restore-key + # prefix stops at the version for the same reason. - name: ccache uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 with: - key: rocm-windows-${{ matrix.gfx_target }}-${{ inputs.tag }} + key: rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }} restore-keys: | - rocm-windows-${{ matrix.gfx_target }} + rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }} append-timestamp: false variant: ccache max-size: 2G @@ -485,7 +490,7 @@ jobs: uses: actions/cache/save@v6 with: path: ${{ github.workspace }}\.ccache - key: ccache-rocm-windows-${{ matrix.gfx_target }}-${{ inputs.tag }}- + key: ccache-rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }}- build-ubuntu: name: linux/${{ matrix.gfx_target }} @@ -618,18 +623,25 @@ jobs: echo "ROCm environment variables set successfully" - # See the Windows job for why the key is per gfx target and why save: false. + # See the Windows job for why the key is per gfx target and ROCm version, + # and why save: false. - name: ccache uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 with: - key: rocm-linux-${{ matrix.gfx_target }}-${{ inputs.tag }} + key: rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }} restore-keys: | - rocm-linux-${{ matrix.gfx_target }} + rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }} append-timestamp: false variant: ccache max-size: 2G save: false + # The action already sets this on Windows and macOS but leaves Linux on the + # default mtime check. ROCm is re-extracted from the nightly tarball every + # run, so metadata alone is not a reliable compiler identity here. + - name: Hash the compiler by content, not mtime + run: ccache --set-config=compiler_check=content + # The parent's resolve job built the source tree (upstream base + any mix # PRs, with the build number/commit and Unsloth fingerprint baked # into cmake/build-info.cmake) and uploaded it as an artifact; extract it @@ -839,4 +851,4 @@ jobs: uses: actions/cache/save@v6 with: path: ${{ github.workspace }}/.ccache - key: ccache-rocm-linux-${{ matrix.gfx_target }}-${{ inputs.tag }}- + key: ccache-rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }}- From 4030abacf6aaf0c9c78f1d3ec80699abaf0800a0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 9 Aug 2026 08:47:00 +0000 Subject: [PATCH 2/6] prebuilt: hold one ROCm alpha per week instead of taking a new one nightly TheRock publishes a dev alpha every day, between about 03:00Z and 09:00Z, and the nightly cron fires at 20:13Z, so every scheduled run picked up a new toolchain and rebuilt ROCm from scratch. The GPU targets we ship are all covered by much older builds, so the daily churn bought nothing. rocm_version now defaults to weekly: take the newest alpha dated on or before the most recent Sunday in SF time. That is a pure function of the date, so it needs no state and no bot, and every run from Sunday to Saturday resolves the same build. Combined with the version-keyed ccache the whole week hits. latest and explicit versions still work. --- .github/workflows/unsloth-prebuilt-rocm.yml | 45 ++++++++++++++++----- .github/workflows/unsloth-prebuilt.yml | 6 +-- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index 8fb229b814a9..81c7e4224ef7 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -49,9 +49,9 @@ on: default: 'windows,ubuntu' type: string rocm_version: - description: 'TheRock ROCm version (e.g., 10.1.0a20260807) or "latest"' + description: 'TheRock ROCm version (e.g., 10.1.0a20260807), "weekly" or "latest"' required: false - default: 'latest' + default: 'weekly' type: string permissions: @@ -202,8 +202,21 @@ jobs: # TheRock publishes nightlies to the multi-arch tarball index. The # static HTML page embeds a JSON `files` array with names and mtimes. $baseUrl = "https://rocm.nightlies.amd.com/tarball-multi-arch" - if ($rocmVersion -eq "latest") { - Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" + if ($rocmVersion -eq "latest" -or $rocmVersion -eq "weekly") { + # weekly: see the Linux job. Cutoff is the most recent Sunday in SF + # time, so the whole week resolves to one build and the ccache hits. + $cutoff = "99999999" + if ($rocmVersion -eq "weekly") { + # Windows uses its own zone ids; fall back to the IANA name so this + # keeps working if the step ever moves off a Windows runner. + try { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("Pacific Standard Time") } + catch { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("America/Los_Angeles") } + $sf = [System.TimeZoneInfo]::ConvertTimeFromUtc([DateTime]::UtcNow, $tz) + $cutoff = $sf.Date.AddDays(-[int]$sf.DayOfWeek).ToString("yyyyMMdd") + Write-Host "Weekly pin: newest build dated on or before $cutoff (SF Sunday)" + } else { + Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" + } $indexHtml = (Invoke-WebRequest "$baseUrl/" -UseBasicParsing).Content $filesMatch = [regex]::Match($indexHtml, 'const files = (\[.*?\]);', [System.Text.RegularExpressions.RegexOptions]::Singleline) if (-not $filesMatch.Success) { @@ -217,10 +230,11 @@ jobs: $versionPattern = "^$([regex]::Escape($prefix))\d+\.\d+\.\d+(a|rc)\d+\.tar\.gz$" $latest = $allFiles | Where-Object { $_.name -match $versionPattern } | + Where-Object { [regex]::Match($_.name, '(\d{8})\.tar\.gz$').Groups[1].Value -le $cutoff } | Sort-Object { [regex]::Match($_.name, '(\d{8})\.tar\.gz$').Groups[1].Value } | Select-Object -Last 1 if (-not $latest) { - Write-Error "No tarball found for prefix '$prefix' at $baseUrl/" + Write-Error "No tarball found for prefix '$prefix' at or before $cutoff at $baseUrl/" exit 1 } $latestFile = $latest.name @@ -555,8 +569,19 @@ jobs: # TheRock publishes nightlies to the multi-arch tarball index. The # static HTML page embeds a JSON `files` array with names and mtimes. base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" - if [ "$rocm_version" = "latest" ]; then - echo "Auto-detecting latest ROCm version for target: $current_target" + if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then + # weekly: hold one alpha for the whole week instead of taking a new + # toolchain every night. The cutoff is the most recent Sunday in SF + # time, so every run Sun-Sat resolves the same build and the ccache + # (keyed on this version) hits all week. TheRock never backfills a + # skipped day, so a later publish cannot change an earlier pick. + cutoff=99999999 + if [ "$rocm_version" = "weekly" ]; then + cutoff="$(TZ=America/Los_Angeles date -d "-$(TZ=America/Los_Angeles date +%w) days" +%Y%m%d)" + echo "Weekly pin: newest build dated on or before $cutoff (SF Sunday)" + else + echo "Auto-detecting latest ROCm version for target: $current_target" + fi prefix="therock-dist-linux-${archive_target}-" files_json=$(curl -s "$base_url/" | tr '\n' ' ' | grep -oP 'const files = \K\[.*?\](?=\s*;)') if [ -z "$files_json" ]; then @@ -565,10 +590,10 @@ jobs: fi # Pick the newest build date and exclude sibling test archives. - latest_file=$(echo "$files_json" | jq -r --arg p "$prefix" \ - '[.[] | select(.name | test("^" + $p + "[0-9]+\\.[0-9]+\\.[0-9]+(a|rc)[0-9]+\\.tar\\.gz$"))] | sort_by(.name | capture("(?[0-9]{8})\\.tar\\.gz$").d) | last | .name // empty') + latest_file=$(echo "$files_json" | jq -r --arg p "$prefix" --arg c "$cutoff" \ + '[.[] | select(.name | test("^" + $p + "[0-9]+\\.[0-9]+\\.[0-9]+(a|rc)[0-9]+\\.tar\\.gz$")) | select((.name | capture("(?[0-9]{8})\\.tar\\.gz$").d) <= $c)] | sort_by(.name | capture("(?[0-9]{8})\\.tar\\.gz$").d) | last | .name // empty') if [ -z "$latest_file" ]; then - echo "No tarball found for prefix '$prefix' at $base_url/" + echo "No tarball found for prefix '$prefix' at or before $cutoff at $base_url/" exit 1 fi echo "Found latest file: $latest_file" diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml index 9c41d0f48932..3ccb086f3c84 100644 --- a/.github/workflows/unsloth-prebuilt.yml +++ b/.github/workflows/unsloth-prebuilt.yml @@ -58,8 +58,8 @@ on: required: false type: string rocm_version: - description: 'ROCm version (or "latest")' - default: 'latest' + description: 'ROCm version, "weekly" (newest alpha as of the last SF Sunday) or "latest"' + default: 'weekly' required: false type: string publish: @@ -507,7 +507,7 @@ jobs: source_artifact: ${{ needs.resolve.outputs.source_artifact }} matrix: ${{ needs.resolve.outputs.rocm_matrix }} operating_systems: ${{ github.event.inputs.operating_systems || 'windows,ubuntu' }} - rocm_version: ${{ github.event.inputs.rocm_version || 'latest' }} + rocm_version: ${{ github.event.inputs.rocm_version || 'weekly' }} build-macos: name: macOS From 9aefdfd933877e03bd2d12578a37edaafdc6256e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 9 Aug 2026 09:07:17 +0000 Subject: [PATCH 3/6] prebuilt: prune ROCm cache generations across versions, resolve the weekly cutoff once The cache pruner groups by the restore prefix, which is the key minus its tag suffix. Putting the ROCm version in the key made every weekly toolchain its own group, so it kept two caches per version forever and none of them could ever be restored again. Strip the version in the pruner too, so all generations of a leg share one group and only the newest two survive. The weekly cutoff was also computed per matrix leg off each leg's own clock, so a run crossing the SF Saturday-to-Sunday boundary could ship some bundles from last week's toolchain and some from this week's. Resolve it once in the parent and pass it down; a standalone call to the child still computes its own. Also unwrap the comment sentences added by this branch, per AGENTS.md. --- .github/workflows/unsloth-prebuilt-rocm.yml | 48 +++++++++++---------- .github/workflows/unsloth-prebuilt.yml | 10 ++++- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index 81c7e4224ef7..5cb1b8b89890 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -53,6 +53,11 @@ on: required: false default: 'weekly' type: string + rocm_cutoff: + description: 'For "weekly": YYYYMMDD cutoff resolved once by the parent. Blank means each leg computes its own.' + required: false + default: '' + type: string permissions: contents: read @@ -203,16 +208,17 @@ jobs: # static HTML page embeds a JSON `files` array with names and mtimes. $baseUrl = "https://rocm.nightlies.amd.com/tarball-multi-arch" if ($rocmVersion -eq "latest" -or $rocmVersion -eq "weekly") { - # weekly: see the Linux job. Cutoff is the most recent Sunday in SF - # time, so the whole week resolves to one build and the ccache hits. + # weekly: see the Linux job. Cutoff is the most recent Sunday in SF time, so the whole week resolves to one build and the ccache hits. $cutoff = "99999999" if ($rocmVersion -eq "weekly") { - # Windows uses its own zone ids; fall back to the IANA name so this - # keeps working if the step ever moves off a Windows runner. - try { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("Pacific Standard Time") } - catch { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("America/Los_Angeles") } - $sf = [System.TimeZoneInfo]::ConvertTimeFromUtc([DateTime]::UtcNow, $tz) - $cutoff = $sf.Date.AddDays(-[int]$sf.DayOfWeek).ToString("yyyyMMdd") + $cutoff = "${{ inputs.rocm_cutoff }}" + if (-not $cutoff) { + # Windows uses its own zone ids; fall back to the IANA name so this keeps working if the step ever moves off a Windows runner. + try { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("Pacific Standard Time") } + catch { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("America/Los_Angeles") } + $sf = [System.TimeZoneInfo]::ConvertTimeFromUtc([DateTime]::UtcNow, $tz) + $cutoff = $sf.Date.AddDays(-[int]$sf.DayOfWeek).ToString("yyyyMMdd") + } Write-Host "Weekly pin: newest build dated on or before $cutoff (SF Sunday)" } else { Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" @@ -275,10 +281,8 @@ jobs: # runs after packaging, so a failed package step does not persist a cache # for a bundle that never shipped (same pattern as the CPU/CUDA children). # - # The ROCm version is in the key because ccache hashes compiler identity - # into every entry: a cache written by a different nightly can never hit, - # so restoring one only costs a download and a re-save. The restore-key - # prefix stops at the version for the same reason. + # The ROCm version is in the key because ccache hashes compiler identity into every entry: a cache written by a different nightly can never hit, so restoring one only costs a download and a re-save. + # The restore-key prefix stops at the version for the same reason. - name: ccache uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 with: @@ -570,14 +574,14 @@ jobs: # static HTML page embeds a JSON `files` array with names and mtimes. base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then - # weekly: hold one alpha for the whole week instead of taking a new - # toolchain every night. The cutoff is the most recent Sunday in SF - # time, so every run Sun-Sat resolves the same build and the ccache - # (keyed on this version) hits all week. TheRock never backfills a - # skipped day, so a later publish cannot change an earlier pick. + # weekly: hold one alpha for the whole week instead of taking a new toolchain every night. + # The cutoff is the most recent Sunday in SF time, so every run Sun-Sat resolves the same build and the ccache (keyed on this version) hits all week. + # TheRock never backfills a skipped day, so a later publish cannot change an earlier pick. + # The parent resolves the cutoff once and passes it in, so all legs of one run agree even if the run crosses midnight; the fallback is for a standalone call. cutoff=99999999 if [ "$rocm_version" = "weekly" ]; then - cutoff="$(TZ=America/Los_Angeles date -d "-$(TZ=America/Los_Angeles date +%w) days" +%Y%m%d)" + cutoff="${{ inputs.rocm_cutoff }}" + [ -n "$cutoff" ] || cutoff="$(TZ=America/Los_Angeles date -d "-$(TZ=America/Los_Angeles date +%w) days" +%Y%m%d)" echo "Weekly pin: newest build dated on or before $cutoff (SF Sunday)" else echo "Auto-detecting latest ROCm version for target: $current_target" @@ -648,8 +652,7 @@ jobs: echo "ROCm environment variables set successfully" - # See the Windows job for why the key is per gfx target and ROCm version, - # and why save: false. + # See the Windows job for why the key is per gfx target and ROCm version, and why save: false. - name: ccache uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 with: @@ -661,9 +664,8 @@ jobs: max-size: 2G save: false - # The action already sets this on Windows and macOS but leaves Linux on the - # default mtime check. ROCm is re-extracted from the nightly tarball every - # run, so metadata alone is not a reliable compiler identity here. + # The action already sets this on Windows and macOS but leaves Linux on the default mtime check. + # ROCm is re-extracted from the nightly tarball every run, so metadata alone is not a reliable compiler identity here. - name: Hash the compiler by content, not mtime run: ccache --set-config=compiler_check=content diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml index 3ccb086f3c84..eb6c55ac1ff9 100644 --- a/.github/workflows/unsloth-prebuilt.yml +++ b/.github/workflows/unsloth-prebuilt.yml @@ -107,6 +107,7 @@ jobs: cuda_matrix: ${{ steps.r.outputs.cuda_matrix }} win_cuda_matrix: ${{ steps.r.outputs.win_cuda_matrix }} rocm_matrix: ${{ steps.r.outputs.rocm_matrix }} + rocm_cutoff: ${{ steps.r.outputs.rocm_cutoff }} macos_matrix: ${{ steps.r.outputs.macos_matrix }} env: GH_TOKEN: ${{ github.token }} @@ -389,6 +390,10 @@ jobs: [ -n "$GFX" ] || { echo "refusing empty gfx_target (would publish a CUDA-only release labeled CUDA + ROCm)" >&2; exit 1; } ROCM_MATRIX="$(jq -cn --arg g "$GFX" '{gfx_target: ($g | split(",") | map(gsub("^\\s+|\\s+$"; "")))}')" + # One weekly cutoff for the whole run, resolved here before fan-out. + # Each ROCm leg reads its own clock otherwise, so a run that crosses the SF Saturday-to-Sunday boundary would ship some bundles from last week's toolchain and some from this week's. + ROCM_CUTOFF="$(TZ=America/Los_Angeles date -d "-$(TZ=America/Los_Angeles date +%w) days" +%Y%m%d)" + # macOS slices are static: two fixed runners with per-slice deployment # targets. arm64 builds on macos-26 (newest Metal SDK; avoids the # M5/A19 "error compiling source" the macos-14 SDK emits) while both @@ -412,6 +417,7 @@ jobs: echo "cuda_matrix={\"include\":$CUDA_INCLUDE}" echo "win_cuda_matrix={\"include\":$WIN_CUDA_INCLUDE}" echo "rocm_matrix=$ROCM_MATRIX" + echo "rocm_cutoff=$ROCM_CUTOFF" echo "macos_matrix={\"include\":$MACOS_INCLUDE}" } >> "$GITHUB_OUTPUT" echo "Resolved $REQ -> $TAG ($COMMIT); prs=$PRS; source_artifact=${SRC_ARTIFACT:-none}; release exists=$EXISTS; only=$ONLY; gfx=$GFX" @@ -508,6 +514,7 @@ jobs: matrix: ${{ needs.resolve.outputs.rocm_matrix }} operating_systems: ${{ github.event.inputs.operating_systems || 'windows,ubuntu' }} rocm_version: ${{ github.event.inputs.rocm_version || 'weekly' }} + rocm_cutoff: ${{ needs.resolve.outputs.rocm_cutoff }} build-macos: name: macOS @@ -1084,10 +1091,11 @@ jobs: # Group by the restore-keys prefix: the key minus its -- suffix. # Newest first, so anything past $KEEP is unreachable by restore-keys. + # The ROCm version comes off too: it is part of the restore prefix, so leaving it on would make every weekly toolchain its own group and keep 2 caches per group forever, none of which can ever hit again. freed=0; deleted=0 while IFS=$'\t' read -r id created size key; do [ -z "${id:-}" ] && continue - pre="$(printf '%s' "$key" | sed -E 's/-b[0-9]+-mix-[0-9a-f]+-?$//')" + pre="$(printf '%s' "$key" | sed -E 's/-b[0-9]+-mix-[0-9a-f]+-?$//; s/-[0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+$//')" printf '%s\t%s\t%s\t%s\n' "$pre" "$created" "$id" "$size" done < "$all" | sort -t"$(printf '\t')" -k1,1 -k2,2r > "$RUNNER_TEMP/grouped.tsv" From 542e6205c89a0236b05b49f5c03bc489047dbce7 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 9 Aug 2026 09:09:03 +0000 Subject: [PATCH 4/6] prebuilt: tighten the comments added by this branch --- .github/workflows/unsloth-prebuilt-rocm.yml | 16 ++++++---------- .github/workflows/unsloth-prebuilt.yml | 6 +++--- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index 5cb1b8b89890..e86e83f84237 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -208,12 +208,12 @@ jobs: # static HTML page embeds a JSON `files` array with names and mtimes. $baseUrl = "https://rocm.nightlies.amd.com/tarball-multi-arch" if ($rocmVersion -eq "latest" -or $rocmVersion -eq "weekly") { - # weekly: see the Linux job. Cutoff is the most recent Sunday in SF time, so the whole week resolves to one build and the ccache hits. + # weekly: see the Linux job. $cutoff = "99999999" if ($rocmVersion -eq "weekly") { $cutoff = "${{ inputs.rocm_cutoff }}" if (-not $cutoff) { - # Windows uses its own zone ids; fall back to the IANA name so this keeps working if the step ever moves off a Windows runner. + # Windows uses its own zone ids; the IANA name is the fallback. try { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("Pacific Standard Time") } catch { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("America/Los_Angeles") } $sf = [System.TimeZoneInfo]::ConvertTimeFromUtc([DateTime]::UtcNow, $tz) @@ -281,8 +281,7 @@ jobs: # runs after packaging, so a failed package step does not persist a cache # for a bundle that never shipped (same pattern as the CPU/CUDA children). # - # The ROCm version is in the key because ccache hashes compiler identity into every entry: a cache written by a different nightly can never hit, so restoring one only costs a download and a re-save. - # The restore-key prefix stops at the version for the same reason. + # The ROCm version is in the key, and in the restore prefix, because ccache hashes the compiler into every entry: a cache from another nightly can never hit. - name: ccache uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 with: @@ -574,10 +573,8 @@ jobs: # static HTML page embeds a JSON `files` array with names and mtimes. base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then - # weekly: hold one alpha for the whole week instead of taking a new toolchain every night. - # The cutoff is the most recent Sunday in SF time, so every run Sun-Sat resolves the same build and the ccache (keyed on this version) hits all week. - # TheRock never backfills a skipped day, so a later publish cannot change an earlier pick. - # The parent resolves the cutoff once and passes it in, so all legs of one run agree even if the run crosses midnight; the fallback is for a standalone call. + # weekly: take the newest alpha up to the last SF Sunday, so a whole week uses one toolchain and the ccache hits. + # The parent sends one cutoff for the run, so every leg agrees; the fallback is for a standalone call. cutoff=99999999 if [ "$rocm_version" = "weekly" ]; then cutoff="${{ inputs.rocm_cutoff }}" @@ -664,8 +661,7 @@ jobs: max-size: 2G save: false - # The action already sets this on Windows and macOS but leaves Linux on the default mtime check. - # ROCm is re-extracted from the nightly tarball every run, so metadata alone is not a reliable compiler identity here. + # The action sets this on Windows and macOS but leaves Linux on mtime, and ROCm is re-extracted every run, so mtime is not a reliable compiler identity. - name: Hash the compiler by content, not mtime run: ccache --set-config=compiler_check=content diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml index eb6c55ac1ff9..13e910974611 100644 --- a/.github/workflows/unsloth-prebuilt.yml +++ b/.github/workflows/unsloth-prebuilt.yml @@ -390,8 +390,8 @@ jobs: [ -n "$GFX" ] || { echo "refusing empty gfx_target (would publish a CUDA-only release labeled CUDA + ROCm)" >&2; exit 1; } ROCM_MATRIX="$(jq -cn --arg g "$GFX" '{gfx_target: ($g | split(",") | map(gsub("^\\s+|\\s+$"; "")))}')" - # One weekly cutoff for the whole run, resolved here before fan-out. - # Each ROCm leg reads its own clock otherwise, so a run that crosses the SF Saturday-to-Sunday boundary would ship some bundles from last week's toolchain and some from this week's. + # One weekly cutoff for the whole run, before fan-out. + # Each leg reads its own clock otherwise, so a run crossing the SF Sat-to-Sun boundary would mix two toolchains in one release. ROCM_CUTOFF="$(TZ=America/Los_Angeles date -d "-$(TZ=America/Los_Angeles date +%w) days" +%Y%m%d)" # macOS slices are static: two fixed runners with per-slice deployment @@ -1091,7 +1091,7 @@ jobs: # Group by the restore-keys prefix: the key minus its -- suffix. # Newest first, so anything past $KEEP is unreachable by restore-keys. - # The ROCm version comes off too: it is part of the restore prefix, so leaving it on would make every weekly toolchain its own group and keep 2 caches per group forever, none of which can ever hit again. + # The ROCm version comes off too, else every weekly toolchain becomes its own group and keeps 2 caches that can never hit again. freed=0; deleted=0 while IFS=$'\t' read -r id created size key; do [ -z "${id:-}" ] && continue From 7e019777b223f4e8697bea0f7b73b810627eb262 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 9 Aug 2026 09:19:39 +0000 Subject: [PATCH 5/6] prebuilt: move the weekly cutoff back a day so the cutoff day is settled The cutoff was the most recent Sunday, which is the day the first run of the week executes, so the index can still gain a qualifying build after that run has read it. TheRock names a build for the next day and usually publishes the evening before, but on 2026-07-05 the Sunday build landed at 18:33 PT, over five hours after the 13:13 PT cron. That Sunday resolved 20260704 and the rest of the week resolved 20260705, so the week used two toolchains and paid two cold builds, which is what weekly exists to avoid. It is 1 of the last 12 Sundays, and four more cleared the cron by only a few hours. Use the Saturday before instead. It has had at least 17 hours to settle in every observed case, and every day Sun-Sat still maps to one cutoff. Also let the prune regex match a vanilla tag: it required -mix-, so on a build with an empty pr-set every key became its own group and nothing was ever pruned. --- .github/workflows/unsloth-prebuilt-rocm.yml | 11 ++++++----- .github/workflows/unsloth-prebuilt.yml | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index e86e83f84237..1c27f39e6501 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -217,9 +217,9 @@ jobs: try { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("Pacific Standard Time") } catch { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("America/Los_Angeles") } $sf = [System.TimeZoneInfo]::ConvertTimeFromUtc([DateTime]::UtcNow, $tz) - $cutoff = $sf.Date.AddDays(-[int]$sf.DayOfWeek).ToString("yyyyMMdd") + $cutoff = $sf.Date.AddDays(-([int]$sf.DayOfWeek + 1)).ToString("yyyyMMdd") } - Write-Host "Weekly pin: newest build dated on or before $cutoff (SF Sunday)" + Write-Host "Weekly pin: newest build dated on or before $cutoff (week of the last SF Sunday)" } else { Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" } @@ -573,13 +573,14 @@ jobs: # static HTML page embeds a JSON `files` array with names and mtimes. base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then - # weekly: take the newest alpha up to the last SF Sunday, so a whole week uses one toolchain and the ccache hits. + # weekly: take the newest alpha up to the Saturday before the last SF Sunday, so a whole week uses one toolchain and the ccache hits. + # The cutoff day must be settled before the first run that uses it: TheRock names a build for the next day and usually publishes the evening before, but it has landed as late as 18:33 PT on the named day, which would split the week. # The parent sends one cutoff for the run, so every leg agrees; the fallback is for a standalone call. cutoff=99999999 if [ "$rocm_version" = "weekly" ]; then cutoff="${{ inputs.rocm_cutoff }}" - [ -n "$cutoff" ] || cutoff="$(TZ=America/Los_Angeles date -d "-$(TZ=America/Los_Angeles date +%w) days" +%Y%m%d)" - echo "Weekly pin: newest build dated on or before $cutoff (SF Sunday)" + [ -n "$cutoff" ] || cutoff="$(TZ=America/Los_Angeles date -d "-$(( $(TZ=America/Los_Angeles date +%w) + 1 )) days" +%Y%m%d)" + echo "Weekly pin: newest build dated on or before $cutoff (week of the last SF Sunday)" else echo "Auto-detecting latest ROCm version for target: $current_target" fi diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml index 13e910974611..ffc5fb5fe11e 100644 --- a/.github/workflows/unsloth-prebuilt.yml +++ b/.github/workflows/unsloth-prebuilt.yml @@ -390,9 +390,9 @@ jobs: [ -n "$GFX" ] || { echo "refusing empty gfx_target (would publish a CUDA-only release labeled CUDA + ROCm)" >&2; exit 1; } ROCM_MATRIX="$(jq -cn --arg g "$GFX" '{gfx_target: ($g | split(",") | map(gsub("^\\s+|\\s+$"; "")))}')" - # One weekly cutoff for the whole run, before fan-out. + # One weekly cutoff for the whole run, before fan-out, set to the Saturday before the last SF Sunday so the day is already settled when the first run of the week reads it. # Each leg reads its own clock otherwise, so a run crossing the SF Sat-to-Sun boundary would mix two toolchains in one release. - ROCM_CUTOFF="$(TZ=America/Los_Angeles date -d "-$(TZ=America/Los_Angeles date +%w) days" +%Y%m%d)" + ROCM_CUTOFF="$(TZ=America/Los_Angeles date -d "-$(( $(TZ=America/Los_Angeles date +%w) + 1 )) days" +%Y%m%d)" # macOS slices are static: two fixed runners with per-slice deployment # targets. arm64 builds on macos-26 (newest Metal SDK; avoids the @@ -1095,7 +1095,7 @@ jobs: freed=0; deleted=0 while IFS=$'\t' read -r id created size key; do [ -z "${id:-}" ] && continue - pre="$(printf '%s' "$key" | sed -E 's/-b[0-9]+-mix-[0-9a-f]+-?$//; s/-[0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+$//')" + pre="$(printf '%s' "$key" | sed -E 's/-b[0-9]+(-mix-[0-9a-f]+)?-?$//; s/-[0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+$//')" printf '%s\t%s\t%s\t%s\n' "$pre" "$created" "$id" "$size" done < "$all" | sort -t"$(printf '\t')" -k1,1 -k2,2r > "$RUNNER_TEMP/grouped.tsv" From 89cc69885bdfcdaa05c057aa5d12d9fffcb1c7a4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 9 Aug 2026 09:36:10 +0000 Subject: [PATCH 6/6] prebuilt: tighten two comments added by the cutoff fix --- .github/workflows/unsloth-prebuilt-rocm.yml | 2 +- .github/workflows/unsloth-prebuilt.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index 1c27f39e6501..883f8e87c15a 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -574,7 +574,7 @@ jobs: base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then # weekly: take the newest alpha up to the Saturday before the last SF Sunday, so a whole week uses one toolchain and the ccache hits. - # The cutoff day must be settled before the first run that uses it: TheRock names a build for the next day and usually publishes the evening before, but it has landed as late as 18:33 PT on the named day, which would split the week. + # The cutoff day must be settled before the first run reads it: TheRock usually publishes the evening before, but has landed as late as 18:33 PT on the named day, which splits the week. # The parent sends one cutoff for the run, so every leg agrees; the fallback is for a standalone call. cutoff=99999999 if [ "$rocm_version" = "weekly" ]; then diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml index ffc5fb5fe11e..fc86eb9f318e 100644 --- a/.github/workflows/unsloth-prebuilt.yml +++ b/.github/workflows/unsloth-prebuilt.yml @@ -390,7 +390,7 @@ jobs: [ -n "$GFX" ] || { echo "refusing empty gfx_target (would publish a CUDA-only release labeled CUDA + ROCm)" >&2; exit 1; } ROCM_MATRIX="$(jq -cn --arg g "$GFX" '{gfx_target: ($g | split(",") | map(gsub("^\\s+|\\s+$"; "")))}')" - # One weekly cutoff for the whole run, before fan-out, set to the Saturday before the last SF Sunday so the day is already settled when the first run of the week reads it. + # One weekly cutoff for the whole run, before fan-out. See the ROCm child for why it is the Saturday before. # Each leg reads its own clock otherwise, so a run crossing the SF Sat-to-Sun boundary would mix two toolchains in one release. ROCM_CUTOFF="$(TZ=America/Los_Angeles date -d "-$(( $(TZ=America/Los_Angeles date +%w) + 1 )) days" +%Y%m%d)"