diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c55847d..643ff719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,42 +13,31 @@ on: merge_group: branches: [main] types: [checks_requested] - # Manual trigger for on-demand E2E runs on the self-hosted GPU runners, without - # waiting on a push/PR or the full gate. Dispatch against any ref - # (`gh workflow run ci.yml --ref -f platform=... -f tier=...`) — GitHub - # runs THAT ref's copy of this file, so the E2E jobs it selects live on the ref. - # A dispatch skips build-and-test (see its `if:`) for a fast loop. This trigger - # must exist on the default branch for the workflow to be dispatchable at all. + # Manual trigger for an on-demand mock (GitHub-hosted) E2E run, without waiting + # on a push/PR or the full gate. Dispatch against any ref + # (`gh workflow run ci.yml --ref -f platform=mock`) — GitHub runs THAT + # ref's copy of this file. A dispatch skips build-and-test (see the e2e `if:`) + # for a fast loop. This trigger must exist on the default branch for the + # workflow to be dispatchable at all. The self-hosted GPU lanes moved to their + # own workflow (e2e-selfhosted.yml) — dispatch THAT for GPU/Strix runs. workflow_dispatch: inputs: platform: description: Which runner(s) to target type: choice default: all - options: [all, mock, app-dev-gpu, strix-ubuntu, strix-windows] + options: [all, mock] # No tier input: the harness resolves pass/xfail/skip per scenario, so a # platform runs one job covering everything applicable to it. - # Scenario-name regex forwarded to the cucumber harness - # (`cargo xtask e2e -- --name `) so a dispatch can run only the - # selected scenarios. Empty = full suite. - name_filter: - description: "Scenario-name regex (cucumber --name); empty = full suite" - type: string - default: "" - # Opt a manual dispatch into the expensive `@nightly` scenarios (large-model - # serve, cold devel install) that the per-PR run skips. Off by default so a - # normal dispatch is unchanged; combine with name_filter to probe just one. - include_nightly: - description: "Include @nightly scenarios (large-model serve, cold install)" - type: boolean - default: false concurrency: - # Manual dispatches get a UNIQUE group (run_id) so a run that gets stuck — e.g. - # a job queued on a temporarily offline self-hosted runner, which GitHub cannot - # cancel — never holds the shared group and blocks later dispatches. push / PR / - # merge_group keep the shared per-ref group so a new commit still supersedes the - # previous in-flight run. + # Manual dispatches get a UNIQUE group (run_id) so a stuck run never holds the + # shared group and blocks later dispatches. push / PR / merge_group keep the + # shared per-ref group so a new commit still supersedes the previous in-flight + # run. The offline-self-hosted-runner stall that motivated this is now + # structurally avoided: the self-hosted GPU lanes live in e2e-selfhosted.yml + # with their own group, so an uncancellable queued job can never stall THIS + # workflow's merge-required checks. group: >- ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'shared' }} @@ -641,11 +630,13 @@ jobs: BASE: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || github.event.merge_group.base_sha }} run: cargo xtask verify-commits --base "$BASE" --require-verified - # Hardware (real AMD GPU) smoke tests run on dedicated self-hosted runners — - # see the e2e-gpu, e2e-gpu-strix-ubuntu, and e2e-gpu-strix-windows jobs below. + # Hardware (real AMD GPU) smoke tests run on dedicated self-hosted runners in a + # SEPARATE workflow, e2e-selfhosted.yml (jobs e2e-gpu, e2e-gpu-strix-ubuntu, + # e2e-gpu-strix-windows). They live there — not here — so a job queued on an + # offline self-hosted runner (which GitHub cannot cancel) can never hold this + # workflow's concurrency group and stall its merge-required checks (EAI-7548). # They are non-blocking (continue-on-error) and fork-safe (self-hosted runners - # do not execute untrusted fork PRs). See docs/ci-hardware-testing.md for the - # design. + # do not execute untrusted fork PRs). See docs/ci-hardware-testing.md. # ── E2E tests (cucumber-rs) ─────────────────────────────────────────────── # @@ -720,520 +711,15 @@ jobs: name: e2e-report path: tests/e2e-cucumber/results/ - # app-dev MI300X (Instinct data-center GPU, `amd-gpu` label). Non-blocking. - # One job runs every applicable scenario (tiers collapsed). Cost drivers: - # `install sdk` re-running per scenario (isolated data dirs), several vLLM - # cold-starts, and — since it was added — the large-model (Qwen3.6-27B) serve, - # whose ~54 GiB load plus the post-serve VRAM-drain wait pushed a full run past - # the old 35min cap (it was cancelled at 35m17s). Cap raised to 90min so the job - # COMPLETES and writes platform.json (a cancelled job produces none → no grid - # column). The heaviest scenarios are being moved to a nightly-only tag so the - # per-PR run stays short; see the `@nightly` handling in xtask/the harness. - e2e-gpu: - name: E2E tests (GPU) - timeout-minutes: 90 - runs-on: [self-hosted, linux, amd-gpu] - needs: [changes, build-and-test] - # See `e2e`: dispatch tolerates skipped build-and-test; app-dev-gpu. - if: >- - always() - && needs.changes.result == 'success' - && ( - (github.event_name != 'workflow_dispatch' - && needs.build-and-test.result == 'success' - && needs.changes.outputs.heavy == 'true') - || (github.event_name == 'workflow_dispatch' - && (inputs.platform == 'all' || inputs.platform == 'app-dev-gpu')) - ) - continue-on-error: true - env: - # Bound serve readiness below the 35-min job cap so a serve that never comes - # ready fails the scenario with a real error instead of hanging until the - # job is cancelled. 300s is ample for a real MI300X vLLM cold-start; - # per-scenario overrides in expectations.toml / a `@serve-timeout` tag adjust - # it (shorter for known bugs, longer for large models). - E2E_SERVE_TIMEOUT_SECS: "300" - # Opt-in @nightly scenarios on a manual dispatch (default off). The nightly - # workflow sets this unconditionally; here it lets a scoped dispatch confirm - # a single nightly scenario (e.g. the 27B serve) without the full nightly run. - E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # Reclaim the GPU before running: a serve leaked by a killed/timed-out prior - # run (its Drop teardown never executed) can keep an engine process spinning - # on the GPU, starving this job's serves until it hits the timeout. Kill only - # e2e leftovers — scoped to /tmp/rocm-e2e-* and the e2e-target/e2e-shared - # trees — never the runner or any /workload manual-testing processes. - - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - # NOTE: no unanchored `pkill -f 'vulkan/llama-server'` — the lemonade - # Vulkan assistant an e2e scenario spawns lives under /tmp/rocm-e2e-* - # and is already caught by the first line; an unanchored pattern would - # also kill a legitimate /workload manual-testing serve on this shared - # self-hosted runner. - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" - - # GPU preflight: fail fast (~90s) instead of hanging to the job cap when the - # GPU is missing, the driver is wedged, or VRAM is still saturated by a - # leftover serve. A BOUNDED POLL, not a one-shot check: transient contention - # (e.g. the reclaim step's kills still draining VRAM) self-heals within - # seconds, so we retry up to a ceiling and succeed the moment the GPU is both - # responsive AND has enough free VRAM. Only a genuinely absent/wedged/held - # GPU reaches the ceiling and fails — with a per-reason message. - - name: GPU preflight (bounded wait for an available GPU) - run: | - # A serve here needs most of the card; require a generous free-VRAM - # floor so a leftover serve (which the reclaim step should have killed) - # is caught, while normal baseline (~300 MB used) passes immediately. - MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" - CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" - min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) - deadline=$(( SECONDS + CEILING_SECS )) - reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" - while [ "$SECONDS" -lt "$deadline" ]; do - # rocm-smi itself can hang on a wedged driver — bound it with timeout. - out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } - # Parse the byte value AFTER the colon; the line prefix "GPU[0]" would - # otherwise make a naive first-number match pick up the "0". - total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) - used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) - if [ -z "$total" ] || [ -z "$used" ]; then - reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" - sleep 5; continue - fi - free=$(( total - used )) - if [ "$free" -ge "$min_free" ]; then - echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." - exit 0 - fi - reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" - echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" - sleep 5 - done - echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" - exit 1 - - # cache: false — on this self-hosted runner we persist the build cache - # ourselves via CARGO_TARGET_DIR (below). The action's built-in - # Swatinem/rust-cache otherwise tries to SAVE the large target dir to - # GitHub's cache service in a post-step (slow/hangs) and its cleanup wipes - # the local target. - - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 - with: - cache: false - - # `actions/checkout` runs `git clean -ffdx`, which deletes the gitignored - # `target/` inside the repo every job → a full ~15min rebuild each run. - # Point CARGO_TARGET_DIR at a sibling of the checkout ($RUNNER_WORKSPACE is - # the checkout's parent — untouched by git clean and persistent between jobs - # on a self-hosted runner), so cargo rebuilds incrementally. - - name: Run E2E tests on GPU hardware - run: | - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - # Share heavy immutable artifacts (TheRock runtimes ~3.3GB, HF weights, - # vLLM venv) across scenarios so they download once per runner, not per - # scenario. Persistent path; service state stays isolated per scenario. - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Share uv's wheel download/build cache so `rocm install sdk` is a warm - # ~34s per scenario instead of a cold ~160s (measured on MI300X). Kept - # OFF the RUNNER_WORKSPACE Longhorn PVC (near-full) and on the roomy `/` - # overlay; ~23GB, one cold fill per pod-life. Not git-cleaned (outside - # the checkout) and outside the reclaim step's /tmp/rocm-e2e-* glob. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" - # Share ONE installed managed runtime across the serve/chat scenarios so - # `rocm install sdk` runs once per runner, not once per scenario (the - # per-scenario install count — each a multi-GiB TheRock SDK whose probe - # unpacks an ~8.8 GiB devel tarball — is what blew the time cap). The - # "a managed runtime is active" precondition symlinks each scenario's - # data/runtimes here (see use_shared_runtimes); clean-slate scenarios - # stay isolated. Persisted across runs on RUNNER_WORKSPACE, so after the - # first run ever the pre-warm below is a no-op. - # - # CRITICAL: the shared dir IS the pre-warm's own `data/runtimes`, and we - # NEVER move it. `install sdk` bakes ABSOLUTE paths (install_root, - # python_executable) into the runtime manifest; a post-install `mv` would - # leave those pointing at a deleted location and every serve would fail - # instantly (observed on run 29320025393). Installing in place keeps the - # baked paths valid, and each scenario's data/runtimes symlink resolves to - # this same real tree. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # Build the rocm binary ONCE and reuse it for both the pre-warm and the - # suite (via ROCM_CLI_BINARY) so xtask does not rebuild. Honors - # CARGO_TARGET_DIR set above. - cargo build --release -p rocm - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - - # Pre-warm the shared runtime ONCE, SERIALLY, before the suite — never - # lazily inside a concurrent scenario (two multi-GiB installs racing the - # same dir). Skipped once the tree is populated (it persists across runs). - # Uses the prebuilt binary (no cargo run --release) and the shared uv + HF - # caches. Installs directly into the persistent pre-warm data dir (no mv), - # so the manifest's absolute install_root stays valid for every scenario. - if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then - echo "pre-warming shared runtime (first run on this runner)…" - mkdir -p "$prewarm"/{data,config,cache} - ROCM_CLI_CONFIG_DIR="$prewarm/config" \ - ROCM_CLI_DATA_DIR="$prewarm/data" \ - ROCM_CLI_CACHE_DIR="$prewarm/cache" \ - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - "$ROCM_CLI_BINARY" install sdk - if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then - echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" - else - echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 - fi - else - echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" - fi - - # Optional scenario-name filter for a scoped dispatch — lets a manual - # run select only the large-model scenario instead of the whole suite. - NAME_FILTER="${{ github.event.inputs.name_filter }}" - if [ -n "$NAME_FILTER" ]; then - echo "name filter active: $NAME_FILTER" - cargo xtask e2e -- --name "$NAME_FILTER" - else - cargo xtask e2e - fi - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e2e-gpu-report - path: tests/e2e-cucumber/results/ - - # Second AMD GPU architecture: the Strix Halo (gfx1151) Ubuntu runner, targeted - # by the `strix-halo` label (app-dev-gpu carries `amd-gpu`, not `strix-halo`). - # Non-blocking while this hardware is proven out. - e2e-gpu-strix-ubuntu: - name: E2E tests (Strix Halo, Ubuntu) - # 35min: see e2e-gpu — one collapsed job runs all serves + per-scenario - # install sdk; the cap must exceed the run so the job writes platform.json. - timeout-minutes: 35 - # `native` disambiguates the two Linux Strix runners: the WSL host also - # carries `strix-halo`, but the paths below exist only on the native one. - runs-on: [self-hosted, linux, strix-halo, native] - needs: [changes, build-and-test] - # See `e2e`: dispatch tolerates skipped build-and-test; strix-ubuntu. - if: >- - always() - && needs.changes.result == 'success' - && ( - (github.event_name != 'workflow_dispatch' - && needs.build-and-test.result == 'success' - && needs.changes.outputs.heavy == 'true') - || (github.event_name == 'workflow_dispatch' - && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')) - ) - continue-on-error: true - # On this runner `/`, `/home/ubuntu`, and `/tmp` are ALL on a full root - # partition; only /home/ubuntu/actions-runner (a 1.7T nvme) has space. So - # EVERYTHING the job writes must land on the nvme. Point HOME there (catches - # ~/.cache/pip, ~/.config, and any other $HOME writer — pip's cache under the - # real /home/ubuntu is what previously failed `install sdk` with ENOSPC), - # plus the toolchain, temp, and pip cache. The rustup bootstrap still uses - # --no-modify-path so it doesn't touch $HOME/.profile. These paths are - # specific to that host, which is why `runs-on` pins `native` above. - env: - HOME: /home/ubuntu/actions-runner/e2e-home - CARGO_HOME: /home/ubuntu/actions-runner/.cargo - RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup - TMPDIR: /home/ubuntu/actions-runner/tmp - PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache - E2E_SERVE_TIMEOUT_SECS: "300" - # Match the MI300X dispatch path: opt into the platform-adaptive large-model - # scenario only when the manual include_nightly input is enabled. - E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs - run: | - mkdir -p /home/ubuntu/actions-runner/e2e-home /home/ubuntu/actions-runner/tmp /home/ubuntu/actions-runner/pip-cache - # Reclaim the GPU from any serve leaked by a killed/timed-out prior run - # (see e2e-gpu). Scoped to e2e leftovers only. - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "prepared + reclaimed" - - # GPU preflight: bounded wait so a missing/wedged/held GPU fails fast (~90s) - # instead of hanging to the job cap. Transient contention (VRAM still - # draining from the reclaim above) self-heals within the ceiling. See the - # e2e-gpu job for the rationale. gfx1151 shares system memory, so the free - # floor is smaller than the Instinct card but still catches a leftover serve. - - name: GPU preflight (bounded wait for an available GPU) - run: | - MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" - CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" - min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) - deadline=$(( SECONDS + CEILING_SECS )) - reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" - while [ "$SECONDS" -lt "$deadline" ]; do - out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } - total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) - used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) - if [ -z "$total" ] || [ -z "$used" ]; then - reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" - sleep 5; continue - fi - free=$(( total - used )) - if [ "$free" -ge "$min_free" ]; then - echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." - exit 0 - fi - reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" - echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" - sleep 5 - done - echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" - exit 1 - - # Bootstrap rustup ourselves with --no-modify-path so it never writes to - # $HOME/.profile (setup-rust-toolchain doesn't expose that flag). - # rust-toolchain.toml pins the exact toolchain, installed on first cargo - # use. Idempotent. - - name: Ensure Rust toolchain - run: | - if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then - curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ - | sh -s -- -y --no-modify-path --default-toolchain none - fi - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - - - name: Run E2E tests on Strix Halo - run: | - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Share ONE installed managed runtime across serve/chat scenarios, and - # PRE-WARM it in place before the suite (mirrors e2e-gpu). This is not an - # optimization — it is REQUIRED for correctness. `install sdk` bakes - # ABSOLUTE paths (install_root, python_executable, rocm_sdk.*) into the - # runtime manifest. If the first scenario installs into its own isolated - # /tmp/rocm-e2e-XXXX data dir and only the registry is shared onward, those - # baked paths point at that scenario's temp dir — which is deleted when the - # scenario ends. Every later serve then sees `status=unusable (install root - # is missing)` and fails (diagnosed on the box 2026-07-15). Installing in - # place, directly into the persistent shared dir, keeps the baked paths - # valid for all scenarios and for the end-of-run version probe. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # Build the rocm binary once; reuse for pre-warm + suite so xtask doesn't - # rebuild. - cargo build --release -p rocm - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - - # Pre-warm once, serially, in place (no mv/symlink). Skipped once the tree - # is populated (persists across runs on RUNNER_WORKSPACE). - if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then - echo "pre-warming shared runtime (first run on this runner)…" - mkdir -p "$prewarm"/{data,config,cache} - ROCM_CLI_CONFIG_DIR="$prewarm/config" \ - ROCM_CLI_DATA_DIR="$prewarm/data" \ - ROCM_CLI_CACHE_DIR="$prewarm/cache" \ - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - "$ROCM_CLI_BINARY" install sdk - if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then - echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" - else - echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 - fi - else - echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" - fi - - # Optional scenario-name filter for a scoped manual dispatch. - NAME_FILTER="${{ github.event.inputs.name_filter }}" - if [ -n "$NAME_FILTER" ]; then - echo "name filter active: $NAME_FILTER" - cargo xtask e2e -- --name "$NAME_FILTER" - else - cargo xtask e2e - fi - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e2e-gpu-strix-ubuntu-report - path: tests/e2e-cucumber/results/ - - # First real Windows GPU coverage: the Strix Halo Windows 11 runner. The - # existing windows-build-and-test uses GitHub-hosted windows-latest, which has - # no GPU. Non-blocking so it never gates the PR. - e2e-gpu-strix-windows: - name: E2E tests (Strix Halo, Windows) - # 35min: see e2e-gpu — one collapsed job runs all serves + per-scenario - # install sdk; the cap must exceed the run so the job writes platform.json. - timeout-minutes: 35 - runs-on: [self-hosted, windows, strix-halo, native] - needs: [changes, build-and-test] - # See `e2e`: dispatch tolerates skipped build-and-test; strix-windows. - if: >- - always() - && needs.changes.result == 'success' - && ( - (github.event_name != 'workflow_dispatch' - && needs.build-and-test.result == 'success' - && needs.changes.outputs.heavy == 'true') - || (github.event_name == 'workflow_dispatch' - && (inputs.platform == 'all' || inputs.platform == 'strix-windows')) - ) - continue-on-error: true - env: - # Match the Linux Strix dispatch path: opt into the platform-adaptive - # large-model scenario only when the manual input is enabled. - E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # setup-rust-toolchain runs an internal bash script, which this Windows - # runner lacks (bash: command not found). Bootstrap rustup with the - # PowerShell-native installer instead; idempotent, so it only downloads on - # a runner that doesn't already have the toolchain. Use `powershell` - # (Windows PowerShell 5.1, always present) rather than `pwsh` (PowerShell 7), - # which this self-hosted runner does not have installed. - # Reclaim the GPU from any E2E serve leaked by a killed/timed-out prior run - # (see e2e-gpu). PowerShell (5.1) equivalent; best-effort. - - name: Reclaim GPU from stray E2E processes - shell: powershell - run: | - Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | - Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | - ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } - Write-Host "reclaimed" - - # GPU preflight (best-effort on Windows): bounded wait for the GPU to be - # available so a wedged/held GPU fails fast instead of hanging to the cap. - # BEST-EFFORT because the Windows ROCm GPU query tool isn't verified here — - # if no rocm-smi is found we WARN and continue rather than false-fail a - # working runner. When rocm-smi IS present we poll free VRAM the same way as - # the Linux jobs and fail only after the ceiling. - - name: GPU preflight (bounded wait for an available GPU) - shell: powershell - run: | - $minFreeGiB = if ($env:GPU_PREFLIGHT_MIN_FREE_GIB) { [int]$env:GPU_PREFLIGHT_MIN_FREE_GIB } else { 8 } - $ceilingSecs = if ($env:GPU_PREFLIGHT_CEILING_SECS) { [int]$env:GPU_PREFLIGHT_CEILING_SECS } else { 90 } - if (-not (Get-Command rocm-smi -ErrorAction SilentlyContinue)) { - Write-Host "rocm-smi not found on this Windows runner; skipping GPU preflight (best-effort)." - exit 0 - } - $minFree = [int64]$minFreeGiB * 1GB - $deadline = (Get-Date).AddSeconds($ceilingSecs) - $reason = "rocm-smi never returned usable VRAM figures" - while ((Get-Date) -lt $deadline) { - $out = (rocm-smi --showmeminfo vram 2>$null | Out-String) - $total = ([regex]::Matches($out, 'VRAM Total Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value - $used = ([regex]::Matches($out, 'VRAM Total Used Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value - if (-not $total -or -not $used) { - $reason = "rocm-smi returned no VRAM figures (no AMD GPU detected)" - Start-Sleep -Seconds 5; continue - } - $free = [int64]$total - [int64]$used - if ($free -ge $minFree) { - Write-Host "GPU ready: $([math]::Floor($free/1GB)) GiB free (>= $minFreeGiB GiB)." - exit 0 - } - $reason = "VRAM never dropped below the floor: only $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB) - a serve is likely still holding the GPU" - Write-Host "waiting: $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB)..." - Start-Sleep -Seconds 5 - } - Write-Host "::error::GPU preflight failed after ${ceilingSecs}s: $reason" - exit 1 - - - name: Ensure Rust toolchain (PowerShell) - shell: powershell - run: | - if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) { - Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe - # --default-toolchain none: rust-toolchain.toml pins the exact - # version (1.96.0 + components), auto-installed on first cargo use. - & $env:TEMP\rustup-init.exe -y --default-toolchain none - "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Append - } - - - name: Run E2E tests on Strix Halo Windows - shell: powershell - run: | - # Share ONE installed managed runtime across serve/chat scenarios, and - # PRE-WARM it in place before the suite (mirrors e2e-gpu / strix-ubuntu). - # REQUIRED for correctness, not just speed: `install sdk` bakes ABSOLUTE - # paths (install_root, python_executable, rocm_sdk.*, .rocm-cli-runtime.json) - # into the runtime manifest. If the first scenario installs into its own - # isolated temp data dir and only the registry is shared onward, those baked - # paths point at that scenario's dir — deleted when the scenario ends — so - # every later serve sees status=unusable and fails (diagnosed on the Linux - # box 2026-07-15; the Windows scenario-8 cold-download failure is the same - # class). Install in place so the baked paths stay valid for all scenarios. - $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm" - $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" - - # Build the rocm binary once; reuse for pre-warm + suite so xtask doesn't - # rebuild. This job does not set CARGO_TARGET_DIR, so the binary lands in - # the default target\release (fall back to it when the env var is unset). - cargo build --release -p rocm - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" } - $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe" - - # Pre-warm once, in place (no move/symlink). Skipped once the tree is - # populated (persists across runs on RUNNER_WORKSPACE). - if (-not (Test-Path "$env:E2E_SHARED_RUNTIMES_DIR\registry")) { - Write-Host "pre-warming shared runtime (first run on this runner)..." - New-Item -ItemType Directory -Force -Path "$prewarm\data","$prewarm\config","$prewarm\cache" | Out-Null - $env:ROCM_CLI_CONFIG_DIR = "$prewarm\config" - $env:ROCM_CLI_DATA_DIR = "$prewarm\data" - $env:ROCM_CLI_CACHE_DIR = "$prewarm\cache" - & $env:ROCM_CLI_BINARY install sdk - Remove-Item Env:\ROCM_CLI_CONFIG_DIR,Env:\ROCM_CLI_DATA_DIR,Env:\ROCM_CLI_CACHE_DIR -ErrorAction SilentlyContinue - if (Test-Path "$env:E2E_SHARED_RUNTIMES_DIR\registry") { - Write-Host "shared runtime pre-warmed at $env:E2E_SHARED_RUNTIMES_DIR" - } else { - Write-Host "pre-warm did not produce a runtimes registry; scenarios will install their own" - } - } else { - Write-Host "shared runtime already present at $env:E2E_SHARED_RUNTIMES_DIR - skipping pre-warm" - } - - # Optional scenario-name filter for a scoped manual dispatch. - $nameFilter = "${{ github.event.inputs.name_filter }}" - if ($nameFilter) { - Write-Host "name filter active: $nameFilter" - cargo xtask e2e -- --name "$nameFilter" - } else { - cargo xtask e2e - } - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e2e-gpu-strix-windows-report - path: tests/e2e-cucumber/results/ - - # Consolidate every platform/tier's report into ONE cross-platform report: - # a platform × tier matrix in the run Summary plus a single merged HTML - # artifact. `if: always()` so a failing/non-blocking platform still appears; - # runs on GitHub-hosted ubuntu (no GPU needed — it only parses report.json). + # Consolidate the mock (GitHub-hosted) platform's report into the cross-platform + # report: a platform × tier matrix in the run Summary plus a single merged HTML + # artifact. `if: always()` so a failing platform still appears; runs on + # GitHub-hosted ubuntu (no GPU needed — it only parses report.json). # - # Extensibility: the report CONTENT auto-includes any new platform via the - # `*-report` artifact glob below — no change needed here. The one manual step - # when adding a platform is appending its job name to `needs:` (for run ORDERING - # only; GitHub Actions has no wildcard `needs`). + # The self-hosted GPU platforms are consolidated by e2e-selfhosted.yml's own + # report job (they moved there so an offline runner can't stall this workflow's + # required checks). This job only depends on the mock `e2e` lane; the `*-report` + # glob still auto-includes any new GitHub-hosted platform added here. e2e-report: name: E2E consolidated report runs-on: ubuntu-latest @@ -1243,9 +729,6 @@ jobs: needs: - changes - e2e - - e2e-gpu - - e2e-gpu-strix-ubuntu - - e2e-gpu-strix-windows # Consolidate whatever ran. On dispatch `heavy` is unset, so also run when the # trigger was manual; `always()` still lets it collect partial/failed tiers. if: >- @@ -1257,9 +740,15 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 - # Pull every e2e artifact. Each extracts to e2e-artifacts//, - # which `xtask e2e-report` turns into one labeled platform. The `*-report` - # glob is what makes new platforms appear automatically. + # Pull every e2e artifact matching the glob. `xtask e2e-report` turns each + # into one labeled platform and the `*-report` glob makes new platforms + # appear automatically. Layout note: download-artifact@v8 extracts a match + # into e2e-artifacts// when several match, but flattens + # straight into e2e-artifacts/ when EXACTLY ONE matches (its source picks + # the root path when `artifacts.length === 1`). After the ci.yml ⇄ + # e2e-selfhosted.yml split this job has one artifact, so the flattened + # layout is the norm here; `discover()` handles both (labeling the root + # file from platform.json's slug). - name: Download all E2E reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml new file mode 100644 index 00000000..3d0ceb73 --- /dev/null +++ b/.github/workflows/e2e-selfhosted.yml @@ -0,0 +1,649 @@ +name: E2E self-hosted + +# Self-hosted GPU E2E lanes, split out of ci.yml on purpose. A job queued on an +# OFFLINE self-hosted runner cannot be cancelled by GitHub, so if it shared +# ci.yml's concurrency group a superseded run would hold that group forever and +# the newer run's merge-required (GitHub-hosted) checks would sit pending with +# zero jobs (observed on PR #138). Isolating these lanes in their own workflow — +# with their OWN concurrency group — means an offline runner can only ever stall +# THIS workflow's supersession, never the required checks in ci.yml. +# +# These lanes are non-blocking (continue-on-error). NOTE: their check names +# (`E2E tests (GPU)` etc.) are still in main's required-status-check list, so +# while a runner is offline they report as missing and can still block a merge; +# fully closing that requires removing them from the required list (a separate +# branch-protection change, out of scope for this workflow). + +on: + push: + branches: [main] + # Mirror ci.yml: run on PRs against ANY base branch so a stacked PR still gets + # the GPU lanes. Affected-crate/heavy selection diffs against github.base_ref. + pull_request: + merge_group: + branches: [main] + types: [checks_requested] + # On-demand GPU E2E without a push/PR. Dispatch against any ref + # (`gh workflow run e2e-selfhosted.yml --ref -f platform=... `) — the + # mock lane's dispatch stays on ci.yml; this one covers the self-hosted runners. + workflow_dispatch: + inputs: + platform: + description: Which self-hosted runner(s) to target + type: choice + default: all + options: [all, app-dev-gpu, strix-ubuntu, strix-windows] + name_filter: + description: "Scenario-name regex (cucumber --name); empty = full suite" + type: string + default: "" + include_nightly: + description: "Include @nightly scenarios (large-model serve, cold install)" + type: boolean + default: false + +concurrency: + # Own group, namespaced by this workflow so it is DISTINCT from ci.yml's shared + # group — the whole point of the split. Manual dispatches still get a unique + # (run_id) group so a stuck dispatch never blocks later dispatches. + group: >- + ${{ github.workflow }}-${{ github.ref }}-${{ + github.event_name == 'workflow_dispatch' && github.run_id || 'shared' }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + # Trimmed copy of ci.yml's `changes` job: cross-workflow `needs` is impossible, + # so this workflow computes its own `heavy` gate to avoid running the expensive + # GPU lanes on a doc-only PR. Off pull_request (push/merge_group) heavy is + # forced true, matching ci.yml. + changes: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Filter changed paths (pull requests only) + id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + with: + filters: | + # Same `heavy` set as ci.yml: anything that can affect the E2E suite. + heavy: + - '**/*.rs' + - '**/Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain*' + - 'scripts/**' + - 'xtask/**' + - 'engines/**' + - '**/*.py' + - '**/*.sh' + - '**/*.ps1' + - '**/*.feature' + - 'tests/e2e-cucumber/**' + - 'install*' + - 'docs/keys/**' + - '.github/workflows/**' + + - name: Force full run off pull requests + id: all + if: github.event_name != 'pull_request' + run: echo "forced=true" >> "$GITHUB_OUTPUT" + + e2e-gpu: + name: E2E tests (GPU) + timeout-minutes: 90 + runs-on: [self-hosted, linux, amd-gpu] + needs: [changes] + # No build-and-test gate (cross-workflow needs is unavailable): the job builds + # the rocm binary itself and is continue-on-error; ci.yml's required + # build-and-test / mock e2e remain the authoritative pre-merge build gate. + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.heavy == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'app-dev-gpu')) + ) + continue-on-error: true + env: + # Bound serve readiness below the 35-min job cap so a serve that never comes + # ready fails the scenario with a real error instead of hanging until the + # job is cancelled. 300s is ample for a real MI300X vLLM cold-start; + # per-scenario overrides in expectations.toml / a `@serve-timeout` tag adjust + # it (shorter for known bugs, longer for large models). + E2E_SERVE_TIMEOUT_SECS: "300" + # Opt-in @nightly scenarios on a manual dispatch (default off). The nightly + # workflow sets this unconditionally; here it lets a scoped dispatch confirm + # a single nightly scenario (e.g. the 27B serve) without the full nightly run. + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Reclaim the GPU before running: a serve leaked by a killed/timed-out prior + # run (its Drop teardown never executed) can keep an engine process spinning + # on the GPU, starving this job's serves until it hits the timeout. Kill only + # e2e leftovers — scoped to /tmp/rocm-e2e-* and the e2e-target/e2e-shared + # trees — never the runner or any /workload manual-testing processes. + - name: Reclaim GPU from stray E2E processes + run: | + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + # NOTE: no unanchored `pkill -f 'vulkan/llama-server'` — the lemonade + # Vulkan assistant an e2e scenario spawns lives under /tmp/rocm-e2e-* + # and is already caught by the first line; an unanchored pattern would + # also kill a legitimate /workload manual-testing serve on this shared + # self-hosted runner. + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # GPU preflight: fail fast (~90s) instead of hanging to the job cap when the + # GPU is missing, the driver is wedged, or VRAM is still saturated by a + # leftover serve. A BOUNDED POLL, not a one-shot check: transient contention + # (e.g. the reclaim step's kills still draining VRAM) self-heals within + # seconds, so we retry up to a ceiling and succeed the moment the GPU is both + # responsive AND has enough free VRAM. Only a genuinely absent/wedged/held + # GPU reaches the ceiling and fails — with a per-reason message. + - name: GPU preflight (bounded wait for an available GPU) + run: | + # A serve here needs most of the card; require a generous free-VRAM + # floor so a leftover serve (which the reclaim step should have killed) + # is caught, while normal baseline (~300 MB used) passes immediately. + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + while [ "$SECONDS" -lt "$deadline" ]; do + # rocm-smi itself can hang on a wedged driver — bound it with timeout. + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + # Parse the byte value AFTER the colon; the line prefix "GPU[0]" would + # otherwise make a naive first-number match pick up the "0". + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + exit 1 + + # cache: false — on this self-hosted runner we persist the build cache + # ourselves via CARGO_TARGET_DIR (below). The action's built-in + # Swatinem/rust-cache otherwise tries to SAVE the large target dir to + # GitHub's cache service in a post-step (slow/hangs) and its cleanup wipes + # the local target. + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + + # `actions/checkout` runs `git clean -ffdx`, which deletes the gitignored + # `target/` inside the repo every job → a full ~15min rebuild each run. + # Point CARGO_TARGET_DIR at a sibling of the checkout ($RUNNER_WORKSPACE is + # the checkout's parent — untouched by git clean and persistent between jobs + # on a self-hosted runner), so cargo rebuilds incrementally. + - name: Run E2E tests on GPU hardware + run: | + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + # Share heavy immutable artifacts (TheRock runtimes ~3.3GB, HF weights, + # vLLM venv) across scenarios so they download once per runner, not per + # scenario. Persistent path; service state stays isolated per scenario. + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Share uv's wheel download/build cache so `rocm install sdk` is a warm + # ~34s per scenario instead of a cold ~160s (measured on MI300X). Kept + # OFF the RUNNER_WORKSPACE Longhorn PVC (near-full) and on the roomy `/` + # overlay; ~23GB, one cold fill per pod-life. Not git-cleaned (outside + # the checkout) and outside the reclaim step's /tmp/rocm-e2e-* glob. + export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Share ONE installed managed runtime across the serve/chat scenarios so + # `rocm install sdk` runs once per runner, not once per scenario (the + # per-scenario install count — each a multi-GiB TheRock SDK whose probe + # unpacks an ~8.8 GiB devel tarball — is what blew the time cap). The + # "a managed runtime is active" precondition symlinks each scenario's + # data/runtimes here (see use_shared_runtimes); clean-slate scenarios + # stay isolated. Persisted across runs on RUNNER_WORKSPACE, so after the + # first run ever the pre-warm below is a no-op. + # + # CRITICAL: the shared dir IS the pre-warm's own `data/runtimes`, and we + # NEVER move it. `install sdk` bakes ABSOLUTE paths (install_root, + # python_executable) into the runtime manifest; a post-install `mv` would + # leave those pointing at a deleted location and every serve would fail + # instantly (observed on run 29320025393). Installing in place keeps the + # baked paths valid, and each scenario's data/runtimes symlink resolves to + # this same real tree. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # Build the rocm binary ONCE and reuse it for both the pre-warm and the + # suite (via ROCM_CLI_BINARY) so xtask does not rebuild. Honors + # CARGO_TARGET_DIR set above. + cargo build --release -p rocm + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + + # Pre-warm the shared runtime ONCE, SERIALLY, before the suite — never + # lazily inside a concurrent scenario (two multi-GiB installs racing the + # same dir). Skipped once the tree is populated (it persists across runs). + # Uses the prebuilt binary (no cargo run --release) and the shared uv + HF + # caches. Installs directly into the persistent pre-warm data dir (no mv), + # so the manifest's absolute install_root stays valid for every scenario. + if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "pre-warming shared runtime (first run on this runner)…" + mkdir -p "$prewarm"/{data,config,cache} + ROCM_CLI_CONFIG_DIR="$prewarm/config" \ + ROCM_CLI_DATA_DIR="$prewarm/data" \ + ROCM_CLI_CACHE_DIR="$prewarm/cache" \ + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ + "$ROCM_CLI_BINARY" install sdk + if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" + else + echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 + fi + else + echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" + fi + + # Optional scenario-name filter for a scoped dispatch — lets a manual + # run select only the large-model scenario instead of the whole suite. + NAME_FILTER="${{ github.event.inputs.name_filter }}" + if [ -n "$NAME_FILTER" ]; then + echo "name filter active: $NAME_FILTER" + cargo xtask e2e -- --name "$NAME_FILTER" + else + cargo xtask e2e + fi + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-report + path: tests/e2e-cucumber/results/ + + # Second AMD GPU architecture: the Strix Halo (gfx1151) Ubuntu runner, targeted + # by the `strix-halo` label (app-dev-gpu carries `amd-gpu`, not `strix-halo`). + # Non-blocking while this hardware is proven out. + e2e-gpu-strix-ubuntu: + name: E2E tests (Strix Halo, Ubuntu) + # 35min: see e2e-gpu — one collapsed job runs all serves + per-scenario + # install sdk; the cap must exceed the run so the job writes platform.json. + timeout-minutes: 35 + # `native` disambiguates the two Linux Strix runners: the WSL host also + # carries `strix-halo`, but the paths below exist only on the native one. + runs-on: [self-hosted, linux, strix-halo, native] + needs: [changes] + # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.heavy == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')) + ) + continue-on-error: true + # On this runner `/`, `/home/ubuntu`, and `/tmp` are ALL on a full root + # partition; only /home/ubuntu/actions-runner (a 1.7T nvme) has space. So + # EVERYTHING the job writes must land on the nvme. Point HOME there (catches + # ~/.cache/pip, ~/.config, and any other $HOME writer — pip's cache under the + # real /home/ubuntu is what previously failed `install sdk` with ENOSPC), + # plus the toolchain, temp, and pip cache. The rustup bootstrap still uses + # --no-modify-path so it doesn't touch $HOME/.profile. These paths are + # specific to that host, which is why `runs-on` pins `native` above. + env: + HOME: /home/ubuntu/actions-runner/e2e-home + CARGO_HOME: /home/ubuntu/actions-runner/.cargo + RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup + TMPDIR: /home/ubuntu/actions-runner/tmp + PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache + E2E_SERVE_TIMEOUT_SECS: "300" + # Match the MI300X dispatch path: opt into the platform-adaptive large-model + # scenario only when the manual include_nightly input is enabled. + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs + run: | + mkdir -p /home/ubuntu/actions-runner/e2e-home /home/ubuntu/actions-runner/tmp /home/ubuntu/actions-runner/pip-cache + # Reclaim the GPU from any serve leaked by a killed/timed-out prior run + # (see e2e-gpu). Scoped to e2e leftovers only. + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "prepared + reclaimed" + + # GPU preflight: bounded wait so a missing/wedged/held GPU fails fast (~90s) + # instead of hanging to the job cap. Transient contention (VRAM still + # draining from the reclaim above) self-heals within the ceiling. See the + # e2e-gpu job for the rationale. gfx1151 shares system memory, so the free + # floor is smaller than the Instinct card but still catches a leftover serve. + - name: GPU preflight (bounded wait for an available GPU) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + exit 1 + + # Bootstrap rustup ourselves with --no-modify-path so it never writes to + # $HOME/.profile (setup-rust-toolchain doesn't expose that flag). + # rust-toolchain.toml pins the exact toolchain, installed on first cargo + # use. Idempotent. + - name: Ensure Rust toolchain + run: | + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ + | sh -s -- -y --no-modify-path --default-toolchain none + fi + echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + + - name: Run E2E tests on Strix Halo + run: | + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Share ONE installed managed runtime across serve/chat scenarios, and + # PRE-WARM it in place before the suite (mirrors e2e-gpu). This is not an + # optimization — it is REQUIRED for correctness. `install sdk` bakes + # ABSOLUTE paths (install_root, python_executable, rocm_sdk.*) into the + # runtime manifest. If the first scenario installs into its own isolated + # /tmp/rocm-e2e-XXXX data dir and only the registry is shared onward, those + # baked paths point at that scenario's temp dir — which is deleted when the + # scenario ends. Every later serve then sees `status=unusable (install root + # is missing)` and fails (diagnosed on the box 2026-07-15). Installing in + # place, directly into the persistent shared dir, keeps the baked paths + # valid for all scenarios and for the end-of-run version probe. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # Build the rocm binary once; reuse for pre-warm + suite so xtask doesn't + # rebuild. + cargo build --release -p rocm + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + + # Pre-warm once, serially, in place (no mv/symlink). Skipped once the tree + # is populated (persists across runs on RUNNER_WORKSPACE). + if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "pre-warming shared runtime (first run on this runner)…" + mkdir -p "$prewarm"/{data,config,cache} + ROCM_CLI_CONFIG_DIR="$prewarm/config" \ + ROCM_CLI_DATA_DIR="$prewarm/data" \ + ROCM_CLI_CACHE_DIR="$prewarm/cache" \ + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + "$ROCM_CLI_BINARY" install sdk + if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then + echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" + else + echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 + fi + else + echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" + fi + + # Optional scenario-name filter for a scoped manual dispatch. + NAME_FILTER="${{ github.event.inputs.name_filter }}" + if [ -n "$NAME_FILTER" ]; then + echo "name filter active: $NAME_FILTER" + cargo xtask e2e -- --name "$NAME_FILTER" + else + cargo xtask e2e + fi + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-strix-ubuntu-report + path: tests/e2e-cucumber/results/ + + # First real Windows GPU coverage: the Strix Halo Windows 11 runner. The + # existing windows-build-and-test uses GitHub-hosted windows-latest, which has + # no GPU. Non-blocking so it never gates the PR. + e2e-gpu-strix-windows: + name: E2E tests (Strix Halo, Windows) + # 35min: see e2e-gpu — one collapsed job runs all serves + per-scenario + # install sdk; the cap must exceed the run so the job writes platform.json. + timeout-minutes: 35 + runs-on: [self-hosted, windows, strix-halo, native] + needs: [changes] + # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-windows. + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.heavy == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'strix-windows')) + ) + continue-on-error: true + env: + # Match the Linux Strix dispatch path: opt into the platform-adaptive + # large-model scenario only when the manual input is enabled. + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # setup-rust-toolchain runs an internal bash script, which this Windows + # runner lacks (bash: command not found). Bootstrap rustup with the + # PowerShell-native installer instead; idempotent, so it only downloads on + # a runner that doesn't already have the toolchain. Use `powershell` + # (Windows PowerShell 5.1, always present) rather than `pwsh` (PowerShell 7), + # which this self-hosted runner does not have installed. + # Reclaim the GPU from any E2E serve leaked by a killed/timed-out prior run + # (see e2e-gpu). PowerShell (5.1) equivalent; best-effort. + - name: Reclaim GPU from stray E2E processes + shell: powershell + run: | + Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | + ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } + Write-Host "reclaimed" + + # GPU preflight (best-effort on Windows): bounded wait for the GPU to be + # available so a wedged/held GPU fails fast instead of hanging to the cap. + # BEST-EFFORT because the Windows ROCm GPU query tool isn't verified here — + # if no rocm-smi is found we WARN and continue rather than false-fail a + # working runner. When rocm-smi IS present we poll free VRAM the same way as + # the Linux jobs and fail only after the ceiling. + - name: GPU preflight (bounded wait for an available GPU) + shell: powershell + run: | + $minFreeGiB = if ($env:GPU_PREFLIGHT_MIN_FREE_GIB) { [int]$env:GPU_PREFLIGHT_MIN_FREE_GIB } else { 8 } + $ceilingSecs = if ($env:GPU_PREFLIGHT_CEILING_SECS) { [int]$env:GPU_PREFLIGHT_CEILING_SECS } else { 90 } + if (-not (Get-Command rocm-smi -ErrorAction SilentlyContinue)) { + Write-Host "rocm-smi not found on this Windows runner; skipping GPU preflight (best-effort)." + exit 0 + } + $minFree = [int64]$minFreeGiB * 1GB + $deadline = (Get-Date).AddSeconds($ceilingSecs) + $reason = "rocm-smi never returned usable VRAM figures" + while ((Get-Date) -lt $deadline) { + $out = (rocm-smi --showmeminfo vram 2>$null | Out-String) + $total = ([regex]::Matches($out, 'VRAM Total Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value + $used = ([regex]::Matches($out, 'VRAM Total Used Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value + if (-not $total -or -not $used) { + $reason = "rocm-smi returned no VRAM figures (no AMD GPU detected)" + Start-Sleep -Seconds 5; continue + } + $free = [int64]$total - [int64]$used + if ($free -ge $minFree) { + Write-Host "GPU ready: $([math]::Floor($free/1GB)) GiB free (>= $minFreeGiB GiB)." + exit 0 + } + $reason = "VRAM never dropped below the floor: only $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB) - a serve is likely still holding the GPU" + Write-Host "waiting: $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB)..." + Start-Sleep -Seconds 5 + } + Write-Host "::error::GPU preflight failed after ${ceilingSecs}s: $reason" + exit 1 + + - name: Ensure Rust toolchain (PowerShell) + shell: powershell + run: | + if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) { + Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe + # --default-toolchain none: rust-toolchain.toml pins the exact + # version (1.96.0 + components), auto-installed on first cargo use. + & $env:TEMP\rustup-init.exe -y --default-toolchain none + "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Append + } + + - name: Run E2E tests on Strix Halo Windows + shell: powershell + run: | + # Share ONE installed managed runtime across serve/chat scenarios, and + # PRE-WARM it in place before the suite (mirrors e2e-gpu / strix-ubuntu). + # REQUIRED for correctness, not just speed: `install sdk` bakes ABSOLUTE + # paths (install_root, python_executable, rocm_sdk.*, .rocm-cli-runtime.json) + # into the runtime manifest. If the first scenario installs into its own + # isolated temp data dir and only the registry is shared onward, those baked + # paths point at that scenario's dir — deleted when the scenario ends — so + # every later serve sees status=unusable and fails (diagnosed on the Linux + # box 2026-07-15; the Windows scenario-8 cold-download failure is the same + # class). Install in place so the baked paths stay valid for all scenarios. + $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm" + $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" + + # Build the rocm binary once; reuse for pre-warm + suite so xtask doesn't + # rebuild. This job does not set CARGO_TARGET_DIR, so the binary lands in + # the default target\release (fall back to it when the env var is unset). + cargo build --release -p rocm + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" } + $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe" + + # Pre-warm once, in place (no move/symlink). Skipped once the tree is + # populated (persists across runs on RUNNER_WORKSPACE). + if (-not (Test-Path "$env:E2E_SHARED_RUNTIMES_DIR\registry")) { + Write-Host "pre-warming shared runtime (first run on this runner)..." + New-Item -ItemType Directory -Force -Path "$prewarm\data","$prewarm\config","$prewarm\cache" | Out-Null + $env:ROCM_CLI_CONFIG_DIR = "$prewarm\config" + $env:ROCM_CLI_DATA_DIR = "$prewarm\data" + $env:ROCM_CLI_CACHE_DIR = "$prewarm\cache" + & $env:ROCM_CLI_BINARY install sdk + Remove-Item Env:\ROCM_CLI_CONFIG_DIR,Env:\ROCM_CLI_DATA_DIR,Env:\ROCM_CLI_CACHE_DIR -ErrorAction SilentlyContinue + if (Test-Path "$env:E2E_SHARED_RUNTIMES_DIR\registry") { + Write-Host "shared runtime pre-warmed at $env:E2E_SHARED_RUNTIMES_DIR" + } else { + Write-Host "pre-warm did not produce a runtimes registry; scenarios will install their own" + } + } else { + Write-Host "shared runtime already present at $env:E2E_SHARED_RUNTIMES_DIR - skipping pre-warm" + } + + # Optional scenario-name filter for a scoped manual dispatch. + $nameFilter = "${{ github.event.inputs.name_filter }}" + if ($nameFilter) { + Write-Host "name filter active: $nameFilter" + cargo xtask e2e -- --name "$nameFilter" + } else { + cargo xtask e2e + } + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-strix-windows-report + path: tests/e2e-cucumber/results/ + + # Consolidate this workflow's self-hosted platform reports into one GPU-side + # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's + # required `E2E consolidated report` so it does NOT collide with that required + # check; this one is advisory and lives with the lanes it summarizes. Runs on + # GitHub-hosted ubuntu (no GPU needed — it only parses report.json). `always()` + # so a failing/skipped self-hosted platform still appears in the grid. + e2e-report: + name: E2E consolidated report (self-hosted) + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: + - changes + - e2e-gpu + - e2e-gpu-strix-ubuntu + - e2e-gpu-strix-windows + if: >- + always() + && (needs.changes.outputs.heavy == 'true' + || github.event_name == 'workflow_dispatch') + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + + # Pull every self-hosted e2e artifact matching the glob; `xtask e2e-report` + # turns each into one labeled platform and the `*-report` glob picks up any + # new self-hosted platform. Layout note: download-artifact@v8 uses a + # per-artifact subdir (e2e-artifacts//) when several match, but + # flattens into e2e-artifacts/ when EXACTLY ONE matches (its source picks + # the root path when `artifacts.length === 1`, regardless of merge-multiple). + # A single-platform dispatch hits the flattened case; `discover()` handles + # both, labeling a root-level report from platform.json's slug. + - name: Download all self-hosted E2E reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: '*-report' + path: e2e-artifacts + + - name: Build consolidated report + step summary + run: | + mkdir -p consolidated + cargo xtask e2e-report \ + --artifacts-dir e2e-artifacts \ + --html-out consolidated/index.html >> "$GITHUB_STEP_SUMMARY" + + - name: Upload consolidated report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-consolidated-report-selfhosted + path: consolidated/ diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index c5867e76..f0ea1281 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -414,6 +414,12 @@ fn parse_descriptor(name: &str) -> Descriptor { "gpu" => ("MI300X", "Linux"), "gpu-strix-ubuntu" => ("Strix Halo", "Ubuntu"), "gpu-strix-windows" => ("Strix Halo", "Windows"), + // `e2e-unknown-report`: a report whose platform.json sidecar was missing + // or unrecognized (e.g. a GPU run that errored before writing it). The OS + // is genuinely unknown here — a Windows GPU run that erupted early must NOT + // be reported as Linux — so render Unknown / Unknown rather than defaulting + // OS to Linux the way `fallback_descriptor` does for a titlecased platform. + "unknown" => ("Unknown", "Unknown"), other => return fallback_descriptor(other, known_bugs), }; @@ -1999,6 +2005,34 @@ mod tests { assert_eq!(s.passed, 0); } + #[test] + fn parse_descriptor_maps_known_artifacts() { + for (name, platform, os) in [ + ("e2e-report", "Mock", "Linux"), + ("e2e-gpu-report", "MI300X", "Linux"), + ("e2e-gpu-strix-ubuntu-report", "Strix Halo", "Ubuntu"), + ("e2e-gpu-strix-windows-report", "Strix Halo", "Windows"), + ] { + let d = parse_descriptor(name); + assert_eq!( + (d.platform.as_str(), d.os.as_str()), + (platform, os), + "{name}" + ); + } + } + + #[test] + fn parse_descriptor_unknown_is_not_falsely_linux() { + // A report whose platform.json was missing (e.g. a GPU run that errored + // before writing the sidecar) is labeled `e2e-unknown-report`. Its OS is + // genuinely unknown — a Windows GPU run must NOT be reported as Linux — so + // both platform AND os render "Unknown", never a default "Linux". + let d = parse_descriptor("e2e-unknown-report"); + assert_eq!(d.platform, "Unknown"); + assert_eq!(d.os, "Unknown"); + } + #[test] fn scenario_status_undefined_step_is_not_passed() { // Regression: an undefined step must fail the scenario, not pass it. diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 7b0861cf..6b92f364 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -8,9 +8,18 @@ SPDX-License-Identifier: MIT The hosted CI (`ubuntu-latest`, `windows-latest`) builds and unit-tests every shipping target natively, but GitHub-hosted runners have no AMD GPU. A -dedicated hardware layer in `.github/workflows/ci.yml` covers that gap by -running the same cucumber-rs E2E suite on dedicated self-hosted runners with -real AMD GPUs. +dedicated hardware layer covers that gap by running the same cucumber-rs E2E +suite on dedicated self-hosted runners with real AMD GPUs. + +That hardware layer lives in its **own workflow**, `.github/workflows/e2e-selfhosted.yml`, +separate from the main `ci.yml`. The split is deliberate: a job queued on an +**offline** self-hosted runner cannot be cancelled by GitHub, so if it shared +`ci.yml`'s concurrency group a superseded run would hold that group and the +newer run's merge-required (GitHub-hosted) checks would sit pending forever +(observed on PR #138). Giving the self-hosted lanes their own workflow — and +thus their own concurrency group — means an offline runner can only ever stall +that workflow's own supersession, never `ci.yml`'s required checks. See +`EAI-7548`. ## Platforms @@ -20,12 +29,16 @@ scenario to pass / xfail / skip for that host from its `@id` and `@requires-*` tags, a capability probe, and `expectations.toml` — there is no separate tier flag or tag filter to maintain. -| Job | Platform | Runner labels | -|---|---|---| -| `e2e` | Mock (no GPU) | GitHub-hosted `ubuntu-latest` | -| `e2e-gpu` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, amd-gpu]` | -| `e2e-gpu-strix-ubuntu` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, strix-halo]` | -| `e2e-gpu-strix-windows` | Strix Halo (gfx1151) on native Windows 11 | self-hosted `[self-hosted, windows, strix-halo]` | +| Job | Workflow | Platform | Runner labels | +|---|---|---|---| +| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-latest` | +| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, amd-gpu]` | +| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, strix-halo, native]` | +| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on native Windows 11 | self-hosted `[self-hosted, windows, strix-halo, native]` | + +The Strix Halo lanes pin the extra `native` label because two Linux runners +share the `strix-halo` label (a native host and a WSL host) and the jobs' +hardcoded `/home/ubuntu/actions-runner` paths exist only on the native one. `e2e` is the blocking, GitHub-hosted mock job: `@requires-gpu` scenarios resolve to skip here, and known bugs resolve to xfail from @@ -36,50 +49,61 @@ run on dedicated self-hosted runners with a real AMD GPU attached, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios that the mock job cannot. -An `e2e-report` job consolidates every platform's report — including partial -or failed runs — into one HTML report and GitHub step summary, joined by -scenario id, so the (scenario × platform) expectation grid is visible in one -place. +Each workflow has its own consolidated report job (both named `e2e-report` +internally). `ci.yml`'s `E2E consolidated report` covers the mock platform; +`e2e-selfhosted.yml`'s `E2E consolidated report (self-hosted)` covers the three +GPU platforms. Each joins its platforms' reports — including partial or failed +runs — by scenario id into one HTML report and GitHub step summary. ## Triggers -The GPU jobs run automatically on `push`, `pull_request`, and `merge_group` -when both of the following hold: - -- the `changes` job's `heavy` path filter is `true` (the change touches code - that can affect runtime behavior, not just docs or unrelated files), and -- the hosted `build-and-test` job succeeded. - -They can also be triggered manually via `workflow_dispatch`, independent of -the `heavy` gate, with these inputs: - -- `platform` (choice: `all`, `mock`, `app-dev-gpu`, `strix-ubuntu`, - `strix-windows`) — which job(s) to run. `mock` maps to `e2e`, - `app-dev-gpu` to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, and - `strix-windows` to `e2e-gpu-strix-windows`. +The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, +`pull_request`, and `merge_group` when the workflow's own `changes` job's +`heavy` path filter is `true` (the change touches code that can affect runtime +behavior, not just docs or unrelated files). Unlike the pre-split layout they do +**not** gate on the hosted `build-and-test` job — cross-workflow `needs` is not +possible, so each GPU job builds the `rocm` binary itself as its first real step +(a broken build fails that job fast and non-fatally). `ci.yml`'s required +`build-and-test` and mock `e2e` remain the authoritative pre-merge build gate. + +They can also be triggered manually via `e2e-selfhosted.yml`'s +`workflow_dispatch`, independent of the `heavy` gate, with these inputs: + +- `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`) — + which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, + `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, and `strix-windows` to + `e2e-gpu-strix-windows`. (The mock lane has its own `platform` input on + `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a - single scenario instead of the full suite. Only wired into the three - self-hosted GPU jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, - `e2e-gpu-strix-windows`); `platform=mock` always runs the full mock suite - and ignores it. Empty runs everything applicable to the selected - platform(s). + single scenario instead of the full suite. Empty runs everything applicable + to the selected platform(s). - `include_nightly` (boolean, default `false`) — opts a dispatch into `@nightly`-tagged scenarios (e.g. large-model serves, cold installs) that - are otherwise skipped on a normal push/PR run to keep it fast. Same scope - as `name_filter`: only the three self-hosted GPU jobs read it; - `platform=mock` ignores it. + are otherwise skipped on a normal push/PR run to keep it fast. + +Dispatch the GPU lanes with, e.g.: -A manual dispatch skips the hosted `build-and-test` job for a faster loop; the -E2E jobs run directly against the dispatched ref. +```bash +gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu +``` ## Blocking vs. non-blocking -Only `e2e` (the GitHub-hosted mock job) is a required, blocking check. The -three hardware jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, and +The three hardware jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, and `e2e-gpu-strix-windows` — all run with `continue-on-error: true`, so a -hardware failure never gates a PR merge. Their results still surface in the -consolidated `e2e-report` for visibility. +hardware failure that RUNS never gates a PR merge. Their results still surface +in the self-hosted consolidated report for visibility. + +**Required-check caveat.** These three job names (plus, historically, a +consolidated-report name) are still in `main`'s required-status-check list. +`continue-on-error` neutralizes a job that ran and failed, but a required check +that *never reports* — because its self-hosted runner is offline — is treated as +missing and still blocks the merge. The workflow split removes the catastrophic +concurrency stall (an offline runner can no longer freeze `ci.yml`'s hosted +required checks), but fully unblocking merges while a runner is offline +additionally requires removing these self-hosted checks from the required list — +a branch-protection change tracked separately from the workflow split. ## Fork safety @@ -96,6 +120,11 @@ which requires write access to trigger). not performance, so this is not a performance benchmark. Release-fidelity, `manylinux2014` (glibc 2.17) packaging validation is handled separately by the nightly/release pipeline. -- `e2e-report` collects whatever ran, including partial results from a - cancelled or failed job, and renders one HTML report plus a step summary so - all platforms are visible together. +- Each workflow's `e2e-report` job collects whatever ran in that workflow — + including partial results from a cancelled or failed job — and renders one + HTML report plus a step summary. `download-artifact@v8` flattens a + single-match download straight into the artifacts directory (it uses the root + path when exactly one artifact matches, regardless of the pattern), so after + the split each report job usually has one artifact; `xtask e2e-report`'s + discovery handles both the flattened and per-subdirectory layouts, labeling a + root-level report from its `platform.json` slug. diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 77700135..362d3272 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -122,18 +122,22 @@ serve does not compete with the first for device memory, and the failure quotes the service log tail plus the device's free-VRAM state, which is where the engine's own reason for the stall is recorded. -CI runs one job per platform, each executing the full suite: - -| Job | Platform | Blocking | -|---|---|---| -| `e2e` | Mock (no GPU, GitHub-hosted) | yes | -| `e2e-gpu` | MI300X (self-hosted) | no | -| `e2e-gpu-strix-ubuntu` | Strix Halo / Ubuntu (self-hosted) | no | -| `e2e-gpu-strix-windows` | Strix Halo / Windows (self-hosted) | no | +CI runs one job per platform, each executing the full suite. The mock job lives +in the `CI` workflow (`ci.yml`); the self-hosted GPU jobs live in a separate +`E2E self-hosted` workflow (`e2e-selfhosted.yml`) so a job queued on an offline +self-hosted runner can never stall `ci.yml`'s merge-required checks: + +| Job | Workflow | Platform | Blocking | +|---|---|---|---| +| `e2e` | `ci.yml` | Mock (no GPU, GitHub-hosted) | yes | +| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (self-hosted) | no | +| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | +| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | The blocking mock job passes when every applicable scenario is pass-or-xfail with -no XPASS or unexpected failure; the GPU jobs are non-blocking. The `e2e-report` -job consolidates all platforms' results into one cross-platform report. +no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow has +its own `e2e-report` job: `ci.yml`'s consolidates the mock platform, and +`e2e-selfhosted.yml`'s consolidates the self-hosted platforms. The nightly workflow runs three non-blocking jobs — the existing MI300X job and new Strix Halo jobs on Ubuntu and Windows — with `E2E_INCLUDE_NIGHTLY=1`. The @@ -141,23 +145,25 @@ shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on MI300X and the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint through Lemonade on Strix Halo. -Use the CI workflow dispatch to run either model independently on a ref: +Use the self-hosted E2E workflow dispatch to run either model independently on a +ref (the GPU platform / `include_nightly` / `name_filter` inputs live on +`e2e-selfhosted.yml`, not `ci.yml`): ```bash # MI300X / vLLM / Qwen3.6-27B -gh workflow run ci.yml --ref \ +gh workflow run e2e-selfhosted.yml --ref \ -f platform=app-dev-gpu \ -f include_nightly=true \ -f name_filter='large platform-specific model' # Strix Halo Linux / Lemonade / Qwen3.6-35B-A3B-GGUF (UD-Q4_K_XL) -gh workflow run ci.yml --ref \ +gh workflow run e2e-selfhosted.yml --ref \ -f platform=strix-ubuntu \ -f include_nightly=true \ -f name_filter='large platform-specific model' # Strix Halo Windows / Lemonade / Qwen3.6-35B-A3B-GGUF (UD-Q4_K_XL) -gh workflow run ci.yml --ref \ +gh workflow run e2e-selfhosted.yml --ref \ -f platform=strix-windows \ -f include_nightly=true \ -f name_filter='large platform-specific model' diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index ddcfcec6..a6d7b353 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -58,8 +58,20 @@ pub fn run(artifacts_dir: &Path, html_out: &Path) -> Result<()> { Ok(()) } -/// Return `(label, report_json_path)` for each immediate subdir of `dir` that -/// contains a `report.json`, sorted by label for stable output. +/// Return `(label, report_json_path)` for each platform report under `dir`, +/// sorted by label for stable output. +/// +/// Two layouts are handled, because `actions/download-artifact@v8` does NOT +/// always create a per-artifact subdirectory: +/// * multi-artifact download → `dir//report.json` (one subdir +/// per artifact); the subdir name is the label. +/// * single-artifact download → `dir/report.json` at the ROOT. When exactly +/// one artifact matches the download `pattern`, v8 extracts it straight into +/// the `path` (its source picks `resolvedPath` when `artifacts.length === 1`, +/// regardless of `pattern`/`merge-multiple`). After the ci.yml ⇄ +/// e2e-selfhosted.yml split each report job has exactly ONE artifact, so this +/// is the normal case there. The root file has no name to label from, so we +/// recover the label from the sibling `platform.json`'s `platform_slug`. fn discover(dir: &Path) -> Result> { let mut inputs = Vec::new(); let entries = match std::fs::read_dir(dir) { @@ -70,6 +82,12 @@ fn discover(dir: &Path) -> Result> { Err(e) => return Err(e).context("reading artifacts dir")?, }; + // Single-artifact layout: a report.json sits directly in `dir`. + let root_report = dir.join("report.json"); + if root_report.is_file() { + inputs.push((label_for_root_report(dir), root_report)); + } + for entry in entries { let entry = entry.context("reading dir entry")?; if !entry.file_type().context("stat dir entry")?.is_dir() { @@ -89,6 +107,38 @@ fn discover(dir: &Path) -> Result> { Ok(inputs) } +/// Label for a root-level (single-artifact) `report.json`. Recovers the platform +/// from the sibling `platform.json`'s `platform_slug` and maps it to the same +/// artifact-name shape `e2e_report::parse_descriptor` expects, so a flattened +/// download renders identically to a per-subdir one. +/// +/// The sidecar can be ABSENT even for a GPU run: the harness writes `report.json` +/// first and, on a parsing/hook error, exits BEFORE writing `platform.json` (see +/// tests/e2e-cucumber/tests/e2e.rs). So a missing/unrecognized slug must NOT be +/// labeled `mock` — that would misattribute a hardware failure to Mock/Linux in +/// the grid. Only an explicit `mock` slug maps to the mock artifact; anything +/// missing or unknown gets a neutral `e2e-unknown-report`, which +/// `parse_descriptor` renders as "Unknown" rather than claiming a real platform. +fn label_for_root_report(dir: &Path) -> String { + let slug = std::fs::read_to_string(dir.join("platform.json")) + .ok() + .and_then(|t| serde_json::from_str::(&t).ok()) + .and_then(|v| { + v.get("platform_slug") + .and_then(|s| s.as_str()) + .map(str::to_owned) + }); + match slug.as_deref() { + Some("mock") => "e2e-report".to_owned(), + Some("mi300x") => "e2e-gpu-report".to_owned(), + Some("strix-halo-linux") => "e2e-gpu-strix-ubuntu-report".to_owned(), + Some("strix-halo-windows") => "e2e-gpu-strix-windows-report".to_owned(), + // Missing sidecar (e.g. a GPU run that errored before writing it) or an + // unrecognized slug → neutral identity, never a false "Mock". + _ => "e2e-unknown-report".to_owned(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -119,4 +169,66 @@ mod tests { let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); assert_eq!(names, vec!["e2e-gpu-report", "e2e-report"]); } + + // Single-artifact download flattens report.json into the dir root (no + // per-artifact subdir). Discovery must still find it and label it from the + // platform.json sidecar's slug — otherwise the split's one-artifact report + // jobs silently produce an empty report. + #[test] + fn discover_finds_root_level_report_labeled_from_slug() { + for (slug, expected) in [ + ("mi300x", "e2e-gpu-report"), + ("strix-halo-linux", "e2e-gpu-strix-ubuntu-report"), + ("strix-halo-windows", "e2e-gpu-strix-windows-report"), + ("mock", "e2e-report"), + ] { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::write(root.join("report.json"), "[]").unwrap(); + std::fs::write( + root.join("platform.json"), + format!(r#"{{"platform_slug":"{slug}"}}"#), + ) + .unwrap(); + + let got = discover(root).expect("discover"); + let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec![expected], "slug {slug}"); + } + } + + // A root report.json with no platform.json still resolves (rather than being + // dropped) — but to a NEUTRAL "unknown" label, never "mock". This is the + // GPU-run-that-errored case: the harness writes report.json, then exits on a + // parsing/hook error BEFORE writing platform.json. Labeling it "mock" would + // misattribute a hardware failure to Mock/Linux in the consolidated grid. + #[test] + fn discover_root_report_without_sidecar_is_neutral_not_mock() { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::write(root.join("report.json"), "[]").unwrap(); + + let got = discover(root).expect("discover"); + let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["e2e-unknown-report"]); + // Must NOT masquerade as the mock platform. + assert_ne!(names, vec!["e2e-report"]); + } + + // An unrecognized slug is likewise neutral, never a false real platform. + #[test] + fn discover_root_report_unknown_slug_is_neutral() { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::write(root.join("report.json"), "[]").unwrap(); + std::fs::write( + root.join("platform.json"), + r#"{"platform_slug":"some-future-gpu"}"#, + ) + .unwrap(); + + let got = discover(root).expect("discover"); + let names: Vec<&str> = got.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["e2e-unknown-report"]); + } } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 0cde5efe..fb9eec8b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,6 +22,7 @@ mod signing; mod tpn; mod verify_commits; mod verify_pinned_keys; +mod workflow_contract; use std::path::PathBuf; use std::process::ExitCode; diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs new file mode 100644 index 00000000..9d2c574b --- /dev/null +++ b/xtask/src/workflow_contract.rs @@ -0,0 +1,253 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract tests for the CI workflow split. +//! +//! Regression guard for EAI-7548: the self-hosted GPU E2E lanes must live in a +//! SEPARATE workflow from `ci.yml`, with a DISTINCT concurrency group, so a job +//! queued on an offline self-hosted runner (which GitHub cannot cancel) can never +//! hold `ci.yml`'s concurrency group and stall its merge-required checks. +//! +//! There is no YAML dependency in this crate, so instead of a whole-file +//! substring match (which can false-pass — a label hidden across a multiline +//! `runs-on`, or `github.workflow` found only in a comment) these helpers +//! extract the COMPLETE value of each `runs-on` and of the top-level +//! `concurrency.group`, then assert on those extracted values. + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + fn repo_root() -> PathBuf { + // CARGO_MANIFEST_DIR is the xtask/ crate dir; its parent is the repo root + // (same idiom as verify_pinned_keys::repo_root). + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + fn read_workflow(name: &str) -> String { + let p = repo_root().join(".github/workflows").join(name); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) + } + + /// The self-hosted runner labels that must not appear in a `runs-on`. + const SELF_HOSTED_LABELS: [&str; 3] = ["self-hosted", "amd-gpu", "strix-halo"]; + + /// Strip a trailing `# …` comment from a YAML line (best-effort: our + /// workflows never put a literal `#` inside a runs-on/group value). + fn strip_comment(line: &str) -> &str { + line.split_once(" #").map_or(line, |(v, _)| v) + } + + fn indent_of(line: &str) -> usize { + line.len() - line.trim_start().len() + } + + /// Extract the COMPLETE value of every `runs-on:` in the workflow, joining any + /// block/flow continuation lines so a label split across lines can't hide. + /// Returns one flattened string per `runs-on` key. + fn runs_on_values(text: &str) -> Vec { + let lines: Vec<&str> = text.lines().collect(); + let mut out = Vec::new(); + for (i, raw) in lines.iter().enumerate() { + let line = strip_comment(raw); + let trimmed = line.trim_start(); + let Some(rest) = trimmed.strip_prefix("runs-on:") else { + continue; + }; + let key_indent = indent_of(line); + let mut value = rest.trim().to_owned(); + // Gather deeper-indented continuation lines (block list `- x`, or a + // flow list `[…]` wrapped across lines). + for cont in &lines[i + 1..] { + let c = strip_comment(cont); + if c.trim().is_empty() { + continue; + } + if indent_of(c) <= key_indent { + break; + } + value.push(' '); + value.push_str(c.trim()); + } + out.push(value); + } + out + } + + /// Extract the top-level `concurrency.group` value, joining folded (`>-`) + /// continuation lines. Returns the whole group expression as one string. + fn concurrency_group(text: &str) -> String { + let lines: Vec<&str> = text.lines().collect(); + // Find a top-level (column-0) `concurrency:` key. + let start = lines + .iter() + .position(|l| *l == "concurrency:") + .expect("workflow declares a top-level concurrency:"); + // Within that block, find the `group:` key. + let mut group_val = String::new(); + let mut in_group = false; + let mut group_indent = 0; + for line in &lines[start + 1..] { + // A new column-0 key ends the concurrency block. + if !line.is_empty() && indent_of(line) == 0 { + break; + } + let stripped = strip_comment(line); + let trimmed = stripped.trim_start(); + if !in_group { + if let Some(rest) = trimmed.strip_prefix("group:") { + in_group = true; + group_indent = indent_of(stripped); + group_val = rest.trim().to_owned(); + } + continue; + } + // Collecting folded continuation lines under group:. + if trimmed.is_empty() { + continue; + } + if indent_of(stripped) <= group_indent { + break; + } + group_val.push(' '); + group_val.push_str(trimmed); + } + assert!(in_group, "concurrency block has no group: key"); + group_val + } + + fn workflow_name(text: &str) -> String { + text.lines() + .find_map(|l| l.strip_prefix("name:")) + .map(|n| strip_comment(n).trim().to_owned()) + .expect("workflow declares a top-level name:") + } + + #[test] + fn ci_yml_schedules_no_self_hosted_job() { + let ci = read_workflow("ci.yml"); + let values = runs_on_values(&ci); + assert!( + !values.is_empty(), + "expected at least one runs-on in ci.yml (extractor sanity check)" + ); + for value in values { + for label in SELF_HOSTED_LABELS { + assert!( + !value.contains(label), + "ci.yml runs-on `{value}` references the self-hosted label {label:?}: \ + the GPU E2E lanes belong in e2e-selfhosted.yml so a job queued on an \ + offline runner can't stall ci.yml's required checks (EAI-7548)" + ); + } + } + } + + #[test] + fn self_hosted_workflow_owns_the_gpu_lanes() { + let sh = read_workflow("e2e-selfhosted.yml"); + // Each GPU lane must be present AND actually target a self-hosted runner. + let values = runs_on_values(&sh); + assert!( + values.iter().any(|v| v.contains("self-hosted")), + "e2e-selfhosted.yml must schedule at least one `self-hosted` runner (EAI-7548)" + ); + for job in [ + "e2e-gpu:", + "e2e-gpu-strix-ubuntu:", + "e2e-gpu-strix-windows:", + ] { + assert!( + sh.contains(job), + "e2e-selfhosted.yml must define the self-hosted job `{job}` (EAI-7548)" + ); + } + } + + #[test] + fn workflows_use_distinct_concurrency_groups() { + // Isolation comes from the group KEY differing per workflow. Extract the + // actual concurrency.group value from each and assert (a) both keep + // supersession, (b) both key on github.workflow, and (c) the two group + // expressions and workflow names differ — so at runtime the groups are + // distinct and an offline-runner stall in one can't hold the other. + let ci = read_workflow("ci.yml"); + let sh = read_workflow("e2e-selfhosted.yml"); + let ci_group = concurrency_group(&ci); + let sh_group = concurrency_group(&sh); + + for (label, group, text) in [ + ("ci.yml", &ci_group, &ci), + ("e2e-selfhosted.yml", &sh_group, &sh), + ] { + assert!( + group.contains("github.workflow"), + "{label} concurrency.group must be namespaced by github.workflow \ + (got `{group}`) (EAI-7548)" + ); + assert!( + text.contains("cancel-in-progress: true"), + "{label} must keep cancel-in-progress: true (EAI-7548)" + ); + } + // The github.workflow-keyed groups resolve via the workflow `name:`; those + // names must differ for the runtime groups to be distinct. + assert_ne!( + workflow_name(&ci), + workflow_name(&sh), + "the two workflows must have different `name:` values so their \ + github.workflow-keyed concurrency groups are distinct (EAI-7548)" + ); + } + + // Extractor guards: prove the helpers actually parse multiline forms, so the + // contract tests above can't silently false-pass on a shape they don't handle. + #[test] + fn runs_on_extractor_flattens_multiline_forms() { + let yaml = "\ +jobs: + a: + runs-on: ubuntu-latest + b: + runs-on: + - self-hosted + - linux + - amd-gpu + c: + runs-on: [self-hosted, windows, + strix-halo, native] +"; + let vals = runs_on_values(yaml); + assert_eq!(vals.len(), 3); + assert!(vals[0].contains("ubuntu-latest")); + assert!(vals[1].contains("self-hosted") && vals[1].contains("amd-gpu")); + // The flow list split across lines must be joined so `strix-halo` is seen. + assert!(vals[2].contains("self-hosted") && vals[2].contains("strix-halo")); + } + + #[test] + fn concurrency_group_extractor_reads_folded_value() { + let yaml = "\ +name: X + +concurrency: + # a comment mentioning github.workflow that must NOT count + group: >- + ${{ github.workflow }}-${{ github.ref }}-${{ + github.event_name == 'workflow_dispatch' && github.run_id || 'shared' }} + cancel-in-progress: true + +permissions: + contents: read +"; + let g = concurrency_group(yaml); + assert!(g.contains("github.workflow")); + assert!(g.contains("github.run_id")); + // Must stop at the next key, not swallow permissions. + assert!(!g.contains("permissions")); + } +}