ROCm · fredespi · Jul 31, 2026 · Aug 4, 2026
@@ -82,6 +82,9 @@ jobs:
       # filter step is skipped off-PR), otherwise the forced 'true'.
       rust: ${{ steps.filter.outputs.rust || steps.all.outputs.forced }}
       heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }}
+      # Narrow GPU-serve gate; forced true off-PR so the merge queue always runs
+      # the full matrix and its required checks are never starved.
+      serve: ${{ steps.filter.outputs.serve || steps.all.outputs.forced }}
       lint: ${{ steps.filter.outputs.lint || steps.all.outputs.forced }}
       tpn: ${{ steps.filter.outputs.tpn || steps.all.outputs.forced }}
     steps:
@@ -125,6 +128,33 @@ jobs:
               # `cargo xtask verify-pinned-keys`.
               - 'docs/keys/**'
               - '.github/workflows/**'
+            # The real-GPU serve matrix (e2e-gpu*, non-blocking). Narrower than
+            # `heavy`: only paths that can change serve BEHAVIOUR or the GPU E2E
+            # harness — NOT a blanket `**/*.rs`. Compile coverage for every crate
+            # already runs on the always-on build/test lanes, so a dash-only or
+            # unrelated-crate Rust PR need not fire the heavy serve matrix. Err
+            # toward inclusion — Cargo.lock, the toolchain, and the workflow
+            # itself are broad safety nets so a transitive-dep or CI change still
+            # runs the matrix. (Excluded on purpose: crates/rocm-dash-* — they
+            # build into `rocm` but cannot change serve behaviour.)
+            serve:
+              - 'engines/**'
+              - 'crates/rocm-core/**'
+              - 'crates/rocm-engine-protocol/**'
+              - 'apps/rocm/**'
+              - 'apps/rocmd/**'
+              - 'tests/e2e-cucumber/**'
+              - 'crates/e2e-report/**'
+              - 'xtask/**'
+              - '**/*.feature'
+              - 'scripts/**'
+              # Root manifest only (NOT `**/Cargo.toml` — that would re-include
+              # the excluded dash crates): `[workspace.dependencies]` edits here
+              # can change a serving crate's deps without touching Cargo.lock.
+              - 'Cargo.toml'
+              - 'Cargo.lock'
+              - 'rust-toolchain*'
+              - '.github/workflows/**'
             # THIRD_PARTY_NOTICES.txt staleness gate: anything that changes the
             # dependency tree, the cargo-about config/template, the generator, or
             # the generated file itself.
@@ -741,7 +771,7 @@ jobs:
       && (
         (github.event_name != 'workflow_dispatch'
           && needs.build-and-test.result == 'success'
-          && needs.changes.outputs.heavy == 'true')
+          && needs.changes.outputs.serve == 'true')
         || (github.event_name == 'workflow_dispatch'
           && (inputs.platform == 'all' || inputs.platform == 'app-dev-gpu'))
       )
@@ -757,6 +787,12 @@ jobs:
       # workflow sets this unconditionally; here it lets a scoped dispatch confirm
       # a single nightly scenario (e.g. the 27B serve) without the full nightly run.
       E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}"
+      # PR canary mode: on a pull_request this MI300X lane serves ONLY the single
+      # @canary scenario (a minimal real vLLM serve) as a fast pre-merge smoke;
+      # the full serve matrix runs in the merge queue (merge_group forces this
+      # off). Every other @requires-gpu scenario resolves to Skip in the harness,
+      # so platform.json stays valid and the consolidated report reconciles.
+      E2E_PR_CANARY: "${{ github.event_name == 'pull_request' && '1' || '' }}"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
@@ -925,14 +961,28 @@ jobs:
     # carries `strix-halo`, but the paths below exist only on the native one.
     runs-on: [self-hosted, linux, strix-halo, native]
     needs: [changes, build-and-test]
-    # See `e2e`: dispatch tolerates skipped build-and-test; strix-ubuntu.
+    # Merge-queue-only heavy lane: this slow serve matrix runs in the merge queue
+    # (merge_group) and on push to main, NOT on every pull_request — the PR
+    # pre-merge serve smoke is the MI300X @canary lane above. A manual dispatch
+    # can still target it. The job is required-but-continue-on-error, so skipping
+    # it on a PR reports the required check as skipped (branch protection
+    # satisfied) without running the hardware.
+    #
+    # NOT a gating backstop: this lane is `continue-on-error` on merge_group too,
+    # so a Strix (lemonade / Windows) serve regression skips on the PR, fails
+    # non-blocking in the queue, and still lands on main — first hard signal is
+    # the nightly run, a day later, decoupled from the causing PR. Deliberate
+    # trade for scarce serial Strix hardware (a per-PR real Strix serve is the
+    # cost this change removes); the Strix lanes are proving-out and non-blocking
+    # by design. Drop `continue-on-error` on the merge_group path if/when they
+    # graduate to a true pre-merge gate.
     if: >-
       always()
       && needs.changes.result == 'success'
       && (
-        (github.event_name != 'workflow_dispatch'
+        ((github.event_name == 'merge_group' || github.event_name == 'push')
           && needs.build-and-test.result == 'success'
-          && needs.changes.outputs.heavy == 'true')
+          && needs.changes.outputs.serve == 'true')
         || (github.event_name == 'workflow_dispatch'
           && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu'))
       )
@@ -1082,14 +1132,17 @@ jobs:
     timeout-minutes: 35
     runs-on: [self-hosted, windows, strix-halo, native]
     needs: [changes, build-and-test]
-    # See `e2e`: dispatch tolerates skipped build-and-test; strix-windows.
+    # Merge-queue-only heavy lane (see e2e-gpu-strix-ubuntu): runs on merge_group
+    # and push to main, NOT on every pull_request; a manual dispatch can target
+    # it. Required-but-continue-on-error, so a PR skip satisfies branch
+    # protection without running the hardware.
     if: >-
       always()
       && needs.changes.result == 'success'
       && (
-        (github.event_name != 'workflow_dispatch'
+        ((github.event_name == 'merge_group' || github.event_name == 'push')
           && needs.build-and-test.result == 'success'
-          && needs.changes.outputs.heavy == 'true')
+          && needs.changes.outputs.serve == 'true')
         || (github.event_name == 'workflow_dispatch'
           && (inputs.platform == 'all' || inputs.platform == 'strix-windows'))
       )
@@ -1246,11 +1299,15 @@ jobs:
       - e2e-gpu
       - e2e-gpu-strix-ubuntu
       - e2e-gpu-strix-windows
-    # Consolidate whatever ran. On dispatch `heavy` is unset, so also run when the
-    # trigger was manual; `always()` still lets it collect partial/failed tiers.
+    # Consolidate whatever ran. Gate on heavy OR serve: the mock e2e lane runs on
+    # `heavy`, the GPU lanes on `serve`, and `serve` matches paths `heavy` does
+    # not — so a serve-only change runs GPU jobs whose report must still be
+    # produced. On dispatch both are unset, so also run when the trigger was
+    # manual; `always()` still lets it collect partial/failed tiers.
     if: >-
       always()
       && (needs.changes.outputs.heavy == 'true'
+        || needs.changes.outputs.serve == 'true'
         || github.event_name == 'workflow_dispatch')
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

@@ -88,10 +88,14 @@ E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e
 
 ## Tags and per-scenario expectations
 
-There is no tag-filter tiering. Each CI job runs the **whole** suite
-(`cargo xtask e2e`, no `-t` filter); the harness resolves every scenario to
-**pass / xfail / skip** at runtime from its capability tags plus the known-bug
-matrix, then reconciles the actual result against that expectation.
+There is no tag-filter tiering: a job runs `cargo xtask e2e` with no `-t` filter,
+and the harness resolves every scenario to **pass / xfail / skip** at runtime from
+its capability tags plus the known-bug matrix, then reconciles the actual result
+against that expectation. Two env-gated modes narrow *which* scenarios actually
+run (still via resolution, not a CLI filter): `E2E_PR_CANARY=1` on the per-PR
+MI300X lane runs only the `@canary` serve (see the job table below), and
+`E2E_INCLUDE_NIGHTLY` / `E2E_INCLUDE_LIFECYCLE` opt expensive scenarios in. Off
+those modes (merge queue, push, nightly, dispatch) a job runs the whole suite.
 
 Scenarios carry stable-id and capability tags:
 
@@ -104,6 +108,8 @@ Scenarios carry stable-id and capability tags:
 | `@serve-timeout:<secs>` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). |
 | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. |
 | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. |
+| `@canary` | The single minimal real-serve scenario used as the per-PR pre-merge smoke on the MI300X lane. In canary mode (`E2E_PR_CANARY=1`) it is the **only** GPU-serving scenario that runs; every other one resolves to skip. Exactly one scenario must carry this tag (enforced by a unit test). |
+| `@serves-on-gpu` | Marks a scenario that does a **real** `rocm serve` on a GPU host even though it is not `@requires-gpu` (its assertion is engine-agnostic, so it also runs mock-backed on the no-GPU lane). Needed so canary mode skips it too; without the tag it would launch an extra real serve on every PR canary run. |
 
 Known bugs are **not** tagged in the `.feature` files — they live in
 `expectations.toml`, keyed by `@id`, each with a `when = { ... }` condition (e.g.
@@ -122,18 +128,28 @@ serve does not compete with the first for device memory, and the failure quotes
 the service log tail plus the device's free-VRAM state, which is where the
 engine's own reason for the stall is recorded.
 
-CI runs one job per platform, each executing the full suite:
-
-| Job | Platform | Blocking |
-|---|---|---|
-| `e2e` | Mock (no GPU, GitHub-hosted) | yes |
-| `e2e-gpu` | MI300X (self-hosted) | no |
-| `e2e-gpu-strix-ubuntu` | Strix Halo / Ubuntu (self-hosted) | no |
-| `e2e-gpu-strix-windows` | Strix Halo / Windows (self-hosted) | no |
-
-The blocking mock job passes when every applicable scenario is pass-or-xfail with
-no XPASS or unexpected failure; the GPU jobs are non-blocking. The `e2e-report`
-job consolidates all platforms' results into one cross-platform report.
+CI runs one job per platform. Because real GPU serves are the wall-clock long
+pole on scarce serial hardware, the heavy GPU lanes are narrowed on a
+`pull_request` (see the `changes` job's `serve` paths-filter and the per-job
+`if:`/`E2E_PR_CANARY` gating in `ci.yml`):
+
+| Job | Platform | On `pull_request` | On merge_group / push / dispatch | Blocking |
+|---|---|---|---|---|
+| `e2e` | Mock (no GPU, GitHub-hosted) | full suite | full suite | yes |
+| `e2e-gpu` | MI300X (self-hosted) | `@canary` scenario only (canary mode) | full suite | no |
+| `e2e-gpu-strix-ubuntu` | Strix Halo / Ubuntu (self-hosted) | skipped | full suite | no |
+| `e2e-gpu-strix-windows` | Strix Halo / Windows (self-hosted) | skipped | full suite | no |
+
+Only serve-affecting PRs run the GPU lanes at all: they are gated on the `serve`
+paths-filter, so a PR that can't touch serving skips the whole matrix. When a GPU
+lane does run on a PR, only the MI300X `@canary` serve executes as a pre-merge
+smoke; the two Strix lanes skip on PRs (their required checks report as skipped,
+satisfying branch protection) and run the full suite in the merge queue and on
+push. The blocking mock job passes when every applicable scenario is pass-or-xfail
+with no XPASS or unexpected failure; the GPU jobs are non-blocking
+(`continue-on-error`) on every trigger — including merge_group — so they surface
+regressions without gating the queue. The `e2e-report` job consolidates all
+platforms' results into one cross-platform report.
 
 The nightly workflow runs three non-blocking jobs — the existing MI300X job and
 new Strix Halo jobs on Ubuntu and Windows — with `E2E_INCLUDE_NIGHTLY=1`. The

@@ -45,7 +45,9 @@ Feature: Chat and endpoint detection
   # real `rocm serve`, on the no-GPU mock lane it's backed by MockServer. The
   # assertion (a tools-bearing request is accepted) is engine-agnostic, so no GPU
   # is required — dropping @requires-gpu gives this per-PR mock-lane coverage.
-  @id:chat-tool-definitions-accepted
+  # @serves-on-gpu: it DOES real-serve on a GPU host, so PR canary mode skips it
+  # (the mock lane still covers it every PR).
+  @id:chat-tool-definitions-accepted @serves-on-gpu
   Scenario: 5 - Chat requests that include tool definitions are accepted
     Given a managed runtime is active
     And a model is served in the background
@@ -55,8 +57,9 @@ Feature: Chat and endpoint detection
   # Runs on every lane (see scenario 5): real serve on a GPU host, MockServer on
   # the no-GPU mock lane. Asserts only that a served model returns a non-empty
   # reply, which is engine-agnostic — real generation is covered by the
-  # @requires-gpu serve-*-inference scenarios.
-  @id:chat-end-to-end-local-model
+  # @requires-gpu serve-*-inference scenarios. @serves-on-gpu: real-serves on a
+  # GPU host, so PR canary mode skips it (mock lane still covers it every PR).
+  @id:chat-end-to-end-local-model @serves-on-gpu
   Scenario: 6 - End-to-end chat through a locally served model
     Given a managed runtime is active
     And a model is served in the background

@@ -5,12 +5,15 @@ Feature: Model serving
   # @expected-failure tags. Engine requirements are declared via @requires-engine
   # so the harness can skip a scenario whose engine can't start on this host.
 
-  @id:serve-short-name-expansion
+  # @serves-on-gpu: the step does a real `rocm serve` on a GPU host, so PR canary
+  # mode skips it (mock lane still covers name expansion every PR).
+  @id:serve-short-name-expansion @serves-on-gpu
   Scenario: 1 - Short model names are expanded to their full name
     When the user serves a model using its short name
     Then the output shows the full model name
 
-  @id:serve-short-name-consistent-across-engines
+  # @serves-on-gpu: real `rocm serve` on a GPU host (see scenario 1).
+  @id:serve-short-name-consistent-across-engines @serves-on-gpu
   Scenario: 2 - Short name expansion is consistent across engines
     When the user serves the same short name with different engines
     Then all engines expand to the same full model name
@@ -73,8 +76,12 @@ Feature: Model serving
     Then an engine is selected automatically
     And the model is reachable
 
-  # The inference half of scenario 6.
-  @id:serve-default-engine-inference @requires-gpu
+  # The inference half of scenario 6. Also the PR pre-merge serve canary
+  # (@canary): a real default-engine serve + inference that is ExpectPass on
+  # MI300X (only xfail on lemonade+linux, EAI-7423). In canary mode (the MI300X
+  # pull_request lane, E2E_PR_CANARY) this is the ONLY GPU scenario that serves;
+  # the full matrix runs in the merge queue. See resolve()/@canary in expectation.rs.
+  @id:serve-default-engine-inference @requires-gpu @canary
   Scenario: 6b - A default-engine served model responds to inference requests
     Given a managed runtime is active
     When the user serves a model without specifying an engine