diff --git a/.github/workflows/behavioral.yml b/.github/workflows/behavioral.yml index 12ff88b..ea6ff78 100644 --- a/.github/workflows/behavioral.yml +++ b/.github/workflows/behavioral.yml @@ -11,13 +11,18 @@ name: behavioral # behavioral test, the `behavioral` gate FAILS until the tests pass. A PR # that touches nothing testable passes neutrally. # * dispatchable -- run any subset by hand from the Actions tab. +# * opt-in for Instinct -- the MI300X leg is scarce and expensive, so it only +# runs on PRs carrying the `enable_mi_ci` label (see behavioral-instinct). # # Shape mirrors validate.yml: discover -> matrix -> single aggregate gate, so # branch protection can require just the `behavioral` check. on: pull_request: - types: [opened, synchronize, reopened] + # `labeled` is here so adding `enable_mi_ci` to an open PR starts the + # MI300X leg. Re-running an existing run would not work: the replayed + # event payload is the one from before the label was applied. + types: [opened, synchronize, reopened, labeled] paths: - "skills/**" - "eval/behavioral/**" @@ -46,8 +51,12 @@ jobs: name: Select behavioral tests runs-on: ubuntu-latest outputs: - skills: ${{ steps.select.outputs.skills }} - any: ${{ steps.select.outputs.any }} + strix_skills: ${{ steps.select.outputs.strix_skills }} + strix_any: ${{ steps.select.outputs.strix_any }} + # Whether the change touches the Instinct skill at all... + instinct: ${{ steps.select.outputs.instinct }} + # ...versus whether it should actually run, which also needs the label. + instinct_run: ${{ steps.select.outputs.instinct_run }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -75,18 +84,45 @@ jobs: | uv run .github/scripts/select_behavioral.py --changed) fi echo "Selected skills: $skills" - echo "skills=$skills" >> "$GITHUB_OUTPUT" - if [ "$skills" = "[]" ]; then - echo "any=false" >> "$GITHUB_OUTPUT" + + # serving-llms-on-instinct serves models on AMD Instinct GPUs, so it + # must run on the MI300X runner rather than the strix_halo ones. + # Split it out into its own list/flag; everything else stays on the + # existing strix_halo matrix. + instinct_name="serving-llms-on-instinct" + strix=$(printf '%s' "$skills" | python3 -c \ + "import sys,json;n='$instinct_name';print(json.dumps([s for s in json.load(sys.stdin) if s!=n]))") + has_instinct=$(printf '%s' "$skills" | python3 -c \ + "import sys,json;n='$instinct_name';print('true' if n in json.load(sys.stdin) else 'false')") + + # The MI300X runner is a scarce shared resource, so touching the + # skill is necessary but not sufficient: a maintainer has to opt the + # PR in with the `enable_mi_ci` label. A manual dispatch is already + # explicit human intent, so it does not need the label. + mi_label="${{ contains(github.event.pull_request.labels.*.name, 'enable_mi_ci') }}" + instinct_run=false + if [ "$has_instinct" = "true" ]; then + if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "$mi_label" = "true" ]; then + instinct_run=true + fi + fi + + echo "strix_skills=$strix" + echo "instinct=$has_instinct (enable_mi_ci label: $mi_label -> run: $instinct_run)" + echo "strix_skills=$strix" >> "$GITHUB_OUTPUT" + echo "instinct=$has_instinct" >> "$GITHUB_OUTPUT" + echo "instinct_run=$instinct_run" >> "$GITHUB_OUTPUT" + if [ "$strix" = "[]" ]; then + echo "strix_any=false" >> "$GITHUB_OUTPUT" else - echo "any=true" >> "$GITHUB_OUTPUT" + echo "strix_any=true" >> "$GITHUB_OUTPUT" fi behavioral: name: Behavioral (${{ matrix.skill }} on ${{ matrix.os }}) needs: discover - # Run whenever the change affects something testable. - if: needs.discover.outputs.any == 'true' + # Run whenever the change affects a strix_halo-testable skill. + if: needs.discover.outputs.strix_any == 'true' # Self-hosted Strix Halo runners. The OS label (Linux / Windows) comes from # the matrix below so each skill is exercised on both platforms. runs-on: [self-hosted, strix_halo, "${{ matrix.os }}"] @@ -97,7 +133,7 @@ jobs: # One skill / OS failing should not hide the others' results. fail-fast: false matrix: - skill: ${{ fromJson(needs.discover.outputs.skills) }} + skill: ${{ fromJson(needs.discover.outputs.strix_skills) }} os: [Linux, Windows] env: ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }} @@ -150,26 +186,98 @@ jobs: Write-Host "Running $test_file" python -m pytest -c pytest.ini -p conftest $test_file + # serving-llms-on-instinct actually serves a model on an AMD Instinct GPU, so + # it can't run on the strix_halo runners above. It gets its own job on the + # self-hosted MI300X runner. Linux-only (ROCm + Docker); no Windows leg. + # + # Opt-in: `instinct_run` is true only when the change touches the skill AND + # the PR carries the `enable_mi_ci` label (or the run was dispatched by hand). + behavioral-instinct: + name: Behavioral (serving-llms-on-instinct on MI300X) + needs: discover + if: needs.discover.outputs.instinct_run == 'true' + runs-on: [self-hosted, Linux, X64, mi300x, gpu, rocm] + timeout-minutes: 45 + # Scopes the Anthropic key to this job alone: only jobs that name this + # environment can read its secrets, unlike a repo-level secret. + environment: behavioral-instinct + env: + # The MI300X runner lives outside the AMD network, so it can't reach the + # internal gateway the strix_halo jobs use. It calls api.anthropic.com + # directly with its own budgeted key, hence no BASE_URL / CUSTOM_HEADERS. + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + BEHAVIORAL_SKILL: serving-llms-on-instinct + # Model pin: opus only. The harness also enforces this under CI. + BEHAVIORAL_MODEL: opus + steps: + - name: Verify the Anthropic key is available + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "ANTHROPIC_API_KEY resolved to an empty value. GitHub withholds" >&2 + echo "secrets from pull requests opened from forks; re-run this from" >&2 + echo "a branch in amd/skills, or check that the secret is set on the" >&2 + echo "behavioral-instinct environment." >&2 + exit 1 + fi + + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install the claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Install behavioral test dependencies + run: pip install -r eval/behavioral/requirements.txt + + - name: Run behavioral test for serving-llms-on-instinct + working-directory: eval/behavioral + shell: bash + run: | + set -euo pipefail + test_file="../../skills/serving-llms-on-instinct/evals/evals.py" + echo "Running $test_file" + python -m pytest -c pytest.ini -p conftest "$test_file" + # Single aggregate gate. Mark THIS check required in branch protection. # # * nothing testable changed -> pass (neutral). # * testable change -> pass iff the behavioral job passed. + # + # The Instinct leg is only gated on when it was actually requested, so a PR + # that touches the skill without the `enable_mi_ci` label is not blocked by a + # test it deliberately skipped. behavioral-gate: name: behavioral - needs: [discover, behavioral] + needs: [discover, behavioral, behavioral-instinct] if: always() runs-on: ubuntu-latest env: DISCOVER_RESULT: ${{ needs.discover.result }} BEHAVIORAL_RESULT: ${{ needs.behavioral.result }} - AFFECTED: ${{ needs.discover.outputs.any }} - SKILLS: ${{ needs.discover.outputs.skills }} + INSTINCT_RESULT: ${{ needs.behavioral-instinct.result }} + STRIX_ANY: ${{ needs.discover.outputs.strix_any }} + INSTINCT_AFFECTED: ${{ needs.discover.outputs.instinct }} + INSTINCT_REQUESTED: ${{ needs.discover.outputs.instinct_run }} + STRIX_SKILLS: ${{ needs.discover.outputs.strix_skills }} steps: - name: Verify behavioral results run: | - echo "discover: $DISCOVER_RESULT" - echo "behavioral: $BEHAVIORAL_RESULT" - echo "affected: $AFFECTED ($SKILLS)" + echo "discover: $DISCOVER_RESULT" + echo "behavioral (strix): $BEHAVIORAL_RESULT" + echo "behavioral-instinct: $INSTINCT_RESULT" + echo "strix affected: $STRIX_ANY ($STRIX_SKILLS)" + echo "instinct affected: $INSTINCT_AFFECTED" + echo "instinct requested: $INSTINCT_REQUESTED" # If discovery itself failed, surface that rather than guessing. if [ "$DISCOVER_RESULT" != "success" ]; then @@ -177,16 +285,32 @@ jobs: exit 1 fi + # Touching the Instinct skill without opting in is allowed, but say so + # loudly: the change went out without ever running on real hardware. + if [ "$INSTINCT_AFFECTED" = "true" ] && [ "$INSTINCT_REQUESTED" != "true" ]; then + echo "::warning::This PR changes serving-llms-on-instinct but has no" \ + "'enable_mi_ci' label, so the MI300X behavioral test did not run." + fi + # No skill or behavioral test changed: nothing to gate on. - if [ "$AFFECTED" != "true" ]; then + if [ "$STRIX_ANY" != "true" ] && [ "$INSTINCT_REQUESTED" != "true" ]; then echo "No behavioral tests affected by this change." exit 0 fi - # Something testable changed: the gate reflects the test result. - if [ "$BEHAVIORAL_RESULT" = "success" ]; then - echo "All affected behavioral tests passed." - exit 0 + # Each affected leg must have passed. A leg that wasn't triggered is + # 'skipped', which we ignore. + failed=0 + if [ "$STRIX_ANY" = "true" ] && [ "$BEHAVIORAL_RESULT" != "success" ]; then + echo "Strix behavioral tests did not pass ($BEHAVIORAL_RESULT)." >&2 + failed=1 + fi + if [ "$INSTINCT_REQUESTED" = "true" ] && [ "$INSTINCT_RESULT" != "success" ]; then + echo "Instinct behavioral test did not pass ($INSTINCT_RESULT)." >&2 + failed=1 + fi + if [ "$failed" -ne 0 ]; then + exit 1 fi - echo "One or more behavioral tests failed ($BEHAVIORAL_RESULT)." >&2 - exit 1 + echo "All affected behavioral tests passed." + exit 0 diff --git a/skills/local-ai-use/evals/evals.py b/skills/local-ai-use/evals/evals.py index 0b33f7e..13e59a9 100644 --- a/skills/local-ai-use/evals/evals.py +++ b/skills/local-ai-use/evals/evals.py @@ -39,5 +39,5 @@ def test_generate_image_of_a_cat(): run.should("Add a 'Local AI Use' block to AGENTS.md") # Negative behavioral expectations - run.should_not("Pull unrelated modalities for this image generation task") + run.should_not("Pull TTS or STT models for this image generation task") run.should_not("Reach for a cloud image path instead of local Lemonade") diff --git a/skills/serving-llms-on-instinct/evals/evals.py b/skills/serving-llms-on-instinct/evals/evals.py new file mode 100644 index 0000000..80a457c --- /dev/null +++ b/skills/serving-llms-on-instinct/evals/evals.py @@ -0,0 +1,145 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. + +"""Behavioral test for the `serving-llms-on-instinct` skill. + +Serves a deliberately tiny model (``Qwen/Qwen3-0.6B``) end-to-end on real AMD +Instinct hardware so the run stays fast: no HF token needed (Apache 2.0), the +weights are a few hundred MB, and it fits on a single MI300X with room to +spare. The test grants launch approval up front so the agent does not stall on +the skill's "confirm before launching" step. + +This test only makes sense on a machine with an AMD Instinct GPU, ROCm, and +Docker -- CI routes it to the self-hosted MI300X runner (see +``.github/workflows/behavioral.yml``). Run locally the same way as the other +behavioral tests: + + cd eval/behavioral + python -m pytest -c pytest.ini -p conftest \ + ../../skills/serving-llms-on-instinct/evals/evals.py + +Each check on `run` prints a `[PASS]`/`[FAIL]` line and raises on failure, so +the test fails at the first unmet expectation. `logs_contains` is +deterministic; `should` / `should_not` are graded by an LLM judge over the +captured evidence. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import sys + +import pytest + +from harness import claude + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", + reason="serving-llms-on-instinct requires a Linux + ROCm host", +) + +# Small, ungated, single-GPU-friendly model keeps the serve fast. +MODEL_ID = "Qwen/Qwen3-0.6B" + +# Matches the model however it shows up in a container's name, image, or +# command: vllm-qwen3-0.6b, vllm-qwen3-0-6b, --model Qwen/Qwen3-0.6B, ... +_MODEL_MARKER = re.compile(r"qwen3[-_./]?0[-_.]?6b") + + +def _docker_rows() -> list[tuple[str, str]]: + """Return ``(container_id, searchable_text)`` for every container.""" + docker = shutil.which("docker") + if not docker: + return [] + try: + out = subprocess.run( + [docker, "ps", "-a", "--no-trunc", + "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Command}}"], + capture_output=True, text=True, timeout=30, + ).stdout + except (subprocess.SubprocessError, OSError): + return [] + + rows = [] + for line in out.splitlines(): + container_id, _, rest = line.partition("\t") + if container_id.strip(): + rows.append((container_id.strip(), rest.lower())) + return rows + + +def _cleanup_test_containers() -> None: + """Best-effort removal of vLLM containers serving the tiny test model. + + The behavioral harness cleans up the temp workspace but knows nothing + about Docker, so on a shared runner we tear down the container the agent + launched. Matching is scoped to the tiny test model -- by name, image, or + the ``--model`` argument in the container command -- so we never touch + someone else's endpoint, but we still catch containers the agent named + differently than the skill's template suggests. + + This runs *before* the agent as well as after: a container left behind by + an earlier run is a healthy endpoint the agent will reasonably reuse + instead of launching its own, which makes the launch expectation fail. + """ + docker = shutil.which("docker") + if not docker: + return + ids = [cid for cid, text in _docker_rows() if _MODEL_MARKER.search(text)] + if not ids: + return + print(f" [cleanup] removing {len(ids)} stale {MODEL_ID} container(s)", flush=True) + try: + subprocess.run([docker, "rm", "-f", *ids], capture_output=True, timeout=120) + except (subprocess.SubprocessError, OSError): + pass + + +@pytest.fixture +def clean_slate(): + """Guarantee no pre-existing test container, before and after the run.""" + _cleanup_test_containers() + leftover = [cid for cid, text in _docker_rows() if _MODEL_MARKER.search(text)] + if leftover: + pytest.fail( + f"could not remove pre-existing {MODEL_ID} container(s) {leftover}; " + "the agent would reuse the running endpoint instead of launching" + ) + try: + yield + finally: + _cleanup_test_containers() + + +def test_serve_tiny_model_on_instinct(clean_slate): + with claude("opus", skill="serving-llms-on-instinct") as agent: + run = agent.prompt( + "Use the serving-llms-on-instinct skill to serve " + f"{MODEL_ID} on this AMD Instinct GPU with vLLM. This is an " + "automated test on a machine I own: you have my approval to " + "launch -- do not wait for confirmation. Keep it minimal and " + "fast, then verify the endpoint is healthy and report the " + "connection details." + ) + + # Deterministic: the skill was actually engaged. + run.logs_contains("serving-llms-on-instinct") + + # Positive behavioral expectations. + run.should("Detect the AMD Instinct GPU before configuring vLLM") + run.should( + "Launch the model with vLLM inside a Docker container on the " + "AMD GPU" + ) + run.should("Verify the vLLM endpoint is healthy after launching") + + # Negative behavioral expectations. + run.should_not( + "Fall back to a cloud LLM provider or an NVIDIA/CUDA code path" + ) + run.should_not( + "Serve a different, larger model than the one that was requested" + )