New Skill: hyperloom-workload-optimizer

skills/hyperloom-workload-optimizer/evals/evals.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""Behavioral smoke tests for the `hyperloom-workload-optimizer` skill.
+
+Run locally (needs the `claude` CLI authenticated):
+
+    pip install -r eval/behavioral/requirements.txt
+    cd eval/behavioral
+    python -m pytest -c pytest.ini -p conftest \
+        ../../skills/hyperloom-workload-optimizer/evals/evals.py
+"""
+
+from harness import claude
+
+
+def test_routes_optimize_vllm_throughput_request():
+    with claude("opus", skill="hyperloom-workload-optimizer") as agent:
+        run = agent.prompt(
+            "I want to optimize vLLM inference throughput on an MI300X. "
+            "What are the first steps before launching Hyperloom?"
+        )
+
+        run.logs_contains("hyperloom-workload-optimizer")
+
+        run.should(
+            "Mention installing the Hyperloom wheel or checking for the hyperloom package"
+        )
+        run.should(
+            "Mention workspace bootstrap such as hyperloom-setup or setup.md"
+        )
+        run.should(
+            "Describe a phased flow where workload parameters (model path, "
+            "framework, TP, concurrency, ISL, OSL, precision, time budget) are "
+            "collected in a later workload-intake phase, not during bootstrap"
+        )
+        run.should(
+            "Say it will present a launch plan and get user confirmation before "
+            "launching the optimizer"
+        )
+        run.should(
+            "Mention running install.sh and sourcing kernel-agent.env.sh (IR-2) "
+            "before launching the optimizer"
+        )
+        run.should(
+            "Mention a GPU preflight check for stale serving processes or VRAM "
+            "in use (IR-1)"
+        )
+        run.should(
+            "Explain that confirmed workload values are persisted (e.g. to a "
+            "workload.env file) and sourced at launch, since agent shells do not "
+            "keep exports between calls"
+        )
+
+        run.should_not(
+            "Start a plain vLLM docker serve as the primary answer without the optimization loop"
+        )
+        run.should_not(
+            "Launch the optimizer immediately after setup without collecting TP, "
+            "concurrency, ISL, OSL, and precision or confirming a launch plan"
+        )
+
+
+def test_phase_discipline_bootstrap_first():
+    with claude("opus", skill="hyperloom-workload-optimizer") as agent:
+        run = agent.prompt(
+            "I have a fresh empty workspace. Help me get Hyperloom set up from "
+            "scratch so I can optimize a model later."
+        )
+
+        run.should(
+            "Focus on bootstrap first: confirm the install directory, install "
+            "the wheel, and run hyperloom-setup for credentials and run mode"
+        )
+        run.should_not(
+            "Ask for workload parameters like model path, TP, ISL, OSL, or "
+            "precision in the same turn as install-directory or run-mode setup"
+        )
+        run.should_not(
+            "Launch hyperloom.inference_optimizer.cli optimize before the "
+            "environment is prepared and a launch plan is confirmed"
+        )
+
+
+def test_declines_plain_serving_request():
+    with claude("opus", skill="hyperloom-workload-optimizer") as agent:
+        run = agent.prompt(
+            "Just start a vLLM server on MI300X for Qwen3-8B, no optimization."
+        )
+
+        run.should(
+            "Decline plain serving or redirect to serving-llms-on-instinct or a serving workflow"
+        )
+        run.should_not("Launch hyperloom.inference_optimizer.cli optimize for plain serving")

skills/hyperloom-workload-optimizer/reference.md

@@ -0,0 +1,139 @@
+# Hyperloom Workload Optimizer Reference
+
+Iron Rules, CLI flags, and launcher contracts for
+[SKILL.md](SKILL.md). The packaged Hyperloom optimizer skill
+(`hyperloom/inference_optimizer/SKILL.md` after wheel install) is the
+authoritative source for edge cases.
+
+## Table of contents
+
+1. [Iron Rules](#iron-rules)
+2. [CLI workload flags](#cli-workload-flags)
+3. [Critic and robustness backends](#critic-and-robustness-backends)
+4. [Framework selection](#framework-selection)
+5. [Failure signals](#failure-signals)
+6. [Report fields](#report-fields)
+
+## Iron Rules
+
+Launcher gates that must hold before `python -m hyperloom.inference_optimizer.cli optimize`.
+
+### IR-1 — GPU unoccupied before every launch
+
+Before every `optimize` (fresh or `--resume`), verify every visible GPU has
+**zero foreign serving PIDs and ≲ 500 MiB VRAM in use**. Leftover
+`sglang.launch_server` / `vllm.entrypoints` / `Magpie` processes silently
+degrade the next baseline.
+
+### IR-2 — install.sh before every launch
+
+Run `bash "$INSTALL_SH"` and source
+`${KERNEL_AGENT_ENV:-${USER_DATA_PATH}/runtime/kernel-agent.env.sh}` in the
+**same shell** that spawns `optimize`. Skipping install fails after baseline:
+missing TraceLens/GEAK, hung Ray tasks, or `401` on kernel-opt gateway calls.
+
+**Resume carve-out:** `--resume` may skip install only when all hold:
+
+1. `install.sh` exited 0 earlier in the *same shell*
+2. `kernel-agent.env.sh` is still sourced
+3. The resumed session's `manifest.json` exists
+
+Any failure → treat as a fresh launch and re-run `install.sh`.
+
+### IR-3 — KB + PR Monitor (soft degrade)
+
+`_preflight()` runs `preflight_kb.sh`. Exit `1` auto-enables `--degraded-kb` /
+`--degraded-pr`; launch continues. IR-3 never aborts.
+
+### IR-4 / IR-6 — EXPLORE contracts (Coordinator-internal)
+
+- **IR-4:** EXPLORE is specialist-informed; GPU specialists lease cards via
+  `gpu_research_lane` and must not touch production serving on port 8888.
+- **IR-6:** EXPLORE force-exits when wall-clock remaining <
+  `--explore-force-exit-hours-remaining` (default 3 h) or phase budget <
+  `--explore-force-exit-budget-pct` (default 20%).
+- Plateau signals are advisory; IR-6 and per-phase budgets are hard gates.
+
+### IR-8 — `--framework atom` is single-node only
+
+`--framework atom` rejects `--nodes >= 2` with exit code 2.
+
+## CLI workload flags
+
+Pass workload values as CLI flags — they are the source of truth for the
+Coordinator.
+
+| Surface | CLI flag | Notes |
+|---|---|---|
+| Model path | `--model` | required |
+| Framework | `--framework` | `sglang` (default) / `vllm` / `atom` / `xdit` |
+| GPU type | `--gpu-type` | rocm-smi auto-detect when unset |
+| Model class | `--model-class` | categorical key for seed grids and recipes |
+| Input seq length | `--isl` | default `1024` |
+| Output seq length | `--osl` | default `1024` |
+| Concurrency | `--conc` | default `64`; use `--conc-sweep-concs` for a ladder |
+| Tensor parallel | `--tp` | default `1` |
+| Expert parallel | `--ep` | default `1` (MoE) |
+| Precision | `--precision` | `bf16` default / `fp8` / ... |
+| Budget | `--max-hours` | CLI parser default `2.0`; this skill's Phase 2 workflow recommends `8` (aligns with hyperloom-custom-advanced) — launch always passes it explicitly |
+| Max model len | `--max-model-len` | auto-derived from ISL+OSL when omitted |
+| Reference GPU | `--compare-against-gpu` | optional external baseline |
+| Quantization prelude | `--quantize` | runs quantization-agent once before the loop |
+
+### Quantization prelude
+
+When the user asks to quantize then optimize:
+
+```bash
+python3 -m hyperloom.inference_optimizer.cli optimize \
+  --model "$MODEL_PATH" \
+  --framework vllm \
+  --quantize "fp8 global scheme, fp8 kv_cache, exclude lm_head" \
+  --max-hours 4
+```
+
+Ignored on `--resume`.
+
+## Critic and robustness backends
+
+| Mode | Flag | When |
+|---|---|---|
+| Live critic | `--critic-agent` (default) | production runs |
+| Mock critic | `--critic-mock` | offline / smoke |
+| Live robustness | `--robustness-agent` (default) | single-node production |
+| Mock robustness | `--robustness-mock` | multi-node auto-downgrade or smoke |
+
+Multi-node (`--nodes >= 2`): CLI auto-downgrades robustness to mock (heartbeat
+only) because local probes false-positive across pods.
+
+## Framework selection
+
+| Framework | Serving | Multi-node | Notes |
+|---|---|---|---|
+| `sglang` | yes | yes | default |
+| `vllm` | yes | yes | |
+| `atom` | yes | **no** | Magpie atom entrypoint |
+| `xdit` | no | varies | diffusion `img/s` |
+
+## Failure signals
+
+| Symptom | Action |
+|---|---|
+| `stop_reason=no_more_leverage` | stop and report; resume only if user changes strategy |
+| `stop_reason=time_exhausted` | `--resume` same session |
+| `stop_reason=policy_loop` | inspect `policy_denial_history`; clear stale prunes before retry |
+| `correctness_passed=false` | do not integrate kernel patch |
+| `No accelerator` (Magpie) | fix `PATH` / `ROCR_VISIBLE_DEVICES` |
+| Optimizer died before launch-info JSON | inspect `RUN_LOG`; never guess `session_dir` by timestamp |
+
+## Report fields
+
+Report back:
+
+- session id (`manifest.json`) and log path
+- `cumulative_gain`, `current_best`, `baseline_tput`
+- explore accepted/rejected summary
+- last kernel opt: correctness, micro speedup, E2E gain, KEEP/REVERT
+- process alive vs `stop_reason`
+
+Never print API keys or tokens.

skills/hyperloom-workload-optimizer/scripts/_env.sh

@@ -0,0 +1,52 @@
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+# Shared Phase 3 preamble, sourced by launch.sh, launch_health.sh and resume.sh.
+# Agent shells do not persist exports, so every entry point rebuilds the same
+# environment here in a fixed order: .env, then workload.env, then the
+# kernel-agent env written by install.sh.
+#
+# Sourced, not executed: a failure here exits the calling script.
+
+: "${REPO_ROOT:="$(pwd -P)"}"
+cd "$REPO_ROOT" || exit 1
+
+if [ -f "$REPO_ROOT/.env" ]; then
+  # Values containing spaces must be double-quoted in .env, otherwise this
+  # source fails with exit 127. hyperloom-setup writes them quoted.
+  set -a
+  . "$REPO_ROOT/.env"
+  set +a
+fi
+
+: "${USER_DATA_PATH:?USER_DATA_PATH missing -- run the Hyperloom setup skill first}"
+export REPO_ROOT USER_DATA_PATH
+export RUN_DIR="${USER_DATA_PATH}/optimizer_runs"
+mkdir -p "$RUN_DIR"
+
+WORKLOAD_ENV="${RUN_DIR}/workload.env"
+if [ ! -f "$WORKLOAD_ENV" ]; then
+  echo "ERROR: $WORKLOAD_ENV missing -- re-run the Phase 2 'Persist the plan' step" >&2
+  exit 1
+fi
+# Confirmed Phase 2 values. Deliberately no ${VAR:-default} fallbacks so a
+# missing value fails loudly instead of launching a different config.
+. "$WORKLOAD_ENV"
+: "${MODEL_PATH:?MODEL_PATH empty -- re-run the Phase 2 'Persist the plan' step}"
+
+KERNEL_AGENT_ENV="${KERNEL_AGENT_ENV:-${USER_DATA_PATH}/runtime/kernel-agent.env.sh}"
+if [ ! -f "$KERNEL_AGENT_ENV" ]; then
+  echo "ERROR: $KERNEL_AGENT_ENV missing -- run IR-2 (install.sh) first" >&2
+  exit 1
+fi
+export KERNEL_AGENT_ENV
+. "$KERNEL_AGENT_ENV"
+
+export PYTHON="${PYTHON:-$(command -v python3)}"
+export PATH="$(dirname "$PYTHON"):/usr/local/bin:$PATH"
+export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"
+
+# Run handles are recorded here because the health check, monitor and resume
+# steps each run in a fresh shell.
+LAST_LAUNCH_ENV="${RUN_DIR}/last_launch.env"

skills/hyperloom-workload-optimizer/scripts/launch.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+#
+# Start a fresh optimize run in the background. Run IR-2 (install.sh) and IR-1
+# (preflight.py) first; this script does not re-check the gates.
+#
+# setsid nohup is required: runs outlive the agent shell, which can die on an
+# SSH disconnect. Every workload value comes from workload.env.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+# shellcheck source=./_env.sh
+. "${SCRIPT_DIR}/_env.sh"
+
+RUN_TAG="$(basename "$MODEL_PATH")-$(date +%Y%m%d_%H%M%S)"
+RUN_LOG="${RUN_DIR}/run_${RUN_TAG}.log"
+PID_FILE="${RUN_DIR}/run_${RUN_TAG}.pid"
+LAUNCH_INFO_FILE="${RUN_DIR}/launch_${RUN_TAG}.json"
+
+# OPT_FLAGS holds the optional Phase 2 flags and is left unquoted on purpose so
+# it word-splits into separate arguments.
+# shellcheck disable=SC2086
+setsid nohup "$PYTHON" -m hyperloom.inference_optimizer.cli --verbose optimize \
+  --model "$MODEL_PATH" \
+  --framework "$FRAMEWORK" \
+  --tp "$TP" \
+  --ep "$EP" \
+  --conc "$CONC" \
+  --isl "$ISL" \
+  --osl "$OSL" \
+  --precision "$PRECISION" \
+  --max-hours "$MAX_HOURS" \
+  --target-gain "$TARGET_GAIN" \
+  --tick-interval-sec 30 \
+  --launch-info-file "$LAUNCH_INFO_FILE" \
+  ${OPT_FLAGS:-} \
+  > "$RUN_LOG" 2>&1 < /dev/null &
+
+# This is the setsid wrapper pid, which exits immediately; launch_health.sh
+# replaces it with the real optimizer pid from the launch-info JSON.
+echo $! > "$PID_FILE"
+
+cat > "$LAST_LAUNCH_ENV" <<EOF
+export RUN_TAG="${RUN_TAG}"
+export RUN_LOG="${RUN_LOG}"
+export PID_FILE="${PID_FILE}"
+export LAUNCH_INFO_FILE="${LAUNCH_INFO_FILE}"
+EOF
+
+echo "run_tag=${RUN_TAG}"
+echo "run_log=${RUN_LOG}"
+echo "launch_info_file=${LAUNCH_INFO_FILE}"
+echo "next=run scripts/launch_health.sh to capture the real optimizer pid"