serving-llms-on-epyc: update skill

skills/serving-llms-on-epyc/data/epyc.json

@@ -1,18 +1,17 @@
 {
-  "vllm_version": "0.22.0",
+  "vllm_version": "0.25.1",
   "container": {
-    "image": "amdih/zendnn_zentorch:vllm_v0.22.0_zentorch_v2.11.0.1_ubuntu22.04_2026_ww23",
+    "image": "amdih/zendnn_zentorch:vllm_v0.25.1_zentorch_v2.11.0.3_ubuntu22.04_2026_ww30",
     "runtimes": ["docker", "podman"],
-    "comment": "Public vLLM + zentorch CPU image on Docker Hub (amdih/zendnn_zentorch) -- no internal-registry access needed. Tags are vllm_v<ver>_zentorch_v<ver>_<os>_<build>; prefer the newest ubuntu22.04 stable. Both docker and podman are supported; the skill prefers docker and falls back to podman.",
+    "comment": "Pinned public vLLM + zentorch CPU image on Docker Hub (amdih/zendnn_zentorch) -- no internal-registry access needed. Update this image and vllm_version together so model-registry checks match the served runtime. Both docker and podman are supported; the skill prefers docker and falls back to podman.",
     "run_flags": [
       "--ipc=host",
-      "--shm-size=16g",
       "--network=host"
     ],
     "hf_cache_mount": "-v ~/.cache/huggingface:/root/.cache/huggingface",
     "flag_notes": {
-      "--ipc=host": "vLLM workers use host IPC/shared memory.",
-      "--shm-size=16g": "vLLM needs a large /dev/shm; default 64MB is not enough.",
+      "--ipc=host": "vLLM workers need a large /dev/shm; --ipc=host shares the host's (large) shared memory, which covers it. Do NOT also pass --shm-size: podman rejects '--shm-size' together with '--ipc=host' (cannot set shmsize in host IPC namespace), and it is redundant on docker too.",
+      "shm_alternative": "If you must isolate IPC (drop --ipc=host), then add --shm-size=16g instead (the 64MB container default is too small for vLLM). Use one or the other, never both.",
       "--network=host": "Expose the served port directly. Alternative: -p <port>:<port>.",
       "numa": "A single instance is pinned to ONE socket plus its memory. cpu_tune.py picks a free socket by CPU load on dual-socket hosts (warns if both busy; --socket N forces), sizes KV from that socket's local RAM, and emits --cpuset-cpus + --cpuset-mems (container) or numactl --cpunodebind/--membind (conda). True multi-socket scaling = multiple instances (one per socket), out of scope here."
     }
@@ -34,7 +33,7 @@
   "default_model": "Qwen/Qwen3-0.6B",
   "default_model_notes": "Ungated (Apache-2.0), tiny, fast first success on CPU. For a real workload pick a larger Qwen3 / Llama once the flow is verified.",
   "smoke_model": "Qwen/Qwen3-0.6B",
-  "smoke_model_notes": "Current small Qwen, chat-capable (ships a chat template, so /v1/chat/completions works -- unlike base models such as opt-125m).",
+  "smoke_model_notes": "Current small Qwen, chat-capable (ships a chat template, so /v1/chat/completions works -- unlike base models such as allenai/OLMo-2-0425-1B, which have no chat template and serve /v1/completions).",
   "env_defaults": {
     "VLLM_CPU_OMP_THREADS_BIND": "set by cpu_tune.py (physical cores of the chosen socket)",
     "VLLM_CPU_KVCACHE_SPACE": "set by cpu_tune.py (GB)",

skills/serving-llms-on-epyc/reference.md

@@ -1,14 +1,60 @@
 # serving-llms-on-epyc -- Reference
 
 ## Table of Contents
-1. [Runtime selection](#runtime-selection)
-2. [Container run flags (CPU)](#container-run-flags-cpu)
-3. [Precision and modality](#precision-and-modality)
-4. [CPU sizing](#cpu-sizing)
-5. [Known quirks](#known-quirks)
+1. [Hardware support](#hardware-support)
+2. [Runtime and stack compatibility](#runtime-and-stack-compatibility)
+3. [Runtime selection](#runtime-selection)
+4. [Container run flags (CPU)](#container-run-flags-cpu)
+5. [Precision and modality](#precision-and-modality)
+6. [Client endpoints and parameters](#client-endpoints-and-parameters)
+7. [CPU sizing](#cpu-sizing)
+8. [Known quirks](#known-quirks)
 
 ---
 
+## Hardware support
+
+This recipe supports the **AMD EPYC 9000 server series** for now: Genoa (9004),
+Turin (9005), and 6th Gen [Venice (9006)](https://ir.amd.com/news-events/press-releases/detail/1294/aai-2026-amd-delivers-full-stack-compute-for-the-agentic-ai-era)
+(launched at Advancing AI 2026). `scripts/detect.py` reports only these three
+generations as `is_supported_epyc: true`.
+
+AVX-512 is necessary but not sufficient for this support policy. Other EPYC parts
+-- Bergamo and Siena, and the AM5 EPYC 4004/4005 -- expose the required ISA but are
+outside this skill's current 9000-series scope; the detector still names them but
+reports `is_supported_epyc: false`. Do not infer support from AVX-512 alone.
+
+The presence of AMD Instinct GPUs does not change CPU support. Use this skill
+when the requested endpoint should execute on EPYC; use
+`serving-llms-on-instinct` when it should execute on a GPU. Both serving engines
+may coexist on the same host.
+
+## Runtime and stack compatibility
+
+Detecting a supported CPU is not the same as running a validated software stack.
+`scripts/validate.py --generation <gen>` probes the **selected** runtime (the
+container image when present, else the conda/host env) for its exact
+`vllm`/`zentorch`/`torch` versions and the **active vLLM platform**, then reports
+`compatibility.status`:
+
+- `proceed` -- a Zen platform is active (zentorch acceleration on) and the stack is
+  the validated default or a validated family.
+- `blocked` (error) -- the stock `CpuPlatform` is active, so serving would run
+  **without** zentorch. Two vLLM paths select a Zen platform: the in-tree
+  `ZenCpuPlatform` (vLLM detects an AMD AVX-512 CPU with `zentorch` importable) and
+  the out-of-tree `zentorch` plugin. If neither is active, fix the environment or
+  use the pinned image; do not serve an unaccelerated CPU stack.
+- `confirmation_required` (`requires_confirmation: true`) -- **Venice on a vLLM
+  other than the pinned default**. AMD documents 6th Gen EPYC as a zentorch target,
+  but this recipe has not validated Venice end-to-end on an off-default version.
+  Venice on the pinned `vllm_version` proceeds with no warning; on any other
+  version, stop, recommend the pinned image, and get an explicit user go/no-go.
+
+The probe only runs once the image is local, so after a first `pull` re-run
+`validate.py` to gate on the real stack rather than the tag. The container tag
+pins the AMD-published integration stack; a conda env may differ, so
+`check_model.py` should use the probed `stack.vllm` for that path.
+
 ## Runtime selection
 
 `scripts/validate.py` resolves a runtime the **agent can drive
@@ -41,19 +87,22 @@ From `data/epyc.json`. Unlike the Instinct (GPU) skill there are **no**
 
 | Flag | Why |
 |---|---|
-| `--ipc=host` | vLLM workers use host IPC / shared memory |
-| `--shm-size=16g` | vLLM needs a large `/dev/shm`; the 64MB default is too small |
+| `--ipc=host` | vLLM workers need a large `/dev/shm`; sharing the host IPC namespace provides it. **Do not also pass `--shm-size`** -- podman rejects the combination, and it is redundant on docker |
+| `--shm-size=16g` | **only if you drop `--ipc=host`** (isolated IPC). The 64MB container default is too small for vLLM. Use one or the other, never both |
 | `--network=host` | expose the served port directly (or use `-p <port>:<port>`) |
 | `--cpuset-cpus` / `--cpuset-mems` | pin the container to the chosen socket's physical cores and its NUMA node(s); from `cpu_tune.py` |
 | `-v ~/.cache/huggingface:/root/.cache/huggingface` | reuse the host model cache |
 
 Image: `amdih/zendnn_zentorch:<tag>` -- the public vLLM + zentorch CPU image on
 Docker Hub (no internal-registry access needed). The exact tag lives in
-`data/epyc.json`; read it, never hardcode it.
+`data/epyc.json`; read it, never hardcode it. The image and `vllm_version` are
+pinned together so `check_model.py` reads the registry for the runtime that will
+actually serve the model. This reproducibility pin applies to the default
+container recipe; it does not replace or modify an existing conda environment.
 
 ## Precision and modality
 
-| Dtype | EPYC (Zen) | Notes |
+| Dtype | Supported EPYC server target | Notes |
 |---|---|---|
 | BF16 | Native (default) | throughput default |
 | FP16 | Native | |
@@ -67,6 +116,39 @@ reranker and non-LLM architectures are rejected (not chat/completion endpoints).
 A vLLM-supported multimodal arch may still hit a GPU-only kernel on CPU -- that
 surfaces at load, where the no-retry rule applies.
 
+## Client endpoints and parameters
+
+`check_model.py` reports the endpoint the model actually supports so the handoff
+matches reality instead of always assuming chat:
+
+| Model | `chat_template.status` | `primary_endpoint` | Client call |
+|---|---|---|---|
+| Instruct/chat (ships a template) | `present` | `chat_completions` | `POST /v1/chat/completions` with `messages` |
+| Base text (no template) | `absent` | `completions` | `POST /v1/completions` with `prompt` |
+| Multiple named templates, no `default` | `ambiguous` | `completions` | completions now; chat needs `--chat-template`/a chosen name |
+| Template unreadable (gated/offline) | `unknown` | `completions` | completions; chat also works if a template exists |
+| Multimodal, no usable template | `absent`/`ambiguous` | none (`launchable: false`) | stop -- supply `--chat-template` or another model |
+
+`/v1/chat/completions` applies the model's chat template to structured `messages`
+and returns `choices[0].message.content`. `/v1/completions` takes a raw `prompt`
+(no template) and returns `choices[0].text`. Never invent a chat template; only
+enable chat when a real one is present or the user supplies `--chat-template`.
+
+Request parameters worth surfacing to users:
+
+| Parameter | Meaning |
+|---|---|
+| `max_tokens` | Output-token cap. `prompt_tokens + max_tokens` must be `<= --max-model-len`. |
+| `temperature` | Randomness; `0` is deterministic/greedy. |
+| `top_p` | Nucleus sampling; tune this **or** `temperature`, not both. |
+| `stream` | `true` streams tokens over SSE instead of one blocking reply. |
+| `stop` | String(s) that end generation early. |
+
+The base URL always ends in `/v1`. The OpenAI Python SDK requires a non-empty
+`api_key`, so pass a placeholder (e.g. `"EMPTY"`) when the server has no auth.
+The model repo's `generation_config.json` can set sampling defaults, so pass
+explicit values when determinism matters.
+
 ## CPU sizing
 
 Policy: a single instance is pinned to **one socket plus its memory** (vLLM scales
@@ -107,8 +189,11 @@ between the failing and passing runs was `VLLM_USE_AOT_COMPILE`. Never set
 `FREEZING=1` without `VLLM_USE_AOT_COMPILE=0`. The base recipe leaves both unset.
 
 **`/dev/shm` too small**
-Without `--shm-size=16g` (or `--ipc=host`), vLLM workers fail to allocate shared
-memory at startup.
+vLLM workers need a large `/dev/shm` or they fail to allocate shared memory at
+startup. The base recipe uses `--ipc=host` (shares the host's large shared memory).
+**Do not combine `--ipc=host` with `--shm-size`** -- podman errors *"cannot set
+shmsize when running in the host IPC Namespace"*, and it is redundant on docker. If
+you drop `--ipc=host`, use `--shm-size=16g` instead -- one or the other, never both.
 
 **RAM is the ceiling, not VRAM**
 CPU serving keeps weights + KV cache in system RAM. `estimate_memory.py` checks

skills/serving-llms-on-epyc/scripts/check_model.py

@@ -13,12 +13,18 @@
 (text + multimodal) are supported; pooling/embedding/reranker and non-LLM
 architectures are not chat/completion endpoints and are rejected.
 
+It also inspects the model repo for a chat template (`chat_template.jinja`, or
+`chat_template` in `tokenizer_config.json`) and picks the client endpoint:
+`/v1/chat/completions` when a usable template is present, else `/v1/completions`
+for raw prompts. A multimodal model with no usable template cannot serve and is
+rejected (`launchable: false`).
+
     check_model.py --model-id Qwen/Qwen3-0.6B
-    check_model.py --model-id <id> --vllm-version 0.22.0
+    check_model.py --model-id <id> --vllm-version 0.25.1
 
 Exit 0 if vLLM serves it as a generation endpoint (or support is undeterminable
--- launch confirms), 1 if it is positively unsupported. JSON to stdout.
-Env: HF_TOKEN for gated models.
+-- launch confirms), 1 if it is positively unsupported OR a multimodal model has
+no usable chat template. JSON to stdout. Env: HF_TOKEN for gated models.
 """
 
 import argparse
@@ -29,10 +35,15 @@
 import sys
 import urllib.request
 import urllib.error
+from pathlib import Path
 
 HF = "https://huggingface.co"
 GH_RAW = "https://raw.githubusercontent.com/vllm-project/vllm"
 REG_PATH = "vllm/model_executor/models/registry.py"
+TOKENIZER_CFG = "tokenizer_config.json"
+CHAT_TEMPLATE_FILE = "chat_template.jinja"
+DATA_PATH = Path(__file__).resolve().parent.parent / "data" / "epyc.json"
+DEFAULT_VLLM_VERSION = json.loads(DATA_PATH.read_text(encoding="utf-8"))["vllm_version"]
 
 # registry.py dict name -> kind we care about
 _SECTIONS = {
@@ -73,6 +84,65 @@ def model_architectures(model, rev, token):
     return cfg.get("architectures") or [], None
 
 
+def classify_template_field(ct):
+    """Classify a tokenizer_config `chat_template` value (pure).
+
+    Returns (status, selected_name, names):
+    - a non-empty string -> ("present", None, [])
+    - a list of {name, template} -> "present" if it has a single template or a
+      "default", else "ambiguous" (multiple named, no default)
+    - anything else / empty -> ("absent", None, [])
+    """
+    if isinstance(ct, str) and ct.strip():
+        return "present", None, []
+    if isinstance(ct, list) and ct:
+        names = [e.get("name") for e in ct if isinstance(e, dict) and e.get("name")]
+        if "default" in names:
+            return "present", "default", names
+        if len(names) == 1:
+            return "present", names[0], names
+        return "ambiguous", None, names
+    return "absent", None, []
+
+
+def chat_template_info(model, rev, token):
+    """Inspect the model repo for a usable chat template. Returns a dict with
+    `status` (present/ambiguous/absent/unknown), `source`, `selected_name`, `names`.
+
+    A standalone `chat_template.jinja` (Transformers v5) takes precedence over the
+    `chat_template` field in `tokenizer_config.json`. When neither file can be read
+    (gated/offline) the status is `unknown`."""
+    tmpl, terr = _get(f"{HF}/{model}/resolve/{rev}/{CHAT_TEMPLATE_FILE}", token)
+    if tmpl is not None and tmpl.strip():
+        return {"status": "present", "source": CHAT_TEMPLATE_FILE, "selected_name": None, "names": []}
+
+    cfg_text, cerr = _get(f"{HF}/{model}/resolve/{rev}/{TOKENIZER_CFG}", token)
+    if cfg_text is None:
+        return {"status": "unknown", "source": None, "selected_name": None, "names": [],
+                "detail": cerr or terr or "no tokenizer_config.json"}
+    cfg = json.loads(cfg_text) if cfg_text.strip().startswith("{") else {}
+    status, selected, names = classify_template_field(cfg.get("chat_template"))
+    return {"status": status, "source": TOKENIZER_CFG if status != "absent" else None,
+            "selected_name": selected, "names": names}
+
+
+def endpoints_for(kind, template_status):
+    """Pick client endpoints from modality + template status (pure).
+
+    Returns (supported_endpoints, primary_endpoint, needs_template):
+    - multimodal needs a chat template; without one it cannot serve (needs_template).
+    - text with a usable template -> chat preferred, completions also available.
+    - text without an auto-usable template (absent/ambiguous/unknown) -> completions
+      (chat still possible via an explicit --chat-template)."""
+    if kind == "multimodal":
+        if template_status == "present":
+            return ["chat_completions"], "chat_completions", False
+        return [], None, True
+    if template_status == "present":
+        return ["chat_completions", "completions"], "chat_completions", False
+    return ["completions"], "completions", False
+
+
 def registry_from_github(version):
     """Parse vLLM's registry.py at v<version>. Returns ({arch: kind}, source) or (None, err)."""
     src, err = _get(f"{GH_RAW}/v{version}/{REG_PATH}")
@@ -120,7 +190,11 @@ def main():
     p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     p.add_argument("--model-id", required=True)
     p.add_argument("--revision", default="main")
-    p.add_argument("--vllm-version", default="0.22.0", help="pin the registry to this vLLM version (from data/epyc.json)")
+    p.add_argument(
+        "--vllm-version",
+        default=DEFAULT_VLLM_VERSION,
+        help=f"registry version (default: {DEFAULT_VLLM_VERSION} from data/epyc.json)",
+    )
     a = p.parse_args()
     token = os.environ.get("HF_TOKEN", "")
 
@@ -155,10 +229,36 @@ def main():
 
     if any(k in ("text", "multimodal") for k in known):
         kind = "multimodal" if "multimodal" in known else "text"
-        msg = f"vLLM supports {archs} as a {kind} generation endpoint."
+        tmpl = chat_template_info(a.model_id, a.revision, token)
+        eps, primary, needs_tmpl = endpoints_for(kind, tmpl["status"])
+        out.update(supported=True, kind=kind, chat_template=tmpl,
+                   supported_endpoints=eps, primary_endpoint=primary)
+
+        if kind == "multimodal" and needs_tmpl:
+            out.update(launchable=False,
+                       message=(f"{archs} is a multimodal model but no usable chat template was found "
+                                f"(chat_template.status={tmpl['status']}). vLLM needs a chat template to "
+                                "serve it; pass --chat-template <file> or choose a model that ships one. Stop."))
+            print(json.dumps(out, indent=2))
+            sys.exit(1)
+
+        if primary == "chat_completions":
+            msg = (f"vLLM supports {archs} as a {kind} generation endpoint; a chat template is present "
+                   f"({tmpl['source']}). Serve /v1/chat/completions.")
+        elif tmpl["status"] == "ambiguous":
+            msg = (f"vLLM supports {archs} as a {kind} generation endpoint, but it ships multiple named chat "
+                   f"templates with no 'default' ({tmpl['names']}). /v1/completions works now; for chat, pass "
+                   "--chat-template <file> or select one of those names.")
+        elif tmpl["status"] == "unknown":
+            msg = (f"vLLM supports {archs} as a {kind} generation endpoint, but the chat template could not be "
+                   "read (gated/offline). Defaulting to /v1/completions; if this is an instruct model with a "
+                   "template, /v1/chat/completions will also work.")
+        else:
+            msg = (f"vLLM supports {archs} as a {kind} generation endpoint, but no chat template was found. "
+                   "Serve base text via /v1/completions; for chat, pass --chat-template <file>.")
         if kind == "multimodal":
             msg += " A multimodal arch may still hit a GPU-only kernel on CPU; that surfaces at load (no-retry rule applies)."
-        out.update(supported=True, kind=kind, message=msg)
+        out.update(launchable=True, message=msg)
         print(json.dumps(out, indent=2))
         sys.exit(0)