From bd3339bbbe4bcd2b555995c10d9e76e6c369c83d Mon Sep 17 00:00:00 2001 From: Ognjen Date: Thu, 11 Jun 2026 11:33:59 +0200 Subject: [PATCH] Enable Strix Halo profiling support - Add persistent vLLM and Triton cache mounts to reduce compile overhead - Fix path handling to support execution from repository root - Enable GPU config auto-generation via generate_gpu_yaml.sh - Improve orchestrator reliability (absolute paths, python execution) - Disable Qwen3-VL on Strix Halo due to OOM during profiling - Adjust model configuration for safer memory usage - Update README and gitignore for improved usability and clarity --- .gitignore | 17 +++++++-- README.md | 36 +++++++++++++++++- scripts/host/docker_tool.py | 46 +++++++++++++++------- scripts/host/generate_gpu_yaml.sh | 63 ++++++++++++++++--------------- scripts/host/orchestrator.py | 11 +++++- yaml/env_vars.yaml | 1 + yaml/models.yaml | 4 +- 7 files changed, 128 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore index b9075fa..0440b79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,17 @@ -**/*cache*/ +# venv +venv/ +.venv/ + +# python cache __pycache__/ +*.pyc + +# generic caches +**/*cache*/ -*.*/ +#files +images/ -Dockerfile +# logs (optional but recommended) +.logs/ +.config/ \ No newline at end of file diff --git a/README.md b/README.md index 687f2af..f991a8f 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,16 @@ Purpose: * Preserve downloaded models between runs * Avoid repeated downloads across container executions +### `.cache/vllm/` and `.cache/triton/` + +These directories are mounted into Docker containers to persist vLLM and Triton compile caches. + +Purpose: + +* Reduce repeated `torch.compile` and CUDA graph capture time across runs +* Make Strix Halo profiling runs closer to steady-state behavior after the first cold start +* Improve repeatability when comparing baselines across models + --- ### `.config/` @@ -145,7 +155,13 @@ Run directly on the host system: * `generate_gpu_yaml.sh` Helper script to auto-generate a `gpus.yaml` template - **NOTE**: *This script is currently defunct and you will have to create your own .yaml config based on the one provided in the README and your own setup* + Usage: + + ```bash + scripts/host/generate_gpu_yaml.sh + ``` + + This generates `.config/gpus.yaml` from the local `/dev/dri/renderD*` devices. * Future host utilities will also live here @@ -209,3 +225,21 @@ Notes: * Improved automation and validation * Expanded profiling support * Potential non-Linux support + +--- + +## Portability Notes + +* Host scripts now resolve workspace paths from the repository root, so running commands from different working directories is supported. +* Use `disabled_on` in `yaml/models.yaml` to prevent known-problematic models on specific devices. + +Example: + +```yaml +models: + - name: Qwen/Qwen3-VL-4B-Instruct + type: multimodal + script: qwen_vl.py + disabled_on: + - Strix Halo +``` diff --git a/scripts/host/docker_tool.py b/scripts/host/docker_tool.py index 5cd224b..2fc3f98 100755 --- a/scripts/host/docker_tool.py +++ b/scripts/host/docker_tool.py @@ -40,6 +40,10 @@ def prepare_env(): def run_container(args, script_args): env = prepare_env() + hf_cache_dir = Path(args.hf_cache_dir) + if not hf_cache_dir.is_absolute(): + hf_cache_dir = (ROOT_DIR / hf_cache_dir).resolve() + cmd = [ "docker", "run", @@ -81,34 +85,50 @@ def run_container(args, script_args): # huggingface cache dir hf_cache_container = "/root/.cache/huggingface" - Path(args.hf_cache_dir).mkdir(parents=True, exist_ok=True) - cmd.extend(["-v", f"{args.hf_cache_dir}:{hf_cache_container}"]) - print(f"Mounting HuggingFace cache: {args.hf_cache_dir} -> {hf_cache_container}") + hf_cache_dir.mkdir(parents=True, exist_ok=True) + cmd.extend(["-v", f"{hf_cache_dir}:{hf_cache_container}"]) + print(f"Mounting HuggingFace cache: {hf_cache_dir} -> {hf_cache_container}") + + # vLLM and Triton compile caches + cache_mounts = { + "/root/.cache/vllm": ROOT_DIR / ".cache" / "vllm", + "/root/.cache/triton": ROOT_DIR / ".cache" / "triton", + } + for container_cache_dir, host_cache_dir in cache_mounts.items(): + host_cache_dir.mkdir(parents=True, exist_ok=True) + cmd.extend(["-v", f"{host_cache_dir}:{container_cache_dir}"]) + print(f"Mounting cache: {host_cache_dir} -> {container_cache_dir}") # local container scripts dir scripts_container = str(container_workspace / "scripts") - cmd.extend(["-v", f"./scripts/container:{scripts_container}"]) - print(f"Mounting ./scripts/container -> {scripts_container}") + host_scripts_dir = ROOT_DIR / "scripts" / "container" + cmd.extend(["-v", f"{host_scripts_dir}:{scripts_container}"]) + print(f"Mounting {host_scripts_dir} -> {scripts_container}") # local prompts dir prompts_container = str(container_workspace / "prompts") - cmd.extend(["-v", f"./prompts:{prompts_container}"]) - print(f"Mounting ./prompts -> {prompts_container}") + host_prompts_dir = ROOT_DIR / "prompts" + cmd.extend(["-v", f"{host_prompts_dir}:{prompts_container}"]) + print(f"Mounting {host_prompts_dir} -> {prompts_container}") # logs dir logs_container = str(container_workspace / "logs") - cmd.extend(["-v", f"./.logs:{logs_container}"]) - print(f"Mounting ./.logs -> {logs_container}") + host_logs_dir = ROOT_DIR / ".logs" + host_logs_dir.mkdir(parents=True, exist_ok=True) + cmd.extend(["-v", f"{host_logs_dir}:{logs_container}"]) + print(f"Mounting {host_logs_dir} -> {logs_container}") # images dir images_container = str(container_workspace / "images") - cmd.extend(["-v", f"./images:{images_container}"]) - print(f"Mounting ./images -> {images_container}") + host_images_dir = ROOT_DIR / "images" + cmd.extend(["-v", f"{host_images_dir}:{images_container}"]) + print(f"Mounting {host_images_dir} -> {images_container}") # yaml dir yaml_container = str(container_workspace / "yaml") - cmd.extend(["-v", f"./yaml:{yaml_container}"]) - print(f"Mounting ./yaml -> {yaml_container}") + host_yaml_dir = ROOT_DIR / "yaml" + cmd.extend(["-v", f"{host_yaml_dir}:{yaml_container}"]) + print(f"Mounting {host_yaml_dir} -> {yaml_container}") shell_cmd = [] diff --git a/scripts/host/generate_gpu_yaml.sh b/scripts/host/generate_gpu_yaml.sh index 46f5a4d..d7d49e4 100755 --- a/scripts/host/generate_gpu_yaml.sh +++ b/scripts/host/generate_gpu_yaml.sh @@ -1,33 +1,36 @@ #!/usr/bin/env bash -echo "THE SCRIPT generate_gpu_yaml.sh IS CURRENTLY DEFUNCT" -echo "GENERATE THE FILE MANUALLY" -echo "EXAMPLE:" -echo "gpus: - 0: - name: Radeon RX 7900 XTX - device: /dev/dri/renderD128 - 1: - name: AMD Radeon RX 9070 XT - device: /dev/dri/renderD129 - 2: - name: AMD Radeon RX 6700 XT - device: /dev/dri/renderD130" -# TODO: ENABLE USING ROCMINFO - -exit 1 - -echo "gpus:" - -i=0 -for dev in /dev/dri/renderD*; do - name=$(udevadm info --query=property --name="$dev" \ - | grep ID_MODEL= \ - | cut -d= -f2) - - echo " gpu$i:" - echo " name: ${name:-unknown}" - echo " device: $dev" - - ((i++)) +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +OUT_FILE="${ROOT_DIR}/.config/gpus.yaml" + +mkdir -p "${ROOT_DIR}/.config" + +mapfile -t render_devices < <(ls /dev/dri/renderD* 2>/dev/null | sort || true) + +if [[ ${#render_devices[@]} -eq 0 ]]; then + echo "No /dev/dri/renderD* devices found; cannot generate ${OUT_FILE}" >&2 + exit 1 +fi + +echo "gpus:" > "${OUT_FILE}" + +for dev in "${render_devices[@]}"; do + name=$(udevadm info --query=property --name="$dev" 2>/dev/null \ + | awk -F= '/^ID_MODEL_FROM_DATABASE=/{print $2; found=1; exit} /^ID_MODEL=/{if (!found) print $2}' \ + | sed 's/_/ /g' \ + || true) + + if [[ -z "${name}" ]]; then + name="${dev##*/}" + fi + + cat >> "${OUT_FILE}" <