From 7704fd08851c074c15c7b434514822c04481e770 Mon Sep 17 00:00:00 2001 From: anubhutiv Date: Mon, 3 Aug 2026 23:20:51 -0700 Subject: [PATCH] fix(customizer): fix CVEs + vllm venv issue Signed-off-by: anubhutiv --- docker-bake.hcl | 2 +- docker/rl/Dockerfile.nmp-rl-base | 68 ++++++++++++++++++++++++++++++-- docker/rl/README.md | 41 +++++++++++++++++-- 3 files changed, 103 insertions(+), 8 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 7839541fd6..640f89d730 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -100,7 +100,7 @@ variable "NEMO_RL_REPO" { # RL pins Gym as a git submodule (-> soluwalana/Gym over https), so Gym rides in with the RL git ADD # - no separate Gym pin needed. variable "NEMO_RL_REF" { - default = "706382658e8b3b6292d0b96e83102a610b5f5491" # soluwalana/RL nmp/customizer + default = "edba0f96edcecd8349302a3d93b1249d8a067d0e" # soluwalana/RL nmp/customizer } variable "RL_BASE_CONTEXT" { default = "" diff --git a/docker/rl/Dockerfile.nmp-rl-base b/docker/rl/Dockerfile.nmp-rl-base index abbd8e6056..356acd1585 100644 --- a/docker/rl/Dockerfile.nmp-rl-base +++ b/docker/rl/Dockerfile.nmp-rl-base @@ -22,7 +22,9 @@ # build-only tooling in the publish stage; the CUDA devel toolkit (nvcc) and gcc/build-essential # stay because deep_ep JIT and native-v1 env installs need them at runtime. -ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.03-cuda13.2-devel-ubuntu24.04 +# Kept in lockstep with the pinned NEMO_RL_REF: RL's own Dockerfile builds on this tag, and its lock +# overrides nvidia-nccl-cu13 to the version that ships here. Bump both together. +ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.05-cuda13.2-devel-ubuntu24.04 # Source. Default clones the soluwalana/RL fork at the given ref; override with a local checkout via # `--build-context nemo-rl=` or point elsewhere via the NEMO_RL_{REPO,REF} build-args. Gym is @@ -33,8 +35,10 @@ ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.03-cuda13.2-devel-ubuntu24.04 ARG NEMO_RL_REPO=https://github.com/soluwalana/RL.git ARG NEMO_RL_REF=nmp/customizer -ARG UV_VERSION=0.11.18 -ARG PYTHON_VERSION=3.13.13 +# CVE bumps: uv 0.11.18 bundles a vulnerable quinn-proto (0.12 has breaking changes, so stay on +# 0.11.x); Python 3.13.14 clears 5 CVEs. Both are scanned at their installed path - see README.md. +ARG UV_VERSION=0.11.33 +ARG PYTHON_VERSION=3.13.14 ARG CMAKE_VERSION=4.0.3 # Reuse the shared FFmpeg/VLM wheels (cp313 av / opencv-headless / decord2), supplied by bake as a @@ -55,11 +59,14 @@ FROM ${BASE_IMAGE} AS base USER root ENV DEBIAN_FRONTEND=noninteractive +# openssl is the only dpkg package here with an out-of-SLA CVE. Upgraded explicitly rather than via +# a blanket `apt-get upgrade`, which would also pull newer CUDA packages from the NVIDIA repos. RUN apt-get update && \ apt-get install -y --no-install-recommends \ jq curl git rsync wget less vim ccache gnupg \ build-essential ca-certificates \ libibverbs-dev && \ + apt-get install -y --only-upgrade openssl libssl3t64 && \ apt-get clean && rm -rf /var/lib/apt/lists/* # CMake (CUDA extension builds: transformer-engine, mamba-ssm, ...). @@ -80,7 +87,11 @@ RUN ARCH="$(uname -m)" && \ # "No module named 'encodings'". Copy the uv binary to /usr/local/bin for the same reason (runtime env installs). ARG UV_VERSION ARG PYTHON_VERSION +# UV_PYTHON pins the interpreter for every uv call, at build and at runtime (node-built venvs too). +# Required, not cosmetic: RL ships a `.python-version` that otherwise wins, so bumping PYTHON_VERSION +# alone leaves every venv on RL's patch release and the requested one unused on disk. ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python \ + UV_PYTHON=${PYTHON_VERSION} \ PATH="/usr/local/bin:/root/.local/bin:${PATH}" RUN curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh && \ cp /root/.local/bin/uv /usr/local/bin/uv && \ @@ -234,6 +245,10 @@ uv pip install --python /opt/nemo_rl_venv/bin/python "opensandbox>=0.1.9" "tenac # CVE GHSA-mqqc-3gqh-h2x8: drop ray's bundled old aiohttp from the uv cache. find "${UV_CACHE_DIR}" -type d -path "*ray/_private/runtime_env/agent/thirdparty_files/aiohttp*" -exec rm -rf {} + || true + +# jackson-databind / jackson-core CVEs: drop ray's bundled ray_dist.jar, which only backs Ray's Java +# worker support (unused - no JVM here). Removed before the prefetch so no venv symlinks to it. +find "${UV_CACHE_DIR}" -type d -path "*/ray/jars" -exec rm -rf {} + || true EOF # Replace the FFmpeg-bundling av / opencv / decord2 copies with vendored cp313 wheels (CVE). @@ -318,10 +333,15 @@ uv run nemo_rl/utils/prefetch_venvs.py \ # Fail loudly if the registry moved an actor out from under a filter above - otherwise the venv # would silently be built on the node at first use instead, turning a build error into a slow job. +# The vLLM-tier actors are listed too: the publish stage privatizes `vllm/` in whatever venvs exist, +# so a filter that stops producing one would leave that worker sharing ray_executor.py again. for actor in \ nemo_rl.environments.nemo_gym.NemoGym \ nemo_rl.environments.sandbox.nemo_gym_actor.SandboxedGymActor \ - nemo_rl.environments.sandbox.broker_actor.SandboxEpisodeBrokerActor + nemo_rl.environments.sandbox.broker_actor.SandboxEpisodeBrokerActor \ + nemo_rl.models.generation.vllm.vllm_worker.VllmGenerationWorker \ + nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker \ + nemo_rl.experience.sync_rollout_actor.SyncRolloutActor do if [ ! -x "/opt/ray_venvs/${actor}/bin/python" ]; then echo "ERROR: expected prefetched venv at /opt/ray_venvs/${actor}" >&2 @@ -408,6 +428,11 @@ FROM builder AS nmp-rl-base # any /opt/gym_venvs environment baked in via NEMO_GYM_PREFETCH_CONFIGS. It also doubles as a warm # cache for everything installed at runtime (Gym environments, user FileSets, non-prefetched # actors), making those installs faster. +# +# Nsight Systems / Nsight Compute come in with cuda-dl-base and carry Go stdlib CVEs (3 Critical, 25 +# High). They are profilers - nothing on the DPO/GRPO path runs nsys/ncu. This clears them from the +# flattened rootfs the scanner walks; it does NOT shrink the pulled image, since they live in an +# ancestor layer and this stage inherits the builder filesystem (see header NOTE). RUN apt-get purge -y ccache vim vim-common less >/dev/null 2>&1 || true; \ apt-get autoremove -y >/dev/null 2>&1 || true; \ rm -rf \ @@ -415,6 +440,9 @@ RUN apt-get purge -y ccache vim vim-common less >/dev/null 2>&1 || true; \ /usr/local/share/cmake-* \ /root/.cache/ccache \ /var/lib/apt/lists/* \ + /usr/local/cuda*/NsightSystems-cli-* /usr/local/cuda*/nsight-compute-* \ + /opt/nvidia/nsight-systems-cli /opt/nvidia/nsight-compute \ + /usr/local/bin/nsys /usr/local/bin/nsys-ui /usr/local/bin/ncu /usr/local/bin/ncu-ui \ || true # NO_VCS_VERSION=1: with no .git present, nemo_rl/package_info.py's `git rev-parse` would fail on every @@ -448,6 +476,38 @@ ARG RUNTIME_UID=1000 ARG RUNTIME_GID=1000 RUN chown ${RUNTIME_UID}:${RUNTIME_GID} /opt/ray_venvs /opt/gym_venvs +# NeMo-RL patches vLLM in place at worker startup (models/generation/vllm/patches.py), which +# symlinking breaks two ways: the venvs are root-owned so the patch lock fails with EACCES as UID +# 1000, and every vLLM venv symlinks to ONE copy in /opt/uv_cache while the patch writes a per-venv +# py_executable into it. Giving each venv a private vllm/ fixes both. Scoped to vllm/ (~506 MB x 3) +# rather than UV_LINK_MODE=copy globally (~56 GB) - see README.md, "Link mode". +RUN <<"EOF" bash -exu +privatized=0 +for sp in /opt/ray_venvs/*/lib/python*/site-packages; do + [ -d "${sp}/vllm" ] || continue + # -L turns the per-file symlinks into real files; the uv cache entries stay untouched. + cp -rL --preserve=mode,timestamps "${sp}/vllm" "${sp}/.vllm.private" + rm -rf "${sp}/vllm" + mv "${sp}/.vllm.private" "${sp}/vllm" + chmod -R a+rX "${sp}/vllm" + chown -R ${RUNTIME_UID}:${RUNTIME_GID} "${sp}/vllm" + privatized=$((privatized + 1)) +done +# Fail loudly: if the vllm prefetch filters change, this must not become a silent no-op. +if [ "${privatized}" -eq 0 ]; then + echo "ERROR: no prefetched venv contains a vllm package." >&2 + echo " check the vllm prefetch filters against ray_actor_environment_registry.py" >&2 + exit 1 +fi +echo "privatized vllm/ in ${privatized} venv(s)" +EOF + +# Gym falls back to this in-tree cache when a process does not inherit NRL_CONTAINER / UV_CACHE_DIR +# (the sandboxed Gym host's per-app servers), and fails with EACCES on a root-owned tree. Keeping +# the fallback writable is the safety net; the real fix is env propagation, which lives in NeMo-RL. +RUN mkdir -p /opt/nemo-rl/3rdparty/Gym-workspace/Gym/cache && \ + chown ${RUNTIME_UID}:${RUNTIME_GID} /opt/nemo-rl/3rdparty/Gym-workspace/Gym/cache + WORKDIR /opt/nemo-rl # Default to the non-root runtime user. This stage is published on its own (nmp-rl-base), so a diff --git a/docker/rl/README.md b/docker/rl/README.md index 48ed864e9f..eb43d6cbba 100644 --- a/docker/rl/README.md +++ b/docker/rl/README.md @@ -361,6 +361,12 @@ the uv cache + venv prefetch rather than via wheel images: and reinstalls clean `cp313` wheels built against a patched FFmpeg (from `docker/base/Dockerfile.python-wheels`). - **Ray's bundled aiohttp** is removed from the uv cache to fully address its CVE. +- **The interpreter needs `UV_PYTHON`, not just `PYTHON_VERSION`.** NeMo-RL ships a + `.python-version` pinning an exact patch release, which uv honours over whatever + `uv python install` provisioned. Bumping `PYTHON_VERSION` alone therefore fixed nothing: + every venv came up on RL's version while ours sat unused on disk, so the image shipped + two interpreters and ran the vulnerable one — silently. `UV_PYTHON` overrides the file + and persists into the runtime image, so node-built venvs agree too. ## Layering for fast CI rebuilds @@ -384,9 +390,11 @@ whenever the *dependency graph* hasn't changed: ### Prefetching the per-worker venvs (build once, not per job) -The warmup `uv sync --extra …` calls populate the **uv cache**, which is a build-time -cache mount and never enters the image. The venvs training actually runs in are the -per-worker ones under `/opt/ray_venvs`, so the base runs +The warmup `uv sync --extra …` calls populate the **uv cache at `/opt/uv_cache`, which +ships inside the image** — it has to, because the prefetched venvs symlink into it (see +"Link mode" below). Do not confuse it with the `--mount=type=cache` the training image +uses for its editable install, which is build-only and never enters the image. The venvs +training actually runs in are the per-worker ones under `/opt/ray_venvs`, so the base runs `nemo_rl/utils/prefetch_venvs.py` after the source copy to bake them in — the same approach NeMo-RL's own release stage uses. @@ -442,6 +450,33 @@ readable by the non-root user. Hence `UV_CACHE_DIR=/opt/uv_cache`: A useful side effect: environments installed at runtime (user Gym FileSets, actors that were not prefetched) resolve against a warm cache instead of downloading from scratch. +#### `vllm/` is a private copy per vLLM venv + +NeMo-RL **patches vLLM in place at worker startup** (`nemo_rl/models/generation/vllm/patches.py` +rewrites `v1/executor/ray_executor.py`, `model_executor/models/llama_eagle3.py` and +`tool_parsers/hermes_tool_parser.py`, taking a `.patch_lock` beside each). Symlinking breaks +that in two independent ways, both observed on the shipped image: + +1. **Permission** — the prefetched venvs are root-owned and the runtime is UID 1000, so creating + `ray_executor.py.patch_lock` fails with `EACCES` and the `VllmAsyncGenerationWorker` actor dies + in its creation task. +2. **Sharing** — every vLLM-tier venv symlinks that file to *one* physical copy in `/opt/uv_cache`. + The patch writes a **per-venv `py_executable`** into it, so two venvs cannot share it. No amount + of `chown` fixes this; the file must not be shared at all. + +So the publish stage replaces `site-packages/vllm/` with a real, private, UID-1000-owned copy in +each vLLM-tier venv. + +The two need separate fixes: **ownership** solves (1), **private copies** solve (2). Running as root +would silence the `EACCES` but leaves the sharing intact — the first venv to patch wins and the next +one launches under another venv's interpreter. + +Note this is *not* what upstream NeMo-RL does. Its published image symlinks too (74,567 symlinks vs +925 real files in one vLLM venv, both vLLM venvs sharing one `ray_executor.py`), and it runs as +**root** — so (1) never surfaces there and (2) stays latent, because vLLM 0.25 defaults +`VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1` and the V2 executor never reads the patched call site. Running +non-root is required here, so we can inherit neither the root workaround nor that assumption. + ## Image size For operators sizing nodes and registries, and for anyone deciding what to trim. This is a