fix(ci): preflight the build toolchain on the self-hosted runner

.github/workflows/gpu-conformance-amd.yml

@@ -90,6 +90,21 @@ jobs:
 
     env:
       BUILD_TYPE: Release
+      # Build width, set here rather than as a `--parallel` flag so it applies
+      # to EVERY `cmake --build` in the job, including any added later that
+      # forget the flag. `cmake --build` reads this whenever `--parallel` is
+      # absent; an explicit `--parallel` on a step would override it.
+      #
+      # This must be set explicitly. Dropping the flag does NOT fall back to
+      # something conservative — with the Ninja generator the default is
+      # cores + 2, which is 18 on this 16-core host, i.e. MORE than the
+      # `--parallel 12` that already ran it out of memory.
+      #
+      # 6, not 16: this host also runs the gh-runner-1/2/3 runners for another
+      # repository. A nightly at ~02:47 local should normally find them idle,
+      # but "normally" is not "always" — leave headroom so a concurrent build
+      # elsewhere on the box doesn't turn into swap thrash or an OOM.
+      CMAKE_BUILD_PARALLEL_LEVEL: 6
       # Goldens are baselined on NVIDIA hardware; GoldenImageTests.cpp reads this
       # to select a per-vendor golden directory (assets/tests/golden/amd/) so an
       # AMD run can never clobber the NVIDIA baseline set.
@@ -154,6 +169,53 @@ jobs:
           # it — the same reasoning as Windows.yml's checkout.
           persist-credentials: false
 
+      # A self-hosted runner has whatever the box happens to have, and the runner
+      # user's PATH is NOT the admin's. cmake and ninja were originally pip
+      # installs under /home/obueker/.local/bin — on this host /home/obueker is
+      # mode 0700, so gh-runner-olo could not see them and the first run died
+      # with a bare `cmake: command not found` after a 39 s checkout. Everything
+      # the build needs must be system-wide (/usr, /opt), never in a person's
+      # home. Check that up front so the error names the cause.
+      #
+      # This runs FIRST, before the GL probe below, because that probe compiles
+      # its test program with gcc. With the order reversed a missing compiler
+      # surfaces as a bare `gcc: command not found` from the GL step instead of
+      # the remediation message here — do not reorder these two.
+      - name: Preflight — build toolchain
+        run: |
+          set -euo pipefail
+          missing=0
+          for tool in cmake ninja ccache gcc g++ python3; do
+            if command -v "$tool" >/dev/null 2>&1; then
+              printf '  %-8s %s\n' "$tool" "$(command -v "$tool")"
+            else
+              echo "::error::$tool not found on the runner's PATH"
+              missing=1
+            fi
+          done
+          # Python modules count as toolchain too. glad2's code generation
+          # imports jinja2, and a `pip install --user` puts it in the INSTALLING
+          # user's home — invisible to the runner user for the same 0700 reason
+          # as everything else. Checking only binaries let this through: the
+          # preflight passed and the build then died on ModuleNotFoundError
+          # four minutes in. Check what the build actually imports.
+          for mod in jinja2; do
+            if python3 -c "import $mod" 2>/dev/null; then
+              printf '  %-8s %s\n' "$mod" "$(python3 -c "import $mod,os; print(os.path.dirname($mod.__file__))")"
+            else
+              echo "::error::python module '$mod' not importable by $(id -un)"
+              missing=1
+            fi
+          done
+
+          if [ "$missing" -ne 0 ]; then
+            echo "::error::Provision the runner: sudo dnf install -y cmake ninja-build ccache gcc gcc-c++ python3 python3-jinja2"
+            echo "::error::See docs/ops/self-hosted-gpu-runner.md. Everything must be system-wide —"
+            echo "::error::not under a 0700 home, and not a 'pip install --user' in someone else's account."
+            exit 1
+          fi
+          cmake --version | head -1
+
       # ---------------------------------------------------------------------
       # Preflight: prove we have a HARDWARE context before spending an hour
       # building. This is the anti-silent-failure guard, and it is the whole
@@ -230,7 +292,21 @@ jobs:
           : "${VULKAN_SDK:?VULKAN_SDK is not set - see docs/ops/self-hosted-gpu-runner.md}"
           test -d "$VULKAN_SDK/include/vulkan" || { echo "::error::no headers under $VULKAN_SDK"; exit 1; }
           test -e "$VULKAN_SDK/lib/libshaderc_shared.so" || { echo "::error::no shaderc under $VULKAN_SDK"; exit 1; }
+
+          # The LunarG SDK does NOT place the loader in lib/ — it nests it under
+          # lib/VulkanLoader/lib/. CMAKE_PREFIX_PATH therefore resolves the
+          # headers and the *version* but not Vulkan_LIBRARY, and configure dies
+          # with "Could NOT find Vulkan (missing: Vulkan_LIBRARY) (found version
+          # 1.4.350)" — found-but-not-found, which reads like a version problem
+          # and is not one. Locate the loader and hand the configure step an
+          # exact path rather than letting it guess at a layout that varies
+          # between SDK versions.
+          loader=$(find "$VULKAN_SDK" -name 'libvulkan.so' -print -quit 2>/dev/null || true)
+          [ -n "$loader" ] || { echo "::error::no libvulkan.so anywhere under $VULKAN_SDK"; exit 1; }
+
           echo "Vulkan SDK: $VULKAN_SDK"
+          echo "  loader:   $loader"
+          echo "OLO_VULKAN_LIBRARY=$loader" >> "$GITHUB_ENV"
 
       - name: Configure CMake
         # Heavy optional interchange deps (USD/Alembic/MaterialX) and the FFmpeg
@@ -246,18 +322,31 @@ jobs:
           cmake -S . -B build -G Ninja
           -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
           -DCMAKE_PREFIX_PATH="$VULKAN_SDK"
+          -DVulkan_LIBRARY="$OLO_VULKAN_LIBRARY"
           -DCMAKE_C_COMPILER_LAUNCHER=ccache
           -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
           -DOLO_VIDEO_FFMPEG=OFF
           -DOLO_WITH_USD=OFF -DOLO_WITH_ALEMBIC=OFF -DOLO_WITH_MATERIALX=OFF
+          -DOLO_LINK_JOBS=1
 
       - name: Build tests
-        # --parallel 12, not nproc (16): this host also runs the gh-runner-1/2/3
-        # runners for another repository. A nightly at ~02:47 local should
-        # normally find them idle, but "normally" is not "always" — leave
-        # headroom so a concurrent build on the other runners doesn't turn into
-        # swap thrash or an OOM during the OloEngine-Tests link.
-        run: cmake --build build --target OloEngine-Tests --parallel 12
+        # Width comes from CMAKE_BUILD_PARALLEL_LEVEL in the job env — see the
+        # note there for why it must be set explicitly rather than left to the
+        # generator default.
+        #
+        # Link concurrency is the second half of it, and the more important
+        # half for this failure: linking OloEngine-Tests is the memory spike,
+        # and a link peaks far higher than a compile. A plain -j cap still
+        # permits several concurrent links.
+        #
+        # That cap now belongs to the project — the root CMakeLists owns an
+        # olo_link job pool sized by OLO_LINK_JOBS (default 2), so `ninja`
+        # obeys it too, not just `cmake --build`. The configure step above
+        # therefore sets OLO_LINK_JOBS=1 rather than declaring a second,
+        # workflow-local pool: two mechanisms binding CMAKE_JOB_POOL_LINK
+        # would have the workflow's silently win, leaving the project's
+        # setting looking effective while doing nothing.
+        run: cmake --build build --target OloEngine-Tests
 
       # ---------------------------------------------------------------------
       # The suite runs from OloEditor/ — OloEditor, OloRuntime and the test
@@ -278,8 +367,15 @@ jobs:
           MESA_LOADER_DRIVER_OVERRIDE: radeonsi
         run: |
           mkdir -p ../test_results
+          # NetworkIntegrationTest is excluded by EVERY other job in this repo
+          # (see asan.yml's --exclude-regex on all three Linux sanitiser jobs):
+          # it needs real socket plumbing that CI environments don't provide, and
+          # it failed all six cases here for the same reason. Excluding it keeps
+          # this job's red/green meaningful; it is not a GPU test and this job
+          # has no business gating on it.
           ../build/OloEngine/tests/OloEngine-Tests \
             --gtest_catch_exceptions=1 \
+            --gtest_filter=-NetworkIntegrationTest.* \
             --gtest_output=xml:../test_results/gpu_amd.xml
 
       # ---------------------------------------------------------------------

CLAUDE.md

@@ -86,16 +86,40 @@ CMake presets ([CMakePresets.json](CMakePresets.json)) — note all three requir
 scripts\Win-GenerateProjectVS2022.bat   # or VS2026
 
 # Build a target
-cmake --build build --target OloEditor       --config Debug --parallel
-cmake --build build --target OloEngine-Tests --config Debug --parallel
-cmake --build build --target OloRuntime      --config Debug --parallel
-cmake --build build --target OloServer       --config Debug --parallel
+cmake --build build --target OloEditor       --config Debug --parallel 6
+cmake --build build --target OloEngine-Tests --config Debug --parallel 6
+cmake --build build --target OloRuntime      --config Debug --parallel 6
+cmake --build build --target OloServer       --config Debug --parallel 6
 
 # ClangCL (configure once, then build)
 cmake --preset clangcl
-cmake --build build-clang --target OloEngine-Tests --config Debug --parallel
+cmake --build build-clang --target OloEngine-Tests --config Debug --parallel 6
 ```
 
+### Cap build parallelism — a full-width build can OOM this machine
+
+**Never build uncapped.** Either pass an explicit job count — `--parallel 6`, or `ninja -j6` — or set `CMAKE_BUILD_PARALLEL_LEVEL`, which `cmake --build` uses whenever no `--parallel` is given (this is how the nightly workflow caps itself):
+
+```powershell
+$env:CMAKE_BUILD_PARALLEL_LEVEL = "6"   # PowerShell (the primary dev shell here)
+```
+```bash
+export CMAKE_BUILD_PARALLEL_LEVEL=6      # POSIX shell / the Linux GPU runner
+```
+
+An explicit `--parallel N` overrides the environment variable, so don't set one expecting the other to win.
+
+**`CMAKE_BUILD_PARALLEL_LEVEL` caps `cmake --build` only — `ninja` does not read it.** A direct `ninja` invocation must always carry a numeric `-jN` of its own; setting the variable and then running bare `ninja` gives you the full 18-wide default with no warning. What is never acceptable is a bare `--parallel`, or a direct `ninja` without `-jN`.
+
+This is not a style preference. The dev box is 16 cores / 31 GB and *also* hosts the `gh-runner-1/2/3` runners for another repository, so a build never has the machine to itself. Neither default is a cap:
+
+- `cmake --build … --parallel` with **no number** does not pick a number itself — it forwards the omission to the native build tool, whose own default applies (unless `CMAKE_BUILD_PARALLEL_LEVEL` is set). So the width you get depends on the generator, and it is never *lower* than the tool's default.
+- With Ninja that default is `cores + 2` — 18 on this host, confirmed by `ninja --help` reporting `[default=18 on this system]`. Dropping a `--parallel N` flag therefore *raises* the width rather than lowering it.
+
+An agent session running repeated uncapped builds — especially with a test suite running alongside — has already OOM-killed this host once. If you need it faster, use ccache (already wired in), not more jobs.
+
+Link steps are capped separately and automatically: the root `CMakeLists.txt` puts them in a Ninja job pool (`OLO_LINK_JOBS`, default 2) because linking the full static engine is the memory spike. That pool lives in the generated `build.ninja`, so it protects a bare `ninja` too — but it does **not** cap compilation, which is what the job count above is for.
+
 VS Code tasks ([.vscode/tasks.json](.vscode/tasks.json)) wrap the above: `build-oloeditor-debug`, `run-oloeditor-debug`, `build-tests-debug`, `run-tests-debug`, `build-clangcl-tests-debug`, `configure-clangcl`, etc.
 
 **Working directory matters.** `OloEditor`, `OloRuntime`, and `OloServer` resolve assets, shaders, and Mono assemblies relative to `OloEditor/`. Always run with `cwd = OloEditor/` (the VS Code tasks already do this; the test binary runs from repo root instead).

CMakeLists.txt

@@ -77,6 +77,44 @@ list(APPEND CMAKE_MODULE_PATH
     ${CMAKE_CURRENT_BINARY_DIR}
 )
 
+# --------------------------------------------------------------------------
+# Link concurrency (Ninja only).
+#
+# Linking is this project's memory spike, not compiling: OloEngine-Tests and
+# OloEditor pull in the whole static engine plus its vendored dependencies, and
+# each concurrent link holds that peak at once. Ninja's default width is
+# `cores + 2` (18 on a 16-core host), so an unqualified `ninja` will happily
+# start several of those together and take the machine out — which is exactly
+# how this host was OOM-killed while three other CI runners shared it.
+#
+# A pool is the right tool rather than a smaller `-j`: it throttles ONLY the
+# link step, leaving compilation at full width. It also lives in the generated
+# build.ninja, so it protects a bare `ninja` invocation too — not just
+# `cmake --build`, which is the only thing CMAKE_BUILD_PARALLEL_LEVEL reaches.
+#
+# Raise it on a machine with memory to spare: -DOLO_LINK_JOBS=4.
+if(CMAKE_GENERATOR MATCHES "Ninja")
+    set(OLO_LINK_JOBS "2" CACHE STRING "Maximum concurrent link steps (Ninja generators)")
+    # THIS PROJECT requires a finite positive depth — that is a stricter rule
+    # than the toolchain's, deliberately.
+    #
+    # Measured, rather than assumed: CMake already rejects a non-numeric depth
+    # at configure time (build.ninja is never generated), so that case needs no
+    # help here. But `depth = 0` passes straight through CMake AND ninja, and
+    # the build then runs happily with the cap doing nothing. Since the entire
+    # point of this pool is to bound peak memory during linking, a value that
+    # quietly disables it is worse than one that fails — so reject it here and
+    # say why, instead of leaving a green build that is not actually capped.
+    if(NOT OLO_LINK_JOBS MATCHES "^[1-9][0-9]*$")
+        message(FATAL_ERROR
+            "OLO_LINK_JOBS must be a positive integer (got '${OLO_LINK_JOBS}'). "
+            "It bounds concurrent link steps to limit peak memory; 0 or empty "
+            "would leave links effectively uncapped.")
+    endif()
+    set_property(GLOBAL PROPERTY JOB_POOLS olo_link=${OLO_LINK_JOBS})
+    set(CMAKE_JOB_POOL_LINK olo_link)
+endif()
+
 set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS ON)

OloEditor/assets/shaders/AtmosphereSky.glsl

@@ -117,16 +117,46 @@ vec3 dayLayer(vec3 viewDir)
 
 // ── Night half — mirrored CPU-side in AtmosphereSky.cpp ──
 
-// Mirrors Hash13 (AtmosphereSky.cpp).
-float hash13(vec3 p)
+// Integer bit-mixer (PCG output permutation). Mirrors PcgHash
+// (AtmosphereSky.cpp) EXACTLY: unsigned wrap, shift and xor are bit-defined
+// operations, so every vendor and the CPU produce identical results.
+//
+// This replaces the classic `fract(sin(dot(p, k)) * 43758.5453)` hash, which
+// is NOT portable. That hash feeds sin() an argument in the tens of thousands,
+// where a 1-ULP difference in the argument moves the result by a large
+// fraction of a period; NVIDIA and Mesa do not implement sin() to identical
+// precision there, so the *= 43758 and fract() amplified the disagreement into
+// completely different values. The stars therefore landed in different places
+// on different GPUs, and the C++ mirror (std::sin) could match neither.
+uint pcgHash(uint v)
+{
+    uint state = v * 747796405u + 2891336453u;
+    uint word = ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u;
+    return (word >> 22u) ^ word;
+}
+
+// Mirrors HashCell (AtmosphereSky.cpp). Integer lattice cell -> uint.
+uint hashCell(ivec3 c, uint seed)
+{
+    uint h = pcgHash(uint(c.x) ^ 0x9E3779B9u);
+    h = pcgHash(h ^ uint(c.y) ^ 0x85EBCA6Bu);
+    h = pcgHash(h ^ uint(c.z) ^ 0xC2B2AE35u);
+    return pcgHash(h ^ seed);
+}
+
+// Mirrors Hash1 (AtmosphereSky.cpp). Result in [0,1).
+// Masked to 24 bits so the uint->float conversion is EXACT on every
+// implementation (a float mantissa holds 24 bits), and scaled by a power of
+// two so the divide introduces no rounding of its own.
+float hash1(ivec3 c, uint seed)
 {
-    return fract(sin(dot(p, vec3(127.1, 311.7, 74.7))) * 43758.5453);
+    return float(hashCell(c, seed) & 0xFFFFFFu) * (1.0 / 16777216.0);
 }
 
-// Mirrors Hash33 (AtmosphereSky.cpp).
-vec3 hash33(vec3 p)
+// Mirrors Hash3 (AtmosphereSky.cpp).
+vec3 hash3(ivec3 c)
 {
-    return vec3(hash13(p), hash13(p + vec3(19.19, 0.0, 0.0)), hash13(p + vec3(0.0, 47.31, 0.0)));
+    return vec3(hash1(c, 0u), hash1(c, 1u), hash1(c, 2u));
 }
 
 // Mirrors StarField (AtmosphereSky.cpp).
@@ -136,12 +166,16 @@ float starField(vec3 dir, float rotation, float intensity)
     float s = sin(rotation);
     vec3 d = vec3(c * dir.x + s * dir.z, dir.y, -s * dir.x + c * dir.z);
 
+    // `dir` is unit, so p stays within +/-60 and the cell index converts to
+    // int exactly. A 1-ULP disagreement in cos/sin between vendors now only
+    // matters exactly on a cell boundary, instead of re-rolling every star.
     vec3 p = d * 60.0;
-    vec3 cell = floor(p);
-    vec3 f = p - cell;
-    vec3 starPos = hash33(cell);
+    vec3 cellF = floor(p);
+    ivec3 cell = ivec3(cellF);
+    vec3 f = p - cellF;
+    vec3 starPos = hash3(cell);
     float dist = length(f - starPos);
-    float lum = pow(hash13(cell + vec3(17.0)), 14.0);
+    float lum = pow(hash1(cell, 3u), 14.0);
     float star = smoothstep(0.18, 0.0, dist) * lum;
     return star * intensity * 60.0;
 }

OloEditor/assets/shaders/InfiniteGrid.glsl

@@ -143,7 +143,24 @@ void main() {
         }
 
         FragColor = gridColor;
-        gl_FragDepth = depth;
+
+        // Break the coplanar tie with scene ground geometry.
+        //
+        // The grid lies on Y=0, and scenes routinely put a ground plane on Y=0
+        // too. The ground mesh gets the rasteriser's INTERPOLATED depth; this
+        // shader writes gl_FragDepth from a position that was unprojected and
+        // then re-projected. Those two paths compute the same surface and
+        // disagree by ~1 ULP, so the depth test resolves it per-pixel on float
+        // noise. NVIDIA happens to resolve it consistently; Mesa/radeonsi does
+        // not, and the grid lines break into dashes toward the horizon
+        // ("z-precision-dashed", RMSE ~22 against the NVIDIA goldens).
+        //
+        // Nudging the grid a hair toward the camera makes it win the tie on
+        // every vendor instead of by luck. 1e-5 in [0,1] depth is ~170 quanta
+        // of a 24-bit buffer -- comfortably decisive, and far too small to lift
+        // the grid visibly off the ground.
+        const float kCoplanarBias = 1e-5;
+        gl_FragDepth = clamp(depth - kCoplanarBias, 0.0, 1.0);
         EntityID = -1;  // Grid is not pickable
         o_ViewNormal = vec2(-2.0);