Benchmark the forward pass and optimize CPU+GPU; portable build by default

CMakeLists.txt

@@ -31,6 +31,54 @@ endif()
 set(GGML_BUILD_TESTS    OFF)
 set(GGML_BUILD_EXAMPLES OFF)
 
+# tinyBLAS (llamafile) cache-blocked GEMM for the CPU backend. ggml ships it OFF
+# by default, leaving mul_mat on the per-row ggml_vec_dot fallback; turning it on
+# gives a cache-blocked F16/F32 GEMM that runs the transformer Linears ~1.6x
+# faster (53%->32% of CPU time at S=8192). It is numerically a no-op (same f32
+# accumulation) and is parity-verified layer-by-layer against the f64 reference
+# at the strict CPU-f32 gate. Set as the ggml *default* (not forced) so it covers
+# every preset yet still yields to an explicit -DGGML_LLAMAFILE=OFF.
+set(GGML_LLAMAFILE_DEFAULT ON)
+
+# Portable, fast CPU build by DEFAULT -- including a preset-less `cmake -B build`.
+# Build every ggml CPU ISA variant (SSE..AVX-512) as runtime-dispatched dynamic
+# backends and pick the best at load time, instead of a single -march=native
+# target. Native is fragile across machines and, under Nix, NIX_ENFORCE_NO_NATIVE
+# strips it -- silently leaving a ~13x slower SCALAR build. The presets used to be
+# the only portable path; this makes the bare build "just work" too.
+#
+# Scope: only a plain CPU Release/RelWithDebInfo (or the typeless bare build).
+# Skipped for GPU/fuzz/debug builds (kept simple and quick to compile), and each
+# knob is guarded so an explicit -D or a preset cacheVariable still wins:
+#   - release-native (GGML_NATIVE=ON) keeps its single native target -- the
+#     `AND NOT GGML_NATIVE` guards avoid ggml's NATIVE-vs-BACKEND_DL conflict;
+#   - release / release-portable already set these ON (DEFINED -> left as-is).
+# (GGML_BACKEND_DL's BUILD_SHARED_LIBS requirement is met by ggml's own
+# default-ON shared-libs option on Linux.)
+if ((NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE MATCHES "Release|RelWithDebInfo")
+        AND NOT FREE_SPLATTER_VULKAN AND NOT FREE_SPLATTER_CUDA AND NOT FREE_SPLATTER_FUZZ)
+    # CACHE (no FORCE): visible/overridable, and an explicit -D or preset value
+    # already in the cache wins. ggml's own option() then honors these.
+    set(GGML_NATIVE OFF CACHE BOOL "ggml: optimize for the build CPU (off => portable multi-variant build)")
+    if (NOT GGML_NATIVE)
+        set(GGML_BACKEND_DL       ON CACHE BOOL "ggml: build backends as dynamic libs (enables runtime CPU ISA dispatch)")
+        set(GGML_CPU_ALL_VARIANTS ON CACHE BOOL "ggml: build every CPU ISA variant; best is picked at load time")
+    endif()
+endif()
+
+# Co-locate our executables with ggml's dynamically-loaded backend .so files. A
+# GGML_BACKEND_DL build emits libggml-*.so to <build>/bin and loads them by path
+# at runtime; ggml sets CMAKE_RUNTIME_OUTPUT_DIRECTORY for that, but only in its
+# own (subdirectory) scope -- so without this our cli/bench land in the build
+# root, away from the backends, and startup fails with "CPU backend init failed".
+# The presets set this explicitly; an explicit value still wins (if-not-set).
+# Gated on GGML_BACKEND_DL: only a dynamic-backend build needs the co-location,
+# so static builds (vulkan/debug/native presets) keep their default binary
+# location and existing script paths (e.g. bench.sh's build/vulkan/...) hold.
+if (GGML_BACKEND_DL AND NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+endif()
+
 # --- Optional custom-op patch slot (LocalVQE pattern) -----------------------
 # If pieces 1-3 ever need a ggml op not upstream, drop a patch in patches/ and it
 # is applied to the submodule working tree at configure time, idempotently.

CMakePresets.json

@@ -15,20 +15,30 @@
     {
       "name": "release",
       "binaryDir": "${sourceDir}/build/release",
-      "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" }
-    },
-    {
-      "name": "release-portable",
-      "binaryDir": "${sourceDir}/build/release-portable",
-      "description": "Portable + fast CPU: build every ggml-cpu ISA variant and pick the best at runtime. Avoids -march=native (fragile, and stripped by Nix's NIX_ENFORCE_NO_NATIVE).",
+      "description": "Default release build. Portable + fast CPU: builds every ggml-cpu ISA variant (SSE..AVX-512) and selects the best at runtime. We deliberately do NOT default to -march=native: it is fragile across machines, and Nix's NIX_ENFORCE_NO_NATIVE strips it, leaving a SCALAR build ~13x slower. For a single-target native binary on non-Nix, use `release-native`.",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "GGML_NATIVE": "OFF",
         "GGML_BACKEND_DL": "ON",
         "GGML_CPU_ALL_VARIANTS": "ON",
+        "CMAKE_RUNTIME_OUTPUT_DIRECTORY": "${sourceDir}/build/release/bin"
+      }
+    },
+    {
+      "name": "release-portable",
+      "inherits": "release",
+      "binaryDir": "${sourceDir}/build/release-portable",
+      "description": "Alias of `release` (the portable default), kept for scripts/muscle-memory; identical flags in a separate build dir.",
+      "cacheVariables": {
         "CMAKE_RUNTIME_OUTPUT_DIRECTORY": "${sourceDir}/build/release-portable/bin"
       }
     },
+    {
+      "name": "release-native",
+      "binaryDir": "${sourceDir}/build/release-native",
+      "description": "Single-target -march=native build (one self-contained binary, no runtime ISA dispatch). Fastest on the build machine, but NOT portable, and scalar under Nix (native is stripped) -- use only off-Nix.",
+      "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "GGML_NATIVE": "ON" }
+    },
     {
       "name": "vulkan",
       "binaryDir": "${sourceDir}/build/vulkan",
@@ -58,6 +68,7 @@
     { "name": "debug",            "configurePreset": "debug" },
     { "name": "release",          "configurePreset": "release" },
     { "name": "release-portable", "configurePreset": "release-portable" },
+    { "name": "release-native",   "configurePreset": "release-native" },
     { "name": "vulkan",           "configurePreset": "vulkan" },
     { "name": "profile",          "configurePreset": "profile" },
     { "name": "fuzz",             "configurePreset": "fuzz" }

README.md

@@ -12,6 +12,23 @@ network on your CPU (or a Vulkan GPU).
 You give it N images; it gives you, for every pixel, a 3D Gaussian (position,
 colour, opacity, size, orientation) that a Gaussian-splatting viewer can render.
 
+## Speed
+
+One scene — 2 photos at 512×512 — on a desktop (AMD Ryzen 9 7900, NVIDIA
+RTX 5070 Ti), median time for a single forward pass:
+
+| run on | free-splatter.cpp | reference PyTorch¹ |
+|---|--:|--:|
+| **GPU** | **0.22 s** · Vulkan, f16 | 1.37 s · CUDA, fp16 |
+| **CPU** | **14 s** · 12 threads, f16 | 54 s · fp32 |
+
+¹ the upstream FreeSplatter transformer in eager PyTorch — the same forward pass on
+the same machine (reproduce with `scripts/bench.sh`).
+
+A GPU turns a handful of photos into a splat in well under a second; even the
+no-GPU, no-Python CPU path beats the reference by ~4×. Run time scales with the
+number of input views.
+
 ## Build
 
 ```sh
@@ -20,6 +37,12 @@ cmake --preset release
 cmake --build --preset release
 ```
 
+The `release` preset builds a **portable** CPU binary — every ggml CPU ISA
+variant (up to AVX-512), with the best picked at runtime. (It avoids
+`-march=native`, which Nix strips, leaving a ~13× slower scalar build; off-Nix
+you can use `--preset release-native` for a single-target binary.) The
+executables land in `build/release/bin/`.
+
 No Nix? You just need CMake, a C++17 compiler, and the bundled `ggml` submodule
 (`git submodule update --init`).
 

bench/free_splatter-bench.cpp

@@ -1,36 +1,130 @@
-// free_splatter-bench: load a model and time end-to-end forward passes.
-// (Forward is built incrementally across M1-M3; until then this reports load +
-// geometry so the target links and the harness exists.)
+// free_splatter-bench: time end-to-end forward passes of the engine.
+//
+// Loads a model, builds an N-view input (a deterministic synthetic batch, or a
+// raw view-major NCHW .f32 file via --input), then runs free_splatter_run a few
+// times and reports per-iteration latency (min/median/mean/max) and throughput.
+// Load time is reported separately and excluded from the forward timings.
+//
+//   free_splatter-bench [--device DEV] [--views N] [--input FILE.f32]
+//                       [--iters K] [--warmup W] [--threads T] MODEL.gguf
+//
+// The last line is machine-parseable (scripts/bench.sh reads it):
+//   RESULT engine device=DEV views=N gc=23 load_ms=.. min_ms=.. median_ms=.. \
+//          mean_ms=.. max_ms=.. views_per_s=..
 #include "free_splatter.h"
 
+#include <algorithm>
+#include <chrono>
+#include <cstdint>
 #include <cstdio>
+#include <cstring>
+#include <fstream>
 #include <string>
+#include <vector>
+
+using clk = std::chrono::steady_clock;
+static double ms_since(clk::time_point t0) {
+    return std::chrono::duration<double, std::milli>(clk::now() - t0).count();
+}
+
+static int usage(const char * a0) {
+    std::fprintf(stderr,
+        "usage: %s [--device DEV] [--views N] [--input FILE.f32]\n"
+        "          [--iters K] [--warmup W] [--threads T] MODEL.gguf\n", a0);
+    return 2;
+}
 
 int main(int argc, char ** argv) {
-    if (argc < 2) {
-        std::fprintf(stderr, "usage: %s MODEL.gguf [--device DEV]\n", argv[0]);
-        return 2;
-    }
-    const char * device = nullptr;
-    for (int i = 2; i < argc; i++) {
-        if (std::string(argv[i]) == "--device" && i + 1 < argc) device = argv[++i];
+    const char * device = nullptr, * model = nullptr, * input = nullptr;
+    int views = 2, iters = 10, warmup = 2, threads = 0;
+
+    for (int i = 1; i < argc; i++) {
+        std::string a = argv[i];
+        if      (a == "--device" && i+1 < argc)  device  = argv[++i];
+        else if (a == "--views"  && i+1 < argc)  views   = std::atoi(argv[++i]);
+        else if (a == "--input"  && i+1 < argc)  input   = argv[++i];
+        else if (a == "--iters"  && i+1 < argc)  iters   = std::atoi(argv[++i]);
+        else if (a == "--warmup" && i+1 < argc)  warmup  = std::atoi(argv[++i]);
+        else if (a == "--threads" && i+1 < argc) threads = std::atoi(argv[++i]);
+        else if (a == "-h" || a == "--help")     return usage(argv[0]);
+        else if (!model)                         model   = argv[i];
+        else                                     return usage(argv[0]);
     }
+    if (!model || iters < 1 || views < 1) return usage(argv[0]);
 
     free_splatter_options * opts = free_splatter_options_new();
-    if (device) free_splatter_options_set_device(opts, device);
-    free_splatter_ctx * ctx = free_splatter_load(argv[1], opts);
-    free_splatter_options_free(opts);
+    if (device)      free_splatter_options_set_device(opts, device);
+    if (threads > 0) free_splatter_options_set_threads(opts, threads);
 
-    if (!ctx || free_splatter_last_error(ctx)) {
-        std::fprintf(stderr, "load failed: %s\n",
-                     ctx ? free_splatter_last_error(ctx) : "oom");
-        free_splatter_free(ctx);
-        return 1;
+    const auto t_load0 = clk::now();
+    free_splatter_ctx * ctx = free_splatter_load(model, opts);
+    const double load_ms = ms_since(t_load0);
+    free_splatter_options_free(opts);
+    if (!ctx) { std::fprintf(stderr, "load: out of memory\n"); return 1; }
+    if (const char * err = free_splatter_last_error(ctx)) {
+        std::fprintf(stderr, "load failed: %s\n", err); free_splatter_free(ctx); return 1;
     }
+
     free_splatter_geometry geo;
     free_splatter_geometry_of(ctx, &geo);
-    std::printf("loaded: %dx%d gaussian_channels=%d\n",
-                geo.image_width, geo.image_height, geo.gaussian_channels);
+    const int64_t per_view = (int64_t) geo.in_channels * geo.image_height * geo.image_width;
+
+    // Build the input: a raw .f32 (its size fixes the view count), or a
+    // deterministic synthetic batch in [0,1]. Values don't affect timing; an
+    // LCG keeps it reproducible and free of denormals.
+    std::vector<float> buf;
+    if (input) {
+        std::ifstream f(input, std::ios::binary | std::ios::ate);
+        if (!f) { std::fprintf(stderr, "cannot open %s\n", input); free_splatter_free(ctx); return 1; }
+        const std::streamsize bytes = f.tellg(); f.seekg(0);
+        buf.resize(bytes / sizeof(float)); f.read((char *) buf.data(), bytes);
+        if (per_view == 0 || buf.size() % per_view != 0) {
+            std::fprintf(stderr, "input size is not a whole number of views\n"); free_splatter_free(ctx); return 1; }
+        views = (int) (buf.size() / per_view);
+    } else {
+        buf.resize((size_t) views * per_view);
+        uint32_t s = 0x9e3779b9u;
+        for (float & v : buf) { s = s * 1664525u + 1013904223u; v = (s >> 8) * (1.0f / 16777216.0f); }
+    }
+
+    std::printf("model: %s  device=%s  %dx%d  in=%d  gc=%d  views=%d  S=%lld  load=%.1f ms\n",
+                model, device ? device : "cpu", geo.image_width, geo.image_height,
+                geo.in_channels, geo.gaussian_channels, views,
+                (long long) ((int64_t) views * (geo.image_height/8) * (geo.image_width/8)), load_ms);
+
+    auto once = [&](double * out_ms) -> bool {
+        float * out = nullptr; size_t n_out = 0;
+        const auto t0 = clk::now();
+        const int rc = free_splatter_run(ctx, buf.data(), (int32_t) views,
+                                         geo.image_height, geo.image_width, &out, &n_out);
+        if (out_ms) *out_ms = ms_since(t0);
+        if (rc != 0) { std::fprintf(stderr, "run failed: %s\n", free_splatter_last_error(ctx)); return false; }
+        free_splatter_buf_free(out);
+        return true;
+    };
+
+    for (int i = 0; i < warmup; i++) { if (!once(nullptr)) { free_splatter_free(ctx); return 1; } }
+
+    std::vector<double> t(iters);
+    for (int i = 0; i < iters; i++) {
+        if (!once(&t[i])) { free_splatter_free(ctx); return 1; }
+        std::printf("  iter %2d: %8.1f ms\n", i, t[i]);
+    }
     free_splatter_free(ctx);
+
+    std::vector<double> sorted = t;
+    std::sort(sorted.begin(), sorted.end());
+    const double mn = sorted.front(), mx = sorted.back();
+    const double med = sorted[iters / 2];
+    double sum = 0; for (double v : t) sum += v;
+    const double mean = sum / iters;
+    const double vps = views / (med / 1000.0);
+
+    std::printf("\nsummary: min=%.1f  median=%.1f  mean=%.1f  max=%.1f ms   "
+                "(%.2f views/s, %.2f scenes/s)\n", mn, med, mean, mx, vps, 1000.0 / med);
+    std::printf("RESULT engine device=%s views=%d gc=%d load_ms=%.1f min_ms=%.1f "
+                "median_ms=%.1f mean_ms=%.1f max_ms=%.1f views_per_s=%.3f\n",
+                device ? device : "cpu", views, geo.gaussian_channels, load_ms,
+                mn, med, mean, mx, vps);
     return 0;
 }

scripts/bench.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Benchmark free-splatter.cpp against the upstream FreeSplatter PyTorch model.
+#
+# Times one forward pass (N views -> the [N,512,512,23] transformer logits) and
+# compares the ggml engine (CPU with AVX/AVX512, and Vulkan GPU) against PyTorch
+# (CPU, and CUDA GPU), at matching precisions, then prints a table.
+#
+#   nix develop -c scripts/bench.sh                 # default: 2 views
+#   VIEWS=4 ITERS=10 nix develop -c scripts/bench.sh
+#
+# Notes:
+#  - CPU uses the `release-portable` build (every ggml CPU ISA variant, best
+#    picked at runtime). The plain `release` preset is SCALAR under Nix
+#    (NIX_ENFORCE_NO_NATIVE strips -march=native) and ~13x slower -- do not use
+#    it for CPU timing.
+#  - PyTorch-on-CUDA needs a cu1xx torch build (.venv-torch-cu128) and the host
+#    NVIDIA driver libs (/run/opengl-driver/lib) on LD_LIBRARY_PATH; both are set
+#    up below. The CPU-only .venv-torch cannot use the GPU.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+ROOT=$(pwd)
+
+VIEWS=${VIEWS:-2}
+ITERS=${ITERS:-8}
+CPU_ITERS=${CPU_ITERS:-3}
+GGUF_F16=${GGUF_F16:-$ROOT/.cache/freesplatter-scene-f16.gguf}
+GGUF_F32=${GGUF_F32:-$ROOT/.cache/freesplatter-scene-f32.gguf}
+CKPT=${CKPT:-$ROOT/.cache/freesplatter-scene.safetensors}
+TORCH=${TORCH_PY:-$ROOT/.venv-torch-cu128/bin/python}
+
+# pip-CUDA torch finds libcuda.so.1 from the host driver here (NixOS).
+export LD_LIBRARY_PATH="/run/opengl-driver/lib:${LD_LIBRARY_PATH:-}"
+
+CPU_BIN=$ROOT/build/release-portable/bin/free_splatter-bench
+VK_BIN=$ROOT/build/vulkan/free_splatter-bench
+
+echo "building bench binaries (if needed) ..."
+[ -x "$CPU_BIN" ] || { cmake --preset release-portable >/dev/null && \
+    cmake --build --preset release-portable --target free_splatter-bench -j >/dev/null; }
+[ -x "$VK_BIN" ]  || { cmake --preset vulkan >/dev/null && \
+    cmake --build build/vulkan --target free_splatter-bench -j >/dev/null; }
+
+RESULTS=$(mktemp)
+trap 'rm -f "$RESULTS"' EXIT
+
+run() {  # label, command...
+    local label=$1; shift
+    printf '>>> %s\n' "$label" >&2
+    local line
+    line=$("$@" 2>/dev/null | grep '^RESULT' || true)
+    if [ -n "$line" ]; then echo "$label|$line" >> "$RESULTS"
+    else printf '    (no result -- skipped or failed)\n' >&2; fi
+}
+
+# --- engine (ggml) ---
+[ -f "$GGUF_F32" ] && run "engine | cpu    | f32 " "$CPU_BIN" --device cpu    --views "$VIEWS" --iters "$CPU_ITERS" --warmup 1 "$GGUF_F32"
+[ -f "$GGUF_F16" ] && run "engine | cpu    | f16 " "$CPU_BIN" --device cpu    --views "$VIEWS" --iters "$CPU_ITERS" --warmup 1 "$GGUF_F16"
+[ -f "$GGUF_F16" ] && run "engine | vulkan | f16 " "$VK_BIN"  --device vulkan --views "$VIEWS" --iters "$ITERS"     --warmup 2 "$GGUF_F16"
+[ -f "$GGUF_F32" ] && run "engine | vulkan | f32 " "$VK_BIN"  --device vulkan --views "$VIEWS" --iters "$ITERS"     --warmup 2 "$GGUF_F32"
+
+# --- PyTorch reference ---
+if [ -x "$TORCH" ] && [ -f "$CKPT" ]; then
+    run "torch  | cpu    | fp32" "$TORCH" scripts/bench_torch.py --ckpt "$CKPT" --device cpu  --dtype fp32 --views "$VIEWS" --iters "$CPU_ITERS" --warmup 1
+    if "$TORCH" -c 'import torch,sys; sys.exit(0 if torch.cuda.is_available() else 1)' 2>/dev/null; then
+        run "torch  | cuda   | fp16" "$TORCH" scripts/bench_torch.py --ckpt "$CKPT" --device cuda --dtype fp16 --views "$VIEWS" --iters "$ITERS" --warmup 3
+        run "torch  | cuda   | fp32" "$TORCH" scripts/bench_torch.py --ckpt "$CKPT" --device cuda --dtype fp32 --views "$VIEWS" --iters "$ITERS" --warmup 3
+        run "torch  | cuda   | bf16" "$TORCH" scripts/bench_torch.py --ckpt "$CKPT" --device cuda --dtype bf16 --views "$VIEWS" --iters "$ITERS" --warmup 3
+    else
+        echo ">>> torch CUDA unavailable (need .venv-torch-cu128 + /run/opengl-driver/lib)" >&2
+    fi
+else
+    echo ">>> torch skipped (no $TORCH or $CKPT)" >&2
+fi
+
+# --- table ---
+echo
+printf '%-26s | %12s | %10s\n' "engine | device | dtype" "median (ms)" "scenes/s"
+printf '%-26s-+-%12s-+-%10s\n' "$(printf '%.0s-' {1..26})" "------------" "----------"
+sort "$RESULTS" | while IFS='|' read -r who dev dt rest; do
+    med=$(echo "$rest" | grep -o 'median_ms=[0-9.]*' | cut -d= -f2)
+    vps=$(echo "$rest" | grep -o 'views_per_s=[0-9.]*' | cut -d= -f2)
+    sps=$(awk -v v="$vps" -v n="$VIEWS" 'BEGIN{ if (v>0) printf "%.3f", v/n; else print "-" }')
+    printf '%-7s|%-8s|%-5s | %12s | %10s\n' "$who" "$dev" "$dt" "$med" "$sps"
+done
+echo
+echo "(N=$VIEWS views, S=$((VIEWS*4096)) tokens; median of timed iters; lower ms / higher scenes/s is better)"