From ac55665243c72de283b428c0b903f73bf354d360 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 19 Jul 2026 16:22:15 +0200 Subject: [PATCH 1/5] Add Nix flake with CPU/CUDA/Metal builds and a NixOS module --- flake.lock | 85 +++++++++ flake.nix | 510 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 595 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..cc14f0c9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,85 @@ +{ + "nodes": { + "crane": { + "locked": { + "lastModified": 1784407669, + "narHash": "sha256-gcFMcRjw0ZSn380Rx2QLlU1goUQeSrKX/DF12omI6+o=", + "owner": "ipetkov", + "repo": "crane", + "rev": "1316b7d278ad77a16aec024b71d971366e123bec", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1784356753, + "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "crane": "crane", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784438913, + "narHash": "sha256-NYF7ZM5ip0u+w1pBFDpIGEbrbgN/wpnLFAmBkWkYMXw=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "afacd6819d3765a05814ee8e3de74c77d42ac799", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..ed53d219 --- /dev/null +++ b/flake.nix @@ -0,0 +1,510 @@ +{ + description = "Paddler — open-source LLMOps platform for hosting and scaling LLMs in your own infrastructure"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + crane.url = "github:ipetkov/crane"; + }; + + outputs = + inputs@{ + self, + nixpkgs, + flake-parts, + rust-overlay, + crane, + }: + flake-parts.lib.mkFlake { inherit inputs; } { + systems = [ + "x86_64-linux" + "aarch64-darwin" + ]; + + perSystem = + { system, ... }: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + lib = pkgs.lib; + + rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain; + craneLibCuda = craneLib.overrideScope ( + _final: _prev: { + stdenvSelector = eachPkgs: eachPkgs.cudaPackages.backendStdenv; + } + ); + + version = "4.1.0"; + + webAdminPanelAssets = pkgs.buildNpmPackage { + pname = "paddler-web-admin-panel"; + inherit version; + src = self; + npmDepsHash = "sha256-sBDdMf388qFQVIjQ3t/BL3KC/yAqF1qj47a/40axgF8="; + dontNpmBuild = true; + nativeBuildInputs = [ pkgs.nodejs ]; + buildPhase = '' + runHook preBuild + node jarmuz-static.mjs + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p "$out" + cp -r static "$out/static" + cp esbuild-meta.json "$out/esbuild-meta.json" + runHook postInstall + ''; + }; + + injectWebAdminPanelAssets = '' + cp -r --no-preserve=mode,ownership ${webAdminPanelAssets}/static ./static + cp --no-preserve=mode,ownership ${webAdminPanelAssets}/esbuild-meta.json ./esbuild-meta.json + ''; + + acceleratorInputs = + accelerator: + if accelerator == "cpu" then + { + cargoFeatures = [ ]; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + env = { }; + } + else if accelerator == "cuda" then + { + cargoFeatures = [ "cuda" ]; + nativeBuildInputs = [ + pkgs.cudaPackages.cuda_nvcc + pkgs.autoAddDriverRunpath + ]; + buildInputs = [ + pkgs.cudaPackages.cuda_cudart + pkgs.cudaPackages.libcublas + (lib.getOutput "static" pkgs.cudaPackages.libcublas) + pkgs.cudaPackages.cccl + ]; + env = { + CMAKE_CUDA_ARCHITECTURES = pkgs.cudaPackages.flags.cmakeCudaArchitecturesString; + CARGO_BUILD_RUSTFLAGS = lib.concatStringsSep " " [ + "-L native=${pkgs.cudaPackages.cuda_cudart}/lib" + "-L native=${pkgs.cudaPackages.cuda_cudart}/lib/stubs" + "-L native=${lib.getOutput "static" pkgs.cudaPackages.libcublas}/lib" + ]; + }; + } + else if accelerator == "metal" then + { + cargoFeatures = [ "metal" ]; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + env = { }; + } + else + throw "paddler: unsupported accelerator '${accelerator}'"; + + mkPaddler = + { + accelerator ? "cpu", + webAdminPanel ? true, + }: + let + accel = acceleratorInputs accelerator; + craneLibEff = if accelerator == "cuda" then craneLibCuda else craneLib; + features = accel.cargoFeatures ++ lib.optional webAdminPanel "web_admin_panel"; + featureFlags = lib.optionals (features != [ ]) [ + "--features" + (lib.concatStringsSep "," features) + ]; + cargoExtraArgs = lib.escapeShellArgs ( + [ + "-p" + "paddler_cli" + ] + ++ featureFlags + ); + + pname = "paddler${lib.optionalString (accelerator != "cpu") "-${accelerator}"}${ + lib.optionalString (!webAdminPanel) "-headless" + }"; + + commonArgs = { + inherit cargoExtraArgs version pname; + src = self; + strictDeps = true; + doCheck = false; + nativeBuildInputs = [ + pkgs.cmake + pkgs.pkg-config + pkgs.llvmPackages.clang + ] + ++ accel.nativeBuildInputs; + buildInputs = [ pkgs.openssl ] ++ accel.buildInputs; + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + } + // accel.env; + + cargoArtifacts = craneLibEff.buildDepsOnly ( + commonArgs // { src = craneLibEff.cleanCargoSource self; } + ); + in + craneLibEff.buildPackage ( + commonArgs + // { + inherit cargoArtifacts; + meta = { + description = "Paddler ${accelerator} build (web admin panel ${ + if webAdminPanel then "enabled" else "disabled" + })"; + homepage = "https://paddler.intentee.com/"; + license = lib.licenses.asl20; + mainProgram = "paddler"; + }; + } + // lib.optionalAttrs webAdminPanel { preBuild = injectWebAdminPanelAssets; } + ); + + allowUnfree = (pkgs.config.allowUnfree or false) || (builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1"); + + linuxPackages = { + paddler = mkPaddler { }; + paddler-headless = mkPaddler { webAdminPanel = false; }; + } + // lib.optionalAttrs allowUnfree { + paddler-cuda = mkPaddler { accelerator = "cuda"; }; + paddler-cuda-headless = mkPaddler { + accelerator = "cuda"; + webAdminPanel = false; + }; + }; + + darwinPackages = { + paddler = mkPaddler { accelerator = "metal"; }; + paddler-headless = mkPaddler { + accelerator = "metal"; + webAdminPanel = false; + }; + }; + + accelPackages = if pkgs.stdenv.hostPlatform.isDarwin then darwinPackages else linuxPackages; + in + { + packages = accelPackages // { + default = accelPackages.paddler; + }; + + apps.default = { + type = "app"; + program = "${lib.getExe accelPackages.paddler}"; + }; + + checks = { + paddler = accelPackages.paddler; + paddler-headless = accelPackages.paddler-headless; + }; + + devShells.default = craneLib.devShell { + packages = [ + pkgs.nodejs + pkgs.cmake + pkgs.pkg-config + pkgs.llvmPackages.clang + pkgs.openssl + ]; + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + }; + + formatter = pkgs.nixfmt-rfc-style; + }; + + flake = + let + paddlerNixosModule = + { + config, + lib, + pkgs, + utils, + ... + }: + let + cfg = config.services.paddler; + + defaultPackage = self.packages.${pkgs.stdenv.hostPlatform.system}.paddler; + + socketAddrType = lib.types.str; + + balancerArgs = + let + balancer = cfg.balancer; + in + [ + "balancer" + "--management-addr" + balancer.managementAddr + "--inference-addr" + balancer.inferenceAddr + "--state-database" + balancer.stateDatabase + ] + ++ lib.optionals (balancer.webAdminPanelAddr != null) [ + "--web-admin-panel-addr" + balancer.webAdminPanelAddr + ] + ++ lib.optionals (balancer.openaiCompatAddr != null) [ + "--compat-openai-addr" + balancer.openaiCompatAddr + ] + ++ lib.concatMap (host: [ + "--management-cors-allowed-host" + host + ]) balancer.managementCorsAllowedHosts + ++ lib.concatMap (host: [ + "--inference-cors-allowed-host" + host + ]) balancer.inferenceCorsAllowedHosts + ++ balancer.extraArgs; + + agentArgs = + let + agent = cfg.agent; + in + [ + "agent" + "--management-addr" + agent.managementAddr + "--slots" + (toString agent.slots) + ] + ++ lib.optionals (agent.name != null) [ + "--name" + agent.name + ] + ++ agent.extraArgs; + in + { + options.services.paddler = { + balancer = { + enable = lib.mkEnableOption "the Paddler balancer service"; + + package = lib.mkOption { + type = lib.types.package; + default = defaultPackage; + defaultText = lib.literalExpression "paddler.packages.\${system}.paddler"; + description = "The paddler package used for the balancer."; + }; + + managementAddr = lib.mkOption { + type = socketAddrType; + default = "127.0.0.1:8060"; + description = '' + Address of the management server. Agents connect here and the web admin + panel calls it directly from the browser, so if the panel is used remotely + this must be an address the browser can actually reach. + ''; + }; + + inferenceAddr = lib.mkOption { + type = socketAddrType; + default = "127.0.0.1:8061"; + description = '' + Address of the inference server. The web admin panel calls it directly from + the browser, so if the panel is used remotely this must be browser-reachable. + ''; + }; + + webAdminPanelAddr = lib.mkOption { + type = lib.types.nullOr socketAddrType; + default = null; + example = "127.0.0.1:8062"; + description = '' + Address of the web admin panel. When null the panel is disabled. Requires a + package built with the web admin panel feature (the default package). + ''; + }; + + openaiCompatAddr = lib.mkOption { + type = lib.types.nullOr socketAddrType; + default = null; + description = "Address of the OpenAI-compatible API server. When null it is disabled."; + }; + + stateDatabase = lib.mkOption { + type = lib.types.str; + default = "file:///var/lib/paddler/state.db"; + description = '' + Balancer state database URL. Either memory:// or file:///absolute/path. + A file database persists the runtime model assignment across restarts. + ''; + }; + + managementCorsAllowedHosts = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Allowed CORS hosts for the management service."; + }; + + inferenceCorsAllowedHosts = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Allowed CORS hosts for the inference service."; + }; + + extraArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra command-line arguments passed to the balancer."; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open the management, inference, web admin panel and OpenAI-compatible ports in the firewall."; + }; + }; + + agent = { + enable = lib.mkEnableOption '' + the Paddler agent service. Run a single agent per host: one agent already + saturates the host's inference hardware with its slots, so additional agents + on the same host would contend for the same GPU or CPU + ''; + + package = lib.mkOption { + type = lib.types.package; + default = defaultPackage; + defaultText = lib.literalExpression "paddler.packages.\${system}.paddler"; + description = "The paddler package used for the agent (e.g. paddler-cuda for GPU)."; + }; + + managementAddr = lib.mkOption { + type = socketAddrType; + example = "127.0.0.1:8060"; + description = "Management address of the balancer to connect to."; + }; + + slots = lib.mkOption { + type = lib.types.ints.positive; + example = 4; + description = "Number of parallel requests this agent can handle at once."; + }; + + name = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Human-readable name reported to the balancer."; + }; + + hfTokenFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + example = "/run/secrets/paddler-hf-token"; + description = '' + Path to a file containing a HuggingFace access token (the raw token on a + single line), used to download gated repositories. It is loaded as a + systemd credential and installed into the agent's HuggingFace cache + (HF_HOME/token) before the agent starts. + ''; + }; + + environment = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + description = "Extra environment variables for the agent process."; + }; + + extraArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra command-line arguments passed to the agent."; + }; + }; + }; + + config = lib.mkMerge [ + (lib.mkIf cfg.balancer.enable { + systemd.services.paddler-balancer = { + description = "Paddler balancer"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = utils.escapeSystemdExecArgs ([ (lib.getExe cfg.balancer.package) ] ++ balancerArgs); + DynamicUser = true; + StateDirectory = "paddler"; + Restart = "on-failure"; + RestartSec = 5; + ProtectSystem = "strict"; + ProtectHome = true; + NoNewPrivileges = true; + PrivateTmp = true; + }; + }; + + networking.firewall = lib.mkIf cfg.balancer.openFirewall { + allowedTCPPorts = + let + portOf = addr: lib.toInt (lib.last (lib.splitString ":" addr)); + in + [ + (portOf cfg.balancer.managementAddr) + (portOf cfg.balancer.inferenceAddr) + ] + ++ lib.optional (cfg.balancer.webAdminPanelAddr != null) (portOf cfg.balancer.webAdminPanelAddr) + ++ lib.optional (cfg.balancer.openaiCompatAddr != null) (portOf cfg.balancer.openaiCompatAddr); + }; + }) + + (lib.mkIf cfg.agent.enable { + systemd.services.paddler-agent = { + description = "Paddler agent"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + environment = { + PADDLER_CACHE_DIR = "/var/cache/paddler"; + HF_HOME = "/var/cache/paddler/huggingface"; + } + // cfg.agent.environment; + serviceConfig = { + ExecStart = utils.escapeSystemdExecArgs ([ (lib.getExe cfg.agent.package) ] ++ agentArgs); + DynamicUser = true; + CacheDirectory = "paddler"; + Restart = "on-failure"; + RestartSec = 5; + ProtectSystem = "strict"; + ProtectHome = true; + NoNewPrivileges = true; + PrivateTmp = true; + } + // lib.optionalAttrs (cfg.agent.hfTokenFile != null) { + LoadCredential = [ "hf-token:${toString cfg.agent.hfTokenFile}" ]; + ExecStartPre = "${lib.getExe' pkgs.coreutils "install"} -D -m0600 %d/hf-token /var/cache/paddler/huggingface/token"; + }; + }; + }) + ]; + }; + in + { + nixosModules.default = paddlerNixosModule; + nixosModules.paddler = paddlerNixosModule; + + overlays.default = final: _prev: { + paddler = self.packages.${final.stdenv.hostPlatform.system}.paddler; + paddler-headless = self.packages.${final.stdenv.hostPlatform.system}.paddler-headless; + }; + }; + }; +} From 78a2e08cd9b9f77ce58fbd131b5f71560f527e0b Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 19 Jul 2026 17:19:46 +0200 Subject: [PATCH 2/5] gate CUDA behind explicit cudaSupport opt-in and support aarch64-linux --- flake.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index ed53d219..82afc2e8 100644 --- a/flake.nix +++ b/flake.nix @@ -25,15 +25,20 @@ flake-parts.lib.mkFlake { inherit inputs; } { systems = [ "x86_64-linux" + "aarch64-linux" "aarch64-darwin" ]; perSystem = { system, ... }: let + allowUnfree = builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1"; + cudaSupport = builtins.getEnv "PADDLER_ENABLE_CUDA" == "1"; + pkgs = import nixpkgs { inherit system; overlays = [ (import rust-overlay) ]; + config = { inherit allowUnfree cudaSupport; }; }; lib = pkgs.lib; @@ -175,13 +180,13 @@ // lib.optionalAttrs webAdminPanel { preBuild = injectWebAdminPanelAssets; } ); - allowUnfree = (pkgs.config.allowUnfree or false) || (builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1"); + enableCuda = system == "x86_64-linux" && allowUnfree && cudaSupport; linuxPackages = { paddler = mkPaddler { }; paddler-headless = mkPaddler { webAdminPanel = false; }; } - // lib.optionalAttrs allowUnfree { + // lib.optionalAttrs enableCuda { paddler-cuda = mkPaddler { accelerator = "cuda"; }; paddler-cuda-headless = mkPaddler { accelerator = "cuda"; From 92918dfb9e6e5e35c9dff896f0f0432951a2640a Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 19 Jul 2026 21:28:22 +0200 Subject: [PATCH 3/5] upgrade to llama-cpp-bindings 0.12.0 --- Cargo.lock | 24 ++++++++++++------------ Cargo.toml | 6 +++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2faa372..f70c3122 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3728,9 +3728,9 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llama-cpp-bindings" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b3e674002002689fae8b895a8efdf1e851f46b657c32325473f375d0286b9b" +checksum = "dd2ca73bb0bda4720b8eceb8932605730f4abf7a91bdcb9c708eabb5708b94fd" dependencies = [ "encoding_rs", "enumflags2", @@ -3748,9 +3748,9 @@ dependencies = [ [[package]] name = "llama-cpp-bindings-build" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bef3ed70c4633858122ca28bdb683b28b6d036a5c2b9cdc6aab976f39a19441" +checksum = "b09a396d180c93d56342594f60b3fae78d2a517a9ed0be02d36f33c3db65e507" dependencies = [ "bindgen", "cc", @@ -3763,18 +3763,18 @@ dependencies = [ [[package]] name = "llama-cpp-bindings-sys" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf63ee8a9808ee4adf3ae5cc0da3105c3d91e53fcee8894f8254c5b5d03ab3e" +checksum = "24867ff3fe6a53c3aa2e5f47273abda1b3ffd1676fdd2785b544376609e3bdd0" dependencies = [ "llama-cpp-bindings-build", ] [[package]] name = "llama-cpp-bindings-types" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e764c657a21c313bee323869ccb0039f40f877128a6d9ecc7e2f23103a7a5d3" +checksum = "59bfea302da4827659a9eaadb587f2bbaf30b47294bf3331502bf0ca952c3b57" dependencies = [ "serde", "serde_json", @@ -3783,15 +3783,15 @@ dependencies = [ [[package]] name = "llama-cpp-error-recorder" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f454fc2ae5ca82d66bf1e22050fc25796ea1675536d1784ced70fc754458fc01" +checksum = "15b606bbd0c9ed0a82420441cab10fc9818fda77626daf488154102b4a4cf0ad" [[package]] name = "llama-cpp-log-decoder" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2508cccd00a43f0cdc0bf0904ecf7ac6ab0aa3f85b4729b1c5c97ca6ac89deb3" +checksum = "18699860211dd95bcf6752a89b637c764029be383e487112b6a64bd200609988" [[package]] name = "llguidance" diff --git a/Cargo.toml b/Cargo.toml index 47829b3a..2a1b8b53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,9 +57,9 @@ http = "1" image = "0.25" indoc = "2" jsonschema = { version = "0.37", default-features = false } -llama-cpp-bindings = "=0.11.0" -llama-cpp-bindings-sys = "=0.11.0" -llama-cpp-bindings-types = "=0.11.0" +llama-cpp-bindings = "=0.12.0" +llama-cpp-bindings-sys = "=0.12.0" +llama-cpp-bindings-types = "=0.12.0" base64 = "0.22" log = "0.4" mime_guess = "2" From 60ddd10ae8c02d59a56032f005fa311f19821f94 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 19 Jul 2026 21:28:22 +0200 Subject: [PATCH 4/5] build CUDA and Metal agents via the NixOS module with explicit GPU architectures and bounded compile parallelism --- flake.nix | 322 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 205 insertions(+), 117 deletions(-) diff --git a/flake.nix b/flake.nix index 82afc2e8..347daef7 100644 --- a/flake.nix +++ b/flake.nix @@ -22,35 +22,56 @@ rust-overlay, crane, }: - flake-parts.lib.mkFlake { inherit inputs; } { - systems = [ - "x86_64-linux" - "aarch64-linux" - "aarch64-darwin" - ]; + let + version = "4.1.0"; - perSystem = - { system, ... }: - let - allowUnfree = builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1"; - cudaSupport = builtins.getEnv "PADDLER_ENABLE_CUDA" == "1"; + paddlerPkgs = + { + system, + allowUnfree ? false, + cudaSupport ? false, + cudaCapabilities ? [ ], + }: + import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + config = { + inherit allowUnfree cudaSupport; + } + // (if cudaCapabilities == [ ] then { } else { inherit cudaCapabilities; }); + }; - pkgs = import nixpkgs { - inherit system; - overlays = [ (import rust-overlay) ]; - config = { inherit allowUnfree cudaSupport; }; - }; - lib = pkgs.lib; + craneLibFor = pkgs: (crane.mkLib pkgs).overrideToolchain (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml); - rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; - craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain; - craneLibCuda = craneLib.overrideScope ( - _final: _prev: { - stdenvSelector = eachPkgs: eachPkgs.cudaPackages.backendStdenv; - } - ); + defaultAccelerator = + pkgs: + if pkgs.stdenv.hostPlatform.isDarwin then + "metal" + else if (pkgs.config.cudaSupport or false) then + "cuda" + else + "cpu"; - version = "4.1.0"; + buildPaddler = + pkgs: + { + accelerator ? defaultAccelerator pkgs, + webAdminPanel ? true, + cudaBuildParallelism ? 4, + }: + let + lib = pkgs.lib; + + craneLib = craneLibFor pkgs; + craneLibEff = + if accelerator == "cuda" then + craneLib.overrideScope ( + _final: _prev: { + stdenvSelector = eachPkgs: eachPkgs.cudaPackages.backendStdenv; + } + ) + else + craneLib; webAdminPanelAssets = pkgs.buildNpmPackage { pname = "paddler-web-admin-panel"; @@ -78,8 +99,7 @@ cp --no-preserve=mode,ownership ${webAdminPanelAssets}/esbuild-meta.json ./esbuild-meta.json ''; - acceleratorInputs = - accelerator: + accel = if accelerator == "cpu" then { cargoFeatures = [ ]; @@ -102,6 +122,8 @@ ]; env = { CMAKE_CUDA_ARCHITECTURES = pkgs.cudaPackages.flags.cmakeCudaArchitecturesString; + CMAKE_BUILD_PARALLEL_LEVEL = toString cudaBuildParallelism; + CARGO_BUILD_JOBS = toString cudaBuildParallelism; CARGO_BUILD_RUSTFLAGS = lib.concatStringsSep " " [ "-L native=${pkgs.cudaPackages.cuda_cudart}/lib" "-L native=${pkgs.cudaPackages.cuda_cudart}/lib/stubs" @@ -119,107 +141,91 @@ else throw "paddler: unsupported accelerator '${accelerator}'"; - mkPaddler = - { - accelerator ? "cpu", - webAdminPanel ? true, - }: - let - accel = acceleratorInputs accelerator; - craneLibEff = if accelerator == "cuda" then craneLibCuda else craneLib; - features = accel.cargoFeatures ++ lib.optional webAdminPanel "web_admin_panel"; - featureFlags = lib.optionals (features != [ ]) [ - "--features" - (lib.concatStringsSep "," features) - ]; - cargoExtraArgs = lib.escapeShellArgs ( - [ - "-p" - "paddler_cli" - ] - ++ featureFlags - ); - - pname = "paddler${lib.optionalString (accelerator != "cpu") "-${accelerator}"}${ - lib.optionalString (!webAdminPanel) "-headless" - }"; - - commonArgs = { - inherit cargoExtraArgs version pname; - src = self; - strictDeps = true; - doCheck = false; - nativeBuildInputs = [ - pkgs.cmake - pkgs.pkg-config - pkgs.llvmPackages.clang - ] - ++ accel.nativeBuildInputs; - buildInputs = [ pkgs.openssl ] ++ accel.buildInputs; - LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; - } - // accel.env; - - cargoArtifacts = craneLibEff.buildDepsOnly ( - commonArgs // { src = craneLibEff.cleanCargoSource self; } - ); - in - craneLibEff.buildPackage ( - commonArgs - // { - inherit cargoArtifacts; - meta = { - description = "Paddler ${accelerator} build (web admin panel ${ - if webAdminPanel then "enabled" else "disabled" - })"; - homepage = "https://paddler.intentee.com/"; - license = lib.licenses.asl20; - mainProgram = "paddler"; - }; - } - // lib.optionalAttrs webAdminPanel { preBuild = injectWebAdminPanelAssets; } - ); + features = accel.cargoFeatures ++ lib.optional webAdminPanel "web_admin_panel"; + featureFlags = lib.optionals (features != [ ]) [ + "--features" + (lib.concatStringsSep "," features) + ]; + cargoExtraArgs = lib.escapeShellArgs ( + [ + "-p" + "paddler_cli" + ] + ++ featureFlags + ); - enableCuda = system == "x86_64-linux" && allowUnfree && cudaSupport; + pname = "paddler${lib.optionalString (accelerator != "cpu") "-${accelerator}"}${ + lib.optionalString (!webAdminPanel) "-headless" + }"; - linuxPackages = { - paddler = mkPaddler { }; - paddler-headless = mkPaddler { webAdminPanel = false; }; + commonArgs = { + inherit cargoExtraArgs version pname; + src = self; + strictDeps = true; + doCheck = false; + nativeBuildInputs = [ + pkgs.cmake + pkgs.pkg-config + pkgs.llvmPackages.clang + ] + ++ accel.nativeBuildInputs; + buildInputs = [ pkgs.openssl ] ++ accel.buildInputs; + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; } - // lib.optionalAttrs enableCuda { - paddler-cuda = mkPaddler { accelerator = "cuda"; }; - paddler-cuda-headless = mkPaddler { - accelerator = "cuda"; - webAdminPanel = false; - }; - }; + // accel.env; - darwinPackages = { - paddler = mkPaddler { accelerator = "metal"; }; - paddler-headless = mkPaddler { - accelerator = "metal"; - webAdminPanel = false; + cargoArtifacts = craneLibEff.buildDepsOnly ( + commonArgs // { src = craneLibEff.cleanCargoSource self; } + ); + in + craneLibEff.buildPackage ( + commonArgs + // { + inherit cargoArtifacts; + meta = { + description = "Paddler ${accelerator} build (web admin panel ${ + if webAdminPanel then "enabled" else "disabled" + })"; + homepage = "https://paddler.intentee.com/"; + license = lib.licenses.asl20; + mainProgram = "paddler"; }; - }; + } + // lib.optionalAttrs webAdminPanel { preBuild = injectWebAdminPanelAssets; } + ); + in + flake-parts.lib.mkFlake { inherit inputs; } { + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ]; + + perSystem = + { system, ... }: + let + pkgs = paddlerPkgs { inherit system; }; + lib = pkgs.lib; - accelPackages = if pkgs.stdenv.hostPlatform.isDarwin then darwinPackages else linuxPackages; + paddler = buildPaddler pkgs { }; + paddler-headless = buildPaddler pkgs { webAdminPanel = false; }; in { - packages = accelPackages // { - default = accelPackages.paddler; + packages = { + default = paddler; + inherit paddler paddler-headless; }; apps.default = { type = "app"; - program = "${lib.getExe accelPackages.paddler}"; + program = "${lib.getExe paddler}"; }; checks = { - paddler = accelPackages.paddler; - paddler-headless = accelPackages.paddler-headless; + inherit paddler paddler-headless; }; - devShells.default = craneLib.devShell { + devShells.default = (craneLibFor pkgs).devShell { packages = [ pkgs.nodejs pkgs.cmake @@ -246,7 +252,30 @@ let cfg = config.services.paddler; - defaultPackage = self.packages.${pkgs.stdenv.hostPlatform.system}.paddler; + boxSystem = pkgs.stdenv.hostPlatform.system; + + balancerPackage = buildPaddler (paddlerPkgs { system = boxSystem; }) { }; + + agentPackage = + let + agentCuda = cfg.agent.cuda; + in + if agentCuda.enable then + buildPaddler + (paddlerPkgs { + system = boxSystem; + allowUnfree = true; + cudaSupport = true; + cudaCapabilities = agentCuda.capabilities; + }) + { + accelerator = "cuda"; + cudaBuildParallelism = agentCuda.buildParallelism; + } + else if cfg.agent.metal.enable then + buildPaddler (paddlerPkgs { system = boxSystem; }) { accelerator = "metal"; } + else + buildPaddler (paddlerPkgs { system = boxSystem; }) { accelerator = "cpu"; }; socketAddrType = lib.types.str; @@ -305,8 +334,8 @@ package = lib.mkOption { type = lib.types.package; - default = defaultPackage; - defaultText = lib.literalExpression "paddler.packages.\${system}.paddler"; + default = balancerPackage; + defaultText = lib.literalExpression "the CPU paddler build for this host"; description = "The paddler package used for the balancer."; }; @@ -388,9 +417,51 @@ package = lib.mkOption { type = lib.types.package; - default = defaultPackage; - defaultText = lib.literalExpression "paddler.packages.\${system}.paddler"; - description = "The paddler package used for the agent (e.g. paddler-cuda for GPU)."; + default = agentPackage; + defaultText = lib.literalExpression "the paddler build selected by the agent's cuda/metal options"; + description = '' + The paddler package used for the agent. Defaults to a build matching the + agent's acceleration options (cuda, metal, otherwise CPU). + ''; + }; + + cuda = { + enable = lib.mkEnableOption '' + CUDA acceleration for the agent (NVIDIA GPUs). Builds paddler with the cuda + feature; the unfree license and cudaSupport are scoped to paddler's own build + and do not affect the rest of the system + ''; + + capabilities = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "12.0" ]; + description = '' + CUDA compute capabilities to build kernels for, matching the target GPU + (e.g. "8.9", "9.0", "12.0"). Required when cuda.enable is set: only the + listed architectures are compiled. + Building for the wrong or for many architectures is what makes the CUDA compile + enormous, so there is no default fallback. + ''; + }; + + buildParallelism = lib.mkOption { + type = lib.types.ints.positive; + default = 4; + description = '' + Maximum number of CUDA compiler (nvcc) and rustc jobs run in parallel while + building the agent. The CUDA kernels are memory-heavy to compile (~2-2.5GB + each), so this bounds peak build RAM independently of the build host's core + count. The default of 4 keeps the build under 16GB; lower it on a tighter box. + ''; + }; + }; + + metal = { + enable = lib.mkEnableOption '' + Metal acceleration for the agent (Apple GPUs). Only supported when building for + a Darwin host + ''; }; managementAddr = lib.mkOption { @@ -438,6 +509,23 @@ }; config = lib.mkMerge [ + (lib.mkIf cfg.agent.enable { + assertions = [ + { + assertion = !(cfg.agent.cuda.enable && cfg.agent.metal.enable); + message = "services.paddler.agent: cuda and metal acceleration cannot both be enabled."; + } + { + assertion = cfg.agent.cuda.enable -> cfg.agent.cuda.capabilities != [ ]; + message = "services.paddler.agent.cuda.capabilities must list the target GPU's compute capabilities (e.g. [ \"12.0\" ]) when cuda.enable is set."; + } + { + assertion = cfg.agent.metal.enable -> pkgs.stdenv.hostPlatform.isDarwin; + message = "services.paddler.agent.metal is only supported on Darwin hosts."; + } + ]; + }) + (lib.mkIf cfg.balancer.enable { systemd.services.paddler-balancer = { description = "Paddler balancer"; From b5ee22d8795fbe53f25e8cd4a3f1331ab5f1cc31 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 19 Jul 2026 21:35:15 +0200 Subject: [PATCH 5/5] trim NixOS module option descriptions to essentials --- flake.nix | 65 ++++++++++--------------------------------------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/flake.nix b/flake.nix index 347daef7..d6c2e4d0 100644 --- a/flake.nix +++ b/flake.nix @@ -342,30 +342,20 @@ managementAddr = lib.mkOption { type = socketAddrType; default = "127.0.0.1:8060"; - description = '' - Address of the management server. Agents connect here and the web admin - panel calls it directly from the browser, so if the panel is used remotely - this must be an address the browser can actually reach. - ''; + description = "Address of the management server."; }; inferenceAddr = lib.mkOption { type = socketAddrType; default = "127.0.0.1:8061"; - description = '' - Address of the inference server. The web admin panel calls it directly from - the browser, so if the panel is used remotely this must be browser-reachable. - ''; + description = "Address of the inference server."; }; webAdminPanelAddr = lib.mkOption { type = lib.types.nullOr socketAddrType; default = null; example = "127.0.0.1:8062"; - description = '' - Address of the web admin panel. When null the panel is disabled. Requires a - package built with the web admin panel feature (the default package). - ''; + description = "Address of the web admin panel. Disabled when null."; }; openaiCompatAddr = lib.mkOption { @@ -377,10 +367,7 @@ stateDatabase = lib.mkOption { type = lib.types.str; default = "file:///var/lib/paddler/state.db"; - description = '' - Balancer state database URL. Either memory:// or file:///absolute/path. - A file database persists the runtime model assignment across restarts. - ''; + description = "Balancer state database URL: memory:// or file:///absolute/path."; }; managementCorsAllowedHosts = lib.mkOption { @@ -409,59 +396,34 @@ }; agent = { - enable = lib.mkEnableOption '' - the Paddler agent service. Run a single agent per host: one agent already - saturates the host's inference hardware with its slots, so additional agents - on the same host would contend for the same GPU or CPU - ''; + enable = lib.mkEnableOption "the Paddler agent service"; package = lib.mkOption { type = lib.types.package; default = agentPackage; defaultText = lib.literalExpression "the paddler build selected by the agent's cuda/metal options"; - description = '' - The paddler package used for the agent. Defaults to a build matching the - agent's acceleration options (cuda, metal, otherwise CPU). - ''; + description = "The paddler package used for the agent."; }; cuda = { - enable = lib.mkEnableOption '' - CUDA acceleration for the agent (NVIDIA GPUs). Builds paddler with the cuda - feature; the unfree license and cudaSupport are scoped to paddler's own build - and do not affect the rest of the system - ''; + enable = lib.mkEnableOption "CUDA acceleration for the agent"; capabilities = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; example = [ "12.0" ]; - description = '' - CUDA compute capabilities to build kernels for, matching the target GPU - (e.g. "8.9", "9.0", "12.0"). Required when cuda.enable is set: only the - listed architectures are compiled. - Building for the wrong or for many architectures is what makes the CUDA compile - enormous, so there is no default fallback. - ''; + description = "CUDA compute capabilities to build kernels for. Required when cuda.enable is set."; }; buildParallelism = lib.mkOption { type = lib.types.ints.positive; default = 4; - description = '' - Maximum number of CUDA compiler (nvcc) and rustc jobs run in parallel while - building the agent. The CUDA kernels are memory-heavy to compile (~2-2.5GB - each), so this bounds peak build RAM independently of the build host's core - count. The default of 4 keeps the build under 16GB; lower it on a tighter box. - ''; + description = "Maximum number of parallel nvcc and rustc jobs while building the agent."; }; }; metal = { - enable = lib.mkEnableOption '' - Metal acceleration for the agent (Apple GPUs). Only supported when building for - a Darwin host - ''; + enable = lib.mkEnableOption "Metal acceleration for the agent (Darwin only)"; }; managementAddr = lib.mkOption { @@ -486,12 +448,7 @@ type = lib.types.nullOr lib.types.path; default = null; example = "/run/secrets/paddler-hf-token"; - description = '' - Path to a file containing a HuggingFace access token (the raw token on a - single line), used to download gated repositories. It is loaded as a - systemd credential and installed into the agent's HuggingFace cache - (HF_HOME/token) before the agent starts. - ''; + description = "Path to a file containing a HuggingFace access token for downloading gated repositories."; }; environment = lib.mkOption {