diff --git a/Cargo.lock b/Cargo.lock index d2faa372..f70c3122 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3728,9 +3728,9 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llama-cpp-bindings" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b3e674002002689fae8b895a8efdf1e851f46b657c32325473f375d0286b9b" +checksum = "dd2ca73bb0bda4720b8eceb8932605730f4abf7a91bdcb9c708eabb5708b94fd" dependencies = [ "encoding_rs", "enumflags2", @@ -3748,9 +3748,9 @@ dependencies = [ [[package]] name = "llama-cpp-bindings-build" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bef3ed70c4633858122ca28bdb683b28b6d036a5c2b9cdc6aab976f39a19441" +checksum = "b09a396d180c93d56342594f60b3fae78d2a517a9ed0be02d36f33c3db65e507" dependencies = [ "bindgen", "cc", @@ -3763,18 +3763,18 @@ dependencies = [ [[package]] name = "llama-cpp-bindings-sys" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf63ee8a9808ee4adf3ae5cc0da3105c3d91e53fcee8894f8254c5b5d03ab3e" +checksum = "24867ff3fe6a53c3aa2e5f47273abda1b3ffd1676fdd2785b544376609e3bdd0" dependencies = [ "llama-cpp-bindings-build", ] [[package]] name = "llama-cpp-bindings-types" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e764c657a21c313bee323869ccb0039f40f877128a6d9ecc7e2f23103a7a5d3" +checksum = "59bfea302da4827659a9eaadb587f2bbaf30b47294bf3331502bf0ca952c3b57" dependencies = [ "serde", "serde_json", @@ -3783,15 +3783,15 @@ dependencies = [ [[package]] name = "llama-cpp-error-recorder" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f454fc2ae5ca82d66bf1e22050fc25796ea1675536d1784ced70fc754458fc01" +checksum = "15b606bbd0c9ed0a82420441cab10fc9818fda77626daf488154102b4a4cf0ad" [[package]] name = "llama-cpp-log-decoder" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2508cccd00a43f0cdc0bf0904ecf7ac6ab0aa3f85b4729b1c5c97ca6ac89deb3" +checksum = "18699860211dd95bcf6752a89b637c764029be383e487112b6a64bd200609988" [[package]] name = "llguidance" diff --git a/Cargo.toml b/Cargo.toml index 47829b3a..2a1b8b53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,9 +57,9 @@ http = "1" image = "0.25" indoc = "2" jsonschema = { version = "0.37", default-features = false } -llama-cpp-bindings = "=0.11.0" -llama-cpp-bindings-sys = "=0.11.0" -llama-cpp-bindings-types = "=0.11.0" +llama-cpp-bindings = "=0.12.0" +llama-cpp-bindings-sys = "=0.12.0" +llama-cpp-bindings-types = "=0.12.0" base64 = "0.22" log = "0.4" mime_guess = "2" diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..cc14f0c9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,85 @@ +{ + "nodes": { + "crane": { + "locked": { + "lastModified": 1784407669, + "narHash": "sha256-gcFMcRjw0ZSn380Rx2QLlU1goUQeSrKX/DF12omI6+o=", + "owner": "ipetkov", + "repo": "crane", + "rev": "1316b7d278ad77a16aec024b71d971366e123bec", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1784356753, + "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "crane": "crane", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784438913, + "narHash": "sha256-NYF7ZM5ip0u+w1pBFDpIGEbrbgN/wpnLFAmBkWkYMXw=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "afacd6819d3765a05814ee8e3de74c77d42ac799", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..d6c2e4d0 --- /dev/null +++ b/flake.nix @@ -0,0 +1,560 @@ +{ + description = "Paddler — open-source LLMOps platform for hosting and scaling LLMs in your own infrastructure"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + crane.url = "github:ipetkov/crane"; + }; + + outputs = + inputs@{ + self, + nixpkgs, + flake-parts, + rust-overlay, + crane, + }: + let + version = "4.1.0"; + + paddlerPkgs = + { + system, + allowUnfree ? false, + cudaSupport ? false, + cudaCapabilities ? [ ], + }: + import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + config = { + inherit allowUnfree cudaSupport; + } + // (if cudaCapabilities == [ ] then { } else { inherit cudaCapabilities; }); + }; + + craneLibFor = pkgs: (crane.mkLib pkgs).overrideToolchain (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml); + + defaultAccelerator = + pkgs: + if pkgs.stdenv.hostPlatform.isDarwin then + "metal" + else if (pkgs.config.cudaSupport or false) then + "cuda" + else + "cpu"; + + buildPaddler = + pkgs: + { + accelerator ? defaultAccelerator pkgs, + webAdminPanel ? true, + cudaBuildParallelism ? 4, + }: + let + lib = pkgs.lib; + + craneLib = craneLibFor pkgs; + craneLibEff = + if accelerator == "cuda" then + craneLib.overrideScope ( + _final: _prev: { + stdenvSelector = eachPkgs: eachPkgs.cudaPackages.backendStdenv; + } + ) + else + craneLib; + + webAdminPanelAssets = pkgs.buildNpmPackage { + pname = "paddler-web-admin-panel"; + inherit version; + src = self; + npmDepsHash = "sha256-sBDdMf388qFQVIjQ3t/BL3KC/yAqF1qj47a/40axgF8="; + dontNpmBuild = true; + nativeBuildInputs = [ pkgs.nodejs ]; + buildPhase = '' + runHook preBuild + node jarmuz-static.mjs + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p "$out" + cp -r static "$out/static" + cp esbuild-meta.json "$out/esbuild-meta.json" + runHook postInstall + ''; + }; + + injectWebAdminPanelAssets = '' + cp -r --no-preserve=mode,ownership ${webAdminPanelAssets}/static ./static + cp --no-preserve=mode,ownership ${webAdminPanelAssets}/esbuild-meta.json ./esbuild-meta.json + ''; + + accel = + if accelerator == "cpu" then + { + cargoFeatures = [ ]; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + env = { }; + } + else if accelerator == "cuda" then + { + cargoFeatures = [ "cuda" ]; + nativeBuildInputs = [ + pkgs.cudaPackages.cuda_nvcc + pkgs.autoAddDriverRunpath + ]; + buildInputs = [ + pkgs.cudaPackages.cuda_cudart + pkgs.cudaPackages.libcublas + (lib.getOutput "static" pkgs.cudaPackages.libcublas) + pkgs.cudaPackages.cccl + ]; + env = { + CMAKE_CUDA_ARCHITECTURES = pkgs.cudaPackages.flags.cmakeCudaArchitecturesString; + CMAKE_BUILD_PARALLEL_LEVEL = toString cudaBuildParallelism; + CARGO_BUILD_JOBS = toString cudaBuildParallelism; + CARGO_BUILD_RUSTFLAGS = lib.concatStringsSep " " [ + "-L native=${pkgs.cudaPackages.cuda_cudart}/lib" + "-L native=${pkgs.cudaPackages.cuda_cudart}/lib/stubs" + "-L native=${lib.getOutput "static" pkgs.cudaPackages.libcublas}/lib" + ]; + }; + } + else if accelerator == "metal" then + { + cargoFeatures = [ "metal" ]; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + env = { }; + } + else + throw "paddler: unsupported accelerator '${accelerator}'"; + + features = accel.cargoFeatures ++ lib.optional webAdminPanel "web_admin_panel"; + featureFlags = lib.optionals (features != [ ]) [ + "--features" + (lib.concatStringsSep "," features) + ]; + cargoExtraArgs = lib.escapeShellArgs ( + [ + "-p" + "paddler_cli" + ] + ++ featureFlags + ); + + pname = "paddler${lib.optionalString (accelerator != "cpu") "-${accelerator}"}${ + lib.optionalString (!webAdminPanel) "-headless" + }"; + + commonArgs = { + inherit cargoExtraArgs version pname; + src = self; + strictDeps = true; + doCheck = false; + nativeBuildInputs = [ + pkgs.cmake + pkgs.pkg-config + pkgs.llvmPackages.clang + ] + ++ accel.nativeBuildInputs; + buildInputs = [ pkgs.openssl ] ++ accel.buildInputs; + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + } + // accel.env; + + cargoArtifacts = craneLibEff.buildDepsOnly ( + commonArgs // { src = craneLibEff.cleanCargoSource self; } + ); + in + craneLibEff.buildPackage ( + commonArgs + // { + inherit cargoArtifacts; + meta = { + description = "Paddler ${accelerator} build (web admin panel ${ + if webAdminPanel then "enabled" else "disabled" + })"; + homepage = "https://paddler.intentee.com/"; + license = lib.licenses.asl20; + mainProgram = "paddler"; + }; + } + // lib.optionalAttrs webAdminPanel { preBuild = injectWebAdminPanelAssets; } + ); + in + flake-parts.lib.mkFlake { inherit inputs; } { + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ]; + + perSystem = + { system, ... }: + let + pkgs = paddlerPkgs { inherit system; }; + lib = pkgs.lib; + + paddler = buildPaddler pkgs { }; + paddler-headless = buildPaddler pkgs { webAdminPanel = false; }; + in + { + packages = { + default = paddler; + inherit paddler paddler-headless; + }; + + apps.default = { + type = "app"; + program = "${lib.getExe paddler}"; + }; + + checks = { + inherit paddler paddler-headless; + }; + + devShells.default = (craneLibFor pkgs).devShell { + packages = [ + pkgs.nodejs + pkgs.cmake + pkgs.pkg-config + pkgs.llvmPackages.clang + pkgs.openssl + ]; + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + }; + + formatter = pkgs.nixfmt-rfc-style; + }; + + flake = + let + paddlerNixosModule = + { + config, + lib, + pkgs, + utils, + ... + }: + let + cfg = config.services.paddler; + + boxSystem = pkgs.stdenv.hostPlatform.system; + + balancerPackage = buildPaddler (paddlerPkgs { system = boxSystem; }) { }; + + agentPackage = + let + agentCuda = cfg.agent.cuda; + in + if agentCuda.enable then + buildPaddler + (paddlerPkgs { + system = boxSystem; + allowUnfree = true; + cudaSupport = true; + cudaCapabilities = agentCuda.capabilities; + }) + { + accelerator = "cuda"; + cudaBuildParallelism = agentCuda.buildParallelism; + } + else if cfg.agent.metal.enable then + buildPaddler (paddlerPkgs { system = boxSystem; }) { accelerator = "metal"; } + else + buildPaddler (paddlerPkgs { system = boxSystem; }) { accelerator = "cpu"; }; + + socketAddrType = lib.types.str; + + balancerArgs = + let + balancer = cfg.balancer; + in + [ + "balancer" + "--management-addr" + balancer.managementAddr + "--inference-addr" + balancer.inferenceAddr + "--state-database" + balancer.stateDatabase + ] + ++ lib.optionals (balancer.webAdminPanelAddr != null) [ + "--web-admin-panel-addr" + balancer.webAdminPanelAddr + ] + ++ lib.optionals (balancer.openaiCompatAddr != null) [ + "--compat-openai-addr" + balancer.openaiCompatAddr + ] + ++ lib.concatMap (host: [ + "--management-cors-allowed-host" + host + ]) balancer.managementCorsAllowedHosts + ++ lib.concatMap (host: [ + "--inference-cors-allowed-host" + host + ]) balancer.inferenceCorsAllowedHosts + ++ balancer.extraArgs; + + agentArgs = + let + agent = cfg.agent; + in + [ + "agent" + "--management-addr" + agent.managementAddr + "--slots" + (toString agent.slots) + ] + ++ lib.optionals (agent.name != null) [ + "--name" + agent.name + ] + ++ agent.extraArgs; + in + { + options.services.paddler = { + balancer = { + enable = lib.mkEnableOption "the Paddler balancer service"; + + package = lib.mkOption { + type = lib.types.package; + default = balancerPackage; + defaultText = lib.literalExpression "the CPU paddler build for this host"; + description = "The paddler package used for the balancer."; + }; + + managementAddr = lib.mkOption { + type = socketAddrType; + default = "127.0.0.1:8060"; + description = "Address of the management server."; + }; + + inferenceAddr = lib.mkOption { + type = socketAddrType; + default = "127.0.0.1:8061"; + description = "Address of the inference server."; + }; + + webAdminPanelAddr = lib.mkOption { + type = lib.types.nullOr socketAddrType; + default = null; + example = "127.0.0.1:8062"; + description = "Address of the web admin panel. Disabled when null."; + }; + + openaiCompatAddr = lib.mkOption { + type = lib.types.nullOr socketAddrType; + default = null; + description = "Address of the OpenAI-compatible API server. When null it is disabled."; + }; + + stateDatabase = lib.mkOption { + type = lib.types.str; + default = "file:///var/lib/paddler/state.db"; + description = "Balancer state database URL: memory:// or file:///absolute/path."; + }; + + managementCorsAllowedHosts = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Allowed CORS hosts for the management service."; + }; + + inferenceCorsAllowedHosts = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Allowed CORS hosts for the inference service."; + }; + + extraArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra command-line arguments passed to the balancer."; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open the management, inference, web admin panel and OpenAI-compatible ports in the firewall."; + }; + }; + + agent = { + enable = lib.mkEnableOption "the Paddler agent service"; + + package = lib.mkOption { + type = lib.types.package; + default = agentPackage; + defaultText = lib.literalExpression "the paddler build selected by the agent's cuda/metal options"; + description = "The paddler package used for the agent."; + }; + + cuda = { + enable = lib.mkEnableOption "CUDA acceleration for the agent"; + + capabilities = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "12.0" ]; + description = "CUDA compute capabilities to build kernels for. Required when cuda.enable is set."; + }; + + buildParallelism = lib.mkOption { + type = lib.types.ints.positive; + default = 4; + description = "Maximum number of parallel nvcc and rustc jobs while building the agent."; + }; + }; + + metal = { + enable = lib.mkEnableOption "Metal acceleration for the agent (Darwin only)"; + }; + + managementAddr = lib.mkOption { + type = socketAddrType; + example = "127.0.0.1:8060"; + description = "Management address of the balancer to connect to."; + }; + + slots = lib.mkOption { + type = lib.types.ints.positive; + example = 4; + description = "Number of parallel requests this agent can handle at once."; + }; + + name = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Human-readable name reported to the balancer."; + }; + + hfTokenFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + example = "/run/secrets/paddler-hf-token"; + description = "Path to a file containing a HuggingFace access token for downloading gated repositories."; + }; + + environment = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + description = "Extra environment variables for the agent process."; + }; + + extraArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra command-line arguments passed to the agent."; + }; + }; + }; + + config = lib.mkMerge [ + (lib.mkIf cfg.agent.enable { + assertions = [ + { + assertion = !(cfg.agent.cuda.enable && cfg.agent.metal.enable); + message = "services.paddler.agent: cuda and metal acceleration cannot both be enabled."; + } + { + assertion = cfg.agent.cuda.enable -> cfg.agent.cuda.capabilities != [ ]; + message = "services.paddler.agent.cuda.capabilities must list the target GPU's compute capabilities (e.g. [ \"12.0\" ]) when cuda.enable is set."; + } + { + assertion = cfg.agent.metal.enable -> pkgs.stdenv.hostPlatform.isDarwin; + message = "services.paddler.agent.metal is only supported on Darwin hosts."; + } + ]; + }) + + (lib.mkIf cfg.balancer.enable { + systemd.services.paddler-balancer = { + description = "Paddler balancer"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = utils.escapeSystemdExecArgs ([ (lib.getExe cfg.balancer.package) ] ++ balancerArgs); + DynamicUser = true; + StateDirectory = "paddler"; + Restart = "on-failure"; + RestartSec = 5; + ProtectSystem = "strict"; + ProtectHome = true; + NoNewPrivileges = true; + PrivateTmp = true; + }; + }; + + networking.firewall = lib.mkIf cfg.balancer.openFirewall { + allowedTCPPorts = + let + portOf = addr: lib.toInt (lib.last (lib.splitString ":" addr)); + in + [ + (portOf cfg.balancer.managementAddr) + (portOf cfg.balancer.inferenceAddr) + ] + ++ lib.optional (cfg.balancer.webAdminPanelAddr != null) (portOf cfg.balancer.webAdminPanelAddr) + ++ lib.optional (cfg.balancer.openaiCompatAddr != null) (portOf cfg.balancer.openaiCompatAddr); + }; + }) + + (lib.mkIf cfg.agent.enable { + systemd.services.paddler-agent = { + description = "Paddler agent"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + environment = { + PADDLER_CACHE_DIR = "/var/cache/paddler"; + HF_HOME = "/var/cache/paddler/huggingface"; + } + // cfg.agent.environment; + serviceConfig = { + ExecStart = utils.escapeSystemdExecArgs ([ (lib.getExe cfg.agent.package) ] ++ agentArgs); + DynamicUser = true; + CacheDirectory = "paddler"; + Restart = "on-failure"; + RestartSec = 5; + ProtectSystem = "strict"; + ProtectHome = true; + NoNewPrivileges = true; + PrivateTmp = true; + } + // lib.optionalAttrs (cfg.agent.hfTokenFile != null) { + LoadCredential = [ "hf-token:${toString cfg.agent.hfTokenFile}" ]; + ExecStartPre = "${lib.getExe' pkgs.coreutils "install"} -D -m0600 %d/hf-token /var/cache/paddler/huggingface/token"; + }; + }; + }) + ]; + }; + in + { + nixosModules.default = paddlerNixosModule; + nixosModules.paddler = paddlerNixosModule; + + overlays.default = final: _prev: { + paddler = self.packages.${final.stdenv.hostPlatform.system}.paddler; + paddler-headless = self.packages.${final.stdenv.hostPlatform.system}.paddler-headless; + }; + }; + }; +}