diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index 351743ec7..67959137b 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -289,3 +289,10 @@ gcc [Tt]eardown repo buildbox +subgraph +genrule +hardcoded +shellexpand +pluggable +[Cc]allout +hostnames diff --git a/.vale.ini b/.vale.ini index d6a0112ca..2a5ab2484 100644 --- a/.vale.ini +++ b/.vale.ini @@ -27,6 +27,10 @@ TokenIgnores = ['"]\.\/[\w-]+\.mdx['"], (?<=\])\([^)]+\) # Ignore filenames and directory names in tree visualizations. BlockIgnores = (?s)(.*?) +# Mermaid diagram source (node ids, `subgraph`/`end` keywords) isn't prose — +# scanning it produces spurious Vale.Repetition hits on short node labels. +BlockIgnores = (?s)(.*?) + # Too harsh. The `write-good.Passive` check already covers many cases. write-good.E-Prime = NO diff --git a/nativelink-config/examples/local_rbe_self_test.json5 b/nativelink-config/examples/local_rbe_self_test.json5 new file mode 100644 index 000000000..3e3149bc6 --- /dev/null +++ b/nativelink-config/examples/local_rbe_self_test.json5 @@ -0,0 +1,172 @@ +// A complete NativeLink cluster — CAS, AC, scheduler, and one worker — in a +// single process on localhost. Everything a Bazel client needs (cache AND +// execution) is on one port; nothing here talks to any external service. +// +// Use this to prove your Bazel setup can do remote caching and remote +// execution before you ever point it at a real cluster. See +// docs/rbe/local-remote-execution for the full walkthrough. +// +// Run it: +// nativelink ./nativelink-config/examples/local_rbe_self_test.json5 +// +// Then, from a Bazel workspace: +// bazel test --remote_cache=grpc://127.0.0.1:50051 \ +// --remote_executor=grpc://127.0.0.1:50051 \ +// --remote_default_exec_properties=cpu_count=1 \ +// //some:target +{ + stores: [ + { + name: "CAS_MAIN_STORE", + filesystem: { + content_path: "/tmp/nativelink-local-rbe-test/data/content_path-cas", + temp_path: "/tmp/nativelink-local-rbe-test/data/tmp_path-cas", + eviction_policy: { + max_bytes: 2000000000, + }, + }, + }, + { + name: "AC_MAIN_STORE", + filesystem: { + content_path: "/tmp/nativelink-local-rbe-test/data/content_path-ac", + temp_path: "/tmp/nativelink-local-rbe-test/data/tmp_path-ac", + eviction_policy: { + max_bytes: 200000000, + }, + }, + }, + { + name: "WORKER_FAST_SLOW_STORE", + fast_slow: { + // "fast" must be a filesystem store — the worker hardlinks out of it + // to build each action's sandbox. + fast: { + filesystem: { + content_path: "/tmp/nativelink-local-rbe-test/data/content_path-worker", + temp_path: "/tmp/nativelink-local-rbe-test/data/tmp_path-worker", + eviction_policy: { + max_bytes: 2000000000, + }, + }, + }, + + // The worker and the client-facing CAS share one store instance, so + // whatever the worker produces is immediately visible to Bazel. + slow: { + ref_store: { + name: "CAS_MAIN_STORE", + }, + }, + }, + }, + ], + schedulers: [ + { + name: "MAIN_SCHEDULER", + simple: { + supported_platform_properties: { + cpu_count: "minimum", + OSFamily: "priority", + "container-image": "priority", + }, + }, + }, + ], + workers: [ + { + local: { + worker_api_endpoint: { + uri: "grpc://127.0.0.1:50061", + }, + cas_fast_slow_store: "WORKER_FAST_SLOW_STORE", + upload_action_result: { + ac_store: "AC_MAIN_STORE", + }, + work_directory: "/tmp/nativelink-local-rbe-test/work", + platform_properties: { + cpu_count: { + values: [ + "1", + ], + }, + OSFamily: { + values: [ + "", + ], + }, + "container-image": { + values: [ + "", + ], + }, + }, + + // use_namespaces / use_mount_namespace default to false, which is + // required here — they're Linux-only and this config is meant to + // also run on macOS. + }, + }, + ], + servers: [ + { + // Everything a Bazel client needs — cache AND execution — on one + // local endpoint. No separate/remote cache service involved. + name: "local", + listener: { + http: { + socket_address: "0.0.0.0:50051", + }, + }, + services: { + cas: [ + { + cas_store: "CAS_MAIN_STORE", + }, + ], + ac: [ + { + ac_store: "AC_MAIN_STORE", + }, + ], + bytestream: [ + { + cas_store: "CAS_MAIN_STORE", + }, + ], + execution: [ + { + cas_store: "CAS_MAIN_STORE", + scheduler: "MAIN_SCHEDULER", + }, + ], + capabilities: [ + { + remote_execution: { + scheduler: "MAIN_SCHEDULER", + }, + }, + ], + }, + }, + { + // Workers talk to this port, clients never do — keep it off the + // client-facing listener (see nativelink-config/examples/README.md). + name: "worker_api", + listener: { + http: { + socket_address: "0.0.0.0:50061", + }, + }, + services: { + worker_api: { + scheduler: "MAIN_SCHEDULER", + }, + health: {}, + }, + }, + ], + global: { + max_open_files: 24576, + }, +} diff --git a/nativelink-config/examples/stores-config.json5 b/nativelink-config/examples/stores-config.json5 index 4c6a1b7d3..0748ce406 100644 --- a/nativelink-config/examples/stores-config.json5 +++ b/nativelink-config/examples/stores-config.json5 @@ -85,6 +85,38 @@ }, { name: "6", + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + }, + { + name: "7", + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + }, + { + name: "8", "ontap_s3_existence_cache": { "index_path": "/path/to/cache/index.json", "sync_interval_seconds": 300, @@ -97,7 +129,7 @@ } }, { - name: "7", + name: "9", "verify": { "backend": { "memory": { @@ -111,7 +143,7 @@ } }, { - name: "8", + name: "10", "completeness_checking": { "backend": { "filesystem": { @@ -130,7 +162,7 @@ } }, { - name: "9", + name: "11", "compression": { "compression_algorithm": { "lz4": {} @@ -147,7 +179,7 @@ } }, { - name: "10", + name: "12", "dedup": { "index_store": { "memory": { @@ -186,7 +218,7 @@ } }, { - name: "11", + name: "13", "existence_cache": { "backend": { "memory": { @@ -202,7 +234,7 @@ } }, { - name: "12", + name: "14", "fast_slow": { "fast": { "filesystem": { @@ -225,7 +257,7 @@ } }, { - name: "13", + name: "15", "shard": { "stores": [ { @@ -241,7 +273,7 @@ } }, { - name: "14", + name: "16", "filesystem": { "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", @@ -251,13 +283,13 @@ } }, { - name: "15", + name: "17", "ref_store": { "name": "FS_CONTENT_STORE" } }, { - name: "16", + name: "18", "size_partitioning": { "size": "128mib", "lower_store": { @@ -274,7 +306,7 @@ } }, { - name: "17", + name: "19", "grpc": { "instance_name": "main", "endpoints": [ @@ -295,7 +327,7 @@ } }, { - name: "18", + name: "20", "redis_store": { "addresses": [ "redis://127.0.0.1:6379/", @@ -304,11 +336,11 @@ } }, { - name: "19", + name: "21", "noop": {} }, { - name: "20", + name: "22", "experimental_mongo": { "connection_string": "mongodb://localhost:27017", "database": "nativelink", diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index 9cc0aff1b..e67db1bc6 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -181,6 +181,55 @@ pub enum StoreSpec { /// "multipart_max_concurrent_uploads": 10 /// } /// ``` + /// + /// 5. **Cloudflare R2:** + /// R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + /// S3 API, so this is a thin wrapper that derives the account-scoped + /// endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + /// + /// **Example JSON Config:** + /// ```json + /// "experimental_cloud_object_store": { + /// "provider": "r2", + /// "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + /// "bucket": "nativelink-cas", + /// "key_prefix": "test-prefix/", + /// "retry": { + /// "max_retries": 6, + /// "delay": 0.3, + /// "jitter": 0.5 + /// }, + /// "multipart_max_concurrent_uploads": 10 + /// } + /// ``` + /// + /// 6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + /// OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + /// Storage API. The path-style endpoint is derived from your Object + /// Storage `namespace` and `region` as + /// `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + /// Authenticate with a Customer Secret Key (Access Key/Secret Key pair + /// created under User Settings -> Customer secret keys in the OCI + /// console); the secret cannot be retrieved after generation, so read + /// it from an env var via shellexpand. + /// + /// **Example JSON Config:** + /// ```json + /// "experimental_cloud_object_store": { + /// "provider": "oci", + /// "namespace": "your-object-storage-namespace", + /// "region": "us-phoenix-1", + /// "bucket": "nativelink-cas", + /// "access_key_id": "oci_access_key_id", + /// "secret_access_key": "oci_secret_access_key", + /// "key_prefix": "test-prefix/", + /// "retry": { + /// "max_retries": 6, + /// "delay": 0.3, + /// "jitter": 0.5 + /// } + /// } + /// ``` ExperimentalCloudObjectStore(ExperimentalCloudObjectSpec), /// ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store diff --git a/web/apps/docs/content/docs/explanations/lre.mdx b/web/apps/docs/content/docs/explanations/lre.mdx index 295868e8f..ff7ad1d21 100644 --- a/web/apps/docs/content/docs/explanations/lre.mdx +++ b/web/apps/docs/content/docs/explanations/lre.mdx @@ -66,39 +66,63 @@ The recommended flow: is the easiest path.
  • - **Pull the NativeLink LRE flake template.** + **Pull the NativeLink Bazel flake template.** ```bash - nix flake init -t github:TraceMachina/nativelink#lre + nix flake init -t github:TraceMachina/nativelink#bazel ```
  • - **Enter the dev shell.** This downloads the pinned toolchain on - first run. + **Enter the dev shell.** This downloads the Nix-pinned toolchain on + first run and generates `lre.bazelrc`, which the template's + `.bazelrc` already `try-import`s. ```bash nix develop ```
  • - **Start the local worker.** The template ships with a - `nativelink-lre.json5` that binds CAS, AC, scheduler, and a single - worker to `localhost:50051`. + **Point `user.bazelrc` at a NativeLink cache and executor.** The + template ships it with placeholders: ```bash - nativelink ./nativelink-lre.json5 + build --remote_cache=grpcs://TODO + build --bes_backend=grpcs://TODO + build --remote_timeout=600 + build --remote_executor=grpcs://TODO ``` + + Fill these in with either your [dev.nativelink.com](https://dev.nativelink.com) + credentials or a self-hosted cluster with a worker capable of the + platform the example needs. A plain + [local instance](/rbe/local-testing) (drop the `s` in `grpcs://`, + point at `127.0.0.1`) is enough to validate `remote_cache`, but + the C++ example's `lre-cc` platform still needs a real worker + running the matching container image — see + [Local Remote Execution](/rbe/local-remote-execution) for the + exact commands and what each one actually requires.
  • - **Point your build system at it.** The flake includes a - `.bazelrc.lre` that's pre-configured; for non-Bazel clients see - [Getting Started → Other build systems](/getting-started/other-build-systems). + **Build the example.** + + ```bash + bazel build hello-world + ```
    • The first build will be the same wall-time as a normal local build. The second one will be near-instant — that's the cache doing its job. + + The template above is the fastest way to see LRE working, but it + starts a new project. To add the same Nix-pinned toolchains to a + project you already have, see the full flake-side and Bazel-side + wiring — the `nativelink.flakeModule` import, `lre.installationScript`, + and the `local-remote-execution` Bazel module override — in + [`local-remote-execution/README.md`](https://github.com/TraceMachina/nativelink/blob/main/local-remote-execution/README.md). + + ## Why Nix, specifically? LRE needs every input to a build action to be hashable. The compiler, @@ -116,9 +140,15 @@ For background, see [What is Nix?](/faq/nix) and ## What's next +- [RBE → Local Remote Execution](/rbe/local-remote-execution) — the + Setup steps above, run for real, plus what's actually + local-vs-remote once you leave `x86_64-linux`. +- [Local cache and executor](/rbe/local-testing) — a cache and + executor to point the setup above at, no Nix required for the + server side. - [Architecture](/explanations/architecture) — the full RE-API picture LRE plugs into. - [Configuration → Basic](/configuration/basic) — the JSON5 the local worker reads. -- [RBE → Nix templates](/rbe/nix-templates) — runnable LRE - reference setups. +- [RBE → Nix templates](/rbe/nix-templates) — the `bazel` flake + template this page walks through. diff --git a/web/apps/docs/content/docs/rbe/local-remote-execution.mdx b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx new file mode 100644 index 000000000..ccc194eec --- /dev/null +++ b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx @@ -0,0 +1,190 @@ +--- +title: Local Remote Execution +description: The Nix-pinned toolchain workflow behind LRE, walked through and tested end to end — including exactly what runs fully local and what still needs a real worker. +--- + +[Explanations → LRE](/explanations/lre) covers what Local Remote +Execution is and why it exists. This page is the hands-on version: +the exact commands, run for real, plus two gotchas the flake alone +doesn't mention. + + + `@local-remote-execution` ships **two** toolchain families, and they + behave differently once you leave `x86_64-linux`: + + - **Rust** (`@local-remote-execution//rust/...`) has a real, natively + built toolchain for `aarch64-darwin`, `aarch64-linux`, + `x86_64-darwin`, and `x86_64-linux`. Actions using it run fully + local — no network, no container. + - **C++** (`@local-remote-execution//generated-cc/...`) — what the + `bazel` template's own example uses — is a single Linux container + config (`ghcr.io/tracemachina/nativelink-worker-lre-cc:...`). + On any host, actions using it dispatch to a real worker running + that container. On macOS this means a network round-trip even + though the toolchain is Nix-pinned — LRE gives you hermeticity + here, not offline-ness. + + Below is what actually happened running the C++ example on + aarch64-darwin. If your project is Rust, swap the target and you get + the fully-local experience the intro promises. + + +## 1. Install Nix + +Flakes enabled. The +[next-gen installer](https://github.com/NixOS/experimental-nix-installer) +is the easiest path. + +## 2. Pull the template + +```bash +mkdir my-lre-test && cd my-lre-test +nix flake init -t github:TraceMachina/nativelink#bazel +``` + +This writes a `flake.nix` that imports `nativelink.flakeModules.lre` +and pins `lre = { inherit (pkgs.lre.lre-cc.meta) Env; };`, plus the +`hello-world.cpp` example, its `BUILD.bazel`, and `platforms/BUILD.bazel`. + +## 3. Init git — this step is not optional + +```bash +git init && git add -A +``` + + + The flake module's install script checks for a `.git` directory + before generating anything. Running `nix develop` in a plain + (non-git) folder prints this and moves on without writing + `lre.bazelrc` — no error, no toolchain: + + ```text + WARNING: lre: .git not found; skipping installation. + ``` + + +## 4. Enter the dev shell + +```bash +nix develop +``` + +This fetches the Nix-pinned Clang and writes `lre.bazelrc`. On +aarch64-darwin, the generated file looked like this (paths and hashes +will differ on your machine): + +```bash +# These flags are dynamically generated by the lre flake module. +# +# PATH=/nix/store/.../llvm-binutils-wrapper-20.1.1/bin:/nix/store/.../customClang/bin:... +# CC=/nix/store/.../customClang/bin/customClang + +# Bazel-side configuration for LRE. +build --define=EXECUTOR=remote +build --extra_execution_platforms=@local-remote-execution//rust/platforms:aarch64-apple-darwin +build --extra_toolchains=@local-remote-execution//rust:rust-aarch64-darwin +build --extra_toolchains=@local-remote-execution//rust:rustfmt-aarch64-darwin +build --platforms=@local-remote-execution//rust/platforms:aarch64-apple-darwin +``` + +Notice this registers the **Rust** platform even though the flake +requested the `lre-cc` `Env` — there's no local aarch64-darwin config +under `generated-cc`, so the module falls back to the toolchain that +does have one. The template's `.bazelrc` separately adds +`--extra_execution_platforms=@//platforms:lre-cc` for the C++ example, +which is the container-based platform from the callout above. + +## 5. Point `user.bazelrc` at a real cache and executor + +The template's own generator writes `user.bazelrc` with the literal +word `TODO` as every value — this isn't a placeholder left in this +page, it's what `nix flake init` puts on disk for you to edit: + +```bash +build --remote_cache=grpcs://TODO +build --bes_backend=grpcs://TODO +build --remote_timeout=600 +build --remote_executor=grpcs://TODO +``` + +Replace all three `TODO`s with either your +[dev.nativelink.com](https://dev.nativelink.com) credentials or your +own cluster's endpoint. Concretely, filled in for a plain local +cluster (no TLS, so `grpc://` instead of `grpcs://`, and no BES +endpoint to point at) — this is the exact `user.bazelrc` used to +produce the verified output in step 4 above: + +```bash +build --remote_cache=grpc://127.0.0.1:50051 +build --remote_timeout=600 +build --remote_executor=grpc://127.0.0.1:50051 +``` + +A [dev.nativelink.com](https://dev.nativelink.com) or self-hosted +cluster looks the same shape, just with `grpcs://` and your cluster's +real hostnames in place of `127.0.0.1:50051`, plus a `bes_backend` +line if you want build results streamed there. + +If you build before editing `user.bazelrc` at all, Bazel fails fast on +the literal hostname `TODO` — here's the exact error that produces, +captured from a real run, so you recognize it as "I forgot to edit +`user.bazelrc`" rather than something else being wrong: + +```text +[8 / 10] Compiling src/hello-world.cpp; 0s remote, remote-cache +ERROR: BUILD.bazel:3:10: Compiling src/hello-world.cpp failed: + Failed to query remote execution capabilities: UNAVAILABLE: Unable to resolve host TODO +``` + +Everything before that DNS failure worked correctly — the Nix +toolchain, the Bazel platform wiring, and the `lre-cc` execution +platform selection (`0s remote, remote-cache` means Bazel committed to +executing it remotely). Once real values replace `TODO`, this step +just works. + +## 6. Build + +```bash +bazel build hello-world +``` + +With a real endpoint in `user.bazelrc` and a worker running the +`nativelink-worker-lre-cc` image available to it, this compiles +`src/hello-world.cpp` on that worker and returns the binary through +the same CAS your local Bazel reads from. Re-run and it's a cache hit. + + + You don't need a full cluster to see the cache side of this working. + [Local cache and executor](/rbe/local-testing) starts a plain + NativeLink instance on `localhost` — enough to validate + `remote_cache` against real credentials before you also need a + worker that matches `lre-cc`'s platform requirements. + + +## Troubleshooting + + + If cache hits between local and remote execution stop happening, + compare `lre.bazelrc`'s paths against the toolchain configs in the + `@local-remote-execution` module at the commit pinned in your + `MODULE.bazel` / `flake.nix`. A version drift between the two is the + most common cause. + + + + The template above bootstraps a new project. For the flake-parts + module wiring, `bazel_dep` override, and `try-import` setup to add + the same toolchains to a project you already have, see + [`local-remote-execution/README.md`](https://github.com/TraceMachina/nativelink/blob/main/local-remote-execution/README.md) + — it also covers verifying the setup against a Kubernetes cluster. + + +## What's next + +- [Explanations → LRE](/explanations/lre) — the conceptual write-up. +- [Local cache and executor](/rbe/local-testing) — validate the + protocol side without Nix or a container-capable worker. +- [Nix templates](/rbe/nix-templates) — what else ships in the + `bazel` flake template. +- [Deployment → Kubernetes](/deployment/kubernetes) — running a real + worker fleet, including `lre-cc` workers, in production. diff --git a/web/apps/docs/content/docs/rbe/local-testing.mdx b/web/apps/docs/content/docs/rbe/local-testing.mdx new file mode 100644 index 000000000..28e4dccf9 --- /dev/null +++ b/web/apps/docs/content/docs/rbe/local-testing.mdx @@ -0,0 +1,204 @@ +--- +title: Local cache and executor +description: Run a full NativeLink cache and executor on your own machine and prove Bazel's remote caching and remote execution work — no Nix, no shared cluster. +--- + +Before pointing a build at a shared cluster — or before setting up +[Local Remote Execution](/rbe/local-remote-execution) — prove the +client side works against something you fully control: a NativeLink +cache and executor running on your own machine, using whatever +toolchain is already on your `PATH`. Nothing here talks to any +external or shared service, and nothing here requires Nix. + + + [Local Remote Execution](/rbe/local-remote-execution) is a specific + NativeLink feature: Nix-pinned toolchains that make local and remote + actions hash-identical. This page is narrower and has no toolchain + opinion at all — it's the fastest way to confirm a Bazel client can + talk to a real NativeLink `Execute` and CAS implementation before + you add Nix into the mix. + + +## What you get + +One `nativelink` process, one port for the client, a private port for +the worker: + + +{`flowchart LR + subgraph client[Your machine] + B[Bazel] + subgraph nl["nativelink process"] + direction LR + L["0.0.0.0:50051\\ncas · ac · bytestream\\nexecution · capabilities"] + W["0.0.0.0:50061\\nworker_api (private)"] + WK[local worker] + end + end + B -->|remote_cache\\n+ remote_executor| L + L --- W + W --- WK`} + + +Bazel's `--remote_cache` and `--remote_executor` both point at +`127.0.0.1:50051`. `50061` only exists so the worker can register with +the scheduler — Bazel never talks to it. + +## 1. Build `nativelink` + +```bash +bazel build //:nativelink +# or: cargo build --bin nativelink +``` + +## 2. Grab the config + +[`nativelink-config/examples/local_rbe_self_test.json5`](https://github.com/TraceMachina/nativelink/blob/main/nativelink-config/examples/local_rbe_self_test.json5) +is a complete, tested cluster — CAS, AC, a `simple` scheduler, and one +`local` worker, entirely on `localhost`: + +```json5 +{ + stores: [ + { name: "CAS_MAIN_STORE", filesystem: { /* ... */ } }, + { name: "AC_MAIN_STORE", filesystem: { /* ... */ } }, + { + // The worker's fast tier and the client-facing CAS share one store, + // so anything the worker produces is immediately visible to Bazel. + name: "WORKER_FAST_SLOW_STORE", + fast_slow: { + fast: { filesystem: { /* ... */ } }, + slow: { ref_store: { name: "CAS_MAIN_STORE" } }, + }, + }, + ], + schedulers: [{ + name: "MAIN_SCHEDULER", + simple: { supported_platform_properties: { cpu_count: "minimum" /* ... */ } }, + }], + workers: [{ + local: { + worker_api_endpoint: { uri: "grpc://127.0.0.1:50061" }, + cas_fast_slow_store: "WORKER_FAST_SLOW_STORE", + upload_action_result: { ac_store: "AC_MAIN_STORE" }, + platform_properties: { cpu_count: { values: ["1"] } /* ... */ }, + }, + }], + servers: [ + { + // Cache AND execution on one client-facing port. + name: "local", + listener: { http: { socket_address: "0.0.0.0:50051" } }, + services: { + cas: [{ cas_store: "CAS_MAIN_STORE" }], + ac: [{ ac_store: "AC_MAIN_STORE" }], + bytestream: [{ cas_store: "CAS_MAIN_STORE" }], + execution: [{ cas_store: "CAS_MAIN_STORE", scheduler: "MAIN_SCHEDULER" }], + capabilities: [{ remote_execution: { scheduler: "MAIN_SCHEDULER" } }], + }, + }, + { + // Private — only the worker connects here. + name: "worker_api", + listener: { http: { socket_address: "0.0.0.0:50061" } }, + services: { worker_api: { scheduler: "MAIN_SCHEDULER" }, health: {} }, + }, + ], +} +``` + + + Every service above omits `instance_name`, so it defaults to `""` — + the same default Bazel uses when you don't pass + `--remote_instance_name`. If you add an `instance_name` on the server + side, pass the matching `--remote_instance_name` flag, or every + request will fail with `'instance_name' not configured for ''`. + + +## 3. Start it + +```bash +nativelink ./nativelink-config/examples/local_rbe_self_test.json5 +``` + +```text +INFO nativelink: Ready, listening on 0.0.0.0:50051 +INFO nativelink: Ready, listening on 0.0.0.0:50061 +INFO nativelink_worker::local_worker: Worker registered with scheduler, worker_id: 1f175824-... +``` + +## 4. Point Bazel at it + +```bash +bazel test \ + --remote_cache=grpc://127.0.0.1:50051 \ + --remote_executor=grpc://127.0.0.1:50051 \ + --remote_default_exec_properties=cpu_count=1 \ + //your:target +``` + + + It's tempting to assume `--remote_executor` alone is enough — the + executor has to read/write CAS anyway. It isn't: without an explicit + `--remote_cache`, Bazel calls `Execute` with an action digest it + never uploaded, and the server correctly rejects it — + `FAILED_PRECONDITION: Action ... is missing from CAS`. Set both + flags. Pointing both at the same local address is exactly the point: + one machine, one cache, no separate remote service. + + +This repo tests itself exactly this way — see `build:self_test` and +`build:self_execute` in +[`.bazelrc`](https://github.com/TraceMachina/nativelink/blob/main/.bazelrc), +combined as `bazel test --config=self_test --config=self_execute`. +Copy that pattern into your own `.bazelrc` once the raw flags work. + +## 5. Verify + +First run — nothing cached yet, both actions run remote: + +```text +[4 / 5] 1 / 1 tests; Testing //:dummy_test; 0s remote, remote-cache +INFO: 5 processes: 3 internal, 3 remote. +//:dummy_test PASSED in 0.4s +``` + +`bazel clean`, run again — the genrule is a cache hit; the test +re-executes remotely because `--nocache_test_results` was set: + +```text +[7 / 8] Testing //:dummy_test; 0s remote +INFO: 8 processes: 2 remote cache hit, 6 internal, 1 remote. +//:dummy_test PASSED in 0.5s +``` + +Both runs were captured against the exact config above, on macOS +(arm64) — no Linux, no containers, no Nix. + +## Troubleshooting + + + The scheduler's `supported_platform_properties` must be satisfiable + by at least one worker's `platform_properties`, or matching actions + sit in the queue indefinitely with no error. Start with just + `cpu_count: "minimum"` on the scheduler and `cpu_count: { values: ["1"] }` + (or higher) on the worker — that alone unblocks most local setups. + + + + The worker's `use_namespaces` / `use_mount_namespace` options + sandbox actions with Linux namespaces and default to `false`. Leave + them unset on macOS — setting either to `true` on an unsupported + platform makes the worker exit immediately. + + +## What's next + +- [Local Remote Execution](/rbe/local-remote-execution) — add + Nix-pinned, hermetic toolchains on top of this same idea. +- [Classic RBE examples](/rbe/examples) — the same three patterns + (cache-only, full RE, hybrid) once you're pointed at a shared cluster. +- [Nix templates](/rbe/nix-templates) — the `bazel` flake template, + for wiring a whole new project instead of the existing NativeLink repo. +- [Configuration → Introduction](/configuration/intro) — what every + field in the config above actually does. diff --git a/web/apps/docs/content/docs/rbe/meta.json b/web/apps/docs/content/docs/rbe/meta.json index 7b27cf714..0a9b612e5 100644 --- a/web/apps/docs/content/docs/rbe/meta.json +++ b/web/apps/docs/content/docs/rbe/meta.json @@ -1,5 +1,7 @@ { "pages": [ + "local-testing", + "local-remote-execution", "examples", "nix-templates" ], diff --git a/web/apps/docs/content/docs/rbe/nix-templates.mdx b/web/apps/docs/content/docs/rbe/nix-templates.mdx index 6235bd9f0..0e2dcf8e5 100644 --- a/web/apps/docs/content/docs/rbe/nix-templates.mdx +++ b/web/apps/docs/content/docs/rbe/nix-templates.mdx @@ -1,12 +1,12 @@ --- title: Nix templates -description: Reproducible NativeLink testbeds in a single flake init command. +description: The Nix flake template NativeLink ships — Local Remote Execution with Bazel in one flake init command. --- -NativeLink ships Nix flake templates that spin up a complete -remote-execution environment in one command. Useful for -experimenting with configuration changes, demoing the system, or -debugging a regression on a known-good baseline. +NativeLink ships a Nix flake template that wires a Bazel project to +Local Remote Execution in one command. Useful for experimenting with +LRE, demoing the system, or debugging a regression on a known-good +baseline. ## Prerequisites @@ -21,102 +21,101 @@ debugging a regression on a known-good baseline. nix flake show github:TraceMachina/nativelink ``` -The templates we ship and maintain: + + Templates get added and removed between releases. Run the command + above rather than trusting a hardcoded list — including the one + below, which only reflects what's shipping as of this page's last + update. + -| Template | What it gives you | -| ------------- | ---------------------------------------------------------- | -| `bazel` | Bazel monorepo wired to a local NativeLink cluster. | -| `cargo` | Pure Cargo project with sccache pointing at NativeLink. | -| `lre` | Local Remote Execution — hermetic builds, no network. | -| `kubernetes` | `kind` cluster running the NativeLink Helm chart. | +| Template | What it gives you | +| --- | --- | +| `bazel` | A Bazel `cc_binary` project pre-wired for [Local Remote Execution](/explanations/lre): Nix-pinned toolchain, LRE Bazel module, and a `user.bazelrc` you point at a cache and executor. | ## The Bazel template -The fastest path to a working RBE setup: - ```bash mkdir my-rbe-test && cd my-rbe-test nix flake init -t github:TraceMachina/nativelink#bazel nix develop ``` -Inside the dev shell you have: - -- A pinned Bazel + Bazelisk. -- A NativeLink server binary. -- A `.bazelrc.local` already pointed at `localhost:50051`. - -Start the NativeLink server in one terminal: - -```bash -nativelink ./config/nativelink.json5 -``` - -Build the included sample target in another: - -```bash -bazel build //hello -``` - -Re-run; everything should hit the cache. - -## The LRE template - -For [Local Remote Execution](/explanations/lre): +`nix develop` downloads the Nix-pinned toolchain and generates +`lre.bazelrc`, which the template's `.bazelrc` already `try-import`s. +It does **not** bundle a NativeLink server — you point it at one you +already have: ```bash -mkdir my-lre-test && cd my-lre-test -nix flake init -t github:TraceMachina/nativelink#lre -nix develop +# In user.bazelrc +build --remote_cache=grpcs://TODO +build --bes_backend=grpcs://TODO +build --remote_timeout=600 +build --remote_executor=grpcs://TODO ``` -The dev shell provides a fully-hermetic Nix-pinned toolchain. -NativeLink runs on `localhost`; Bazel is pre-configured to use it. +Three ways to fill in `TODO` — [Local Remote Execution → Point +`user.bazelrc` at a real cache and executor](/rbe/local-remote-execution) +has a worked, filled-in example of the file below, whichever option +you pick: + +- **[dev.nativelink.com](https://dev.nativelink.com)** — paste your + cloud credentials, use `grpcs://`. +- **A self-hosted cluster** — your own endpoint, `grpcs://` if it + terminates TLS. +- **Nothing but your own machine** — drop the `s`, point both + `remote_cache` and `remote_executor` at `grpc://127.0.0.1:50051`. + [Local cache and executor](/rbe/local-testing) gets you a + from-scratch, credential-free instance in one command — enough to + validate `remote_cache`, though the included C++ example's `lre-cc` + platform still needs a worker running the matching container image + to actually execute (see [Local Remote Execution](/rbe/local-remote-execution) + for what that involves). + +Then build the included example: ```bash -nativelink ./config/lre.json5 -bazel build //hello +bazel build hello-world ``` -The first build of any target is the cost of compilation. Every -subsequent build is the cost of one network round-trip to -`localhost`. +Re-run; the cached actions should skip straight to a cache hit. -## The Kubernetes template +## Not shipped as flake templates today -For prototyping a real deployment without leaving your laptop: - -```bash -mkdir my-k8s-test && cd my-k8s-test -nix flake init -t github:TraceMachina/nativelink#kubernetes -nix develop -``` - -The flake includes `kind`, `kubectl`, `helm`, and the NativeLink -chart. Bring up a local cluster: - -```bash -kind create cluster -helm install nativelink ./chart -kubectl wait --for=condition=Ready pods --all -``` +Earlier drafts of this page also described `cargo`, `lre`, and +`kubernetes` templates. None exist in `flake.nix` right now — running +`nix flake init -t github:TraceMachina/nativelink#lre` (or `#cargo`, +`#kubernetes`) fails with `does not provide attribute 'templates.X'`. +If you followed a link here looking for one of those: -You now have CAS + scheduler + workers running in a local cluster -on your laptop. Port-forward `:50051` and point Bazel at it. +- **A local Bazel cluster with no Nix at all** — [Local cache and + executor](/rbe/local-testing) starts CAS, AC, a scheduler, and a + worker in one `nativelink` process. +- **A real Kubernetes deployment** — [Deployment → Kubernetes](/deployment/kubernetes) + covers the Helm chart directly; `kind` + that chart gets you the + same result a `kubernetes` template would have. +- **Cargo / sccache** — not currently packaged as a template; the + [`bazel` template](#the-bazel-template) above is the maintained path. ## Customising -Every template is a flake — fork and edit. The most common +The template is a flake — fork and edit. The most common modifications: -- Swap the `nativelink.json5` for a config matching your real - environment. -- Add custom worker platform properties. -- Pin to a specific NativeLink release (`inputs.nativelink.url`). +- Point `user.bazelrc` at your real cluster instead of a throwaway one. +- Add custom worker platform properties in `platforms/BUILD.bazel`. +- Pin to a specific NativeLink commit (`inputs.nativelink.url` in + `flake.nix`, kept in sync with the `local-remote-execution` module + override in `MODULE.bazel`). ## What's next -- [Architecture](/explanations/architecture) — the data flow - these templates exercise. -- [Configuration → Basic](/configuration/basic) — the JSON5 - the templates ship. +- [Local Remote Execution](/rbe/local-remote-execution) — this same + template, walked through step by step and tested for real. +- [Local cache and executor](/rbe/local-testing) — the fastest way to + get something running on `127.0.0.1` to point this template at. +- [Explanations → LRE](/explanations/lre) — what the toolchain pinning + this template sets up actually buys you. +- [Architecture](/explanations/architecture) — the data flow this + template exercises. +- [Configuration → Basic](/configuration/basic) — the JSON5 shape a + server you point this at should speak. diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index d8c56476f..02431ba98 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ main (cfb8141e) + Source: nativelink-config @ main (ce3a919e) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -180,6 +180,55 @@ It supports the following backends: } ``` +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "${OCI_ACCESS_KEY_ID}", + "secret_access_key": "${OCI_SECRET_ACCESS_KEY}", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + **Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) ### `ontap_s3_existence_cache` @@ -859,8 +908,8 @@ Configuration for `ExperimentalMongoDB` store. | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | | `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | | `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | -| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. When an action completes, the worker waits for the directory to be cleaned up before reusing it. If cleanup takes longer than this, the action fails with `DeadlineExceeded`. Value in seconds. | -| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. When waiting for a previous operation's cleanup to complete, the worker uses exponential backoff starting from a small value and increasing up to this maximum. Value in milliseconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | | `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | | `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | | `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | @@ -1114,6 +1163,55 @@ It supports the following backends: } ``` +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "${OCI_ACCESS_KEY_ID}", + "secret_access_key": "${OCI_SECRET_ACCESS_KEY}", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + **Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) ### `ontap_s3_existence_cache` diff --git a/web/apps/docs/content/docs/reference/nativelink-config/meta.json b/web/apps/docs/content/docs/reference/nativelink-config/meta.json index 35333832f..22e3bb1a6 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/meta.json +++ b/web/apps/docs/content/docs/reference/nativelink-config/meta.json @@ -1,6 +1,7 @@ { "pages": [ - "index" + "index", + "store-overview" ], "title": "NativeLink config" } diff --git a/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx new file mode 100644 index 000000000..ca4b343fa --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx @@ -0,0 +1,171 @@ +--- +title: Store overview +description: The mental model behind every store in the configuration reference — what holds data, what wraps another store, and how they compose. +--- + +The [full reference](/reference/nativelink-config) lists every field on +every store. This page is the map you need before that list is useful: +what a store actually is, the two families every store falls into, and +the compositions people actually run in production. + +## What a store is + +A store is a named, pluggable backend for CAS or AC data. You declare +each one once in the top-level `stores` array, then reference it by +name from `servers`, `workers`, or from inside another store: + +```json5 +{ + stores: [ + { name: "CAS_MAIN_STORE", filesystem: { /* ... */ } }, + ], + servers: [{ + services: { + cas: [{ instance_name: "main", cas_store: "CAS_MAIN_STORE" }], + }, + }], +} +``` + +The store's *type* is whichever key you put inside its object — +`filesystem`, `memory`, `compression`, `fast_slow`, and so on. NativeLink +picks the implementation from that key; everything else in the object is +that implementation's config. + +## Two families + +Every store type is either **terminal** (it actually holds or fetches +bytes) or a **wrapper** (it adds behavior in front of another store, +which it embeds as a nested `StoreSpec`). Wrappers nest arbitrarily +deep — a `verify` can wrap a `compression`, which wraps a `fast_slow`, +which wraps two more terminal stores: + + +{`flowchart LR + V["verify\\n(hash + size check)"] --> C["compression\\n(lz4)"] + C --> F["fast_slow"] + F -->|fast| M[("memory")] + F -->|slow| D[("filesystem")]`} + + +Reads and writes flow through every wrapper in order. Nothing about the +client changes — it still speaks plain CAS/AC RPCs against whatever +store name the server exposes. + +### Terminal stores — where bytes actually live + +| Store key | Holds data in | Notes | +| --- | --- | --- | +| `memory` | An in-process hash map | Fastest, gone on restart. Good for AC tiers and dev. | +| `filesystem` | Local disk | Survives restarts; scans and rebuilds its index on startup. | +| `experimental_cloud_object_store` | A cloud object bucket | One store type, six `provider`s: `aws` (S3), `gcs`, `azure`, `ontap` (NetApp ONTAP S3), `r2` (Cloudflare), `oci` (Oracle Cloud Infrastructure). All speak an S3-compatible API except `gcs` and `azure`. | +| `redis_store` | Any Redis-API-compatible service | Pairs well with `size_partitioning` — most Redis services cap uploads around 256–512 MB. | +| `experimental_mongo` | MongoDB | Supports CAS and scheduler data, with optional change streams for scheduler subscriptions. | +| `grpc` | Another NativeLink-compatible gRPC endpoint | Proxies calls upstream. Useful for a regional cache in front of a central cluster. | +| `ref_store` | Nothing itself — points at another store by name | Lets two composition trees share one underlying store instance (see below). | +| `noop` | Nothing | Reads 404. Writes vanish. Used to explicitly discard a partition of data. | + +### Wrapper stores — behavior layered on a backend + +| Store key | Wraps | What it adds | +| --- | --- | --- | +| `compression` | one store | LZ4-compresses on write, decompresses on read. | +| `dedup` | two stores (`index_store` + `content_store`) | Rolling-hash chunking so only changed slices upload. | +| `fast_slow` | two stores (`fast` + `slow`) | Reads try `fast` first, fall back to `slow`, and backfill `fast`. Writes mirror to both. | +| `shard` | N stores | Routes by digest hash. The standard shape for scaling CAS past one backend. | +| `size_partitioning` | two stores (`lower_store` + `upper_store`) | Routes by blob size instead of hash. CAS-only — see the gotcha below. | +| `verify` | one store | Rejects uploads that fail a hash and/or size check before they ever reach the backend. | +| `existence_cache` | one store | Caches `has()` results. CAS-only. | +| `completeness_checking` | one store + a CAS store | Confirms an `ActionResult`'s output digests exist in CAS before returning it. AC-only. | +| `cache_metrics` | one store | Emits OpenTelemetry cache-hit/miss metrics for the wrapped store. Opt-in — stores you don't wrap pay nothing extra. | +| `ontap_s3_existence_cache` | (built-in ONTAP S3 backend) | Purpose-built existence cache for ONTAP S3 specifically, with disk-persisted sync instead of a generic wrapped backend. | + + + - Put `dedup` *inside* `compression` (compression wraps dedup's output), + never the other way — `compression` as `dedup`'s `content_store` negates + dedup's gains, since every chunk becomes a differently-compressed blob. + - `fast_slow` never checks whether an object in `fast` also exists in + `slow`. If you need a durability guarantee — e.g. remote execution + artifacts that must survive a `fast`-tier wipe — write-through both + tiers deliberately rather than assuming it. + - `existence_cache` and `size_partitioning` are CAS-only; `completeness_checking` + is AC-only. Using them on the other store type produces confusing + correctness bugs, not a config error. + + +## Choosing a terminal backend + +| Backend | Durability | Shared across instances | Typical role | +| --- | --- | --- | --- | +| `memory` | None (restart wipes it) | No | Fast tier in a `fast_slow`, or a whole AC store for a short-lived CI runner. | +| `filesystem` | Survives restarts | No (single node) | Single-node dev/CI cache, or the fast/local tier in front of shared storage. | +| `experimental_cloud_object_store` | Durable, provider-managed | Yes | The shared, multi-node backing store in most production clusters. | +| `redis_store` | As durable as your Redis deployment | Yes | Low-latency shared tier, usually fronted by `size_partitioning` to keep large blobs out of it. | +| `experimental_mongo` | Durable | Yes | Alternative shared backend when you already operate MongoDB and want scheduler-state change streams. | + +## Common compositions + +**Single-node dev cache** — no wrapping at all. See +[Configuration → Basic](/configuration/basic). + +**Validated, compressed, tiered CAS** — the shape most self-hosted +production clusters converge on: + +```json5 +{ + name: "CAS_MAIN_STORE", + verify: { + verify_size: true, + verify_hash: true, + backend: { + compression: { + compression_algorithm: { lz4: {} }, + backend: { + fast_slow: { + fast: { memory: { eviction_policy: { max_bytes: "2gb" } } }, + slow: { + experimental_cloud_object_store: { + provider: "aws", + region: "us-east-1", + bucket: "nativelink-cas", + }, + }, + }, + }, + }, + }, + }, +} +``` + +**Sharing one store instance across two trees** — use `ref_store` +instead of declaring the same backend twice: + +```json5 +{ + stores: [ + { + name: "FS_CONTENT_STORE", + filesystem: { content_path: "/var/lib/nativelink/cas", temp_path: "/var/lib/nativelink/tmp" }, + }, + { + // The worker's fast/local tier and the AC's fast tier now share + // one filesystem store instead of each scanning their own copy. + name: "AC_MAIN_STORE", + fast_slow: { + fast: { ref_store: { name: "FS_CONTENT_STORE" } }, + slow: { noop: {} }, + }, + }, + ], +} +``` + +## What's next + +- [Configuration reference](/reference/nativelink-config) — every + field, default, and JSON5 example for every store type above. +- [Configuration → Introduction](/configuration/intro) — how stores + fit alongside servers, schedulers, and workers. +- [Configuration → Production](/configuration/production) — sharded, + multi-region store topologies end to end. diff --git a/web/apps/docs/scripts/gen-config-reference.mjs b/web/apps/docs/scripts/gen-config-reference.mjs index fcf137f7d..051cc4859 100644 --- a/web/apps/docs/scripts/gen-config-reference.mjs +++ b/web/apps/docs/scripts/gen-config-reference.mjs @@ -252,12 +252,17 @@ export const CONFIG_VERSIONS: ConfigVersion[] = ${JSON.stringify(entries, null, writeFileSync(manifestFile, body); } +// Hand-written pages that live alongside the autogenerated reference in this +// folder. Keep these out of `resolveVersions()`/`generateOne()` — they're not +// versioned schema dumps — but they must survive `writeMeta()` below. +const HAND_WRITTEN_PAGES = ["store-overview"]; + function writeMeta() { - // Only the canonical page appears in the sidebar; other versions are reached - // through the in-page version switcher. + // The canonical page plus any hand-written companion pages appear in the + // sidebar; other versions are reached through the in-page version switcher. writeFileSync( join(contentDir, "meta.json"), - `${JSON.stringify({ pages: ["index"], title: "NativeLink config" }, null, 2)}\n`, + `${JSON.stringify({ pages: ["index", ...HAND_WRITTEN_PAGES], title: "NativeLink config" }, null, 2)}\n`, ); }