diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c000835..b01358d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,28 @@ on: pull_request: jobs: + msrv: + runs-on: ubuntu-latest + steps: + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + - name: Checkout + uses: actions/checkout@v4 + - name: Install MSRV + uses: dtolnay/rust-toolchain@1.88.0 + - name: Cache Rust + uses: Swatinem/rust-cache@v2 + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + version: "24.x" + - name: Check all targets at MSRV + run: cargo check --locked --all-targets --all-features + build-and-test: runs-on: ubuntu-latest steps: @@ -29,8 +51,14 @@ jobs: with: version: "24.x" - name: Test - run: cargo test + run: cargo test --locked --all-features + - name: Check all targets + run: cargo check --locked --all-targets --all-features - name: Format run: cargo fmt --check - name: Clippy - run: cargo clippy -- -D warnings + run: cargo clippy --locked --all-targets --all-features -- -D warnings + - name: Validate deployment manifests + run: | + bash -n scripts/*.sh + docker compose -f compose.yaml config --quiet diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 724b1c2..7caa983 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,9 +19,15 @@ jobs: with: version: "24.x" - name: cargo test - run: cargo test + run: cargo test --locked --all-features + - name: cargo fmt + run: cargo fmt --check + - name: cargo clippy + run: cargo clippy --locked --all-targets --all-features -- -D warnings + - name: cargo package + run: cargo package --locked - name: cargo publish - run: cargo publish + run: cargo publish --locked env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - name: GitHub Release diff --git a/.gitignore b/.gitignore index c2838f3..277c439 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Rust /target/ -Cargo.lock # Generated files *.dot diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a827e39 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,61 @@ +# Changelog + +All notable changes to this project are documented here. This project follows +Semantic Versioning. + +## [Unreleased] + +### Fixed + +- Ingest acknowledgement now waits for a synchronous RocksDB WAL flush. The + external ingest WAL is removed only after records and entity-resolution state + have reached stable storage, closing a power-loss window for acknowledged data. +- Ingest WAL files now carry a version, payload length, and checksum. WAL rename, + removal, and quarantine operations sync the parent directory; corrupt or + truncated WALs fail shard startup and are preserved for recovery. +- Cross-shard merge redirects now persist before their RPC reports success and + reload when persistent linker state is reconstructed. +- Persistent reset now clears DSU, tiered-index, and linker state together with + records and assignments in one RocksDB write batch, preventing stale clusters + from reappearing after reset or restart. +- Explicit checkpoints sync the RocksDB WAL before creating a checkpoint. +- Persistent shards no longer send large batches to in-memory partition stores. + Acknowledged records and their entity-resolution assignments now survive a + shard restart regardless of batch size. +- Duplicate source identities within one staged persistent batch are now + idempotent instead of creating multiple records. +- Ingest APIs now return persistence failures instead of logging them and + returning a successful acknowledgement. +- Large partitioned requests now reject invalid records consistently instead of + silently dropping them during parallel conversion. +- Partitioned linking errors are propagated instead of producing cluster zero + assignments. +- Persistent shard reset now releases the active RocksDB handle before reopening + the database, avoiding self-deadlock on the database lock. +- Interner persistence watermarks now advance only after a successful RocksDB + write, so retries cannot omit attribute or value mappings. +- Load-test sent and acknowledged counters no longer double-count generated + records or classify failed requests as successful. Reported latency is now + measured across successful RPCs instead of inferred from aggregate throughput, + and incomplete runs exit unsuccessfully after writing their diagnostic report. + +### Removed + +- Removed the partitioned `ultra_fast` API, which skipped both storage and entity + resolution and violated the engine's ingest contract. +- Removed standalone in-memory and single-node examples. The maintained example + now demonstrates the persistent distributed deployment model. + +### Changed + +- Adopted Rust 2024 with a documented MSRV of Rust 1.88. +- Updated the active gRPC, terminal UI, cache, and compression dependencies and + removed unused direct HTTP, TOML, and protobuf-types dependencies. +- The package version is now `0.2.0` in preparation for the next release. +- The application lockfile is tracked for reproducible binary and container + builds. +- Distributed integration tests now use temporary persistent shard stores; the + in-memory store remains limited to unit tests. +- Replaced the non-durable historical performance claim with a verified + five-shard, power-loss-durable baseline of 50,598 records/second for the + documented 10-million-record workload. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c1d308b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3814 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.1", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.112", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec 0.6.3", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "by_address" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64fa3c856b712db6612c019f14756e64e4bcea13337a6b33b696333a9eaa2d06" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.13.1", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csscolorparser" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a7d3066da2de787b7f032c736763eb7ae5d355f81a68bab2675a96008b0bf" +dependencies = [ + "lab", + "phf", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.112", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "deltae" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.112", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set 0.5.3", + "regex", +] + +[[package]] +name = "fast-srgb8" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd2e7510819d6fbf51a5545c8f922716ecfb14df168a3242f7d33e0239efe6a1" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "toml", + "uncased", + "version_check", +] + +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" + +[[package]] +name = "finl_unicode" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + +[[package]] +name = "instability" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "kasuari" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde5057d6143cc94e861d90f591b9303d6716c6b9602309150bd068853c10899" +dependencies = [ + "hashbrown 0.16.1", + "portable-atomic", + "thiserror 2.0.17", +] + +[[package]] +name = "lab" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "librocksdb-sys" +version = "0.17.3+10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "line-clipping" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f50e8f47623268b5407192d26876c4d7f89d686ca130fdc53bced4814cd29f8" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "mac_address" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" +dependencies = [ + "nix", + "winapi", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memmem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "palette" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbf71184cc5ecc2e4e1baccdb21026c20e5fc3dcf63028a086131b3ab00b6e6" +dependencies = [ + "approx", + "fast-srgb8", + "libm", + "palette_derive", +] + +[[package]] +name = "palette_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5030daf005bface118c096f510ffb781fc28f9ab6a32ab224d8631be6851d30" +dependencies = [ + "by_address", + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "pest_meta" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +dependencies = [ + "pest", +] + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset 0.5.7", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.112", +] + +[[package]] +name = "proc-macro2" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", + "version_check", + "yansi", +] + +[[package]] +name = "proptest" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" +dependencies = [ + "bit-set 0.8.0", + "bit-vec 0.8.0", + "bitflags 2.13.1", + "num-traits", + "rand 0.9.2", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.112", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags 2.13.1", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.3", +] + +[[package]] +name = "ratatui" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3274ba0a2c5e1bcad2a2005d20f4dc59dad26b2eb0940fb094500dba4099d57d" +dependencies = [ + "instability", + "ratatui-core", + "ratatui-crossterm", + "ratatui-macros", + "ratatui-termina", + "ratatui-termwiz", + "ratatui-widgets", + "serde", +] + +[[package]] +name = "ratatui-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb175c433c8e28a809d1f5773a2ae96e68c0ce40db865cbab1020bf33ae479c" +dependencies = [ + "bitflags 2.13.1", + "compact_str", + "critical-section", + "hashbrown 0.17.1", + "itertools 0.14.0", + "kasuari", + "lru", + "palette", + "serde", + "strum", + "thiserror 2.0.17", + "unicode-segmentation", + "unicode-truncate", + "unicode-width", +] + +[[package]] +name = "ratatui-crossterm" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567584a3b0e6a8203c23de40b4861497266725eb5363dbfd18a1edd603cca9f0" +dependencies = [ + "cfg-if", + "crossterm", + "instability", + "ratatui-core", +] + +[[package]] +name = "ratatui-macros" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7dc68daa7498a43e4d68e0eb078427e10c38fbcfbb1e42d955f1fa2140d814" +dependencies = [ + "ratatui-core", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-termina" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0bf912d9e66f057a759d92e386a280ea886b352ab757d6ac4d653c7ed2c43c2" +dependencies = [ + "instability", + "ratatui-core", + "termina", +] + +[[package]] +name = "ratatui-termwiz" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf03e0380b7744054d6cb74224fe3adf062a029754933f575ca1e3b4c2ce977" +dependencies = [ + "ratatui-core", + "termwiz", +] + +[[package]] +name = "ratatui-widgets" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e3d19bcc9130ca376277d93b60767ff121ace3be06f5f95f81dd68956407d1" +dependencies = [ + "bitflags 2.13.1", + "hashbrown 0.17.1", + "indoc", + "instability", + "itertools 0.14.0", + "line-clipping", + "ratatui-core", + "serde", + "strum", + "time", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rcgen" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rocksdb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +dependencies = [ + "libc", + "librocksdb-sys", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "serde_json" +version = "1.0.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "string-interner" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23de088478b31c349c9ba67816fa55d9355232d63c3afea8bf513e31f0f1d2c0" +dependencies = [ + "hashbrown 0.15.5", + "serde", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21f182278bf2d2bcb3c88b1b08a37df029d71ce3d3ae26168e3c653b213b99d4" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termina" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" +dependencies = [ + "bitflags 2.13.1", + "parking_lot", + "rustix", + "signal-hook", + "windows-sys 0.61.2", +] + +[[package]] +name = "terminfo" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" +dependencies = [ + "fnv", + "nom", + "phf", + "phf_codegen", +] + +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + +[[package]] +name = "termwiz" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" +dependencies = [ + "anyhow", + "base64", + "bitflags 2.13.1", + "fancy-regex", + "filedescriptor", + "finl_unicode", + "fixedbitset 0.4.2", + "hex", + "lazy_static", + "libc", + "log", + "memmem", + "nix", + "num-derive", + "num-traits", + "ordered-float", + "pest", + "pest_derive", + "phf", + "sha2", + "signal-hook", + "siphasher", + "terminfo", + "termios", + "thiserror 1.0.69", + "ucd-trie", + "unicode-segmentation", + "vtparse", + "wezterm-bidi", + "wezterm-blob-leases", + "wezterm-color-types", + "wezterm-dynamic", + "wezterm-input-types", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.112", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" +dependencies = [ + "itertools 0.14.0", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "unirust-rs" +version = "0.2.0" +dependencies = [ + "anyhow", + "async-channel", + "bincode", + "chrono", + "crc32fast", + "criterion", + "crossbeam-channel", + "crossbeam-queue", + "crossterm", + "dashmap", + "figment", + "futures", + "hashbrown 0.16.1", + "lru", + "lz4_flex", + "parking_lot", + "proptest", + "prost", + "rand 0.9.2", + "ratatui", + "rayon", + "rcgen", + "rocksdb", + "rustc-hash", + "serde", + "serde_json", + "sha2", + "smallvec", + "string-interner", + "tempfile", + "thiserror 2.0.17", + "time", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tonic-prost-build", + "tracing", + "tracing-subscriber", + "unirust-rs", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "atomic", + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vtparse" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9b2acfb050df409c972a37d3b8e08cdea3bddb0c09db9d53137e504cfabed0" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.112", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wezterm-bidi" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0a6e355560527dd2d1cf7890652f4f09bb3433b6aadade4c9b5ed76de5f3ec" +dependencies = [ + "log", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-blob-leases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692daff6d93d94e29e4114544ef6d5c942a7ed998b37abdc19b17136ea428eb7" +dependencies = [ + "getrandom 0.3.4", + "mac_address", + "sha2", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "wezterm-color-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de81ef35c9010270d63772bebef2f2d6d1f2d20a983d27505ac850b8c4b4296" +dependencies = [ + "csscolorparser", + "deltae", + "lazy_static", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-dynamic" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2ab60e120fd6eaa68d9567f3226e876684639d22a4219b313ff69ec0ccd5ac" +dependencies = [ + "log", + "ordered-float", + "strsim", + "thiserror 1.0.69", + "wezterm-dynamic-derive", +] + +[[package]] +name = "wezterm-dynamic-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c0cf2d539c645b448eaffec9ec494b8b19bd5077d9e58cb1ae7efece8d575b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wezterm-input-types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7012add459f951456ec9d6c7e6fc340b1ce15d6fc9629f8c42853412c029e57e" +dependencies = [ + "bitflags 1.3.2", + "euclid", + "lazy_static", + "serde", + "wezterm-dynamic", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.112", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zmij" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9211a9f64b825911bdf0240f58b7a8dac217fe260fc61f080a07f61372fbd5" + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index be4515c..6831ade 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "unirust-rs" -version = "0.1.0" -edition = "2021" +version = "0.2.0" +edition = "2024" +rust-version = "1.88" description = "High-performance temporal entity resolution engine with conflict detection, knowledge graph export, and perspective-weighted data mastering for multi-source datasets." authors = ["unirust contributors"] license = "MIT" @@ -28,22 +29,22 @@ rayon = "1.10" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.147" bincode = "1.3.3" -toml = "0.8" +crc32fast = "1.5.0" +sha2 = "0.10" # Configuration figment = { version = "0.10", features = ["toml", "env"] } # Compression (Bigtable-style fast compression) -lz4_flex = "0.11" +lz4_flex = "0.14" rocksdb = "0.24.0" -reqwest = { version = "0.11", features = ["json", "rustls-tls"] } -tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs", "io-util", "time"] } -tonic = "0.11" -prost = "0.12" -prost-types = "0.12" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs", "io-util", "time", "signal"] } +tonic = { version = "0.14", features = ["tls-ring"] } +tonic-prost = "0.14" +prost = "0.14" tokio-stream = { version = "0.1", features = ["net"] } futures = "0.3" -lru = "0.12.3" +lru = "0.18" # Property-based testing proptest = "1.9.0" @@ -60,8 +61,8 @@ tracing-subscriber = "0.3.19" string-interner = "0.19.0" # TUI -ratatui = "0.29" -crossterm = "0.28" +ratatui = { version = "0.30", features = ["crossterm_0_29"] } +crossterm = "0.29" # Async channels for MPMC async-channel = "2.3" @@ -79,10 +80,11 @@ rand = { version = "0.9.2", optional = true } criterion = { version = "0.8.1", features = ["html_reports"] } proptest = "1.9.0" tempfile = "3.12.0" +rcgen = "0.14" unirust-rs = { path = ".", features = ["test-support"] } [build-dependencies] -tonic-build = "0.11" +tonic-prost-build = "0.14" # Quick benchmarks (~30s) - CI/dev feedback [[bench]] @@ -120,3 +122,7 @@ default = [] profiling = [] # Enable test support module (used by integration tests and benchmarks) test-support = ["rand"] + +[lints.clippy] +# Keep nested conditionals where they make multi-stage validation easier to audit. +collapsible_if = "allow" diff --git a/Containerfile b/Containerfile index 9265bd0..1ddd933 100644 --- a/Containerfile +++ b/Containerfile @@ -4,12 +4,12 @@ # # Usage: # podman build -t unirust -f Containerfile . -# podman run --rm -p 50061:50061 unirust shard +# podman run --rm -p 50061:50061 -v unirust-data:/data unirust shard # podman run --rm -p 50060:50060 unirust router --shards shard-0:50061 # # Or use with compose.yaml for full cluster deployment -FROM rust:1.82 AS builder +FROM rust:1.88-bookworm AS builder WORKDIR /app @@ -27,9 +27,11 @@ COPY src ./src COPY benches ./benches # Build release binaries -RUN cargo build --release --no-default-features \ +RUN cargo build --release --locked --features test-support \ --bin unirust_shard \ --bin unirust_router \ + --bin unirust_healthcheck \ + --bin unirust_client \ --bin unirust_loadtest # Production image @@ -46,10 +48,12 @@ RUN useradd -m -u 1000 unirust # Copy binaries COPY --from=builder /app/target/release/unirust_shard /usr/local/bin/ COPY --from=builder /app/target/release/unirust_router /usr/local/bin/ +COPY --from=builder /app/target/release/unirust_healthcheck /usr/local/bin/ +COPY --from=builder /app/target/release/unirust_client /usr/local/bin/ COPY --from=builder /app/target/release/unirust_loadtest /usr/local/bin/ -# Create data directory -RUN mkdir -p /data && chown unirust:unirust /data +# Create persistent data and checkpoint mount points +RUN mkdir -p /data /backup && chown unirust:unirust /data /backup WORKDIR /app USER unirust diff --git a/DESIGN.md b/DESIGN.md index ec4c405..66b6c78 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -581,7 +581,8 @@ fn wal_writer_loop(rx: Receiver, config: WalConfig) { ### Partitioned Processing -For maximum throughput, records are partitioned by identity key hash for parallel processing. Each partition uses `process_batch_optimized()`: +For non-persistent shards, records can be partitioned by identity key hash for +parallel processing. Each partition uses `process_batch_optimized()`: ``` ┌──────────────────────┐ @@ -614,6 +615,13 @@ For maximum throughput, records are partitioned by identity key hash for paralle Each partition runs independently with its own `Mutex`. Rayon processes all partitions in parallel—no global lock contention. +Partition stores are currently in-memory. A shard configured with `data_dir` +therefore does not use this path: it routes every batch through the shard's +`PersistentStore`, where records, indexes, and cluster assignments are durable +and immediately available to queries. Enabling durable partition-local stores +requires an explicit on-disk layout and recovery protocol; routing persistent +traffic to in-memory partitions is not a valid optimization. + --- ## Data Flow @@ -624,18 +632,28 @@ Each partition runs independently with its own `Mutex`. Rayon process 1. Client sends RecordInput batch via gRPC 2. Router hashes identity keys → partition records to shards 3. Each shard receives its partition of the batch -4. dispatch_ingest_partitioned() routes to ParallelPartitionedUnirust -5. Records partitioned again by identity_key_hash % partition_count +4. Non-persistent shards may route large batches to ParallelPartitionedUnirust +5. On that path, records are partitioned again by identity_key_hash % partition_count 6. Each partition (in parallel via Rayon): a. Batch insert all records to store b. Parallel extract: identity keys + strong ID summaries (Rayon) c. Sequential link: DSU merges with temporal guards d. Update identity index -7. Merge partition results, sort by original index -8. Update boundary index for cross-shard tracking -9. Return cluster assignments +7. Persistent shards instead resolve through Unirust backed by PersistentStore +8. Before resolution, write and fsync a versioned, checksummed binary ingest WAL +9. Persist records, indexes, cluster assignments, and request metadata +10. Sync the RocksDB WAL to stable storage +11. Remove the ingest WAL and fsync its parent directory +12. Update boundary indexes for cross-shard tracking +13. Return cluster assignments ``` +If a shard stops before step 11, restart replays the ingest WAL using source +identity idempotency. If framing, length, or checksum validation fails, startup +fails closed and preserves the corrupt file for operator recovery. Cross-shard +merge redirects use the linker metadata column family and receive the same +stable-storage barrier before their RPC reports success. + ### Query Path ``` @@ -675,7 +693,7 @@ Each partition runs independently with its own `Mutex`. Rayon process | Balanced | 2,000 | 50,000 | General purpose | | LowLatency | 1,000 | 20,000 | Fast responses | | HighThroughput | 4,000 | 100,000 | Batch processing | -| BulkIngest | 500 | 10,000 | Maximum speed | +| BulkIngest | 500 | 10,000 | Large loads; full resolution with lower candidate caps | | MemorySaver | 500 | 5,000 | Reduced memory | | BillionScale | 2,000 + persistent DSU | 100,000 | Huge datasets | @@ -683,14 +701,18 @@ Each partition runs independently with its own `Mutex`. Rayon process ## Appendix: Performance Characteristics -### Throughput (5-shard cluster, 16 streams, batch size 5000) +### Verified Throughput (5-shard persistent cluster) + +The release audit on 2026-07-22 measured 50,598 records/sec for 10,000,000 +records, 16 streams, 5,000-record batches, and 10% overlap on an Apple M5 with +32 GB RAM. All 10,000,000 records were acknowledged with zero stream errors. +The measurement includes an `fsync` of the RocksDB WAL before each successful +batch acknowledgement. -| Workload | Records/sec | Notes | -|----------|-------------|-------| -| Pure ingest, no overlap | ~500K | Extraction-dominated | -| 10% overlap probability | ~410K | Moderate DSU merge overhead | -| 50% overlap probability | ~280K | Heavy merge workload | -| With cross-shard reconciliation | ~380K | Periodic boundary sync | +Earlier 280K-500K figures measured an in-memory partition path that did not +persist or expose its records through the shard's primary store. They are not +valid production baselines. Performance work must preserve full entity +resolution and acknowledge records only after durable storage. ### Memory Usage diff --git a/README.md b/README.md index c8db01d..948f9ea 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Unirust will: - **Conflict Detection**: Automatic detection of attribute conflicts within clusters - **Distributed**: Router + multi-shard architecture for horizontal scaling - **Persistent**: RocksDB storage with crash recovery -- **High Performance**: 400K+ records/sec via batch-parallel processing, lock-free DSU, SIMD hashing +- **Measured Performance**: Release baselines use persistent shards and include full entity resolution; see the benchmark below ## Quick Start @@ -52,7 +52,13 @@ cargo build --release ```bash # Use the cluster script -SHARDS=5 ./scripts/cluster.sh start +SHARDS=5 ONTOLOGY=/etc/unirust/ontology.json ./scripts/cluster.sh start + +# Restart the same persistent cluster without deleting records +SHARDS=5 ONTOLOGY=/etc/unirust/ontology.json ./scripts/cluster.sh restart + +# Destructive reset is separate and requires explicit confirmation +UNIRUST_CONFIRM_RESET=1 ./scripts/cluster.sh reset # Or start manually: ./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /data/shard0 @@ -132,7 +138,12 @@ write_buffer_mb = 256 | `UNIRUST_PROFILE` | Tuning profile | | `UNIRUST_SHARD_LISTEN` | Shard listen address | | `UNIRUST_SHARD_ID` | Shard ID | +| `UNIRUST_SHARD_DATA_DIR` | Persistent shard data directory | +| `UNIRUST_SHARD_BACKUP_DIR` | External checkpoint root | | `UNIRUST_ROUTER_SHARDS` | Comma-separated shard addresses | +| `UNIRUST_ROUTER_CHECKPOINT_INTERVAL_SECS` | Coordinated checkpoint interval (`0` disables) | +| `UNIRUST_ROUTER_SHARD_CONNECT_TIMEOUT_SECS` | Shard connection timeout | +| `UNIRUST_ROUTER_SHARD_REQUEST_TIMEOUT_SECS` | Per-RPC shard timeout | ### Tuning Profiles @@ -141,9 +152,9 @@ write_buffer_mb = 256 | `balanced` | General purpose (default for library) | | `low-latency` | Interactive queries, fast responses | | `high-throughput` | Batch processing (default for binaries) | -| `bulk-ingest` | Maximum ingest speed, reduced matching | +| `bulk-ingest` | Large initial loads with lower candidate caps; entity resolution remains enabled | | `memory-saver` | Constrained environments | -| `billion-scale` | Persistent DSU for huge datasets | +| `billion-scale` | Disk-backed DSU/index; see the recovery and memory limits below | ## API Reference @@ -187,43 +198,235 @@ See [DESIGN.md](DESIGN.md) for detailed architecture documentation, including: ## Examples -The `examples/` directory contains comprehensive examples: +The `examples/` directory demonstrates the supported sharded deployment model: -- `in_memory.rs` - In-memory entity resolution, perfect for learning -- `persistent_shard.rs` - Single-shard with RocksDB persistence - `cluster.rs` - Full 3-shard distributed cluster with router -- `unirust.toml` - Example configuration file +- `unirust.toml` - Persistent router and shard configuration Run examples: ```bash -# Simple in-memory example -cargo run --example in_memory - -# Persistent storage with single shard -cargo run --example persistent_shard - -# Distributed cluster (requires cluster running first) +# Distributed cluster (requires the persistent cluster running first) SHARDS=3 ./scripts/cluster.sh start cargo run --example cluster ./scripts/cluster.sh stop ``` +## Durability + +Persistent shards use two recovery layers for every ingest request: + +1. The request is written to a versioned, checksummed binary ingest WAL. The + file and its parent directory are synced before entity resolution begins. +2. Records, indexes, cluster assignments, and all other state produced by the + request are written to RocksDB, then its WAL is synced to stable storage. +3. Only after that sync succeeds is the ingest WAL removed and the request + acknowledged. Its directory is synced again after removal. + +On restart, a remaining ingest WAL is replayed idempotently. A pending WAL is +never overwritten by a later request; a failed ingest therefore requires shard +restart and replay before more traffic is accepted. A truncated or corrupt WAL +is preserved with a `.corrupt.*` suffix and shard startup fails with a data-loss +error instead of accepting traffic with an unknown recovery gap. Cross-shard +merge redirects are also persisted before acknowledgement and reloaded when the +streaming linker is reconstructed. + +The tuple `(entity_type, perspective, uid)` identifies one immutable source +record. Retrying that identity with the same descriptors is idempotent, including +when descriptor order differs. Reusing it with different values or validity +intervals is rejected instead of acknowledging data that would be discarded. +Temporal revisions must use a new source-record UID; entity resolution links the +snapshots through their configured identity keys. + +In a multi-shard cluster, the router first durably reserves that source identity +on a shard selected only from the source tuple. The reservation stores a +canonical binary payload digest and the original ingest target before the record +enters the normal shard entity-resolution path. This prevents a changed +identity-key value from routing the same source UID to another shard. If target +ingest fails after reservation, retry the exact request; the reservation remains +valid and no record has bypassed entity resolution. + +Router startup performs a crash-resumable one-time reservation backfill for +records written by an earlier release. Each target shard is marked complete only +after every exported record has been reserved durably. Conflicting legacy +duplicates make startup fail closed for operator repair. The internal router and +shard protocol is versioned, and mixed versions are rejected, so upgrades must +replace the full cluster with one coordinated Unirust version before restarting +the router. Shard gRPC ports are internal APIs; client ingest must use the router. +The persisted reservation directory is also bound to the configured shard count. +Router startup rejects shard-count changes because the current import API cannot +atomically move a record, update its reservation, and delete the old copy. +Cross-shard copy imports therefore fail closed; online scale-out and scale-in +require a future transactional relocation protocol or an offline rebuild. + +Cross-shard redirects are durably applied to every shard and are idempotent. If +any shard fails while a reconciliation result is being applied, the router +latches the cluster closed for ingest, query, and administrative traffic rather +than serving a partially updated global view. Retrying `Reconcile` repairs the +retained dirty keys in place. After a router or full-cluster restart, router +startup performs that repair before returning a serviceable node and clears the +dirty generation only after every shard converges. + +The shard reconstructs all derived linker state before opening its gRPC +listener. Recovery currently scans all persisted records, so recovery time is +O(record count). This is a correctness-first crash-recovery path, not a bounded +recovery-time guarantee. Measure restart time at the intended dataset size and +set orchestration startup probes accordingly. + +`LinkerStateConfig` cache capacities are not enforced because the current LRU +backend has no durable spill/read-through path. Evicting cluster IDs, strong-ID +summaries, or record perspectives would change entity-resolution results. +Persistent profiles therefore retain this correctness-critical working set in +memory even though their DSU and identity index are disk-backed. The +`billion-scale` profile must not be treated as proof that a billion-record +deployment fits a given memory or recovery-time budget. + +`scripts/cluster.sh start` and `restart` preserve `DATA_DIR`. Only the explicit +`reset` action deletes shard data, and it requires `UNIRUST_CONFIRM_RESET=1`. +The script defaults to `examples/loadtest-ontology.json`; set `ONTOLOGY` to the +same immutable configuration on every shard and router for another deployment. +Router startup compares the complete ontology reported by every shard and fails +closed on a mismatch. + +The destructive gRPC `Reset` method is disabled by default because a sequential +multi-shard reset cannot be atomic. The supported production reset is the +confirmed offline script action. Test or isolated admin deployments can opt in +with the shard flag `--allow-destructive-admin`. + +The shard and router binaries handle SIGINT/SIGTERM with graceful gRPC shutdown. +The shard drains active mutations and flushes dirty DSU and authoritative store +state before exit. Acknowledged ingests are additionally covered by an +executable-level SIGKILL, restart, SIGTERM, and second-restart integration test. + +Use `unirust_healthcheck --shard ` and +`unirust_healthcheck --router ` for readiness probes. These call the gRPC +health methods rather than checking only for an open socket. Shard readiness +requires completed WAL recovery and a healthy persistent store; router +readiness additionally requires a consistent reconciliation state and a +successful health response from every configured shard. The provided Compose +deployment and cluster script use these semantic probes. + +Router-to-shard calls are bounded by configurable transport settings under +`[router]`: `shard_connect_timeout_secs` (10 seconds by default), +`shard_request_timeout_secs` (120 seconds), and +`shard_tcp_keepalive_secs` (30 seconds). A deadline error is not proof that a +mutation was rolled back; the shard may have committed just before the response +was lost. Retry ingest with the same immutable source-record identity and +payload so the operation is resolved idempotently. + +### External Backups + +Configure each shard with a unique checkpoint root on storage outside its data +directory and, for real volume-loss protection, in a separate failure domain: + +```toml +[shard] +data_dir = "/var/lib/unirust/shard-0" +backup_dir = "/var/backups/unirust/shard-0" + +[router] +# Select this from the deployment RPO. Zero disables automatic checkpoints. +checkpoint_interval_secs = 3600 +``` + +Trigger checkpoints through the router so router-mediated mutations remain +blocked for the complete cluster snapshot. A supplied name is created beneath +every shard's configured backup root: + +```bash +grpcurl -plaintext \ + -d '{"path":"backup-2026-07-24T1300Z"}' \ + 127.0.0.1:50060 unirust.RouterService/Checkpoint +``` + +Checkpoint creation uses a two-phase prepare/commit protocol. Every shard first +flushes its in-memory linker state and creates a RocksDB snapshot. Only after +all shards prepare successfully does the router write a binary commit marker to +every snapshot. The response includes the shared `generation` and +`committed: true`. A failed generation remains uncommitted and cannot be +restored; retrying the same name is idempotent and completes any missing +prepare or commit steps. Do not call the shard checkpoint RPC directly for a +production backup. + +The router scheduler waits one configured interval before its first checkpoint. +If a shard fails during prepare or commit, the scheduler retains and retries the +same immutable generation instead of creating a stream of unrelated partial +snapshots. Successful and failed generations are emitted to structured logs. +The container deployment enables hourly checkpoints by default; set +`UNIRUST_CHECKPOINT_INTERVAL_SECS` explicitly to choose another RPO or `0` to +disable them. + +To recover from lost data volumes, stop the complete cluster and restore every +shard from the same checkpoint generation into an empty replacement directory: + +```bash +unirust_shard \ + --shard-id 0 \ + --data-dir /replacement/shard-0 \ + --backup-dir /var/backups/unirust/shard-0 \ + --restore-from /var/backups/unirust/shard-0/backup-2026-07-24T1300Z \ + --ontology /etc/unirust/ontology.json +``` + +Restore refuses a non-RocksDB source, symlinks, a nonempty destination, and an +existing partial staging directory. It also requires matching binary +prepare/commit markers, verifies the checkpoint belongs to the requested shard, +and opens both the source and staged copy read-only with RocksDB paranoid checks. +It copies and syncs into a sibling staging directory before publishing the +replacement with one rename. Restore the whole cluster together; restoring +only one older shard beside newer peers can violate the cluster snapshot +boundary. Each shard retains the committed checkpoint provenance in its +replacement data directory and refuses a manifest for another shard. Router +startup requires every shard to be either unrestored or restored from the same +generation and shard count, then verifies the topology, ontology, and protocol +versions before accepting traffic. Mixed restored/unrestored volumes and +different committed generations fail closed. + +The built-in scheduler only creates local coordinated checkpoints. Unirust does +not replicate, encrypt, transfer, retain, or verify off-host backups. Without +storage-layer replication, the recovery point is the last completed coordinated +checkpoint, so acknowledged writes after it can be lost in a volume failure. +Production operators must provide off-host replication, retention, monitoring, +and periodic restore drills according to their RPO and RTO. Process crashes and +ordinary restarts remain covered by the synced ingest WAL independently of this +backup path. + +## Deployment Security + +The gRPC services do not currently terminate TLS or authenticate callers. +Production deployments must place the router and every shard on private +networks and enforce authenticated TLS with a service mesh or reverse proxy. +Never expose a shard port directly to an untrusted network. Destructive RPCs +remain disabled by default, but ingest, query, export, reconciliation, and other +administrative surfaces are otherwise unauthenticated. + +The shard binary requires a persistent `--data-dir`. An in-memory shard can only +be started with the explicit `--ephemeral` flag and loses all records when the +process exits. + ## Performance -Typical throughput on a 5-shard cluster (16 concurrent streams, batch size 5000): +Release verification on an Apple M5 with 32 GB RAM, five persistent shards, +16 concurrent streams, 5,000-record batches, and 10% overlap: -| Workload | Records/sec | Batch Latency | -|----------|-------------|---------------| -| Pure ingest | ~500K | 8ms | -| 10% overlap | ~410K | 12ms | -| With conflicts | ~300K | 16ms | +| Records | Records/sec | Stream Errors | +|---------|-------------|---------------| +| 10,000,000 | 50,598 | 0 | + +This is an end-to-end power-loss-durability measurement: records and cluster +assignments are persisted and the RocksDB WAL is synchronously flushed before +acknowledgement, and every record goes through entity resolution. Results depend +on storage hardware and ontology complexity; rerun the command below on the +release target rather than treating this number as a service-level guarantee. ## Development ```bash -# Run tests +# Fast correctness gate (unit and integration tests) cargo test +# Compile examples, binaries, and benchmarks without executing benchmarks +cargo check --all-targets --all-features + # Run quick benchmarks (~30s) cargo bench --bench bench_quick @@ -236,9 +439,25 @@ cargo bench --bench bench_quick # Format and lint cargo fmt -cargo clippy --all-targets +cargo clippy --all-targets --all-features -- -D warnings ``` +`cargo test --all-targets` executes Criterion benchmark binaries on some Cargo +versions and is intentionally not the default correctness gate. Run benchmarks +explicitly with `cargo bench --bench `. + +### Test Strategy + +- Unit tests cover temporal algebra, matching, DSU behavior, indexes, and focused + persistence failure modes. +- Integration tests exercise router and shard RPCs with temporary + `PersistentStore` databases, including restart, WAL, reconciliation, streaming, + reset, and rebalance behavior. +- `cargo check --all-targets --all-features` compiles examples and benchmarks + without mixing performance workloads into the correctness suite. +- `bench_quick` is the local performance smoke test. The distributed load test is + the release benchmark and must be run against persistent shards. + ## Container Deployment ```bash @@ -269,6 +488,9 @@ podman-compose logs -f router podman-compose run --rm loadtest # Stop and clean up +podman-compose down + +# Explicitly delete all persistent shard volumes podman-compose down -v ``` diff --git a/benches/bench_diagnostic.rs b/benches/bench_diagnostic.rs index 6c5f3d3..cac9c6b 100644 --- a/benches/bench_diagnostic.rs +++ b/benches/bench_diagnostic.rs @@ -87,7 +87,9 @@ fn diagnose_dsu_scaling() { /// Test index via full pipeline at different scales fn diagnose_index_scaling() { println!("--- Index Scaling via Pipeline Diagnostic ---"); - println!(" (Index performance is measured via full pipeline - see diagnose_full_pipeline for details)"); + println!( + " (Index performance is measured via full pipeline - see diagnose_full_pipeline for details)" + ); println!(); } diff --git a/benches/bench_distributed.rs b/benches/bench_distributed.rs index 8603e3b..6c0ffe1 100644 --- a/benches/bench_distributed.rs +++ b/benches/bench_distributed.rs @@ -387,7 +387,9 @@ fn bench_partitioned_ingest(c: &mut Criterion) { (partitioned, indexed) }, |(partitioned, indexed)| { - let results = partitioned.ingest_batch_with_partitions(indexed); + let results = partitioned + .ingest_batch_with_partitions(indexed) + .expect("partitioned ingest"); black_box(results); }, BatchSize::LargeInput, @@ -408,9 +410,11 @@ fn bench_shardnode_ingest(c: &mut Criterion) { group.warm_up_time(Duration::from_secs(1)); group.measurement_time(Duration::from_secs(6)); - std::env::set_var("UNIRUST_PARTITIONED", "1"); + // SAFETY: benchmark configuration runs before this function creates any worker threads. + unsafe { std::env::set_var("UNIRUST_PARTITIONED", "1") }; let partition_count = env_usize("UNIRUST_DIST_PARTITIONS", 8); - std::env::set_var("UNIRUST_PARTITION_COUNT", partition_count.to_string()); + // SAFETY: benchmark configuration runs before this function creates any worker threads. + unsafe { std::env::set_var("UNIRUST_PARTITION_COUNT", partition_count.to_string()) }; let count = env_u32("UNIRUST_DIST_RECORDS", 50_000); let overlap = env_f64("UNIRUST_DIST_OVERLAP", 0.05); @@ -430,7 +434,10 @@ fn bench_shardnode_ingest(c: &mut Criterion) { |(rt, shard, records)| { let response = rt.block_on(async { shard - .ingest_records(Request::new(proto::IngestRecordsRequest { records })) + .ingest_records(Request::new(proto::IngestRecordsRequest { + internal_protocol_version: 2, + records, + })) .await .expect("ingest") }); @@ -450,9 +457,11 @@ fn bench_shardnode_streaming_simulated(c: &mut Criterion) { group.warm_up_time(Duration::from_secs(1)); group.measurement_time(Duration::from_secs(6)); - std::env::set_var("UNIRUST_PARTITIONED", "1"); + // SAFETY: benchmark configuration runs before this function creates any worker threads. + unsafe { std::env::set_var("UNIRUST_PARTITIONED", "1") }; let partition_count = env_usize("UNIRUST_DIST_PARTITIONS", 8); - std::env::set_var("UNIRUST_PARTITION_COUNT", partition_count.to_string()); + // SAFETY: benchmark configuration runs before this function creates any worker threads. + unsafe { std::env::set_var("UNIRUST_PARTITION_COUNT", partition_count.to_string()) }; let total = env_u32("UNIRUST_DIST_STREAM_TOTAL", 20_000); let chunk = env_u32("UNIRUST_DIST_STREAM_CHUNK", 512).max(1); @@ -479,6 +488,7 @@ fn bench_shardnode_streaming_simulated(c: &mut Criterion) { let batch = records[offset..end].to_vec(); let resp = shard .ingest_records(Request::new(proto::IngestRecordsRequest { + internal_protocol_version: 2, records: batch, })) .await diff --git a/build.rs b/build.rs index 8286bd6..e4172c2 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,7 @@ fn main() -> Result<(), Box> { - tonic_build::configure() + tonic_prost_build::configure() .build_server(true) .build_client(true) - .compile(&["proto/unirust.proto"], &["proto"])?; + .compile_protos(&["proto/unirust.proto"], &["proto"])?; Ok(()) } diff --git a/compose.yaml b/compose.yaml index 0d74eff..a7db127 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,13 +5,13 @@ # podman-compose up -d # podman-compose logs -f router # -# Scale to more shards: -# SHARDS=5 podman-compose up -d --scale shard=5 -# # Run loadtest: -# podman-compose run --rm loadtest --endpoint http://router:50060 --count 100000 +# podman-compose run --rm loadtest --router http://router:50060 --count 100000 +# +# Stop cluster while preserving data: +# podman-compose down # -# Stop cluster: +# Explicitly delete all shard data: # podman-compose down -v services: @@ -21,15 +21,18 @@ services: build: context: . dockerfile: Containerfile - command: ["shard", "--shard-id", "0", "--data-dir", "/data"] volumes: - shard-0-data:/data + - shard-0-backup:/backup + - ./examples/loadtest-ontology.json:/etc/unirust/ontology.json:ro + command: + ["shard", "--shard-id", "0", "--data-dir", "/data", "--backup-dir", "/backup", "--ontology", "/etc/unirust/ontology.json"] environment: UNIRUST_PROFILE: high-throughput networks: - unirust-net healthcheck: - test: ["CMD-SHELL", "echo > /dev/tcp/localhost/50061 || exit 1"] + test: ["CMD", "unirust_healthcheck", "--shard", "http://127.0.0.1:50061"] interval: 5s timeout: 3s retries: 5 @@ -40,15 +43,18 @@ services: build: context: . dockerfile: Containerfile - command: ["shard", "--shard-id", "1", "--data-dir", "/data"] + command: + ["shard", "--shard-id", "1", "--data-dir", "/data", "--backup-dir", "/backup", "--ontology", "/etc/unirust/ontology.json"] volumes: - shard-1-data:/data + - shard-1-backup:/backup + - ./examples/loadtest-ontology.json:/etc/unirust/ontology.json:ro environment: UNIRUST_PROFILE: high-throughput networks: - unirust-net healthcheck: - test: ["CMD-SHELL", "echo > /dev/tcp/localhost/50061 || exit 1"] + test: ["CMD", "unirust_healthcheck", "--shard", "http://127.0.0.1:50061"] interval: 5s timeout: 3s retries: 5 @@ -59,15 +65,18 @@ services: build: context: . dockerfile: Containerfile - command: ["shard", "--shard-id", "2", "--data-dir", "/data"] + command: + ["shard", "--shard-id", "2", "--data-dir", "/data", "--backup-dir", "/backup", "--ontology", "/etc/unirust/ontology.json"] volumes: - shard-2-data:/data + - shard-2-backup:/backup + - ./examples/loadtest-ontology.json:/etc/unirust/ontology.json:ro environment: UNIRUST_PROFILE: high-throughput networks: - unirust-net healthcheck: - test: ["CMD-SHELL", "echo > /dev/tcp/localhost/50061 || exit 1"] + test: ["CMD", "unirust_healthcheck", "--shard", "http://127.0.0.1:50061"] interval: 5s timeout: 3s retries: 5 @@ -79,7 +88,18 @@ services: build: context: . dockerfile: Containerfile - command: ["router", "--shards", "shard-0:50061,shard-1:50061,shard-2:50061"] + command: + [ + "router", + "--shards", + "shard-0:50061,shard-1:50061,shard-2:50061", + "--ontology", + "/etc/unirust/ontology.json", + "--checkpoint-interval-secs", + "${UNIRUST_CHECKPOINT_INTERVAL_SECS:-3600}", + ] + volumes: + - ./examples/loadtest-ontology.json:/etc/unirust/ontology.json:ro ports: - "50060:50060" depends_on: @@ -92,7 +112,7 @@ services: networks: - unirust-net healthcheck: - test: ["CMD-SHELL", "echo > /dev/tcp/localhost/50060 || exit 1"] + test: ["CMD", "unirust_healthcheck", "--router", "http://127.0.0.1:50060"] interval: 5s timeout: 3s retries: 5 @@ -104,7 +124,8 @@ services: build: context: . dockerfile: Containerfile - command: ["loadtest", "--endpoint", "http://router:50060", "--count", "100000", "--streams", "8"] + command: + ["loadtest", "--router", "http://router:50060", "--count", "100000", "--streams", "8", "--headless"] depends_on: router: condition: service_healthy @@ -119,5 +140,8 @@ networks: volumes: shard-0-data: + shard-0-backup: shard-1-data: + shard-1-backup: shard-2-data: + shard-2-backup: diff --git a/examples/cluster.rs b/examples/cluster.rs index ca89b54..e05f59f 100644 --- a/examples/cluster.rs +++ b/examples/cluster.rs @@ -218,7 +218,10 @@ async fn main() -> anyhow::Result<()> { let start = std::time::Instant::now(); let response = client - .ingest_records(IngestRecordsRequest { records }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 0, + records, + }) .await? .into_inner(); diff --git a/examples/in_memory.rs b/examples/in_memory.rs deleted file mode 100644 index a0daf23..0000000 --- a/examples/in_memory.rs +++ /dev/null @@ -1,228 +0,0 @@ -//! # In-Memory Entity Resolution Example -//! -//! This example demonstrates unirust's core entity resolution capability -//! using only in-memory storage. Perfect for learning, testing, and -//! small datasets. -//! -//! ## What This Example Shows -//! -//! 1. Creating an ontology (matching rules) -//! 2. Ingesting records from multiple source systems -//! 3. Automatic entity clustering based on identity keys -//! 4. Conflict detection when records disagree -//! 5. Querying the unified view -//! -//! ## Run It -//! -//! ```bash -//! cargo run --example in_memory -//! ``` - -use unirust_rs::model::{Descriptor, Record, RecordId, RecordIdentity}; -use unirust_rs::ontology::{IdentityKey, Ontology, StrongIdentifier}; -use unirust_rs::temporal::Interval; -use unirust_rs::{QueryDescriptor, Unirust}; - -fn main() -> anyhow::Result<()> { - println!("╔══════════════════════════════════════════════════════════════╗"); - println!("║ Unirust In-Memory Entity Resolution ║"); - println!("╚══════════════════════════════════════════════════════════════╝\n"); - - // ========================================================================= - // Step 1: Create an Ontology - // ========================================================================= - // - // The ontology defines HOW records are matched: - // - Identity Keys: Attributes that, when matching, indicate same entity - // - Strong Identifiers: Unique attributes that prevent false merges - // - // Think of identity keys as "these records MIGHT be the same entity" - // and strong identifiers as "these records are DEFINITELY different if this differs" - - let mut ontology = Ontology::new(); - - // Identity Key: Records with same (name + email) are candidates for merging - // Using from_names() - no need to pre-intern attributes! - ontology.add_identity_key(IdentityKey::from_names(vec!["name", "email"], "name_email")); - - // Strong Identifier: SSN uniquely identifies a person - // If two records have different SSNs, they CANNOT be merged - ontology.add_strong_identifier(StrongIdentifier::from_name("ssn", "ssn_unique")); - - println!("Ontology configured:"); - println!(" - Identity Key: name + email (records match if both are equal)"); - println!(" - Strong ID: SSN (prevents merging if different)\n"); - - // Create the engine - ontology attributes are automatically interned - let mut engine = Unirust::new(ontology); - - // ========================================================================= - // Step 2: Create Records from Multiple Source Systems - // ========================================================================= - // - // Imagine we have data from three systems: CRM, ERP, and a Web app. - // Each record has: - // - RecordId: Unique identifier within unirust - // - RecordIdentity: Source system info (entity_type, perspective, uid) - // - Descriptors: Attribute values with temporal validity intervals - - // Intern attributes and values for use in records - let name_attr = engine.intern_attr("name"); - let email_attr = engine.intern_attr("email"); - let phone_attr = engine.intern_attr("phone"); - let ssn_attr = engine.intern_attr("ssn"); - - let john_name = engine.intern_value("John Doe"); - let john_email = engine.intern_value("john@example.com"); - let ssn_123 = engine.intern_value("123-45-6789"); - let phone_1 = engine.intern_value("555-1234"); - let phone_2 = engine.intern_value("555-5678"); // Different phone - will cause conflict! - - let jane_name = engine.intern_value("Jane Smith"); - let jane_email = engine.intern_value("jane@example.com"); - let ssn_987 = engine.intern_value("987-65-4321"); - let phone_3 = engine.intern_value("555-9999"); - - // Time intervals: [start, end) representing when data is valid - let interval_2022 = Interval::new(202201, 202301).unwrap(); - let interval_2023 = Interval::new(202301, 202401).unwrap(); - - println!("Creating records from multiple source systems...\n"); - - // Record 1: John from CRM (2022) - let john_crm = Record::new( - RecordId(1), - RecordIdentity::new("person".into(), "crm".into(), "CRM-001".into()), - vec![ - Descriptor::new(name_attr, john_name, interval_2022), - Descriptor::new(email_attr, john_email, interval_2022), - Descriptor::new(phone_attr, phone_1, interval_2022), - Descriptor::new(ssn_attr, ssn_123, interval_2022), - ], - ); - - // Record 2: John from ERP (2023) - Same person, different phone! - let john_erp = Record::new( - RecordId(2), - RecordIdentity::new("person".into(), "erp".into(), "ERP-001".into()), - vec![ - Descriptor::new(name_attr, john_name, interval_2023), - Descriptor::new(email_attr, john_email, interval_2023), - Descriptor::new(phone_attr, phone_2, interval_2023), // <-- Different phone - Descriptor::new(ssn_attr, ssn_123, interval_2023), - ], - ); - - // Record 3: Jane from CRM - Different person entirely - let jane_crm = Record::new( - RecordId(3), - RecordIdentity::new("person".into(), "crm".into(), "CRM-002".into()), - vec![ - Descriptor::new(name_attr, jane_name, interval_2022), - Descriptor::new(email_attr, jane_email, interval_2022), - Descriptor::new(phone_attr, phone_3, interval_2022), - Descriptor::new(ssn_attr, ssn_987, interval_2022), - ], - ); - - // ========================================================================= - // Step 3: Ingest Records - // ========================================================================= - - let result = engine.ingest(vec![john_crm, john_erp, jane_crm])?; - - println!("Ingestion Results:"); - println!(" - Records processed: {}", result.assignments.len()); - println!(" - Clusters formed: {}", result.cluster_count); - println!(" - Conflicts detected: {}", result.conflicts.len()); - println!(); - - // Show cluster assignments - println!("Cluster Assignments:"); - for assignment in &result.assignments { - println!( - " Record {} -> Cluster {}", - assignment.record_id.0, assignment.cluster_id.0 - ); - } - println!(); - - // ========================================================================= - // Step 4: Examine Conflicts - // ========================================================================= - - if !result.conflicts.is_empty() { - println!("Conflicts Detected:"); - for conflict in &result.conflicts { - let attr_name = conflict.attribute.as_deref().unwrap_or("unknown"); - let cause = conflict.cause.as_deref().unwrap_or("value mismatch"); - println!( - " - {} conflict at {}: {}", - attr_name, conflict.interval, cause - ); - println!(" Records involved: {:?}", conflict.records); - } - println!(); - } - - // ========================================================================= - // Step 5: Query the Unified View - // ========================================================================= - - println!("Querying for 'John Doe' with email 'john@example.com'..."); - - let query_name = engine.intern_value("John Doe"); - let query_email = engine.intern_value("john@example.com"); - - let query_result = engine.query( - &[ - QueryDescriptor { - attr: name_attr, - value: query_name, - }, - QueryDescriptor { - attr: email_attr, - value: query_email, - }, - ], - Interval::new(202201, 202401).unwrap(), - )?; - - match query_result { - unirust_rs::QueryOutcome::Matches(matches) => { - if matches.is_empty() { - println!(" No matches found"); - } else { - println!(" Found {} matching cluster(s):", matches.len()); - for m in &matches { - println!( - " Cluster {} at interval {} with {} golden attributes", - m.cluster_id.0, - m.interval, - m.golden.len() - ); - } - } - } - unirust_rs::QueryOutcome::Conflict(conflict) => { - println!(" Query resulted in conflict: {:?}", conflict); - } - } - - // ========================================================================= - // Step 6: Get Statistics - // ========================================================================= - - println!("\nEngine Statistics:"); - let stats = engine.stats(); - println!(" - Total records: {}", stats.record_count); - println!(" - Total clusters: {}", stats.cluster_count); - println!(" - Merges performed: {}", stats.merges_performed); - - println!("\n✓ Example completed successfully!"); - println!("\nNext steps:"); - println!(" - Try 'cargo run --example persistent_shard' for persistence"); - println!(" - Try 'cargo run --example cluster' for distributed mode"); - - Ok(()) -} diff --git a/examples/loadtest-ontology.json b/examples/loadtest-ontology.json new file mode 100644 index 0000000..6139e33 --- /dev/null +++ b/examples/loadtest-ontology.json @@ -0,0 +1,55 @@ +{ + "identity_keys": [ + { + "name": "user_upn_key", + "attributes": ["user_upn"] + }, + { + "name": "user_identity", + "attributes": ["user_sid"] + }, + { + "name": "session_identity", + "attributes": ["session_id"] + }, + { + "name": "process_identity", + "attributes": ["process_id", "asset_id"] + }, + { + "name": "connection_identity", + "attributes": ["flow_id"] + }, + { + "name": "asset_hostname", + "attributes": ["hostname"] + }, + { + "name": "asset_identity", + "attributes": ["asset_id"] + }, + { + "name": "vuln_identity", + "attributes": ["cve_id", "asset_id"] + }, + { + "name": "alert_identity", + "attributes": ["alert_id"] + }, + { + "name": "file_identity", + "attributes": ["file_hash"] + } + ], + "strong_identifiers": [ + "user_sid", + "session_id", + "process_hash", + "flow_id", + "asset_id", + "cve_id", + "alert_id", + "file_hash" + ], + "constraints": [] +} diff --git a/examples/persistent_shard.rs b/examples/persistent_shard.rs deleted file mode 100644 index 64fee87..0000000 --- a/examples/persistent_shard.rs +++ /dev/null @@ -1,237 +0,0 @@ -//! # Persistent Single-Shard Example -//! -//! This example demonstrates unirust with persistent storage using RocksDB. -//! Data survives restarts and can handle larger datasets than fit in memory. -//! -//! ## What This Example Shows -//! -//! 1. Opening a persistent store -//! 2. Ingesting records with durability -//! 3. Checkpointing for crash recovery -//! 4. Restarting and recovering state -//! 5. Performance tuning profiles -//! -//! ## Run It -//! -//! ```bash -//! # First run - creates data -//! cargo run --example persistent_shard -//! -//! # Second run - recovers from disk -//! cargo run --example persistent_shard -//! ``` - -use tempfile::tempdir; -use unirust_rs::model::{Descriptor, Record, RecordId, RecordIdentity}; -use unirust_rs::ontology::{IdentityKey, Ontology, StrongIdentifier}; -use unirust_rs::temporal::Interval; -use unirust_rs::{PersistentStore, StreamingTuning, TuningProfile, Unirust}; - -fn main() -> anyhow::Result<()> { - println!("╔══════════════════════════════════════════════════════════════╗"); - println!("║ Unirust Persistent Single-Shard Example ║"); - println!("╚══════════════════════════════════════════════════════════════╝\n"); - - // For this example, we use a temp directory. In production, use a fixed path. - // let data_dir = PathBuf::from("/var/lib/unirust/shard-0"); - let temp_dir = tempdir()?; - let data_dir = temp_dir.path().to_path_buf(); - - println!("Data directory: {}\n", data_dir.display()); - - // ========================================================================= - // Step 1: Open Persistent Store - // ========================================================================= - // - // PersistentStore uses RocksDB under the hood, providing: - // - ACID transactions - // - Crash recovery - // - Efficient compression - // - Background compaction - - println!("Opening persistent store..."); - let store = PersistentStore::open(&data_dir)?; - println!(" Store opened successfully\n"); - - // ========================================================================= - // Step 2: Choose a Tuning Profile - // ========================================================================= - // - // Tuning profiles optimize for different workloads: - // - // | Profile | Best For | - // |---------------|---------------------------------------| - // | Balanced | General purpose, moderate loads | - // | LowLatency | Interactive queries, fast responses | - // | HighThroughput| Batch processing, maximum ingest rate | - // | BulkIngest | Initial data loading, minimal matching| - // | MemorySaver | Memory-constrained environments | - // | BillionScale | Huge datasets with persistent DSU | - - let profile = TuningProfile::HighThroughput; - let tuning = StreamingTuning::from_profile(profile); - println!("Using tuning profile: {:?}\n", profile); - - // ========================================================================= - // Step 3: Configure Ontology - // ========================================================================= - // - // Using from_names() - no need to pre-intern attributes! - // The engine will automatically intern them when created. - - let mut ontology = Ontology::new(); - - ontology.add_identity_key(IdentityKey::from_names(vec!["name", "email"], "name_email")); - - ontology.add_strong_identifier(StrongIdentifier::from_name("ssn", "ssn_unique")); - - println!("Ontology configured with identity key (name+email) and strong ID (ssn)\n"); - - // ========================================================================= - // Step 4: Create Engine with Store and Ontology - // ========================================================================= - // - // The engine automatically interns the ontology's attribute names. - - let mut engine = Unirust::with_store_and_tuning(ontology, store, tuning); - - // Intern attributes for use in records - let name_attr = engine.intern_attr("name"); - let email_attr = engine.intern_attr("email"); - let phone_attr = engine.intern_attr("phone"); - let ssn_attr = engine.intern_attr("ssn"); - let dept_attr = engine.intern_attr("department"); - - // ========================================================================= - // Step 5: Generate and Ingest Records - // ========================================================================= - - println!("Generating sample records..."); - - // Simulate records from multiple departments - let departments = ["Engineering", "Sales", "Marketing", "Support"]; - let mut records = Vec::new(); - - // Create 100 records across departments - for i in 0..100 { - let dept = departments[i % departments.len()]; - let name_val = engine.intern_value(&format!("Employee {}", i)); - let email_val = engine.intern_value(&format!("emp{}@company.com", i)); - let phone_val = engine.intern_value(&format!("555-{:04}", i)); - let ssn_val = engine.intern_value(&format!("{:03}-{:02}-{:04}", i / 100, i % 100, i)); - let dept_val = engine.intern_value(dept); - - let interval = Interval::new(202401, 202501).unwrap(); - - records.push(Record::new( - RecordId(i as u32), - RecordIdentity::new( - "employee".into(), - "hr_system".into(), - format!("HR-{:04}", i), - ), - vec![ - Descriptor::new(name_attr, name_val, interval), - Descriptor::new(email_attr, email_val, interval), - Descriptor::new(phone_attr, phone_val, interval), - Descriptor::new(ssn_attr, ssn_val, interval), - Descriptor::new(dept_attr, dept_val, interval), - ], - )); - } - - // Add some duplicate records from a different system (will be merged) - for i in 0..10 { - let name_val = engine.intern_value(&format!("Employee {}", i)); - let email_val = engine.intern_value(&format!("emp{}@company.com", i)); - let phone_val = engine.intern_value(&format!("555-{:04}", 1000 + i)); // Different phone - let ssn_val = engine.intern_value(&format!("{:03}-{:02}-{:04}", i / 100, i % 100, i)); - - let interval = Interval::new(202406, 202501).unwrap(); - - records.push(Record::new( - RecordId(100 + i as u32), - RecordIdentity::new( - "employee".into(), - "payroll_system".into(), - format!("PAY-{:04}", i), - ), - vec![ - Descriptor::new(name_attr, name_val, interval), - Descriptor::new(email_attr, email_val, interval), - Descriptor::new(phone_attr, phone_val, interval), - Descriptor::new(ssn_attr, ssn_val, interval), - ], - )); - } - - println!(" Created {} records\n", records.len()); - - println!("Ingesting records..."); - let start = std::time::Instant::now(); - let result = engine.ingest(records)?; - let elapsed = start.elapsed(); - - println!("Ingestion complete:"); - println!(" - Records: {}", result.assignments.len()); - println!(" - Clusters: {}", result.cluster_count); - println!(" - Conflicts: {}", result.conflicts.len()); - println!(" - Time: {:?}", elapsed); - println!( - " - Rate: {:.0} records/sec\n", - result.assignments.len() as f64 / elapsed.as_secs_f64() - ); - - // ========================================================================= - // Step 6: Checkpoint for Crash Recovery - // ========================================================================= - // - // Checkpointing ensures all data is durably written to disk. - // Call this periodically in production to limit data loss on crash. - - println!("Creating checkpoint..."); - engine.checkpoint()?; - println!(" Checkpoint complete - data is durable\n"); - - // ========================================================================= - // Step 7: Query and Statistics - // ========================================================================= - - println!("Engine Statistics:"); - let stats = engine.stats(); - println!(" - Total records: {}", stats.record_count); - println!(" - Total clusters: {}", stats.cluster_count); - println!(" - Merges performed: {}", stats.merges_performed); - println!(" - Conflicts detected: {}", stats.conflicts_detected); - - // Show conflicts (phone number changes between systems) - if !result.conflicts.is_empty() { - println!("\nConflicts detected (phone changes between systems):"); - for (i, conflict) in result.conflicts.iter().take(5).enumerate() { - let attr = conflict.attribute.as_deref().unwrap_or("unknown"); - let cause = conflict.cause.as_deref().unwrap_or("value mismatch"); - println!(" {}. {} at {}: {}", i + 1, attr, conflict.interval, cause); - } - if result.conflicts.len() > 5 { - println!(" ... and {} more", result.conflicts.len() - 5); - } - } - - // ========================================================================= - // Step 8: Demonstrate Recovery (Simulated) - // ========================================================================= - - println!("\n--- Simulating Restart ---"); - println!("In production, you would:"); - println!(" 1. Close the engine (drop it)"); - println!(" 2. Restart your application"); - println!(" 3. Re-open with PersistentStore::open()"); - println!(" 4. All data would be recovered automatically"); - - println!("\n✓ Example completed successfully!"); - println!("\nNext steps:"); - println!(" - Try 'cargo run --example cluster' for distributed mode"); - println!(" - Use a fixed data_dir path for real persistence"); - - Ok(()) -} diff --git a/examples/unirust.toml b/examples/unirust.toml index e09ad4d..1a0be98 100644 --- a/examples/unirust.toml +++ b/examples/unirust.toml @@ -17,6 +17,8 @@ listen = "0.0.0.0:50061" id = 0 # Data directory for persistence (comment out for in-memory only) # data_dir = "/var/lib/unirust/shard-0" +# Checkpoint root on an independent volume (use a unique path per shard) +# backup_dir = "/var/backups/unirust/shard-0" # Path to ontology configuration (JSON) # ontology = "/etc/unirust/ontology.json" # Run repair on startup (recovers from unclean shutdown) @@ -36,6 +38,12 @@ shards = [ # shards_file = "/etc/unirust/shards.txt" # Path to ontology configuration (JSON) # ontology = "/etc/unirust/ontology.json" +# Router-to-shard failure bounds +shard_connect_timeout_secs = 10 +shard_request_timeout_secs = 120 +shard_tcp_keepalive_secs = 30 +# Automatic coordinated checkpoint interval (0 disables) +# checkpoint_interval_secs = 3600 # Advanced storage tuning (RocksDB) # These settings control the underlying storage engine performance. diff --git a/proto/unirust.proto b/proto/unirust.proto index e801df9..1c94ad9 100644 --- a/proto/unirust.proto +++ b/proto/unirust.proto @@ -33,6 +33,7 @@ message RecordInput { message IngestRecordsRequest { repeated RecordInput records = 1; + uint32 internal_protocol_version = 2; // Router-to-shard only; clients leave this unset. } message IngestAssignment { @@ -258,16 +259,60 @@ message ConfigVersionRequest {} message ConfigVersionResponse { string version = 1; + OntologyConfig ontology_config = 2; + uint32 protocol_version = 3; + uint32 source_reservation_backfill_version = 4; + uint32 source_reservation_shard_count = 5; + uint32 checkpoint_protocol_version = 6; + // Empty/zero for a volume that has never been restored from a checkpoint. + string restore_generation = 7; + uint32 restore_shard_count = 8; } +// --- Durable Source Identity Reservation --- + +message SourceRecordReservation { + uint32 index = 1; + RecordIdentity identity = 2; + bytes payload_digest = 3; + uint32 target_shard_id = 4; +} + +message ReserveSourceRecordsRequest { + repeated SourceRecordReservation reservations = 1; +} + +message SourceRecordReservationResult { + uint32 index = 1; + uint32 target_shard_id = 2; +} + +message ReserveSourceRecordsResponse { + repeated SourceRecordReservationResult reservations = 1; +} + +message MarkSourceReservationsBackfilledRequest { + uint32 protocol_version = 1; + uint32 shard_count = 2; +} + +message MarkSourceReservationsBackfilledResponse {} + // --- Checkpoint --- message CheckpointRequest { string path = 1; + // Internal router-to-shard coordination fields. External clients call the + // router with these left at their zero values. + uint32 checkpoint_protocol_version = 2; + uint32 shard_count = 3; + bool finalize = 4; } message CheckpointResponse { repeated string paths = 1; + string generation = 2; + bool committed = 3; } // --- Metrics --- @@ -336,10 +381,12 @@ message ExportRecordsChunk { message ImportRecordsRequest { repeated RecordSnapshot records = 1; + uint32 internal_protocol_version = 2; } message ImportRecordsChunk { repeated RecordSnapshot records = 1; + uint32 internal_protocol_version = 2; } message ImportRecordsResponse { @@ -374,6 +421,14 @@ message IdentityKeySignature { bytes signature = 1; // 32-byte hash } +message BoundaryStrongId { + string perspective = 1; + string attribute = 2; + string value = 3; + int64 interval_start = 4; + int64 interval_end = 5; +} + message ClusterBoundaryEntry { GlobalClusterId cluster_id = 1; int64 interval_start = 2; @@ -382,7 +437,10 @@ message ClusterBoundaryEntry { // Strong ID hashes per perspective for cross-shard conflict detection. // Key: perspective name hash, Value: hash of strong ID values. // Two clusters conflict if they share a perspective but have different values. + // Deprecated: retained for rolling compatibility with pre-0.2 nodes. map perspective_strong_ids = 5; + // Exact temporal observations used for authoritative merge guards. + repeated BoundaryStrongId strong_ids = 6; } message BoundaryMetadata { @@ -481,6 +539,7 @@ message StoreCrossShardConflictsResponse { message IngestRecordsChunk { repeated RecordInput records = 1; + uint32 internal_protocol_version = 2; } // ============================================================================= @@ -493,6 +552,8 @@ service ShardService { rpc SetOntology(ApplyOntologyRequest) returns (ApplyOntologyResponse); // Core operations + rpc ReserveSourceRecords(ReserveSourceRecordsRequest) returns (ReserveSourceRecordsResponse); + rpc MarkSourceReservationsBackfilled(MarkSourceReservationsBackfilledRequest) returns (MarkSourceReservationsBackfilledResponse); rpc IngestRecords(IngestRecordsRequest) returns (IngestRecordsResponse); rpc IngestRecordsStream(stream IngestRecordsChunk) returns (IngestRecordsResponse); rpc IngestRecordsFromUrl(IngestRecordsFromUrlRequest) returns (IngestRecordsResponse); diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..8153a3d --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +style_edition = "2021" diff --git a/scripts/cluster.sh b/scripts/cluster.sh index f096cb9..58af550 100755 --- a/scripts/cluster.sh +++ b/scripts/cluster.sh @@ -9,37 +9,46 @@ SHARDS="${SHARDS:-3}" ROUTER_PORT="${ROUTER_PORT:-50060}" SHARD_PORT_BASE="${SHARD_PORT_BASE:-50061}" DATA_DIR="${DATA_DIR:-$ROOT_DIR/cluster_data}" +BACKUP_DIR="${BACKUP_DIR:-$ROOT_DIR/cluster_backups}" LOG_DIR="${LOG_DIR:-$ROOT_DIR/cluster_logs}" RUN_DIR="${RUN_DIR:-$ROOT_DIR/.cluster}" -ONTOLOGY="${ONTOLOGY:-}" +ONTOLOGY="${ONTOLOGY:-$ROOT_DIR/examples/loadtest-ontology.json}" CONFIG_VERSION="${CONFIG_VERSION:-}" +CHECKPOINT_INTERVAL_SECS="${CHECKPOINT_INTERVAL_SECS:-0}" PROFILE="${PROFILE:-high-throughput}" REPAIR="${REPAIR:-0}" CARGO_FEATURES="${CARGO_FEATURES:-}" SHARD_WAIT_SECS="${SHARD_WAIT_SECS:-10}" +STOP_WAIT_SECS="${STOP_WAIT_SECS:-10}" +UNIRUST_CONFIRM_RESET="${UNIRUST_CONFIRM_RESET:-0}" usage() { cat </dev/null 2>&1 || return 1 + + local command + command="$(ps -p "$pid" -o command= 2>/dev/null || true)" + [[ "$command" == *"$expected_binary"* ]] +} + +assert_cluster_stopped() { + if pid_matches "$RUN_DIR/router.pid" "unirust_router"; then + echo "Router is already running. Use '$0 restart' or '$0 stop' first." + return 1 + fi + + local pid_file + for pid_file in "$RUN_DIR"/shard-*.pid; do + [[ -f "$pid_file" ]] || continue + if pid_matches "$pid_file" "unirust_shard"; then + echo "$(basename "$pid_file" .pid) is already running. Use '$0 restart' or '$0 stop' first." + return 1 + fi + done +} + +wait_for_health() { + local service="$1" + local endpoint="$2" local timeout_secs="$3" local start start="$(date +%s)" while true; do - if (echo >/dev/tcp/"$host"/"$port") >/dev/null 2>&1; then + if "$BIN_DIR/unirust_healthcheck" "$service" "$endpoint" >/dev/null 2>&1; then return 0 fi if [[ $(( $(date +%s) - start )) -ge "$timeout_secs" ]]; then @@ -78,17 +118,19 @@ wait_for_port() { } start_cluster() { + assert_cluster_stopped build_bins - remove_data_dir - mkdir -p "$DATA_DIR" "$LOG_DIR" "$RUN_DIR" + mkdir -p "$DATA_DIR" "$BACKUP_DIR" "$LOG_DIR" "$RUN_DIR" local shard_list="" local shard_ports=() for i in $(seq 0 $((SHARDS - 1))); do local port=$((SHARD_PORT_BASE + i)) local shard_dir="$DATA_DIR/shard-$i" + local backup_dir="$BACKUP_DIR/shard-$i" mkdir -p "$shard_dir" - local shard_args=(--listen "127.0.0.1:${port}" --shard-id "$i" --data-dir "$shard_dir") + mkdir -p "$backup_dir" + local shard_args=(--listen "127.0.0.1:${port}" --shard-id "$i" --data-dir "$shard_dir" --backup-dir "$backup_dir") if [[ -n "$ONTOLOGY" ]]; then shard_args+=(--ontology "$ONTOLOGY") fi @@ -116,9 +158,10 @@ start_cluster() { done for port in "${shard_ports[@]}"; do - if ! wait_for_port "127.0.0.1" "$port" "$SHARD_WAIT_SECS"; then - echo "Shard on port ${port} failed to start within ${SHARD_WAIT_SECS}s." - exit 1 + if ! wait_for_health --shard "http://127.0.0.1:${port}" "$SHARD_WAIT_SECS"; then + echo "Shard on port ${port} did not become healthy within ${SHARD_WAIT_SECS}s." + stop_cluster + return 1 fi done @@ -129,35 +172,77 @@ start_cluster() { if [[ -n "$CONFIG_VERSION" ]]; then router_args+=(--config-version "$CONFIG_VERSION") fi + router_args+=(--checkpoint-interval-secs "$CHECKPOINT_INTERVAL_SECS") "$BIN_DIR/unirust_router" "${router_args[@]}" >"$LOG_DIR/router.log" 2>&1 & echo $! >"$RUN_DIR/router.pid" + if ! wait_for_health --router "http://127.0.0.1:${ROUTER_PORT}" "$SHARD_WAIT_SECS"; then + echo "Router on port ${ROUTER_PORT} did not become healthy within ${SHARD_WAIT_SECS}s." + stop_cluster + return 1 + fi + echo "Cluster started with ${SHARDS} shards." echo "Router listening on 127.0.0.1:${ROUTER_PORT}" + echo "Persistent data directory: $DATA_DIR" +} + +stop_pid_file() { + local pid_file="$1" + local expected_binary="$2" + [[ -f "$pid_file" ]] || return 0 + + if ! pid_matches "$pid_file" "$expected_binary"; then + echo "Removing stale or mismatched PID file: $pid_file" + rm -f "$pid_file" + return 0 + fi + + local pid + read -r pid <"$pid_file" + kill -TERM "$pid" >/dev/null 2>&1 || true + + local attempts=$((STOP_WAIT_SECS * 10)) + for ((attempt = 0; attempt < attempts; attempt++)); do + if ! kill -0 "$pid" >/dev/null 2>&1; then + rm -f "$pid_file" + return 0 + fi + sleep 0.1 + done + + echo "$expected_binary did not stop within ${STOP_WAIT_SECS}s; sending SIGKILL." + kill -KILL "$pid" >/dev/null 2>&1 || true + rm -f "$pid_file" } stop_cluster() { - pkill -f "/unirust_shard" >/dev/null 2>&1 || true - pkill -f "/unirust_router" >/dev/null 2>&1 || true if [[ ! -d "$RUN_DIR" ]]; then echo "No PID directory found." return 0 fi - for pid_file in "$RUN_DIR"/shard-*.pid "$RUN_DIR"/router.pid; do - if [[ -f "$pid_file" ]]; then - pid="$(cat "$pid_file")" - if ps -p "$pid" >/dev/null 2>&1; then - kill "$pid" >/dev/null 2>&1 || true - fi - rm -f "$pid_file" - fi + stop_pid_file "$RUN_DIR/router.pid" "unirust_router" + local pid_file + for pid_file in "$RUN_DIR"/shard-*.pid; do + [[ -f "$pid_file" ]] || continue + stop_pid_file "$pid_file" "unirust_shard" done echo "Cluster stopped." } +reset_cluster() { + assert_cluster_stopped + if [[ "$UNIRUST_CONFIRM_RESET" != "1" ]]; then + echo "Refusing to delete persistent data without UNIRUST_CONFIRM_RESET=1." + return 1 + fi + remove_data_dir + echo "Deleted persistent data directory: $DATA_DIR" +} + status_cluster() { if [[ ! -d "$RUN_DIR" ]]; then echo "No PID directory found." @@ -165,17 +250,21 @@ status_cluster() { fi local found=0 - for pid_file in "$RUN_DIR"/shard-*.pid "$RUN_DIR"/router.pid; do - if [[ -f "$pid_file" ]]; then - pid="$(cat "$pid_file")" - if ps -p "$pid" >/dev/null 2>&1; then - echo "$(basename "$pid_file" .pid) running (pid $pid)" - found=1 - else - echo "$(basename "$pid_file" .pid) not running (stale pid $pid)" - found=1 - fi + local pid_file + for pid_file in "$RUN_DIR"/router.pid "$RUN_DIR"/shard-*.pid; do + [[ -f "$pid_file" ]] || continue + local expected_binary="unirust_shard" + if [[ "$(basename "$pid_file")" == "router.pid" ]]; then + expected_binary="unirust_router" + fi + local pid + read -r pid <"$pid_file" || pid="invalid" + if pid_matches "$pid_file" "$expected_binary"; then + echo "$(basename "$pid_file" .pid) running (pid $pid)" + else + echo "$(basename "$pid_file" .pid) not running (stale or mismatched pid $pid)" fi + found=1 done if [[ "$found" -eq 0 ]]; then @@ -190,12 +279,19 @@ case "$ACTION" in start) start_cluster ;; + restart) + stop_cluster + start_cluster + ;; stop) stop_cluster ;; status) status_cluster ;; + reset) + reset_cluster + ;; *) usage exit 1 diff --git a/scripts/podman_cluster.sh b/scripts/podman_cluster.sh index 2008c11..71e6983 100755 --- a/scripts/podman_cluster.sh +++ b/scripts/podman_cluster.sh @@ -1,11 +1,18 @@ #!/usr/bin/env bash set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" IMAGE="${IMAGE:-unirust-distributed}" NETWORK="${NETWORK:-unirust-net}" SHARDS="${SHARDS:-2}" ROUTER_PORT="${ROUTER_PORT:-50060}" -ONTOLOGY="${ONTOLOGY:-}" +ONTOLOGY="${ONTOLOGY:-$ROOT_DIR/examples/loadtest-ontology.json}" +CONFIG_VERSION="${CONFIG_VERSION:-}" +PROFILE="${PROFILE:-high-throughput}" +CONTAINER_PREFIX="${CONTAINER_PREFIX:-unirust}" +VOLUME_PREFIX="${VOLUME_PREFIX:-unirust-data}" +STOP_WAIT_SECS="${STOP_WAIT_SECS:-10}" +UNIRUST_CONFIRM_RESET="${UNIRUST_CONFIRM_RESET:-0}" action="${1:-start}" @@ -21,29 +28,52 @@ start_cluster() { podman network create "$NETWORK" fi - podman rm -f unirust-router >/dev/null 2>&1 || true + if podman container exists "${CONTAINER_PREFIX}-router"; then + echo "Cluster containers already exist. Use '$0 restart' or '$0 stop' first." + return 1 + fi for i in $(seq 0 $((SHARDS - 1))); do - podman rm -f "unirust-shard-$i" >/dev/null 2>&1 || true + if podman container exists "${CONTAINER_PREFIX}-shard-$i"; then + echo "Cluster containers already exist. Use '$0 restart' or '$0 stop' first." + return 1 + fi done - for i in $(seq 0 $((SHARDS - 1))); do - ontology_args=() - if [[ -n "$ONTOLOGY" ]]; then - ontology_args=(--ontology "$ONTOLOGY") + ontology_mount=() + ontology_args=() + if [[ -n "$ONTOLOGY" ]]; then + if [[ ! -f "$ONTOLOGY" ]]; then + echo "Ontology file does not exist: $ONTOLOGY" + return 1 fi + ontology_mount=(-v "$ONTOLOGY:/etc/unirust/ontology.json:ro") + ontology_args=(--ontology /etc/unirust/ontology.json) + fi + + version_args=() + if [[ -n "$CONFIG_VERSION" ]]; then + version_args=(--config-version "$CONFIG_VERSION") + fi + + for i in $(seq 0 $((SHARDS - 1))); do podman run -d \ - --name "unirust-shard-$i" \ + --name "${CONTAINER_PREFIX}-shard-$i" \ --network "$NETWORK" \ + --restart=unless-stopped \ + -v "${VOLUME_PREFIX}-shard-$i:/data" \ + "${ontology_mount[@]}" \ "$IMAGE" \ - unirust_shard \ - --listen "0.0.0.0:50061" \ + shard \ --shard-id "$i" \ - "${ontology_args[@]}" + --data-dir /data \ + --profile "$PROFILE" \ + "${ontology_args[@]}" \ + "${version_args[@]}" done shard_list="" for i in $(seq 0 $((SHARDS - 1))); do - entry="unirust-shard-$i:50061" + entry="${CONTAINER_PREFIX}-shard-$i:50061" if [[ -z "$shard_list" ]]; then shard_list="$entry" else @@ -51,34 +81,63 @@ start_cluster() { fi done - ontology_args=() - if [[ -n "$ONTOLOGY" ]]; then - ontology_args=(--ontology "$ONTOLOGY") - fi podman run -d \ - --name unirust-router \ + --name "${CONTAINER_PREFIX}-router" \ --network "$NETWORK" \ + --restart=unless-stopped \ -p "$ROUTER_PORT:50060" \ + "${ontology_mount[@]}" \ "$IMAGE" \ - unirust_router \ - --listen "0.0.0.0:50060" \ + router \ --shards "$shard_list" \ - "${ontology_args[@]}" + "${ontology_args[@]}" \ + "${version_args[@]}" echo "Cluster started." echo "Router listening on localhost:${ROUTER_PORT}" + echo "Persistent volumes use prefix: ${VOLUME_PREFIX}-shard-" } stop_cluster() { - podman rm -f -t 3 unirust-router >/dev/null 2>&1 || true - for container in $(podman ps -a --format "{{.Names}}" | grep -E "^unirust-shard-" || true); do - podman rm -f -t 3 "$container" >/dev/null 2>&1 || true + containers=("${CONTAINER_PREFIX}-router") + while IFS= read -r container; do + [[ -n "$container" ]] && containers+=("$container") + done < <( + podman ps -a --format "{{.Names}}" | + while IFS= read -r name; do + [[ "$name" == "${CONTAINER_PREFIX}-shard-"* ]] && printf '%s\n' "$name" + done + ) + + for container in "${containers[@]}"; do + if podman container exists "$container"; then + podman stop --time "$STOP_WAIT_SECS" "$container" >/dev/null 2>&1 || true + podman rm "$container" >/dev/null 2>&1 || true + fi done echo "Cluster stopped." } +reset_cluster() { + if [[ "$UNIRUST_CONFIRM_RESET" != "1" ]]; then + echo "Refusing to delete persistent volumes without UNIRUST_CONFIRM_RESET=1." + return 1 + fi + stop_cluster + while IFS= read -r volume; do + if [[ "$volume" == "${VOLUME_PREFIX}-shard-"* ]]; then + podman volume rm "$volume" + fi + done < <(podman volume ls --format "{{.Name}}") + echo "Deleted persistent volumes with prefix: ${VOLUME_PREFIX}-shard-" +} + status_cluster() { - podman ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -E "unirust-(router|shard)" + podman ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | + while IFS= read -r line; do + [[ "$line" == NAMES* || "$line" == "${CONTAINER_PREFIX}-router"* || "$line" == "${CONTAINER_PREFIX}-shard-"* ]] && + printf '%s\n' "$line" + done } case "$action" in @@ -88,14 +147,21 @@ case "$action" in start) start_cluster ;; + restart) + stop_cluster + start_cluster + ;; stop) stop_cluster ;; status) status_cluster ;; + reset) + reset_cluster + ;; *) - echo "Usage: $0 [build|start|stop|status]" + echo "Usage: $0 [build|start|restart|stop|status|reset]" exit 1 ;; esac diff --git a/src/bin/unirust_client.rs b/src/bin/unirust_client.rs index f15d8c5..2e07fbd 100644 --- a/src/bin/unirust_client.rs +++ b/src/bin/unirust_client.rs @@ -1,5 +1,7 @@ use std::fs; +use std::path::PathBuf; +use tonic::transport::Endpoint; use unirust_rs::distributed::proto::router_service_client::RouterServiceClient; use unirust_rs::distributed::proto::{ ApplyOntologyRequest, ConstraintConfig, ConstraintKind, IdentityKeyConfig, @@ -7,6 +9,7 @@ use unirust_rs::distributed::proto::{ RecordIdentity, RecordInput, }; use unirust_rs::distributed::DistributedOntologyConfig; +use unirust_rs::transport_security::load_client_mtls; fn parse_arg(flag: &str) -> Option { let mut args = std::env::args(); @@ -23,7 +26,23 @@ async fn main() -> anyhow::Result<()> { let router = parse_arg("--router").unwrap_or_else(|| "http://127.0.0.1:50060".to_string()); let ontology_path = parse_arg("--ontology").ok_or_else(|| anyhow::anyhow!("--ontology is required"))?; - let mut client = RouterServiceClient::connect(router).await?; + let tls_ca = parse_arg("--tls-ca").map(PathBuf::from); + let tls_cert = parse_arg("--tls-cert").map(PathBuf::from); + let tls_key = parse_arg("--tls-key").map(PathBuf::from); + let mtls = load_client_mtls(tls_ca.as_deref(), tls_cert.as_deref(), tls_key.as_deref())?; + if mtls.is_some() && router.starts_with("http://") { + anyhow::bail!("mTLS requires an https:// router address"); + } + if mtls.is_none() && router.starts_with("https://") { + anyhow::bail!("https:// router addresses require mTLS certificate options"); + } + let endpoint = Endpoint::from_shared(router)?; + let endpoint = if let Some(mtls) = mtls { + endpoint.tls_config(mtls)? + } else { + endpoint + }; + let mut client = RouterServiceClient::new(endpoint.connect().await?); let ontology = load_ontology(ontology_path)?; client @@ -80,7 +99,10 @@ async fn main() -> anyhow::Result<()> { ]; let response = client - .ingest_records(IngestRecordsRequest { records }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 0, + records, + }) .await? .into_inner(); println!("Assignments: {:?}", response.assignments); diff --git a/src/bin/unirust_healthcheck.rs b/src/bin/unirust_healthcheck.rs new file mode 100644 index 0000000..296617d --- /dev/null +++ b/src/bin/unirust_healthcheck.rs @@ -0,0 +1,172 @@ +use std::path::PathBuf; +use std::time::Duration; + +use tonic::transport::Endpoint; +use unirust_rs::distributed::proto::{ + self, router_service_client::RouterServiceClient, shard_service_client::ShardServiceClient, +}; +use unirust_rs::transport_security::load_client_mtls; + +enum Service { + Router(String), + Shard(String), +} + +struct Options { + service: Service, + timeout: Duration, + ca_cert: Option, + client_cert: Option, + client_key: Option, +} + +fn print_help() { + eprintln!( + r#"unirust_healthcheck - Unirust gRPC readiness probe + +USAGE: + unirust_healthcheck (--router | --shard ) [--timeout-secs ] + +OPTIONS: + --router Check a router and all of its shards + --shard Check one shard's recovery and store health + --timeout-secs Connection and RPC timeout [default: 2] + --ca-cert PEM CA used to verify the service certificate + --client-cert + PEM client certificate + --client-key PEM client private key + -h, --help Print help +"# + ); +} + +fn parse_options() -> anyhow::Result> { + let mut service = None; + let mut timeout_secs = 2u64; + let mut ca_cert = None; + let mut client_cert = None; + let mut client_key = None; + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "-h" | "--help" => { + print_help(); + return Ok(None); + } + "--router" => { + let uri = args + .next() + .ok_or_else(|| anyhow::anyhow!("--router requires a URI"))?; + if service.is_some() { + anyhow::bail!("exactly one of --router or --shard is required"); + } + service = Some(Service::Router(uri)); + } + "--shard" => { + let uri = args + .next() + .ok_or_else(|| anyhow::anyhow!("--shard requires a URI"))?; + if service.is_some() { + anyhow::bail!("exactly one of --router or --shard is required"); + } + service = Some(Service::Shard(uri)); + } + "--timeout-secs" => { + timeout_secs = args + .next() + .ok_or_else(|| anyhow::anyhow!("--timeout-secs requires a value"))? + .parse()?; + if timeout_secs == 0 { + anyhow::bail!("--timeout-secs must be greater than zero"); + } + } + "--ca-cert" => { + ca_cert = Some( + args.next() + .ok_or_else(|| anyhow::anyhow!("--ca-cert requires a path"))? + .into(), + ); + } + "--client-cert" => { + client_cert = Some( + args.next() + .ok_or_else(|| anyhow::anyhow!("--client-cert requires a path"))? + .into(), + ); + } + "--client-key" => { + client_key = Some( + args.next() + .ok_or_else(|| anyhow::anyhow!("--client-key requires a path"))? + .into(), + ); + } + _ => anyhow::bail!("unknown option {arg}"), + } + } + + let service = + service.ok_or_else(|| anyhow::anyhow!("exactly one of --router or --shard is required"))?; + Ok(Some(Options { + service, + timeout: Duration::from_secs(timeout_secs), + ca_cert, + client_cert, + client_key, + })) +} + +fn normalize_uri(uri: String, mtls: bool) -> anyhow::Result { + if uri.starts_with("http://") || uri.starts_with("https://") { + if mtls && uri.starts_with("http://") { + anyhow::bail!("mTLS health checks require an https:// endpoint"); + } + if !mtls && uri.starts_with("https://") { + anyhow::bail!("https:// health checks require CA, client certificate, and client key"); + } + Ok(uri) + } else if mtls { + Ok(format!("https://{uri}")) + } else { + Ok(format!("http://{uri}")) + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let Some(options) = parse_options()? else { + return Ok(()); + }; + let mtls = load_client_mtls( + options.ca_cert.as_deref(), + options.client_cert.as_deref(), + options.client_key.as_deref(), + )?; + let uri = match &options.service { + Service::Router(uri) | Service::Shard(uri) => normalize_uri(uri.clone(), mtls.is_some())?, + }; + let endpoint = Endpoint::from_shared(uri)? + .connect_timeout(options.timeout) + .timeout(options.timeout); + let endpoint = if let Some(mtls) = mtls { + endpoint.tls_config(mtls)? + } else { + endpoint + }; + let channel = endpoint.connect().await?; + let response = match options.service { + Service::Router(_) => RouterServiceClient::new(channel) + .health_check(proto::HealthCheckRequest {}) + .await? + .into_inner(), + Service::Shard(_) => ShardServiceClient::new(channel) + .health_check(proto::HealthCheckRequest {}) + .await? + .into_inner(), + }; + if response.status != "ok" { + anyhow::bail!("service reported unhealthy status: {}", response.status); + } + println!("ok"); + Ok(()) +} diff --git a/src/bin/unirust_loadtest.rs b/src/bin/unirust_loadtest.rs index 775f0c0..a21d51e 100644 --- a/src/bin/unirust_loadtest.rs +++ b/src/bin/unirust_loadtest.rs @@ -21,7 +21,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; -use anyhow::Result; +use anyhow::{bail, Result}; use crossterm::{ event::{self, Event, KeyCode, KeyEventKind}, execute, @@ -37,7 +37,7 @@ use ratatui::{ Frame, Terminal, }; use tokio::sync::RwLock; -use tonic::transport::Channel; +use tonic::transport::{Channel, Endpoint}; use unirust_rs::distributed::proto::router_service_client::RouterServiceClient; use unirust_rs::distributed::proto::shard_service_client::ShardServiceClient; @@ -45,6 +45,7 @@ use unirust_rs::distributed::proto::{ ApplyOntologyRequest, IdentityKeyConfig, IngestRecordsRequest, MetricsRequest, OntologyConfig, RecordDescriptor, RecordIdentity, RecordInput, StatsRequest, }; +use unirust_rs::transport_security::load_client_mtls; // ============================================================================= // CONFIGURATION @@ -63,6 +64,9 @@ pub struct LoadTestConfig { pub log_file: Option, pub seed: u64, pub headless: bool, + pub tls_ca: Option, + pub tls_cert: Option, + pub tls_key: Option, } impl Default for LoadTestConfig { @@ -79,10 +83,49 @@ impl Default for LoadTestConfig { log_file: None, seed: 42, headless: false, + tls_ca: None, + tls_cert: None, + tls_key: None, } } } +impl LoadTestConfig { + fn validate(&self) -> Result<()> { + if self.stream_count == 0 { + bail!("--streams must be greater than zero"); + } + if self.batch_size == 0 { + bail!("--batch must be greater than zero"); + } + for (name, value) in [ + ("--overlap", self.overlap_probability), + ("--conflict", self.conflict_probability), + ("--cross-shard", self.cross_shard_probability), + ] { + if !(0.0..=1.0).contains(&value) { + bail!("{name} must be between 0.0 and 1.0"); + } + } + let tls_paths = [ + self.tls_ca.as_ref(), + self.tls_cert.as_ref(), + self.tls_key.as_ref(), + ]; + let configured = tls_paths.iter().filter(|path| path.is_some()).count(); + if configured != 0 && configured != tls_paths.len() { + bail!("--tls-ca, --tls-cert, and --tls-key must be provided together"); + } + if configured > 0 && self.router_addr.starts_with("http://") { + bail!("mTLS load tests require an https:// router address"); + } + if configured == 0 && self.router_addr.starts_with("https://") { + bail!("https:// router addresses require mTLS certificate options"); + } + Ok(()) + } +} + fn parse_args() -> LoadTestConfig { let mut config = LoadTestConfig::default(); let mut args = std::env::args().skip(1); @@ -140,6 +183,15 @@ fn parse_args() -> LoadTestConfig { "--headless" => { config.headless = true; } + "--tls-ca" => { + config.tls_ca = args.next().map(PathBuf::from); + } + "--tls-cert" => { + config.tls_cert = args.next().map(PathBuf::from); + } + "--tls-key" => { + config.tls_key = args.next().map(PathBuf::from); + } "--help" | "-h" => { print_help(); std::process::exit(0); @@ -159,7 +211,7 @@ USAGE: unirust_loadtest [OPTIONS] OPTIONS: - -c, --count Total entities to generate (default: 20000000) + -c, --count Total records to generate (default: 20000000) -r, --router Router gRPC address (default: http://127.0.0.1:50060) -s, --shards Comma-separated shard addresses for direct metrics --streams Number of concurrent streams (default: 32) @@ -172,10 +224,13 @@ OPTIONS: --log Log file path (logs to file only) --seed Random seed for reproducibility (default: 42) --headless Run without TUI (for automation/CI) + --tls-ca PEM CA used to verify router/shard certificates + --tls-cert PEM client certificate + --tls-key PEM client private key -h, --help Print help EXAMPLES: - # Basic test with 100K entities + # Basic test with 100K records unirust_loadtest --count 100000 --router http://127.0.0.1:50060 # Large scale with custom parallelism @@ -372,7 +427,6 @@ fn build_ontology_config() -> OntologyConfig { /// Seed data for creating overlapping entities struct UserSeed { - uid: String, user_sid: String, user_upn: String, employee_id: String, @@ -381,7 +435,6 @@ struct UserSeed { } struct AssetSeed { - uid: String, asset_id: String, hostname: String, ip_address: String, @@ -391,7 +444,6 @@ struct AssetSeed { /// Generic entity seed for other types struct EntitySeed { - uid: String, #[allow(dead_code)] identity_attr: String, identity_value: String, @@ -443,7 +495,7 @@ impl CyberEntityGenerator { ) -> Self { let total_weight: u32 = EntityType::all().iter().map(|e| e.weight()).sum(); // Pool size determines cluster density: smaller pool = denser clusters - // For ~10 entities per cluster with 10% overlap, use pool_size ~= total_entities / 100 + // For ~10 records per cluster with 10% overlap, use pool_size ~= total_records / 100 let pool_size = 10_000; // Supports up to 1M entities with ~10 per cluster at 10% overlap Self { @@ -569,7 +621,7 @@ impl CyberEntityGenerator { } /// Generate a user entity. Returns (uid, descriptors, perspective_override). - /// When overlapping, reuses UID from pool to create clusters. + /// When overlapping, reuses identity keys from the pool to create clusters. /// When conflicting, uses same perspective and different strong identifier. /// When cross-shard, uses different user_upn (partitioning key) but same user_sid /// (secondary identity key) to create cross-shard merge candidates. @@ -614,7 +666,6 @@ impl CyberEntityGenerator { } else if is_overlap { let idx = self.rng.random_range(0..self.user_pool.len()); // Clone seed values upfront to avoid borrow conflicts - let seed_uid = self.user_pool[idx].uid.clone(); let seed_user_sid = self.user_pool[idx].user_sid.clone(); let seed_user_upn = self.user_pool[idx].user_upn.clone(); let seed_employee_id = self.user_pool[idx].employee_id.clone(); @@ -640,9 +691,9 @@ impl CyberEntityGenerator { Some(seed_perspective), // SAME perspective - required for conflict detection ) } else { - // Exact overlap - same cluster, no conflict + // Same identity keys link a distinct source record into the cluster. ( - seed_uid, + self.next_uid(), seed_user_sid, seed_user_upn, seed_employee_id, @@ -661,7 +712,6 @@ impl CyberEntityGenerator { if self.user_pool.len() < self.pool_size { self.user_pool.push(UserSeed { - uid: uid.clone(), user_sid: user_sid.clone(), user_upn: user_upn.clone(), employee_id: employee_id.clone(), @@ -716,14 +766,13 @@ impl CyberEntityGenerator { let (uid, session_id) = if is_overlap { let idx = self.rng.random_range(0..self.session_pool.len()); - let seed = &self.session_pool[idx]; - (seed.uid.clone(), seed.identity_value.clone()) + let session_id = self.session_pool[idx].identity_value.clone(); + (self.next_uid(), session_id) } else { let uid = self.next_uid(); let session_id = format!("SES-{:016X}", self.rng.random::()); if self.session_pool.len() < self.pool_size { self.session_pool.push(EntitySeed { - uid: uid.clone(), identity_attr: "session_id".to_string(), identity_value: session_id.clone(), }); @@ -778,14 +827,13 @@ impl CyberEntityGenerator { let (uid, process_hash) = if is_overlap { let idx = self.rng.random_range(0..self.process_pool.len()); - let seed = &self.process_pool[idx]; - (seed.uid.clone(), seed.identity_value.clone()) + let process_hash = self.process_pool[idx].identity_value.clone(); + (self.next_uid(), process_hash) } else { let uid = self.next_uid(); let process_hash = format!("{:064x}", self.rng.random::()); if self.process_pool.len() < self.pool_size { self.process_pool.push(EntitySeed { - uid: uid.clone(), identity_attr: "process_hash".to_string(), identity_value: process_hash.clone(), }); @@ -853,14 +901,13 @@ impl CyberEntityGenerator { let (uid, flow_id) = if is_overlap { let idx = self.rng.random_range(0..self.connection_pool.len()); - let seed = &self.connection_pool[idx]; - (seed.uid.clone(), seed.identity_value.clone()) + let flow_id = self.connection_pool[idx].identity_value.clone(); + (self.next_uid(), flow_id) } else { let uid = self.next_uid(); let flow_id = format!("FLOW-{:016X}", self.rng.random::()); if self.connection_pool.len() < self.pool_size { self.connection_pool.push(EntitySeed { - uid: uid.clone(), identity_attr: "flow_id".to_string(), identity_value: flow_id.clone(), }); @@ -938,7 +985,6 @@ impl CyberEntityGenerator { let (uid, asset_id, hostname, ip_address, os_type, perspective_override) = if is_overlap { let idx = self.rng.random_range(0..self.asset_pool.len()); // Clone seed values upfront to avoid borrow conflicts - let seed_uid = self.asset_pool[idx].uid.clone(); let seed_asset_id = self.asset_pool[idx].asset_id.clone(); let seed_hostname = self.asset_pool[idx].hostname.clone(); let seed_ip_address = self.asset_pool[idx].ip_address.clone(); @@ -960,7 +1006,7 @@ impl CyberEntityGenerator { ) } else { ( - seed_uid, + self.next_uid(), seed_asset_id, seed_hostname, seed_ip_address, @@ -990,7 +1036,6 @@ impl CyberEntityGenerator { if self.asset_pool.len() < self.pool_size { self.asset_pool.push(AssetSeed { - uid: uid.clone(), asset_id: asset_id.clone(), hostname: hostname.clone(), ip_address: ip_address.clone(), @@ -1052,26 +1097,17 @@ impl CyberEntityGenerator { let (uid, cve_id, severity, status) = if is_overlap { let idx = self.rng.random_range(0..self.vuln_pool.len()); - let seed = &self.vuln_pool[idx]; + let cve_id = self.vuln_pool[idx].identity_value.clone(); + let uid = self.next_uid(); if is_conflict { let severities = ["Critical", "High", "Medium", "Low", "Info"]; let new_severity = severities[self.rng.random_range(0..severities.len())].to_string(); let statuses = ["Open", "In Progress", "Remediated", "Accepted"]; let new_status = statuses[self.rng.random_range(0..statuses.len())].to_string(); - ( - seed.uid.clone(), - seed.identity_value.clone(), - new_severity, - new_status, - ) + (uid, cve_id, new_severity, new_status) } else { - ( - seed.uid.clone(), - seed.identity_value.clone(), - "Medium".to_string(), - "Open".to_string(), - ) + (uid, cve_id, "Medium".to_string(), "Open".to_string()) } } else { let uid = self.next_uid(); @@ -1087,7 +1123,6 @@ impl CyberEntityGenerator { if self.vuln_pool.len() < self.pool_size { self.vuln_pool.push(EntitySeed { - uid: uid.clone(), identity_attr: "cve_id".to_string(), identity_value: cve_id.clone(), }); @@ -1143,14 +1178,13 @@ impl CyberEntityGenerator { let (uid, alert_id) = if is_overlap { let idx = self.rng.random_range(0..self.alert_pool.len()); - let seed = &self.alert_pool[idx]; - (seed.uid.clone(), seed.identity_value.clone()) + let alert_id = self.alert_pool[idx].identity_value.clone(); + (self.next_uid(), alert_id) } else { let uid = self.next_uid(); let alert_id = format!("ALERT-{:016X}", self.rng.random::()); if self.alert_pool.len() < self.pool_size { self.alert_pool.push(EntitySeed { - uid: uid.clone(), identity_attr: "alert_id".to_string(), identity_value: alert_id.clone(), }); @@ -1211,14 +1245,13 @@ impl CyberEntityGenerator { let (uid, file_hash) = if is_overlap { let idx = self.rng.random_range(0..self.file_pool.len()); - let seed = &self.file_pool[idx]; - (seed.uid.clone(), seed.identity_value.clone()) + let file_hash = self.file_pool[idx].identity_value.clone(); + (self.next_uid(), file_hash) } else { let uid = self.next_uid(); let file_hash = format!("{:064x}", self.rng.random::()); if self.file_pool.len() < self.pool_size { self.file_pool.push(EntitySeed { - uid: uid.clone(), identity_attr: "file_hash".to_string(), identity_value: file_hash.clone(), }); @@ -1288,6 +1321,8 @@ pub struct StreamStats { pub records_sent: AtomicU64, pub records_acked: AtomicU64, pub last_latency_micros: AtomicU64, + pub latency_total_micros: AtomicU64, + pub successful_batches: AtomicU64, pub errors: AtomicU64, } @@ -1311,9 +1346,9 @@ pub struct ShardStats { pub struct LoadTestMetrics { pub start_time: Instant, - pub total_entities: u64, - pub entities_sent: AtomicU64, - pub entities_acked: AtomicU64, + pub total_records: u64, + pub records_sent: AtomicU64, + pub records_acked: AtomicU64, pub cluster_count: AtomicU64, pub conflicts_detected: AtomicU64, pub merges_performed: AtomicU64, @@ -1328,7 +1363,7 @@ pub struct LoadTestMetrics { } impl LoadTestMetrics { - pub fn new(stream_count: usize, shard_count: usize, total_entities: u64) -> Self { + pub fn new(stream_count: usize, shard_count: usize, total_records: u64) -> Self { let stream_stats = (0..stream_count).map(|_| StreamStats::default()).collect(); let shard_stats = (0..shard_count) .map(|i| ShardStats { @@ -1339,9 +1374,9 @@ impl LoadTestMetrics { Self { start_time: Instant::now(), - total_entities, - entities_sent: AtomicU64::new(0), - entities_acked: AtomicU64::new(0), + total_records, + records_sent: AtomicU64::new(0), + records_acked: AtomicU64::new(0), cluster_count: AtomicU64::new(0), conflicts_detected: AtomicU64::new(0), merges_performed: AtomicU64::new(0), @@ -1360,16 +1395,16 @@ impl LoadTestMetrics { } pub fn progress(&self) -> f64 { - let acked = self.entities_acked.load(Ordering::Relaxed); - if self.total_entities == 0 { + let acked = self.records_acked.load(Ordering::Relaxed); + if self.total_records == 0 { 1.0 } else { - acked as f64 / self.total_entities as f64 + acked as f64 / self.total_records as f64 } } pub fn throughput(&self) -> f64 { - let acked = self.entities_acked.load(Ordering::Relaxed); + let acked = self.records_acked.load(Ordering::Relaxed); let elapsed = self.elapsed().as_secs_f64(); if elapsed > 0.0 { acked as f64 / elapsed @@ -1379,8 +1414,8 @@ impl LoadTestMetrics { } pub fn eta(&self) -> Option { - let acked = self.entities_acked.load(Ordering::Relaxed); - let remaining = self.total_entities.saturating_sub(acked); + let acked = self.records_acked.load(Ordering::Relaxed); + let remaining = self.total_records.saturating_sub(acked); let throughput = self.throughput(); if throughput > 0.0 && remaining > 0 { Some(Duration::from_secs_f64(remaining as f64 / throughput)) @@ -1389,12 +1424,40 @@ impl LoadTestMetrics { } } + fn successful_batches(&self) -> u64 { + self.stream_stats + .iter() + .map(|stream| stream.successful_batches.load(Ordering::Relaxed)) + .sum() + } + + fn average_request_latency_millis(&self) -> f64 { + let total_micros: u64 = self + .stream_stats + .iter() + .map(|stream| stream.latency_total_micros.load(Ordering::Relaxed)) + .sum(); + let batches = self.successful_batches(); + if batches == 0 { + 0.0 + } else { + total_micros as f64 / batches as f64 / 1000.0 + } + } + + fn request_errors(&self) -> u64 { + self.stream_stats + .iter() + .map(|stream| stream.errors.load(Ordering::Relaxed)) + .sum() + } + /// Generate a detailed report for analysis pub fn generate_report(&self, config: &LoadTestConfig) -> String { let elapsed = self.elapsed(); let elapsed_secs = elapsed.as_secs_f64(); - let entities_sent = self.entities_sent.load(Ordering::Relaxed); - let entities_acked = self.entities_acked.load(Ordering::Relaxed); + let records_sent = self.records_sent.load(Ordering::Relaxed); + let records_acked = self.records_acked.load(Ordering::Relaxed); let cluster_count = self.cluster_count.load(Ordering::Relaxed); let conflicts = self.conflicts_detected.load(Ordering::Relaxed); let throughput = self.throughput(); @@ -1441,18 +1504,14 @@ impl LoadTestMetrics { // Summary report.push_str("## Summary\n"); - report.push_str(&format!(" Entities sent: {}\n", entities_sent)); - report.push_str(&format!(" Entities acked: {}\n", entities_acked)); + report.push_str(&format!(" Records sent: {}\n", records_sent)); + report.push_str(&format!(" Records acked: {}\n", records_acked)); report.push_str(&format!(" Clusters: {}\n", cluster_count)); report.push_str(&format!(" Conflicts: {}\n", conflicts)); report.push_str(&format!(" Throughput: {:.2} rec/sec\n", throughput)); report.push_str(&format!( - " Avg latency: {:.2} ms/batch\n\n", - if throughput > 0.0 { - (config.batch_size as f64 / throughput) * 1000.0 - } else { - 0.0 - } + " Avg RPC latency: {:.2} ms/request\n\n", + self.average_request_latency_millis() )); // Cross-shard reconciliation stats @@ -1572,20 +1631,16 @@ impl LoadTestMetrics { // Performance metrics report.push_str("## Performance Analysis\n"); - let entities_per_stream = entities_acked as f64 / config.stream_count as f64; - let batches_total = entities_acked as f64 / config.batch_size as f64; - let batch_latency_avg = if batches_total > 0.0 { - elapsed_secs * 1000.0 / batches_total - } else { - 0.0 - }; - report.push_str(&format!(" Total batches: {:.0}\n", batches_total)); + let records_per_stream = records_acked as f64 / config.stream_count as f64; + let batches_total = self.successful_batches(); + let batch_latency_avg = self.average_request_latency_millis(); + report.push_str(&format!(" Successful batches: {}\n", batches_total)); report.push_str(&format!( - " Entities/stream: {:.0}\n", - entities_per_stream + " Records/stream: {:.0}\n", + records_per_stream )); report.push_str(&format!( - " Batch latency avg: {:.2} ms\n", + " RPC latency avg: {:.2} ms\n", batch_latency_avg )); report.push_str(&format!( @@ -1602,7 +1657,7 @@ impl LoadTestMetrics { report.push_str(" - Check disk I/O with iostat\n"); } if batch_latency_avg > 500.0 { - report.push_str(" ⚠ Batch latency > 500ms - processing bottleneck\n"); + report.push_str(" ⚠ RPC latency > 500ms - processing bottleneck\n"); report.push_str(" - Check conflict detection overhead\n"); report.push_str(" - Consider profiling with UNIRUST_PROFILE=1\n"); } @@ -1655,6 +1710,40 @@ fn write_report(config: &LoadTestConfig, metrics: &LoadTestMetrics) -> Result Result { + let mtls = load_client_mtls( + config.tls_ca.as_deref(), + config.tls_cert.as_deref(), + config.tls_key.as_deref(), + )?; + let address = if address.starts_with("http://") || address.starts_with("https://") { + address + } else if mtls.is_some() { + format!("https://{address}") + } else { + format!("http://{address}") + }; + if mtls.is_some() && address.starts_with("http://") { + bail!("mTLS requires an https:// endpoint"); + } + if mtls.is_none() && address.starts_with("https://") { + bail!("https:// endpoints require mTLS certificate options"); + } + let endpoint = Endpoint::from_shared(address)?; + let endpoint = if let Some(mtls) = mtls { + endpoint.tls_config(mtls)? + } else { + endpoint + }; + Ok(endpoint.connect().await?) +} + +async fn connect_router(config: &LoadTestConfig) -> Result> { + Ok(RouterServiceClient::new( + connect_channel(config, config.router_addr.clone()).await?, + )) +} + async fn run_parallel_streams( config: &LoadTestConfig, metrics: Arc, @@ -1663,7 +1752,6 @@ async fn run_parallel_streams( // Generator task let gen_config = config.clone(); - let gen_metrics = metrics.clone(); let gen_tx = tx.clone(); let generator_handle = tokio::spawn(async move { let mut generator = CyberEntityGenerator::new( @@ -1678,10 +1766,6 @@ async fn run_parallel_streams( let batch_size = gen_config.batch_size.min(remaining as usize); let batch = generator.generate_batch(batch_size); remaining -= batch_size as u64; - gen_metrics - .entities_sent - .fetch_add(batch_size as u64, Ordering::Relaxed); - if gen_tx.send(batch).await.is_err() { break; // Channel closed } @@ -1693,10 +1777,10 @@ async fn run_parallel_streams( .map(|i| { let rx = rx.clone(); let metrics = metrics.clone(); - let router = config.router_addr.clone(); + let connection_config = config.clone(); tokio::spawn(async move { - let mut client = match RouterServiceClient::connect(router).await { + let mut client = match connect_router(&connection_config).await { Ok(c) => c, Err(e) => { tracing::error!(stream = i, error = %e, "Failed to connect to router"); @@ -1712,7 +1796,7 @@ async fn run_parallel_streams( metrics.stream_stats[i] .records_sent .fetch_add(batch_len as u64, Ordering::Relaxed); - metrics.entities_sent.fetch_add(batch_len as u64, Ordering::Relaxed); + metrics.records_sent.fetch_add(batch_len as u64, Ordering::Relaxed); let start = Instant::now(); // Retry with exponential backoff @@ -1730,21 +1814,38 @@ async fn run_parallel_streams( match client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 0, records: batch.clone(), }) .await { Ok(response) => { let count = response.into_inner().assignments.len(); + if count != batch_len { + tracing::error!( + stream = i, + expected = batch_len, + acknowledged = count, + "Ingest returned an incomplete assignment set" + ); + metrics.stream_stats[i] + .errors + .fetch_add(1, Ordering::Relaxed); + } metrics.stream_stats[i] .records_acked .fetch_add(count as u64, Ordering::Relaxed); - metrics - .entities_acked - .fetch_add(count as u64, Ordering::Relaxed); + metrics.records_acked.fetch_add(count as u64, Ordering::Relaxed); + let latency_micros = start.elapsed().as_micros() as u64; metrics.stream_stats[i] .last_latency_micros - .store(start.elapsed().as_micros() as u64, Ordering::Relaxed); + .store(latency_micros, Ordering::Relaxed); + metrics.stream_stats[i] + .latency_total_micros + .fetch_add(latency_micros, Ordering::Relaxed); + metrics.stream_stats[i] + .successful_batches + .fetch_add(1, Ordering::Relaxed); success = true; break; } @@ -1761,16 +1862,16 @@ async fn run_parallel_streams( metrics.stream_stats[i] .errors .fetch_add(1, Ordering::Relaxed); - // Still count for progress tracking - metrics - .entities_acked - .fetch_add(batch_len as u64, Ordering::Relaxed); } } }) }) .collect(); + // If every connection fails, dropping the original receiver lets the + // generator observe the closed channel instead of blocking forever. + drop(rx); + // Wait for generator to finish let _ = generator_handle.await; @@ -1798,16 +1899,15 @@ async fn poll_metrics_task( let mut interval = tokio::time::interval(Duration::from_millis(500)); // Connect to router - let mut router_client = RouterServiceClient::connect(config.router_addr.clone()) - .await - .ok(); + let mut router_client = connect_router(config).await.ok(); // Connect to shards let mut shard_clients: Vec>> = Vec::new(); for addr in &config.shard_addrs { - let client = ShardServiceClient::connect(format!("http://{}", addr)) + let client = connect_channel(config, addr.clone()) .await - .ok(); + .ok() + .map(ShardServiceClient::new); shard_clients.push(client); } @@ -1828,7 +1928,7 @@ async fn poll_metrics_task( // Poll individual shards for detailed metrics for (i, client_opt) in shard_clients.iter_mut().enumerate() { - if let Some(ref mut client) = client_opt { + if let Some(client) = client_opt { if let Ok(response) = client.get_metrics(MetricsRequest {}).await { let m = response.into_inner(); if i < metrics.shard_stats.len() { @@ -1839,9 +1939,11 @@ async fn poll_metrics_task( if let Some(latency) = m.ingest_latency { shard.ingest_latency_total.store(latency.total_micros, Ordering::Relaxed); shard.ingest_latency_max.store(latency.max_micros, Ordering::Relaxed); - if latency.count > 0 { + if let Some(average) = + latency.total_micros.checked_div(latency.count) + { shard.ingest_latency_avg.store( - latency.total_micros / latency.count, + average, Ordering::Relaxed, ); } @@ -1970,7 +2072,7 @@ fn draw_ui(frame: &mut Frame, config: &LoadTestConfig, metrics: &LoadTestMetrics // Progress let progress = metrics.progress(); - let acked = metrics.entities_acked.load(Ordering::Relaxed); + let acked = metrics.records_acked.load(Ordering::Relaxed); let eta_str = metrics .eta() .map(|d| format!("ETA: {}", format_duration(d))) @@ -1996,17 +2098,10 @@ fn draw_ui(frame: &mut Frame, config: &LoadTestConfig, metrics: &LoadTestMetrics .split(chunks[3]); let throughput = metrics.throughput(); - let avg_latency: u64 = metrics - .stream_stats - .iter() - .map(|s| s.last_latency_micros.load(Ordering::Relaxed)) - .sum::() - / metrics.stream_stats.len().max(1) as u64; - let throughput_text = format!( "Records/sec: {:.0}\nAvg Latency: {:.2} ms", throughput, - avg_latency as f64 / 1000.0 + metrics.average_request_latency_millis() ); let throughput_para = Paragraph::new(throughput_text) .style(Style::default().fg(Color::Green)) @@ -2038,13 +2133,14 @@ fn draw_ui(frame: &mut Frame, config: &LoadTestConfig, metrics: &LoadTestMetrics .iter() .enumerate() .map(|(i, s)| { + let sent = s.records_sent.load(Ordering::Relaxed); let acked = s.records_acked.load(Ordering::Relaxed); let latency = s.last_latency_micros.load(Ordering::Relaxed); let errors = s.errors.load(Ordering::Relaxed); Row::new(vec![ format!("#{}", i), - format!("{} sent", format_number(acked)), + format!("{} sent", format_number(sent)), format!("{} ack", format_number(acked)), format!("{:.1}ms", latency as f64 / 1000.0), if errors > 0 { @@ -2167,7 +2263,7 @@ async fn run_headless( shutdown_tx: tokio::sync::watch::Sender, ) -> Result<()> { println!( - "Running in headless mode - streaming {} entities...", + "Running in headless mode - streaming {} records...", config.count ); let start = Instant::now(); @@ -2178,8 +2274,8 @@ async fn run_headless( // Print progress every 5 seconds if last_print.elapsed() >= Duration::from_secs(5) { - let sent = metrics.entities_sent.load(Ordering::Relaxed); - let acked = metrics.entities_acked.load(Ordering::Relaxed); + let sent = metrics.records_sent.load(Ordering::Relaxed); + let acked = metrics.records_acked.load(Ordering::Relaxed); let elapsed = start.elapsed().as_secs_f64(); let rate = if elapsed > 0.0 { acked as f64 / elapsed @@ -2214,6 +2310,7 @@ async fn run_headless( #[tokio::main] async fn main() -> Result<()> { let config = parse_args(); + config.validate()?; // Setup file logging if specified if let Some(log_path) = &config.log_file { @@ -2236,7 +2333,7 @@ async fn main() -> Result<()> { // Set ontology on router first { - let mut client = RouterServiceClient::connect(config.router_addr.clone()).await?; + let mut client = connect_router(&config).await?; client .set_ontology(ApplyOntologyRequest { config: Some(build_ontology_config()), @@ -2251,6 +2348,7 @@ async fn main() -> Result<()> { if let Err(e) = run_parallel_streams(&stream_config, stream_metrics.clone()).await { tracing::error!(error = %e, "Streaming failed"); *stream_metrics.error_message.write().await = Some(e.to_string()); + stream_metrics.completed.store(true, Ordering::Relaxed); } }); @@ -2274,8 +2372,117 @@ async fn main() -> Result<()> { let _ = streaming_handle.await; let _ = metrics_handle.await; - // Generate and save report - write_report(&config, &metrics)?; + // Always preserve the diagnostic report, even when the run is incomplete. + let report_path = write_report(&config, &metrics)?; + if let Some(error) = metrics.error_message.read().await.clone() { + bail!( + "load test failed: {error}; report: {}", + report_path.display() + ); + } + + let sent = metrics.records_sent.load(Ordering::Relaxed); + let acked = metrics.records_acked.load(Ordering::Relaxed); + let errors = metrics.request_errors(); + if errors > 0 || sent != config.count || acked != config.count { + bail!( + "load test incomplete: sent {sent}/{}, acknowledged {acked}/{}, request errors {errors}; report: {}", + config.count, + config.count, + report_path.display() + ); + } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + #[test] + fn report_uses_single_record_counters_and_measured_rpc_latency() { + let config = LoadTestConfig { + count: 100, + stream_count: 1, + batch_size: 50, + ..LoadTestConfig::default() + }; + let metrics = LoadTestMetrics::new(1, 0, config.count); + metrics.records_sent.store(100, Ordering::Relaxed); + metrics.records_acked.store(90, Ordering::Relaxed); + metrics.stream_stats[0] + .successful_batches + .store(2, Ordering::Relaxed); + metrics.stream_stats[0] + .latency_total_micros + .store(5_000, Ordering::Relaxed); + + let report = metrics.generate_report(&config); + + assert!(report.contains("Records sent: 100")); + assert!(report.contains("Records acked: 90")); + assert!(report.contains("Avg RPC latency: 2.50 ms/request")); + assert!(report.contains("Successful batches: 2")); + assert_eq!(metrics.progress(), 0.9); + } + + #[test] + fn config_rejects_values_that_cannot_make_progress() { + let mut config = LoadTestConfig { + batch_size: 0, + ..LoadTestConfig::default() + }; + assert!(config.validate().is_err()); + + config.batch_size = 5_000; + config.overlap_probability = 1.1; + assert!(config.validate().is_err()); + } + + #[test] + fn overlap_records_keep_source_identities_unique() { + let mut generator = CyberEntityGenerator::new(42, 1.0, 0.0, 0.0); + let records = generator + .generate_batch(2_000) + .into_iter() + .chain(generator.generate_batch(2_000)) + .collect::>(); + + let source_identities = records + .iter() + .map(|record| { + let identity = record.identity.as_ref().expect("generated identity"); + ( + identity.entity_type.as_str(), + identity.perspective.as_str(), + identity.uid.as_str(), + ) + }) + .collect::>(); + + assert_eq!(source_identities.len(), records.len()); + } + + #[test] + fn deployment_ontology_matches_loadtest_ontology() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("examples/loadtest-ontology.json"); + let deployed: unirust_rs::distributed::DistributedOntologyConfig = + serde_json::from_slice(&std::fs::read(path).expect("deployment ontology")) + .expect("valid deployment ontology"); + let generated = build_ontology_config(); + + assert_eq!(deployed.identity_keys.len(), generated.identity_keys.len()); + for (deployed_key, generated_key) in + deployed.identity_keys.iter().zip(&generated.identity_keys) + { + assert_eq!(deployed_key.name, generated_key.name); + assert_eq!(deployed_key.attributes, generated_key.attributes); + } + assert_eq!(deployed.strong_identifiers, generated.strong_identifiers); + assert!(deployed.constraints.is_empty()); + assert!(generated.constraints.is_empty()); + } +} diff --git a/src/bin/unirust_rebalance.rs b/src/bin/unirust_rebalance.rs index 0ff0b68..99cc9c0 100644 --- a/src/bin/unirust_rebalance.rs +++ b/src/bin/unirust_rebalance.rs @@ -83,6 +83,13 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } + if router.is_none() { + anyhow::bail!( + "--router is required for imports; direct shard-to-shard copies bypass durable source \ + reservations and are not a safe rebalance operation" + ); + } + let source = parse_arg("--source").ok_or_else(|| anyhow::anyhow!("--source is required"))?; let target = parse_arg("--target").ok_or_else(|| anyhow::anyhow!("--target is required"))?; let start_id: u32 = parse_arg("--start-id") @@ -197,6 +204,7 @@ async fn main() -> anyhow::Result<()> { let count = chunk.records.len() as u64; tx.send(ImportRecordsChunk { records: chunk.records, + internal_protocol_version: unirust_rs::distributed::DISTRIBUTED_PROTOCOL_VERSION, }) .await .map_err(|_| anyhow::anyhow!("import stream closed"))?; @@ -253,6 +261,8 @@ async fn main() -> anyhow::Result<()> { .expect("target client") .import_records(ImportRecordsRequest { records: response.records, + internal_protocol_version: + unirust_rs::distributed::DISTRIBUTED_PROTOCOL_VERSION, }) .await? .into_inner() diff --git a/src/bin/unirust_router.rs b/src/bin/unirust_router.rs index ebbdc69..1925326 100644 --- a/src/bin/unirust_router.rs +++ b/src/bin/unirust_router.rs @@ -2,7 +2,10 @@ use std::fs; use tonic::transport::Server; use unirust_rs::config::{normalize_shard_addrs, ConfigOverrides, RouterOverrides, UniConfig}; -use unirust_rs::distributed::{proto, DistributedOntologyConfig, RouterNode}; +use unirust_rs::distributed::{ + proto, AdaptiveReconciliationConfig, DistributedOntologyConfig, RouterNode, RouterRpcConfig, +}; +use unirust_rs::transport_security::{load_client_mtls, load_server_mtls}; fn parse_arg(flag: &str) -> Option { let mut args = std::env::args(); @@ -32,12 +35,26 @@ OPTIONS: --shards-file Path to file containing shard addresses (one per line) -o, --ontology Path to ontology config (JSON) --config-version Config version for compatibility checking + --checkpoint-interval-secs + Automatic coordinated checkpoint interval (0 disables) + --tls-cert PEM router server certificate + --tls-key PEM router server private key + --tls-client-ca + PEM CA used to require client certificates + --shard-tls-ca + PEM CA used to verify shard certificates + --shard-tls-cert + PEM router client certificate presented to shards + --shard-tls-key + PEM router client private key -h, --help Print help ENVIRONMENT: UNIRUST_CONFIG Path to config file UNIRUST_ROUTER_LISTEN Listen address UNIRUST_ROUTER_SHARDS Comma-separated shard addresses + UNIRUST_ROUTER_CHECKPOINT_INTERVAL_SECS + Automatic coordinated checkpoint interval CONFIG FILE (unirust.toml): [router] @@ -57,6 +74,24 @@ fn load_ontology(path: Option<&std::path::Path>) -> anyhow::Result {} + _ = terminate.recv() => {} + } + } + + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} + #[tokio::main] async fn main() -> anyhow::Result<()> { if has_flag("-h") || has_flag("--help") { @@ -91,10 +126,39 @@ async fn main() -> anyhow::Result<()> { router_overrides.ontology = Some(ontology.into()); } + if let Some(interval) = parse_arg("--checkpoint-interval-secs") { + router_overrides.checkpoint_interval_secs = Some(interval.parse()?); + } + if let Some(path) = parse_arg("--tls-cert") { + router_overrides.tls_cert = Some(path.into()); + } + if let Some(path) = parse_arg("--tls-key") { + router_overrides.tls_key = Some(path.into()); + } + if let Some(path) = parse_arg("--tls-client-ca") { + router_overrides.tls_client_ca = Some(path.into()); + } + if let Some(path) = parse_arg("--shard-tls-ca") { + router_overrides.shard_tls_ca = Some(path.into()); + } + if let Some(path) = parse_arg("--shard-tls-cert") { + router_overrides.shard_tls_cert = Some(path.into()); + } + if let Some(path) = parse_arg("--shard-tls-key") { + router_overrides.shard_tls_key = Some(path.into()); + } + if router_overrides.listen.is_some() || router_overrides.shards.is_some() || router_overrides.shards_file.is_some() || router_overrides.ontology.is_some() + || router_overrides.checkpoint_interval_secs.is_some() + || router_overrides.tls_cert.is_some() + || router_overrides.tls_key.is_some() + || router_overrides.tls_client_ca.is_some() + || router_overrides.shard_tls_ca.is_some() + || router_overrides.shard_tls_cert.is_some() + || router_overrides.shard_tls_key.is_some() { overrides.router = Some(router_overrides); } @@ -104,6 +168,16 @@ async fn main() -> anyhow::Result<()> { .or_else(|| parse_arg("-c")) .or_else(|| std::env::var("UNIRUST_CONFIG").ok()); let config = UniConfig::load(config_path.as_deref(), overrides)?; + let server_mtls = load_server_mtls( + config.router.tls_cert.as_deref(), + config.router.tls_key.as_deref(), + config.router.tls_client_ca.as_deref(), + )?; + let shard_mtls = load_client_mtls( + config.router.shard_tls_ca.as_deref(), + config.router.shard_tls_cert.as_deref(), + config.router.shard_tls_key.as_deref(), + )?; // Load ontology let ontology = load_ontology(config.router.ontology.as_deref())?; @@ -113,21 +187,72 @@ async fn main() -> anyhow::Result<()> { // Normalize shard addresses let shard_addrs = normalize_shard_addrs(&config.router.shards); + let reconciliation = AdaptiveReconciliationConfig { + key_count_threshold: config.reconciliation.key_count_threshold, + max_staleness: std::time::Duration::from_secs(config.reconciliation.max_staleness_secs), + idle_ingest_rate: config.reconciliation.idle_ingest_rate, + min_reconcile_interval: std::time::Duration::from_secs( + config.reconciliation.min_interval_secs, + ), + }; + let rpc = RouterRpcConfig { + connect_timeout: std::time::Duration::from_secs(config.router.shard_connect_timeout_secs), + request_timeout: std::time::Duration::from_secs(config.router.shard_request_timeout_secs), + tcp_keepalive: std::time::Duration::from_secs(config.router.shard_tcp_keepalive_secs), + shard_mtls, + }; // Create router node let router = if let Some(path) = &config.router.shards_file { - RouterNode::connect_from_file(path.to_str().unwrap(), ontology, config_version).await? + RouterNode::connect_from_file_with_runtime_config( + path, + ontology, + config_version, + reconciliation, + rpc, + ) + .await? + } else { + RouterNode::connect_with_runtime_config( + shard_addrs, + ontology, + config_version, + reconciliation, + rpc, + ) + .await? + }; + let checkpoint_task = if config.router.checkpoint_interval_secs > 0 { + tracing::info!( + interval_secs = config.router.checkpoint_interval_secs, + "automatic coordinated checkpoints enabled" + ); + Some( + router + .clone() + .start_checkpoint_scheduler(std::time::Duration::from_secs( + config.router.checkpoint_interval_secs, + ))?, + ) } else { - RouterNode::connect_with_version(shard_addrs, ontology, config_version).await? + None }; println!("Unirust router listening on {}", config.router.listen); - Server::builder() + let mut server = Server::builder(); + if let Some(server_mtls) = server_mtls { + server = server.tls_config(server_mtls)?; + } + server .add_service(proto::router_service_server::RouterServiceServer::new( router, )) - .serve(config.router.listen) + .serve_with_shutdown(config.router.listen, shutdown_signal()) .await?; + if let Some(checkpoint_task) = checkpoint_task { + checkpoint_task.abort(); + let _ = checkpoint_task.await; + } Ok(()) } diff --git a/src/bin/unirust_shard.rs b/src/bin/unirust_shard.rs index 58ebeeb..4d4f11e 100644 --- a/src/bin/unirust_shard.rs +++ b/src/bin/unirust_shard.rs @@ -3,7 +3,8 @@ use std::fs; use tonic::transport::Server; use unirust_rs::config::{ConfigOverrides, Profile, ShardOverrides, UniConfig}; use unirust_rs::distributed::{proto, DistributedOntologyConfig, ShardNode}; -use unirust_rs::StreamingTuning; +use unirust_rs::transport_security::load_server_mtls; +use unirust_rs::{restore_checkpoint_for_shard, StreamingTuning}; fn parse_arg(flag: &str) -> Option { let mut args = std::env::args(); @@ -31,11 +32,21 @@ OPTIONS: -l, --listen Override listen address [default: 127.0.0.1:50061] -i, --shard-id Override shard ID [default: 0] -d, --data-dir Override data directory + --backup-dir Checkpoint root on an independent volume + --restore-from + Restore a checkpoint into an empty data directory before startup -o, --ontology Path to ontology config (JSON) -p, --profile Tuning profile: balanced, low-latency, high-throughput, bulk-ingest, memory-saver, billion-scale, billion-scale-high-performance --repair Run repair on startup + --ephemeral Allow an in-memory shard; all data is lost on process exit + --allow-destructive-admin + Enable destructive admin RPCs such as Reset + --tls-cert PEM shard server certificate + --tls-key PEM shard server private key + --tls-client-ca + PEM CA used to require router client certificates --config-version Config version for compatibility checking -h, --help Print help @@ -45,6 +56,8 @@ ENVIRONMENT: UNIRUST_SHARD_LISTEN Listen address UNIRUST_SHARD_ID Shard ID UNIRUST_SHARD_DATA_DIR Data directory + UNIRUST_SHARD_BACKUP_DIR + Checkpoint root CONFIG FILE (unirust.toml): profile = "billion-scale-high-performance" @@ -53,6 +66,7 @@ CONFIG FILE (unirust.toml): listen = "0.0.0.0:50061" id = 0 data_dir = "/var/lib/unirust" + backup_dir = "/var/backups/unirust/shard-0" "# ); } @@ -80,6 +94,24 @@ fn load_ontology(path: Option<&std::path::Path>) -> anyhow::Result {} + _ = terminate.recv() => {} + } + } + + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} + #[tokio::main] async fn main() -> anyhow::Result<()> { if has_flag("-h") || has_flag("--help") { @@ -114,6 +146,10 @@ async fn main() -> anyhow::Result<()> { shard_overrides.data_dir = Some(data_dir.into()); } + if let Some(backup_dir) = parse_arg("--backup-dir") { + shard_overrides.backup_dir = Some(backup_dir.into()); + } + if let Some(ontology) = parse_arg("--ontology").or_else(|| parse_arg("-o")) { shard_overrides.ontology = Some(ontology.into()); } @@ -121,12 +157,25 @@ async fn main() -> anyhow::Result<()> { if has_flag("--repair") { shard_overrides.repair = Some(true); } + if let Some(path) = parse_arg("--tls-cert") { + shard_overrides.tls_cert = Some(path.into()); + } + if let Some(path) = parse_arg("--tls-key") { + shard_overrides.tls_key = Some(path.into()); + } + if let Some(path) = parse_arg("--tls-client-ca") { + shard_overrides.tls_client_ca = Some(path.into()); + } if shard_overrides.listen.is_some() || shard_overrides.id.is_some() || shard_overrides.data_dir.is_some() + || shard_overrides.backup_dir.is_some() || shard_overrides.ontology.is_some() || shard_overrides.repair.is_some() + || shard_overrides.tls_cert.is_some() + || shard_overrides.tls_key.is_some() + || shard_overrides.tls_client_ca.is_some() { overrides.shard = Some(shard_overrides); } @@ -136,6 +185,29 @@ async fn main() -> anyhow::Result<()> { .or_else(|| parse_arg("-c")) .or_else(|| std::env::var("UNIRUST_CONFIG").ok()); let config = UniConfig::load(config_path.as_deref(), overrides)?; + let server_mtls = load_server_mtls( + config.shard.tls_cert.as_deref(), + config.shard.tls_key.as_deref(), + config.shard.tls_client_ca.as_deref(), + )?; + if config.shard.data_dir.is_none() && !has_flag("--ephemeral") { + anyhow::bail!( + "persistent shard storage is required; configure --data-dir (or use --ephemeral \ + explicitly for disposable development data)" + ); + } + if let Some(source) = parse_arg("--restore-from") { + let data_dir = config + .shard + .data_dir + .as_deref() + .ok_or_else(|| anyhow::anyhow!("--restore-from requires --data-dir"))?; + restore_checkpoint_for_shard( + std::path::Path::new(&source), + data_dir, + Some(u32::from(config.shard.id)), + )?; + } // Get tuning from profile let profile = config.profile.to_tuning_profile(); @@ -150,23 +222,32 @@ async fn main() -> anyhow::Result<()> { let config_version = parse_arg("--config-version").or(config.shard.config_version.clone()); // Create shard node - let shard = ShardNode::new_with_data_dir( + let shard = ShardNode::new_with_storage_paths( config.shard.id as u32, ontology, tuning, config.shard.data_dir.clone(), + config.shard.backup_dir.clone(), config.shard.repair, config_version, - )?; + )? + .with_destructive_admin(has_flag("--allow-destructive-admin")); println!( "Unirust shard {} listening on {}", config.shard.id, config.shard.listen ); - Server::builder() - .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) - .serve(config.shard.listen) + let mut server = Server::builder(); + if let Some(server_mtls) = server_mtls { + server = server.tls_config(server_mtls)?; + } + server + .add_service(proto::shard_service_server::ShardServiceServer::new( + shard.clone(), + )) + .serve_with_shutdown(config.shard.listen, shutdown_signal()) .await?; + shard.shutdown().await?; Ok(()) } diff --git a/src/config/defaults.rs b/src/config/defaults.rs index f848296..4548bcc 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -21,6 +21,19 @@ pub const DEFAULT_SHARD_COUNT: usize = 5; /// Default router listen address pub const DEFAULT_ROUTER_ADDR: &str = "127.0.0.1:50060"; +/// Maximum time to establish a router-to-shard connection. +pub const DEFAULT_SHARD_CONNECT_TIMEOUT_SECS: u64 = 10; + +/// Maximum time for one router-to-shard RPC. +pub const DEFAULT_SHARD_REQUEST_TIMEOUT_SECS: u64 = 120; + +/// TCP keepalive interval for router-to-shard connections. +pub const DEFAULT_SHARD_TCP_KEEPALIVE_SECS: u64 = 30; + +/// Automatic coordinated checkpoints are disabled until backup storage and an +/// operator-selected RPO have been configured. +pub const DEFAULT_CHECKPOINT_INTERVAL_SECS: u64 = 0; + // ============================================================================= // Storage Defaults (RocksDB) // ============================================================================= @@ -58,6 +71,12 @@ pub const DEFAULT_MAX_STALENESS_SECS: u64 = 60; /// Prevents reconciliation thrashing under high load. pub const DEFAULT_MIN_RECONCILE_INTERVAL_SECS: u64 = 5; +/// Number of dirty identity keys that triggers reconciliation immediately. +pub const DEFAULT_RECONCILE_KEY_COUNT_THRESHOLD: usize = 1_000; + +/// Ingest rate below which pending cross-shard work is reconciled while idle. +pub const DEFAULT_RECONCILE_IDLE_INGEST_RATE: f64 = 1_000.0; + // ============================================================================= // Partitioning Defaults // ============================================================================= diff --git a/src/config/mod.rs b/src/config/mod.rs index 812cab5..d527917 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -10,6 +10,7 @@ //! listen = "0.0.0.0:50061" //! id = 0 //! data_dir = "/var/lib/unirust" +//! backup_dir = "/var/backups/unirust/shard-0" //! //! [router] //! listen = "0.0.0.0:50060" @@ -30,6 +31,106 @@ use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::path::PathBuf; +fn config_env_path(key: &str) -> Option<&'static str> { + match key.to_ascii_uppercase().as_str() { + "PROFILE" => Some("profile"), + "SHARD_LISTEN" => Some("shard.listen"), + "SHARD_ID" => Some("shard.id"), + "SHARD_DATA_DIR" => Some("shard.data_dir"), + "SHARD_BACKUP_DIR" => Some("shard.backup_dir"), + "SHARD_ONTOLOGY" => Some("shard.ontology"), + "SHARD_REPAIR" => Some("shard.repair"), + "SHARD_CONFIG_VERSION" => Some("shard.config_version"), + "SHARD_TLS_CERT" => Some("shard.tls_cert"), + "SHARD_TLS_KEY" => Some("shard.tls_key"), + "SHARD_TLS_CLIENT_CA" => Some("shard.tls_client_ca"), + "ROUTER_LISTEN" => Some("router.listen"), + "ROUTER_SHARDS_FILE" => Some("router.shards_file"), + "ROUTER_ONTOLOGY" => Some("router.ontology"), + "ROUTER_CONFIG_VERSION" => Some("router.config_version"), + "ROUTER_SHARD_CONNECT_TIMEOUT_SECS" => Some("router.shard_connect_timeout_secs"), + "ROUTER_SHARD_REQUEST_TIMEOUT_SECS" => Some("router.shard_request_timeout_secs"), + "ROUTER_SHARD_TCP_KEEPALIVE_SECS" => Some("router.shard_tcp_keepalive_secs"), + "ROUTER_CHECKPOINT_INTERVAL_SECS" => Some("router.checkpoint_interval_secs"), + "ROUTER_TLS_CERT" => Some("router.tls_cert"), + "ROUTER_TLS_KEY" => Some("router.tls_key"), + "ROUTER_TLS_CLIENT_CA" => Some("router.tls_client_ca"), + "ROUTER_SHARD_TLS_CA" => Some("router.shard_tls_ca"), + "ROUTER_SHARD_TLS_CERT" => Some("router.shard_tls_cert"), + "ROUTER_SHARD_TLS_KEY" => Some("router.shard_tls_key"), + "STORAGE_BLOCK_CACHE_MB" => Some("storage.block_cache_mb"), + "STORAGE_WRITE_BUFFER_MB" => Some("storage.write_buffer_mb"), + "STORAGE_RATE_LIMIT_MBPS" => Some("storage.rate_limit_mbps"), + "STORAGE_MAX_BACKGROUND_JOBS" => Some("storage.max_background_jobs"), + "RECONCILIATION_KEY_COUNT_THRESHOLD" => Some("reconciliation.key_count_threshold"), + "RECONCILIATION_MAX_STALENESS_SECS" => Some("reconciliation.max_staleness_secs"), + "RECONCILIATION_IDLE_INGEST_RATE" => Some("reconciliation.idle_ingest_rate"), + "RECONCILIATION_MIN_INTERVAL_SECS" => Some("reconciliation.min_interval_secs"), + // Parsed separately because the documented value is a comma-separated list. + "ROUTER_SHARDS" => None, + _ => None, + } +} + +fn validate_config_environment() -> Result<(), ConfigError> { + const CONFIG_NAMESPACES: [&str; 4] = [ + "UNIRUST_SHARD_", + "UNIRUST_ROUTER_", + "UNIRUST_STORAGE_", + "UNIRUST_RECONCILIATION_", + ]; + + for (key, _) in std::env::vars_os() { + let Some(key) = key.to_str() else { + continue; + }; + let normalized = key.to_ascii_uppercase(); + if CONFIG_NAMESPACES + .iter() + .any(|namespace| normalized.starts_with(namespace)) + && normalized != "UNIRUST_ROUTER_SHARDS" + && config_env_path(normalized.trim_start_matches("UNIRUST_")).is_none() + { + return Err(ConfigError { + message: format!("unknown Unirust configuration variable {key}"), + }); + } + } + + Ok(()) +} + +fn router_shards_from_env() -> Result>, ConfigError> { + let Some(value) = std::env::var_os("UNIRUST_ROUTER_SHARDS") else { + return Ok(None); + }; + let value = value.into_string().map_err(|_| ConfigError { + message: "UNIRUST_ROUTER_SHARDS must be valid UTF-8".to_string(), + })?; + let shards: Vec = value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(str::to_string) + .collect(); + if shards.is_empty() { + return Err(ConfigError { + message: "UNIRUST_ROUTER_SHARDS must contain at least one address".to_string(), + }); + } + Ok(Some(shards)) +} + +fn validate_mtls_group(name: &str, paths: [Option<&PathBuf>; 3]) -> Result<(), ConfigError> { + let configured = paths.iter().filter(|path| path.is_some()).count(); + if configured != 0 && configured != paths.len() { + return Err(ConfigError { + message: format!("{name} requires all three certificate paths together"), + }); + } + Ok(()) +} + /// Main configuration for unirust components. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(default)] @@ -56,6 +157,7 @@ impl UniConfig { config_path: Option<&str>, overrides: ConfigOverrides, ) -> Result { + validate_config_environment()?; let mut figment = Figment::new().merge(Serialized::defaults(UniConfig::default())); // Layer 1: Config file (if provided) @@ -64,12 +166,49 @@ impl UniConfig { } // Layer 2: Environment variables with UNIRUST_ prefix - figment = figment.merge(Env::prefixed("UNIRUST_").split("_")); + figment = figment.merge( + Env::prefixed("UNIRUST_") + .filter_map(|key| config_env_path(key.as_str()).map(Into::into)), + ); + if let Some(shards) = router_shards_from_env()? { + figment = figment.merge(Serialized::defaults(ConfigOverrides { + router: Some(RouterOverrides { + shards: Some(shards), + ..RouterOverrides::default() + }), + ..ConfigOverrides::default() + })); + } // Layer 3: CLI overrides figment = figment.merge(Serialized::defaults(overrides)); - figment.extract().map_err(ConfigError::from) + let config: Self = figment.extract().map_err(ConfigError::from)?; + validate_mtls_group( + "shard mTLS", + [ + config.shard.tls_cert.as_ref(), + config.shard.tls_key.as_ref(), + config.shard.tls_client_ca.as_ref(), + ], + )?; + validate_mtls_group( + "router mTLS", + [ + config.router.tls_cert.as_ref(), + config.router.tls_key.as_ref(), + config.router.tls_client_ca.as_ref(), + ], + )?; + validate_mtls_group( + "router-to-shard mTLS", + [ + config.router.shard_tls_ca.as_ref(), + config.router.shard_tls_cert.as_ref(), + config.router.shard_tls_key.as_ref(), + ], + )?; + Ok(config) } /// Load from environment and optional config file only (no CLI overrides) @@ -124,12 +263,20 @@ pub struct ShardConfig { pub id: u16, /// Data directory for persistence pub data_dir: Option, + /// Checkpoint root, ideally on storage independent from the data directory + pub backup_dir: Option, /// Path to ontology configuration file (JSON) pub ontology: Option, /// Run repair on startup pub repair: bool, /// Config version for compatibility checking pub config_version: Option, + /// PEM server certificate for mutually authenticated TLS + pub tls_cert: Option, + /// PEM server private key for mutually authenticated TLS + pub tls_key: Option, + /// PEM CA used to require and verify client certificates + pub tls_client_ca: Option, } impl Default for ShardConfig { @@ -138,9 +285,13 @@ impl Default for ShardConfig { listen: DEFAULT_SHARD_ADDR.parse().unwrap(), id: 0, data_dir: None, + backup_dir: None, ontology: None, repair: false, config_version: None, + tls_cert: None, + tls_key: None, + tls_client_ca: None, } } } @@ -159,6 +310,26 @@ pub struct RouterConfig { pub ontology: Option, /// Config version for compatibility checking pub config_version: Option, + /// Maximum time to establish a shard connection + pub shard_connect_timeout_secs: u64, + /// Maximum time for one shard RPC + pub shard_request_timeout_secs: u64, + /// TCP keepalive interval for shard connections + pub shard_tcp_keepalive_secs: u64, + /// Interval between automatic coordinated checkpoints (0 disables) + pub checkpoint_interval_secs: u64, + /// PEM server certificate for mutually authenticated TLS + pub tls_cert: Option, + /// PEM server private key for mutually authenticated TLS + pub tls_key: Option, + /// PEM CA used to require and verify client certificates + pub tls_client_ca: Option, + /// PEM CA used to verify shard server certificates + pub shard_tls_ca: Option, + /// PEM router client certificate presented to shards + pub shard_tls_cert: Option, + /// PEM router client private key presented to shards + pub shard_tls_key: Option, } impl Default for RouterConfig { @@ -169,6 +340,16 @@ impl Default for RouterConfig { shards_file: None, ontology: None, config_version: None, + shard_connect_timeout_secs: DEFAULT_SHARD_CONNECT_TIMEOUT_SECS, + shard_request_timeout_secs: DEFAULT_SHARD_REQUEST_TIMEOUT_SECS, + shard_tcp_keepalive_secs: DEFAULT_SHARD_TCP_KEEPALIVE_SECS, + checkpoint_interval_secs: DEFAULT_CHECKPOINT_INTERVAL_SECS, + tls_cert: None, + tls_key: None, + tls_client_ca: None, + shard_tls_ca: None, + shard_tls_cert: None, + shard_tls_key: None, } } } @@ -202,8 +383,12 @@ impl Default for StorageConfig { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct ReconciliationConfig { + /// Number of dirty identity keys that triggers reconciliation + pub key_count_threshold: usize, /// Maximum staleness before forced reconcile (seconds) pub max_staleness_secs: u64, + /// Ingest rate below which pending keys are reconciled while idle + pub idle_ingest_rate: f64, /// Minimum interval between reconciles (seconds) pub min_interval_secs: u64, } @@ -211,7 +396,9 @@ pub struct ReconciliationConfig { impl Default for ReconciliationConfig { fn default() -> Self { Self { + key_count_threshold: DEFAULT_RECONCILE_KEY_COUNT_THRESHOLD, max_staleness_secs: DEFAULT_MAX_STALENESS_SECS, + idle_ingest_rate: DEFAULT_RECONCILE_IDLE_INGEST_RATE, min_interval_secs: DEFAULT_MIN_RECONCILE_INTERVAL_SECS, } } @@ -239,9 +426,17 @@ pub struct ShardOverrides { #[serde(skip_serializing_if = "Option::is_none")] pub data_dir: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub backup_dir: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ontology: Option, #[serde(skip_serializing_if = "Option::is_none")] pub repair: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tls_cert: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tls_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tls_client_ca: Option, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -255,6 +450,20 @@ pub struct RouterOverrides { pub shards_file: Option, #[serde(skip_serializing_if = "Option::is_none")] pub ontology: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub checkpoint_interval_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tls_cert: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tls_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tls_client_ca: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub shard_tls_ca: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub shard_tls_cert: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub shard_tls_key: Option, } /// Configuration error. @@ -303,7 +512,20 @@ mod tests { let config = UniConfig::default(); assert_eq!(config.profile, Profile::BillionScaleHighPerformance); assert_eq!(config.shard.id, 0); + assert!(config.shard.backup_dir.is_none()); assert_eq!(config.storage.block_cache_mb, DEFAULT_BLOCK_CACHE_MB); + assert_eq!( + config.router.shard_connect_timeout_secs, + DEFAULT_SHARD_CONNECT_TIMEOUT_SECS + ); + assert_eq!( + config.router.shard_request_timeout_secs, + DEFAULT_SHARD_REQUEST_TIMEOUT_SECS + ); + assert_eq!( + config.router.checkpoint_interval_secs, + DEFAULT_CHECKPOINT_INTERVAL_SECS + ); } #[test] @@ -327,4 +549,21 @@ mod tests { let profile: Profile = serde_json::from_str("\"low-latency\"").unwrap(); assert_eq!(profile, Profile::LowLatency); } + + #[test] + fn test_config_env_paths_preserve_field_underscores() { + assert_eq!( + config_env_path("ROUTER_CHECKPOINT_INTERVAL_SECS"), + Some("router.checkpoint_interval_secs") + ); + assert_eq!( + config_env_path("SHARD_BACKUP_DIR"), + Some("shard.backup_dir") + ); + assert_eq!( + config_env_path("RECONCILIATION_KEY_COUNT_THRESHOLD"), + Some("reconciliation.key_count_threshold") + ); + assert_eq!(config_env_path("ROUTER_UNKNOWN"), None); + } } diff --git a/src/config/tuning.rs b/src/config/tuning.rs index 34817e1..1daf6d3 100644 --- a/src/config/tuning.rs +++ b/src/config/tuning.rs @@ -302,7 +302,10 @@ impl StreamingTuning { use_tiered_index: true, shard_id: 0, enable_boundary_tracking: false, - linker_state_config: Some(LinkerStateConfig::default()), + // LinkerState's bounded LRU backend has no durable spill path yet. + // Evicting cluster summaries or ID mappings changes resolution results, + // so persistent profiles must remain unbounded until spill is implemented. + linker_state_config: None, } } @@ -326,7 +329,8 @@ impl StreamingTuning { use_tiered_index: true, shard_id: 0, enable_boundary_tracking: false, - linker_state_config: Some(LinkerStateConfig::high_performance()), + // Correctness-critical linker state cannot be evicted without durable spill. + linker_state_config: None, } } @@ -343,8 +347,11 @@ impl StreamingTuning { } } -/// Configuration for linker state memory management. -/// Controls LRU cache sizes for cluster IDs, summaries, and perspectives. +/// Reserved configuration for linker-state memory management. +/// +/// Capacities are currently not enforced because correctness-critical evictions +/// require a durable spill/read-through backend. Supplying this configuration +/// retains unbounded state and emits a warning. #[derive(Debug, Clone)] pub struct LinkerStateConfig { /// Maximum number of cluster ID mappings to keep in memory. diff --git a/src/conflicts.rs b/src/conflicts.rs index 4d578da..ad843e7 100644 --- a/src/conflicts.rs +++ b/src/conflicts.rs @@ -9,7 +9,7 @@ use crate::ontology::{Constraint, ConstraintViolation, IdentityKey, Ontology}; use crate::store::RecordStore; use crate::temporal::Interval; use anyhow::Result; -use hashbrown::HashMap; +use hashbrown::{HashMap, HashSet}; use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; // use std::collections::HashSet; @@ -35,7 +35,7 @@ impl Participants { Participants::One(a) => { *self = Participants::Many(vec![*a, b]); } - Participants::Many(ref mut v) => { + Participants::Many(v) => { v.push(b); } }, @@ -44,7 +44,7 @@ impl Participants { other_v.push(*a); *self = Participants::Many(other_v); } - Participants::Many(ref mut v) => { + Participants::Many(v) => { v.append(&mut other_v); } }, @@ -53,7 +53,7 @@ impl Participants { #[inline] fn sort_dedup(&mut self) { - if let Participants::Many(ref mut v) = self { + if let Participants::Many(v) = self { v.sort(); v.dedup(); } @@ -74,6 +74,8 @@ struct ValueInterval { participants: Participants, } +type ConflictDescriptor = (RecordId, ValueId, Interval); + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] enum EventKind { End, @@ -414,31 +416,61 @@ impl ConflictDetector { fn detect_direct_conflicts( store: &dyn RecordStore, cluster: &crate::dsu::Cluster, - _ontology: &Ontology, + ontology: &Ontology, ) -> Result> { let mut conflicts = Vec::new(); - // Group descriptors by attribute - only extract needed fields (no clone) - // Use (ValueId, Interval) tuple instead of full Descriptor - let mut descriptors_by_attr: HashMap> = - HashMap::new(); + // A perspective-scoped constraint explicitly permits different perspectives + // to hold different values. A global Unique constraint, when both are + // configured, remains the stricter rule. + let mut perspective_scoped_attrs = HashSet::new(); + let mut globally_unique_attrs = HashSet::new(); + for constraint in &ontology.constraints { + match constraint { + Constraint::Unique { attribute, .. } => { + globally_unique_attrs.insert(*attribute); + } + Constraint::UniqueWithinPerspective { attribute, .. } => { + perspective_scoped_attrs.insert(*attribute); + } + } + } + perspective_scoped_attrs.retain(|attr| !globally_unique_attrs.contains(attr)); + + // Group descriptors by attribute, and by perspective only where required. + let mut descriptors_by_attr: HashMap> = HashMap::new(); + let mut descriptors_by_attr_and_perspective: HashMap< + (AttrId, String), + Vec, + > = HashMap::new(); for record_id in &cluster.records { if let Some(record) = store.get_record(*record_id) { for descriptor in &record.descriptors { - descriptors_by_attr - .entry(descriptor.attr) - .or_default() - .push((*record_id, descriptor.value, descriptor.interval)); + let entry = (*record_id, descriptor.value, descriptor.interval); + if perspective_scoped_attrs.contains(&descriptor.attr) { + descriptors_by_attr_and_perspective + .entry((descriptor.attr, record.identity.perspective.clone())) + .or_default() + .push(entry); + } else { + descriptors_by_attr + .entry(descriptor.attr) + .or_default() + .push(entry); + } } } } - // Check each attribute for conflicts for (attr, descriptors) in descriptors_by_attr { let conflicts_for_attr = Self::detect_conflicts_for_attribute_fast(descriptors, attr)?; conflicts.extend(conflicts_for_attr); } + for ((attr, _perspective), descriptors) in descriptors_by_attr_and_perspective { + let conflicts_for_attr = Self::detect_conflicts_for_attribute_fast(descriptors, attr)?; + conflicts.extend(conflicts_for_attr); + } Ok(conflicts) } @@ -680,7 +712,7 @@ impl ConflictDetector { return intervals; } - intervals.sort_by(|a, b| a.interval.start.cmp(&b.interval.start)); + intervals.sort_by_key(|value| value.interval.start); let mut merged: Vec = Vec::with_capacity(intervals.len()); for current in intervals { @@ -858,7 +890,7 @@ impl ConflictDetector { return conflicts; } - conflicts.sort_by(|a, b| a.interval.start.cmp(&b.interval.start)); + conflicts.sort_by_key(|conflict| conflict.interval.start); let mut merged: Vec = Vec::with_capacity(conflicts.len()); for current in conflicts { @@ -1380,6 +1412,16 @@ impl ConflictDetector { cluster: &crate::dsu::Cluster, attribute: AttrId, name: &str, + ) -> Result> { + Self::check_unique_constraint_for_records(store, &cluster.records, attribute, name, false) + } + + fn check_unique_constraint_for_records( + store: &dyn RecordStore, + record_ids: &[RecordId], + attribute: AttrId, + name: &str, + within_perspective: bool, ) -> Result> { let mut violations = Vec::new(); @@ -1387,7 +1429,7 @@ impl ConflictDetector { let mut descriptors_by_value: HashMap> = HashMap::new(); - for record_id in &cluster.records { + for record_id in record_ids { if let Some(record) = store.get_record(*record_id) { for descriptor in &record.descriptors { if descriptor.attr == attribute { @@ -1422,8 +1464,13 @@ impl ConflictDetector { participants.sort(); participants.dedup(); + let constraint = if within_perspective { + Constraint::unique_within_perspective(attribute, name.to_string()) + } else { + Constraint::unique(attribute, name.to_string()) + }; let violation = ConstraintViolation::new( - Constraint::unique(attribute, name.to_string()), + constraint, overlap, participants, format!( @@ -1446,9 +1493,23 @@ impl ConflictDetector { attribute: AttrId, name: &str, ) -> Result> { - // Similar to check_unique_constraint but filtered by perspective - // For now, we'll use the same logic - Self::check_unique_constraint(store, cluster, attribute, name) + let mut records_by_perspective: HashMap> = HashMap::new(); + for record_id in &cluster.records { + if let Some(record) = store.get_record(*record_id) { + records_by_perspective + .entry(record.identity.perspective) + .or_default() + .push(*record_id); + } + } + + let mut violations = Vec::new(); + for record_ids in records_by_perspective.values() { + violations.extend(Self::check_unique_constraint_for_records( + store, record_ids, attribute, name, true, + )?); + } + Ok(violations) } } @@ -2352,6 +2413,98 @@ mod tests { ); } + #[test] + fn direct_conflicts_respect_perspective_scoped_constraints() { + let mut store = Store::new(); + let attr = store.interner_mut().intern_attr("source_code"); + let value_a = store.interner_mut().intern_value("A"); + let value_b = store.interner_mut().intern_value("B"); + let interval = Interval::new(0, 100).unwrap(); + + store + .insert_record(Record::new( + RecordId(1), + RecordIdentity::new( + "instrument".to_string(), + "source_a".to_string(), + "1".to_string(), + ), + vec![Descriptor::new(attr, value_a, interval)], + )) + .unwrap(); + store + .insert_record(Record::new( + RecordId(2), + RecordIdentity::new( + "instrument".to_string(), + "source_b".to_string(), + "2".to_string(), + ), + vec![Descriptor::new(attr, value_b, interval)], + )) + .unwrap(); + let cluster = + crate::dsu::Cluster::new(ClusterId(1), RecordId(1), vec![RecordId(1), RecordId(2)]); + + let mut scoped = Ontology::new(); + scoped.add_constraint(Constraint::unique_within_perspective( + attr, + "source_code_per_source".to_string(), + )); + assert!( + ConflictDetector::detect_direct_conflicts(&store, &cluster, &scoped) + .unwrap() + .is_empty(), + "different perspectives may hold different values" + ); + + scoped.add_constraint(Constraint::unique(attr, "source_code_global".to_string())); + assert_eq!( + ConflictDetector::detect_direct_conflicts(&store, &cluster, &scoped) + .unwrap() + .len(), + 1, + "a global constraint must override perspective scoping" + ); + } + + #[test] + fn perspective_scoped_constraint_still_conflicts_within_one_perspective() { + let mut store = Store::new(); + let attr = store.interner_mut().intern_attr("source_code"); + let value_a = store.interner_mut().intern_value("A"); + let value_b = store.interner_mut().intern_value("B"); + let interval = Interval::new(0, 100).unwrap(); + + for (id, value) in [(1, value_a), (2, value_b)] { + store + .insert_record(Record::new( + RecordId(id), + RecordIdentity::new( + "instrument".to_string(), + "source_a".to_string(), + id.to_string(), + ), + vec![Descriptor::new(attr, value, interval)], + )) + .unwrap(); + } + let cluster = + crate::dsu::Cluster::new(ClusterId(1), RecordId(1), vec![RecordId(1), RecordId(2)]); + let mut ontology = Ontology::new(); + ontology.add_constraint(Constraint::unique_within_perspective( + attr, + "source_code_per_source".to_string(), + )); + + assert_eq!( + ConflictDetector::detect_direct_conflicts(&store, &cluster, &ontology) + .unwrap() + .len(), + 1 + ); + } + #[test] fn test_can_handle_indirect_conflict() { // This test demonstrates indirect conflict handling with three entities: @@ -2677,7 +2830,11 @@ mod tests { // With the optimized linker, records with strong identifier conflicts are kept separate // so there should be no indirect conflicts detected - assert_eq!(indirect_conflicts.len(), 0, "No indirect conflicts should be detected when records are kept separate due to strong identifier conflicts"); + assert_eq!( + indirect_conflicts.len(), + 0, + "No indirect conflicts should be detected when records are kept separate due to strong identifier conflicts" + ); // Verify conflict details for conflict in &indirect_conflicts { @@ -3296,7 +3453,11 @@ mod tests { records, |_, clusters, observations| { // Should have 2 separate clusters (entities should NOT be merged) - assert_eq!(clusters.clusters.len(), 2, "Should have 2 separate clusters - entities with same email but different UIDs should not be merged"); + assert_eq!( + clusters.clusters.len(), + 2, + "Should have 2 separate clusters - entities with same email but different UIDs should not be merged" + ); // Verify that the clusters contain the correct records let mut found_entity1 = false; @@ -3412,10 +3573,18 @@ mod tests { records, |_, clusters, observations| { // Should have 2 separate clusters (entities should NOT be merged) - assert_eq!(clusters.clusters.len(), 2, "Should have 2 separate clusters - entities from different perspectives should not be merged"); + assert_eq!( + clusters.clusters.len(), + 2, + "Should have 2 separate clusters - entities from different perspectives should not be merged" + ); // Should have NO conflict observations (different perspectives can have same unique values) - assert_eq!(observations.len(), 0, "Should have no conflict observations - entities from different perspectives can have the same unique identifier"); + assert_eq!( + observations.len(), + 0, + "Should have no conflict observations - entities from different perspectives can have the same unique identifier" + ); }, ); } diff --git a/src/distributed.rs b/src/distributed.rs index 702d403..a362987 100644 --- a/src/distributed.rs +++ b/src/distributed.rs @@ -1,19 +1,24 @@ use crate::conflicts::ConflictSummary; use crate::graph::GoldenDescriptor; -use crate::model::{AttrId, GlobalClusterId, Record, RecordId, RecordIdentity}; +use crate::model::{GlobalClusterId, Record, RecordId, RecordIdentity}; use crate::ontology::{Constraint, IdentityKey, Ontology, StrongIdentifier}; use crate::partitioned::{ParallelPartitionedUnirust, PartitionConfig}; use crate::perf::ConcurrentInterner; -use crate::persistence::PersistentOpenOptions; +use crate::persistence::{ + commit_cluster_checkpoint, prepare_cluster_checkpoint, read_restored_checkpoint_manifest, + validate_prepared_cluster_checkpoint, ClusterCheckpointManifest, PersistentOpenOptions, + CHECKPOINT_PROTOCOL_VERSION, +}; use crate::query::{QueryDescriptor, QueryOutcome}; use crate::sharding::{BloomFilter, IdentityKeySignature}; -use crate::store::StoreMetrics; +use crate::store::{SourceRecordReservation, SourceReservationError, StoreMetrics}; use crate::temporal::Interval; use crate::{PersistentStore, StreamingTuning, Unirust}; use anyhow::Result as AnyResult; use lru::LruCache; use rayon::prelude::*; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::fs; use std::hash::{Hash, Hasher}; @@ -22,7 +27,7 @@ use std::num::NonZeroUsize; use std::path::Path; use std::path::PathBuf; use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, RwLock as StdRwLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::{mpsc, oneshot}; @@ -59,6 +64,112 @@ struct IngestWal { temp_path: PathBuf, } +const INGEST_WAL_MAGIC: &[u8; 8] = b"UNIRWAL\0"; +const INGEST_WAL_VERSION: u32 = 1; +const INGEST_WAL_HEADER_LEN: usize = 24; +pub const DISTRIBUTED_PROTOCOL_VERSION: u32 = 3; + +#[allow(clippy::result_large_err)] +fn validate_distributed_protocol(protocol_version: u32) -> Result<(), Status> { + if protocol_version == DISTRIBUTED_PROTOCOL_VERSION { + Ok(()) + } else { + Err(Status::failed_precondition(format!( + "distributed protocol mismatch: router {}, shard {}; deploy one coordinated Unirust \ + version across the complete cluster", + DISTRIBUTED_PROTOCOL_VERSION, protocol_version + ))) + } +} + +#[allow(clippy::result_large_err)] +fn validate_checkpoint_protocol(protocol_version: u32) -> Result<(), Status> { + if protocol_version == CHECKPOINT_PROTOCOL_VERSION { + Ok(()) + } else { + Err(Status::failed_precondition(format!( + "checkpoint protocol mismatch: router {}, shard {}; deploy one coordinated Unirust \ + version across the complete cluster", + CHECKPOINT_PROTOCOL_VERSION, protocol_version + ))) + } +} + +fn encode_wal_batch(inputs: &[WalRecordInput]) -> Result, Status> { + let body = bincode::serialize(inputs).map_err(|err| Status::internal(err.to_string()))?; + let body_len = u64::try_from(body.len()) + .map_err(|_| Status::resource_exhausted("ingest WAL batch is too large"))?; + let mut encoded = Vec::with_capacity(INGEST_WAL_HEADER_LEN + body.len()); + encoded.extend_from_slice(INGEST_WAL_MAGIC); + encoded.extend_from_slice(&INGEST_WAL_VERSION.to_be_bytes()); + encoded.extend_from_slice(&body_len.to_be_bytes()); + encoded.extend_from_slice(&crc32fast::hash(&body).to_be_bytes()); + encoded.extend_from_slice(&body); + Ok(encoded) +} + +fn decode_wal_batch(bytes: &[u8]) -> Result, String> { + // Accept the pre-0.2 unframed format so an upgrade can replay an existing WAL. + if !bytes.starts_with(INGEST_WAL_MAGIC) { + return bincode::deserialize(bytes).map_err(|err| err.to_string()); + } + if bytes.len() < INGEST_WAL_HEADER_LEN { + return Err("truncated ingest WAL header".to_string()); + } + + let version = u32::from_be_bytes( + bytes[8..12] + .try_into() + .map_err(|_| "invalid ingest WAL version field".to_string())?, + ); + if version != INGEST_WAL_VERSION { + return Err(format!("unsupported ingest WAL version {version}")); + } + let declared_len = u64::from_be_bytes( + bytes[12..20] + .try_into() + .map_err(|_| "invalid ingest WAL payload length field".to_string())?, + ); + let declared_len = usize::try_from(declared_len) + .map_err(|_| "ingest WAL payload length exceeds this platform".to_string())?; + let expected_len = INGEST_WAL_HEADER_LEN + .checked_add(declared_len) + .ok_or_else(|| "ingest WAL payload length overflow".to_string())?; + if bytes.len() != expected_len { + return Err(format!( + "ingest WAL length mismatch: expected {expected_len} bytes, found {}", + bytes.len() + )); + } + + let expected_checksum = u32::from_be_bytes( + bytes[20..24] + .try_into() + .map_err(|_| "invalid ingest WAL checksum field".to_string())?, + ); + let body = &bytes[INGEST_WAL_HEADER_LEN..]; + let actual_checksum = crc32fast::hash(body); + if actual_checksum != expected_checksum { + return Err("ingest WAL checksum mismatch".to_string()); + } + bincode::deserialize(body).map_err(|err| err.to_string()) +} + +#[cfg(unix)] +fn sync_parent_directory(path: &Path) -> Result<(), Status> { + let parent = path + .parent() + .ok_or_else(|| Status::internal("ingest WAL path has no parent directory"))?; + fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|err| Status::internal(format!("failed to sync WAL directory: {err}"))) +} + +#[cfg(not(unix))] +fn sync_parent_directory(_path: &Path) -> Result<(), Status> { + Ok(()) +} + #[allow(clippy::result_large_err)] impl IngestWal { fn new(data_dir: &Path) -> Self { @@ -69,21 +180,27 @@ impl IngestWal { fn write_batch(&self, records: &[proto::RecordInput]) -> Result<(), Status> { if records.is_empty() { - return self.clear(); + return Ok(()); } let inputs: Vec = records .iter() .map(wal_from_proto_input) .collect::>()?; - let payload = - bincode::serialize(&inputs).map_err(|err| Status::internal(err.to_string()))?; + let payload = encode_wal_batch(&inputs)?; + if self.path.exists() || self.temp_path.exists() { + return Err(Status::failed_precondition( + "an earlier ingest WAL is still pending; restart the shard to replay it", + )); + } let mut file = fs::File::create(&self.temp_path).map_err(|err| Status::internal(err.to_string()))?; file.write_all(&payload) .map_err(|err| Status::internal(err.to_string()))?; file.sync_all() .map_err(|err| Status::internal(err.to_string()))?; + drop(file); fs::rename(&self.temp_path, &self.path).map_err(|err| Status::internal(err.to_string()))?; + sync_parent_directory(&self.path)?; Ok(()) } @@ -99,20 +216,28 @@ impl IngestWal { return Ok(None); }; let bytes = fs::read(path).map_err(|err| Status::internal(err.to_string()))?; - if bytes.is_empty() { - return Ok(None); - } - let inputs: Vec = match bincode::deserialize(&bytes) { + let inputs = match decode_wal_batch(&bytes) { Ok(inputs) => inputs, - Err(_) => { - self.quarantine_corrupt()?; - return Ok(None); + Err(reason) => { + let quarantined = self.quarantine_corrupt()?; + return Err(Status::data_loss(format!( + "ingest WAL is corrupt ({reason}); preserved as {}", + quarantined + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", ") + ))); } }; let records = inputs.into_iter().map(proto_from_wal_input).collect(); Ok(Some(records)) } + fn has_pending(&self) -> bool { + self.path.exists() || self.temp_path.exists() + } + fn replay(&self, unirust: &mut Unirust, shard_id: u32) -> Result<(), Status> { if let Some(records) = self.load_batch()? { process_ingest_batch(unirust, shard_id, &records)?; @@ -122,33 +247,43 @@ impl IngestWal { } fn clear(&self) -> Result<(), Status> { + let mut removed = false; if self.path.exists() { fs::remove_file(&self.path).map_err(|err| Status::internal(err.to_string()))?; + removed = true; } if self.temp_path.exists() { fs::remove_file(&self.temp_path).map_err(|err| Status::internal(err.to_string()))?; + removed = true; + } + if removed { + sync_parent_directory(&self.path)?; } Ok(()) } - fn quarantine_corrupt(&self) -> Result<(), Status> { + fn quarantine_corrupt(&self) -> Result, Status> { let suffix = SystemTime::now() .duration_since(UNIX_EPOCH) .map_err(|err| Status::internal(err.to_string()))? - .as_secs(); - let corrupt_path = self.path.with_extension(format!("bin.corrupt.{}", suffix)); - if self.path.exists() { - if let Err(err) = fs::rename(&self.path, &corrupt_path) { - if self.path.exists() { - fs::remove_file(&self.path).map_err(|err| Status::internal(err.to_string()))?; - } - return Err(Status::internal(err.to_string())); + .as_nanos(); + let mut quarantined = Vec::new(); + for source in [&self.path, &self.temp_path] { + if !source.exists() { + continue; } + let file_name = source + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| Status::internal("ingest WAL filename is not valid UTF-8"))?; + let destination = source.with_file_name(format!("{file_name}.corrupt.{suffix}")); + fs::rename(source, &destination).map_err(|err| Status::internal(err.to_string()))?; + quarantined.push(destination); } - if self.temp_path.exists() { - fs::remove_file(&self.temp_path).map_err(|err| Status::internal(err.to_string()))?; + if !quarantined.is_empty() { + sync_parent_directory(&self.path)?; } - Ok(()) + Ok(quarantined) } } @@ -384,27 +519,27 @@ impl PerfMetrics { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct IdentityKeyConfig { pub name: String, pub attributes: Vec, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ConstraintKind { Unique, UniqueWithinPerspective, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ConstraintConfig { pub name: String, pub attribute: String, pub kind: ConstraintKind, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DistributedOntologyConfig { pub identity_keys: Vec, pub strong_identifiers: Vec, @@ -420,35 +555,31 @@ impl DistributedOntologyConfig { } } - pub fn build_ontology(&self, store: &mut crate::Store) -> Ontology { + fn ontology_template(&self) -> Ontology { let mut ontology = Ontology::new(); for key in &self.identity_keys { - let attrs: Vec = key - .attributes - .iter() - .map(|attr| store.interner_mut().intern_attr(attr)) - .collect(); - // Store both AttrIds and string names for partitioning support - ontology.add_identity_key(IdentityKey::with_attribute_names( - attrs, - key.attributes.clone(), + ontology.add_identity_key(IdentityKey::from_names( + key.attributes.iter().map(String::as_str).collect(), key.name.clone(), )); } for attr in &self.strong_identifiers { - let attr_id = store.interner_mut().intern_attr(attr); ontology - .add_strong_identifier(StrongIdentifier::new(attr_id, format!("{attr}_strong"))); + .add_strong_identifier(StrongIdentifier::from_name(attr, format!("{attr}_strong"))); } for constraint in &self.constraints { - let attr_id = store.interner_mut().intern_attr(&constraint.attribute); let constraint = match constraint.kind { - ConstraintKind::Unique => Constraint::unique(attr_id, constraint.name.clone()), + ConstraintKind::Unique => { + Constraint::unique_from_name(&constraint.attribute, constraint.name.clone()) + } ConstraintKind::UniqueWithinPerspective => { - Constraint::unique_within_perspective(attr_id, constraint.name.clone()) + Constraint::unique_within_perspective_from_name( + &constraint.attribute, + constraint.name.clone(), + ) } }; ontology.add_constraint(constraint); @@ -457,42 +588,17 @@ impl DistributedOntologyConfig { ontology } + pub fn build_ontology(&self, store: &mut crate::Store) -> Ontology { + let mut ontology = self.ontology_template(); + ontology.intern_attributes(|name| store.interner_mut().intern_attr(name)); + ontology + } + /// Build ontology using a ConcurrentInterner for partitioned mode. /// This ensures AttrIds match between ontology and records built with the same interner. pub fn build_ontology_with_interner(&self, interner: &ConcurrentInterner) -> Ontology { - let mut ontology = Ontology::new(); - - for key in &self.identity_keys { - let attrs: Vec = key - .attributes - .iter() - .map(|attr| interner.intern_attr(attr)) - .collect(); - // Store both AttrIds and string names for partitioning support - ontology.add_identity_key(IdentityKey::with_attribute_names( - attrs, - key.attributes.clone(), - key.name.clone(), - )); - } - - for attr in &self.strong_identifiers { - let attr_id = interner.intern_attr(attr); - ontology - .add_strong_identifier(StrongIdentifier::new(attr_id, format!("{attr}_strong"))); - } - - for constraint in &self.constraints { - let attr_id = interner.intern_attr(&constraint.attribute); - let constraint = match constraint.kind { - ConstraintKind::Unique => Constraint::unique(attr_id, constraint.name.clone()), - ConstraintKind::UniqueWithinPerspective => { - Constraint::unique_within_perspective(attr_id, constraint.name.clone()) - } - }; - ontology.add_constraint(constraint); - } - + let mut ontology = self.ontology_template(); + ontology.intern_attributes(|name| interner.intern_attr(name)); ontology } } @@ -528,6 +634,34 @@ fn map_proto_config(config: &proto::OntologyConfig) -> DistributedOntologyConfig } } +fn to_proto_config(config: &DistributedOntologyConfig) -> proto::OntologyConfig { + proto::OntologyConfig { + identity_keys: config + .identity_keys + .iter() + .map(|entry| proto::IdentityKeyConfig { + name: entry.name.clone(), + attributes: entry.attributes.clone(), + }) + .collect(), + strong_identifiers: config.strong_identifiers.clone(), + constraints: config + .constraints + .iter() + .map(|entry| proto::ConstraintConfig { + name: entry.name.clone(), + attribute: entry.attribute.clone(), + kind: match entry.kind { + ConstraintKind::Unique => proto::ConstraintKind::Unique.into(), + ConstraintKind::UniqueWithinPerspective => { + proto::ConstraintKind::UniqueWithinPerspective.into() + } + }, + }) + .collect(), + } +} + #[allow(clippy::result_large_err)] fn wal_from_proto_input(input: &proto::RecordInput) -> Result { let identity = input @@ -580,72 +714,199 @@ fn proto_from_wal_input(input: WalRecordInput) -> proto::RecordInput { } } +type CanonicalDescriptor = (String, String, i64, i64); + +fn validate_record_inputs( + records: &[proto::RecordInput], +) -> Result>, Status> { + let mut unique_records = HashMap::with_capacity(records.len()); + for record in records { + let identity = record + .identity + .as_ref() + .ok_or_else(|| Status::invalid_argument("record identity is required"))?; + if identity.entity_type.is_empty() + || identity.perspective.is_empty() + || identity.uid.is_empty() + { + return Err(Status::invalid_argument( + "record identity fields must not be empty", + )); + } + + let mut descriptors = Vec::with_capacity(record.descriptors.len()); + for descriptor in &record.descriptors { + Interval::new(descriptor.start, descriptor.end) + .map_err(|err| Status::invalid_argument(err.to_string()))?; + descriptors.push(( + descriptor.attr.clone(), + descriptor.value.clone(), + descriptor.start, + descriptor.end, + )); + } + descriptors.sort_unstable(); + + let identity = RecordIdentity::new( + identity.entity_type.clone(), + identity.perspective.clone(), + identity.uid.clone(), + ); + if let Some(previous) = unique_records.insert(identity, descriptors.clone()) { + if previous != descriptors { + return Err(Status::invalid_argument( + "source record identity appears more than once with different payloads", + )); + } + } + } + Ok(unique_records) +} + +fn hash_route_component(hasher: &mut Sha256, value: &str) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value.as_bytes()); +} + +fn source_identity_hash(identity: &proto::RecordIdentity) -> u64 { + let mut hasher = Sha256::new(); + hasher.update(b"unirust.source-owner.v1"); + hash_route_component(&mut hasher, &identity.entity_type); + hash_route_component(&mut hasher, &identity.perspective); + hash_route_component(&mut hasher, &identity.uid); + let digest = hasher.finalize(); + u64::from_be_bytes( + digest[..8] + .try_into() + .expect("SHA-256 digest contains eight routing bytes"), + ) +} + +#[doc(hidden)] +pub fn hash_source_identity_to_shard( + identity: &proto::RecordIdentity, + shard_count: usize, +) -> usize { + (source_identity_hash(identity) % shard_count as u64) as usize +} + +fn canonical_record_payload_digest(record: &proto::RecordInput) -> Result<[u8; 32], Status> { + let identity = record + .identity + .as_ref() + .ok_or_else(|| Status::invalid_argument("record identity is required"))?; + let mut descriptors = record + .descriptors + .iter() + .map(|descriptor| { + ( + descriptor.attr.as_str(), + descriptor.value.as_str(), + descriptor.start, + descriptor.end, + ) + }) + .collect::>(); + descriptors.sort_unstable(); + + let mut hasher = Sha256::new(); + hasher.update(b"unirust.source-payload.v1"); + hash_route_component(&mut hasher, &identity.entity_type); + hash_route_component(&mut hasher, &identity.perspective); + hash_route_component(&mut hasher, &identity.uid); + hasher.update((descriptors.len() as u64).to_be_bytes()); + for (attribute, value, start, end) in descriptors { + hash_route_component(&mut hasher, attribute); + hash_route_component(&mut hasher, value); + hasher.update(start.to_be_bytes()); + hasher.update(end.to_be_bytes()); + } + Ok(hasher.finalize().into()) +} + fn hash_record_to_u64(config: &DistributedOntologyConfig, record: &proto::RecordInput) -> u64 { let identity = record.identity.as_ref(); - let mut descriptors_by_attr: HashMap<&str, &str> = HashMap::new(); + let mut descriptors_by_attr: HashMap<&str, Vec<&str>> = HashMap::new(); for descriptor in &record.descriptors { descriptors_by_attr .entry(descriptor.attr.as_str()) - .or_insert(descriptor.value.as_str()); + .or_default() + .push(descriptor.value.as_str()); + } + for values in descriptors_by_attr.values_mut() { + values.sort_unstable(); + values.dedup(); } + let mut hasher = Sha256::new(); for key in &config.identity_keys { - let mut values = Vec::new(); - let mut has_all = true; - for attr in &key.attributes { - if let Some(value) = descriptors_by_attr.get(attr.as_str()) { - values.push(*value); - } else { - has_all = false; - break; - } - } - if has_all { - let mut state = std::collections::hash_map::DefaultHasher::new(); + let values = key + .attributes + .iter() + .map(|attribute| { + descriptors_by_attr + .get(attribute.as_str()) + .and_then(|values| values.first()) + .copied() + }) + .collect::>>(); + if let Some(values) = values { + hasher.update(b"unirust.identity-route.v1"); if let Some(identity) = identity { - identity.entity_type.hash(&mut state); + hash_route_component(&mut hasher, &identity.entity_type); } - key.name.hash(&mut state); - for value in values { - value.hash(&mut state); + hash_route_component(&mut hasher, &key.name); + for (attribute, value) in key.attributes.iter().zip(values) { + hash_route_component(&mut hasher, attribute); + hash_route_component(&mut hasher, value); } - return state.finish(); + let digest = hasher.finalize(); + return u64::from_be_bytes( + digest[..8] + .try_into() + .expect("SHA-256 digest contains eight routing bytes"), + ); } } for constraint in &config.constraints { - if let Some(value) = descriptors_by_attr.get(constraint.attribute.as_str()) { - let mut state = std::collections::hash_map::DefaultHasher::new(); + if let Some(value) = descriptors_by_attr + .get(constraint.attribute.as_str()) + .and_then(|values| values.first()) + { + hasher.update(b"unirust.constraint-route.v1"); if let Some(identity) = identity { - identity.entity_type.hash(&mut state); + hash_route_component(&mut hasher, &identity.entity_type); if matches!(constraint.kind, ConstraintKind::UniqueWithinPerspective) { - identity.perspective.hash(&mut state); + hash_route_component(&mut hasher, &identity.perspective); } } - constraint.name.hash(&mut state); - constraint.attribute.hash(&mut state); - value.hash(&mut state); - return state.finish(); + hash_route_component(&mut hasher, &constraint.name); + hash_route_component(&mut hasher, &constraint.attribute); + hash_route_component(&mut hasher, value); + let digest = hasher.finalize(); + return u64::from_be_bytes( + digest[..8] + .try_into() + .expect("SHA-256 digest contains eight routing bytes"), + ); } } - let mut state = std::collections::hash_map::DefaultHasher::new(); + hasher.update(b"unirust.source-route.v1"); if let Some(identity) = identity { - identity.entity_type.hash(&mut state); - identity.perspective.hash(&mut state); - identity.uid.hash(&mut state); - } - state.finish() -} - -/// Mix hash bits for better distribution with small modulo values. -/// Uses fibonacci hashing to reduce clustering. -#[inline] -fn mix_hash(hash: u64) -> u64 { - // Fibonacci hashing: multiply by golden ratio and take high bits - // This provides excellent distribution for small modulo values - const GOLDEN_RATIO: u64 = 0x9E3779B97F4A7C15; - hash.wrapping_mul(GOLDEN_RATIO) + hash_route_component(&mut hasher, &identity.entity_type); + hash_route_component(&mut hasher, &identity.perspective); + hash_route_component(&mut hasher, &identity.uid); + } else { + hasher.update(record.index.to_be_bytes()); + } + let digest = hasher.finalize(); + u64::from_be_bytes( + digest[..8] + .try_into() + .expect("SHA-256 digest contains eight routing bytes"), + ) } pub fn hash_record_to_shard( @@ -654,9 +915,115 @@ pub fn hash_record_to_shard( shard_count: usize, ) -> usize { let hash = hash_record_to_u64(config, record); - let mixed = mix_hash(hash); - // Use high bits which have better distribution after mixing - ((mixed >> 32) as usize) % shard_count + (hash % shard_count as u64) as usize +} + +fn global_cluster_id_to_proto(id: GlobalClusterId) -> proto::GlobalClusterId { + proto::GlobalClusterId { + shard_id: u32::from(id.shard_id), + local_id: id.local_id, + version: u32::from(id.version), + } +} + +fn cross_shard_conflict_to_proto( + conflict: &crate::sharding::CrossShardConflict, +) -> proto::CrossShardConflict { + proto::CrossShardConflict { + identity_key_signature: Some(proto::IdentityKeySignature { + signature: conflict.identity_key_signature.to_bytes().to_vec(), + }), + cluster1: Some(global_cluster_id_to_proto(conflict.cluster1)), + cluster2: Some(global_cluster_id_to_proto(conflict.cluster2)), + interval_start: conflict.interval.start, + interval_end: conflict.interval.end, + perspective_hash: conflict.perspective_hash, + strong_id_hash1: conflict.strong_id_hash1, + strong_id_hash2: conflict.strong_id_hash2, + } +} + +fn boundary_strong_id_to_proto( + strong_id: crate::sharding::BoundaryStrongId, +) -> proto::BoundaryStrongId { + proto::BoundaryStrongId { + perspective: strong_id.perspective, + attribute: strong_id.attribute, + value: strong_id.value, + interval_start: strong_id.interval.start, + interval_end: strong_id.interval.end, + } +} + +#[allow(clippy::result_large_err)] +fn boundary_index_from_metadata( + metadata: &proto::BoundaryMetadata, +) -> Result { + let shard_id = u16::try_from(metadata.shard_id) + .map_err(|_| Status::invalid_argument("boundary shard_id exceeds u16"))?; + let mut boundary_index = crate::sharding::ClusterBoundaryIndex::new_small(shard_id); + + for key_entries in &metadata.entries { + let signature = key_entries + .signature + .as_ref() + .ok_or_else(|| Status::data_loss("boundary entry is missing its signature"))?; + let signature_bytes: [u8; 32] = signature + .signature + .as_slice() + .try_into() + .map_err(|_| Status::data_loss("boundary signature must be 32 bytes"))?; + let signature = IdentityKeySignature::from_bytes(signature_bytes); + + for entry in &key_entries.entries { + if entry.shard_id != metadata.shard_id { + return Err(Status::data_loss(format!( + "boundary entry shard_id {} does not match metadata shard_id {}", + entry.shard_id, metadata.shard_id + ))); + } + let cluster = entry + .cluster_id + .as_ref() + .ok_or_else(|| Status::data_loss("boundary entry is missing its cluster ID"))?; + let cluster_shard = u16::try_from(cluster.shard_id) + .map_err(|_| Status::data_loss("boundary cluster shard_id exceeds u16"))?; + let cluster_version = u16::try_from(cluster.version) + .map_err(|_| Status::data_loss("boundary cluster version exceeds u16"))?; + let interval = Interval::new(entry.interval_start, entry.interval_end) + .map_err(|err| Status::data_loss(err.to_string()))?; + let strong_ids = entry + .strong_ids + .iter() + .map(|strong_id| { + if strong_id.perspective.is_empty() + || strong_id.attribute.is_empty() + || strong_id.value.is_empty() + { + return Err(Status::data_loss( + "boundary strong-ID fields must not be empty", + )); + } + Ok(crate::sharding::BoundaryStrongId { + perspective: strong_id.perspective.clone(), + attribute: strong_id.attribute.clone(), + value: strong_id.value.clone(), + interval: Interval::new(strong_id.interval_start, strong_id.interval_end) + .map_err(|err| Status::data_loss(err.to_string()))?, + }) + }) + .collect::, Status>>()?; + boundary_index.register_boundary_key_with_conflict_data( + signature, + GlobalClusterId::new(cluster_shard, cluster.local_id, cluster_version), + interval, + entry.perspective_strong_ids.clone(), + strong_ids, + ); + } + } + + Ok(boundary_index) } #[derive(Clone)] @@ -674,11 +1041,17 @@ pub struct ShardNode { tuning: StreamingTuning, ontology_config: Arc>, data_dir: Option, + restored_checkpoint: Option, + checkpoint_root: Option, ingest_wal: Option>, ingest_wal_lock: Arc>, + mutation_gate: Arc>, ingest_txs: Vec>, + ingest_worker_handles: Arc>>>, config_version: String, metrics: Arc, + /// Destructive RPCs are disabled unless explicitly enabled by the embedding process. + allow_destructive_admin: bool, /// Cross-shard conflicts detected during reconciliation. /// These are indirect conflicts where clusters on different shards share /// an identity key but have conflicting strong identifiers. @@ -744,11 +1117,36 @@ impl ShardNode { repair_on_start: bool, config_version: Option, ) -> AnyResult { + Self::new_with_storage_paths( + shard_id, + ontology_config, + tuning, + data_dir, + None, + repair_on_start, + config_version, + ) + } + + pub fn new_with_storage_paths( + shard_id: u32, + ontology_config: DistributedOntologyConfig, + tuning: StreamingTuning, + data_dir: Option, + checkpoint_root: Option, + repair_on_start: bool, + config_version: Option, + ) -> AnyResult { + if data_dir.is_none() && checkpoint_root.is_some() { + anyhow::bail!("checkpoint root requires persistent shard storage"); + } + let shard_id_u16 = u16::try_from(shard_id) + .map_err(|_| anyhow::anyhow!("shard_id {shard_id} exceeds u16"))?; // Set shard_id in tuning and enable boundary tracking for cross-shard reconciliation. // Boundary tracking is REQUIRED for distributed mode - it enables cross-shard conflict // detection. Without it, conflicts between records on different shards go undetected. let tuning = tuning - .with_shard_id(shard_id as u16) + .with_shard_id(shard_id_u16) .with_boundary_tracking(true); // Defensive check: fail fast if boundary tracking is somehow disabled. @@ -763,26 +1161,71 @@ impl ShardNode { let config_version = config_version.unwrap_or_else(|| "unversioned".to_string()); let worker_count = ingest_worker_count(); - let use_partitioned = is_partitioned_enabled(); + let partitioned_requested = is_partitioned_enabled(); + // Partition stores are currently in-memory. Persistent shards must keep every + // ingest on the PersistentStore path until partitions have a durable backend. + let use_partitioned = partitioned_requested && data_dir.is_none(); let num_partitions = partition_count(); if use_partitioned { tracing::info!(shard_id, num_partitions, "Partitioned processing enabled"); + } else if partitioned_requested && data_dir.is_some() { + tracing::info!( + shard_id, + "Partitioned processing disabled for persistent shard" + ); } // Create concurrent interner FIRST - used for both ontology and records let concurrent_interner = Arc::new(ConcurrentInterner::new()); if let Some(path) = data_dir.clone() { + let checkpoint_root = if let Some(checkpoint_root) = checkpoint_root { + fs::create_dir_all(&path)?; + fs::create_dir_all(&checkpoint_root)?; + let data_path = path.canonicalize()?; + let backup_path = checkpoint_root.canonicalize()?; + if backup_path.starts_with(&data_path) || data_path.starts_with(&backup_path) { + anyhow::bail!( + "external checkpoint root {} and data directory {} must be disjoint", + backup_path.display(), + data_path.display() + ); + } + backup_path + } else { + let checkpoint_root = path.join("checkpoints"); + fs::create_dir_all(&checkpoint_root)?; + checkpoint_root.canonicalize()? + }; + let restored_checkpoint = read_restored_checkpoint_manifest(&path)?; + if let Some(manifest) = &restored_checkpoint { + if manifest.shard_id() != shard_id { + anyhow::bail!( + "restored checkpoint belongs to shard {}, not configured shard {}", + manifest.shard_id(), + shard_id + ); + } + } let (store, config, ontology) = load_persistent_state(&path, ontology_config, repair_on_start)?; let ingest_wal = Some(Arc::new(IngestWal::new(&path))); let mut unirust = Unirust::with_store_and_tuning(ontology.clone(), store, tuning.clone()); + let recovery_started = Instant::now(); + unirust.initialize_streaming()?; if let Some(wal) = ingest_wal.as_ref() { wal.replay(&mut unirust, shard_id) .map_err(|err| anyhow::anyhow!(err.to_string()))?; } + tracing::info!( + shard_id, + records = unirust.record_count(), + elapsed_ms = recovery_started.elapsed().as_millis(), + "persistent linker recovery completed" + ); + let recovered_cross_shard_conflicts = unirust.load_cross_shard_conflicts()?; let unirust = Arc::new(parking_lot::RwLock::new(unirust)); // Create partitioned processor if enabled - no RwLock needed! @@ -800,7 +1243,8 @@ impl ShardNode { None }; - let ingest_txs = spawn_ingest_workers(unirust.clone(), shard_id, worker_count); + let (ingest_txs, ingest_worker_handles) = + spawn_ingest_workers(unirust.clone(), shard_id, worker_count); return Ok(Self { shard_id, unirust, @@ -809,12 +1253,19 @@ impl ShardNode { tuning, ontology_config: Arc::new(Mutex::new(config)), data_dir: Some(path), + restored_checkpoint, + checkpoint_root: Some(checkpoint_root), ingest_wal, ingest_wal_lock: Arc::new(Mutex::new(())), + mutation_gate: Arc::new(tokio::sync::RwLock::new(())), ingest_txs, + ingest_worker_handles: Arc::new(Mutex::new(ingest_worker_handles)), config_version, metrics: Arc::new(PerfMetrics::new()), - cross_shard_conflicts: Arc::new(parking_lot::RwLock::new(Vec::new())), + allow_destructive_admin: false, + cross_shard_conflicts: Arc::new(parking_lot::RwLock::new( + recovered_cross_shard_conflicts, + )), }); } @@ -852,7 +1303,8 @@ impl ShardNode { None }; - let ingest_txs = spawn_ingest_workers(unirust.clone(), shard_id, worker_count); + let (ingest_txs, ingest_worker_handles) = + spawn_ingest_workers(unirust.clone(), shard_id, worker_count); Ok(Self { shard_id, unirust, @@ -861,15 +1313,148 @@ impl ShardNode { tuning, ontology_config: Arc::new(Mutex::new(ontology_config)), data_dir: None, + restored_checkpoint: None, + checkpoint_root: None, ingest_wal, ingest_wal_lock: Arc::new(Mutex::new(())), + mutation_gate: Arc::new(tokio::sync::RwLock::new(())), ingest_txs, + ingest_worker_handles: Arc::new(Mutex::new(ingest_worker_handles)), config_version, metrics: Arc::new(PerfMetrics::new()), + allow_destructive_admin: false, cross_shard_conflicts: Arc::new(parking_lot::RwLock::new(Vec::new())), }) } + /// Explicitly enable destructive admin RPCs such as `Reset`. + /// + /// Production binaries leave these disabled by default. Prefer an offline + /// reset of stopped shard data directories whenever possible. + pub fn with_destructive_admin(mut self, allow: bool) -> Self { + self.allow_destructive_admin = allow; + self + } + + /// Drain mutations and durably flush derived state before process exit. + pub async fn shutdown(&self) -> AnyResult<()> { + let _mutation_guard = self.mutation_gate.write().await; + let _wal_guard = self.ingest_wal_lock.lock().await; + if self.data_dir.is_some() { + let mut unirust = write_unirust!(self); + unirust.checkpoint_for_shutdown()?; + if self + .ingest_wal + .as_ref() + .is_some_and(|wal| wal.has_pending()) + { + tracing::warn!( + shard_id = self.shard_id, + "shutdown completed with a pending ingest WAL; it will replay on restart" + ); + } + } + + let worker_handles = { + let mut handles = self.ingest_worker_handles.lock().await; + std::mem::take(&mut *handles) + }; + for handle in worker_handles { + handle.abort(); + let _ = handle.await; + } + Ok(()) + } + + fn ensure_no_pending_ingest(&self) -> Result<(), Status> { + if self + .ingest_wal + .as_ref() + .is_some_and(|wal| wal.has_pending()) + { + return if self.ingest_wal_lock.try_lock().is_ok() { + Err(Status::failed_precondition( + "ingest recovery is pending; restart the shard before serving more traffic", + )) + } else { + Err(Status::unavailable("an ingest commit is in progress")) + }; + } + let unirust = read_unirust!(self); + unirust.ensure_store_healthy().map_err(|err| { + Status::failed_precondition(format!( + "persistent store is unhealthy; restart the shard: {err}" + )) + }) + } + + fn ensure_recovery_healthy(&self) -> Result<(), Status> { + if self + .ingest_wal + .as_ref() + .is_some_and(|wal| wal.has_pending()) + && self.ingest_wal_lock.try_lock().is_ok() + { + return Err(Status::failed_precondition( + "ingest recovery is pending; restart the shard", + )); + } + let unirust = read_unirust!(self); + unirust.ensure_store_healthy().map_err(|err| { + Status::failed_precondition(format!( + "persistent store is unhealthy; restart the shard: {err}" + )) + }) + } + + fn ensure_store_healthy(&self) -> Result<(), Status> { + let unirust = read_unirust!(self); + unirust.ensure_store_healthy().map_err(|err| { + Status::failed_precondition(format!( + "persistent store is unhealthy; restart the shard: {err}" + )) + }) + } + + fn ensure_ingest_payloads_idempotent( + &self, + records: &[proto::RecordInput], + ) -> Result<(), Status> { + let unique_records = validate_record_inputs(records)?; + let unirust = read_unirust!(self); + for (identity, incoming_descriptors) in unique_records { + let Some(existing) = unirust + .get_record_by_identity(&identity) + .map_err(|err| Status::failed_precondition(err.to_string()))? + else { + continue; + }; + + let mut existing_descriptors = Vec::with_capacity(existing.descriptors.len()); + for descriptor in &existing.descriptors { + let attr = unirust.resolve_attr(descriptor.attr).ok_or_else(|| { + Status::data_loss("stored record references an unknown attribute") + })?; + let value = unirust.resolve_value(descriptor.value).ok_or_else(|| { + Status::data_loss("stored record references an unknown value") + })?; + existing_descriptors.push(( + attr, + value, + descriptor.interval.start, + descriptor.interval.end, + )); + } + existing_descriptors.sort_unstable(); + if existing_descriptors != incoming_descriptors { + return Err(Status::already_exists( + "source record identity already exists with a different payload", + )); + } + } + Ok(()) + } + /// Build a record using the concurrent interner - NO LOCK REQUIRED! #[allow(clippy::result_large_err)] fn build_record_concurrent( @@ -984,49 +1569,126 @@ impl ShardNode { } } - fn to_proto_match( - shard_id: u32, - cluster_id: crate::model::ClusterId, - interval: Interval, - golden: &[GoldenDescriptor], - cluster_key: Option, - cluster_key_identity: Option, - ) -> proto::QueryMatch { - proto::QueryMatch { - shard_id, - cluster_id: cluster_id.0, - start: interval.start, - end: interval.end, - cluster_key: cluster_key.unwrap_or_default(), - cluster_key_identity: cluster_key_identity.unwrap_or_default(), - golden: golden - .iter() - .map(|descriptor| proto::GoldenDescriptor { - attr: descriptor.attr.clone(), - value: descriptor.value.clone(), - start: descriptor.interval.start, - end: descriptor.interval.end, - }) - .collect(), - } - } + #[allow(clippy::result_large_err)] + fn build_import_records( + unirust: &mut Unirust, + snapshots: &[proto::RecordSnapshot], + ) -> Result, Status> { + let mut records = Vec::with_capacity(snapshots.len()); + let mut batch_ids: HashMap = HashMap::new(); + let mut batch_identities: HashMap<(String, String, String), u32> = HashMap::new(); + + for snapshot in snapshots { + let identity = snapshot + .identity + .as_ref() + .ok_or_else(|| Status::invalid_argument("record identity is required"))?; + if let Some(previous) = batch_ids.get(&snapshot.record_id) { + if **previous != *snapshot { + return Err(Status::already_exists(format!( + "import batch contains conflicting content for record ID {}", + snapshot.record_id + ))); + } + continue; + } + batch_ids.insert(snapshot.record_id, snapshot); + + let identity_key = ( + identity.entity_type.clone(), + identity.perspective.clone(), + identity.uid.clone(), + ); + if let Some(previous_id) = + batch_identities.insert(identity_key.clone(), snapshot.record_id) + { + if previous_id != snapshot.record_id { + return Err(Status::already_exists(format!( + "import batch maps one record identity to IDs {previous_id} and {}", + snapshot.record_id + ))); + } + } + + if let Some(existing) = unirust.get_record(RecordId(snapshot.record_id)) { + if Self::record_to_snapshot(unirust, &existing) != *snapshot { + return Err(Status::already_exists(format!( + "record ID {} already exists with different content", + snapshot.record_id + ))); + } + continue; + } + + let record_identity = + RecordIdentity::new(identity_key.0, identity_key.1, identity_key.2); + if let Some(existing_id) = unirust.store().get_record_id_by_identity(&record_identity) { + return Err(Status::already_exists(format!( + "record identity already exists under ID {} instead of requested ID {}", + existing_id.0, snapshot.record_id + ))); + } + records.push(Self::build_record_with_id( + unirust, + snapshot.record_id, + identity, + &snapshot.descriptors, + )?); + } + unirust + .ensure_store_healthy() + .map_err(|err| Status::failed_precondition(err.to_string()))?; + Ok(records) + } + + fn to_proto_match( + shard_id: u32, + cluster_id: crate::model::ClusterId, + interval: Interval, + golden: &[GoldenDescriptor], + cluster_key: Option, + cluster_key_identity: Option, + ) -> proto::QueryMatch { + proto::QueryMatch { + shard_id, + cluster_id: cluster_id.0, + start: interval.start, + end: interval.end, + cluster_key: cluster_key.unwrap_or_default(), + cluster_key_identity: cluster_key_identity.unwrap_or_default(), + golden: golden + .iter() + .map(|descriptor| proto::GoldenDescriptor { + attr: descriptor.attr.clone(), + value: descriptor.value.clone(), + start: descriptor.interval.start, + end: descriptor.interval.end, + }) + .collect(), + } + } } #[allow(clippy::result_large_err)] -fn resolve_checkpoint_path(data_dir: &Path, requested: &str) -> Result { +fn resolve_checkpoint_path(checkpoint_root: &Path, requested: &str) -> Result { if requested.is_empty() { let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .map_err(|err| Status::internal(err.to_string()))? - .as_secs(); - return Ok(data_dir.join("checkpoints").join(format!("{timestamp}"))); + .as_nanos(); + return Ok(checkpoint_root.join(format!("{timestamp}"))); } let candidate = PathBuf::from(requested); - if candidate.is_absolute() { - Ok(candidate) - } else { - Ok(data_dir.join(candidate)) + if candidate.is_absolute() + || candidate + .components() + .any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err(Status::invalid_argument( + "checkpoint path must be relative and remain within the shard checkpoint directory", + )); } + Ok(checkpoint_root.join(candidate)) } fn ingest_worker_count() -> usize { @@ -1055,12 +1717,16 @@ fn spawn_ingest_workers( unirust: Arc>, shard_id: u32, worker_count: usize, -) -> Vec> { +) -> ( + Vec>, + Vec>, +) { let mut senders = Vec::with_capacity(worker_count); + let mut handles = Vec::with_capacity(worker_count); for _ in 0..worker_count { let (tx, mut rx) = mpsc::channel::(INGEST_QUEUE_CAPACITY); let worker_unirust = unirust.clone(); - tokio::spawn(async move { + let handle = tokio::spawn(async move { while let Some(job) = rx.recv().await { let result = { let mut guard = worker_unirust.write(); @@ -1070,8 +1736,9 @@ fn spawn_ingest_workers( } }); senders.push(tx); + handles.push(handle); } - senders + (senders, handles) } async fn dispatch_ingest_records( @@ -1206,20 +1873,13 @@ async fn dispatch_ingest_partitioned( let indexed_records: Vec<(usize, u32, Record)> = if records.len() > 500 { records .par_iter() - .filter_map(|record| { - ShardNode::build_record_concurrent(interner, record) - .ok() - .map(|r| { - let partition_id = compute_partition_id_for_record( - &r, - ontology, - interner, - partition_count, - ); - (partition_id, record.index, r) - }) + .map(|record| { + let built = ShardNode::build_record_concurrent(interner, record)?; + let partition_id = + compute_partition_id_for_record(&built, ontology, interner, partition_count); + Ok((partition_id, record.index, built)) }) - .collect() + .collect::, Status>>()? } else { let mut indexed_records = Vec::with_capacity(records.len()); for record in &records { @@ -1232,7 +1892,9 @@ async fn dispatch_ingest_partitioned( }; // Phase 2: REAL entity resolution via partitioned processing with pre-computed partition IDs - let partition_results = partitioned.ingest_batch_with_partitions(indexed_records); + let partition_results = partitioned + .ingest_batch_with_partitions(indexed_records) + .map_err(|err| Status::internal(err.to_string()))?; // Phase 3: Convert to proto assignments let mut assignments = Vec::with_capacity(partition_results.len()); @@ -1292,6 +1954,54 @@ fn process_ingest_batch( Ok(assignments) } +fn build_global_conflict_summaries( + config: DistributedOntologyConfig, + snapshots: Vec, +) -> AnyResult> { + let mut store = crate::Store::new(); + let ontology = config.build_ontology(&mut store); + let mut unirust = Unirust::with_store_and_tuning( + ontology, + store, + StreamingTuning::from_profile(crate::TuningProfile::Balanced), + ); + let mut batch = Vec::with_capacity(1_000); + + for snapshot in snapshots { + let identity = snapshot + .identity + .ok_or_else(|| anyhow::anyhow!("exported record is missing its identity"))?; + let mut descriptors = Vec::with_capacity(snapshot.descriptors.len()); + for descriptor in snapshot.descriptors { + descriptors.push(crate::Descriptor::new( + unirust.intern_attr(&descriptor.attr), + unirust.intern_value(&descriptor.value), + Interval::new(descriptor.start, descriptor.end)?, + )); + } + batch.push(Record::new( + RecordId(0), + RecordIdentity::new(identity.entity_type, identity.perspective, identity.uid), + descriptors, + )); + if batch.len() == 1_000 { + unirust.stream_records(std::mem::take(&mut batch))?; + } + } + if !batch.is_empty() { + unirust.stream_records(batch)?; + } + + let clusters = unirust.build_clusters()?; + let observations = unirust.detect_conflicts(&clusters)?; + let summaries = unirust + .summarize_conflicts(&observations) + .into_iter() + .map(to_proto_conflict_summary) + .collect::>(); + Ok(summaries) +} + fn load_persistent_state( path: &Path, fallback_config: DistributedOntologyConfig, @@ -1323,10 +2033,115 @@ impl proto::shard_service_server::ShardService for ShardNode { type ExportRecordsStreamStream = Pin> + Send + 'static>>; + async fn reserve_source_records( + &self, + request: Request, + ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_store_healthy()?; + let request = request.into_inner(); + let mut reservations = Vec::with_capacity(request.reservations.len()); + let mut indices = Vec::with_capacity(request.reservations.len()); + + for reservation in request.reservations { + let identity = reservation + .identity + .ok_or_else(|| Status::invalid_argument("reservation identity is required"))?; + if identity.entity_type.is_empty() + || identity.perspective.is_empty() + || identity.uid.is_empty() + { + return Err(Status::invalid_argument( + "reservation identity fields must not be empty", + )); + } + let payload_digest: [u8; 32] = reservation + .payload_digest + .as_slice() + .try_into() + .map_err(|_| Status::invalid_argument("payload digest must be 32 bytes"))?; + indices.push(reservation.index); + reservations.push(SourceRecordReservation { + identity: RecordIdentity::new( + identity.entity_type, + identity.perspective, + identity.uid, + ), + payload_digest, + target_shard_id: reservation.target_shard_id, + }); + } + + let targets = { + let mut unirust = write_unirust!(self); + unirust + .store_mut() + .reserve_source_records(&reservations) + .map_err(|err| { + if let Some(conflict) = err.downcast_ref::() { + match conflict { + SourceReservationError::PayloadConflict => { + Status::already_exists(conflict.to_string()) + } + SourceReservationError::TargetConflict { .. } => { + Status::failed_precondition(conflict.to_string()) + } + } + } else { + Status::internal(err.to_string()) + } + })? + }; + let reservations = indices + .into_iter() + .zip(targets) + .map( + |(index, target_shard_id)| proto::SourceRecordReservationResult { + index, + target_shard_id, + }, + ) + .collect(); + Ok(Response::new(proto::ReserveSourceRecordsResponse { + reservations, + })) + } + + async fn mark_source_reservations_backfilled( + &self, + request: Request, + ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_store_healthy()?; + let request = request.into_inner(); + if request.protocol_version != DISTRIBUTED_PROTOCOL_VERSION { + return Err(Status::failed_precondition(format!( + "cannot mark source reservations for protocol {}; shard requires {}", + request.protocol_version, DISTRIBUTED_PROTOCOL_VERSION + ))); + } + if request.shard_count == 0 { + return Err(Status::invalid_argument( + "source reservation shard count must be greater than zero", + )); + } + + let mut unirust = write_unirust!(self); + unirust + .store_mut() + .mark_source_reservation_backfill(request.protocol_version, request.shard_count) + .map_err(|err| Status::internal(err.to_string()))?; + Ok(Response::new( + proto::MarkSourceReservationsBackfilledResponse {}, + )) + } + async fn set_ontology( &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.write().await; + self.ensure_no_pending_ingest()?; let config = request .into_inner() .config @@ -1334,7 +2149,20 @@ impl proto::shard_service_server::ShardService for ShardNode { let config = map_proto_config(&config); let mut config_guard = self.ontology_config.lock().await; - *config_guard = config.clone(); + if *config_guard == config { + return Ok(Response::new(proto::ApplyOntologyResponse {})); + } + + let record_count = { + let guard = read_unirust!(self); + guard.record_count() + }; + if record_count != 0 { + return Err(Status::failed_precondition(format!( + "refusing to replace ontology on a nonempty shard ({record_count} records); \ + reset the cluster explicitly before changing ontology" + ))); + } if let Some(path) = &self.data_dir { // Must acquire write lock and drop old Unirust BEFORE opening new store @@ -1373,11 +2201,12 @@ impl proto::shard_service_server::ShardService for ShardNode { let mut guard = write_unirust!(self); *guard = Unirust::with_store_and_tuning(ontology, store, self.tuning.clone()); } + self.cross_shard_conflicts.write().clear(); // Rebuild partitioned processor with the new ontology if enabled // CRITICAL: The partitioned processor needs the ontology with strong identifiers // for conflict detection to work! - if is_partitioned_enabled() { + if is_partitioned_enabled() && self.data_dir.is_none() { let num_partitions = partition_count(); let partition_config = PartitionConfig::for_cores(num_partitions); // Build ontology using concurrent interner to ensure AttrIds match records @@ -1393,8 +2222,11 @@ impl proto::shard_service_server::ShardService for ShardNode { let mut partitioned_guard = self.partitioned.write(); *partitioned_guard = Some(Arc::new(new_partitioned)); tracing::info!("Partitioned processor rebuilt with updated ontology"); + } else { + *self.partitioned.write() = None; } + *config_guard = config; Ok(Response::new(proto::ApplyOntologyResponse {})) } @@ -1402,14 +2234,19 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; let start = Instant::now(); - let records = request.into_inner().records; + let request = request.into_inner(); + validate_distributed_protocol(request.internal_protocol_version)?; + let records = request.records; let record_count = records.len(); let _wal_guard = if self.ingest_wal.is_some() { Some(self.ingest_wal_lock.lock().await) } else { None }; + self.ensure_store_healthy()?; + self.ensure_ingest_payloads_idempotent(&records)?; // Use partitioned processing for large batches (high-throughput mode) // Small batches use sequential path for correctness (query support) @@ -1438,6 +2275,7 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request>, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; let start = Instant::now(); let mut stream = request.into_inner(); let mut assignments = Vec::new(); @@ -1451,12 +2289,15 @@ impl proto::shard_service_server::ShardService for ShardNode { if chunk.records.is_empty() { continue; } + validate_distributed_protocol(chunk.internal_protocol_version)?; record_count += chunk.records.len(); let _wal_guard = if self.ingest_wal.is_some() { Some(self.ingest_wal_lock.lock().await) } else { None }; + self.ensure_store_healthy()?; + self.ensure_ingest_payloads_idempotent(&chunk.records)?; // Use partitioned processing for large batches (high-throughput mode) // Small batches use sequential path for correctness @@ -1497,6 +2338,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let start = Instant::now(); let mut unirust = write_unirust!(self); let request = request.into_inner(); @@ -1523,16 +2366,19 @@ impl proto::shard_service_server::ShardService for ShardNode { matches: matches .into_iter() .map(|entry| { - Self::to_proto_match( - self.shard_id, - entry.cluster_id, + let global_id = unirust + .global_cluster_id_for_record(entry.root_record_id) + .map_err(|err| Status::internal(err.to_string()))?; + Ok(Self::to_proto_match( + u32::from(global_id.shard_id), + crate::model::ClusterId(global_id.local_id), entry.interval, &entry.golden, entry.cluster_key, entry.cluster_key_identity, - ) + )) }) - .collect(), + .collect::, Status>>()?, }, )), }, @@ -1590,6 +2436,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, _request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let unirust = read_unirust!(self); let partitioned_guard = self.partitioned.read(); @@ -1641,7 +2489,7 @@ impl proto::shard_service_server::ShardService for ShardNode { graph_node_count, graph_edge_count, cross_shard_merges, - cross_shard_conflicts: 0, // Tracked at router level during reconcile + cross_shard_conflicts: self.cross_shard_conflicts.read().len() as u64, boundary_keys_tracked: boundary_keys, })) } @@ -1650,6 +2498,7 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, _request: Request, ) -> Result, Status> { + self.ensure_recovery_healthy()?; Ok(Response::new(proto::HealthCheckResponse { status: "ok".to_string(), })) @@ -1659,8 +2508,32 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, _request: Request, ) -> Result, Status> { + let ontology_config = self.ontology_config.lock().await; + let source_reservation_backfill = read_unirust!(self) + .store() + .source_reservation_backfill() + .map_err(|err| Status::internal(err.to_string()))?; Ok(Response::new(proto::ConfigVersionResponse { version: self.config_version.clone(), + ontology_config: Some(to_proto_config(&ontology_config)), + protocol_version: DISTRIBUTED_PROTOCOL_VERSION, + source_reservation_backfill_version: source_reservation_backfill + .map(|value| value.0) + .unwrap_or_default(), + source_reservation_shard_count: source_reservation_backfill + .map(|value| value.1) + .unwrap_or_default(), + checkpoint_protocol_version: CHECKPOINT_PROTOCOL_VERSION, + restore_generation: self + .restored_checkpoint + .as_ref() + .map(|manifest| manifest.generation().to_string()) + .unwrap_or_default(), + restore_shard_count: self + .restored_checkpoint + .as_ref() + .map(ClusterCheckpointManifest::shard_count) + .unwrap_or_default(), })) } @@ -1689,23 +2562,68 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { - let data_dir = self - .data_dir + let _mutation_guard = self.mutation_gate.write().await; + self.ensure_no_pending_ingest()?; + let request = request.into_inner(); + validate_checkpoint_protocol(request.checkpoint_protocol_version)?; + if request.shard_count == 0 || self.shard_id >= request.shard_count { + return Err(Status::invalid_argument(format!( + "checkpoint shard {} is outside cluster shard count {}", + self.shard_id, request.shard_count + ))); + } + let checkpoint_root = self + .checkpoint_root .as_ref() - .ok_or_else(|| Status::failed_precondition("checkpoint requires --data-dir"))?; - let target = resolve_checkpoint_path(data_dir, &request.into_inner().path)?; - fs::create_dir_all( - target - .parent() - .ok_or_else(|| Status::internal("invalid checkpoint path"))?, - ) - .map_err(|err| Status::internal(err.to_string()))?; - let unirust = read_unirust!(self); - unirust - .checkpoint_to_path(&target) + .ok_or_else(|| Status::failed_precondition("checkpoint requires persistent storage"))?; + let target = resolve_checkpoint_path(checkpoint_root, &request.path)?; + let parent = target + .parent() + .ok_or_else(|| Status::internal("invalid checkpoint path"))?; + fs::create_dir_all(parent).map_err(|err| Status::internal(err.to_string()))?; + let parent = parent + .canonicalize() .map_err(|err| Status::internal(err.to_string()))?; + if !parent.starts_with(checkpoint_root) { + return Err(Status::invalid_argument( + "checkpoint path traverses outside the shard checkpoint directory", + )); + } + if target + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(Status::invalid_argument( + "checkpoint target must not be a symbolic link", + )); + } + + if request.finalize { + commit_cluster_checkpoint(&target, &request.path, self.shard_id, request.shard_count) + .map_err(|err| Status::failed_precondition(err.to_string()))?; + } else if target.exists() { + validate_prepared_cluster_checkpoint( + &target, + &request.path, + self.shard_id, + request.shard_count, + ) + .map_err(|err| Status::already_exists(err.to_string()))?; + } else { + let mut unirust = write_unirust!(self); + unirust + .checkpoint_for_shutdown() + .map_err(|err| Status::internal(err.to_string()))?; + unirust + .checkpoint_to_path(&target) + .map_err(|err| Status::internal(err.to_string()))?; + prepare_cluster_checkpoint(&target, &request.path, self.shard_id, request.shard_count) + .map_err(|err| Status::internal(err.to_string()))?; + } Ok(Response::new(proto::CheckpointResponse { paths: vec![target.to_string_lossy().to_string()], + generation: request.path, + committed: request.finalize, })) } @@ -1713,6 +2631,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, _request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let unirust = read_unirust!(self); let record_count = unirust.record_count() as u64; let response = match unirust.record_id_bounds() { @@ -1736,6 +2656,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let request = request.into_inner(); let limit = if request.limit == 0 { EXPORT_DEFAULT_LIMIT @@ -1781,6 +2703,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let mutation_guard = self.mutation_gate.clone().read_owned().await; + self.ensure_no_pending_ingest()?; let request = request.into_inner(); let limit = if request.limit == 0 { EXPORT_DEFAULT_LIMIT @@ -1830,6 +2754,7 @@ impl proto::shard_service_server::ShardService for ShardNode { }; tokio::spawn(async move { + let _mutation_guard = mutation_guard; loop { let (records, has_more, next_start_id) = read_records(unirust.clone(), start_id, end_id, limit).await; @@ -1867,26 +2792,17 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let request = request.into_inner(); + validate_distributed_protocol(request.internal_protocol_version)?; if request.records.is_empty() { return Ok(Response::new(proto::ImportRecordsResponse { imported: 0 })); } let mut unirust = write_unirust!(self); - let mut records = Vec::with_capacity(request.records.len()); - for snapshot in &request.records { - let identity = snapshot - .identity - .as_ref() - .ok_or_else(|| Status::invalid_argument("record identity is required"))?; - records.push(Self::build_record_with_id( - &mut unirust, - snapshot.record_id, - identity, - &snapshot.descriptors, - )?); - } + let records = Self::build_import_records(&mut unirust, &request.records)?; unirust - .ingest(records) + .ingest_with_explicit_ids(records) .map_err(|err| Status::internal(err.to_string()))?; Ok(Response::new(proto::ImportRecordsResponse { imported: request.records.len() as u64, @@ -1897,6 +2813,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request>, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let mut stream = request.into_inner(); let mut imported = 0u64; while let Some(chunk) = stream @@ -1907,22 +2825,11 @@ impl proto::shard_service_server::ShardService for ShardNode { if chunk.records.is_empty() { continue; } + validate_distributed_protocol(chunk.internal_protocol_version)?; let mut unirust = write_unirust!(self); - let mut records = Vec::with_capacity(chunk.records.len()); - for snapshot in &chunk.records { - let identity = snapshot - .identity - .as_ref() - .ok_or_else(|| Status::invalid_argument("record identity is required"))?; - records.push(Self::build_record_with_id( - &mut unirust, - snapshot.record_id, - identity, - &snapshot.descriptors, - )?); - } + let records = Self::build_import_records(&mut unirust, &chunk.records)?; unirust - .ingest(records) + .ingest_with_explicit_ids(records) .map_err(|err| Status::internal(err.to_string()))?; imported += chunk.records.len() as u64; } @@ -1933,6 +2840,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let request = request.into_inner(); let mut summaries = if let Some(cache) = { let guard = read_unirust!(self); @@ -1941,10 +2850,14 @@ impl proto::shard_service_server::ShardService for ShardNode { cache } else if let Some(persisted) = { let guard = read_unirust!(self); - guard.load_conflict_summaries() + guard + .load_conflict_summaries() + .map_err(|err| Status::failed_precondition(err.to_string()))? } { let mut unirust = write_unirust!(self); - unirust.set_conflict_cache(persisted.clone()); + unirust + .set_conflict_cache(persisted.clone()) + .map_err(|err| Status::internal(err.to_string()))?; persisted } else { let mut unirust = write_unirust!(self); @@ -1955,7 +2868,9 @@ impl proto::shard_service_server::ShardService for ShardNode { .detect_conflicts(&clusters) .map_err(|err| Status::internal(err.to_string()))?; let summaries = unirust.summarize_conflicts(&observations); - unirust.set_conflict_cache(summaries.clone()); + unirust + .set_conflict_cache(summaries.clone()) + .map_err(|err| Status::internal(err.to_string()))?; summaries }; @@ -2009,30 +2924,37 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, _request: Request, ) -> Result, Status> { + if !self.allow_destructive_admin { + return Err(Status::permission_denied( + "destructive reset RPC is disabled; stop the cluster and use the confirmed \ + offline reset procedure", + )); + } + let _mutation_guard = self.mutation_gate.write().await; + self.ensure_no_pending_ingest()?; let config = self.ontology_config.lock().await.clone(); - if let Some(path) = &self.data_dir { - let mut store = - PersistentStore::open(path).map_err(|err| Status::internal(err.to_string()))?; - store - .reset_data() - .map_err(|err| Status::internal(err.to_string()))?; - store - .save_ontology_config(&bincode::serialize(&config).map_err(|err| { - Status::internal(format!("failed to encode ontology config: {err}")) - })?) - .map_err(|err| Status::internal(err.to_string()))?; - let ontology = config.build_ontology(store.inner_mut()); - store - .persist_state() - .map_err(|err| Status::internal(err.to_string()))?; + { let mut guard = write_unirust!(self); - *guard = Unirust::with_store_and_tuning(ontology, store, self.tuning.clone()); + guard + .reset_with_ontology(config.ontology_template()) + .map_err(|err| Status::internal(err.to_string()))?; + } + + if is_partitioned_enabled() && self.data_dir.is_none() { + let partition_config = PartitionConfig::for_cores(partition_count()); + let ontology = config.build_ontology_with_interner(&self.concurrent_interner); + let partitioned = ParallelPartitionedUnirust::new_with_interner( + partition_config, + Arc::new(ontology), + self.tuning.clone(), + self.concurrent_interner.clone(), + ) + .map_err(|err| Status::internal(err.to_string()))?; + *self.partitioned.write() = Some(Arc::new(partitioned)); } else { - let mut store = crate::Store::new(); - let ontology = config.build_ontology(&mut store); - let mut guard = write_unirust!(self); - *guard = Unirust::with_store_and_tuning(ontology, store, self.tuning.clone()); + *self.partitioned.write() = None; } + self.cross_shard_conflicts.write().clear(); Ok(Response::new(proto::Empty {})) } @@ -2040,6 +2962,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let since_version = request.into_inner().since_version; // Export boundary metadata from the streaming linker @@ -2079,6 +3003,11 @@ impl proto::shard_service_server::ShardService for ShardNode { interval_end: e.interval.end, shard_id: e.shard_id as u32, perspective_strong_ids: e.perspective_strong_ids.clone(), + strong_ids: e + .strong_ids + .into_iter() + .map(boundary_strong_id_to_proto) + .collect(), }) .collect(), }) @@ -2102,6 +3031,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let req = request.into_inner(); let primary = req @@ -2112,25 +3043,23 @@ impl proto::shard_service_server::ShardService for ShardNode { .ok_or_else(|| Status::invalid_argument("secondary cluster ID is required"))?; let primary_id = GlobalClusterId::new( - primary.shard_id as u16, + u16::try_from(primary.shard_id) + .map_err(|_| Status::invalid_argument("primary shard_id exceeds u16"))?, primary.local_id, - primary.version as u16, + u16::try_from(primary.version) + .map_err(|_| Status::invalid_argument("primary version exceeds u16"))?, ); let secondary_id = GlobalClusterId::new( - secondary.shard_id as u16, + u16::try_from(secondary.shard_id) + .map_err(|_| Status::invalid_argument("secondary shard_id exceeds u16"))?, secondary.local_id, - secondary.version as u16, + u16::try_from(secondary.version) + .map_err(|_| Status::invalid_argument("secondary version exceeds u16"))?, ); - - // Only apply if we own one of the clusters - if primary_id.shard_id != self.shard_id as u16 - && secondary_id.shard_id != self.shard_id as u16 - { - return Ok(Response::new(proto::ApplyMergeResponse { - success: true, - records_updated: 0, - error: String::new(), - })); + if primary_id == secondary_id { + return Err(Status::invalid_argument( + "primary and secondary cluster IDs must differ", + )); } // Get write lock on unirust @@ -2159,6 +3088,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, _request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let unirust = read_unirust!(self); let partitioned_guard = self.partitioned.read(); @@ -2193,6 +3124,12 @@ impl proto::shard_service_server::ShardService for ShardNode { interval_end: e.interval.end, shard_id: e.shard_id as u32, perspective_strong_ids: e.perspective_strong_ids.clone(), + strong_ids: e + .strong_ids + .iter() + .cloned() + .map(boundary_strong_id_to_proto) + .collect(), }) .collect(); @@ -2216,6 +3153,8 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_no_pending_ingest()?; let req = request.into_inner(); let mut unirust = write_unirust!(self); let partitioned_guard = self.partitioned.read(); @@ -2251,36 +3190,54 @@ impl proto::shard_service_server::ShardService for ShardNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.write().await; + self.ensure_no_pending_ingest()?; let req = request.into_inner(); let conflicts: Vec = req .conflicts .into_iter() - .filter_map(|c| { - let sig = c.identity_key_signature.as_ref()?; + .map(|c| { + let sig = c.identity_key_signature.ok_or_else(|| { + Status::invalid_argument("cross-shard conflict signature is required") + })?; if sig.signature.len() != 32 { - return None; + return Err(Status::invalid_argument( + "cross-shard conflict signature must be 32 bytes", + )); } let mut sig_bytes = [0u8; 32]; sig_bytes.copy_from_slice(&sig.signature); - let cluster1 = c.cluster1.as_ref()?; - let cluster2 = c.cluster2.as_ref()?; - - let interval = Interval::new(c.interval_start, c.interval_end).ok()?; + let cluster1 = c.cluster1.ok_or_else(|| { + Status::invalid_argument("cross-shard conflict cluster1 is required") + })?; + let cluster2 = c.cluster2.ok_or_else(|| { + Status::invalid_argument("cross-shard conflict cluster2 is required") + })?; + let cluster1_shard = u16::try_from(cluster1.shard_id) + .map_err(|_| Status::invalid_argument("cluster1 shard_id exceeds u16"))?; + let cluster1_version = u16::try_from(cluster1.version) + .map_err(|_| Status::invalid_argument("cluster1 version exceeds u16"))?; + let cluster2_shard = u16::try_from(cluster2.shard_id) + .map_err(|_| Status::invalid_argument("cluster2 shard_id exceeds u16"))?; + let cluster2_version = u16::try_from(cluster2.version) + .map_err(|_| Status::invalid_argument("cluster2 version exceeds u16"))?; + let interval = Interval::new(c.interval_start, c.interval_end) + .map_err(|err| Status::invalid_argument(err.to_string()))?; - Some(crate::sharding::CrossShardConflict { + Ok(crate::sharding::CrossShardConflict { identity_key_signature: crate::sharding::IdentityKeySignature::from_bytes( sig_bytes, ), cluster1: GlobalClusterId::new( - cluster1.shard_id as u16, + cluster1_shard, cluster1.local_id, - cluster1.version as u16, + cluster1_version, ), cluster2: GlobalClusterId::new( - cluster2.shard_id as u16, + cluster2_shard, cluster2.local_id, - cluster2.version as u16, + cluster2_version, ), interval, perspective_hash: c.perspective_hash, @@ -2288,26 +3245,28 @@ impl proto::shard_service_server::ShardService for ShardNode { strong_id_hash2: c.strong_id_hash2, }) }) - .collect(); - - let stored_count = conflicts.len() as u32; + .collect::>()?; // Store conflicts - keep only those relevant to this shard let shard_id = self.shard_id as u16; let mut conflict_storage = self.cross_shard_conflicts.write(); + let mut updated = conflict_storage.clone(); + let mut stored_count = 0u32; for conflict in conflicts { - // Store if either cluster belongs to this shard if conflict.cluster1.shard_id == shard_id || conflict.cluster2.shard_id == shard_id { - // Avoid duplicates - check if already stored - if !conflict_storage.iter().any(|c| { - c.cluster1 == conflict.cluster1 - && c.cluster2 == conflict.cluster2 - && c.interval == conflict.interval - }) { - conflict_storage.push(conflict); + if !updated.contains(&conflict) { + updated.push(conflict); + stored_count = stored_count.saturating_add(1); } } } + if stored_count != 0 { + let mut unirust = write_unirust!(self); + unirust + .persist_cross_shard_conflicts(&updated) + .map_err(|err| Status::internal(err.to_string()))?; + *conflict_storage = updated; + } Ok(Response::new(proto::StoreCrossShardConflictsResponse { stored_count, @@ -2343,6 +3302,37 @@ impl Default for AdaptiveReconciliationConfig { } } +/// Failure bounds for router-to-shard transport and RPC calls. +#[derive(Clone)] +pub struct RouterRpcConfig { + pub connect_timeout: Duration, + pub request_timeout: Duration, + pub tcp_keepalive: Duration, + pub shard_mtls: Option, +} + +impl std::fmt::Debug for RouterRpcConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RouterRpcConfig") + .field("connect_timeout", &self.connect_timeout) + .field("request_timeout", &self.request_timeout) + .field("tcp_keepalive", &self.tcp_keepalive) + .field("shard_mtls_configured", &self.shard_mtls.is_some()) + .finish() + } +} + +impl Default for RouterRpcConfig { + fn default() -> Self { + Self { + connect_timeout: Duration::from_secs(10), + request_timeout: Duration::from_secs(120), + tcp_keepalive: Duration::from_secs(30), + shard_mtls: None, + } + } +} + /// State for a dirty boundary key. #[derive(Debug)] struct DirtyKeyState { @@ -2450,6 +3440,16 @@ pub struct RouterNode { locality_index: Arc>, /// Reconciliation coordinator (wrapped in mutex for interior mutability) reconciliation_coordinator: Arc>, + /// Serializes destructive cluster changes against router-mediated mutations. + mutation_gate: Arc>, + /// Latches closed if a distributed ontology mutation cannot be rolled back. + cluster_consistent: Arc, + /// Latches closed while a partially applied reconciliation needs repair. + reconciliation_consistent: Arc, + /// Shared base checkpoint generation when the complete cluster was restored. + restore_generation: Option, + /// Authoritative conflict view rebuilt from a consistent all-shard record scan. + global_conflict_cache: Arc>>>, } impl RouterNode { @@ -2465,54 +3465,268 @@ impl RouterNode { ontology_config: DistributedOntologyConfig, config_version: Option, ) -> Result, Status> { - let shard_count = shard_addrs.len(); - let mut shard_clients = Vec::with_capacity(shard_count); - for addr in shard_addrs { - let client = proto::shard_service_client::ShardServiceClient::connect(addr) - .await - .map_err(|err| Status::unavailable(err.to_string()))?; - shard_clients.push(client); - } - let config_version = config_version.unwrap_or_else(|| "unversioned".to_string()); - for client in &shard_clients { - let mut client = client.clone(); - let version = client - .get_config_version(Request::new(proto::ConfigVersionRequest {})) - .await - .map_err(|err| Status::unavailable(err.to_string()))? - .into_inner() - .version; - if version != config_version { - return Err(Status::failed_precondition(format!( - "config version mismatch: router {}, shard {}", - config_version, version - ))); - } - } - let router = Arc::new(Self { - shard_clients, - ontology_config: Arc::new(RwLock::new(ontology_config)), + Self::connect_with_version_and_reconciliation( + shard_addrs, + ontology_config, config_version, - metrics: Arc::new(PerfMetrics::new()), - locality_index: Arc::new(StdRwLock::new(ClusterLocalityIndex::new())), - reconciliation_coordinator: Arc::new(tokio::sync::Mutex::new( - ReconciliationCoordinator::new(AdaptiveReconciliationConfig::default()), - )), - }); - - // Start adaptive reconciliation background task automatically - let reconcile_router = router.clone(); - tokio::spawn(async move { - reconcile_router.run_adaptive_reconciliation_loop().await; - }); - - Ok(router) + AdaptiveReconciliationConfig::default(), + ) + .await + } + + pub async fn connect_with_version_and_reconciliation( + shard_addrs: Vec, + ontology_config: DistributedOntologyConfig, + config_version: Option, + reconciliation_config: AdaptiveReconciliationConfig, + ) -> Result, Status> { + Self::connect_with_runtime_config( + shard_addrs, + ontology_config, + config_version, + reconciliation_config, + RouterRpcConfig::default(), + ) + .await + } + + pub async fn connect_with_runtime_config( + shard_addrs: Vec, + ontology_config: DistributedOntologyConfig, + config_version: Option, + reconciliation_config: AdaptiveReconciliationConfig, + rpc_config: RouterRpcConfig, + ) -> Result, Status> { + if reconciliation_config.key_count_threshold == 0 { + return Err(Status::invalid_argument( + "reconciliation key_count_threshold must be greater than zero", + )); + } + if !reconciliation_config.idle_ingest_rate.is_finite() + || reconciliation_config.idle_ingest_rate < 0.0 + { + return Err(Status::invalid_argument( + "reconciliation idle_ingest_rate must be finite and nonnegative", + )); + } + if rpc_config.connect_timeout.is_zero() + || rpc_config.request_timeout.is_zero() + || rpc_config.tcp_keepalive.is_zero() + { + return Err(Status::invalid_argument( + "router shard RPC timeouts and TCP keepalive must be greater than zero", + )); + } + let shard_count = shard_addrs.len(); + if shard_count == 0 { + return Err(Status::invalid_argument( + "at least one shard address is required", + )); + } + if shard_count > usize::from(u16::MAX) + 1 { + return Err(Status::invalid_argument(format!( + "shard count {shard_count} exceeds the global cluster ID capacity" + ))); + } + let mut shard_clients = Vec::with_capacity(shard_count); + for addr in shard_addrs { + let uses_https = addr.starts_with("https://"); + let endpoint = tonic::transport::Endpoint::from_shared(addr) + .map_err(|err| Status::invalid_argument(err.to_string()))? + .connect_timeout(rpc_config.connect_timeout) + .timeout(rpc_config.request_timeout) + .tcp_keepalive(Some(rpc_config.tcp_keepalive)); + let endpoint = match (&rpc_config.shard_mtls, uses_https) { + (Some(tls), true) => endpoint + .tls_config(tls.clone()) + .map_err(|err| Status::invalid_argument(err.to_string()))?, + (Some(_), false) => { + return Err(Status::invalid_argument( + "router-to-shard mTLS requires https:// shard addresses", + )); + } + (None, true) => { + return Err(Status::invalid_argument( + "https:// shard addresses require explicit router-to-shard mTLS \ + certificate configuration", + )); + } + (None, false) => endpoint, + }; + let channel = endpoint + .connect() + .await + .map_err(|err| Status::unavailable(err.to_string()))?; + let client = proto::shard_service_client::ShardServiceClient::new(channel); + shard_clients.push(client); + } + let config_version = config_version.unwrap_or_else(|| "unversioned".to_string()); + let expected_reservation_shard_count = u32::try_from(shard_count) + .map_err(|_| Status::invalid_argument("shard count exceeds u32"))?; + let mut source_reservation_backfill_required = Vec::with_capacity(shard_count); + let mut cluster_restore_state: Option> = None; + for (expected_shard_id, client) in shard_clients.iter().enumerate() { + let mut client = client.clone(); + let response = client + .get_config_version(Request::new(proto::ConfigVersionRequest {})) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + validate_distributed_protocol(response.protocol_version)?; + validate_checkpoint_protocol(response.checkpoint_protocol_version)?; + let shard_restore_state = match ( + response.restore_generation.is_empty(), + response.restore_shard_count, + ) { + (true, 0) => None, + (false, count) if count > 0 => { + if count != expected_reservation_shard_count { + return Err(Status::failed_precondition(format!( + "restored checkpoint generation {} was created for {} shards, \ + but the router has {}", + response.restore_generation, count, expected_reservation_shard_count + ))); + } + Some((response.restore_generation.clone(), count)) + } + _ => { + return Err(Status::data_loss( + "shard reported incomplete restore checkpoint provenance", + )); + } + }; + if let Some(expected) = &cluster_restore_state { + if expected != &shard_restore_state { + let message = match (expected, &shard_restore_state) { + (Some((expected, _)), Some((actual, _))) => format!( + "shards were restored from different checkpoint generations \ + ({expected} and {actual}); restore the complete cluster from one \ + coordinated generation" + ), + _ => "cluster mixes restored and unrestored shard volumes; restore the \ + complete cluster from one coordinated generation" + .to_string(), + }; + return Err(Status::failed_precondition(message)); + } + } else { + cluster_restore_state = Some(shard_restore_state); + } + if response.version != config_version { + return Err(Status::failed_precondition(format!( + "config version mismatch: router {}, shard {}", + config_version, response.version + ))); + } + let shard_ontology = response.ontology_config.ok_or_else(|| { + Status::failed_precondition("shard did not report its ontology configuration") + })?; + if map_proto_config(&shard_ontology) != ontology_config { + return Err(Status::failed_precondition( + "ontology mismatch between router and shard; restart the router with the \ + same ontology used by every shard", + )); + } + if response.source_reservation_backfill_version == DISTRIBUTED_PROTOCOL_VERSION + && response.source_reservation_shard_count != 0 + && response.source_reservation_shard_count != expected_reservation_shard_count + { + return Err(Status::failed_precondition(format!( + "shard topology mismatch: shard was initialized for {} shards, router has {}; \ + online shard-count changes are unsupported without an atomic relocation \ + protocol", + response.source_reservation_shard_count, expected_reservation_shard_count + ))); + } + source_reservation_backfill_required.push( + response.source_reservation_backfill_version != DISTRIBUTED_PROTOCOL_VERSION + || response.source_reservation_shard_count != expected_reservation_shard_count, + ); + let metadata = client + .get_boundary_metadata(Request::new(proto::GetBoundaryMetadataRequest { + since_version: 0, + })) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner() + .metadata + .ok_or_else(|| Status::data_loss("shard returned no boundary metadata"))?; + let expected_shard_id = u32::try_from(expected_shard_id) + .map_err(|_| Status::invalid_argument("shard index exceeds u32"))?; + if metadata.shard_id != expected_shard_id { + return Err(Status::failed_precondition(format!( + "shard endpoint at index {expected_shard_id} reports shard_id {}; shard \ + addresses must be ordered by contiguous shard ID", + metadata.shard_id + ))); + } + } + let restore_generation = cluster_restore_state + .flatten() + .map(|(generation, _)| generation); + let router = Arc::new(Self { + shard_clients, + ontology_config: Arc::new(RwLock::new(ontology_config)), + config_version, + metrics: Arc::new(PerfMetrics::new()), + locality_index: Arc::new(StdRwLock::new(ClusterLocalityIndex::new())), + reconciliation_coordinator: Arc::new(tokio::sync::Mutex::new( + ReconciliationCoordinator::new(reconciliation_config), + )), + mutation_gate: Arc::new(tokio::sync::RwLock::new(())), + cluster_consistent: Arc::new(AtomicBool::new(true)), + reconciliation_consistent: Arc::new(AtomicBool::new(true)), + restore_generation, + global_conflict_cache: Arc::new(parking_lot::RwLock::new(None)), + }); + + router + .backfill_source_reservations(source_reservation_backfill_required) + .await?; + router.recover_dirty_reconciliation_on_startup().await?; + + // Start adaptive reconciliation without keeping the router alive forever. + router.clone().start_adaptive_reconciliation(); + + Ok(router) } pub async fn connect_from_file( path: impl AsRef, ontology_config: DistributedOntologyConfig, config_version: Option, + ) -> Result, Status> { + Self::connect_from_file_with_reconciliation( + path, + ontology_config, + config_version, + AdaptiveReconciliationConfig::default(), + ) + .await + } + + pub async fn connect_from_file_with_reconciliation( + path: impl AsRef, + ontology_config: DistributedOntologyConfig, + config_version: Option, + reconciliation_config: AdaptiveReconciliationConfig, + ) -> Result, Status> { + Self::connect_from_file_with_runtime_config( + path, + ontology_config, + config_version, + reconciliation_config, + RouterRpcConfig::default(), + ) + .await + } + + pub async fn connect_from_file_with_runtime_config( + path: impl AsRef, + ontology_config: DistributedOntologyConfig, + config_version: Option, + reconciliation_config: AdaptiveReconciliationConfig, + rpc_config: RouterRpcConfig, ) -> Result, Status> { let content = fs::read_to_string(path.as_ref()) .map_err(|err| Status::invalid_argument(err.to_string()))?; @@ -2532,7 +3746,14 @@ impl RouterNode { if shard_addrs.is_empty() { return Err(Status::invalid_argument("no shard addresses found")); } - Self::connect_with_version(shard_addrs, ontology_config, config_version).await + Self::connect_with_runtime_config( + shard_addrs, + ontology_config, + config_version, + reconciliation_config, + rpc_config, + ) + .await } #[allow(clippy::result_large_err)] @@ -2551,6 +3772,505 @@ impl RouterNode { Ok(self.shard_clients[idx].clone()) } + #[allow(clippy::result_large_err)] + fn ensure_ontology_consistent(&self) -> Result<(), Status> { + if self.cluster_consistent.load(Ordering::Acquire) { + Ok(()) + } else { + Err(Status::failed_precondition( + "cluster is blocked after an incomplete ontology change; retry SetOntology with \ + the intended configuration or recover the shard configurations offline", + )) + } + } + + #[allow(clippy::result_large_err)] + fn ensure_cluster_consistent(&self) -> Result<(), Status> { + self.ensure_ontology_consistent()?; + if self.reconciliation_consistent.load(Ordering::Acquire) { + Ok(()) + } else { + Err(Status::failed_precondition( + "cluster is blocked after a partially applied cross-shard reconciliation; retry \ + Reconcile or restart the router to repair retained dirty keys before serving \ + traffic", + )) + } + } + + fn invalidate_global_conflict_cache(&self) { + *self.global_conflict_cache.write() = None; + } + + async fn recover_dirty_reconciliation_on_startup(&self) -> Result<(), Status> { + let mut dirty_keys = std::collections::HashSet::new(); + for (expected_shard_id, client) in self.shard_clients.iter().enumerate() { + let mut client = client.clone(); + let response = client + .get_dirty_boundary_keys(Request::new(proto::GetDirtyBoundaryKeysRequest {})) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + if response.shard_id as usize != expected_shard_id { + return Err(Status::data_loss(format!( + "dirty-key response from endpoint {expected_shard_id} reports shard {}", + response.shard_id + ))); + } + for dirty_key in response.dirty_keys { + let signature = dirty_key.signature.ok_or_else(|| { + Status::data_loss("dirty-key response is missing a signature") + })?; + let signature = <[u8; 32]>::try_from(signature.signature.as_slice()) + .map_err(|_| Status::data_loss("dirty-key signature must be 32 bytes"))?; + dirty_keys.insert(IdentityKeySignature::from_bytes(signature)); + } + } + if dirty_keys.is_empty() { + return Ok(()); + } + + let mut dirty_keys = dirty_keys.into_iter().collect::>(); + dirty_keys.sort_by_key(|signature| *signature.to_bytes()); + tracing::warn!( + dirty_keys = dirty_keys.len(), + "router startup is repairing retained cross-shard reconciliation work" + ); + self.reconcile_dirty_keys(&dirty_keys).await?; + + let keys = dirty_keys + .iter() + .map(|signature| proto::IdentityKeySignature { + signature: signature.to_bytes().to_vec(), + }) + .collect::>(); + for (shard_id, client) in self.shard_clients.iter().enumerate() { + let mut client = client.clone(); + client + .clear_dirty_keys(Request::new(proto::ClearDirtyKeysRequest { + keys: keys.clone(), + })) + .await + .map_err(|err| { + Status::unavailable(format!( + "failed to clear recovered dirty keys on shard {shard_id}: {err}" + )) + })?; + } + Ok(()) + } + + async fn backfill_source_reservations(&self, required: Vec) -> Result<(), Status> { + let shard_count = self.shard_clients.len(); + let shard_count_u32 = u32::try_from(shard_count) + .map_err(|_| Status::invalid_argument("shard count exceeds u32"))?; + + for (target_shard_id, requires_backfill) in required.into_iter().enumerate() { + if !requires_backfill { + continue; + } + let target_shard_id_u32 = u32::try_from(target_shard_id) + .map_err(|_| Status::invalid_argument("target shard exceeds u32"))?; + let mut target_client = self.shard_clients[target_shard_id].clone(); + let mut start_id = 0; + let mut migrated_records = 0u64; + + loop { + let response = target_client + .export_records(Request::new(proto::ExportRecordsRequest { + start_id, + end_id: 0, + limit: 10_000, + })) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + let mut owner_batches = vec![Vec::new(); shard_count]; + let mut expected_targets = Vec::with_capacity(response.records.len()); + + for (position, snapshot) in response.records.into_iter().enumerate() { + let identity = snapshot.identity.ok_or_else(|| { + Status::data_loss("exported record is missing its source identity") + })?; + if identity.entity_type.is_empty() + || identity.perspective.is_empty() + || identity.uid.is_empty() + { + return Err(Status::data_loss( + "exported record has an empty source identity field", + )); + } + let reservation_index = u32::try_from(position).map_err(|_| { + Status::resource_exhausted("reservation migration page exceeds u32 records") + })?; + let record = proto::RecordInput { + index: reservation_index, + identity: Some(identity.clone()), + descriptors: snapshot.descriptors, + }; + validate_record_inputs(std::slice::from_ref(&record)).map_err(|err| { + Status::data_loss(format!( + "stored record is invalid during source reservation migration: {}", + err.message() + )) + })?; + let owner_shard_id = hash_source_identity_to_shard(&identity, shard_count); + owner_batches[owner_shard_id].push(proto::SourceRecordReservation { + index: reservation_index, + identity: Some(identity), + payload_digest: canonical_record_payload_digest(&record)?.to_vec(), + target_shard_id: target_shard_id_u32, + }); + expected_targets.push(target_shard_id_u32); + } + + self.dispatch_source_reservations(owner_batches, &expected_targets) + .await?; + migrated_records = migrated_records.saturating_add(expected_targets.len() as u64); + + if !response.has_more { + break; + } + if response.next_start_id == 0 || response.next_start_id <= start_id { + return Err(Status::data_loss(format!( + "shard {target_shard_id} returned a non-progressing reservation migration \ + cursor" + ))); + } + start_id = response.next_start_id; + } + + target_client + .mark_source_reservations_backfilled(Request::new( + proto::MarkSourceReservationsBackfilledRequest { + protocol_version: DISTRIBUTED_PROTOCOL_VERSION, + shard_count: shard_count_u32, + }, + )) + .await?; + tracing::info!( + target_shard_id, + migrated_records, + "source reservation migration completed" + ); + } + Ok(()) + } + + async fn reserve_and_partition_source_records( + &self, + config: &DistributedOntologyConfig, + records: Vec, + ) -> Result>, Status> { + let shard_count = self.shard_clients.len(); + let mut owner_batches = vec![Vec::new(); shard_count]; + let mut expected_targets = Vec::with_capacity(records.len()); + + for (position, record) in records.iter().enumerate() { + let identity = record + .identity + .as_ref() + .ok_or_else(|| Status::invalid_argument("record identity is required"))?; + let reservation_index = u32::try_from(position) + .map_err(|_| Status::resource_exhausted("ingest batch exceeds u32 records"))?; + let target_shard_id = hash_record_to_shard(config, record, shard_count); + let target_shard_id = u32::try_from(target_shard_id) + .map_err(|_| Status::resource_exhausted("target shard exceeds u32"))?; + let owner_shard_id = hash_source_identity_to_shard(identity, shard_count); + owner_batches[owner_shard_id].push(proto::SourceRecordReservation { + index: reservation_index, + identity: Some(identity.clone()), + payload_digest: canonical_record_payload_digest(record)?.to_vec(), + target_shard_id, + }); + expected_targets.push(target_shard_id); + } + + let confirmed_targets = self + .dispatch_source_reservations(owner_batches, &expected_targets) + .await?; + + let mut shard_batches = vec![Vec::new(); shard_count]; + for (record, target_shard_id) in records.into_iter().zip(confirmed_targets) { + let target_shard_id = target_shard_id as usize; + shard_batches[target_shard_id].push(record); + } + Ok(shard_batches) + } + + async fn reserve_source_snapshots_for_target( + &self, + target_shard_id: u32, + snapshots: &[proto::RecordSnapshot], + ) -> Result<(), Status> { + let shard_count = self.shard_clients.len(); + if target_shard_id as usize >= shard_count { + return Err(Status::invalid_argument("target shard is out of range")); + } + let mut owner_batches = vec![Vec::new(); shard_count]; + let mut expected_targets = Vec::with_capacity(snapshots.len()); + + for (position, snapshot) in snapshots.iter().enumerate() { + let identity = snapshot + .identity + .as_ref() + .ok_or_else(|| Status::invalid_argument("imported record identity is required"))?; + let reservation_index = u32::try_from(position) + .map_err(|_| Status::resource_exhausted("import batch exceeds u32 records"))?; + let record = proto::RecordInput { + index: reservation_index, + identity: Some(identity.clone()), + descriptors: snapshot.descriptors.clone(), + }; + validate_record_inputs(std::slice::from_ref(&record))?; + let owner_shard_id = hash_source_identity_to_shard(identity, shard_count); + owner_batches[owner_shard_id].push(proto::SourceRecordReservation { + index: reservation_index, + identity: Some(identity.clone()), + payload_digest: canonical_record_payload_digest(&record)?.to_vec(), + target_shard_id, + }); + expected_targets.push(target_shard_id); + } + self.dispatch_source_reservations(owner_batches, &expected_targets) + .await?; + Ok(()) + } + + async fn dispatch_source_reservations( + &self, + owner_batches: Vec>, + expected_targets: &[u32], + ) -> Result, Status> { + let reservation_futures = owner_batches + .into_iter() + .enumerate() + .filter(|(_, reservations)| !reservations.is_empty()) + .map(|(owner_shard_id, reservations)| { + let mut client = self.shard_clients[owner_shard_id].clone(); + async move { + client + .reserve_source_records(Request::new(proto::ReserveSourceRecordsRequest { + reservations, + })) + .await + } + }) + .collect::>(); + let responses = futures::future::join_all(reservation_futures).await; + let mut confirmed_targets = vec![None; expected_targets.len()]; + for response in responses { + let response = response?.into_inner(); + for reservation in response.reservations { + let position = reservation.index as usize; + let expected_target = expected_targets.get(position).ok_or_else(|| { + Status::data_loss("source reservation response index is out of range") + })?; + if reservation.target_shard_id != *expected_target { + return Err(Status::data_loss(format!( + "source reservation target mismatch at index {}: expected {}, received {}", + reservation.index, expected_target, reservation.target_shard_id + ))); + } + let confirmed = confirmed_targets.get_mut(position).ok_or_else(|| { + Status::data_loss("source reservation response index is out of range") + })?; + if confirmed.replace(reservation.target_shard_id).is_some() { + return Err(Status::data_loss( + "source reservation response contains a duplicate index", + )); + } + } + } + if confirmed_targets.iter().any(Option::is_none) { + return Err(Status::data_loss( + "source reservation response omitted one or more records", + )); + } + confirmed_targets + .into_iter() + .map(|target| { + target.ok_or_else(|| { + Status::data_loss("source reservation response omitted a verified target") + }) + }) + .collect() + } + + async fn fetch_all_record_snapshots(&self) -> Result, Status> { + let mut snapshots = Vec::new(); + for (shard_id, client) in self.shard_clients.iter().enumerate() { + let mut client = client.clone(); + let range = client + .get_record_id_range(Request::new(proto::RecordIdRangeRequest {})) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + if range.empty { + continue; + } + if range.max_id == u32::MAX { + return Err(Status::data_loss(format!( + "shard {shard_id} contains reserved record ID {}", + u32::MAX + ))); + } + + let mut start_id = 0; + loop { + let response = client + .export_records(Request::new(proto::ExportRecordsRequest { + start_id, + end_id: 0, + limit: 10_000, + })) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + snapshots.extend(response.records); + if !response.has_more { + break; + } + if response.next_start_id == 0 || response.next_start_id <= start_id { + return Err(Status::data_loss(format!( + "shard {shard_id} returned a non-progressing record export cursor" + ))); + } + start_id = response.next_start_id; + } + } + Ok(snapshots) + } + + async fn authoritative_conflict_summaries( + &self, + ) -> Result, Status> { + if let Some(cached) = self.global_conflict_cache.read().clone() { + return Ok(cached); + } + + let snapshots = self.fetch_all_record_snapshots().await?; + let config = self.ontology_config.read().await.clone(); + let summaries = + tokio::task::spawn_blocking(move || build_global_conflict_summaries(config, snapshots)) + .await + .map_err(|err| Status::internal(format!("global conflict scan panicked: {err}")))? + .map_err(|err| Status::internal(err.to_string()))?; + *self.global_conflict_cache.write() = Some(summaries.clone()); + Ok(summaries) + } + + async fn fetch_boundary_metadata(&self) -> Result, Status> { + let mut metadata = Vec::with_capacity(self.shard_clients.len()); + for (expected_shard_id, client) in self.shard_clients.iter().enumerate() { + let mut client = client.clone(); + let response = client + .get_boundary_metadata(Request::new(proto::GetBoundaryMetadataRequest { + since_version: 0, + })) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + let shard_metadata = response + .metadata + .ok_or_else(|| Status::data_loss("shard returned no boundary metadata"))?; + let expected_shard_id = u32::try_from(expected_shard_id) + .map_err(|_| Status::internal("shard index exceeds u32"))?; + if shard_metadata.shard_id != expected_shard_id { + return Err(Status::data_loss(format!( + "shard endpoint at index {expected_shard_id} reports shard_id {}", + shard_metadata.shard_id + ))); + } + metadata.push(shard_metadata); + } + Ok(metadata) + } + + async fn apply_merge_to_shard( + &self, + shard_id: u16, + primary: GlobalClusterId, + secondary: GlobalClusterId, + ) -> Result { + let mut client = self.shard_client(u32::from(shard_id))?; + let response = client + .apply_merge(Request::new(proto::ApplyMergeRequest { + primary: Some(global_cluster_id_to_proto(primary)), + secondary: Some(global_cluster_id_to_proto(secondary)), + })) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + if !response.success { + return Err(Status::aborted(format!( + "shard {shard_id} rejected cross-shard merge: {}", + response.error + ))); + } + Ok(response.records_updated) + } + + async fn store_conflicts_on_shard( + &self, + shard_id: u16, + conflicts: Vec, + ) -> Result { + let mut client = self.shard_client(u32::from(shard_id))?; + let response = client + .store_cross_shard_conflicts(Request::new(proto::StoreCrossShardConflictsRequest { + conflicts, + })) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + Ok(response.stored_count) + } + + async fn apply_reconciliation_result( + &self, + result: &crate::sharding::ReconciliationResult, + ) -> Result<(), Status> { + let outcome = self.apply_reconciliation_result_inner(result).await; + self.reconciliation_consistent + .store(outcome.is_ok(), Ordering::Release); + outcome + } + + async fn apply_reconciliation_result_inner( + &self, + result: &crate::sharding::ReconciliationResult, + ) -> Result<(), Status> { + for (primary, secondary) in &result.merged_clusters { + for shard_id in 0..self.shard_clients.len() { + let shard_id = u16::try_from(shard_id) + .map_err(|_| Status::internal("shard index exceeds u16"))?; + self.apply_merge_to_shard(shard_id, *primary, *secondary) + .await?; + } + } + + let mut conflicts_by_shard: std::collections::HashMap> = + std::collections::HashMap::new(); + for conflict in &result.detected_conflicts { + let proto_conflict = cross_shard_conflict_to_proto(conflict); + conflicts_by_shard + .entry(conflict.cluster1.shard_id) + .or_default() + .push(proto_conflict.clone()); + if conflict.cluster2.shard_id != conflict.cluster1.shard_id { + conflicts_by_shard + .entry(conflict.cluster2.shard_id) + .or_default() + .push(proto_conflict); + } + } + for (shard_id, conflicts) in conflicts_by_shard { + self.store_conflicts_on_shard(shard_id, conflicts).await?; + } + Ok(()) + } + /// Get read access to the cluster locality index. pub fn locality_index(&self) -> std::sync::RwLockReadGuard<'_, ClusterLocalityIndex> { self.locality_index @@ -2579,97 +4299,226 @@ impl RouterNode { /// This task polls shards for dirty boundary keys and triggers reconciliation /// based on adaptive conditions (key count, staleness, idle system). pub fn start_adaptive_reconciliation(self: Arc) -> tokio::task::JoinHandle<()> { - let router = self.clone(); + let router = Arc::downgrade(&self); tokio::spawn(async move { - router.run_adaptive_reconciliation_loop().await; + let mut interval = tokio::time::interval(Duration::from_millis(500)); + loop { + interval.tick().await; + let Some(router) = router.upgrade() else { + return; + }; + router.run_adaptive_reconciliation_once().await; + } }) } - /// Run the adaptive reconciliation loop. - async fn run_adaptive_reconciliation_loop(&self) { - let mut interval = tokio::time::interval(Duration::from_millis(500)); + /// Start periodic coordinated checkpoints. A failed two-phase generation is + /// retried with the same immutable name until it commits successfully. + #[allow(clippy::result_large_err)] + pub fn start_checkpoint_scheduler( + self: Arc, + checkpoint_interval: Duration, + ) -> Result, Status> { + if checkpoint_interval.is_zero() { + return Err(Status::invalid_argument( + "checkpoint interval must be greater than zero", + )); + } + let start = tokio::time::Instant::now() + .checked_add(checkpoint_interval) + .ok_or_else(|| Status::invalid_argument("checkpoint interval is too large"))?; + let router = Arc::downgrade(&self); + Ok(tokio::spawn(async move { + let mut ticker = tokio::time::interval_at(start, checkpoint_interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut pending_generation = None; + + loop { + ticker.tick().await; + let Some(router) = router.upgrade() else { + return; + }; + let generation = match pending_generation.clone() { + Some(generation) => generation, + None => { + let timestamp = match SystemTime::now().duration_since(UNIX_EPOCH) { + Ok(timestamp) => timestamp.as_nanos(), + Err(err) => { + tracing::error!( + error = %err, + "automatic checkpoint skipped because the system clock is \ + before the Unix epoch" + ); + continue; + } + }; + let generation = format!("scheduled-{timestamp}"); + pending_generation = Some(generation.clone()); + generation + } + }; - loop { - interval.tick().await; + let result = + ::checkpoint( + router.as_ref(), + Request::new(proto::CheckpointRequest { + path: generation.clone(), + checkpoint_protocol_version: 0, + shard_count: 0, + finalize: false, + }), + ) + .await; + match result { + Ok(response) => { + let response = response.into_inner(); + tracing::info!( + generation = response.generation, + shard_paths = response.paths.len(), + "automatic cluster checkpoint committed" + ); + pending_generation = None; + } + Err(err) => { + tracing::error!( + error = %err, + generation, + "automatic cluster checkpoint failed; the same generation will retry" + ); + } + } + } + })) + } - // 1. Poll all shards for dirty boundary keys - let mut all_dirty_keys: std::collections::HashMap< - crate::sharding::IdentityKeySignature, - Vec<(u16, proto::DirtyBoundaryKey)>, - > = std::collections::HashMap::new(); + async fn run_adaptive_reconciliation_once(&self) { + let poll_complete = { + let _mutation_guard = self.mutation_gate.read().await; + if self.ensure_ontology_consistent().is_err() { + return; + } - for (idx, client) in self.shard_clients.iter().enumerate() { + // 1. Poll all shards for dirty boundary keys without pausing ingest. + let mut poll_complete = true; + for (expected_shard_id, client) in self.shard_clients.iter().enumerate() { let mut client = client.clone(); - if let Ok(response) = client + match client .get_dirty_boundary_keys(Request::new(proto::GetDirtyBoundaryKeysRequest {})) .await { - let resp = response.into_inner(); - let shard_id = resp.shard_id as u16; - - for dirty_key in resp.dirty_keys { - if let Some(sig_proto) = &dirty_key.signature { - if sig_proto.signature.len() == 32 { - let mut sig_bytes = [0u8; 32]; - sig_bytes.copy_from_slice(&sig_proto.signature); - let sig = IdentityKeySignature::from_bytes(sig_bytes); - - all_dirty_keys - .entry(sig) - .or_default() - .push((shard_id, dirty_key.clone())); - - // Add to coordinator - let mut coordinator = self.reconciliation_coordinator.lock().await; - coordinator.add_dirty_keys_from_shard(shard_id, vec![sig]); + Ok(response) => { + let resp = response.into_inner(); + let shard_id = match u16::try_from(resp.shard_id) { + Ok(shard_id) => shard_id, + Err(_) => { + tracing::error!( + shard_id = resp.shard_id, + "dirty-key response shard_id exceeds u16" + ); + poll_complete = false; + continue; } + }; + if usize::from(shard_id) != expected_shard_id { + tracing::error!( + expected_shard_id, + reported_shard_id = shard_id, + "dirty-key response came from the wrong shard" + ); + poll_complete = false; + continue; + } + + for dirty_key in resp.dirty_keys { + let Some(sig_proto) = &dirty_key.signature else { + tracing::error!("dirty-key response is missing a signature"); + poll_complete = false; + continue; + }; + let Ok(sig_bytes) = + <[u8; 32]>::try_from(sig_proto.signature.as_slice()) + else { + tracing::error!("dirty-key signature must be 32 bytes"); + poll_complete = false; + continue; + }; + let sig = IdentityKeySignature::from_bytes(sig_bytes); + let mut coordinator = self.reconciliation_coordinator.lock().await; + coordinator.add_dirty_keys_from_shard(shard_id, vec![sig]); } } + Err(err) => { + tracing::warn!(error = %err, "failed to poll shard dirty keys"); + poll_complete = false; + } } - let _ = idx; // suppress unused warning } + poll_complete + }; + if !poll_complete { + return; + } + + // 2. Check if we should reconcile + let should_run = { + let coordinator = self.reconciliation_coordinator.lock().await; + let ingest_rate = self.current_ingest_rate(); + !self.reconciliation_consistent.load(Ordering::Acquire) + || coordinator.should_reconcile(ingest_rate) + }; - // 2. Check if we should reconcile - let should_run = { - let coordinator = self.reconciliation_coordinator.lock().await; - let ingest_rate = self.current_ingest_rate(); - coordinator.should_reconcile(ingest_rate) + if should_run { + // Pause router-mediated ingest while refetching metadata, applying merges, + // and clearing exactly the reconciled dirty-key generation. + let _mutation_guard = self.mutation_gate.write().await; + if self.ensure_ontology_consistent().is_err() { + return; + } + // 3. Take dirty keys and perform targeted reconciliation + let dirty_keys = { + let mut coordinator = self.reconciliation_coordinator.lock().await; + coordinator.take_dirty_keys() }; - if should_run { - // 3. Take dirty keys and perform targeted reconciliation - let dirty_keys = { - let mut coordinator = self.reconciliation_coordinator.lock().await; - coordinator.take_dirty_keys() + if !dirty_keys.is_empty() { + let result = match self.reconcile_dirty_keys(&dirty_keys).await { + Ok(result) => result, + Err(err) => { + tracing::error!( + error = %err, + "adaptive reconciliation failed; shard dirty keys retained" + ); + return; + } }; - if !dirty_keys.is_empty() { - let result = self - .reconcile_dirty_keys(&dirty_keys, &all_dirty_keys) - .await; - - // Update metrics - if result.conflicts_blocked > 0 { - self.metrics - .cross_shard_conflicts - .fetch_add(result.conflicts_blocked as u64, Ordering::Relaxed); - } + // Update metrics + if result.conflicts_blocked > 0 { + self.metrics + .cross_shard_conflicts + .fetch_add(result.conflicts_blocked as u64, Ordering::Relaxed); + } - // 4. Clear dirty keys on shards - let proto_keys: Vec = dirty_keys - .iter() - .map(|sig| proto::IdentityKeySignature { - signature: sig.to_bytes().to_vec(), - }) - .collect(); + // 4. Clear dirty keys on shards + let proto_keys: Vec = dirty_keys + .iter() + .map(|sig| proto::IdentityKeySignature { + signature: sig.to_bytes().to_vec(), + }) + .collect(); - for client in &self.shard_clients { - let mut client = client.clone(); - let _ = client - .clear_dirty_keys(Request::new(proto::ClearDirtyKeysRequest { - keys: proto_keys.clone(), - })) - .await; + for client in &self.shard_clients { + let mut client = client.clone(); + if let Err(err) = client + .clear_dirty_keys(Request::new(proto::ClearDirtyKeysRequest { + keys: proto_keys.clone(), + })) + .await + { + tracing::warn!( + error = %err, + "failed to clear reconciled dirty keys; retry is safe" + ); } } } @@ -2680,154 +4529,18 @@ impl RouterNode { async fn reconcile_dirty_keys( &self, keys: &[crate::sharding::IdentityKeySignature], - dirty_key_data: &std::collections::HashMap< - crate::sharding::IdentityKeySignature, - Vec<(u16, proto::DirtyBoundaryKey)>, - >, - ) -> crate::sharding::ReconciliationResult { - use crate::sharding::{ClusterBoundaryIndex, IncrementalReconciler}; - - // Build boundary indices for each shard from dirty key data - let mut shard_boundaries: std::collections::HashMap = - std::collections::HashMap::new(); - - for sig in keys { - if let Some(entries) = dirty_key_data.get(sig) { - for (shard_id, dirty_key) in entries { - let boundary = shard_boundaries - .entry(*shard_id) - .or_insert_with(|| ClusterBoundaryIndex::new_small(*shard_id)); - - for entry in &dirty_key.entries { - if let Some(cluster_id_proto) = &entry.cluster_id { - let cluster_id = GlobalClusterId::new( - cluster_id_proto.shard_id as u16, - cluster_id_proto.local_id, - cluster_id_proto.version as u16, - ); - if let Ok(interval) = crate::temporal::Interval::new( - entry.interval_start, - entry.interval_end, - ) { - boundary.register_boundary_key_with_strong_ids( - *sig, - cluster_id, - interval, - entry.perspective_strong_ids.clone(), - ); - } - } - } - } - } - } + ) -> Result { + use crate::sharding::IncrementalReconciler; - // Run targeted reconciliation let mut reconciler = IncrementalReconciler::new(); - for (_shard_id, boundary) in shard_boundaries { - reconciler.add_shard_boundary(boundary); + for metadata in self.fetch_boundary_metadata().await? { + reconciler.add_shard_boundary(boundary_index_from_metadata(&metadata)?); } let key_set: std::collections::HashSet<_> = keys.iter().copied().collect(); let result = reconciler.reconcile_keys(&key_set); - - // Apply merges to affected shards - for (primary, secondary) in &result.merged_clusters { - // Apply to primary's shard - if let Ok(mut client) = self.shard_client(primary.shard_id as u32) { - let _ = client - .apply_merge(Request::new(proto::ApplyMergeRequest { - primary: Some(proto::GlobalClusterId { - shard_id: primary.shard_id as u32, - local_id: primary.local_id, - version: primary.version as u32, - }), - secondary: Some(proto::GlobalClusterId { - shard_id: secondary.shard_id as u32, - local_id: secondary.local_id, - version: secondary.version as u32, - }), - })) - .await; - } - - // Apply to secondary's shard if different - if secondary.shard_id != primary.shard_id { - if let Ok(mut client) = self.shard_client(secondary.shard_id as u32) { - let _ = client - .apply_merge(Request::new(proto::ApplyMergeRequest { - primary: Some(proto::GlobalClusterId { - shard_id: primary.shard_id as u32, - local_id: primary.local_id, - version: primary.version as u32, - }), - secondary: Some(proto::GlobalClusterId { - shard_id: secondary.shard_id as u32, - local_id: secondary.local_id, - version: secondary.version as u32, - }), - })) - .await; - } - } - } - - // Propagate detected conflicts to affected shards - if !result.detected_conflicts.is_empty() { - // Group conflicts by shard - let mut conflicts_by_shard: std::collections::HashMap< - u16, - Vec, - > = std::collections::HashMap::new(); - - for conflict in &result.detected_conflicts { - let proto_conflict = proto::CrossShardConflict { - identity_key_signature: Some(proto::IdentityKeySignature { - signature: conflict.identity_key_signature.to_bytes().to_vec(), - }), - cluster1: Some(proto::GlobalClusterId { - shard_id: conflict.cluster1.shard_id as u32, - local_id: conflict.cluster1.local_id, - version: conflict.cluster1.version as u32, - }), - cluster2: Some(proto::GlobalClusterId { - shard_id: conflict.cluster2.shard_id as u32, - local_id: conflict.cluster2.local_id, - version: conflict.cluster2.version as u32, - }), - interval_start: conflict.interval.start, - interval_end: conflict.interval.end, - perspective_hash: conflict.perspective_hash, - strong_id_hash1: conflict.strong_id_hash1, - strong_id_hash2: conflict.strong_id_hash2, - }; - - // Send to both shards involved - conflicts_by_shard - .entry(conflict.cluster1.shard_id) - .or_default() - .push(proto_conflict.clone()); - if conflict.cluster2.shard_id != conflict.cluster1.shard_id { - conflicts_by_shard - .entry(conflict.cluster2.shard_id) - .or_default() - .push(proto_conflict); - } - } - - // Send conflicts to each shard - for (shard_id, conflicts) in conflicts_by_shard { - if let Ok(mut client) = self.shard_client(shard_id as u32) { - let _ = client - .store_cross_shard_conflicts(Request::new( - proto::StoreCrossShardConflictsRequest { conflicts }, - )) - .await; - } - } - } - - result + self.apply_reconciliation_result(&result).await?; + Ok(result) } /// Get current ingest rate (records per second) from metrics. @@ -2860,6 +4573,49 @@ impl RouterNode { } } + matches.sort_by_key(|query_match| { + ( + query_match.shard_id, + query_match.cluster_id, + query_match.start, + query_match.end, + ) + }); + let mut consolidated: Vec = Vec::with_capacity(matches.len()); + for mut query_match in matches { + if let Some(previous) = consolidated.last_mut() { + if previous.shard_id == query_match.shard_id + && previous.cluster_id == query_match.cluster_id + && query_match.start <= previous.end + { + previous.start = previous.start.min(query_match.start); + previous.end = previous.end.max(query_match.end); + for golden in query_match.golden.drain(..) { + if !previous.golden.contains(&golden) { + previous.golden.push(golden); + } + } + previous.golden.sort_by(|left, right| { + (&left.attr, &left.value, left.start, left.end).cmp(&( + &right.attr, + &right.value, + right.start, + right.end, + )) + }); + if previous.cluster_key.is_empty() { + previous.cluster_key = query_match.cluster_key; + } + if previous.cluster_key_identity.is_empty() { + previous.cluster_key_identity = query_match.cluster_key_identity; + } + continue; + } + } + consolidated.push(query_match); + } + let mut matches = consolidated; + if matches.len() <= 1 { return proto::QueryEntitiesResponse { outcome: Some(proto::query_entities_response::Outcome::Matches( @@ -2868,7 +4624,7 @@ impl RouterNode { }; } - matches.sort_by(|a, b| a.start.cmp(&b.start)); + matches.sort_by_key(|query_match| query_match.start); for window in matches.windows(2) { let current = &window[0]; @@ -2918,21 +4674,61 @@ impl proto::router_service_server::RouterService for RouterNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.write().await; let payload = request.into_inner(); let config = payload .config .clone() .ok_or_else(|| Status::invalid_argument("ontology config is required"))?; let mapped = map_proto_config(&config); - *self.ontology_config.write().await = mapped; + let previous = self.ontology_config.read().await.clone(); + if previous != mapped { + for client in &self.shard_clients { + let mut client = client.clone(); + let stats = client + .get_stats(Request::new(proto::StatsRequest {})) + .await? + .into_inner(); + if stats.record_count != 0 { + return Err(Status::failed_precondition(format!( + "refusing to replace ontology while the cluster contains records \ + (shard reports {} records); reset the cluster explicitly first", + stats.record_count + ))); + } + } + } + self.invalidate_global_conflict_cache(); for client in &self.shard_clients { let mut client = client.clone(); - client - .set_ontology(Request::new(payload.clone())) - .await - .map_err(|err| Status::unavailable(err.to_string()))?; + if let Err(apply_error) = client.set_ontology(Request::new(payload.clone())).await { + let rollback = proto::ApplyOntologyRequest { + config: Some(to_proto_config(&previous)), + }; + let mut rollback_complete = true; + for rollback_client in &self.shard_clients { + let mut rollback_client = rollback_client.clone(); + if rollback_client + .set_ontology(Request::new(rollback.clone())) + .await + .is_err() + { + rollback_complete = false; + } + } + if !rollback_complete { + self.cluster_consistent.store(false, Ordering::Release); + return Err(Status::aborted(format!( + "ontology update failed and rollback was incomplete; cluster is blocked: \ + {apply_error}" + ))); + } + return Err(apply_error); + } } + *self.ontology_config.write().await = mapped; + self.cluster_consistent.store(true, Ordering::Release); Ok(Response::new(proto::ApplyOntologyResponse {})) } @@ -2941,10 +4737,14 @@ impl proto::router_service_server::RouterService for RouterNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_cluster_consistent()?; let start = Instant::now(); let batch = request.into_inner(); let record_count = batch.records.len(); let shard_count = self.shard_clients.len(); + validate_record_inputs(&batch.records)?; + self.invalidate_global_conflict_cache(); // ULTRA-FAST single-shard path: skip all hashing and sorting if shard_count == 1 { @@ -2952,6 +4752,7 @@ impl proto::router_service_server::RouterService for RouterNode { let response = client .ingest_records(Request::new(proto::IngestRecordsRequest { records: batch.records, + internal_protocol_version: DISTRIBUTED_PROTOCOL_VERSION, })) .await .map_err(|err| Status::unavailable(err.to_string()))?; @@ -2961,14 +4762,12 @@ impl proto::router_service_server::RouterService for RouterNode { return Ok(response); } - // Multi-shard path: hash-based routing + // Multi-shard path: durably bind each immutable source identity to its + // canonical payload and routing destination before entity resolution. let config = self.ontology_config.read().await.clone(); - let mut shard_batches: Vec> = vec![Vec::new(); shard_count]; - - for record in batch.records { - let shard_idx = hash_record_to_shard(&config, &record, shard_count); - shard_batches[shard_idx].push(record); - } + let shard_batches = self + .reserve_and_partition_source_records(&config, batch.records) + .await?; // Parallel shard ingest - spawn all shard requests concurrently let shard_futures: Vec<_> = shard_batches @@ -2979,7 +4778,10 @@ impl proto::router_service_server::RouterService for RouterNode { let mut client = self.shard_clients[idx].clone(); async move { client - .ingest_records(Request::new(proto::IngestRecordsRequest { records })) + .ingest_records(Request::new(proto::IngestRecordsRequest { + records, + internal_protocol_version: DISTRIBUTED_PROTOCOL_VERSION, + })) .await } }) @@ -2990,7 +4792,7 @@ impl proto::router_service_server::RouterService for RouterNode { for response in shard_responses { match response { Ok(resp) => results.extend(resp.into_inner().assignments), - Err(err) => return Err(Status::unavailable(err.to_string())), + Err(err) => return Err(err), } } @@ -3015,6 +4817,8 @@ impl proto::router_service_server::RouterService for RouterNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_cluster_consistent()?; let start = Instant::now(); let request = request.into_inner(); let mut responses = Vec::with_capacity(self.shard_clients.len()); @@ -3037,6 +4841,7 @@ impl proto::router_service_server::RouterService for RouterNode { &self, _request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; let mut totals = proto::StatsResponse { record_count: 0, cluster_count: 0, @@ -3060,16 +4865,13 @@ impl proto::router_service_server::RouterService for RouterNode { totals.conflict_count += response.conflict_count; totals.graph_node_count += response.graph_node_count; totals.graph_edge_count += response.graph_edge_count; - totals.cross_shard_merges += response.cross_shard_merges; - // cross_shard_conflicts from shards is 0, we track it at router level + totals.cross_shard_merges = totals.cross_shard_merges.max(response.cross_shard_merges); + totals.cross_shard_conflicts += response.cross_shard_conflicts; totals.boundary_keys_tracked += response.boundary_keys_tracked; } - // Add router-level cross-shard conflicts (tracked during reconcile) - totals.cross_shard_conflicts = self - .metrics - .cross_shard_conflicts - .load(std::sync::atomic::Ordering::Relaxed); + // Each cross-shard conflict is durably stored on both participating shards. + totals.cross_shard_conflicts = totals.cross_shard_conflicts.div_ceil(2); Ok(Response::new(totals)) } @@ -3078,6 +4880,8 @@ impl proto::router_service_server::RouterService for RouterNode { &self, _request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_cluster_consistent()?; for client in &self.shard_clients { let mut client = client.clone(); client @@ -3095,8 +4899,20 @@ impl proto::router_service_server::RouterService for RouterNode { &self, _request: Request, ) -> Result, Status> { + let ontology_config = self.ontology_config.read().await; Ok(Response::new(proto::ConfigVersionResponse { version: self.config_version.clone(), + ontology_config: Some(to_proto_config(&ontology_config)), + protocol_version: DISTRIBUTED_PROTOCOL_VERSION, + source_reservation_backfill_version: DISTRIBUTED_PROTOCOL_VERSION, + source_reservation_shard_count: self.shard_clients.len() as u32, + checkpoint_protocol_version: CHECKPOINT_PROTOCOL_VERSION, + restore_generation: self.restore_generation.clone().unwrap_or_default(), + restore_shard_count: self + .restore_generation + .as_ref() + .map(|_| self.shard_clients.len() as u32) + .unwrap_or_default(), })) } @@ -3221,11 +5037,17 @@ impl proto::router_service_server::RouterService for RouterNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_cluster_consistent()?; let request = request.into_inner(); + self.invalidate_global_conflict_cache(); + self.reserve_source_snapshots_for_target(request.shard_id, &request.records) + .await?; let mut client = self.shard_client(request.shard_id)?; let response = client .import_records(Request::new(proto::ImportRecordsRequest { records: request.records, + internal_protocol_version: DISTRIBUTED_PROTOCOL_VERSION, })) .await .map_err(|err| Status::unavailable(err.to_string()))? @@ -3237,6 +5059,8 @@ impl proto::router_service_server::RouterService for RouterNode { &self, request: Request>, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.read().await; + self.ensure_cluster_consistent()?; let mut inbound = request.into_inner(); let first = inbound .message() @@ -3246,15 +5070,20 @@ impl proto::router_service_server::RouterService for RouterNode { return Ok(Response::new(proto::ImportRecordsResponse { imported: 0 })); }; + self.invalidate_global_conflict_cache(); let shard_id = first.shard_id; + self.reserve_source_snapshots_for_target(shard_id, &first.records) + .await?; let mut client = self.shard_client(shard_id)?; let (tx, rx) = mpsc::channel(4); let (err_tx, err_rx) = oneshot::channel::>(); + let router = self.clone(); tokio::spawn(async move { if tx .send(proto::ImportRecordsChunk { records: first.records, + internal_protocol_version: DISTRIBUTED_PROTOCOL_VERSION, }) .await .is_err() @@ -3271,9 +5100,17 @@ impl proto::router_service_server::RouterService for RouterNode { ))); return; } + if let Err(err) = router + .reserve_source_snapshots_for_target(shard_id, &chunk.records) + .await + { + let _ = err_tx.send(Err(err)); + return; + } if tx .send(proto::ImportRecordsChunk { records: chunk.records, + internal_protocol_version: DISTRIBUTED_PROTOCOL_VERSION, }) .await .is_err() @@ -3309,34 +5146,83 @@ impl proto::router_service_server::RouterService for RouterNode { &self, request: Request, ) -> Result, Status> { - let payload = request.into_inner(); + let _mutation_guard = self.mutation_gate.write().await; + let mut payload = request.into_inner(); + if payload.path.is_empty() { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| Status::internal(err.to_string()))? + .as_nanos(); + payload.path = format!("cluster-{timestamp}"); + } + let shard_count = u32::try_from(self.shard_clients.len()) + .map_err(|_| Status::resource_exhausted("shard count exceeds u32"))?; + payload.checkpoint_protocol_version = CHECKPOINT_PROTOCOL_VERSION; + payload.shard_count = shard_count; + payload.finalize = false; + let mut paths = Vec::new(); - for client in &self.shard_clients { + for (shard_id, client) in self.shard_clients.iter().enumerate() { let mut client = client.clone(); let response = client .checkpoint(Request::new(payload.clone())) .await .map_err(|err| Status::unavailable(err.to_string()))? .into_inner(); + if response.committed + || response.generation != payload.path + || response.paths.len() != 1 + { + return Err(Status::data_loss(format!( + "shard {shard_id} returned an invalid checkpoint prepare response" + ))); + } paths.extend(response.paths); } - Ok(Response::new(proto::CheckpointResponse { paths })) + + payload.finalize = true; + for (shard_id, client) in self.shard_clients.iter().enumerate() { + let mut client = client.clone(); + let response = client + .checkpoint(Request::new(payload.clone())) + .await + .map_err(|err| Status::unavailable(err.to_string()))? + .into_inner(); + if !response.committed + || response.generation != payload.path + || response.paths.len() != 1 + { + return Err(Status::data_loss(format!( + "shard {shard_id} returned an invalid checkpoint commit response" + ))); + } + } + + Ok(Response::new(proto::CheckpointResponse { + paths, + generation: payload.path, + committed: true, + })) } async fn list_conflicts( &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.write().await; + self.ensure_cluster_consistent()?; let payload = request.into_inner(); - let mut summaries = Vec::new(); - for client in &self.shard_clients { - let mut client = client.clone(); - let response = client - .list_conflicts(Request::new(payload.clone())) - .await - .map_err(|err| Status::unavailable(err.to_string()))? - .into_inner(); - summaries.extend(response.conflicts); + let mut summaries = self.authoritative_conflict_summaries().await?; + if !payload.attribute.is_empty() { + summaries.retain(|summary| summary.attribute == payload.attribute); + } + if payload.end > payload.start { + let filter = Interval::new(payload.start, payload.end) + .map_err(|err| Status::invalid_argument(err.to_string()))?; + summaries.retain(|summary| { + Interval::new(summary.start, summary.end) + .is_ok_and(|interval| crate::temporal::is_overlapping(&interval, &filter)) + }); } summaries.sort_by(|a, b| { @@ -3363,6 +5249,7 @@ impl proto::router_service_server::RouterService for RouterNode { .collect::>(), )) }); + summaries.dedup(); Ok(Response::new(proto::ListConflictsResponse { conflicts: summaries, @@ -3373,12 +5260,11 @@ impl proto::router_service_server::RouterService for RouterNode { &self, _request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.write().await; + self.invalidate_global_conflict_cache(); for client in &self.shard_clients { let mut client = client.clone(); - client - .reset(Request::new(proto::Empty {})) - .await - .map_err(|err| Status::unavailable(err.to_string()))?; + client.reset(Request::new(proto::Empty {})).await?; } Ok(Response::new(proto::Empty {})) } @@ -3387,73 +5273,25 @@ impl proto::router_service_server::RouterService for RouterNode { &self, request: Request, ) -> Result, Status> { + let _mutation_guard = self.mutation_gate.write().await; + self.ensure_ontology_consistent()?; let req = request.into_inner(); + if !req.shard_metadata.is_empty() { + return Err(Status::invalid_argument( + "caller-supplied shard metadata is not accepted; the router fetches authoritative \ + metadata from its configured shards", + )); + } + let shard_metadata = self.fetch_boundary_metadata().await?; - // Collect boundary metadata from all shards if not provided - let shard_metadata = if req.shard_metadata.is_empty() { - let mut metadata = Vec::new(); - for client in &self.shard_clients { - let mut client = client.clone(); - if let Ok(response) = client - .get_boundary_metadata(Request::new(proto::GetBoundaryMetadataRequest { - since_version: 0, - })) - .await - { - if let Some(m) = response.into_inner().metadata { - metadata.push(m); - } - } - } - metadata - } else { - req.shard_metadata - }; - - // Use IncrementalReconciler to find cross-shard merges let mut reconciler = crate::sharding::IncrementalReconciler::new(); - for metadata in &shard_metadata { - let mut boundary_index = - crate::sharding::ClusterBoundaryIndex::new_small(metadata.shard_id as u16); - - for key_entries in &metadata.entries { - if let Some(sig_proto) = &key_entries.signature { - if sig_proto.signature.len() == 32 { - let mut sig_bytes = [0u8; 32]; - sig_bytes.copy_from_slice(&sig_proto.signature); - let sig = IdentityKeySignature::from_bytes(sig_bytes); - - for entry in &key_entries.entries { - if let Some(cluster_id_proto) = &entry.cluster_id { - let cluster_id = GlobalClusterId::new( - cluster_id_proto.shard_id as u16, - cluster_id_proto.local_id, - cluster_id_proto.version as u16, - ); - if let Ok(interval) = crate::temporal::Interval::new( - entry.interval_start, - entry.interval_end, - ) { - boundary_index.register_boundary_key_with_strong_ids( - sig, - cluster_id, - interval, - entry.perspective_strong_ids.clone(), - ); - } - } - } - } - } - } - - reconciler.add_shard_boundary(boundary_index); + reconciler.add_shard_boundary(boundary_index_from_metadata(metadata)?); } let result = reconciler.reconcile(); + self.apply_reconciliation_result(&result).await?; - // Track cross-shard conflicts in router metrics if result.conflicts_blocked > 0 { self.metrics.cross_shard_conflicts.fetch_add( result.conflicts_blocked as u64, @@ -3461,119 +5299,12 @@ impl proto::router_service_server::RouterService for RouterNode { ); } - // Apply merges to affected shards - let mut total_records_updated = 0u32; - for (primary, secondary) in &result.merged_clusters { - // Apply to primary's shard - if let Ok(mut client) = self.shard_client(primary.shard_id as u32) { - let _ = client - .apply_merge(Request::new(proto::ApplyMergeRequest { - primary: Some(proto::GlobalClusterId { - shard_id: primary.shard_id as u32, - local_id: primary.local_id, - version: primary.version as u32, - }), - secondary: Some(proto::GlobalClusterId { - shard_id: secondary.shard_id as u32, - local_id: secondary.local_id, - version: secondary.version as u32, - }), - })) - .await - .map(|resp| total_records_updated += resp.into_inner().records_updated); - } - - // Apply to secondary's shard if different - if secondary.shard_id != primary.shard_id { - if let Ok(mut client) = self.shard_client(secondary.shard_id as u32) { - let _ = client - .apply_merge(Request::new(proto::ApplyMergeRequest { - primary: Some(proto::GlobalClusterId { - shard_id: primary.shard_id as u32, - local_id: primary.local_id, - version: primary.version as u32, - }), - secondary: Some(proto::GlobalClusterId { - shard_id: secondary.shard_id as u32, - local_id: secondary.local_id, - version: secondary.version as u32, - }), - })) - .await - .map(|resp| total_records_updated += resp.into_inner().records_updated); - } - } - } - - // Propagate detected conflicts to affected shards - if !result.detected_conflicts.is_empty() { - // Group conflicts by shard - let mut conflicts_by_shard: std::collections::HashMap< - u16, - Vec, - > = std::collections::HashMap::new(); - - for conflict in &result.detected_conflicts { - let proto_conflict = proto::CrossShardConflict { - identity_key_signature: Some(proto::IdentityKeySignature { - signature: conflict.identity_key_signature.to_bytes().to_vec(), - }), - cluster1: Some(proto::GlobalClusterId { - shard_id: conflict.cluster1.shard_id as u32, - local_id: conflict.cluster1.local_id, - version: conflict.cluster1.version as u32, - }), - cluster2: Some(proto::GlobalClusterId { - shard_id: conflict.cluster2.shard_id as u32, - local_id: conflict.cluster2.local_id, - version: conflict.cluster2.version as u32, - }), - interval_start: conflict.interval.start, - interval_end: conflict.interval.end, - perspective_hash: conflict.perspective_hash, - strong_id_hash1: conflict.strong_id_hash1, - strong_id_hash2: conflict.strong_id_hash2, - }; - - // Send to both shards involved - conflicts_by_shard - .entry(conflict.cluster1.shard_id) - .or_default() - .push(proto_conflict.clone()); - if conflict.cluster2.shard_id != conflict.cluster1.shard_id { - conflicts_by_shard - .entry(conflict.cluster2.shard_id) - .or_default() - .push(proto_conflict); - } - } - - // Send conflicts to each shard - for (shard_id, conflicts) in conflicts_by_shard { - if let Ok(mut client) = self.shard_client(shard_id as u32) { - let _ = client - .store_cross_shard_conflicts(Request::new( - proto::StoreCrossShardConflictsRequest { conflicts }, - )) - .await; - } - } - } - let merges = result .merged_clusters .iter() .map(|(primary, secondary)| proto::ClusterMerge { - primary: Some(proto::GlobalClusterId { - shard_id: primary.shard_id as u32, - local_id: primary.local_id, - version: primary.version as u32, - }), - secondary: Some(proto::GlobalClusterId { - shard_id: secondary.shard_id as u32, - local_id: secondary.local_id, - version: secondary.version as u32, - }), + primary: Some(global_cluster_id_to_proto(*primary)), + secondary: Some(global_cluster_id_to_proto(*secondary)), }) .collect(); @@ -3791,3 +5522,219 @@ fn merge_latency(acc: &mut proto::LatencyMetrics, other: Option WalRecordInput { + WalRecordInput { + index: 7, + identity: WalRecordIdentity { + entity_type: "person".to_string(), + perspective: "crm".to_string(), + uid: "wal-checksum".to_string(), + }, + descriptors: vec![WalRecordDescriptor { + attr: "email".to_string(), + value: "checksum@example.com".to_string(), + start: 0, + end: 10, + }], + } + } + + #[test] + fn framed_wal_round_trips_and_detects_corruption() { + let records = vec![sample_wal_record()]; + let mut encoded = encode_wal_batch(&records).expect("encode WAL"); + let decoded = decode_wal_batch(&encoded).expect("decode WAL"); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].identity.uid, "wal-checksum"); + + let last = encoded.last_mut().expect("WAL payload byte"); + *last ^= 0x01; + assert!(decode_wal_batch(&encoded) + .expect_err("checksum mutation must fail") + .contains("checksum mismatch")); + } + + #[test] + fn legacy_unframed_wal_remains_replayable() { + let encoded = bincode::serialize(&vec![sample_wal_record()]).expect("encode legacy WAL"); + let decoded = decode_wal_batch(&encoded).expect("decode legacy WAL"); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].index, 7); + } + + #[test] + fn source_routing_hash_is_stable_and_payload_independent() { + let config = DistributedOntologyConfig::empty(); + let original = proto_from_wal_input(sample_wal_record()); + assert_eq!( + hash_record_to_u64(&config, &original), + 15_034_823_070_914_786_251 + ); + + let mut changed_payload = original.clone(); + changed_payload.descriptors.reverse(); + changed_payload.descriptors[0].value = "changed@example.com".to_string(); + assert_eq!( + hash_record_to_u64(&config, &original), + hash_record_to_u64(&config, &changed_payload), + "an immutable source identity must always route to the same shard" + ); + + changed_payload.identity.as_mut().expect("identity").uid = "different-source".to_string(); + assert_ne!( + hash_record_to_u64(&config, &original), + hash_record_to_u64(&config, &changed_payload) + ); + } + + #[test] + fn mixed_distributed_protocol_versions_fail_closed() { + validate_distributed_protocol(DISTRIBUTED_PROTOCOL_VERSION) + .expect("matching protocol must be accepted"); + let error = validate_distributed_protocol(DISTRIBUTED_PROTOCOL_VERSION - 1) + .expect_err("mixed router and shard protocols must be rejected"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("coordinated Unirust version")); + } + + #[test] + fn pending_wal_cannot_be_overwritten() { + let temp_dir = tempfile::tempdir().expect("temporary WAL directory"); + let wal = IngestWal::new(temp_dir.path()); + let first = proto_from_wal_input(sample_wal_record()); + wal.write_batch(std::slice::from_ref(&first)) + .expect("write first WAL"); + + let mut second = first; + second.index = 99; + second.identity.as_mut().expect("identity").uid = "replacement".to_string(); + let error = wal + .write_batch(&[second]) + .expect_err("pending WAL must block replacement"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + + let recovered = wal + .load_batch() + .expect("load pending WAL") + .expect("pending batch"); + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].index, 7); + assert_eq!( + recovered[0].identity.as_ref().expect("identity").uid, + "wal-checksum" + ); + } + + #[tokio::test] + async fn pending_wal_fails_health_and_reads_closed() { + use proto::shard_service_server::ShardService; + + let temp_dir = tempfile::tempdir().expect("temporary shard directory"); + let shard = ShardNode::new_with_data_dir( + 0, + DistributedOntologyConfig::empty(), + StreamingTuning::balanced(), + Some(temp_dir.path().to_path_buf()), + false, + None, + ) + .expect("persistent shard"); + let wal = shard.ingest_wal.as_ref().expect("ingest WAL"); + wal.write_batch(&[proto_from_wal_input(sample_wal_record())]) + .expect("write pending WAL"); + + let health_error = + ShardService::health_check(&shard, Request::new(proto::HealthCheckRequest {})) + .await + .expect_err("pending recovery must fail health"); + assert_eq!(health_error.code(), tonic::Code::FailedPrecondition); + + let stats_error = ShardService::get_stats(&shard, Request::new(proto::StatsRequest {})) + .await + .expect_err("pending recovery must block state reads"); + assert_eq!(stats_error.code(), tonic::Code::FailedPrecondition); + + wal.clear().expect("clear test WAL"); + ShardService::health_check(&shard, Request::new(proto::HealthCheckRequest {})) + .await + .expect("health recovers after WAL clears"); + } + + #[tokio::test] + async fn conflicting_source_identity_is_rejected_before_wal_creation() { + use proto::shard_service_server::ShardService; + + let temp_dir = tempfile::tempdir().expect("temporary shard directory"); + let shard = ShardNode::new_with_data_dir( + 0, + DistributedOntologyConfig::empty(), + StreamingTuning::balanced(), + Some(temp_dir.path().to_path_buf()), + false, + None, + ) + .expect("persistent shard"); + let original = proto_from_wal_input(sample_wal_record()); + ShardService::ingest_records( + &shard, + Request::new(proto::IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![original.clone()], + }), + ) + .await + .expect("initial ingest"); + + let mut conflicting = original; + conflicting.descriptors[0].value = "different@example.com".to_string(); + let error = ShardService::ingest_records( + &shard, + Request::new(proto::IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![conflicting], + }), + ) + .await + .expect_err("changed payload must be rejected"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + assert!( + !shard.ingest_wal.as_ref().expect("ingest WAL").has_pending(), + "a deterministic validation error must not leave a recovery WAL" + ); + + let stats = ShardService::get_stats(&shard, Request::new(proto::StatsRequest {})) + .await + .expect("stats") + .into_inner(); + assert_eq!(stats.record_count, 1); + shard.shutdown().await.expect("shutdown"); + } + + #[tokio::test] + async fn shard_rejects_ingest_from_an_older_router_protocol() { + use proto::shard_service_server::ShardService; + + let shard = ShardNode::new( + 0, + DistributedOntologyConfig::empty(), + StreamingTuning::balanced(), + ) + .expect("shard"); + let error = ShardService::ingest_records( + &shard, + Request::new(proto::IngestRecordsRequest { + records: vec![proto_from_wal_input(sample_wal_record())], + internal_protocol_version: DISTRIBUTED_PROTOCOL_VERSION - 1, + }), + ) + .await + .expect_err("a new shard must reject ingest from an old router"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert_eq!(shard.unirust.read().record_count(), 0); + } +} diff --git a/src/dsu.rs b/src/dsu.rs index 1d8d5e8..fcfae99 100644 --- a/src/dsu.rs +++ b/src/dsu.rs @@ -680,30 +680,6 @@ impl PersistentTemporalDSU { Ok(()) } - /// Save metadata to RocksDB - fn save_metadata(&self) -> Result<()> { - let cf = self - .db - .cf_handle(self.cf_metadata) - .ok_or_else(|| anyhow!("missing DSU metadata column family"))?; - - let next_id = self.next_cluster_id.load(Ordering::SeqCst); - self.db.put_cf( - cf, - crate::persistence::dsu_keys::NEXT_CLUSTER_ID, - next_id.to_be_bytes(), - )?; - - let count = self.cluster_count.load(Ordering::SeqCst); - self.db.put_cf( - cf, - crate::persistence::dsu_keys::CLUSTER_COUNT, - count.to_be_bytes(), - )?; - - Ok(()) - } - /// Add a record to the DSU (record becomes its own parent with rank 0) pub fn add_record(&mut self, record_id: RecordId) -> Result<()> { // Check if already exists in cache or disk @@ -1103,9 +1079,9 @@ impl PersistentTemporalDSU { .cf_handle(self.cf_parent) .ok_or_else(|| anyhow!("missing DSU parent column family"))?; - for (record_id, parent_id) in self.dirty_parents.drain() { - let key = crate::persistence::dsu_encoding::encode_record_key(record_id); - let value = crate::persistence::dsu_encoding::encode_parent_value(parent_id); + for (record_id, parent_id) in &self.dirty_parents { + let key = crate::persistence::dsu_encoding::encode_record_key(*record_id); + let value = crate::persistence::dsu_encoding::encode_parent_value(*parent_id); batch.put_cf(parent_cf, key, value); } @@ -1115,9 +1091,9 @@ impl PersistentTemporalDSU { .cf_handle(self.cf_rank) .ok_or_else(|| anyhow!("missing DSU rank column family"))?; - for (record_id, rank) in self.dirty_ranks.drain() { - let key = crate::persistence::dsu_encoding::encode_record_key(record_id); - let value = crate::persistence::dsu_encoding::encode_rank_value(rank); + for (record_id, rank) in &self.dirty_ranks { + let key = crate::persistence::dsu_encoding::encode_record_key(*record_id); + let value = crate::persistence::dsu_encoding::encode_rank_value(*rank); batch.put_cf(rank_cf, key, value); } @@ -1127,17 +1103,35 @@ impl PersistentTemporalDSU { .cf_handle(self.cf_guards) .ok_or_else(|| anyhow!("missing DSU guards column family"))?; - for (record_id, guards) in self.dirty_guards.drain() { - let key = crate::persistence::dsu_encoding::encode_record_key(record_id); - let value = crate::persistence::dsu_encoding::encode_guards(&guards)?; + for (record_id, guards) in &self.dirty_guards { + let key = crate::persistence::dsu_encoding::encode_record_key(*record_id); + let value = crate::persistence::dsu_encoding::encode_guards(guards)?; batch.put_cf(guards_cf, key, value); } - // Write batch - self.db.write(batch)?; + // State and metadata must commit atomically. Keep dirty entries intact until + // RocksDB confirms the write so a transient failure remains retryable. + let metadata_cf = self + .db + .cf_handle(self.cf_metadata) + .ok_or_else(|| anyhow!("missing DSU metadata column family"))?; + let next_id = self.next_cluster_id.load(Ordering::SeqCst); + batch.put_cf( + metadata_cf, + crate::persistence::dsu_keys::NEXT_CLUSTER_ID, + next_id.to_be_bytes(), + ); + let count = self.cluster_count.load(Ordering::SeqCst); + batch.put_cf( + metadata_cf, + crate::persistence::dsu_keys::CLUSTER_COUNT, + count.to_be_bytes(), + ); - // Save metadata - self.save_metadata()?; + self.db.write(batch)?; + self.dirty_parents.clear(); + self.dirty_ranks.clear(); + self.dirty_guards.clear(); Ok(()) } diff --git a/src/graph.rs b/src/graph.rs index 969ca33..9db1aeb 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1199,7 +1199,7 @@ fn key_attributes_for_type(ontology: &Ontology, entity_type: &str) -> (Vec) -> anyhow::Result { + self.ingest_internal(records, false) + } + + /// Ingest snapshots while preserving their explicit record IDs. + #[doc(hidden)] + pub fn ingest_with_explicit_ids( + &mut self, + records: Vec, + ) -> anyhow::Result { + self.ingest_internal(records, true) + } + + fn ingest_internal( + &mut self, + records: Vec, + preserve_record_ids: bool, + ) -> anyhow::Result { self.clear_conflict_cache(); self.clear_query_stats(); if self.streaming.is_none() { @@ -329,7 +360,11 @@ impl Unirust { // Stage all records and collect their IDs let mut staged_info: Vec<(RecordId, bool)> = Vec::with_capacity(records.len()); for record in records { - let (record_id, inserted) = self.store.stage_record_if_absent(record)?; + let (record_id, inserted) = if preserve_record_ids { + self.store.stage_record_with_explicit_id_if_absent(record)? + } else { + self.store.stage_record_if_absent(record)? + }; staged_info.push((record_id, inserted)); } @@ -343,7 +378,12 @@ impl Unirust { .as_mut() .ok_or_else(|| anyhow::anyhow!("Streaming not initialized"))?; - // Get record references for parallel processing + let inserted_ids: Vec<_> = staged_info + .iter() + .filter_map(|(id, inserted)| inserted.then_some(*id)) + .collect(); + // In-memory stores can lend records directly. Persistent stores return + // owned records from their cache for the parallel extraction phase. let records_to_link: Vec<&Record> = staged_info .iter() .filter(|(_, inserted)| *inserted) @@ -351,19 +391,43 @@ impl Unirust { .collect(); // Use parallel batch linking for large batches (>= 100 records) - if records_to_link.len() >= 100 { - let cluster_ids = streaming.link_records_batch_parallel_with_interner( - &records_to_link, - &self.ontology, - self.store.interner(), - )?; + if inserted_ids.len() >= 100 { + let cluster_ids = if records_to_link.len() == inserted_ids.len() { + streaming.link_records_batch_parallel_with_interner( + &records_to_link, + &self.ontology, + self.store.interner(), + )? + } else { + let owned_records: Vec = inserted_ids + .iter() + .map(|id| { + self.store.get_record(*id).ok_or_else(|| { + anyhow::anyhow!("staged record {} is unavailable", id.0) + }) + }) + .collect::>()?; + let owned_refs: Vec<&Record> = owned_records.iter().collect(); + streaming.link_records_batch_parallel_with_interner( + &owned_refs, + &self.ontology, + self.store.interner(), + )? + }; + if cluster_ids.len() != inserted_ids.len() { + anyhow::bail!( + "entity resolution returned {} assignments for {} records", + cluster_ids.len(), + inserted_ids.len() + ); + } let mut cluster_id_iter = cluster_ids.into_iter(); for (record_id, inserted) in &staged_info { let cluster_id = if *inserted { cluster_id_iter .next() - .unwrap_or_else(|| streaming.cluster_id_for(*record_id)) + .ok_or_else(|| anyhow::anyhow!("missing cluster assignment"))? } else { streaming.cluster_id_for(*record_id) }; @@ -408,22 +472,18 @@ impl Unirust { )?; } - // Flush all staged records to DB - if let Err(e) = self.store.flush_staged_records() { - error!(error = %e, "Failed to flush staged records"); - } + // Do not acknowledge an ingest until its records and resolution state persist. + self.store.flush_staged_records()?; // Batch write all cluster assignments let batch_assignments: Vec<_> = assignments .iter() .map(|a| (a.record_id, a.cluster_id)) .collect(); - if let Err(e) = self.store.set_cluster_assignments_batch(&batch_assignments) { - error!(error = %e, "Failed to persist cluster assignments"); - } - if let Err(e) = self.store.set_cluster_count(cluster_count) { - warn!(error = %e, "Failed to persist cluster count"); - } + self.store + .set_cluster_assignments_batch(&batch_assignments)?; + self.store.set_cluster_count(cluster_count)?; + self.store.sync()?; // Summarize conflicts let conflicts = conflicts::summarize_conflicts(self.store.as_ref(), &observations); @@ -450,11 +510,13 @@ impl Unirust { /// Get all cluster assignments. pub fn clusters(&mut self) -> anyhow::Result { - if let Some(streaming) = &mut self.streaming { - Ok(streaming.clusters_with_conflict_splitting(self.store.as_ref(), &self.ontology)?) + let clusters = if let Some(streaming) = &mut self.streaming { + streaming.clusters_with_conflict_splitting(self.store.as_ref(), &self.ontology)? } else { - linker::build_clusters(self.store.as_ref(), &self.ontology) - } + linker::build_clusters(self.store.as_ref(), &self.ontology)? + }; + self.store.ensure_healthy()?; + Ok(clusters) } /// Export the knowledge graph. @@ -462,12 +524,14 @@ impl Unirust { let clusters = self.clusters()?; let observations = conflicts::detect_conflicts(self.store.as_ref(), &clusters, &self.ontology)?; - graph::export_graph( + let graph = graph::export_graph( self.store.as_ref(), &clusters, &observations, &self.ontology, - ) + )?; + self.store.ensure_healthy()?; + Ok(graph) } /// Persist all state to disk (for persistent stores). @@ -483,6 +547,7 @@ impl Unirust { } // Flush any staged records self.store.flush_staged_records()?; + self.store.sync()?; Ok(()) } @@ -530,6 +595,23 @@ impl Unirust { self.store.get_record(id) } + /// Get a record by its immutable source identity. + #[doc(hidden)] + pub fn get_record_by_identity( + &self, + identity: &RecordIdentity, + ) -> anyhow::Result> { + let Some(record_id) = self.store.get_record_id_by_identity(identity) else { + self.store.ensure_healthy()?; + return Ok(None); + }; + let record = self.store.get_record(record_id).ok_or_else(|| { + anyhow::anyhow!("identity index references missing record {}", record_id.0) + })?; + self.store.ensure_healthy()?; + Ok(Some(record)) + } + /// Access the underlying store (for advanced use cases). pub fn store(&self) -> &dyn RecordStore { self.store.as_ref() @@ -602,6 +684,25 @@ impl Unirust { self.store.len() } + #[doc(hidden)] + pub fn ensure_store_healthy(&self) -> anyhow::Result<()> { + self.store.ensure_healthy() + } + + /// Durably clear the store and install a fresh ontology on the same handle. + #[doc(hidden)] + pub fn reset_with_ontology(&mut self, mut ontology: Ontology) -> anyhow::Result<()> { + self.store.reset_data()?; + ontology.intern_attributes(|name| self.store.intern_attr(name)); + self.ontology = ontology; + self.streaming = None; + self.graph_state = None; + self.query_cache = std::sync::Mutex::new(None); + self.conflict_cache = std::sync::Mutex::new(None); + self.query_stats = std::sync::Mutex::new(query::QuerySelectivityStats::default()); + Ok(()) + } + pub fn streaming_cluster_count(&self) -> Option { self.streaming .as_ref() @@ -609,6 +710,18 @@ impl Unirust { .or_else(|| self.store.cluster_count()) } + /// Reconstruct the streaming linker before accepting live traffic. + /// + /// Persistent shards call this during startup so recovery cost and failures are + /// surfaced before the gRPC listener becomes ready. + #[doc(hidden)] + pub fn initialize_streaming(&mut self) -> anyhow::Result<()> { + if self.streaming.is_none() { + self.streaming = Some(self.create_streaming_linker()?); + } + Ok(()) + } + /// Get the number of conflicts detected by the streaming linker pub fn streaming_conflicts_detected(&self) -> u64 { self.streaming @@ -709,7 +822,12 @@ impl Unirust { .as_mut() .ok_or_else(|| anyhow::anyhow!("Streaming not initialized"))?; - // Get record references for parallel processing + let inserted_ids: Vec<_> = staged_info + .iter() + .filter_map(|(id, inserted)| inserted.then_some(*id)) + .collect(); + // In-memory stores can lend records directly. Persistent stores return + // owned records from their cache for the parallel extraction phase. let records_to_link: Vec<&Record> = staged_info .iter() .filter(|(_, inserted)| *inserted) @@ -717,19 +835,43 @@ impl Unirust { .collect(); // Use parallel batch linking for large batches (>= 100 records) - if records_to_link.len() >= 100 { - let cluster_ids = streaming.link_records_batch_parallel_with_interner( - &records_to_link, - &self.ontology, - self.store.interner(), - )?; + if inserted_ids.len() >= 100 { + let cluster_ids = if records_to_link.len() == inserted_ids.len() { + streaming.link_records_batch_parallel_with_interner( + &records_to_link, + &self.ontology, + self.store.interner(), + )? + } else { + let owned_records: Vec = inserted_ids + .iter() + .map(|id| { + self.store.get_record(*id).ok_or_else(|| { + anyhow::anyhow!("staged record {} is unavailable", id.0) + }) + }) + .collect::>()?; + let owned_refs: Vec<&Record> = owned_records.iter().collect(); + streaming.link_records_batch_parallel_with_interner( + &owned_refs, + &self.ontology, + self.store.interner(), + )? + }; + if cluster_ids.len() != inserted_ids.len() { + anyhow::bail!( + "entity resolution returned {} assignments for {} records", + cluster_ids.len(), + inserted_ids.len() + ); + } let mut cluster_id_iter = cluster_ids.into_iter(); for (record_id, inserted) in &staged_info { let cluster_id = if *inserted { cluster_id_iter .next() - .unwrap_or_else(|| streaming.cluster_id_for(*record_id)) + .ok_or_else(|| anyhow::anyhow!("missing cluster assignment"))? } else { streaming.cluster_id_for(*record_id) }; @@ -764,21 +906,17 @@ impl Unirust { }; // Flush all staged records to DB in a single batch write - if let Err(e) = self.store.flush_staged_records() { - error!(error = %e, "Failed to flush staged records"); - } + self.store.flush_staged_records()?; // Batch write all cluster assignments let batch_assignments: Vec<_> = assignments .iter() .map(|a| (a.record_id, a.cluster_id)) .collect(); - if let Err(e) = self.store.set_cluster_assignments_batch(&batch_assignments) { - error!(error = %e, "Failed to persist cluster assignments"); - } - if let Err(e) = self.store.set_cluster_count(cluster_count) { - warn!(error = %e, "Failed to persist cluster count"); - } + self.store + .set_cluster_assignments_batch(&batch_assignments)?; + self.store.set_cluster_count(cluster_count)?; + self.store.sync()?; self.invalidate_query_cache(); Ok(assignments) } @@ -807,9 +945,7 @@ impl Unirust { } else { streaming.cluster_id_for(record_id) }; - if let Err(e) = self.store.set_cluster_assignment(record_id, cluster_id) { - error!(record_id = ?record_id, cluster_id = ?cluster_id, error = %e, "Failed to persist cluster assignment"); - } + self.store.set_cluster_assignment(record_id, cluster_id)?; let assignment = ClusterAssignment { record_id, cluster_id, @@ -829,12 +965,8 @@ impl Unirust { if inserted && !observations.is_empty() { let summaries = conflicts::summarize_conflicts(self.store.as_ref(), &observations); - if let Err(e) = self - .store - .set_cluster_conflict_summaries(cluster_id, &summaries) - { - warn!(cluster_id = ?cluster_id, error = %e, "Failed to persist cluster conflict summaries"); - } + self.store + .set_cluster_conflict_summaries(cluster_id, &summaries)?; } updates.push(StreamedConflictUpdate { assignment, @@ -846,9 +978,8 @@ impl Unirust { }; self.invalidate_query_cache(); - if let Err(e) = self.store.set_cluster_count(cluster_count) { - warn!(cluster_count = cluster_count, error = %e, "Failed to persist cluster count"); - } + self.store.set_cluster_count(cluster_count)?; + self.store.sync()?; Ok(updates) } @@ -940,14 +1071,11 @@ impl Unirust { } // Flush all staged records in a single batch write - if let Err(e) = self.store.flush_staged_records() { - error!(error = %e, "Failed to flush staged records - data may be stale"); - } + self.store.flush_staged_records()?; // Batch write all cluster assignments - if let Err(e) = self.store.set_cluster_assignments_batch(&assignments_batch) { - error!(error = %e, "Failed to persist cluster assignments batch"); - } + self.store + .set_cluster_assignments_batch(&assignments_batch)?; // Final conflict detection and summary (once at end) if let Some(streaming) = self.streaming.as_mut() { @@ -955,16 +1083,13 @@ impl Unirust { let observations = conflicts::detect_conflicts(self.store.as_ref(), &clusters, &self.ontology)?; let summaries = conflicts::summarize_conflicts(self.store.as_ref(), &observations); - if let Err(e) = self.store.set_conflict_summaries(&summaries) { - warn!(error = %e, "Failed to persist conflict summaries"); - } + self.store.set_conflict_summaries(&summaries)?; *self.conflict_cache.lock().unwrap() = Some(summaries); } self.invalidate_query_cache(); - if let Err(e) = self.store.set_cluster_count(cluster_count) { - warn!(cluster_count = cluster_count, error = %e, "Failed to persist cluster count"); - } + self.store.set_cluster_count(cluster_count)?; + self.store.sync()?; Ok(updates) } @@ -999,7 +1124,7 @@ impl Unirust { } let cache = cache_guard.as_ref().expect("cache"); let mut stats_guard = self.query_stats.lock().expect("query stats lock"); - query::query_master_entities_with_cache_selective( + let outcome = query::query_master_entities_with_cache_selective( self.store.as_ref(), &cache.clusters, descriptors, @@ -1008,7 +1133,9 @@ impl Unirust { &cache.cluster_keys, &cache.record_to_cluster, &mut stats_guard, - ) + )?; + self.store.ensure_healthy()?; + Ok(outcome) } fn invalidate_query_cache(&self) { @@ -1029,8 +1156,26 @@ impl Unirust { self.conflict_cache.lock().unwrap().clone() } - pub fn load_conflict_summaries(&self) -> Option> { - self.store.load_conflict_summaries() + pub fn load_conflict_summaries( + &self, + ) -> anyhow::Result>> { + let summaries = self.store.load_conflict_summaries(); + self.store.ensure_healthy()?; + Ok(summaries) + } + + /// Persist the durable cross-shard conflict set. + pub fn persist_cross_shard_conflicts( + &mut self, + conflicts: &[sharding::CrossShardConflict], + ) -> anyhow::Result<()> { + self.store.set_cross_shard_conflicts(conflicts)?; + self.store.sync() + } + + /// Load cross-shard conflicts recovered from persistent storage. + pub fn load_cross_shard_conflicts(&self) -> anyhow::Result> { + self.store.load_cross_shard_conflicts() } pub fn conflict_summary_count(&self) -> Option { @@ -1041,11 +1186,15 @@ impl Unirust { .or_else(|| self.store.conflict_summary_count()) } - pub fn set_conflict_cache(&mut self, summaries: Vec) { - if let Err(err) = self.store.set_conflict_summaries(&summaries) { - eprintln!("Failed to persist conflict summaries: {err}"); - } + pub fn set_conflict_cache( + &mut self, + summaries: Vec, + ) -> anyhow::Result<()> { + self.store.ensure_healthy()?; + self.store.set_conflict_summaries(&summaries)?; + self.store.sync()?; *self.conflict_cache.lock().unwrap() = Some(summaries); + Ok(()) } fn clear_conflict_cache(&self) { @@ -1117,12 +1266,59 @@ impl Unirust { self.streaming = Some(self.create_streaming_linker()?); } + let (primary, secondary) = { + let streaming = self.streaming.as_ref().ok_or_else(|| { + anyhow::anyhow!("Streaming not initialized - call enable_streaming() first") + })?; + ( + streaming.resolve_global_cluster_id(primary), + streaming.resolve_global_cluster_id(secondary), + ) + }; + if primary == secondary { + return Ok(0); + } + if primary.to_u64() >= secondary.to_u64() { + anyhow::bail!( + "cross-shard merge must redirect cluster {} to lower canonical cluster {}", + secondary.to_u64(), + primary.to_u64() + ); + } + + if let Some(db) = self.store.shared_db() { + let persistence = LinkerStatePersistence::new(&db); + persistence.save_cross_shard_merge(secondary, primary)?; + self.store.sync()?; + } let streaming = self.streaming.as_mut().ok_or_else(|| { anyhow::anyhow!("Streaming not initialized - call enable_streaming() first") })?; Ok(streaming.apply_cross_shard_merge(primary, secondary)) } + /// Resolve a global cluster ID through durable cross-shard redirects. + pub fn resolve_global_cluster_id(&self, id: GlobalClusterId) -> Option { + self.streaming + .as_ref() + .map(|streaming| streaming.resolve_global_cluster_id(id)) + } + + /// Return the canonical global cluster ID for a record. + pub fn global_cluster_id_for_record( + &mut self, + record_id: RecordId, + ) -> anyhow::Result { + if self.streaming.is_none() { + self.streaming = Some(self.create_streaming_linker()?); + } + let streaming = self.streaming.as_mut().ok_or_else(|| { + anyhow::anyhow!("Streaming not initialized - call enable_streaming() first") + })?; + let global_id = streaming.global_cluster_id_for(record_id); + Ok(streaming.resolve_global_cluster_id(global_id)) + } + /// Get the number of cross-shard merge mappings tracked. pub fn cross_shard_merge_count(&self) -> usize { self.streaming @@ -1192,7 +1388,18 @@ impl Unirust { /// This is a convenience method that flushes linker state and ensures /// all data is synced to disk. Call before shutdown for complete recovery. pub fn checkpoint_linker_state(&mut self) -> anyhow::Result<()> { - self.flush_linker_state()?; + self.initialize_streaming()?; + let db = self + .store + .shared_db() + .ok_or_else(|| anyhow::anyhow!("Persistence not available - use PersistentStore"))?; + let streaming = self + .streaming + .as_mut() + .ok_or_else(|| anyhow::anyhow!("Streaming initialization failed"))?; + streaming.flush_dsu()?; + let persistence = LinkerStatePersistence::new(&db); + streaming.flush_state(&persistence)?; // Also flush the underlying store to ensure staged records are persisted if let Err(e) = self.store.flush_staged_records() { @@ -1200,6 +1407,176 @@ impl Unirust { return Err(anyhow::anyhow!("Failed to flush staged records: {}", e)); } + self.store.sync()?; + Ok(()) } + + /// Flush the bounded amount of dirty state required for graceful process exit. + /// + /// Full linker-map snapshots grow with total record count and are not needed by + /// the authoritative record-scan recovery path. + #[doc(hidden)] + pub fn checkpoint_for_shutdown(&mut self) -> anyhow::Result<()> { + self.initialize_streaming()?; + let streaming = self + .streaming + .as_mut() + .ok_or_else(|| anyhow::anyhow!("Streaming initialization failed"))?; + streaming.flush_dsu()?; + self.store.flush_staged_records()?; + self.store.sync() + } +} + +#[cfg(test)] +mod persistence_error_tests { + use super::*; + use crate::model::{AttrId, RecordIdentity, StringInterner}; + use crate::ontology::IdentityKey; + use crate::store::RecordStore; + use crate::temporal::Interval; + + struct FailingStore { + inner: Store, + fail_flush: bool, + fail_sync: bool, + } + + impl RecordStore for FailingStore { + fn add_record(&mut self, record: Record) -> anyhow::Result { + self.inner.add_record(record) + } + + fn get_record(&self, id: RecordId) -> Option { + self.inner.get_record(id) + } + + fn get_record_ref(&self, id: RecordId) -> Option<&Record> { + self.inner.get_record_ref(id) + } + + fn get_record_id_by_identity(&self, identity: &RecordIdentity) -> Option { + self.inner.get_record_id_by_identity(identity) + } + + fn get_all_records(&self) -> Vec { + self.inner.get_all_records() + } + + fn get_records_by_entity_type(&self, entity_type: &str) -> Vec { + self.inner.get_records_by_entity_type(entity_type) + } + + fn get_records_by_perspective(&self, perspective: &str) -> Vec { + self.inner.get_records_by_perspective(perspective) + } + + fn get_records_with_attribute(&self, attr: AttrId) -> Vec { + self.inner.get_records_with_attribute(attr) + } + + fn get_records_in_interval(&self, interval: Interval) -> Vec { + self.inner.get_records_in_interval(interval) + } + + fn interner(&self) -> &StringInterner { + self.inner.interner() + } + + fn interner_mut(&mut self) -> &mut StringInterner { + self.inner.interner_mut() + } + + fn len(&self) -> usize { + self.inner.len() + } + + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn records_in_id_range( + &self, + start: RecordId, + end: RecordId, + max_results: usize, + ) -> Vec { + self.inner.records_in_id_range(start, end, max_results) + } + + fn record_id_bounds(&self) -> Option<(RecordId, RecordId)> { + self.inner.record_id_bounds() + } + + fn flush_staged_records(&mut self) -> anyhow::Result { + if self.fail_flush { + anyhow::bail!("injected flush failure"); + } + Ok(0) + } + + fn sync(&self) -> anyhow::Result<()> { + if self.fail_sync { + anyhow::bail!("injected sync failure"); + } + Ok(()) + } + } + + #[test] + fn ingest_propagates_persistence_failure() { + let mut ontology = Ontology::new(); + ontology.add_identity_key(IdentityKey::from_names(vec!["email"], "email")); + let mut engine = Unirust::with_store( + ontology, + FailingStore { + inner: Store::new(), + fail_flush: true, + fail_sync: false, + }, + ); + let email_attr = engine.intern_attr("email"); + let email_value = engine.intern_value("alice@example.com"); + let record = Record::new( + RecordId(0), + RecordIdentity::new("person".into(), "crm".into(), "alice-1".into()), + vec![Descriptor::new( + email_attr, + email_value, + Interval::new(0, 10).unwrap(), + )], + ); + + let error = engine.ingest(vec![record]).unwrap_err(); + assert!(error.to_string().contains("injected flush failure")); + } + + #[test] + fn ingest_propagates_stable_storage_sync_failure() { + let mut ontology = Ontology::new(); + ontology.add_identity_key(IdentityKey::from_names(vec!["email"], "email")); + let mut engine = Unirust::with_store( + ontology, + FailingStore { + inner: Store::new(), + fail_flush: false, + fail_sync: true, + }, + ); + let email_attr = engine.intern_attr("email"); + let email_value = engine.intern_value("alice@example.com"); + let record = Record::new( + RecordId(0), + RecordIdentity::new("person".into(), "crm".into(), "alice-sync".into()), + vec![Descriptor::new( + email_attr, + email_value, + Interval::new(0, 10).unwrap(), + )], + ); + + let error = engine.ingest(vec![record]).unwrap_err(); + assert!(error.to_string().contains("injected sync failure")); + } } diff --git a/src/linker.rs b/src/linker.rs index 75e904c..274bb08 100644 --- a/src/linker.rs +++ b/src/linker.rs @@ -15,8 +15,9 @@ use crate::index::IndexBackend; use crate::model::{ClusterId, GlobalClusterId, InternerLookup, KeyValue, Record, RecordId}; use crate::ontology::Ontology; use crate::perf::bigtable_opts::PartitionOptimizations; -use crate::sharding::IdentityKeySignature as ShardingKeySignature; -use crate::sharding::IdentityKeySignature; +use crate::sharding::{ + BoundaryStrongId, IdentityKeySignature, IdentityKeySignature as ShardingKeySignature, +}; use crate::store::RecordStore; use crate::temporal::Interval; use anyhow::Result; @@ -38,11 +39,71 @@ type CandidateVec = SmallVec<[(RecordId, Interval); 32]>; /// Data stored for boundary signatures to support cross-shard conflict detection. #[derive(Debug, Clone)] struct BoundaryData { - /// Merged interval for this (signature, cluster) combination. - interval: Interval, + /// Coalesced identity-key intervals for this (signature, cluster) combination. + intervals: Vec, /// Strong ID hashes per perspective for conflict detection. /// Key: hash of perspective name, Value: hash of (attr_id, value_id) tuples. perspective_strong_ids: HashMap, + /// Exact temporal observations used by the distributed merge guard. + strong_ids: Vec, +} + +fn coalesce_boundary_intervals(intervals: &mut Vec) { + intervals.sort_by_key(|interval| (interval.start, interval.end)); + let mut coalesced: Vec = Vec::with_capacity(intervals.len()); + for interval in intervals.drain(..) { + if let Some(previous) = coalesced.last_mut() { + if interval.start <= previous.end { + previous.end = previous.end.max(interval.end); + continue; + } + } + coalesced.push(interval); + } + *intervals = coalesced; +} + +fn sort_dedup_boundary_strong_ids(strong_ids: &mut Vec) { + strong_ids.sort_by(|left, right| { + ( + &left.perspective, + &left.attribute, + &left.value, + left.interval.start, + left.interval.end, + ) + .cmp(&( + &right.perspective, + &right.attribute, + &right.value, + right.interval.start, + right.interval.end, + )) + }); + strong_ids.dedup(); +} + +impl BoundaryData { + fn new( + interval: Interval, + perspective_strong_ids: HashMap, + strong_ids: Vec, + ) -> Self { + Self { + intervals: vec![interval], + perspective_strong_ids, + strong_ids, + } + } + + fn merge_from(&mut self, other: &Self) { + self.intervals.extend(other.intervals.iter().copied()); + coalesce_boundary_intervals(&mut self.intervals); + self.perspective_strong_ids + .extend(other.perspective_strong_ids.clone()); + self.strong_ids.extend(other.strong_ids.iter().cloned()); + sort_dedup_boundary_strong_ids(&mut self.strong_ids); + } } /// Metrics for observability of the streaming linker. @@ -448,23 +509,21 @@ impl StreamingLinker { dsu: DsuBackend, identity_index: IndexBackend, ) -> Result { - // Create LinkerState instances based on config (LRU-bounded or HashMap) - let (cluster_ids, global_cluster_ids, strong_id_summaries, record_perspectives) = - if let Some(config) = &tuning.linker_state_config { - ( - LinkerState::bounded(config.cluster_ids_capacity), - LinkerState::bounded(config.global_ids_capacity), - LinkerState::bounded(config.summaries_capacity), - LinkerState::bounded(config.perspectives_capacity), - ) - } else { - ( - LinkerState::unbounded(), - LinkerState::unbounded(), - LinkerState::unbounded(), - LinkerState::unbounded(), - ) - }; + // These mappings and summaries affect resolution correctness. The bounded + // backend cannot be enabled until evictions have a durable spill/read-through + // path; silently dropping an old entry can split clusters after enough ingest. + if tuning.linker_state_config.is_some() { + tracing::warn!( + "bounded linker state requested but durable spill is unavailable; \ + using correctness-preserving unbounded state" + ); + } + let (cluster_ids, global_cluster_ids, strong_id_summaries, record_perspectives) = ( + LinkerState::unbounded(), + LinkerState::unbounded(), + LinkerState::unbounded(), + LinkerState::unbounded(), + ); let mut streamer = Self { dsu, @@ -498,6 +557,7 @@ impl StreamingLinker { } } + store.ensure_healthy()?; Ok(streamer) } @@ -657,7 +717,7 @@ impl StreamingLinker { ); let candidates: CandidateVec = candidates_slice.iter().copied().collect(); // Cache the result for future lookups - if let Some(ref opts) = partition_opts { + if let Some(opts) = partition_opts { opts.cache_candidates(&identity_sig, candidates.to_vec()); } (candidates, is_hot) @@ -758,20 +818,22 @@ impl StreamingLinker { None }; - // Track if we should record boundary after the loop - let mut boundary_to_track: Option = None; - // Use destructuring for separate mutable borrows in candidate processing let StreamingLinker { dsu, identity_index, cluster_ids, next_cluster_id, + global_cluster_ids, strong_id_summaries, tainted_identity_keys, record_perspectives, + boundary_signatures, + dirty_boundary_keys, + cross_shard_merges, metrics, partition_opts, + shard_id, .. } = self; @@ -853,6 +915,16 @@ impl StreamingLinker { root_b, new_root, ); + reconcile_global_cluster_ids( + global_cluster_ids, + cross_shard_merges, + boundary_signatures, + dirty_boundary_keys, + *shard_id, + root_a, + root_b, + new_root, + ); reconcile_cluster_summaries(strong_id_summaries, root_a, root_b, new_root); identity_index.merge_key_clusters( entity_type, @@ -862,13 +934,10 @@ impl StreamingLinker { new_root, ); // Invalidate cache on merge - candidates changed - if let Some(ref opts) = partition_opts { + if let Some(opts) = partition_opts { opts.on_cluster_merge(root_a, root_b); } root_a = new_root; - - // Mark for boundary tracking after borrows are released - boundary_to_track = Some(new_root); } } // Record stats after candidates loop (borrows dropped) @@ -879,22 +948,17 @@ impl StreamingLinker { .or_default() .record(candidate_len, actual_cap); } - // Track boundary signature for merges (borrows now released) - if let Some(new_root) = boundary_to_track { - self.track_merge_boundary( - interner, - entity_type, - key_values, - new_root, - interval, - ); - } } } // Add the record to the index after matching to avoid self-matches. let _add_guard = crate::profile::profile_scope("add_to_index"); let root = self.dsu.find(record_id).unwrap_or(record_id); + for (_, key_values_with_intervals) in &cached_keys { + for (key_values, interval) in key_values_with_intervals { + self.track_merge_boundary(interner, entity_type, key_values, root, *interval); + } + } self.identity_index .add_record_with_cached_keys(record_id, root, entity_type, cached_keys); @@ -971,7 +1035,7 @@ impl StreamingLinker { cluster_ids.push(cluster_id); // Phase 3: Add to index (requires record reference) - self.add_to_index_after_parallel_link(record, ontology)?; + self.add_to_index_after_parallel_link(record, ontology, interner)?; } Ok(cluster_ids) @@ -1023,7 +1087,7 @@ impl StreamingLinker { &mut self, _ontology: &Ontology, extraction: ParallelExtractionResult, - interner: Option<&dyn InternerLookup>, + _interner: Option<&dyn InternerLookup>, ) -> Result { let record_id = extraction.record_id; let entity_type = &extraction.entity_type; @@ -1121,6 +1185,16 @@ impl StreamingLinker { root_b, new_root, ); + reconcile_global_cluster_ids( + &mut self.global_cluster_ids, + &mut self.cross_shard_merges, + &mut self.boundary_signatures, + &mut self.dirty_boundary_keys, + self.shard_id, + root_a, + root_b, + new_root, + ); reconcile_cluster_summaries( &mut self.strong_id_summaries, root_a, @@ -1135,15 +1209,6 @@ impl StreamingLinker { new_root, ); root_a = new_root; - if let Some(interner) = interner { - self.track_merge_boundary( - interner, - entity_type, - &key_values, - new_root, - interval, - ); - } } } } @@ -1162,6 +1227,7 @@ impl StreamingLinker { &mut self, record: &Record, ontology: &Ontology, + interner: Option<&dyn InternerLookup>, ) -> Result<()> { let root = self.dsu.find(record.id).unwrap_or(record.id); let entity_type = &record.identity.entity_type; @@ -1178,6 +1244,14 @@ impl StreamingLinker { } } + if let Some(interner) = interner { + for (_, key_values_with_intervals) in &cached_keys { + for (key_values, interval) in key_values_with_intervals { + self.track_merge_boundary(interner, entity_type, key_values, root, *interval); + } + } + } + self.identity_index .add_record_with_cached_keys(record.id, root, entity_type, cached_keys); @@ -1274,12 +1348,15 @@ impl StreamingLinker { pub fn export_boundary_index(&self) -> crate::sharding::ClusterBoundaryIndex { let mut index = crate::sharding::ClusterBoundaryIndex::new_small(self.shard_id); for ((sig, global_id), data) in &self.boundary_signatures { - index.register_boundary_key_with_strong_ids( - *sig, - *global_id, - data.interval, - data.perspective_strong_ids.clone(), - ); + for interval in &data.intervals { + index.register_boundary_key_with_conflict_data( + *sig, + *global_id, + *interval, + data.perspective_strong_ids.clone(), + data.strong_ids.clone(), + ); + } } index } @@ -1289,14 +1366,15 @@ impl StreamingLinker { pub fn drain_boundaries(&mut self) -> Vec<(ShardingKeySignature, GlobalClusterId, Interval)> { std::mem::take(&mut self.boundary_signatures) .into_iter() - .map(|((sig, global_id), data)| (sig, global_id, data.interval)) + .flat_map(|((sig, global_id), data)| { + data.intervals + .into_iter() + .map(move |interval| (sig, global_id, interval)) + }) .collect() } - /// Track boundary signature when a merge happens. - /// This is the optimized path - we only track boundaries when clusters actually merge, - /// not on every record. This significantly reduces overhead while still capturing - /// the important cross-shard boundary information. + /// Track an identity-key observation for distributed reconciliation. #[inline] fn track_merge_boundary( &mut self, @@ -1327,25 +1405,26 @@ impl StreamingLinker { .peek(&new_root) .map(|summary| summary.compute_perspective_strong_ids(interner)) .unwrap_or_default(); + let strong_ids = self + .strong_id_summaries + .peek(&new_root) + .map(|summary| summary.compute_boundary_strong_ids(interner)) + .unwrap_or_default(); // Merge intervals for the same (signature, cluster) combination self.boundary_signatures .entry(key) .and_modify(|existing| { - let new_start = existing.interval.start.min(interval.start); - let new_end = existing.interval.end.max(interval.end); - if let Ok(merged) = Interval::new(new_start, new_end) { - existing.interval = merged; - } + existing.intervals.push(interval); + coalesce_boundary_intervals(&mut existing.intervals); // Also merge perspective_strong_ids from the updated cluster for (k, v) in &perspective_strong_ids { existing.perspective_strong_ids.entry(*k).or_insert(*v); } + existing.strong_ids.extend(strong_ids.iter().cloned()); + sort_dedup_boundary_strong_ids(&mut existing.strong_ids); }) - .or_insert_with(|| BoundaryData { - interval, - perspective_strong_ids, - }); + .or_insert_with(|| BoundaryData::new(interval, perspective_strong_ids, strong_ids)); // Mark this key as dirty for adaptive reconciliation self.dirty_boundary_keys.insert(sharding_sig); @@ -1377,17 +1456,7 @@ impl StreamingLinker { // Merge with existing BoundaryData for primary if present self.boundary_signatures .entry(new_key) - .and_modify(|existing| { - let new_start = existing.interval.start.min(data.interval.start); - let new_end = existing.interval.end.max(data.interval.end); - if let Ok(merged) = Interval::new(new_start, new_end) { - existing.interval = merged; - } - // Merge perspective strong IDs - for (k, v) in &data.perspective_strong_ids { - existing.perspective_strong_ids.entry(*k).or_insert(*v); - } - }) + .and_modify(|existing| existing.merge_from(&data)) .or_insert(data); // Mark this key as dirty for adaptive reconciliation self.dirty_boundary_keys.insert(sig); @@ -1619,9 +1688,26 @@ impl StreamingLinker { persistence .flush_global_cluster_ids(self.global_cluster_ids.iter().map(|(k, v)| (*k, *v)))?; + for (secondary, primary) in &self.cross_shard_merges { + persistence.save_cross_shard_merge(*secondary, *primary)?; + } + Ok(()) } + /// Restore durable cross-shard redirects and apply them to reconstructed state. + pub fn restore_cross_shard_merges( + &mut self, + persistence: &crate::persistence::LinkerStatePersistence, + ) -> Result { + let mappings = persistence.load_cross_shard_merges()?; + let count = mappings.len(); + for (secondary, primary) in mappings { + self.apply_cross_shard_merge(primary, secondary); + } + Ok(count) + } + /// Restore linker state from persistent storage. /// Call this during initialization to recover cluster ID mappings. /// Returns the number of cluster_ids restored. @@ -1647,6 +1733,8 @@ impl StreamingLinker { self.global_cluster_ids.insert(record_id, global_id); } + self.restore_cross_shard_merges(persistence)?; + Ok(count) } @@ -1689,6 +1777,31 @@ impl StrongIdSummary { } } + /// Export exact string-valued observations for distributed temporal guards. + fn compute_boundary_strong_ids(&self, interner: &dyn InternerLookup) -> Vec { + let mut observations = Vec::new(); + for (perspective, attrs) in &self.by_perspective { + for (attr_id, values) in attrs { + let Some(attribute) = interner.get_attr_string(*attr_id) else { + continue; + }; + for (value_id, intervals) in values { + let Some(value) = interner.get_value_string(*value_id) else { + continue; + }; + observations.extend(intervals.iter().map(|interval| BoundaryStrongId { + perspective: perspective.clone(), + attribute: attribute.clone(), + value: value.clone(), + interval: *interval, + })); + } + } + } + sort_dedup_boundary_strong_ids(&mut observations); + observations + } + /// Compute perspective -> strong_id_hash for cross-shard conflict detection. /// Returns a map from hashed perspective name to hashed strong ID values. /// @@ -1792,6 +1905,81 @@ mod tests { assert!(linker.boundary_count() > 0); assert!(linker.dirty_boundary_key_count() > 0); } + + #[test] + fn local_merge_canonicalizes_previously_exposed_global_ids() { + let mut store = Store::new(); + let mut ontology = Ontology::new(); + let email_attr = store.intern_attr("email"); + let phone_attr = store.intern_attr("phone"); + ontology.add_identity_key(IdentityKey::new(vec![email_attr], "email_key".to_string())); + ontology.add_identity_key(IdentityKey::new(vec![phone_attr], "phone_key".to_string())); + let tuning = crate::StreamingTuning { + shard_id: 1, + enable_boundary_tracking: true, + ..Default::default() + }; + let mut linker = StreamingLinker::new(&store, &ontology, &tuning).expect("linker"); + let interval = Interval::new(0, 10).expect("interval"); + let email_value = store.intern_value("alice@example.com"); + let phone_value = store.intern_value("+15550001"); + + let records = [ + Record::new( + RecordId(0), + RecordIdentity::new("person".to_string(), "crm".to_string(), "email".to_string()), + vec![Descriptor::new(email_attr, email_value, interval)], + ), + Record::new( + RecordId(0), + RecordIdentity::new("person".to_string(), "crm".to_string(), "phone".to_string()), + vec![Descriptor::new(phone_attr, phone_value, interval)], + ), + Record::new( + RecordId(0), + RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "bridge".to_string(), + ), + vec![ + Descriptor::new(email_attr, email_value, interval), + Descriptor::new(phone_attr, phone_value, interval), + ], + ), + ]; + + for record in records { + let record_id = store.add_record(record).expect("record"); + linker + .link_record(&store, &ontology, record_id) + .expect("link"); + } + + let boundary = linker.export_boundary_index(); + let mut global_ids = [email_attr, phone_attr] + .into_iter() + .zip([email_value, phone_value]) + .flat_map(|(attr, value)| { + let signature = ShardingKeySignature::from_key_values_with_interner( + "person", + &[KeyValue::new(attr, value)], + store.interner(), + ) + .expect("signature"); + boundary + .get_boundaries(&signature) + .expect("boundary") + .iter() + .map(|entry| entry.cluster_id) + .collect::>() + }) + .collect::>(); + global_ids.sort_by_key(GlobalClusterId::to_u64); + global_ids.dedup(); + assert_eq!(global_ids.len(), 1); + assert_eq!(linker.cross_shard_merge_count(), 1); + } } /// Identity key signature for linker deduplication. @@ -1918,6 +2106,86 @@ fn reconcile_cluster_ids( cluster_ids.insert(new_root, chosen); } +#[allow(clippy::too_many_arguments)] +fn reconcile_global_cluster_ids( + global_cluster_ids: &mut LinkerState, + cross_shard_merges: &mut HashMap, + boundary_signatures: &mut HashMap<(ShardingKeySignature, GlobalClusterId), BoundaryData>, + dirty_boundary_keys: &mut HashSet, + shard_id: u16, + root_a: RecordId, + root_b: RecordId, + new_root: RecordId, +) { + fn resolve( + redirects: &HashMap, + id: GlobalClusterId, + ) -> GlobalClusterId { + let mut current = id; + let mut seen = HashSet::new(); + while let Some(next) = redirects.get(¤t).copied() { + if !seen.insert(current) { + break; + } + current = next; + } + current + } + + let id_a = global_cluster_ids.get_copy(&root_a); + let id_b = global_cluster_ids.get_copy(&root_b); + if id_a.is_none() && id_b.is_none() { + return; + } + if root_a != new_root { + global_cluster_ids.remove(&root_a); + } + if root_b != new_root { + global_cluster_ids.remove(&root_b); + } + + let mut ids = [id_a, id_b] + .into_iter() + .flatten() + .flat_map(|id| [id, resolve(cross_shard_merges, id)]) + .collect::>(); + ids.sort_by_key(GlobalClusterId::to_u64); + ids.dedup(); + let canonical = ids + .first() + .copied() + .unwrap_or_else(|| GlobalClusterId::from_local(shard_id, ClusterId(new_root.0))); + + for id in &ids { + if *id == canonical { + cross_shard_merges.remove(id); + } else { + cross_shard_merges.insert(*id, canonical); + } + } + for (_, global_id) in global_cluster_ids.iter_mut() { + if ids.contains(global_id) { + *global_id = canonical; + } + } + global_cluster_ids.insert(new_root, canonical); + + let keys_to_update = boundary_signatures + .keys() + .filter(|(_, global_id)| ids.contains(global_id) && *global_id != canonical) + .cloned() + .collect::>(); + for (signature, old_id) in keys_to_update { + if let Some(data) = boundary_signatures.remove(&(signature, old_id)) { + boundary_signatures + .entry((signature, canonical)) + .and_modify(|existing| existing.merge_from(&data)) + .or_insert(data); + dirty_boundary_keys.insert(signature); + } + } +} + fn reconcile_cluster_summaries( summaries: &mut LinkerState, root_a: RecordId, diff --git a/src/ontology.rs b/src/ontology.rs index bacfae0..a48cb75 100644 --- a/src/ontology.rs +++ b/src/ontology.rs @@ -355,7 +355,7 @@ impl Ontology { attribute_name, .. } => { - if let Some(ref name) = attribute_name { + if let Some(name) = attribute_name { *attribute = intern_fn(name); } } @@ -364,7 +364,7 @@ impl Ontology { attribute_name, .. } => { - if let Some(ref name) = attribute_name { + if let Some(name) = attribute_name { *attribute = intern_fn(name); } } diff --git a/src/partitioned.rs b/src/partitioned.rs index d9c3ac1..0a205aa 100644 --- a/src/partitioned.rs +++ b/src/partitioned.rs @@ -295,10 +295,10 @@ impl Partition { &mut self, records: Vec<(u32, Record)>, ontology: &Ontology, - ) -> Vec { + ) -> Result> { let count = records.len(); if count == 0 { - return Vec::new(); + return Ok(Vec::new()); } // Phase 1: Batch add all records to store, collect metadata @@ -309,13 +309,7 @@ impl Partition { // Compute identity key signature BEFORE moving record to store let identity_sig = self.compute_identity_signature(&record, ontology); - let (record_id, inserted) = match self.store.add_record_if_absent_raw(record) { - Ok((id, ins)) => (id, ins), - Err(e) => { - warn!("Failed to add record: {}", e); - continue; - } - }; + let (record_id, inserted) = self.store.add_record_if_absent_raw(record)?; // Update bloom filter if let Some(ref sig) = identity_sig { @@ -349,22 +343,23 @@ impl Partition { // Use batch parallel linking if we have records to link let cluster_ids: Vec = if !records_to_link.is_empty() { - match self.linker.link_records_batch_parallel_with_interner( + self.linker.link_records_batch_parallel_with_interner( &records_to_link, ontology, self.interner.as_ref(), - ) { - Ok(ids) => ids, - Err(e) => { - warn!("Failed to batch link records: {}", e); - // Fall back to returning empty cluster IDs - vec![ClusterId(0); records_to_link.len()] - } - } + )? } else { Vec::new() }; + if cluster_ids.len() != inserted_info.len() { + anyhow::bail!( + "entity resolution returned {} assignments for {} inserted records", + cluster_ids.len(), + inserted_info.len() + ); + } + let conflicts_after = self .linker .metrics() @@ -412,43 +407,7 @@ impl Partition { // Process any pending cross-partition merges self.drain_merge_queue(ontology); - results - } - - /// ULTRA-FAST batch processing - skips entity resolution AND storage. - /// Uses UID hash as cluster ID for maximum throughput. - /// For pure ingest acknowledgment - real storage/resolution done async. - #[inline] - pub fn process_batch_ultra_fast( - &mut self, - records: Vec<(u32, Record)>, - ) -> Vec { - let count = records.len(); - let mut results = Vec::with_capacity(count); - - for (index, record) in records { - // Compute cluster ID from UID hash - deterministic, no linking needed - let cluster_id = { - let mut hasher = FxHasher::default(); - record.identity.uid.hash(&mut hasher); - ClusterId(hasher.finish() as u32) - }; - - // Skip storage entirely for max throughput - // Records can be reconstructed from source or stored async - let _ = &record; // Keep record for future async storage - - results.push(PartitionIngestResult { - index, - cluster_id, - merges: 0, - had_conflicts: false, - }); - } - - self.records_processed - .fetch_add(count as u64, Ordering::Relaxed); - results + Ok(results) } /// Drain and apply pending cross-partition merges @@ -1035,9 +994,9 @@ impl ParallelPartitionedUnirust { /// Ingest a batch of records using TRUE parallel partition processing. /// Each partition is processed independently with its own lock - no global contention! #[instrument(skip(self, records), level = "debug")] - pub fn ingest_batch(&self, records: Vec<(u32, Record)>) -> Vec { + pub fn ingest_batch(&self, records: Vec<(u32, Record)>) -> Result> { if records.is_empty() { - return Vec::new(); + return Ok(Vec::new()); } let record_count = records.len(); @@ -1051,7 +1010,7 @@ impl ParallelPartitionedUnirust { // Each partition has its own Mutex, so no global lock contention! let ontology = &self.ontology; - let all_results: Vec> = partitioned + let all_results: Result>> = partitioned .into_par_iter() .enumerate() .filter(|(_, batch)| !batch.is_empty()) @@ -1064,9 +1023,9 @@ impl ParallelPartitionedUnirust { .collect(); // Phase 3: Flatten and sort results by original index - let mut results: Vec = all_results.into_iter().flatten().collect(); + let mut results: Vec = all_results?.into_iter().flatten().collect(); results.sort_by_key(|r| r.index); - results + Ok(results) } /// Ingest a batch of records with pre-computed partition IDs. @@ -1077,9 +1036,9 @@ impl ParallelPartitionedUnirust { pub fn ingest_batch_with_partitions( &self, records: Vec<(usize, u32, Record)>, - ) -> Vec { + ) -> Result> { if records.is_empty() { - return Vec::new(); + return Ok(Vec::new()); } let record_count = records.len(); @@ -1097,7 +1056,7 @@ impl ParallelPartitionedUnirust { // Phase 2: Process ALL partitions in PARALLEL using rayon let ontology = &self.ontology; - let all_results: Vec> = partitioned + let all_results: Result>> = partitioned .into_par_iter() .enumerate() .filter(|(_, batch)| !batch.is_empty()) @@ -1109,45 +1068,9 @@ impl ParallelPartitionedUnirust { .collect(); // Phase 3: Flatten and sort results by original index - let mut results: Vec = all_results.into_iter().flatten().collect(); + let mut results: Vec = all_results?.into_iter().flatten().collect(); results.sort_by_key(|r| r.index); - results - } - - /// ULTRA-FAST ingest - skips entity resolution for maximum throughput. - /// Uses UID hash as cluster ID. Entity resolution done lazily at query time. - /// This is the fastest possible ingest path. - #[inline] - pub fn ingest_batch_ultra_fast( - &self, - records: Vec<(u32, Record)>, - ) -> Vec { - if records.is_empty() { - return Vec::new(); - } - - let record_count = records.len(); - self.total_records - .fetch_add(record_count as u64, Ordering::Relaxed); - - // Phase 1: Partition records by primary key (parallel for large batches) - let partitioned = self.partition_records(records); - - // Phase 2: Process ALL partitions in PARALLEL - ULTRA FAST mode - let all_results: Vec> = partitioned - .into_par_iter() - .enumerate() - .filter(|(_, batch)| !batch.is_empty()) - .map(|(partition_id, batch)| { - let mut partition = self.partitions[partition_id].lock(); - partition.process_batch_ultra_fast(batch) - }) - .collect(); - - // Phase 3: Flatten and sort results by original index - let mut results: Vec = all_results.into_iter().flatten().collect(); - results.sort_by_key(|r| r.index); - results + Ok(results) } /// Get the total number of clusters across all partitions diff --git a/src/perf/aligned.rs b/src/perf/aligned.rs index 51f373c..5782690 100644 --- a/src/perf/aligned.rs +++ b/src/perf/aligned.rs @@ -178,11 +178,7 @@ impl AlignedMetrics { cache_misses: self.cache_misses.load(Ordering::Relaxed), hot_key_exits: self.hot_key_exits.load(Ordering::Relaxed), queue_depth: self.queue_depth.load(Ordering::Relaxed), - avg_latency_us: if batches > 0 { - total_latency / batches - } else { - 0 - }, + avg_latency_us: total_latency.checked_div(batches).unwrap_or(0), max_latency_us: self.max_latency_us.load(Ordering::Relaxed), dropped_requests: self.dropped_requests.load(Ordering::Relaxed), } diff --git a/src/perf/write_buffer.rs b/src/perf/write_buffer.rs index 3fcbfa2..177ffdf 100644 --- a/src/perf/write_buffer.rs +++ b/src/perf/write_buffer.rs @@ -113,12 +113,8 @@ impl WriteBufferStats { /// Get average stall duration pub fn avg_stall_duration(&self) -> Duration { let stalls = self.stalls.load(Ordering::Relaxed); - if stalls == 0 { - Duration::ZERO - } else { - let total_us = self.stall_duration_us.load(Ordering::Relaxed); - Duration::from_micros(total_us / stalls) - } + let total_us = self.stall_duration_us.load(Ordering::Relaxed); + Duration::from_micros(total_us.checked_div(stalls).unwrap_or(0)) } } diff --git a/src/persistence.rs b/src/persistence.rs index db18a25..375d068 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -1,15 +1,97 @@ -use crate::model::{Record, RecordId, RecordIdentity, StringInterner}; -use crate::store::{RecordStore, Store, StoreMetrics}; +use crate::model::{GlobalClusterId, Record, RecordId, RecordIdentity, StringInterner}; +use crate::store::{ + records_have_same_payload, RecordStore, SourceRecordReservation, SourceReservationError, Store, + StoreMetrics, +}; use anyhow::{anyhow, Result}; use lru::LruCache; use rocksdb::{ checkpoint::Checkpoint, BlockBasedOptions, Cache, ColumnFamilyDescriptor, DBCompressionType, Direction, IteratorMode, Options, SliceTransform, WriteBatch, WriteOptions, DB, }; +use serde::{Deserialize, Serialize}; use std::cell::RefCell; +use std::collections::HashMap; +use std::fs; +use std::io::Write; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; +pub const CHECKPOINT_PROTOCOL_VERSION: u32 = 1; +const CHECKPOINT_MANIFEST_FORMAT_VERSION: u32 = 1; +const CHECKPOINT_MANIFEST_FILE: &str = "UNIRUST_CHECKPOINT_MANIFEST"; +const CHECKPOINT_COMMITTED_FILE: &str = "UNIRUST_CHECKPOINT_COMMITTED"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClusterCheckpointManifest { + format_version: u32, + checkpoint_protocol_version: u32, + generation: String, + shard_id: u32, + shard_count: u32, +} + +impl ClusterCheckpointManifest { + pub fn generation(&self) -> &str { + &self.generation + } + + pub fn shard_id(&self) -> u32 { + self.shard_id + } + + pub fn shard_count(&self) -> u32 { + self.shard_count + } + + fn new(generation: &str, shard_id: u32, shard_count: u32) -> Result { + if generation.is_empty() { + anyhow::bail!("checkpoint generation must not be empty"); + } + if shard_count == 0 || shard_id >= shard_count { + anyhow::bail!( + "checkpoint shard {} is outside cluster shard count {}", + shard_id, + shard_count + ); + } + Ok(Self { + format_version: CHECKPOINT_MANIFEST_FORMAT_VERSION, + checkpoint_protocol_version: CHECKPOINT_PROTOCOL_VERSION, + generation: generation.to_string(), + shard_id, + shard_count, + }) + } + + fn validate(&self) -> Result<()> { + if self.format_version != CHECKPOINT_MANIFEST_FORMAT_VERSION { + anyhow::bail!( + "unsupported checkpoint manifest format version {}", + self.format_version + ); + } + if self.checkpoint_protocol_version != CHECKPOINT_PROTOCOL_VERSION { + anyhow::bail!( + "unsupported checkpoint protocol version {}", + self.checkpoint_protocol_version + ); + } + if self.generation.is_empty() { + anyhow::bail!("checkpoint manifest generation is empty"); + } + if self.shard_count == 0 || self.shard_id >= self.shard_count { + anyhow::bail!( + "checkpoint manifest shard {} is outside cluster shard count {}", + self.shard_id, + self.shard_count + ); + } + Ok(()) + } +} + const CF_RECORDS: &str = "records"; const CF_METADATA: &str = "metadata"; const CF_INTERNER: &str = "interner"; @@ -20,6 +102,7 @@ const CF_INDEX_TEMPORAL_BUCKET: &str = "index_temporal_bucket"; const CF_INDEX_IDENTITY: &str = "index_identity"; const CF_CONFLICT_SUMMARIES: &str = "conflict_summaries"; const CF_CLUSTER_ASSIGNMENTS: &str = "cluster_assignments"; +const CF_SOURCE_RESERVATIONS: &str = "source_reservations"; thread_local! { static RECORD_SER_BUF: RefCell> = const { RefCell::new(Vec::new()) }; @@ -36,6 +119,281 @@ where }) } +#[cfg(unix)] +fn sync_directory(path: &Path) -> Result<()> { + fs::File::open(path)?.sync_all()?; + Ok(()) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> Result<()> { + Ok(()) +} + +fn copy_checkpoint_tree(source: &Path, destination: &Path) -> Result<()> { + fs::create_dir(destination)?; + for entry in fs::read_dir(source)? { + let entry = entry?; + let file_type = entry.file_type()?; + let target = destination.join(entry.file_name()); + if file_type.is_dir() { + copy_checkpoint_tree(&entry.path(), &target)?; + } else if file_type.is_file() { + fs::copy(entry.path(), &target)?; + fs::File::open(&target)?.sync_all()?; + } else { + anyhow::bail!( + "checkpoint contains unsupported non-file entry {}", + entry.path().display() + ); + } + } + sync_directory(destination)?; + Ok(()) +} + +fn validate_rocksdb_checkpoint(path: &Path) -> Result<()> { + let mut options = Options::default(); + options.set_paranoid_checks(true); + let mut column_families = DB::list_cf(&options, path)?; + column_families.retain(|name| name != "default"); + let db = DB::open_cf_for_read_only(&options, path, column_families, false)?; + drop(db); + Ok(()) +} + +fn read_manifest_file(path: &Path) -> Result<(ClusterCheckpointManifest, Vec)> { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + anyhow::bail!("checkpoint marker {} must be a real file", path.display()); + } + let bytes = fs::read(path)?; + let manifest: ClusterCheckpointManifest = bincode::deserialize(&bytes)?; + manifest.validate()?; + Ok((manifest, bytes)) +} + +fn write_marker_once(path: &Path, bytes: &[u8]) -> Result<()> { + if path.exists() { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + anyhow::bail!("checkpoint marker {} must be a real file", path.display()); + } + if fs::read(path)? == bytes { + return Ok(()); + } + anyhow::bail!("checkpoint marker {} does not match", path.display()); + } + + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("checkpoint marker name must be UTF-8"))?; + let staging = path.with_file_name(format!(".{file_name}.tmp")); + if staging.exists() { + fs::remove_file(&staging)?; + } + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&staging)?; + file.write_all(bytes)?; + file.sync_all()?; + fs::rename(&staging, path)?; + sync_directory( + path.parent() + .ok_or_else(|| anyhow!("checkpoint marker has no parent directory"))?, + )?; + Ok(()) +} + +pub(crate) fn prepare_cluster_checkpoint( + checkpoint: &Path, + generation: &str, + shard_id: u32, + shard_count: u32, +) -> Result<()> { + if !checkpoint.is_dir() || !checkpoint.join("CURRENT").is_file() { + anyhow::bail!( + "prepared checkpoint {} is not a RocksDB checkpoint", + checkpoint.display() + ); + } + let expected = ClusterCheckpointManifest::new(generation, shard_id, shard_count)?; + let bytes = bincode::serialize(&expected)?; + write_marker_once(&checkpoint.join(CHECKPOINT_MANIFEST_FILE), &bytes) +} + +pub(crate) fn validate_prepared_cluster_checkpoint( + checkpoint: &Path, + generation: &str, + shard_id: u32, + shard_count: u32, +) -> Result<()> { + if !checkpoint.is_dir() || !checkpoint.join("CURRENT").is_file() { + anyhow::bail!( + "prepared checkpoint {} is not a RocksDB checkpoint", + checkpoint.display() + ); + } + let expected = ClusterCheckpointManifest::new(generation, shard_id, shard_count)?; + let (prepared, _) = read_manifest_file(&checkpoint.join(CHECKPOINT_MANIFEST_FILE))?; + if prepared != expected { + anyhow::bail!( + "prepared checkpoint {} does not match generation {} shard {}/{}", + checkpoint.display(), + generation, + shard_id, + shard_count + ); + } + Ok(()) +} + +pub(crate) fn commit_cluster_checkpoint( + checkpoint: &Path, + generation: &str, + shard_id: u32, + shard_count: u32, +) -> Result<()> { + let expected = ClusterCheckpointManifest::new(generation, shard_id, shard_count)?; + let (prepared, bytes) = read_manifest_file(&checkpoint.join(CHECKPOINT_MANIFEST_FILE))?; + if prepared != expected { + anyhow::bail!( + "prepared checkpoint {} does not match generation {} shard {}/{}", + checkpoint.display(), + generation, + shard_id, + shard_count + ); + } + write_marker_once(&checkpoint.join(CHECKPOINT_COMMITTED_FILE), &bytes) +} + +/// Read and validate a committed cluster checkpoint manifest. +pub fn read_cluster_checkpoint_manifest(checkpoint: &Path) -> Result { + let metadata = fs::symlink_metadata(checkpoint)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + anyhow::bail!("checkpoint source must be a real directory"); + } + let checkpoint = checkpoint.canonicalize()?; + if !checkpoint.join("CURRENT").is_file() { + anyhow::bail!( + "checkpoint source {} is not a RocksDB checkpoint", + checkpoint.display() + ); + } + let (manifest, prepared_bytes) = + read_manifest_file(&checkpoint.join(CHECKPOINT_MANIFEST_FILE))?; + let (_committed, committed_bytes) = + read_manifest_file(&checkpoint.join(CHECKPOINT_COMMITTED_FILE))?; + if committed_bytes != prepared_bytes { + anyhow::bail!( + "checkpoint {} has mismatched prepare and commit markers", + checkpoint.display() + ); + } + Ok(manifest) +} + +/// Return the committed checkpoint provenance copied into a restored RocksDB +/// data directory. A lone or corrupt marker is a recovery integrity failure, +/// not an unrestored directory. +pub(crate) fn read_restored_checkpoint_manifest( + data_dir: &Path, +) -> Result> { + let prepared = data_dir.join(CHECKPOINT_MANIFEST_FILE); + let committed = data_dir.join(CHECKPOINT_COMMITTED_FILE); + let marker_exists = |path: &Path| match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + }; + match (marker_exists(&prepared)?, marker_exists(&committed)?) { + (false, false) => Ok(None), + (true, true) => read_cluster_checkpoint_manifest(data_dir).map(Some), + _ => anyhow::bail!( + "restored data directory {} contains incomplete checkpoint provenance", + data_dir.display() + ), + } +} + +/// Restore a RocksDB checkpoint into an empty replacement data directory. +/// +/// Copying occurs in a sibling staging directory and is made visible with one +/// rename only after every file and directory has been synchronized. Only +/// checkpoints committed by the router's cluster-wide two-phase protocol are +/// accepted. +pub fn restore_checkpoint(source: &Path, destination: &Path) -> Result<()> { + restore_checkpoint_for_shard(source, destination, None) +} + +/// Restore a committed cluster checkpoint and verify its shard identity. +pub fn restore_checkpoint_for_shard( + source: &Path, + destination: &Path, + expected_shard_id: Option, +) -> Result<()> { + let manifest = read_cluster_checkpoint_manifest(source)?; + if let Some(expected_shard_id) = + expected_shard_id.filter(|expected| *expected != manifest.shard_id) + { + anyhow::bail!( + "checkpoint belongs to shard {}, not requested shard {}", + manifest.shard_id, + expected_shard_id + ); + } + + let source = source.canonicalize()?; + validate_rocksdb_checkpoint(&source)?; + + if destination.exists() { + let metadata = fs::symlink_metadata(destination)?; + if !metadata.file_type().is_dir() || metadata.file_type().is_symlink() { + anyhow::bail!("restore destination must be an empty directory"); + } + if fs::read_dir(destination)?.next().transpose()?.is_some() { + anyhow::bail!( + "refusing to restore over nonempty destination {}", + destination.display() + ); + } + } + + let name = destination + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("restore destination must have a UTF-8 directory name"))?; + let parent = destination + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + let parent = parent.canonicalize()?; + if parent.starts_with(&source) { + anyhow::bail!("restore destination must not be inside the checkpoint"); + } + let destination = parent.join(name); + let staging = parent.join(format!(".{name}.restore.tmp")); + if staging.exists() { + anyhow::bail!( + "restore staging directory {} already exists; inspect and remove it before retrying", + staging.display() + ); + } + + copy_checkpoint_tree(&source, &staging)?; + validate_rocksdb_checkpoint(&staging)?; + if destination.exists() { + fs::remove_dir(&destination)?; + } + fs::rename(&staging, &destination)?; + sync_directory(&parent)?; + Ok(()) +} + // DSU persistence column families const CF_DSU_PARENT: &str = "dsu_parent"; const CF_DSU_RANK: &str = "dsu_rank"; @@ -51,6 +409,28 @@ const CF_LINKER_CLUSTER_IDS: &str = "linker_cluster_ids"; const CF_LINKER_GLOBAL_IDS: &str = "linker_global_ids"; const CF_LINKER_METADATA: &str = "linker_metadata"; +const RESET_DATA_CFS: &[&str] = &[ + CF_RECORDS, + CF_INTERNER, + CF_INDEX_ATTR_VALUE, + CF_INDEX_ENTITY_TYPE, + CF_INDEX_PERSPECTIVE, + CF_INDEX_TEMPORAL_BUCKET, + CF_INDEX_IDENTITY, + CF_CONFLICT_SUMMARIES, + CF_CLUSTER_ASSIGNMENTS, + CF_SOURCE_RESERVATIONS, + CF_DSU_PARENT, + CF_DSU_RANK, + CF_DSU_GUARDS, + CF_DSU_METADATA, + CF_INDEX_IDENTITY_KEYS, + CF_INDEX_KEY_STATS, + CF_LINKER_CLUSTER_IDS, + CF_LINKER_GLOBAL_IDS, + CF_LINKER_METADATA, +]; + const KEY_NEXT_RECORD_ID: &[u8] = b"next_record_id"; const KEY_INTERNER: &[u8] = b"interner"; const KEY_ONTOLOGY_CONFIG: &[u8] = b"ontology_config"; @@ -61,6 +441,14 @@ const KEY_NEXT_VALUE_ID: &[u8] = b"next_value_id"; const KEY_RECORD_COUNT: &[u8] = b"record_count"; const KEY_CLUSTER_COUNT: &[u8] = b"cluster_count"; const KEY_CONFLICT_SUMMARY_COUNT: &[u8] = b"conflict_summary_count"; +const KEY_CROSS_SHARD_CONFLICTS: &[u8] = b"cross_shard_conflicts"; +const KEY_SOURCE_RESERVATION_BACKFILL: &[u8] = b"source_reservation_backfill"; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +struct SourceReservationValue { + payload_digest: [u8; 32], + target_shard_id: u32, +} // DSU metadata keys const KEY_DSU_NEXT_CLUSTER_ID: &[u8] = b"dsu_next_cluster_id"; @@ -114,14 +502,17 @@ pub struct PersistentStore { db: Arc, cache: Mutex>, staged_records: Mutex>, + staged_identities: Mutex>, persisted_attr_id: u32, persisted_value_id: u32, record_count: u64, cluster_count: u64, conflict_summary_count: u64, + read_fault: AtomicBool, } -/// Create WriteOptions for fast async writes (no WAL sync) +/// Create WriteOptions for grouped writes. A high-level ingest calls +/// `RecordStore::sync` once after all related batches have been written. fn fast_write_opts() -> WriteOptions { let mut opts = WriteOptions::default(); opts.set_sync(false); @@ -166,11 +557,13 @@ impl PersistentStore { std::num::NonZeroUsize::new(DEFAULT_CACHE_CAPACITY).expect("cache capacity"), )), staged_records: Mutex::new(Vec::new()), + staged_identities: Mutex::new(HashMap::new()), persisted_attr_id, persisted_value_id, record_count, cluster_count, conflict_summary_count, + read_fault: AtomicBool::new(false), }; instance.rebuild_indexes_if_needed()?; if should_persist_count { @@ -187,7 +580,11 @@ impl PersistentStore { &mut self.inner } - pub fn persist_interner(&mut self, batch: &mut WriteBatch) -> Result<()> { + fn mark_read_fault(&self) { + self.read_fault.store(true, Ordering::Release); + } + + fn append_interner_to_batch(&self, batch: &mut WriteBatch) -> Result<(u32, u32)> { let interner_cf = self .db .cf_handle(CF_INTERNER) @@ -216,8 +613,6 @@ impl PersistentStore { } } - self.persisted_attr_id = next_attr; - self.persisted_value_id = next_value; batch.put_cf( self.db .cf_handle(CF_METADATA) @@ -232,7 +627,12 @@ impl PersistentStore { KEY_NEXT_VALUE_ID, bincode::serialize(&next_value)?, ); - Ok(()) + Ok((next_attr, next_value)) + } + + fn commit_interner_watermark(&mut self, watermark: (u32, u32)) { + self.persisted_attr_id = watermark.0; + self.persisted_value_id = watermark.1; } pub fn persist_metadata(&self, batch: &mut WriteBatch) -> Result<()> { @@ -255,20 +655,7 @@ impl PersistentStore { Ok(()) } - pub fn persist_record(&self, record: &Record) -> Result<()> { - let records_cf = self - .db - .cf_handle(CF_RECORDS) - .ok_or_else(|| anyhow!("missing records column family"))?; - let key = record.id.0.to_be_bytes(); - with_record_bytes(record, |bytes| { - self.db.put_cf(records_cf, key, bytes)?; - Ok(()) - })?; - Ok(()) - } - - pub fn index_record(&self, record: &Record) -> Result<()> { + fn index_record(&self, record: &Record) -> Result<()> { let mut batch = WriteBatch::default(); self.index_record_with_batch(record, &mut batch)?; self.db.write(batch)?; @@ -293,36 +680,61 @@ impl PersistentStore { } pub fn reset_data(&mut self) -> Result<()> { - clear_cf(&self.db, CF_RECORDS)?; - clear_cf(&self.db, CF_INTERNER)?; - clear_cf(&self.db, CF_INDEX_ATTR_VALUE)?; - clear_cf(&self.db, CF_INDEX_ENTITY_TYPE)?; - clear_cf(&self.db, CF_INDEX_PERSPECTIVE)?; - clear_cf(&self.db, CF_INDEX_TEMPORAL_BUCKET)?; - clear_cf(&self.db, CF_INDEX_IDENTITY)?; - clear_cf(&self.db, CF_CONFLICT_SUMMARIES)?; - clear_cf(&self.db, CF_CLUSTER_ASSIGNMENTS)?; - remove_metadata_key(&self.db, KEY_NEXT_RECORD_ID)?; - remove_metadata_key(&self.db, KEY_INDEX_VERSION)?; + let mut batch = WriteBatch::default(); + for &cf_name in RESET_DATA_CFS { + clear_cf_in_batch(&self.db, cf_name, &mut batch)?; + } + let metadata_cf = self + .db + .cf_handle(CF_METADATA) + .ok_or_else(|| anyhow!("missing metadata column family"))?; + batch.delete_cf(metadata_cf, KEY_INDEX_VERSION); + batch.delete_cf(metadata_cf, KEY_CROSS_SHARD_CONFLICTS); + batch.delete_cf(metadata_cf, KEY_SOURCE_RESERVATION_BACKFILL); + batch.put_cf(metadata_cf, KEY_NEXT_RECORD_ID, bincode::serialize(&0u32)?); + batch.put_cf(metadata_cf, KEY_RECORD_COUNT, bincode::serialize(&0u64)?); + batch.put_cf(metadata_cf, KEY_CLUSTER_COUNT, bincode::serialize(&0u64)?); + batch.put_cf( + metadata_cf, + KEY_CONFLICT_SUMMARY_COUNT, + bincode::serialize(&0u64)?, + ); + batch.put_cf(metadata_cf, KEY_NEXT_ATTR_ID, bincode::serialize(&0u32)?); + batch.put_cf(metadata_cf, KEY_NEXT_VALUE_ID, bincode::serialize(&0u32)?); + self.db.write(batch)?; + self.db.flush_wal(true)?; + + // Only publish the empty live view after the durable reset commits. self.inner = Store::new(); self.persisted_attr_id = 0; self.persisted_value_id = 0; self.record_count = 0; self.cluster_count = 0; self.conflict_summary_count = 0; - let mut batch = WriteBatch::default(); - self.persist_interner(&mut batch)?; - self.persist_metadata(&mut batch)?; - self.db.write(batch)?; + self.read_fault.store(false, Ordering::Release); + self.staged_records + .get_mut() + .map_err(|_| anyhow!("staged records lock poisoned"))? + .clear(); + self.staged_identities + .get_mut() + .map_err(|_| anyhow!("staged identities lock poisoned"))? + .clear(); + self.cache + .get_mut() + .map_err(|_| anyhow!("record cache lock poisoned"))? + .clear(); Ok(()) } pub fn flush(&self) -> Result<()> { + self.db.flush_wal(true)?; self.db.flush()?; Ok(()) } pub fn checkpoint(&self, path: impl AsRef) -> Result<()> { + self.db.flush_wal(true)?; let checkpoint = Checkpoint::new(&self.db)?; checkpoint.create_checkpoint(path)?; Ok(()) @@ -330,41 +742,119 @@ impl PersistentStore { pub fn persist_state(&mut self) -> Result<()> { let mut batch = WriteBatch::default(); - self.persist_interner(&mut batch)?; + let watermark = self.append_interner_to_batch(&mut batch)?; self.persist_metadata(&mut batch)?; self.db.write(batch)?; + self.db.flush_wal(true)?; + self.commit_interner_watermark(watermark); Ok(()) } /// Stage a record for later batch write. Returns (record_id, inserted). /// The record is added to cache immediately so it's readable, but not yet persisted to DB. pub fn stage_record_if_absent(&mut self, mut record: Record) -> Result<(RecordId, bool)> { + ::ensure_healthy(self)?; if let Some(existing) = self.get_record_id_by_identity(&record.identity) { + self.ensure_idempotent_record(existing, &record)?; + return Ok((existing, false)); + } + ::ensure_healthy(self)?; + + if let Some(existing) = self + .staged_identities + .lock() + .map_err(|_| anyhow!("staged identities lock poisoned"))? + .get(&record.identity) + .copied() + { + self.ensure_idempotent_record(existing, &record)?; return Ok((existing, false)); } let record_id = self.inner.prepare_record(&mut record)?; + let identity = record.identity.clone(); // Add to cache immediately so it's readable - if let Ok(mut cache) = self.cache.lock() { - cache.put(record_id, record.clone()); - } + self.cache + .lock() + .map_err(|_| anyhow!("record cache lock poisoned"))? + .put(record_id, record.clone()); // Stage for later batch write - if let Ok(mut staged) = self.staged_records.lock() { - staged.push(record); - } + self.staged_records + .lock() + .map_err(|_| anyhow!("staged records lock poisoned"))? + .push(record); + self.staged_identities + .lock() + .map_err(|_| anyhow!("staged identities lock poisoned"))? + .insert(identity, record_id); + + Ok((record_id, true)) + } + pub fn stage_record_with_explicit_id_if_absent( + &mut self, + mut record: Record, + ) -> Result<(RecordId, bool)> { + ::ensure_healthy(self)?; + if let Some(existing) = self.get_record_id_by_identity(&record.identity) { + self.ensure_idempotent_record(existing, &record)?; + return Ok((existing, false)); + } + ::ensure_healthy(self)?; + if let Some(existing) = self + .staged_identities + .lock() + .map_err(|_| anyhow!("staged identities lock poisoned"))? + .get(&record.identity) + .copied() + { + self.ensure_idempotent_record(existing, &record)?; + return Ok((existing, false)); + } + if self.get_record(record.id).is_some() { + anyhow::bail!("record ID {} already exists", record.id.0); + } + ::ensure_healthy(self)?; + + let record_id = self.inner.prepare_record_with_explicit_id(&mut record)?; + let identity = record.identity.clone(); + self.cache + .lock() + .map_err(|_| anyhow!("record cache lock poisoned"))? + .put(record_id, record.clone()); + self.staged_records + .lock() + .map_err(|_| anyhow!("staged records lock poisoned"))? + .push(record); + self.staged_identities + .lock() + .map_err(|_| anyhow!("staged identities lock poisoned"))? + .insert(identity, record_id); Ok((record_id, true)) } + fn ensure_idempotent_record(&self, existing_id: RecordId, incoming: &Record) -> Result<()> { + let existing = self + .get_record(existing_id) + .ok_or_else(|| anyhow!("identity index references a missing record"))?; + ::ensure_healthy(self)?; + if records_have_same_payload(&existing, incoming) { + Ok(()) + } else { + anyhow::bail!("source record identity already exists with a different payload") + } + } + /// Flush all staged records to the database in a single batch write. pub fn flush_staged_records(&mut self) -> Result { + ::ensure_healthy(self)?; let records = { let mut staged = self .staged_records .lock() - .map_err(|_| anyhow!("lock error"))?; + .map_err(|_| anyhow!("staged records lock poisoned"))?; std::mem::take(&mut *staged) }; @@ -373,28 +863,43 @@ impl PersistentStore { } let count = records.len(); - let records_cf = self - .db - .cf_handle(CF_RECORDS) - .ok_or_else(|| anyhow!("missing records column family"))?; - - let mut batch = WriteBatch::default(); - - for record in &records { - let key = record.id.0.to_be_bytes(); - with_record_bytes(record, |bytes| { - batch.put_cf(records_cf, key, bytes); - Ok(()) - })?; - self.index_record_with_batch(record, &mut batch)?; - } - let next_count = self.record_count.saturating_add(count as u64); - self.persist_interner(&mut batch)?; - self.persist_metadata_with_count(&mut batch, next_count)?; - - self.db.write_opt(batch, &fast_write_opts())?; + let write_result = (|| -> Result<(u32, u32)> { + let mut batch = WriteBatch::default(); + let watermark = self.append_interner_to_batch(&mut batch)?; + self.persist_metadata_with_count(&mut batch, next_count)?; + let records_cf = self + .db + .cf_handle(CF_RECORDS) + .ok_or_else(|| anyhow!("missing records column family"))?; + for record in &records { + let key = record.id.0.to_be_bytes(); + with_record_bytes(record, |bytes| { + batch.put_cf(records_cf, key, bytes); + Ok(()) + })?; + self.index_record_with_batch(record, &mut batch)?; + } + self.db.write_opt(batch, &fast_write_opts())?; + Ok(watermark) + })(); + + let watermark = match write_result { + Ok(watermark) => watermark, + Err(error) => { + *self + .staged_records + .lock() + .map_err(|_| anyhow!("staged records lock poisoned"))? = records; + return Err(error); + } + }; + self.commit_interner_watermark(watermark); self.record_count = next_count; + self.staged_identities + .lock() + .map_err(|_| anyhow!("staged identities lock poisoned"))? + .clear(); Ok(count) } @@ -407,10 +912,24 @@ impl PersistentStore { } fn lookup_interner_id(&self, prefix: u8, value: &str) -> Option { - let interner_cf = self.db.cf_handle(CF_INTERNER)?; + let interner_cf = match self.db.cf_handle(CF_INTERNER) { + Some(cf) => cf, + None => { + self.mark_read_fault(); + return None; + } + }; let key = encode_interner_lookup_key(prefix, value); - let bytes = self.db.get_cf(interner_cf, key).ok()??; + let bytes = match self.db.get_cf(interner_cf, key) { + Ok(Some(bytes)) => bytes, + Ok(None) => return None, + Err(_) => { + self.mark_read_fault(); + return None; + } + }; if bytes.len() != 4 { + self.mark_read_fault(); return None; } let mut buf = [0u8; 4]; @@ -419,15 +938,48 @@ impl PersistentStore { } fn lookup_interner_value(&self, prefix: u8, id: u32) -> Option { - let interner_cf = self.db.cf_handle(CF_INTERNER)?; + let interner_cf = match self.db.cf_handle(CF_INTERNER) { + Some(cf) => cf, + None => { + self.mark_read_fault(); + return None; + } + }; let key = encode_interner_key(prefix, id); - let bytes = self.db.get_cf(interner_cf, key).ok()??; - String::from_utf8(bytes).ok() + let bytes = match self.db.get_cf(interner_cf, key) { + Ok(Some(bytes)) => bytes, + Ok(None) => return None, + Err(_) => { + self.mark_read_fault(); + return None; + } + }; + match String::from_utf8(bytes) { + Ok(value) => Some(value), + Err(_) => { + self.mark_read_fault(); + None + } + } } } impl RecordStore for PersistentStore { + fn ensure_healthy(&self) -> Result<()> { + if self.read_fault.load(Ordering::Acquire) { + anyhow::bail!( + "persistent store observed an I/O or decode failure; restart before continuing" + ); + } + Ok(()) + } + + fn reset_data(&mut self) -> Result<()> { + PersistentStore::reset_data(self) + } + fn add_record(&mut self, record: Record) -> Result { + self.ensure_healthy()?; let mut record = record; let record_id = self.inner.prepare_record(&mut record)?; let next_count = self.record_count.saturating_add(1); @@ -442,9 +994,10 @@ impl RecordStore for PersistentStore { Ok(()) })?; self.index_record_with_batch(&record, &mut batch)?; - self.persist_interner(&mut batch)?; + let watermark = self.append_interner_to_batch(&mut batch)?; self.persist_metadata_with_count(&mut batch, next_count)?; self.db.write(batch)?; + self.commit_interner_watermark(watermark); self.record_count = next_count; if let Ok(mut cache) = self.cache.lock() { cache.put(record_id, record); @@ -453,6 +1006,7 @@ impl RecordStore for PersistentStore { } fn add_records(&mut self, records: Vec) -> Result<()> { + self.ensure_healthy()?; if records.is_empty() { return Ok(()); } @@ -485,11 +1039,12 @@ impl RecordStore for PersistentStore { let next_count = self .record_count .saturating_add(prepared_records.len() as u64); - self.persist_interner(&mut batch)?; + let watermark = self.append_interner_to_batch(&mut batch)?; self.persist_metadata_with_count(&mut batch, next_count)?; // Single write for all records self.db.write(batch)?; + self.commit_interner_watermark(watermark); self.record_count = next_count; // Update cache @@ -503,14 +1058,18 @@ impl RecordStore for PersistentStore { } fn add_record_if_absent(&mut self, record: Record) -> Result<(RecordId, bool)> { + self.ensure_healthy()?; if let Some(existing) = self.get_record_id_by_identity(&record.identity) { + self.ensure_idempotent_record(existing, &record)?; return Ok((existing, false)); } + self.ensure_healthy()?; let record_id = self.add_record(record)?; Ok((record_id, true)) } fn add_records_if_absent(&mut self, records: Vec) -> Result> { + self.ensure_healthy()?; if records.is_empty() { return Ok(Vec::new()); } @@ -519,16 +1078,29 @@ impl RecordStore for PersistentStore { let mut results = Vec::with_capacity(records.len()); let mut new_records = Vec::new(); let mut new_record_indices = Vec::new(); + let mut pending_identities = HashMap::new(); + let mut pending_duplicates = Vec::new(); for (idx, record) in records.into_iter().enumerate() { if let Some(existing) = self.get_record_id_by_identity(&record.identity) { + self.ensure_idempotent_record(existing, &record)?; results.push((existing, false)); + } else if let Some(&new_record_index) = pending_identities.get(&record.identity) { + if !records_have_same_payload(&new_records[new_record_index], &record) { + anyhow::bail!( + "source record identity appears more than once with different payloads" + ); + } + results.push((RecordId(0), false)); + pending_duplicates.push((idx, new_record_index)); } else { results.push((RecordId(0), true)); // Placeholder, will be filled in new_record_indices.push(idx); + pending_identities.insert(record.identity.clone(), new_records.len()); new_records.push(record); } } + self.ensure_healthy()?; if new_records.is_empty() { return Ok(results); @@ -564,17 +1136,21 @@ impl RecordStore for PersistentStore { let next_count = self .record_count .saturating_add(prepared_records.len() as u64); - self.persist_interner(&mut batch)?; + let watermark = self.append_interner_to_batch(&mut batch)?; self.persist_metadata_with_count(&mut batch, next_count)?; // Single write for all new records self.db.write(batch)?; + self.commit_interner_watermark(watermark); self.record_count = next_count; // Update results with actual record IDs for (i, idx) in new_record_indices.into_iter().enumerate() { results[idx].0 = assigned_ids[i]; } + for (idx, new_record_index) in pending_duplicates { + results[idx].0 = assigned_ids[new_record_index]; + } // Update cache if let Ok(mut cache) = self.cache.lock() { @@ -590,10 +1166,139 @@ impl RecordStore for PersistentStore { PersistentStore::stage_record_if_absent(self, record) } + fn stage_record_with_explicit_id_if_absent( + &mut self, + record: Record, + ) -> Result<(RecordId, bool)> { + PersistentStore::stage_record_with_explicit_id_if_absent(self, record) + } + fn flush_staged_records(&mut self) -> Result { PersistentStore::flush_staged_records(self) } + fn reserve_source_records( + &mut self, + reservations: &[SourceRecordReservation], + ) -> Result> { + self.ensure_healthy()?; + let cf = self + .db + .cf_handle(CF_SOURCE_RESERVATIONS) + .ok_or_else(|| anyhow!("missing source reservations column family"))?; + let mut pending = HashMap::with_capacity(reservations.len()); + let mut targets = Vec::with_capacity(reservations.len()); + + for reservation in reservations { + let key = encode_identity_index(&reservation.identity)?; + let stored = if let Some(value) = pending.get(&key).copied() { + Some(value) + } else { + let bytes = match self.db.get_cf(cf, &key) { + Ok(bytes) => bytes, + Err(err) => { + self.mark_read_fault(); + return Err(err.into()); + } + }; + match bytes { + Some(bytes) => match bincode::deserialize::(&bytes) { + Ok(value) => Some(value), + Err(err) => { + self.mark_read_fault(); + return Err(err.into()); + } + }, + None => None, + } + }; + + if let Some(stored) = stored { + if stored.payload_digest != reservation.payload_digest { + return Err(SourceReservationError::PayloadConflict.into()); + } + if stored.target_shard_id != reservation.target_shard_id { + return Err(SourceReservationError::TargetConflict { + existing_shard: stored.target_shard_id, + requested_shard: reservation.target_shard_id, + } + .into()); + } + targets.push(stored.target_shard_id); + } else { + let value = SourceReservationValue { + payload_digest: reservation.payload_digest, + target_shard_id: reservation.target_shard_id, + }; + pending.insert(key, value); + targets.push(value.target_shard_id); + } + } + + if !pending.is_empty() { + let mut batch = WriteBatch::default(); + for (key, value) in pending { + batch.put_cf(cf, key, bincode::serialize(&value)?); + } + let mut options = WriteOptions::default(); + options.set_sync(true); + self.db.write_opt(batch, &options)?; + } + Ok(targets) + } + + fn source_reservation_backfill(&self) -> Result> { + self.ensure_healthy()?; + let cf = self + .db + .cf_handle(CF_METADATA) + .ok_or_else(|| anyhow!("missing metadata column family"))?; + let bytes = match self.db.get_cf(cf, KEY_SOURCE_RESERVATION_BACKFILL) { + Ok(bytes) => bytes, + Err(err) => { + self.mark_read_fault(); + return Err(err.into()); + } + }; + match bytes { + Some(bytes) => match bincode::deserialize(&bytes) { + Ok(value) => Ok(Some(value)), + Err(err) => { + self.mark_read_fault(); + Err(err.into()) + } + }, + None => Ok(None), + } + } + + fn mark_source_reservation_backfill( + &mut self, + protocol_version: u32, + shard_count: u32, + ) -> Result<()> { + self.ensure_healthy()?; + let cf = self + .db + .cf_handle(CF_METADATA) + .ok_or_else(|| anyhow!("missing metadata column family"))?; + let mut options = WriteOptions::default(); + options.set_sync(true); + self.db.put_cf_opt( + cf, + KEY_SOURCE_RESERVATION_BACKFILL, + bincode::serialize(&(protocol_version, shard_count))?, + &options, + )?; + Ok(()) + } + + fn sync(&self) -> Result<()> { + self.ensure_healthy()?; + self.db.flush_wal(true)?; + Ok(()) + } + fn get_record(&self, id: RecordId) -> Option { if let Ok(mut cache) = self.cache.lock() { if let Some(record) = cache.get(&id) { @@ -601,10 +1306,29 @@ impl RecordStore for PersistentStore { } } - let records_cf = self.db.cf_handle(CF_RECORDS)?; + let records_cf = match self.db.cf_handle(CF_RECORDS) { + Some(cf) => cf, + None => { + self.mark_read_fault(); + return None; + } + }; let key = id.0.to_be_bytes(); - let bytes = self.db.get_cf(records_cf, key).ok()??; - let record: Record = bincode::deserialize(&bytes).ok()?; + let bytes = match self.db.get_cf(records_cf, key) { + Ok(Some(bytes)) => bytes, + Ok(None) => return None, + Err(_) => { + self.mark_read_fault(); + return None; + } + }; + let record: Record = match bincode::deserialize(&bytes) { + Ok(record) => record, + Err(_) => { + self.mark_read_fault(); + return None; + } + }; if let Ok(mut cache) = self.cache.lock() { cache.put(id, record.clone()); } @@ -614,23 +1338,56 @@ impl RecordStore for PersistentStore { fn get_all_records(&self) -> Vec { let records_cf = match self.db.cf_handle(CF_RECORDS) { Some(cf) => cf, - None => return Vec::new(), + None => { + self.mark_read_fault(); + return Vec::new(); + } }; - self.db - .iterator_cf(records_cf, IteratorMode::Start) - .filter_map(|entry| { - entry - .ok() - .and_then(|(_, value)| bincode::deserialize(&value).ok()) - }) - .collect() + let mut records = Vec::new(); + for entry in self.db.iterator_cf(records_cf, IteratorMode::Start) { + let (_, value) = match entry { + Ok(entry) => entry, + Err(_) => { + self.mark_read_fault(); + break; + } + }; + match bincode::deserialize(&value) { + Ok(record) => records.push(record), + Err(_) => { + self.mark_read_fault(); + break; + } + } + } + records } fn get_record_id_by_identity(&self, identity: &RecordIdentity) -> Option { - let identity_cf = self.db.cf_handle(CF_INDEX_IDENTITY)?; - let key = encode_identity_index(identity).ok()?; - let value = self.db.get_cf(identity_cf, key).ok()??; + let identity_cf = match self.db.cf_handle(CF_INDEX_IDENTITY) { + Some(cf) => cf, + None => { + self.mark_read_fault(); + return None; + } + }; + let key = match encode_identity_index(identity) { + Ok(key) => key, + Err(_) => { + self.mark_read_fault(); + return None; + } + }; + let value = match self.db.get_cf(identity_cf, key) { + Ok(Some(value)) => value, + Ok(None) => return None, + Err(_) => { + self.mark_read_fault(); + return None; + } + }; if value.len() != 4 { + self.mark_read_fault(); return None; } let mut bytes = [0u8; 4]; @@ -641,15 +1398,25 @@ impl RecordStore for PersistentStore { fn for_each_record(&self, f: &mut dyn FnMut(Record)) { let records_cf = match self.db.cf_handle(CF_RECORDS) { Some(cf) => cf, - None => return, + None => { + self.mark_read_fault(); + return; + } }; - for (_key, value) in self - .db - .iterator_cf(records_cf, IteratorMode::Start) - .flatten() - { - if let Ok(record) = bincode::deserialize(&value) { - f(record); + for entry in self.db.iterator_cf(records_cf, IteratorMode::Start) { + let (_key, value) = match entry { + Ok(entry) => entry, + Err(_) => { + self.mark_read_fault(); + break; + } + }; + match bincode::deserialize(&value) { + Ok(record) => f(record), + Err(_) => { + self.mark_read_fault(); + break; + } } } } @@ -657,7 +1424,10 @@ impl RecordStore for PersistentStore { fn get_records_by_entity_type(&self, entity_type: &str) -> Vec { let cf = match self.db.cf_handle(CF_INDEX_ENTITY_TYPE) { Some(cf) => cf, - None => return Vec::new(), + None => { + self.mark_read_fault(); + return Vec::new(); + } }; let prefix = encode_string_prefix(entity_type); let iter = self @@ -668,7 +1438,10 @@ impl RecordStore for PersistentStore { for entry in iter { let (key, _) = match entry { Ok(pair) => pair, - Err(_) => break, + Err(_) => { + self.mark_read_fault(); + break; + } }; if !key.starts_with(&prefix) { break; @@ -677,8 +1450,14 @@ impl RecordStore for PersistentStore { if seen.insert(record_id) { if let Some(record) = self.get_record(RecordId(record_id)) { records.push(record); + } else { + self.mark_read_fault(); + break; } } + } else { + self.mark_read_fault(); + break; } } records @@ -687,7 +1466,10 @@ impl RecordStore for PersistentStore { fn get_records_by_perspective(&self, perspective: &str) -> Vec { let cf = match self.db.cf_handle(CF_INDEX_PERSPECTIVE) { Some(cf) => cf, - None => return Vec::new(), + None => { + self.mark_read_fault(); + return Vec::new(); + } }; let prefix = encode_string_prefix(perspective); let iter = self @@ -698,7 +1480,10 @@ impl RecordStore for PersistentStore { for entry in iter { let (key, _) = match entry { Ok(pair) => pair, - Err(_) => break, + Err(_) => { + self.mark_read_fault(); + break; + } }; if !key.starts_with(&prefix) { break; @@ -707,8 +1492,14 @@ impl RecordStore for PersistentStore { if seen.insert(record_id) { if let Some(record) = self.get_record(RecordId(record_id)) { records.push(record); + } else { + self.mark_read_fault(); + break; } } + } else { + self.mark_read_fault(); + break; } } records @@ -717,7 +1508,10 @@ impl RecordStore for PersistentStore { fn get_records_with_attribute(&self, attr: crate::model::AttrId) -> Vec { let cf = match self.db.cf_handle(CF_INDEX_ATTR_VALUE) { Some(cf) => cf, - None => return Vec::new(), + None => { + self.mark_read_fault(); + return Vec::new(); + } }; let prefix = encode_attr_prefix(attr.0); let iter = self @@ -728,7 +1522,10 @@ impl RecordStore for PersistentStore { for entry in iter { let (key, _) = match entry { Ok(pair) => pair, - Err(_) => break, + Err(_) => { + self.mark_read_fault(); + break; + } }; if !key.starts_with(&prefix) { break; @@ -737,8 +1534,14 @@ impl RecordStore for PersistentStore { if seen.insert(record_id) { if let Some(record) = self.get_record(RecordId(record_id)) { records.push(record); + } else { + self.mark_read_fault(); + break; } } + } else { + self.mark_read_fault(); + break; } } records @@ -747,7 +1550,10 @@ impl RecordStore for PersistentStore { fn get_records_in_interval(&self, interval: crate::temporal::Interval) -> Vec { let cf = match self.db.cf_handle(CF_INDEX_TEMPORAL_BUCKET) { Some(cf) => cf, - None => return Vec::new(), + None => { + self.mark_read_fault(); + return Vec::new(); + } }; let mut candidates = std::collections::HashSet::new(); for bucket in buckets_for_interval(interval.start, interval.end) { @@ -758,13 +1564,19 @@ impl RecordStore for PersistentStore { for entry in iter { let (key, _) = match entry { Ok(pair) => pair, - Err(_) => break, + Err(_) => { + self.mark_read_fault(); + break; + } }; if !key.starts_with(&prefix) { break; } if let Some(record_id) = decode_temporal_record_id(&key) { candidates.insert(record_id); + } else { + self.mark_read_fault(); + break; } } } @@ -776,6 +1588,9 @@ impl RecordStore for PersistentStore { }) { records.push(record); } + } else { + self.mark_read_fault(); + break; } } records @@ -789,7 +1604,10 @@ impl RecordStore for PersistentStore { ) -> Vec<(RecordId, crate::temporal::Interval)> { let cf = match self.db.cf_handle(CF_INDEX_ATTR_VALUE) { Some(cf) => cf, - None => return Vec::new(), + None => { + self.mark_read_fault(); + return Vec::new(); + } }; let prefix = encode_attr_value_prefix(attr.0, value.0); let iter = self @@ -799,7 +1617,10 @@ impl RecordStore for PersistentStore { for entry in iter { let (key, _) = match entry { Ok(pair) => pair, - Err(_) => break, + Err(_) => { + self.mark_read_fault(); + break; + } }; if !key.starts_with(&prefix) { break; @@ -808,6 +1629,9 @@ impl RecordStore for PersistentStore { if let Some(overlap) = crate::temporal::intersect(&record_interval, &interval) { matches.push((RecordId(record_id), overlap)); } + } else { + self.mark_read_fault(); + break; } } matches @@ -864,11 +1688,18 @@ impl RecordStore for PersistentStore { } fn set_cluster_count(&mut self, count: usize) -> Result<()> { + let previous = self.cluster_count; self.cluster_count = count as u64; - let mut batch = WriteBatch::default(); - self.persist_metadata(&mut batch)?; - self.db.write(batch)?; - Ok(()) + let result = (|| { + let mut batch = WriteBatch::default(); + self.persist_metadata(&mut batch)?; + self.db.write(batch)?; + Ok(()) + })(); + if result.is_err() { + self.cluster_count = previous; + } + result } fn cluster_count(&self) -> Option { @@ -886,10 +1717,17 @@ impl RecordStore for PersistentStore { let bytes = bincode::serialize(summaries)?; let mut batch = WriteBatch::default(); batch.put_cf(cf, b"latest", bytes); + let previous = self.conflict_summary_count; self.conflict_summary_count = summaries.len() as u64; - self.persist_metadata(&mut batch)?; - self.db.write(batch)?; - Ok(()) + let result = (|| { + self.persist_metadata(&mut batch)?; + self.db.write(batch)?; + Ok(()) + })(); + if result.is_err() { + self.conflict_summary_count = previous; + } + result } fn set_cluster_conflict_summaries( @@ -902,14 +1740,13 @@ impl RecordStore for PersistentStore { .cf_handle(CF_CONFLICT_SUMMARIES) .ok_or_else(|| anyhow!("missing conflict summaries column family"))?; let key = cluster_id.0.to_be_bytes(); - let existing = self.db.get_cf(cf, key).ok().flatten(); - let existing_len = existing - .as_ref() - .and_then(|bytes| { - bincode::deserialize::>(bytes).ok() - }) - .map(|summaries| summaries.len()) - .unwrap_or(0); + let existing = self.db.get_cf(cf, key)?; + let existing_len = match existing { + Some(bytes) => { + bincode::deserialize::>(&bytes)?.len() + } + None => 0, + }; let new_len = summaries.len(); let total = self .conflict_summary_count @@ -918,23 +1755,45 @@ impl RecordStore for PersistentStore { let bytes = bincode::serialize(summaries)?; let mut batch = WriteBatch::default(); batch.put_cf(cf, key, bytes); + let previous = self.conflict_summary_count; self.conflict_summary_count = total; - self.persist_metadata(&mut batch)?; - self.db.write(batch)?; - Ok(()) + let result = (|| { + self.persist_metadata(&mut batch)?; + self.db.write(batch)?; + Ok(()) + })(); + if result.is_err() { + self.conflict_summary_count = previous; + } + result } fn load_conflict_summaries(&self) -> Option> { - let cf = self.db.cf_handle(CF_CONFLICT_SUMMARIES)?; + let cf = match self.db.cf_handle(CF_CONFLICT_SUMMARIES) { + Some(cf) => cf, + None => { + self.mark_read_fault(); + return None; + } + }; let mut summaries = Vec::new(); - for (key, value) in self.db.iterator_cf(cf, IteratorMode::Start).flatten() { + for entry in self.db.iterator_cf(cf, IteratorMode::Start) { + let (key, value) = match entry { + Ok(entry) => entry, + Err(_) => { + self.mark_read_fault(); + break; + } + }; if key.as_ref() == b"latest" { continue; } - if let Ok(mut parsed) = - bincode::deserialize::>(&value) - { - summaries.append(&mut parsed); + match bincode::deserialize::>(&value) { + Ok(mut parsed) => summaries.append(&mut parsed), + Err(_) => { + self.mark_read_fault(); + break; + } } } if summaries.is_empty() { @@ -944,6 +1803,33 @@ impl RecordStore for PersistentStore { } } + fn set_cross_shard_conflicts( + &mut self, + conflicts: &[crate::sharding::CrossShardConflict], + ) -> Result<()> { + let cf = self + .db + .cf_handle(CF_METADATA) + .ok_or_else(|| anyhow!("missing metadata column family"))?; + self.db.put_cf( + cf, + KEY_CROSS_SHARD_CONFLICTS, + bincode::serialize(conflicts)?, + )?; + Ok(()) + } + + fn load_cross_shard_conflicts(&self) -> Result> { + let cf = self + .db + .cf_handle(CF_METADATA) + .ok_or_else(|| anyhow!("missing metadata column family"))?; + self.db + .get_cf(cf, KEY_CROSS_SHARD_CONFLICTS)? + .map(|bytes| bincode::deserialize(&bytes).map_err(Into::into)) + .unwrap_or_else(|| Ok(Vec::new())) + } + fn conflict_summary_count(&self) -> Option { Some(self.conflict_summary_count as usize) } @@ -990,7 +1876,10 @@ impl RecordStore for PersistentStore { ) -> Vec { let records_cf = match self.db.cf_handle(CF_RECORDS) { Some(cf) => cf, - None => return Vec::new(), + None => { + self.mark_read_fault(); + return Vec::new(); + } }; let mut records = Vec::new(); let start_key = start.0.to_be_bytes(); @@ -1001,18 +1890,30 @@ impl RecordStore for PersistentStore { for entry in iter { let (key, value) = match entry { Ok(pair) => pair, - Err(_) => break, + Err(_) => { + self.mark_read_fault(); + break; + } }; let record_id = match decode_record_id_key(&key) { Some(id) => id, - None => continue, + None => { + self.mark_read_fault(); + break; + } }; if record_id >= end.0 { break; } - if let Ok(record) = bincode::deserialize::(&value) { - records.push(record); - if max_results > 0 && records.len() >= max_results { + match bincode::deserialize::(&value) { + Ok(record) => { + records.push(record); + if max_results > 0 && records.len() >= max_results { + break; + } + } + Err(_) => { + self.mark_read_fault(); break; } } @@ -1021,20 +1922,44 @@ impl RecordStore for PersistentStore { } fn record_id_bounds(&self) -> Option<(RecordId, RecordId)> { - let records_cf = self.db.cf_handle(CF_RECORDS)?; + let records_cf = match self.db.cf_handle(CF_RECORDS) { + Some(cf) => cf, + None => { + self.mark_read_fault(); + return None; + } + }; let mut start_iter = self.db.iterator_cf(records_cf, IteratorMode::Start); - let min_id = start_iter - .next() - .and_then(|entry| entry.ok()) - .and_then(|(key, _)| decode_record_id_key(&key)) - .map(RecordId)?; + let min_id = match start_iter.next() { + Some(Ok((key, _))) => match decode_record_id_key(&key) { + Some(id) => RecordId(id), + None => { + self.mark_read_fault(); + return None; + } + }, + Some(Err(_)) => { + self.mark_read_fault(); + return None; + } + None => return None, + }; let mut end_iter = self.db.iterator_cf(records_cf, IteratorMode::End); - let max_id = end_iter - .next() - .and_then(|entry| entry.ok()) - .and_then(|(key, _)| decode_record_id_key(&key)) - .map(RecordId)?; + let max_id = match end_iter.next() { + Some(Ok((key, _))) => match decode_record_id_key(&key) { + Some(id) => RecordId(id), + None => { + self.mark_read_fault(); + return None; + } + }, + Some(Err(_)) => { + self.mark_read_fault(); + return None; + } + None => return None, + }; Some((min_id, max_id)) } @@ -1323,6 +2248,10 @@ fn open_db(path: impl AsRef) -> Result { CF_CLUSTER_ASSIGNMENTS, build_cf_options(&base, &index_block_opts, None, None), ), + ColumnFamilyDescriptor::new( + CF_SOURCE_RESERVATIONS, + build_cf_options(&base, &index_block_opts, None, None), + ), // DSU column families - 4-byte record_id keys, optimized for sequential access ColumnFamilyDescriptor::new( CF_DSU_PARENT, @@ -1740,30 +2669,45 @@ fn load_metadata(db: &DB, key: &[u8]) -> Result< } } -fn clear_cf(db: &DB, cf_name: &str) -> Result<()> { +fn clear_cf_in_batch(db: &DB, cf_name: &str, batch: &mut WriteBatch) -> Result<()> { let cf = db .cf_handle(cf_name) .ok_or_else(|| anyhow!("missing column family {cf_name}"))?; - let keys: Vec> = db - .iterator_cf(cf, IteratorMode::Start) - .map(|entry| entry.map(|(key, _)| key.to_vec())) - .collect::, _>>()?; - if keys.is_empty() { - return Ok(()); - } + // Keys are either fixed-width (under 64 bytes) or begin with a discriminator + // below 0xff, so this exclusive bound covers their complete encoded keyspace. + // A range tombstone avoids materializing millions of keys. + batch.delete_range_cf(cf, &[][..], &[u8::MAX; 64][..]); + Ok(()) +} + +fn clear_cf(db: &DB, cf_name: &str) -> Result<()> { let mut batch = WriteBatch::default(); - for key in keys { - batch.delete_cf(cf, key); - } + clear_cf_in_batch(db, cf_name, &mut batch)?; db.write(batch)?; Ok(()) } -fn remove_metadata_key(db: &DB, key: &[u8]) -> Result<()> { - let metadata_cf = db - .cf_handle(CF_METADATA) - .ok_or_else(|| anyhow!("missing metadata column family"))?; - db.delete_cf(metadata_cf, key)?; +/// Clear linker structures that can be reconstructed from durable records. +/// +/// A crash can leave an older, internally consistent DSU snapshot alongside newer +/// acknowledged records. Reusing that mixed-generation state during a record scan +/// skips merges that are required to rebuild summaries and boundary metadata. Start +/// recovery from an empty derived state instead; records and their persisted +/// assignments remain untouched. +pub(crate) fn clear_rebuildable_linker_state(db: &DB) -> Result<()> { + let mut batch = WriteBatch::default(); + for cf_name in [ + CF_DSU_PARENT, + CF_DSU_RANK, + CF_DSU_GUARDS, + CF_DSU_METADATA, + CF_INDEX_IDENTITY_KEYS, + CF_INDEX_KEY_STATS, + ] { + clear_cf_in_batch(db, cf_name, &mut batch)?; + } + db.write(batch)?; + db.flush_wal(true)?; Ok(()) } @@ -2129,6 +3073,24 @@ pub mod linker_encoding { /// The metadata key for next_cluster_id pub const KEY_NEXT_CLUSTER_ID: &[u8] = b"linker_next_cluster_id"; + + /// Prefix for durable cross-shard merge mappings. + pub const CROSS_SHARD_MERGE_PREFIX: &[u8] = b"cross_shard_merge/"; + + pub fn encode_cross_shard_merge_key(secondary: GlobalClusterId) -> Vec { + let mut key = Vec::with_capacity(CROSS_SHARD_MERGE_PREFIX.len() + 8); + key.extend_from_slice(CROSS_SHARD_MERGE_PREFIX); + key.extend_from_slice(&encode_global_cluster_id(secondary)); + key + } + + pub fn decode_cross_shard_merge_key(bytes: &[u8]) -> Option { + let encoded = bytes.strip_prefix(CROSS_SHARD_MERGE_PREFIX)?; + if encoded.len() != 8 { + return None; + } + decode_global_cluster_id(encoded) + } } /// Linker state persistence operations @@ -2198,6 +3160,47 @@ impl<'a> LinkerStatePersistence<'a> { Ok(()) } + /// Persist a cross-shard redirect from a secondary cluster to its primary. + pub fn save_cross_shard_merge( + &self, + secondary: GlobalClusterId, + primary: GlobalClusterId, + ) -> Result<()> { + let cf = self + .db + .cf_handle(linker_cf::METADATA) + .ok_or_else(|| anyhow::anyhow!("Column family {} not found", linker_cf::METADATA))?; + self.db.put_cf( + &cf, + linker_encoding::encode_cross_shard_merge_key(secondary), + linker_encoding::encode_global_cluster_id(primary), + )?; + Ok(()) + } + + /// Load all durable cross-shard redirects. + pub fn load_cross_shard_merges(&self) -> Result> { + let cf = self + .db + .cf_handle(linker_cf::METADATA) + .ok_or_else(|| anyhow::anyhow!("Column family {} not found", linker_cf::METADATA))?; + let mut mappings = Vec::new(); + let iter = self.db.iterator_cf(&cf, rocksdb::IteratorMode::Start); + for item in iter { + let (key, value) = item?; + if !key.starts_with(linker_encoding::CROSS_SHARD_MERGE_PREFIX) { + continue; + } + let secondary = linker_encoding::decode_cross_shard_merge_key(&key) + .ok_or_else(|| anyhow!("invalid persisted cross-shard merge key"))?; + let primary = linker_encoding::decode_global_cluster_id(&value) + .filter(|_| value.len() == 8) + .ok_or_else(|| anyhow!("invalid persisted cross-shard merge value"))?; + mappings.push((secondary, primary)); + } + Ok(mappings) + } + /// Load the next_cluster_id value. pub fn load_next_cluster_id(&self) -> Result> { let cf = self @@ -2267,8 +3270,17 @@ mod tests { use crate::{StreamingTuning, Unirust}; use tempfile::tempdir; + static PERSISTENT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn lock_persistent_tests() -> std::sync::MutexGuard<'static, ()> { + PERSISTENT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + #[test] fn persistent_store_round_trip() { + let _guard = lock_persistent_tests(); let dir = tempdir().unwrap(); let path = dir.path(); @@ -2289,8 +3301,380 @@ mod tests { assert_eq!(store.len(), 1); } + #[test] + fn source_reservation_survives_restart_and_rejects_changed_payload() { + let _guard = lock_persistent_tests(); + let dir = tempdir().unwrap(); + let identity = RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "durable-source".to_string(), + ); + let reservation = SourceRecordReservation { + identity: identity.clone(), + payload_digest: [7; 32], + target_shard_id: 1, + }; + + { + let mut store = PersistentStore::open(dir.path()).unwrap(); + assert_eq!( + store + .reserve_source_records(std::slice::from_ref(&reservation)) + .unwrap(), + vec![1] + ); + store.mark_source_reservation_backfill(2, 3).unwrap(); + } + + let mut store = PersistentStore::open(dir.path()).unwrap(); + assert_eq!(store.source_reservation_backfill().unwrap(), Some((2, 3))); + assert_eq!( + store + .reserve_source_records(std::slice::from_ref(&reservation)) + .unwrap(), + vec![1] + ); + let error = store + .reserve_source_records(&[SourceRecordReservation { + identity, + payload_digest: [8; 32], + target_shard_id: 1, + }]) + .expect_err("a changed payload must remain rejected after restart"); + assert!(matches!( + error.downcast_ref::(), + Some(SourceReservationError::PayloadConflict) + )); + } + + #[test] + fn external_checkpoint_restores_records_and_reservation_state() { + let _guard = lock_persistent_tests(); + let data_volume = tempdir().unwrap(); + let backup_volume = tempdir().unwrap(); + let original_path = data_volume.path().join("original"); + let checkpoint_path = backup_volume.path().join("checkpoint-1"); + let replacement_path = data_volume.path().join("replacement"); + let identity = RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "restored-source".to_string(), + ); + + { + let mut store = PersistentStore::open(&original_path).unwrap(); + let record_id = store + .add_record(Record::new(RecordId(0), identity.clone(), Vec::new())) + .unwrap(); + assert_eq!(record_id, RecordId(0)); + store + .reserve_source_records(&[SourceRecordReservation { + identity: identity.clone(), + payload_digest: [9; 32], + target_shard_id: 0, + }]) + .unwrap(); + store.mark_source_reservation_backfill(2, 1).unwrap(); + store.checkpoint(&checkpoint_path).unwrap(); + } + prepare_cluster_checkpoint(&checkpoint_path, "unit-restore", 0, 1).unwrap(); + commit_cluster_checkpoint(&checkpoint_path, "unit-restore", 0, 1).unwrap(); + + restore_checkpoint(&checkpoint_path, &replacement_path).unwrap(); + let mut restored = PersistentStore::open(&replacement_path).unwrap(); + assert_eq!( + restored.get_record_id_by_identity(&identity), + Some(RecordId(0)) + ); + assert_eq!( + restored.source_reservation_backfill().unwrap(), + Some((2, 1)) + ); + let error = restored + .reserve_source_records(&[SourceRecordReservation { + identity, + payload_digest: [10; 32], + target_shard_id: 0, + }]) + .expect_err("restored reservation must reject a changed payload"); + assert!(matches!( + error.downcast_ref::(), + Some(SourceReservationError::PayloadConflict) + )); + } + + #[test] + fn checkpoint_restore_refuses_nonempty_destination() { + let _guard = lock_persistent_tests(); + let data_volume = tempdir().unwrap(); + let backup_volume = tempdir().unwrap(); + let source_path = data_volume.path().join("source"); + let checkpoint_path = backup_volume.path().join("checkpoint"); + let destination = data_volume.path().join("replacement"); + + let store = PersistentStore::open(&source_path).unwrap(); + store.checkpoint(&checkpoint_path).unwrap(); + prepare_cluster_checkpoint(&checkpoint_path, "nonempty-destination", 0, 1).unwrap(); + commit_cluster_checkpoint(&checkpoint_path, "nonempty-destination", 0, 1).unwrap(); + fs::create_dir_all(&destination).unwrap(); + fs::write(destination.join("do-not-overwrite"), b"live data").unwrap(); + + let error = restore_checkpoint(&checkpoint_path, &destination) + .expect_err("restore must not overwrite a nonempty directory"); + assert!(error.to_string().contains("refusing to restore")); + assert_eq!( + fs::read(destination.join("do-not-overwrite")).unwrap(), + b"live data" + ); + } + + #[test] + fn checkpoint_restore_rejects_uncommitted_and_wrong_shard_snapshots() { + let _guard = lock_persistent_tests(); + let data_volume = tempdir().unwrap(); + let backup_volume = tempdir().unwrap(); + let source_path = data_volume.path().join("source"); + let checkpoint_path = backup_volume.path().join("checkpoint"); + let destination = data_volume.path().join("replacement"); + + let store = PersistentStore::open(&source_path).unwrap(); + store.checkpoint(&checkpoint_path).unwrap(); + prepare_cluster_checkpoint(&checkpoint_path, "cluster-generation", 0, 2).unwrap(); + + restore_checkpoint(&checkpoint_path, &destination) + .expect_err("prepared but uncommitted checkpoint must be rejected"); + assert!(!destination.exists()); + + commit_cluster_checkpoint(&checkpoint_path, "cluster-generation", 0, 2).unwrap(); + restore_checkpoint_for_shard(&checkpoint_path, &destination, Some(1)) + .expect_err("checkpoint from another shard must be rejected"); + assert!(!destination.exists()); + } + + #[test] + fn restored_data_directory_rejects_incomplete_provenance() { + let _guard = lock_persistent_tests(); + let data_volume = tempdir().unwrap(); + fs::write( + data_volume.path().join(CHECKPOINT_MANIFEST_FILE), + b"incomplete", + ) + .unwrap(); + + let error = read_restored_checkpoint_manifest(data_volume.path()) + .expect_err("one restore marker must fail closed"); + assert!(error + .to_string() + .contains("incomplete checkpoint provenance")); + } + + #[test] + fn checkpoint_restore_validates_rocksdb_before_publish() { + let _guard = lock_persistent_tests(); + let data_volume = tempdir().unwrap(); + let backup_volume = tempdir().unwrap(); + let source_path = data_volume.path().join("source"); + let checkpoint_path = backup_volume.path().join("checkpoint"); + let destination = data_volume.path().join("replacement"); + + let store = PersistentStore::open(&source_path).unwrap(); + store.checkpoint(&checkpoint_path).unwrap(); + prepare_cluster_checkpoint(&checkpoint_path, "corrupt-generation", 0, 1).unwrap(); + commit_cluster_checkpoint(&checkpoint_path, "corrupt-generation", 0, 1).unwrap(); + fs::write(checkpoint_path.join("CURRENT"), b"not-a-valid-manifest\n").unwrap(); + + restore_checkpoint(&checkpoint_path, &destination) + .expect_err("corrupt RocksDB checkpoint must be rejected"); + assert!(!destination.exists()); + } + + #[cfg(unix)] + #[test] + fn checkpoint_restore_rejects_top_level_symlink() { + let _guard = lock_persistent_tests(); + use std::os::unix::fs::symlink; + + let data_volume = tempdir().unwrap(); + let backup_volume = tempdir().unwrap(); + let source_path = data_volume.path().join("source"); + let checkpoint_path = backup_volume.path().join("checkpoint"); + let checkpoint_link = backup_volume.path().join("checkpoint-link"); + let destination = data_volume.path().join("replacement"); + + let store = PersistentStore::open(&source_path).unwrap(); + store.checkpoint(&checkpoint_path).unwrap(); + prepare_cluster_checkpoint(&checkpoint_path, "symlink-generation", 0, 1).unwrap(); + commit_cluster_checkpoint(&checkpoint_path, "symlink-generation", 0, 1).unwrap(); + symlink(&checkpoint_path, &checkpoint_link).unwrap(); + + restore_checkpoint(&checkpoint_link, &destination) + .expect_err("checkpoint symlink must be rejected"); + assert!(!destination.exists()); + } + + #[test] + fn corrupt_record_read_poison_store_fail_closed() { + let _guard = lock_persistent_tests(); + let dir = tempdir().unwrap(); + let path = dir.path(); + let record_id; + { + let mut store = PersistentStore::open(path).unwrap(); + let attr = store.interner_mut().intern_attr("email"); + let value = store.interner_mut().intern_value("corrupt@example.com"); + record_id = store + .add_record(Record::new( + RecordId(0), + RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "corrupt".to_string(), + ), + vec![Descriptor::new(attr, value, Interval::new(0, 10).unwrap())], + )) + .unwrap(); + } + + let store = PersistentStore::open(path).unwrap(); + let records_cf = store.db.cf_handle(CF_RECORDS).unwrap(); + store + .db + .put_cf(records_cf, record_id.0.to_be_bytes(), b"not-a-record") + .unwrap(); + + assert!(store.get_record(record_id).is_none()); + assert!(store.ensure_healthy().is_err()); + assert!( + store.sync().is_err(), + "an ingest acknowledgement must fail after an incomplete read" + ); + } + + #[test] + fn staged_batch_deduplicates_source_identity() { + let _guard = lock_persistent_tests(); + let dir = tempdir().unwrap(); + let path = dir.path(); + + let mut store = PersistentStore::open(path).unwrap(); + let attr = store.interner_mut().intern_attr("email"); + let value = store.interner_mut().intern_value("alice@example.com"); + let identity = RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "alice-1".to_string(), + ); + let make_record = || { + Record::new( + RecordId(0), + identity.clone(), + vec![Descriptor::new(attr, value, Interval::new(0, 10).unwrap())], + ) + }; + + let first = store.stage_record_if_absent(make_record()).unwrap(); + let second = store.stage_record_if_absent(make_record()).unwrap(); + + assert!(first.1); + assert!(!second.1); + assert_eq!(first.0, second.0); + assert_eq!(store.flush_staged_records().unwrap(), 1); + drop(store); + + let store = PersistentStore::open(path).unwrap(); + assert_eq!(store.len(), 1); + assert_eq!(store.get_record_id_by_identity(&identity), Some(first.0)); + } + + #[test] + fn exhausted_record_id_sequence_survives_restart_without_wrapping() { + let _guard = lock_persistent_tests(); + let dir = tempdir().unwrap(); + let path = dir.path(); + let last_identity = RecordIdentity::new( + "person".to_string(), + "import".to_string(), + "last-id".to_string(), + ); + + { + let mut store = PersistentStore::open(path).unwrap(); + let record = Record::new(RecordId(u32::MAX - 1), last_identity.clone(), Vec::new()); + let (record_id, inserted) = store + .stage_record_with_explicit_id_if_absent(record) + .unwrap(); + assert_eq!(record_id, RecordId(u32::MAX - 1)); + assert!(inserted); + assert_eq!(store.flush_staged_records().unwrap(), 1); + store.sync().unwrap(); + } + + let mut store = PersistentStore::open(path).unwrap(); + let next = Record::new( + RecordId(0), + RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "must-not-wrap".to_string(), + ), + Vec::new(), + ); + let error = store + .add_record(next) + .expect_err("persisted record ID exhaustion must fail closed"); + assert!(error.to_string().contains("record ID space exhausted")); + assert_eq!(store.len(), 1); + assert_eq!( + store.get_record_id_by_identity(&last_identity), + Some(RecordId(u32::MAX - 1)) + ); + assert!(store.get_record(RecordId(0)).is_none()); + } + + #[test] + fn batch_insert_deduplicates_source_identity() { + let _guard = lock_persistent_tests(); + let dir = tempdir().unwrap(); + let mut store = PersistentStore::open(dir.path()).unwrap(); + let identity = RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "alice-1".to_string(), + ); + let make_record = || Record::new(RecordId(0), identity.clone(), Vec::new()); + + let results = store + .add_records_if_absent(vec![make_record(), make_record()]) + .unwrap(); + + assert_eq!(results.len(), 2); + assert!(results[0].1); + assert!(!results[1].1); + assert_eq!(results[0].0, results[1].0); + assert_eq!(store.len(), 1); + } + + #[test] + fn interner_watermark_advances_only_after_commit() { + let _guard = lock_persistent_tests(); + let dir = tempdir().unwrap(); + let mut store = PersistentStore::open(dir.path()).unwrap(); + store.interner_mut().intern_attr("email"); + + let mut batch = WriteBatch::default(); + let watermark = store.append_interner_to_batch(&mut batch).unwrap(); + + assert_eq!(store.persisted_attr_id, 0); + assert_eq!(watermark.0, 1); + store.db.write(batch).unwrap(); + store.commit_interner_watermark(watermark); + assert_eq!(store.persisted_attr_id, 1); + } + #[test] fn persistent_store_retains_ontology_and_sequence() { + let _guard = lock_persistent_tests(); let dir = tempdir().unwrap(); let path = dir.path(); @@ -2335,8 +3719,32 @@ mod tests { assert_eq!(store.len(), 2); } + #[test] + fn reset_clears_all_persistent_resolution_state() { + let _guard = lock_persistent_tests(); + let dir = tempdir().unwrap(); + let mut store = PersistentStore::open(dir.path()).unwrap(); + store.save_ontology_config(b"retained-config").unwrap(); + + for &cf_name in RESET_DATA_CFS { + let cf = store.db.cf_handle(cf_name).unwrap(); + store.db.put_cf(cf, b"stale-key", b"stale-value").unwrap(); + } + store.reset_data().unwrap(); + + for &cf_name in RESET_DATA_CFS { + let cf = store.db.cf_handle(cf_name).unwrap(); + assert_eq!(store.db.iterator_cf(cf, IteratorMode::Start).count(), 0); + } + assert_eq!( + store.load_ontology_config().unwrap().as_deref(), + Some(b"retained-config".as_slice()) + ); + } + #[test] fn persistent_store_preserves_conflict_results() { + let _guard = lock_persistent_tests(); let dir = tempdir().unwrap(); let path = dir.path(); @@ -2390,6 +3798,7 @@ mod tests { #[test] fn persistent_store_query_after_restart() { + let _guard = lock_persistent_tests(); let dir = tempdir().unwrap(); let path = dir.path(); diff --git a/src/profile.rs b/src/profile.rs index c385f2d..741c094 100644 --- a/src/profile.rs +++ b/src/profile.rs @@ -59,7 +59,7 @@ pub fn print_report() { .map(|(name, (count, total))| (*name, *count, *total)) .collect(); - entries.sort_by(|a, b| b.2.cmp(&a.2)); + entries.sort_by_key(|entry| std::cmp::Reverse(entry.2)); println!("\nProfiling report:"); for (name, count, total) in entries { diff --git a/src/query.rs b/src/query.rs index 96fcf68..5e48485 100644 --- a/src/query.rs +++ b/src/query.rs @@ -73,6 +73,7 @@ impl QuerySelectivityStats { #[derive(Debug, Clone, PartialEq, Eq)] pub struct QueryMatch { pub cluster_id: ClusterId, + pub root_record_id: RecordId, pub interval: Interval, pub golden: Vec, pub cluster_key: Option, @@ -202,24 +203,31 @@ pub fn query_master_entities_with_cache( } let matches = matches .into_iter() - .map(|entry| QueryMatch { - cluster_id: entry.cluster_id, - interval: entry.interval, - golden: filter_golden_for_interval( - golden_cache + .map(|entry| { + let root_record_id = clusters + .get_cluster(entry.cluster_id) + .map(|cluster| cluster.root) + .ok_or_else(|| anyhow::anyhow!("query cluster has no DSU root"))?; + Ok(QueryMatch { + cluster_id: entry.cluster_id, + root_record_id, + interval: entry.interval, + golden: filter_golden_for_interval( + golden_cache + .get(&entry.cluster_id) + .map(Vec::as_slice) + .unwrap_or(&[]), + entry.interval, + ), + cluster_key: cluster_key_cache .get(&entry.cluster_id) - .map(Vec::as_slice) - .unwrap_or(&[]), - entry.interval, - ), - cluster_key: cluster_key_cache - .get(&entry.cluster_id) - .map(|key| key.value.clone()), - cluster_key_identity: cluster_key_cache - .get(&entry.cluster_id) - .map(|key| key.identity_key.clone()), + .map(|key| key.value.clone()), + cluster_key_identity: cluster_key_cache + .get(&entry.cluster_id) + .map(|key| key.identity_key.clone()), + }) }) - .collect(); + .collect::>>()?; Ok(QueryOutcome::Matches(matches)) } @@ -303,24 +311,31 @@ pub fn query_master_entities_with_cache_selective( } let matches = matches .into_iter() - .map(|entry| QueryMatch { - cluster_id: entry.cluster_id, - interval: entry.interval, - golden: filter_golden_for_interval( - golden_cache + .map(|entry| { + let root_record_id = clusters + .get_cluster(entry.cluster_id) + .map(|cluster| cluster.root) + .ok_or_else(|| anyhow::anyhow!("query cluster has no DSU root"))?; + Ok(QueryMatch { + cluster_id: entry.cluster_id, + root_record_id, + interval: entry.interval, + golden: filter_golden_for_interval( + golden_cache + .get(&entry.cluster_id) + .map(Vec::as_slice) + .unwrap_or(&[]), + entry.interval, + ), + cluster_key: cluster_key_cache .get(&entry.cluster_id) - .map(Vec::as_slice) - .unwrap_or(&[]), - entry.interval, - ), - cluster_key: cluster_key_cache - .get(&entry.cluster_id) - .map(|key| key.value.clone()), - cluster_key_identity: cluster_key_cache - .get(&entry.cluster_id) - .map(|key| key.identity_key.clone()), + .map(|key| key.value.clone()), + cluster_key_identity: cluster_key_cache + .get(&entry.cluster_id) + .map(|key| key.identity_key.clone()), + }) }) - .collect(); + .collect::>>()?; Ok(QueryOutcome::Matches(matches)) } @@ -374,7 +389,7 @@ fn coalesce_matches_per_cluster(matches: Vec) -> Vec { } } - result.sort_by(|a, b| a.interval.start.cmp(&b.interval.start)); + result.sort_by_key(|query_match| query_match.interval.start); result } @@ -389,7 +404,7 @@ fn find_overlap_conflict( } let mut sorted = matches.to_vec(); - sorted.sort_by(|a, b| a.interval.start.cmp(&b.interval.start)); + sorted.sort_by_key(|query_match| query_match.interval.start); for window in sorted.windows(2) { let current = &window[0]; diff --git a/src/sharding.rs b/src/sharding.rs index 01262a2..0cf3c28 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -10,10 +10,16 @@ use crate::temporal::{is_overlapping, Interval}; use crate::ClusterAssignment; use anyhow::Result; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::hash_map::DefaultHasher; use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; +fn hash_length_prefixed(hasher: &mut Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + /// A signature of identity key values for fast lookup. /// This is a 32-byte hash of the identity key's attribute-value pairs. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -22,35 +28,15 @@ pub struct IdentityKeySignature(pub [u8; 32]); impl IdentityKeySignature { /// Create a new signature from identity key values. pub fn from_key_values(entity_type: &str, key_values: &[KeyValue]) -> Self { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - // Use two hashers to fill 32 bytes - let mut hasher1 = DefaultHasher::new(); - let mut hasher2 = DefaultHasher::new(); - - entity_type.hash(&mut hasher1); - entity_type.hash(&mut hasher2); - - for (i, kv) in key_values.iter().enumerate() { - kv.hash(&mut hasher1); - // Mix in index for hasher2 to get different bits - (kv, i).hash(&mut hasher2); - } - - let h1 = hasher1.finish().to_le_bytes(); - let h2 = hasher2.finish().to_le_bytes(); - - let mut bytes = [0u8; 32]; - bytes[0..8].copy_from_slice(&h1); - bytes[8..16].copy_from_slice(&h2); - // Fill remaining with XOR variations - for i in 0..8 { - bytes[16 + i] = h1[i] ^ h2[7 - i]; - bytes[24 + i] = h1[7 - i].wrapping_add(h2[i]); + let mut hasher = Sha256::new(); + hasher.update(b"unirust.identity-key.numeric.v1"); + hash_length_prefixed(&mut hasher, entity_type.as_bytes()); + hasher.update((key_values.len() as u64).to_be_bytes()); + for key_value in key_values { + hasher.update(key_value.attr.0.to_be_bytes()); + hasher.update(key_value.value.0.to_be_bytes()); } - - Self(bytes) + Self(hasher.finalize().into()) } /// Create a signature from identity key values using the interner's string values. @@ -60,35 +46,21 @@ impl IdentityKeySignature { key_values: &[KeyValue], interner: &dyn InternerLookup, ) -> Option { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher1 = DefaultHasher::new(); - let mut hasher2 = DefaultHasher::new(); - - entity_type.hash(&mut hasher1); - entity_type.hash(&mut hasher2); - - for (i, kv) in key_values.iter().enumerate() { - let attr = interner.get_attr_string(kv.attr)?; - let value = interner.get_value_string(kv.value)?; - attr.hash(&mut hasher1); - value.hash(&mut hasher1); - ((attr, value), i).hash(&mut hasher2); - } - - let h1 = hasher1.finish().to_le_bytes(); - let h2 = hasher2.finish().to_le_bytes(); - - let mut bytes = [0u8; 32]; - bytes[0..8].copy_from_slice(&h1); - bytes[8..16].copy_from_slice(&h2); - for i in 0..8 { - bytes[16 + i] = h1[i] ^ h2[7 - i]; - bytes[24 + i] = h1[7 - i].wrapping_add(h2[i]); + let mut hasher = Sha256::new(); + hasher.update(b"unirust.identity-key.text.v1"); + hash_length_prefixed(&mut hasher, entity_type.as_bytes()); + hasher.update((key_values.len() as u64).to_be_bytes()); + for key_value in key_values { + hash_length_prefixed( + &mut hasher, + interner.get_attr_string(key_value.attr)?.as_bytes(), + ); + hash_length_prefixed( + &mut hasher, + interner.get_value_string(key_value.value)?.as_bytes(), + ); } - - Some(Self(bytes)) + Some(Self(hasher.finalize().into())) } /// Convert to bytes for storage. @@ -173,6 +145,15 @@ impl BloomFilter { } } +/// Entry in the cluster boundary index. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct BoundaryStrongId { + pub perspective: String, + pub attribute: String, + pub value: String, + pub interval: Interval, +} + /// Entry in the cluster boundary index. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BoundaryEntry { @@ -184,6 +165,9 @@ pub struct BoundaryEntry { /// Two clusters conflict if they share a perspective key but have different values. #[serde(default)] pub perspective_strong_ids: HashMap, + /// Exact strong-ID observations used for temporal conflict checks. + #[serde(default)] + pub strong_ids: Vec, } /// A cross-shard conflict detected during reconciliation. @@ -191,7 +175,7 @@ pub struct BoundaryEntry { /// This represents an indirect conflict where two clusters on different shards /// share an identity key but have conflicting strong identifiers within the /// same perspective - meaning they cannot be merged. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CrossShardConflict { /// The identity key signature that links the conflicting clusters. pub identity_key_signature: IdentityKeySignature, @@ -270,14 +254,50 @@ impl ClusterBoundaryIndex { cluster_id: GlobalClusterId, interval: Interval, perspective_strong_ids: HashMap, + ) { + self.register_boundary_key_with_conflict_data( + signature, + cluster_id, + interval, + perspective_strong_ids, + Vec::new(), + ); + } + + /// Register a cluster's identity key with exact temporal strong-ID observations. + pub fn register_boundary_key_with_conflict_data( + &mut self, + signature: IdentityKeySignature, + cluster_id: GlobalClusterId, + interval: Interval, + perspective_strong_ids: HashMap, + mut strong_ids: Vec, ) { self.key_bloom.insert(&signature); + strong_ids.sort_by(|left, right| { + ( + &left.perspective, + &left.attribute, + &left.value, + left.interval.start, + left.interval.end, + ) + .cmp(&( + &right.perspective, + &right.attribute, + &right.value, + right.interval.start, + right.interval.end, + )) + }); + strong_ids.dedup(); let entry = BoundaryEntry { cluster_id, interval, shard_id: self.shard_id, perspective_strong_ids, + strong_ids, }; self.boundary_keys.entry(signature).or_default().push(entry); @@ -497,7 +517,7 @@ impl IncrementalReconciler { ) { let mut merges = Vec::new(); let mut merge_candidates = 0; - let mut conflicts_blocked = 0; + let mut conflicts_blocked = 0usize; let mut detected_conflicts = Vec::new(); // Build a map of all signatures to their entries across all shards @@ -539,22 +559,15 @@ impl IncrementalReconciler { // Record the conflict and skip this merge conflicts_blocked += 1; - // Compute the overlapping interval - let overlap_start = e1.interval.start.max(e2.interval.start); - let overlap_end = e1.interval.end.min(e2.interval.end); - if let Ok(overlap_interval) = - Interval::new(overlap_start, overlap_end) - { - detected_conflicts.push(CrossShardConflict { - identity_key_signature: sig, - cluster1: e1.cluster_id, - cluster2: e2.cluster_id, - interval: overlap_interval, - perspective_hash: conflict_detail.perspective_hash, - strong_id_hash1: conflict_detail.strong_id_hash1, - strong_id_hash2: conflict_detail.strong_id_hash2, - }); - } + detected_conflicts.push(CrossShardConflict { + identity_key_signature: sig, + cluster1: e1.cluster_id, + cluster2: e2.cluster_id, + interval: conflict_detail.interval, + perspective_hash: conflict_detail.perspective_hash, + strong_id_hash1: conflict_detail.strong_id_hash1, + strong_id_hash2: conflict_detail.strong_id_hash2, + }); continue; } @@ -571,10 +584,12 @@ impl IncrementalReconciler { } } + let (merges, component_conflicts_blocked) = + canonicalize_merges(merges, &detected_conflicts); ( merges, merge_candidates, - conflicts_blocked, + conflicts_blocked.saturating_add(component_conflicts_blocked), detected_conflicts, ) } @@ -636,7 +651,7 @@ impl IncrementalReconciler { let mut merges = Vec::new(); let mut merge_candidates = 0; - let mut conflicts_blocked = 0; + let mut conflicts_blocked = 0usize; let mut keys_matched = 0; let mut detected_conflicts = Vec::new(); @@ -675,21 +690,15 @@ impl IncrementalReconciler { // Record the conflict and skip this merge conflicts_blocked += 1; - // Compute the overlapping interval - let overlap_start = e1.interval.start.max(e2.interval.start); - let overlap_end = e1.interval.end.min(e2.interval.end); - if let Ok(overlap_interval) = Interval::new(overlap_start, overlap_end) - { - detected_conflicts.push(CrossShardConflict { - identity_key_signature: *sig, - cluster1: e1.cluster_id, - cluster2: e2.cluster_id, - interval: overlap_interval, - perspective_hash: conflict_detail.perspective_hash, - strong_id_hash1: conflict_detail.strong_id_hash1, - strong_id_hash2: conflict_detail.strong_id_hash2, - }); - } + detected_conflicts.push(CrossShardConflict { + identity_key_signature: *sig, + cluster1: e1.cluster_id, + cluster2: e2.cluster_id, + interval: conflict_detail.interval, + perspective_hash: conflict_detail.perspective_hash, + strong_id_hash1: conflict_detail.strong_id_hash1, + strong_id_hash2: conflict_detail.strong_id_hash2, + }); continue; } @@ -705,6 +714,9 @@ impl IncrementalReconciler { } } + let (merges, component_conflicts_blocked) = + canonicalize_merges(merges, &detected_conflicts); + conflicts_blocked = conflicts_blocked.saturating_add(component_conflicts_blocked); self.pending_merges = merges.clone(); ReconciliationResult { @@ -728,6 +740,112 @@ impl IncrementalReconciler { } } +fn canonicalize_merges( + merges: Vec<(GlobalClusterId, GlobalClusterId)>, + conflicts: &[CrossShardConflict], +) -> (Vec<(GlobalClusterId, GlobalClusterId)>, usize) { + fn find(parent: &mut [usize], mut index: usize) -> usize { + while parent[index] != index { + parent[index] = parent[parent[index]]; + index = parent[index]; + } + index + } + + let mut edges = merges + .into_iter() + .filter(|(left, right)| left != right) + .map(|(left, right)| { + if left.to_u64() <= right.to_u64() { + (left, right) + } else { + (right, left) + } + }) + .collect::>(); + edges.sort_by_key(|(left, right)| (left.to_u64(), right.to_u64())); + edges.dedup(); + + let mut nodes = edges + .iter() + .flat_map(|(left, right)| [*left, *right]) + .collect::>(); + nodes.sort_by_key(GlobalClusterId::to_u64); + nodes.dedup(); + let node_index = nodes + .iter() + .enumerate() + .map(|(index, node)| (*node, index)) + .collect::>(); + + let mut blocked_by_node: HashMap> = HashMap::new(); + for conflict in conflicts { + if conflict.cluster1 == conflict.cluster2 { + continue; + } + blocked_by_node + .entry(conflict.cluster1) + .or_default() + .insert(conflict.cluster2); + blocked_by_node + .entry(conflict.cluster2) + .or_default() + .insert(conflict.cluster1); + } + + let mut parent = (0..nodes.len()).collect::>(); + let mut members = nodes + .iter() + .map(|node| HashSet::from([*node])) + .collect::>(); + let mut blocked = nodes + .iter() + .map(|node| blocked_by_node.remove(node).unwrap_or_default()) + .collect::>(); + let mut component_conflicts_blocked = 0usize; + + for (left, right) in edges { + let mut left_root = find(&mut parent, node_index[&left]); + let mut right_root = find(&mut parent, node_index[&right]); + if left_root == right_root { + continue; + } + if !blocked[left_root].is_disjoint(&members[right_root]) + || !blocked[right_root].is_disjoint(&members[left_root]) + { + component_conflicts_blocked = component_conflicts_blocked.saturating_add(1); + continue; + } + if nodes[left_root].to_u64() > nodes[right_root].to_u64() { + std::mem::swap(&mut left_root, &mut right_root); + } + parent[right_root] = left_root; + let right_members = std::mem::take(&mut members[right_root]); + members[left_root].extend(right_members); + let right_blocked = std::mem::take(&mut blocked[right_root]); + blocked[left_root].extend(right_blocked); + } + + let mut canonical = Vec::new(); + for (index, component_members) in members.iter().enumerate() { + if find(&mut parent, index) != index { + continue; + } + let mut component = component_members.iter().copied().collect::>(); + component.sort_by_key(GlobalClusterId::to_u64); + if let Some(primary) = component.first().copied() { + canonical.extend( + component + .into_iter() + .skip(1) + .map(|secondary| (primary, secondary)), + ); + } + } + canonical.sort_by_key(|(primary, secondary)| (primary.to_u64(), secondary.to_u64())); + (canonical, component_conflicts_blocked) +} + impl Default for IncrementalReconciler { fn default() -> Self { Self::new() @@ -739,6 +857,21 @@ struct ConflictDetail { perspective_hash: u64, strong_id_hash1: u64, strong_id_hash2: u64, + interval: Interval, +} + +fn stable_boundary_hash(domain: &[u8], values: &[&str]) -> u64 { + let mut hasher = Sha256::new(); + hasher.update(domain); + for value in values { + hash_length_prefixed(&mut hasher, value.as_bytes()); + } + let digest = hasher.finalize(); + u64::from_be_bytes( + digest[..8] + .try_into() + .expect("SHA-256 digest contains eight boundary hash bytes"), + ) } /// Check if two boundary entries have conflicting strong IDs. @@ -750,6 +883,43 @@ struct ConflictDetail { /// /// Returns `Some(ConflictDetail)` if a conflict is found, `None` otherwise. fn detect_cross_shard_conflict(e1: &BoundaryEntry, e2: &BoundaryEntry) -> Option { + if !e1.strong_ids.is_empty() && !e2.strong_ids.is_empty() { + for strong_id_1 in &e1.strong_ids { + for strong_id_2 in &e2.strong_ids { + if strong_id_1.perspective != strong_id_2.perspective + || strong_id_1.attribute != strong_id_2.attribute + || strong_id_1.value == strong_id_2.value + { + continue; + } + let Some(interval) = + crate::temporal::intersect(&strong_id_1.interval, &strong_id_2.interval) + else { + continue; + }; + return Some(ConflictDetail { + perspective_hash: stable_boundary_hash( + b"unirust.boundary-perspective.v1", + &[&strong_id_1.perspective], + ), + strong_id_hash1: stable_boundary_hash( + b"unirust.boundary-strong-id.v1", + &[&strong_id_1.attribute, &strong_id_1.value], + ), + strong_id_hash2: stable_boundary_hash( + b"unirust.boundary-strong-id.v1", + &[&strong_id_2.attribute, &strong_id_2.value], + ), + interval, + }); + } + } + return None; + } + + let interval = crate::temporal::intersect(&e1.interval, &e2.interval)?; + + // Compatibility fallback for metadata from nodes that predate exact observations. // Check each perspective in e1 against e2 for (perspective_hash, strong_id_hash_1) in &e1.perspective_strong_ids { if let Some(strong_id_hash_2) = e2.perspective_strong_ids.get(perspective_hash) { @@ -760,6 +930,7 @@ fn detect_cross_shard_conflict(e1: &BoundaryEntry, e2: &BoundaryEntry) -> Option perspective_hash: *perspective_hash, strong_id_hash1: *strong_id_hash_1, strong_id_hash2: *strong_id_hash_2, + interval, }); } } @@ -857,7 +1028,7 @@ impl ShardedStreamEngine { let shard_count = self.shards.len(); let shards = std::mem::take(&mut self.shards); let mut handles = Vec::with_capacity(shard_count); - for ((idx, shard), bucket) in shards.into_iter().enumerate().zip(buckets.into_iter()) { + for ((idx, shard), bucket) in shards.into_iter().enumerate().zip(buckets) { handles.push(std::thread::spawn( move || -> Result<(usize, ShardState, Vec)> { let mut shard = shard; @@ -1021,6 +1192,131 @@ mod tests { use crate::perf::ConcurrentInterner; use crate::Interval; + #[test] + fn text_identity_signature_has_a_stable_protocol_vector() { + let mut interner = StringInterner::new(); + let attr = interner.intern_attr("email"); + let value = interner.intern_value("alice@example.com"); + let signature = IdentityKeySignature::from_key_values_with_interner( + "person", + &[KeyValue::new(attr, value)], + &interner, + ) + .expect("signature"); + + assert_eq!( + signature.0, + [ + 0x82, 0xb4, 0x0d, 0xdb, 0x19, 0xe7, 0x35, 0xda, 0x21, 0x5e, 0xb6, 0xb4, 0x23, 0x4a, + 0x0d, 0x7f, 0x1b, 0x83, 0xd6, 0x60, 0x06, 0xc0, 0x87, 0xb5, 0xc8, 0x10, 0x6b, 0xb8, + 0x3d, 0x0c, 0x70, 0x41, + ] + ); + } + + #[test] + fn exact_boundary_guards_preserve_attribute_and_interval_semantics() { + let boundary_interval = Interval::new(0, 30).unwrap(); + let cluster0 = GlobalClusterId::new(0, 1, 0); + let cluster1 = GlobalClusterId::new(1, 1, 0); + let entry0 = BoundaryEntry { + cluster_id: cluster0, + interval: boundary_interval, + shard_id: 0, + perspective_strong_ids: HashMap::new(), + strong_ids: vec![BoundaryStrongId { + perspective: "crm".to_string(), + attribute: "employee_id".to_string(), + value: "A".to_string(), + interval: Interval::new(0, 10).unwrap(), + }], + }; + + let different_time = BoundaryEntry { + cluster_id: cluster1, + interval: boundary_interval, + shard_id: 1, + perspective_strong_ids: HashMap::new(), + strong_ids: vec![BoundaryStrongId { + perspective: "crm".to_string(), + attribute: "employee_id".to_string(), + value: "B".to_string(), + interval: Interval::new(10, 20).unwrap(), + }], + }; + assert!( + detect_cross_shard_conflict(&entry0, &different_time).is_none(), + "adjacent strong-ID validity intervals do not conflict" + ); + + let different_attribute = BoundaryEntry { + strong_ids: vec![BoundaryStrongId { + perspective: "crm".to_string(), + attribute: "account_id".to_string(), + value: "B".to_string(), + interval: Interval::new(0, 10).unwrap(), + }], + ..different_time.clone() + }; + assert!( + detect_cross_shard_conflict(&entry0, &different_attribute).is_none(), + "different strong-ID attributes do not conflict" + ); + + let overlapping_value = BoundaryEntry { + strong_ids: vec![BoundaryStrongId { + perspective: "crm".to_string(), + attribute: "employee_id".to_string(), + value: "B".to_string(), + interval: Interval::new(5, 15).unwrap(), + }], + ..different_time + }; + let conflict = detect_cross_shard_conflict(&entry0, &overlapping_value) + .expect("overlapping different values for one strong ID must conflict"); + assert_eq!(conflict.interval, Interval::new(5, 10).unwrap()); + } + + #[test] + fn reconciler_merges_clusters_with_nonoverlapping_strong_id_values() { + let signature = IdentityKeySignature::from_bytes([7; 32]); + let boundary_interval = Interval::new(0, 30).unwrap(); + let mut shard0 = ClusterBoundaryIndex::new_small(0); + shard0.register_boundary_key_with_conflict_data( + signature, + GlobalClusterId::new(0, 1, 0), + boundary_interval, + HashMap::new(), + vec![BoundaryStrongId { + perspective: "crm".to_string(), + attribute: "employee_id".to_string(), + value: "A".to_string(), + interval: Interval::new(0, 10).unwrap(), + }], + ); + let mut shard1 = ClusterBoundaryIndex::new_small(1); + shard1.register_boundary_key_with_conflict_data( + signature, + GlobalClusterId::new(1, 1, 0), + boundary_interval, + HashMap::new(), + vec![BoundaryStrongId { + perspective: "crm".to_string(), + attribute: "employee_id".to_string(), + value: "B".to_string(), + interval: Interval::new(10, 20).unwrap(), + }], + ); + + let mut reconciler = IncrementalReconciler::new(); + reconciler.add_shard_boundary(shard0); + reconciler.add_shard_boundary(shard1); + let result = reconciler.reconcile(); + + assert_eq!(result.conflicts_blocked, 0); + assert_eq!(result.merges_performed, 1); + } + #[test] fn sharded_reconcile_matches_single_linker() { let mut ontology = Ontology::new(); @@ -1305,6 +1601,62 @@ mod tests { assert_eq!(secondary.shard_id, 1); } + #[test] + fn reconciler_redirects_every_component_member_to_one_primary() { + use crate::model::{AttrId, ValueId}; + + let mut reconciler = IncrementalReconciler::new(); + let signature = IdentityKeySignature::from_key_values( + "person", + &[KeyValue::new(AttrId(1), ValueId(2))], + ); + let interval = Interval::new(0, 100).unwrap(); + let clusters = [ + GlobalClusterId::new(0, 10, 0), + GlobalClusterId::new(1, 20, 0), + GlobalClusterId::new(2, 30, 0), + ]; + + for (shard_id, cluster_id) in clusters.into_iter().enumerate() { + let mut boundary = ClusterBoundaryIndex::new_small(shard_id as u16); + boundary.register_boundary_key(signature, cluster_id, interval); + reconciler.add_shard_boundary(boundary); + } + + let result = reconciler.reconcile(); + assert_eq!( + result.merged_clusters, + vec![(clusters[0], clusters[1]), (clusters[0], clusters[2])] + ); + assert_eq!(result.merges_performed, 2); + assert_eq!(result.merge_candidates, 3); + } + + #[test] + fn component_merge_does_not_bridge_a_cannot_link_pair() { + use crate::model::{AttrId, ValueId}; + + let a = GlobalClusterId::new(0, 10, 0); + let b = GlobalClusterId::new(1, 20, 0); + let c = GlobalClusterId::new(2, 30, 0); + let conflict = CrossShardConflict { + identity_key_signature: IdentityKeySignature::from_key_values( + "person", + &[KeyValue::new(AttrId(1), ValueId(2))], + ), + cluster1: a, + cluster2: c, + interval: Interval::new(0, 100).unwrap(), + perspective_hash: 1, + strong_id_hash1: 2, + strong_id_hash2: 3, + }; + + let (merges, blocked) = canonicalize_merges(vec![(a, b), (b, c)], &[conflict]); + assert_eq!(merges, vec![(a, b)]); + assert_eq!(blocked, 1); + } + #[test] fn test_boundary_metadata_export_import() { use crate::model::{AttrId, ValueId}; @@ -1388,6 +1740,7 @@ mod tests { interval, shard_id: 0, perspective_strong_ids: HashMap::new(), + strong_ids: Vec::new(), }; // Strong ID: ts_code="TS1" (represented by hash) entry0.perspective_strong_ids.insert( @@ -1404,6 +1757,7 @@ mod tests { interval, shard_id: 1, perspective_strong_ids: HashMap::new(), + strong_ids: Vec::new(), }; // Strong ID: ts_code="TS2" (DIFFERENT from Record 1!) entry1_a.perspective_strong_ids.insert( @@ -1420,6 +1774,7 @@ mod tests { interval, shard_id: 1, perspective_strong_ids: HashMap::new(), + strong_ids: Vec::new(), }; // Strong ID: axioma_id="AI1" (different perspective, no conflict) entry1_b.perspective_strong_ids.insert( @@ -1511,6 +1866,7 @@ mod tests { interval, shard_id: 0, perspective_strong_ids: HashMap::new(), + strong_ids: Vec::new(), }; entry0 .perspective_strong_ids @@ -1525,6 +1881,7 @@ mod tests { interval, shard_id: 1, perspective_strong_ids: HashMap::new(), + strong_ids: Vec::new(), }; entry1 .perspective_strong_ids diff --git a/src/store.rs b/src/store.rs index 1e67088..7db5a57 100644 --- a/src/store.rs +++ b/src/store.rs @@ -20,8 +20,38 @@ pub struct StoreMetrics { pub block_cache_usage_bytes: u64, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceRecordReservation { + pub identity: RecordIdentity, + pub payload_digest: [u8; 32], + pub target_shard_id: u32, +} + +#[derive(Debug, thiserror::Error)] +pub enum SourceReservationError { + #[error("source record identity is already reserved for a different payload")] + PayloadConflict, + #[error( + "source record identity is already assigned to shard {existing_shard}, not shard {requested_shard}" + )] + TargetConflict { + existing_shard: u32, + requested_shard: u32, + }, +} + /// Persistence abstraction for records and metadata. pub trait RecordStore: Send + Sync { + /// Fail if the store observed an I/O or decode error that made a prior read incomplete. + fn ensure_healthy(&self) -> Result<()> { + Ok(()) + } + + /// Remove all records and derived state while keeping the store usable. + fn reset_data(&mut self) -> Result<()> { + anyhow::bail!("reset is not supported by this record store") + } + /// Add a single record and return its assigned ID. fn add_record(&mut self, record: Record) -> Result; @@ -153,6 +183,19 @@ pub trait RecordStore: Send + Sync { None } + /// Persist cross-shard conflicts detected by distributed reconciliation. + fn set_cross_shard_conflicts( + &mut self, + _conflicts: &[crate::sharding::CrossShardConflict], + ) -> Result<()> { + Ok(()) + } + + /// Load persisted cross-shard conflicts. + fn load_cross_shard_conflicts(&self) -> Result> { + Ok(Vec::new()) + } + /// Load conflict summary count if supported. fn conflict_summary_count(&self) -> Option { None @@ -219,12 +262,55 @@ pub trait RecordStore: Send + Sync { self.add_record_if_absent(record) } + /// Stage a record while preserving its explicit ID. + fn stage_record_with_explicit_id_if_absent( + &mut self, + _record: Record, + ) -> Result<(RecordId, bool)> { + Err(anyhow::anyhow!( + "explicit record ID staging is not supported by this store" + )) + } + /// Flush all staged records to the database. Returns count of flushed records. /// Default implementation returns 0 (no staging support). fn flush_staged_records(&mut self) -> Result { Ok(0) } + /// Make all completed writes durable on stable storage. + /// + /// In-memory stores have nothing to synchronize. Persistent implementations + /// must not return until their recovery log is synced. + fn sync(&self) -> Result<()> { + Ok(()) + } + + /// Atomically reserve immutable source identities before distributed ingest. + /// + /// The returned target shard IDs correspond to the input order. Persistent + /// implementations must make newly created reservations durable before returning. + fn reserve_source_records( + &mut self, + _reservations: &[SourceRecordReservation], + ) -> Result> { + anyhow::bail!("source record reservations are not supported by this record store") + } + + /// Return the completed source-reservation migration protocol and shard count. + fn source_reservation_backfill(&self) -> Result> { + Ok(None) + } + + /// Durably mark all records currently stored on this shard as reserved. + fn mark_source_reservation_backfill( + &mut self, + _protocol_version: u32, + _shard_count: u32, + ) -> Result<()> { + anyhow::bail!("source reservation migration markers are not supported by this record store") + } + /// Optional store-level metrics. fn metrics(&self) -> Option { None @@ -244,6 +330,10 @@ pub struct Store { records: HashMap, /// Identity to record ID mapping (idempotent ingest) identity_index: HashMap, + /// Cluster-wide immutable source identities owned by this shard. + source_reservations: HashMap, + /// Completed distributed reservation migration version and topology size. + source_reservation_backfill: Option<(u32, u32)>, /// String interner for attributes and values interner: StringInterner, /// Attribute-value index for fast lookups @@ -254,12 +344,49 @@ pub struct Store { next_record_id: u32, } +pub(crate) fn records_have_same_payload(left: &Record, right: &Record) -> bool { + if left.identity != right.identity || left.descriptors.len() != right.descriptors.len() { + return false; + } + + let canonical_descriptors = |record: &Record| { + let mut descriptors = record + .descriptors + .iter() + .map(|descriptor| { + ( + descriptor.attr.0, + descriptor.value.0, + descriptor.interval.start, + descriptor.interval.end, + ) + }) + .collect::>(); + descriptors.sort_unstable(); + descriptors + }; + + canonical_descriptors(left) == canonical_descriptors(right) +} + +fn ensure_idempotent_record(existing: &Record, incoming: &Record) -> Result<()> { + if records_have_same_payload(existing, incoming) { + Ok(()) + } else { + anyhow::bail!("source record identity already exists with a different payload") + } +} + impl Store { + const EXHAUSTED_RECORD_ID: u32 = u32::MAX; + /// Create a new store pub fn new() -> Self { Self { records: HashMap::new(), identity_index: HashMap::new(), + source_reservations: HashMap::new(), + source_reservation_backfill: None, interner: StringInterner::new(), attribute_value_index: AttributeValueIndex::new(), temporal_index: TemporalIndex::new(), @@ -272,6 +399,8 @@ impl Store { Self { records: HashMap::new(), identity_index: HashMap::new(), + source_reservations: HashMap::new(), + source_reservation_backfill: None, interner, attribute_value_index: AttributeValueIndex::new(), temporal_index: TemporalIndex::new(), @@ -290,20 +419,45 @@ impl Store { } } + fn allocate_record_id(&mut self) -> Result { + if self.next_record_id == Self::EXHAUSTED_RECORD_ID { + anyhow::bail!("record ID space exhausted"); + } + let record_id = RecordId(self.next_record_id); + self.next_record_id += 1; + Ok(record_id) + } + + fn advance_past_explicit_record_id(&mut self, record_id: RecordId) { + self.next_record_id = self.next_record_id.max(record_id.0.saturating_add(1)); + } + /// Prepare a record for persistence without storing it in memory. pub fn prepare_record(&mut self, record: &mut Record) -> Result { self.intern_record(record); if record.id.0 == 0 { - record.id = RecordId(self.next_record_id); - self.next_record_id += 1; + record.id = self.allocate_record_id()?; } else { - self.next_record_id = self.next_record_id.max(record.id.0 + 1); + if record.id.0 == Self::EXHAUSTED_RECORD_ID { + anyhow::bail!("record ID {} is reserved", Self::EXHAUSTED_RECORD_ID); + } + self.advance_past_explicit_record_id(record.id); } Ok(record.id) } + /// Prepare a record without treating ID zero as an allocation sentinel. + pub fn prepare_record_with_explicit_id(&mut self, record: &mut Record) -> Result { + self.intern_record(record); + if record.id.0 == Self::EXHAUSTED_RECORD_ID { + anyhow::bail!("record ID {} is reserved", Self::EXHAUSTED_RECORD_ID); + } + self.advance_past_explicit_record_id(record.id); + Ok(record.id) + } + /// Add a single record to the store and return its assigned ID. pub fn add_record(&mut self, mut record: Record) -> Result { let record_id = self.prepare_record(&mut record)?; @@ -321,8 +475,11 @@ impl Store { /// Insert a record with an explicit ID without assigning a new one. pub fn insert_record(&mut self, mut record: Record) -> Result { self.intern_record(&mut record); + if record.id.0 == Self::EXHAUSTED_RECORD_ID { + anyhow::bail!("record ID {} is reserved", Self::EXHAUSTED_RECORD_ID); + } - self.next_record_id = self.next_record_id.max(record.id.0 + 1); + self.advance_past_explicit_record_id(record.id); let record_id = record.id; let identity = record.identity.clone(); @@ -346,12 +503,63 @@ impl Store { /// Add a record if its identity is new; returns (id, inserted). pub fn add_record_if_absent(&mut self, record: Record) -> Result<(RecordId, bool)> { if let Some(existing) = self.get_record_id_by_identity(&record.identity) { + let stored = self + .records + .get(&existing) + .ok_or_else(|| anyhow::anyhow!("identity index references a missing record"))?; + ensure_idempotent_record(stored, &record)?; return Ok((existing, false)); } let id = self.add_record(record)?; Ok((id, true)) } + fn reserve_source_records( + &mut self, + reservations: &[SourceRecordReservation], + ) -> Result> { + let mut pending = HashMap::with_capacity(reservations.len()); + let mut targets = Vec::with_capacity(reservations.len()); + + for reservation in reservations { + let existing = self + .source_reservations + .get(&reservation.identity) + .copied() + .or_else(|| pending.get(&reservation.identity).copied()); + if let Some((payload_digest, target_shard_id)) = existing { + if payload_digest != reservation.payload_digest { + return Err(SourceReservationError::PayloadConflict.into()); + } + if target_shard_id != reservation.target_shard_id { + return Err(SourceReservationError::TargetConflict { + existing_shard: target_shard_id, + requested_shard: reservation.target_shard_id, + } + .into()); + } + targets.push(target_shard_id); + } else { + pending.insert( + reservation.identity.clone(), + (reservation.payload_digest, reservation.target_shard_id), + ); + targets.push(reservation.target_shard_id); + } + } + + self.source_reservations.extend(pending); + Ok(targets) + } + + fn source_reservation_backfill(&self) -> Option<(u32, u32)> { + self.source_reservation_backfill + } + + fn mark_source_reservation_backfill(&mut self, protocol_version: u32, shard_count: u32) { + self.source_reservation_backfill = Some((protocol_version, shard_count)); + } + /// Add a record without re-interning. Used when records are pre-interned /// with an external interner (e.g., ConcurrentInterner in partitioned mode). /// @@ -360,10 +568,12 @@ impl Store { pub fn add_record_raw(&mut self, mut record: Record) -> Result { // Assign record ID without interning if record.id.0 == 0 { - record.id = RecordId(self.next_record_id); - self.next_record_id += 1; + record.id = self.allocate_record_id()?; } else { - self.next_record_id = self.next_record_id.max(record.id.0 + 1); + if record.id.0 == Self::EXHAUSTED_RECORD_ID { + anyhow::bail!("record ID {} is reserved", Self::EXHAUSTED_RECORD_ID); + } + self.advance_past_explicit_record_id(record.id); } let record_id = record.id; @@ -381,6 +591,11 @@ impl Store { /// Used when records are pre-interned with an external interner. pub fn add_record_if_absent_raw(&mut self, record: Record) -> Result<(RecordId, bool)> { if let Some(existing) = self.get_record_id_by_identity(&record.identity) { + let stored = self + .records + .get(&existing) + .ok_or_else(|| anyhow::anyhow!("identity index references a missing record"))?; + ensure_idempotent_record(stored, &record)?; return Ok((existing, false)); } let id = self.add_record_raw(record)?; @@ -539,6 +754,11 @@ impl Default for Store { } impl RecordStore for Store { + fn reset_data(&mut self) -> Result<()> { + *self = Store::new(); + Ok(()) + } + fn add_record(&mut self, record: Record) -> Result { Store::add_record(self, record) } @@ -627,9 +847,48 @@ impl RecordStore for Store { Some(self.store_metrics()) } + fn reserve_source_records( + &mut self, + reservations: &[SourceRecordReservation], + ) -> Result> { + Store::reserve_source_records(self, reservations) + } + + fn source_reservation_backfill(&self) -> Result> { + Ok(Store::source_reservation_backfill(self)) + } + + fn mark_source_reservation_backfill( + &mut self, + protocol_version: u32, + shard_count: u32, + ) -> Result<()> { + Store::mark_source_reservation_backfill(self, protocol_version, shard_count); + Ok(()) + } + fn add_record_if_absent(&mut self, record: Record) -> Result<(RecordId, bool)> { Store::add_record_if_absent(self, record) } + + fn stage_record_with_explicit_id_if_absent( + &mut self, + record: Record, + ) -> Result<(RecordId, bool)> { + if let Some(existing) = self.get_record_id_by_identity(&record.identity) { + let stored = self + .records + .get(&existing) + .ok_or_else(|| anyhow::anyhow!("identity index references a missing record"))?; + ensure_idempotent_record(stored, &record)?; + return Ok((existing, false)); + } + if self.records.contains_key(&record.id) { + anyhow::bail!("record ID {} already exists", record.id.0); + } + let id = self.insert_record(record)?; + Ok((id, true)) + } } /// Index for efficient lookup of records by attribute-value pairs @@ -796,6 +1055,81 @@ mod tests { assert_eq!(store.len(), 0); } + #[test] + fn record_id_allocation_fails_closed_at_u32_boundary() { + let mut store = Store::new(); + store.set_next_record_id(u32::MAX - 1); + + let last = Record::new( + RecordId(0), + RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "last-id".to_string(), + ), + vec![], + ); + assert_eq!(store.add_record(last).unwrap(), RecordId(u32::MAX - 1)); + + let overflow = Record::new( + RecordId(0), + RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "overflow".to_string(), + ), + vec![], + ); + let error = store + .add_record(overflow) + .expect_err("record IDs must never wrap"); + assert!(error.to_string().contains("record ID space exhausted")); + assert_eq!(store.len(), 1); + assert!(store.get_record(RecordId(u32::MAX - 1)).is_some()); + assert!(store.get_record(RecordId(0)).is_none()); + } + + #[test] + fn explicit_last_record_id_exhausts_future_allocation() { + let mut store = Store::new(); + let explicit = Record::new( + RecordId(u32::MAX - 1), + RecordIdentity::new( + "person".to_string(), + "import".to_string(), + "max-id".to_string(), + ), + vec![], + ); + store + .stage_record_with_explicit_id_if_absent(explicit) + .unwrap(); + + let next = Record::new( + RecordId(0), + RecordIdentity::new("person".to_string(), "crm".to_string(), "next".to_string()), + vec![], + ); + let error = store + .add_record(next) + .expect_err("allocation after the maximum explicit ID must fail"); + assert!(error.to_string().contains("record ID space exhausted")); + + let reserved = Record::new( + RecordId(u32::MAX), + RecordIdentity::new( + "person".to_string(), + "import".to_string(), + "reserved".to_string(), + ), + vec![], + ); + let error = store + .stage_record_with_explicit_id_if_absent(reserved) + .expect_err("the exhaustion sentinel must not be assignable"); + assert!(error.to_string().contains("is reserved")); + } + #[test] fn test_add_records() { let mut store = Store::new(); @@ -833,6 +1167,111 @@ mod tests { assert_eq!(store.len(), 1); } + #[test] + fn source_reservations_are_atomic_and_immutable() { + let mut store = Store::new(); + let identity_a = + RecordIdentity::new("person".to_string(), "crm".to_string(), "a".to_string()); + let identity_b = + RecordIdentity::new("person".to_string(), "crm".to_string(), "b".to_string()); + let reservation_a = SourceRecordReservation { + identity: identity_a.clone(), + payload_digest: [1; 32], + target_shard_id: 2, + }; + + assert_eq!( + store + .reserve_source_records(std::slice::from_ref(&reservation_a)) + .unwrap(), + vec![2] + ); + assert_eq!( + store + .reserve_source_records(std::slice::from_ref(&reservation_a)) + .unwrap(), + vec![2] + ); + + let error = store + .reserve_source_records(&[ + SourceRecordReservation { + identity: identity_b.clone(), + payload_digest: [2; 32], + target_shard_id: 1, + }, + SourceRecordReservation { + identity: identity_a, + payload_digest: [3; 32], + target_shard_id: 2, + }, + ]) + .expect_err("a conflicting batch must fail atomically"); + assert!(error.downcast_ref::().is_some()); + + assert_eq!( + store + .reserve_source_records(&[SourceRecordReservation { + identity: identity_b, + payload_digest: [4; 32], + target_shard_id: 0, + }]) + .unwrap(), + vec![0], + "the first item from the rejected batch must not be retained" + ); + } + + #[test] + fn repeated_identity_requires_the_same_semantic_payload() { + let mut store = Store::new(); + let email = store.intern_attr("email"); + let phone = store.intern_attr("phone"); + let email_value = store.intern_value("same@example.com"); + let phone_value = store.intern_value("555-0100"); + let interval = Interval::new(0, 10).unwrap(); + let identity = RecordIdentity::new( + "person".to_string(), + "crm".to_string(), + "immutable-1".to_string(), + ); + let first = Record::new( + RecordId(0), + identity.clone(), + vec![ + Descriptor::new(email, email_value, interval), + Descriptor::new(phone, phone_value, interval), + ], + ); + let reordered_retry = Record::new( + RecordId(0), + identity.clone(), + vec![ + Descriptor::new(phone, phone_value, interval), + Descriptor::new(email, email_value, interval), + ], + ); + let changed = Record::new( + RecordId(0), + identity, + vec![Descriptor::new(email, phone_value, interval)], + ); + + let (record_id, inserted) = store.add_record_if_absent(first).unwrap(); + assert!(inserted); + let (retry_id, inserted) = store + .add_record_if_absent(reordered_retry) + .expect("descriptor ordering must not change payload identity"); + assert_eq!(retry_id, record_id); + assert!(!inserted); + + let error = store + .add_record_if_absent(changed) + .expect_err("changed payload must not be silently discarded"); + assert!(error.to_string().contains("different payload")); + assert_eq!(store.len(), 1); + } + #[test] fn test_get_records_by_entity_type() { let mut store = Store::new(); diff --git a/src/transport_security.rs b/src/transport_security.rs new file mode 100644 index 0000000..2e5749a --- /dev/null +++ b/src/transport_security.rs @@ -0,0 +1,61 @@ +use std::fs; +use std::path::Path; + +use anyhow::Result; +use tonic::transport::{Certificate, ClientTlsConfig, Identity, ServerTlsConfig}; + +fn read_nonempty(path: &Path, description: &str) -> Result> { + let bytes = fs::read(path)?; + if bytes.is_empty() { + anyhow::bail!("{description} {} is empty", path.display()); + } + Ok(bytes) +} + +/// Load a server identity and require client certificates signed by `client_ca`. +pub fn load_server_mtls( + cert: Option<&Path>, + key: Option<&Path>, + client_ca: Option<&Path>, +) -> Result> { + match (cert, key, client_ca) { + (None, None, None) => Ok(None), + (Some(cert), Some(key), Some(client_ca)) => { + let cert = read_nonempty(cert, "server certificate")?; + let key = read_nonempty(key, "server private key")?; + let client_ca = read_nonempty(client_ca, "client CA certificate")?; + Ok(Some( + ServerTlsConfig::new() + .identity(Identity::from_pem(cert, key)) + .client_ca_root(Certificate::from_pem(client_ca)), + )) + } + _ => anyhow::bail!( + "mTLS server configuration requires certificate, private key, and client CA together" + ), + } +} + +/// Load a client identity and an explicit CA for server verification. +pub fn load_client_mtls( + ca: Option<&Path>, + cert: Option<&Path>, + key: Option<&Path>, +) -> Result> { + match (ca, cert, key) { + (None, None, None) => Ok(None), + (Some(ca), Some(cert), Some(key)) => { + let ca = read_nonempty(ca, "server CA certificate")?; + let cert = read_nonempty(cert, "client certificate")?; + let key = read_nonempty(key, "client private key")?; + Ok(Some( + ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(ca)) + .identity(Identity::from_pem(cert, key)), + )) + } + _ => anyhow::bail!( + "mTLS client configuration requires CA, certificate, and private key together" + ), + } +} diff --git a/src/utils.rs b/src/utils.rs index 6baa543..2c7b9d3 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -215,7 +215,9 @@ pub fn generate_graph_visualizations( ); } Err(_) => { - eprintln!("Warning: 'dot' command not found. Install Graphviz to generate PNG visualizations."); + eprintln!( + "Warning: 'dot' command not found. Install Graphviz to generate PNG visualizations." + ); } } @@ -236,7 +238,9 @@ pub fn generate_graph_visualizations( ); } Err(_) => { - eprintln!("Warning: 'dot' command not found. Install Graphviz to generate SVG visualizations."); + eprintln!( + "Warning: 'dot' command not found. Install Graphviz to generate SVG visualizations." + ); } } diff --git a/tests/distributed_apply_ontology.rs b/tests/distributed_apply_ontology.rs index 2bc3aee..ae9775e 100644 --- a/tests/distributed_apply_ontology.rs +++ b/tests/distributed_apply_ontology.rs @@ -1,8 +1,10 @@ use std::net::SocketAddr; +use tempfile::tempdir; use tokio::task::JoinHandle; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; +use tonic::Code; use unirust_rs::distributed::proto::{ self, router_service_client::RouterServiceClient, ApplyOntologyRequest, IngestRecordsRequest, QueryDescriptor, QueryEntitiesRequest, RecordDescriptor, RecordIdentity as ProtoRecordIdentity, @@ -19,12 +21,17 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -111,11 +118,12 @@ async fn apply_ontology_enables_queries_distributed() -> anyhow::Result<()> { client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record], }) .await?; - let response = client.query_entities(query).await?.into_inner(); + let response = client.query_entities(query.clone()).await?.into_inner(); match response.outcome { Some(proto::query_entities_response::Outcome::Matches(matches)) => { assert_eq!(matches.matches.len(), 1); @@ -123,5 +131,65 @@ async fn apply_ontology_enables_queries_distributed() -> anyhow::Result<()> { _ => anyhow::bail!("expected match after apply"), } + client + .set_ontology(ApplyOntologyRequest { + config: Some(support::to_proto_config(&config)), + }) + .await?; + + let response = client.query_entities(query.clone()).await?.into_inner(); + match response.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => { + assert_eq!( + matches.matches.len(), + 1, + "reapplying the active ontology must not reset records" + ); + } + _ => anyhow::bail!("expected match after reapplying the active ontology"), + } + + let mut replacement = config.clone(); + replacement + .strong_identifiers + .push("passport_number".to_string()); + let error = client + .set_ontology(ApplyOntologyRequest { + config: Some(support::to_proto_config(&replacement)), + }) + .await + .expect_err("replacing ontology on a nonempty cluster must fail"); + assert_eq!(error.code(), Code::FailedPrecondition); + + let response = client.query_entities(query).await?.into_inner(); + match response.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => { + assert_eq!( + matches.matches.len(), + 1, + "a rejected ontology replacement must preserve records" + ); + } + _ => anyhow::bail!("expected match after rejected ontology replacement"), + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn router_rejects_a_shard_with_a_different_restored_ontology() -> anyhow::Result<()> { + let (shard_addr, _shard_handle) = spawn_shard(0, support::build_iam_config()).await?; + let error = match RouterNode::connect( + vec![format!("http://{shard_addr}")], + DistributedOntologyConfig::empty(), + ) + .await + { + Ok(_) => anyhow::bail!("router unexpectedly accepted a mismatched shard ontology"), + Err(error) => error, + }; + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("ontology mismatch")); Ok(()) } diff --git a/tests/distributed_conflicts_presets.rs b/tests/distributed_conflicts_presets.rs index db76673..6aa89d9 100644 --- a/tests/distributed_conflicts_presets.rs +++ b/tests/distributed_conflicts_presets.rs @@ -53,12 +53,18 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), - )?; + Some(data_dir.path().to_path_buf()), + false, + None, + )? + .with_destructive_admin(true); let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -266,6 +272,7 @@ async fn distributed_conflict_presets_match_local() -> anyhow::Result<()> { if !inputs.is_empty() { client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: inputs.clone(), }) .await diff --git a/tests/distributed_cross_shard_reconciliation.rs b/tests/distributed_cross_shard_reconciliation.rs index a5f9a1d..2ac957f 100644 --- a/tests/distributed_cross_shard_reconciliation.rs +++ b/tests/distributed_cross_shard_reconciliation.rs @@ -10,6 +10,7 @@ use std::net::SocketAddr; +use tempfile::tempdir; use tokio::task::JoinHandle; use tokio::time::{sleep, Duration}; use tokio_stream::wrappers::TcpListenerStream; @@ -26,18 +27,27 @@ use unirust_rs::distributed::{ }; use unirust_rs::{StreamingTuning, TuningProfile}; +// Each scenario opens several RocksDB instances. Serialize this integration file +// so the test process does not exceed its file-descriptor limit. +static PERSISTENT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + async fn spawn_shard( shard_id: u32, config: DistributedOntologyConfig, ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -159,6 +169,7 @@ fn build_instrument_config() -> DistributedOntologyConfig { /// key to trigger a merge, which activates boundary tracking. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cross_shard_conflict_detected_via_reconcile() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -209,6 +220,7 @@ async fn cross_shard_conflict_detected_via_reconcile() -> anyhow::Result<()> { shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![shard0_rec1, shard0_rec2], }) .await?; @@ -239,6 +251,7 @@ async fn cross_shard_conflict_detected_via_reconcile() -> anyhow::Result<()> { shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![shard1_rec1, shard1_rec2], }) .await?; @@ -300,6 +313,7 @@ async fn cross_shard_conflict_detected_via_reconcile() -> anyhow::Result<()> { /// We ingest directly to shards to force cross-shard scenario. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cross_shard_merge_succeeds_without_conflict() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -320,7 +334,7 @@ async fn cross_shard_merge_succeeds_without_conflict() -> anyhow::Result<()> { sleep(Duration::from_millis(100)).await; - // Shard 0: Two records with same identity key, from "msci" perspective + // A singleton on each shard must still participate in distributed resolution. let shard0_rec1 = record_input( 0, "instrument", @@ -332,25 +346,14 @@ async fn cross_shard_merge_succeeds_without_conflict() -> anyhow::Result<()> { ("ts_code", "TS_MSCI", 0, 100), ], ); - let shard0_rec2 = record_input( - 1, - "instrument", - "msci", - "msci_001b", - vec![ - ("isin", "ISIN002", 0, 100), - ("country", "UK", 0, 100), - ("ts_code", "TS_MSCI", 0, 100), - ], - ); - shard0_client .ingest_records(IngestRecordsRequest { - records: vec![shard0_rec1, shard0_rec2], + internal_protocol_version: 3, + records: vec![shard0_rec1], }) .await?; - // Shard 1: Two records with same identity key, from "axioma" perspective + // Shard 1: Same identity key from a different perspective. // Different perspective means no strong ID conflict let shard1_rec1 = record_input( 2, @@ -363,21 +366,10 @@ async fn cross_shard_merge_succeeds_without_conflict() -> anyhow::Result<()> { ("axioma_id", "AX001", 0, 100), ], ); - let shard1_rec2 = record_input( - 3, - "instrument", - "axioma", - "axioma_001b", - vec![ - ("isin", "ISIN002", 0, 100), - ("country", "UK", 0, 100), - ("axioma_id", "AX001", 0, 100), - ], - ); - shard1_client .ingest_records(IngestRecordsRequest { - records: vec![shard1_rec1, shard1_rec2], + internal_protocol_version: 3, + records: vec![shard1_rec1], }) .await?; @@ -413,6 +405,30 @@ async fn cross_shard_merge_succeeds_without_conflict() -> anyhow::Result<()> { reconcile_response.conflicts_blocked ); + let query = router_client + .query_entities(proto::QueryEntitiesRequest { + descriptors: vec![ + proto::QueryDescriptor { + attr: "isin".to_string(), + value: "ISIN002".to_string(), + }, + proto::QueryDescriptor { + attr: "country".to_string(), + value: "UK".to_string(), + }, + ], + start: 0, + end: 100, + }) + .await? + .into_inner(); + let matches = match query.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => matches.matches, + other => anyhow::bail!("expected one reconciled global entity, got {other:?}"), + }; + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].shard_id, 0); + Ok(()) } @@ -420,6 +436,7 @@ async fn cross_shard_merge_succeeds_without_conflict() -> anyhow::Result<()> { /// We ingest directly to shards to force cross-shard scenario. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cross_shard_conflicts_propagated_to_shards() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -466,6 +483,7 @@ async fn cross_shard_conflicts_propagated_to_shards() -> anyhow::Result<()> { shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![shard0_rec1, shard0_rec2], }) .await?; @@ -496,6 +514,7 @@ async fn cross_shard_conflicts_propagated_to_shards() -> anyhow::Result<()> { shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![shard1_rec1, shard1_rec2], }) .await?; @@ -544,10 +563,193 @@ async fn cross_shard_conflicts_propagated_to_shards() -> anyhow::Result<()> { Ok(()) } -/// Test that boundary metadata includes perspective_strong_ids. -/// This directly tests the serialization fix. +/// Different strong-ID values are compatible when their validity intervals do not overlap. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cross_shard_merge_respects_strong_id_validity_intervals() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let config = build_instrument_config(); + let empty_config = DistributedOntologyConfig::empty(); + + let (shard0_addr, _shard0_handle) = spawn_shard(0, empty_config.clone()).await?; + let (shard1_addr, _shard1_handle) = spawn_shard(1, empty_config.clone()).await?; + let (router_addr, _router_handle) = + spawn_router(vec![shard0_addr, shard1_addr], empty_config).await?; + + let mut router_client = RouterServiceClient::connect(format!("http://{}", router_addr)).await?; + let mut shard0_client = ShardServiceClient::connect(format!("http://{}", shard0_addr)).await?; + let mut shard1_client = ShardServiceClient::connect(format!("http://{}", shard1_addr)).await?; + router_client + .set_ontology(ApplyOntologyRequest { + config: Some(to_proto_config(&config)), + }) + .await?; + + shard0_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 0, + "instrument", + "msci", + "historical", + vec![ + ("isin", "ISIN_TEMPORAL", 0, 30), + ("country", "US", 0, 30), + ("ts_code", "TS_OLD", 0, 10), + ], + )], + }) + .await?; + shard1_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 1, + "instrument", + "msci", + "current", + vec![ + ("isin", "ISIN_TEMPORAL", 0, 30), + ("country", "US", 0, 30), + ("ts_code", "TS_NEW", 10, 20), + ], + )], + }) + .await?; + + let reconcile = router_client + .reconcile(proto::ReconcileRequest { + shard_metadata: vec![], + }) + .await? + .into_inner(); + assert_eq!(reconcile.conflicts_blocked, 0); + assert_eq!(reconcile.merges_performed, 1); + + let query = router_client + .query_entities(proto::QueryEntitiesRequest { + descriptors: vec![ + proto::QueryDescriptor { + attr: "isin".to_string(), + value: "ISIN_TEMPORAL".to_string(), + }, + proto::QueryDescriptor { + attr: "country".to_string(), + value: "US".to_string(), + }, + ], + start: 0, + end: 30, + }) + .await? + .into_inner(); + let matches = match query.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => matches.matches, + other => anyhow::bail!("expected one reconciled temporal entity, got {other:?}"), + }; + assert_eq!(matches.len(), 1); + + Ok(()) +} + +/// Disjoint observations of one identity key must not be widened across the temporal gap. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cross_shard_reconciliation_preserves_identity_key_gaps() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let config = DistributedOntologyConfig { + identity_keys: vec![ + IdentityKeyConfig { + name: "account".to_string(), + attributes: vec!["account".to_string()], + }, + IdentityKeyConfig { + name: "bridge".to_string(), + attributes: vec!["bridge".to_string()], + }, + ], + strong_identifiers: Vec::new(), + constraints: Vec::new(), + }; + let empty_config = DistributedOntologyConfig::empty(); + let (shard0_addr, _shard0_handle) = spawn_shard(0, empty_config.clone()).await?; + let (shard1_addr, _shard1_handle) = spawn_shard(1, empty_config.clone()).await?; + let (router_addr, _router_handle) = + spawn_router(vec![shard0_addr, shard1_addr], empty_config).await?; + + let mut router_client = RouterServiceClient::connect(format!("http://{}", router_addr)).await?; + let mut shard0_client = ShardServiceClient::connect(format!("http://{}", shard0_addr)).await?; + let mut shard1_client = ShardServiceClient::connect(format!("http://{}", shard1_addr)).await?; + router_client + .set_ontology(ApplyOntologyRequest { + config: Some(to_proto_config(&config)), + }) + .await?; + + // These two records form one local cluster through `bridge`, but `account=A` + // is valid only before and after the middle gap. + let shard0_ingest = shard0_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![ + record_input( + 0, + "person", + "crm", + "before", + vec![("account", "A", 0, 10), ("bridge", "B", 0, 30)], + ), + record_input( + 1, + "person", + "crm", + "after", + vec![("account", "A", 20, 30), ("bridge", "B", 0, 30)], + ), + ], + }) + .await? + .into_inner(); + assert_eq!( + shard0_ingest.assignments[0].cluster_id, shard0_ingest.assignments[1].cluster_id, + "bridge identity key must form one local cluster" + ); + shard1_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 2, + "person", + "crm", + "middle", + vec![("account", "A", 10, 20)], + )], + }) + .await?; + + let reconcile = router_client + .reconcile(proto::ReconcileRequest { + shard_metadata: vec![], + }) + .await? + .into_inner(); + assert_eq!(reconcile.merges_performed, 0); + + let stats = router_client + .get_stats(proto::StatsRequest {}) + .await? + .into_inner(); + assert_eq!( + stats.cross_shard_merges, 0, + "adaptive reconciliation must not merge through the temporal gap" + ); + + Ok(()) +} + +/// Test that boundary metadata includes exact temporal strong-ID observations. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn boundary_metadata_includes_perspective_strong_ids() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -595,6 +797,7 @@ async fn boundary_metadata_includes_perspective_strong_ids() -> anyhow::Result<( router_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record1, record2], }) .await?; @@ -607,28 +810,32 @@ async fn boundary_metadata_includes_perspective_strong_ids() -> anyhow::Result<( let metadata = metadata_response.metadata.expect("metadata should exist"); - // Find entries and check for perspective_strong_ids - let has_strong_ids = metadata.entries.iter().any(|key_entries| { - key_entries - .entries - .iter() - .any(|entry| !entry.perspective_strong_ids.is_empty()) - }); + let exact_strong_ids = metadata + .entries + .iter() + .flat_map(|key_entries| &key_entries.entries) + .flat_map(|entry| &entry.strong_ids) + .collect::>(); assert!( - has_strong_ids, - "Expected boundary entries to have perspective_strong_ids populated. \ - This indicates the proto serialization is working correctly. \ - Metadata entries: {:?}", + exact_strong_ids.iter().any(|strong_id| { + strong_id.perspective == "msci" + && strong_id.attribute == "ts_code" + && strong_id.value == "TS_DE_001" + && strong_id.interval_start == 0 + && strong_id.interval_end == 100 + }), + "expected exact temporal strong-ID metadata; entries: {:?}", metadata.entries ); Ok(()) } -/// Test dirty boundary keys also include perspective_strong_ids. +/// Test dirty boundary keys also include exact temporal strong-ID observations. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn dirty_boundary_keys_include_perspective_strong_ids() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -675,6 +882,7 @@ async fn dirty_boundary_keys_include_perspective_strong_ids() -> anyhow::Result< router_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record1, record2], }) .await?; @@ -685,18 +893,22 @@ async fn dirty_boundary_keys_include_perspective_strong_ids() -> anyhow::Result< .await? .into_inner(); - // Check if any dirty key entries have perspective_strong_ids - let has_strong_ids = dirty_keys_response.dirty_keys.iter().any(|dirty_key| { - dirty_key - .entries - .iter() - .any(|entry| !entry.perspective_strong_ids.is_empty()) - }); + let exact_strong_ids = dirty_keys_response + .dirty_keys + .iter() + .flat_map(|dirty_key| &dirty_key.entries) + .flat_map(|entry| &entry.strong_ids) + .collect::>(); assert!( - has_strong_ids, - "Expected dirty boundary key entries to have perspective_strong_ids populated. \ - Dirty keys: {:?}", + exact_strong_ids.iter().any(|strong_id| { + strong_id.perspective == "bloomberg" + && strong_id.attribute == "ts_code" + && strong_id.value == "TS_FR_001" + && strong_id.interval_start == 0 + && strong_id.interval_end == 100 + }), + "expected exact temporal strong-ID dirty-key metadata; dirty keys: {:?}", dirty_keys_response.dirty_keys ); @@ -747,6 +959,7 @@ fn build_multi_key_config() -> DistributedOntologyConfig { /// This tests whether reconciliation can detect conflicts through transitive relationships. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn transitive_cross_shard_conflict_detected() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_multi_key_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -801,6 +1014,7 @@ async fn transitive_cross_shard_conflict_detected() -> anyhow::Result<()> { ); shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![a1, a2], }) .await?; @@ -832,6 +1046,7 @@ async fn transitive_cross_shard_conflict_detected() -> anyhow::Result<()> { ); shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![b1, b2], }) .await?; @@ -861,6 +1076,7 @@ async fn transitive_cross_shard_conflict_detected() -> anyhow::Result<()> { ); shard2_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![c1, c2], }) .await?; @@ -927,6 +1143,7 @@ async fn transitive_cross_shard_conflict_detected() -> anyhow::Result<()> { /// Sharded: Each shard thinks its entity legitimately owns the identifier. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn peic_many_entities_claim_same_identifier_across_shards() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; // Use UNIQUE constraint (not UniqueWithinPerspective) for global uniqueness let config = DistributedOntologyConfig { identity_keys: vec![IdentityKeyConfig { @@ -984,6 +1201,7 @@ async fn peic_many_entities_claim_same_identifier_across_shards() -> anyhow::Res ); shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![a1, a2], }) .await?; @@ -1014,6 +1232,7 @@ async fn peic_many_entities_claim_same_identifier_across_shards() -> anyhow::Res ); shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![b1, b2], }) .await?; @@ -1064,6 +1283,7 @@ async fn peic_many_entities_claim_same_identifier_across_shards() -> anyhow::Res /// Sharded: Each shard only knows about its own time ranges. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn temporal_overlap_conflict_across_shards() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -1109,6 +1329,7 @@ async fn temporal_overlap_conflict_across_shards() -> anyhow::Result<()> { ); shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![a1, a2], }) .await?; @@ -1138,6 +1359,7 @@ async fn temporal_overlap_conflict_across_shards() -> anyhow::Result<()> { ); shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![b1, b2], }) .await?; @@ -1185,6 +1407,7 @@ async fn temporal_overlap_conflict_across_shards() -> anyhow::Result<()> { /// Sharded: Requires re-reconciliation to detect the new conflict. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn late_arriving_data_triggers_conflict() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -1230,6 +1453,7 @@ async fn late_arriving_data_triggers_conflict() -> anyhow::Result<()> { ); shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![initial1, initial2], }) .await?; @@ -1274,6 +1498,7 @@ async fn late_arriving_data_triggers_conflict() -> anyhow::Result<()> { ); shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![late1, late2], }) .await?; @@ -1323,6 +1548,7 @@ async fn late_arriving_data_triggers_conflict() -> anyhow::Result<()> { /// Sharded: Requires multiple reconciliation rounds to propagate the chain. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn multi_hop_chain_conflict_across_four_shards() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; // Ontology with 3 identity keys for the chain let config = DistributedOntologyConfig { identity_keys: vec![ @@ -1396,6 +1622,7 @@ async fn multi_hop_chain_conflict_across_four_shards() -> anyhow::Result<()> { ); shard0 .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![a1, a2], }) .await?; @@ -1425,6 +1652,7 @@ async fn multi_hop_chain_conflict_across_four_shards() -> anyhow::Result<()> { ); shard1 .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![b1, b2], }) .await?; @@ -1454,6 +1682,7 @@ async fn multi_hop_chain_conflict_across_four_shards() -> anyhow::Result<()> { ); shard2 .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![c1, c2], }) .await?; @@ -1478,6 +1707,7 @@ async fn multi_hop_chain_conflict_across_four_shards() -> anyhow::Result<()> { ); shard3 .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![d1, d2], }) .await?; @@ -1534,6 +1764,7 @@ async fn multi_hop_chain_conflict_across_four_shards() -> anyhow::Result<()> { /// have different values for the same attribute. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn different_perspectives_no_false_positive_conflict() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_instrument_config(); // Uses UniqueWithinPerspective let empty_config = DistributedOntologyConfig::empty(); @@ -1579,6 +1810,7 @@ async fn different_perspectives_no_false_positive_conflict() -> anyhow::Result<( ); shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![msci1, msci2], }) .await?; @@ -1608,6 +1840,7 @@ async fn different_perspectives_no_false_positive_conflict() -> anyhow::Result<( ); shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![axioma1, axioma2], }) .await?; diff --git a/tests/distributed_cross_shard_reconciliation_persistent.rs b/tests/distributed_cross_shard_reconciliation_persistent.rs index 9c0fb97..ea3d9c6 100644 --- a/tests/distributed_cross_shard_reconciliation_persistent.rs +++ b/tests/distributed_cross_shard_reconciliation_persistent.rs @@ -2,29 +2,125 @@ use std::net::SocketAddr; use std::path::PathBuf; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; use tempfile::TempDir; use tokio::task::JoinHandle; use tokio::time::{sleep, Duration}; use tokio_stream::wrappers::TcpListenerStream; +use tonic::codegen::{http, Service}; +use tonic::server::NamedService; use tonic::transport::Server; use unirust_rs::distributed::proto::{ self, router_service_client::RouterServiceClient, shard_service_client::ShardServiceClient, ApplyOntologyRequest, IngestRecordsRequest, RecordDescriptor, RecordIdentity, RecordInput, }; use unirust_rs::distributed::{ - DistributedOntologyConfig, IdentityKeyConfig, RouterNode, ShardNode, + hash_record_to_shard, hash_source_identity_to_shard, DistributedOntologyConfig, + IdentityKeyConfig, RouterNode, ShardNode, DISTRIBUTED_PROTOCOL_VERSION, }; use unirust_rs::{StreamingTuning, TuningProfile}; mod support; +// Each case opens multiple RocksDB instances with many column families. Running +// them concurrently exceeds the default macOS per-process file descriptor limit. +static PERSISTENT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + struct SpawnedShard { addr: SocketAddr, _handle: JoinHandle<()>, _data_dir: TempDir, } +#[derive(Clone)] +struct FailIngest { + inner: S, +} + +impl Service> for FailIngest +where + S: Service< + http::Request, + Response = http::Response, + Error = std::convert::Infallible, + > + Send, + S::Future: Send + 'static, + B: Send + 'static, +{ + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + if request.uri().path() == "/unirust.ShardService/IngestRecords" { + return Box::pin(async { + Ok(tonic::Status::unavailable("injected target ingest failure").into_http()) + }); + } + Box::pin(self.inner.call(request)) + } +} + +impl NamedService for FailIngest +where + S: NamedService, +{ + const NAME: &'static str = S::NAME; +} + +#[derive(Clone)] +struct FailFirstApplyMerge { + inner: S, + fail_next: Arc, +} + +impl Service> for FailFirstApplyMerge +where + S: Service< + http::Request, + Response = http::Response, + Error = std::convert::Infallible, + > + Send, + S::Future: Send + 'static, + B: Send + 'static, +{ + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + if request.uri().path() == "/unirust.ShardService/ApplyMerge" + && self.fail_next.swap(false, Ordering::AcqRel) + { + return Box::pin(async { + Ok(tonic::Status::unavailable("injected ApplyMerge failure").into_http()) + }); + } + Box::pin(self.inner.call(request)) + } +} + +impl NamedService for FailFirstApplyMerge +where + S: NamedService, +{ + const NAME: &'static str = S::NAME; +} + async fn spawn_shard_persistent( shard_id: u32, config: DistributedOntologyConfig, @@ -57,6 +153,91 @@ async fn spawn_shard_persistent( }) } +async fn spawn_shard_at( + shard_id: u32, + config: DistributedOntologyConfig, + data_path: PathBuf, +) -> anyhow::Result<(SocketAddr, ShardNode, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let shard = ShardNode::new_with_data_dir( + shard_id, + config, + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_path), + false, + None, + )?; + let service_shard = shard.clone(); + let handle = tokio::spawn(async move { + Server::builder() + .add_service(proto::shard_service_server::ShardServiceServer::new( + service_shard, + )) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, shard, handle)) +} + +async fn spawn_apply_merge_failing_shard_at( + shard_id: u32, + config: DistributedOntologyConfig, + data_path: PathBuf, +) -> anyhow::Result<(SocketAddr, ShardNode, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let shard = ShardNode::new_with_data_dir( + shard_id, + config, + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_path), + false, + None, + )?; + let service = proto::shard_service_server::ShardServiceServer::new(shard.clone()); + let service = FailFirstApplyMerge { + inner: service, + fail_next: Arc::new(AtomicBool::new(true)), + }; + let handle = tokio::spawn(async move { + Server::builder() + .add_service(service) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, shard, handle)) +} + +async fn spawn_ingest_failing_shard_at( + shard_id: u32, + config: DistributedOntologyConfig, + data_path: PathBuf, +) -> anyhow::Result<(SocketAddr, ShardNode, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let shard = ShardNode::new_with_data_dir( + shard_id, + config, + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_path), + false, + None, + )?; + let service_shard = shard.clone(); + let service = proto::shard_service_server::ShardServiceServer::new(service_shard); + let handle = tokio::spawn(async move { + Server::builder() + .add_service(FailIngest { inner: service }) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, shard, handle)) +} + async fn spawn_router( shard_addrs: Vec, config: DistributedOntologyConfig, @@ -123,6 +304,7 @@ fn build_email_config() -> DistributedOntologyConfig { /// Ensure cross-shard merges occur with persistent stores. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cross_shard_merge_persistent_store() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = build_email_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -165,6 +347,7 @@ async fn cross_shard_merge_persistent_store() -> anyhow::Result<()> { ); shard0_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![shard0_rec1, shard0_rec2], }) .await?; @@ -191,6 +374,7 @@ async fn cross_shard_merge_persistent_store() -> anyhow::Result<()> { ); shard1_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![shard1_rec1, shard1_rec2], }) .await?; @@ -210,3 +394,667 @@ async fn cross_shard_merge_persistent_store() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn three_shard_singleton_merge_survives_full_restart() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let root = tempfile::tempdir()?; + let config = build_email_config(); + let data_paths = (0..3) + .map(|shard_id| root.path().join(format!("shard-{shard_id}"))) + .collect::>(); + for path in &data_paths { + std::fs::create_dir_all(path)?; + } + + let mut shard_addrs = Vec::new(); + let mut shard_nodes = Vec::new(); + let mut shard_handles = Vec::new(); + for (shard_id, path) in data_paths.iter().enumerate() { + let (addr, shard, handle) = + spawn_shard_at(shard_id as u32, config.clone(), path.clone()).await?; + shard_addrs.push(addr); + shard_nodes.push(shard); + shard_handles.push(handle); + } + let (router_addr, router_handle) = spawn_router(shard_addrs.clone(), config.clone()).await?; + let mut router_client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + + for (shard_id, shard_addr) in shard_addrs.iter().enumerate() { + let mut shard_client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + shard_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + shard_id as u32, + "person", + "hr", + &format!("person-{shard_id}"), + vec![ + ("email", "restart@example.com", 0, 100), + ("ssn", "1234", 0, 100), + ], + )], + }) + .await?; + } + + let reconciliation = router_client + .reconcile(proto::ReconcileRequest { + shard_metadata: Vec::new(), + }) + .await? + .into_inner(); + assert_eq!(reconciliation.merges_performed, 2); + assert_eq!(query_match_count(&mut router_client).await?, 1); + + drop(router_client); + router_handle.abort(); + let _ = router_handle.await; + for shard in &shard_nodes { + shard.shutdown().await?; + } + for handle in shard_handles { + handle.abort(); + let _ = handle.await; + } + drop(shard_nodes); + sleep(Duration::from_millis(100)).await; + + let mut restarted_addrs = Vec::new(); + let mut _restarted_nodes = Vec::new(); + let mut _restarted_handles = Vec::new(); + for (shard_id, path) in data_paths.iter().enumerate() { + let (addr, shard, handle) = + spawn_shard_at(shard_id as u32, config.clone(), path.clone()).await?; + restarted_addrs.push(addr); + _restarted_nodes.push(shard); + _restarted_handles.push(handle); + } + let (router_addr, _router_handle) = spawn_router(restarted_addrs, config).await?; + let mut router_client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + assert_eq!(query_match_count(&mut router_client).await?, 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn source_identity_reservation_survives_routing_change_and_restart() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let root = tempfile::tempdir()?; + let config = build_email_config(); + let data_paths = (0..2) + .map(|shard_id| root.path().join(format!("source-reservation-{shard_id}"))) + .collect::>(); + for path in &data_paths { + std::fs::create_dir_all(path)?; + } + + let mut original = record_input( + 0, + "person", + "crm", + "immutable-source", + vec![("email", "route-original@example.com", 0, 100)], + ); + let original_target = hash_record_to_shard(&config, &original, 2); + let mut changed = None; + for candidate in 0..1_000 { + let record = record_input( + 0, + "person", + "crm", + "immutable-source", + vec![( + "email", + &format!("route-changed-{candidate}@example.com"), + 0, + 100, + )], + ); + if hash_record_to_shard(&config, &record, 2) != original_target { + changed = Some(record); + break; + } + } + let changed = changed.expect("test must find a payload routed to the other shard"); + + let mut shard_nodes = Vec::new(); + let mut shard_handles = Vec::new(); + let mut shard_addrs = Vec::new(); + for (shard_id, path) in data_paths.iter().enumerate() { + let (addr, shard, handle) = + spawn_shard_at(shard_id as u32, config.clone(), path.clone()).await?; + shard_addrs.push(addr); + shard_nodes.push(shard); + shard_handles.push(handle); + } + let (router_addr, router_handle) = spawn_router(shard_addrs.clone(), config.clone()).await?; + let mut router_client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + + let response = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![original.clone()], + }) + .await? + .into_inner(); + assert_eq!(response.assignments.len(), 1); + assert_eq!(response.assignments[0].shard_id as usize, original_target); + + let error = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![changed.clone()], + }) + .await + .expect_err("a routing change must not evade immutable source identity"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + assert_eq!(cluster_record_count(&shard_addrs).await?, 1); + + drop(router_client); + router_handle.abort(); + let _ = router_handle.await; + for shard in &shard_nodes { + shard.shutdown().await?; + } + for handle in shard_handles { + handle.abort(); + let _ = handle.await; + } + drop(shard_nodes); + sleep(Duration::from_millis(100)).await; + + let mut restarted_nodes = Vec::new(); + let mut restarted_handles = Vec::new(); + let mut restarted_addrs = Vec::new(); + for (shard_id, path) in data_paths.iter().enumerate() { + let (addr, shard, handle) = + spawn_shard_at(shard_id as u32, config.clone(), path.clone()).await?; + restarted_addrs.push(addr); + restarted_nodes.push(shard); + restarted_handles.push(handle); + } + let (router_addr, _router_handle) = + spawn_router(restarted_addrs.clone(), config.clone()).await?; + let mut router_client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + + let error = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![changed], + }) + .await + .expect_err("the source reservation must survive a full cluster restart"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + + original.index = 1; + let retry = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![original], + }) + .await? + .into_inner(); + assert_eq!(retry.assignments.len(), 1); + assert_eq!(retry.assignments[0].shard_id as usize, original_target); + assert_eq!(cluster_record_count(&restarted_addrs).await?, 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn router_backfills_legacy_records_before_serving_ingest() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let root = tempfile::tempdir()?; + let config = build_email_config(); + let data_paths = (0..2) + .map(|shard_id| root.path().join(format!("legacy-reservation-{shard_id}"))) + .collect::>(); + for path in &data_paths { + std::fs::create_dir_all(path)?; + } + + let original = record_input( + 0, + "person", + "crm", + "legacy-source", + vec![("email", "legacy-original@example.com", 0, 100)], + ); + let original_target = hash_record_to_shard(&config, &original, 2); + let mut changed = None; + for candidate in 0..1_000 { + let record = record_input( + 0, + "person", + "crm", + "legacy-source", + vec![( + "email", + &format!("legacy-changed-{candidate}@example.com"), + 0, + 100, + )], + ); + if hash_record_to_shard(&config, &record, 2) != original_target { + changed = Some(record); + break; + } + } + let changed = changed.expect("test must find a changed payload routed elsewhere"); + + let mut shard_addrs = Vec::new(); + let mut _shard_nodes = Vec::new(); + let mut _shard_handles = Vec::new(); + for (shard_id, path) in data_paths.iter().enumerate() { + let (addr, shard, handle) = + spawn_shard_at(shard_id as u32, config.clone(), path.clone()).await?; + shard_addrs.push(addr); + _shard_nodes.push(shard); + _shard_handles.push(handle); + } + + let mut original_target_client = + ShardServiceClient::connect(format!("http://{}", shard_addrs[original_target])).await?; + let status_before = original_target_client + .get_config_version(proto::ConfigVersionRequest {}) + .await? + .into_inner(); + assert_eq!(status_before.source_reservation_backfill_version, 0); + original_target_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![original], + }) + .await?; + + let (router_addr, _router_handle) = spawn_router(shard_addrs.clone(), config.clone()).await?; + for shard_addr in &shard_addrs { + let mut shard_client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + let status = shard_client + .get_config_version(proto::ConfigVersionRequest {}) + .await? + .into_inner(); + assert_eq!( + status.source_reservation_backfill_version, + DISTRIBUTED_PROTOCOL_VERSION + ); + assert_eq!(status.source_reservation_shard_count, 2); + } + + let mut router_client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + let error = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![changed], + }) + .await + .expect_err("backfilled legacy source identity must reject a changed payload"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + assert_eq!(cluster_record_count(&shard_addrs).await?, 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reserved_ingest_retries_after_target_failure_and_full_restart() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let root = tempfile::tempdir()?; + let config = build_email_config(); + let data_paths = (0..2) + .map(|shard_id| root.path().join(format!("partial-reservation-{shard_id}"))) + .collect::>(); + for path in &data_paths { + std::fs::create_dir_all(path)?; + } + + let mut selected = None; + for candidate in 0..1_000 { + let record = record_input( + 0, + "person", + "crm", + &format!("partial-source-{candidate}"), + vec![( + "email", + &format!("partial-route-{candidate}@example.com"), + 0, + 100, + )], + ); + let identity = record.identity.as_ref().unwrap(); + let owner = hash_source_identity_to_shard(identity, 2); + let target = hash_record_to_shard(&config, &record, 2); + if owner != target { + selected = Some((record, owner, target)); + break; + } + } + let (mut record, _owner, target) = + selected.expect("test must find distinct reservation owner and ingest target"); + + let mut shard_nodes = Vec::new(); + let mut shard_handles = Vec::new(); + let mut shard_addrs = Vec::new(); + for (shard_id, path) in data_paths.iter().enumerate() { + let (addr, shard, handle) = if shard_id == target { + spawn_ingest_failing_shard_at(shard_id as u32, config.clone(), path.clone()).await? + } else { + spawn_shard_at(shard_id as u32, config.clone(), path.clone()).await? + }; + shard_addrs.push(addr); + shard_nodes.push(shard); + shard_handles.push(handle); + } + let (router_addr, router_handle) = spawn_router(shard_addrs, config.clone()).await?; + let mut router_client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + + let error = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record.clone()], + }) + .await + .expect_err("ingest must fail while its target shard is unavailable"); + assert_eq!(error.code(), tonic::Code::Unavailable); + + drop(router_client); + router_handle.abort(); + let _ = router_handle.await; + for shard in &shard_nodes { + shard.shutdown().await?; + } + for handle in shard_handles { + handle.abort(); + let _ = handle.await; + } + drop(shard_nodes); + sleep(Duration::from_millis(100)).await; + + let mut restarted_nodes = Vec::new(); + let mut restarted_handles = Vec::new(); + let mut restarted_addrs = Vec::new(); + for (shard_id, path) in data_paths.iter().enumerate() { + let (addr, shard, handle) = + spawn_shard_at(shard_id as u32, config.clone(), path.clone()).await?; + restarted_addrs.push(addr); + restarted_nodes.push(shard); + restarted_handles.push(handle); + } + let (router_addr, _router_handle) = + spawn_router(restarted_addrs.clone(), config.clone()).await?; + let mut router_client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + assert_eq!(cluster_record_count(&restarted_addrs).await?, 0); + + record.index = 1; + let retry = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record.clone()], + }) + .await? + .into_inner(); + assert_eq!(retry.assignments.len(), 1); + assert_eq!(retry.assignments[0].shard_id as usize, target); + assert_eq!(cluster_record_count(&restarted_addrs).await?, 1); + + let mut changed = record; + changed.descriptors[0].value = "changed-after-partial-failure@example.com".to_string(); + let error = router_client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![changed], + }) + .await + .expect_err("the durable pre-ingest reservation must reject a changed payload"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn partial_reconciliation_blocks_traffic_and_recovers_after_full_restart( +) -> anyhow::Result<()> { + use proto::router_service_server::RouterService; + + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let root = tempfile::tempdir()?; + let config = build_email_config(); + let path0 = root.path().join("reconcile-shard-0"); + let path1 = root.path().join("reconcile-shard-1"); + let (addr0, shard0, handle0) = spawn_shard_at(0, config.clone(), path0.clone()).await?; + let (addr1, shard1, handle1) = + spawn_apply_merge_failing_shard_at(1, config.clone(), path1.clone()).await?; + let router = RouterNode::connect( + vec![format!("http://{addr0}"), format!("http://{addr1}")], + config.clone(), + ) + .await?; + + let mut client0 = ShardServiceClient::connect(format!("http://{addr0}")).await?; + let mut client1 = ShardServiceClient::connect(format!("http://{addr1}")).await?; + client0 + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 0, + "person", + "hr", + "partial-merge-0", + vec![ + ("email", "partial-merge@example.com", 0, 100), + ("ssn", "1234", 0, 100), + ], + )], + }) + .await?; + client1 + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 1, + "person", + "hr", + "partial-merge-1", + vec![ + ("email", "partial-merge@example.com", 0, 100), + ("ssn", "1234", 0, 100), + ], + )], + }) + .await?; + + let error = RouterService::reconcile( + router.as_ref(), + tonic::Request::new(proto::ReconcileRequest { + shard_metadata: Vec::new(), + }), + ) + .await + .expect_err("second shard must fail the first merge application"); + assert_eq!(error.code(), tonic::Code::Unavailable); + + let blocked = RouterService::ingest_records( + router.as_ref(), + tonic::Request::new(IngestRecordsRequest { + internal_protocol_version: 0, + records: vec![record_input( + 2, + "person", + "hr", + "blocked-during-partial-merge", + vec![("email", "blocked@example.com", 0, 100)], + )], + }), + ) + .await + .expect_err("traffic must fail closed after a partial reconciliation apply"); + assert_eq!(blocked.code(), tonic::Code::FailedPrecondition); + + drop(client0); + drop(client1); + drop(router); + shard0.shutdown().await?; + shard1.shutdown().await?; + handle0.abort(); + handle1.abort(); + let _ = handle0.await; + let _ = handle1.await; + drop(shard0); + drop(shard1); + sleep(Duration::from_millis(100)).await; + + let (restarted_addr0, restarted_shard0, restarted_handle0) = + spawn_shard_at(0, config.clone(), path0).await?; + let (restarted_addr1, restarted_shard1, restarted_handle1) = + spawn_shard_at(1, config.clone(), path1).await?; + let restarted_router = RouterNode::connect( + vec![ + format!("http://{restarted_addr0}"), + format!("http://{restarted_addr1}"), + ], + config, + ) + .await?; + let query = RouterService::query_entities( + restarted_router.as_ref(), + tonic::Request::new(proto::QueryEntitiesRequest { + descriptors: vec![proto::QueryDescriptor { + attr: "email".to_string(), + value: "partial-merge@example.com".to_string(), + }], + start: 0, + end: 100, + }), + ) + .await? + .into_inner(); + match query.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => { + assert_eq!(matches.matches.len(), 1); + } + other => anyhow::bail!("expected one reconciled entity after restart, got {other:?}"), + } + + drop(restarted_router); + restarted_shard0.shutdown().await?; + restarted_shard1.shutdown().await?; + restarted_handle0.abort(); + restarted_handle1.abort(); + let _ = restarted_handle0.await; + let _ = restarted_handle1.await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn partial_reconciliation_can_be_retried_in_place() -> anyhow::Result<()> { + use proto::router_service_server::RouterService; + + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let root = tempfile::tempdir()?; + let config = build_email_config(); + let (addr0, shard0, handle0) = + spawn_shard_at(0, config.clone(), root.path().join("retry-shard-0")).await?; + let (addr1, shard1, handle1) = + spawn_apply_merge_failing_shard_at(1, config.clone(), root.path().join("retry-shard-1")) + .await?; + let router = RouterNode::connect( + vec![format!("http://{addr0}"), format!("http://{addr1}")], + config, + ) + .await?; + for (shard_addr, uid) in [(addr0, "retry-merge-0"), (addr1, "retry-merge-1")] { + let mut client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 0, + "person", + "hr", + uid, + vec![ + ("email", "retry-merge@example.com", 0, 100), + ("ssn", "1234", 0, 100), + ], + )], + }) + .await?; + } + + let request = proto::ReconcileRequest { + shard_metadata: Vec::new(), + }; + RouterService::reconcile(router.as_ref(), tonic::Request::new(request.clone())) + .await + .expect_err("first merge application must fail"); + let retried = RouterService::reconcile(router.as_ref(), tonic::Request::new(request)) + .await? + .into_inner(); + assert_eq!(retried.merges_performed, 1); + + let query = RouterService::query_entities( + router.as_ref(), + tonic::Request::new(proto::QueryEntitiesRequest { + descriptors: vec![proto::QueryDescriptor { + attr: "email".to_string(), + value: "retry-merge@example.com".to_string(), + }], + start: 0, + end: 100, + }), + ) + .await? + .into_inner(); + match query.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => { + assert_eq!(matches.matches.len(), 1); + } + other => anyhow::bail!("expected one reconciled entity after retry, got {other:?}"), + } + + drop(router); + shard0.shutdown().await?; + shard1.shutdown().await?; + handle0.abort(); + handle1.abort(); + let _ = handle0.await; + let _ = handle1.await; + Ok(()) +} + +async fn cluster_record_count(shard_addrs: &[SocketAddr]) -> anyhow::Result { + let mut total = 0; + for shard_addr in shard_addrs { + let mut client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + total += client + .get_stats(proto::StatsRequest {}) + .await? + .into_inner() + .record_count; + } + Ok(total) +} + +async fn query_match_count( + client: &mut RouterServiceClient, +) -> anyhow::Result { + let response = client + .query_entities(proto::QueryEntitiesRequest { + descriptors: vec![proto::QueryDescriptor { + attr: "email".to_string(), + value: "restart@example.com".to_string(), + }], + start: 0, + end: 100, + }) + .await? + .into_inner(); + match response.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => { + Ok(matches.matches.len()) + } + other => anyhow::bail!("expected reconciled query matches, got {other:?}"), + } +} diff --git a/tests/distributed_e2e.rs b/tests/distributed_e2e.rs index a9edf08..e6e89e4 100644 --- a/tests/distributed_e2e.rs +++ b/tests/distributed_e2e.rs @@ -24,12 +24,17 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -203,6 +208,7 @@ async fn distributed_stream_and_query() -> anyhow::Result<()> { let response = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record_a.clone(), record_b.clone()], }) .await? diff --git a/tests/distributed_ingest_stream.rs b/tests/distributed_ingest_stream.rs index 79ce63d..2024906 100644 --- a/tests/distributed_ingest_stream.rs +++ b/tests/distributed_ingest_stream.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; +use tempfile::tempdir; use tokio::task::JoinHandle; use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; use tonic::transport::Server; @@ -19,12 +20,17 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -84,6 +90,7 @@ async fn shard_stream_ingest_accepts_chunks() -> anyhow::Result<()> { }); tx.send(IngestRecordsChunk { + internal_protocol_version: 3, records: vec![record_input( 0, "person", @@ -94,6 +101,7 @@ async fn shard_stream_ingest_accepts_chunks() -> anyhow::Result<()> { }) .await?; tx.send(IngestRecordsChunk { + internal_protocol_version: 3, records: vec![record_input( 1, "person", diff --git a/tests/distributed_metrics.rs b/tests/distributed_metrics.rs index b66fe65..dc8eca9 100644 --- a/tests/distributed_metrics.rs +++ b/tests/distributed_metrics.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; +use tempfile::tempdir; use tokio::task::JoinHandle; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; @@ -19,12 +20,17 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -104,6 +110,7 @@ async fn metrics_report_requests() -> anyhow::Result<()> { router .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record_input( 0, "person", diff --git a/tests/distributed_rebalance.rs b/tests/distributed_rebalance.rs index defe48b..22141ef 100644 --- a/tests/distributed_rebalance.rs +++ b/tests/distributed_rebalance.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; +use tempfile::tempdir; use tokio::task::JoinHandle; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; @@ -19,12 +20,17 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -107,7 +113,10 @@ async fn distributed_export_import_range() -> anyhow::Result<()> { ]; let ingest_response = shard0 - .ingest_records(IngestRecordsRequest { records }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records, + }) .await? .into_inner(); assert_eq!(ingest_response.assignments.len(), 3); @@ -154,6 +163,7 @@ async fn distributed_export_import_range() -> anyhow::Result<()> { shard1 .import_records(ImportRecordsRequest { records: all_records, + internal_protocol_version: 3, }) .await?; @@ -180,5 +190,27 @@ async fn distributed_export_import_range() -> anyhow::Result<()> { assert!(exported_ids.contains(&record_id)); } + let original = exported.records[0].clone(); + let mut collision = original.clone(); + collision.identity.as_mut().expect("exported identity").uid = "different-record".to_string(); + let error = shard1 + .import_records(ImportRecordsRequest { + records: vec![collision], + internal_protocol_version: 3, + }) + .await + .expect_err("import must not overwrite an existing numeric record ID"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + + let after_collision = shard1 + .export_records(ExportRecordsRequest { + start_id: original.record_id, + end_id: original.record_id.saturating_add(1), + limit: 1, + }) + .await? + .into_inner(); + assert_eq!(after_collision.records, vec![original]); + Ok(()) } diff --git a/tests/distributed_rebalance_stream.rs b/tests/distributed_rebalance_stream.rs index 9e8efb1..003ce27 100644 --- a/tests/distributed_rebalance_stream.rs +++ b/tests/distributed_rebalance_stream.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; +use tempfile::tempdir; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tokio_stream::wrappers::TcpListenerStream; @@ -7,11 +8,13 @@ use tokio_stream::StreamExt; use tonic::transport::Server; use unirust_rs::distributed::proto::{ self, router_service_client::RouterServiceClient, shard_service_client::ShardServiceClient, - ApplyOntologyRequest, IngestRecordsRequest, RecordDescriptor, + ApplyOntologyRequest, IngestRecordsRequest, RecordDescriptor, RecordIdRangeRequest, RecordIdentity as ProtoRecordIdentity, RecordInput, RouterExportRecordsRequest, RouterImportRecordsRequest, }; -use unirust_rs::distributed::{DistributedOntologyConfig, RouterNode, ShardNode}; +use unirust_rs::distributed::{ + hash_record_to_shard, DistributedOntologyConfig, RouterNode, ShardNode, +}; use unirust_rs::{StreamingTuning, TuningProfile}; mod support; @@ -22,12 +25,17 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -90,7 +98,7 @@ fn record_input( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn distributed_rebalance_stream_via_router() -> anyhow::Result<()> { +async fn distributed_rebalance_stream_rejects_cross_shard_copy() -> anyhow::Result<()> { let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -100,48 +108,48 @@ async fn distributed_rebalance_stream_via_router() -> anyhow::Result<()> { let (router_addr, _router_handle) = spawn_router(vec![shard0_addr, shard1_addr], empty_config.clone()).await?; - let mut shard0 = ShardServiceClient::connect(format!("http://{}", shard0_addr)).await?; - shard0 - .set_ontology(ApplyOntologyRequest { - config: Some(support::to_proto_config(&config)), - }) - .await?; - let mut shard1 = ShardServiceClient::connect(format!("http://{}", shard1_addr)).await?; - shard1 + let mut router = RouterServiceClient::connect(format!("http://{}", router_addr)).await?; + router .set_ontology(ApplyOntologyRequest { config: Some(support::to_proto_config(&config)), }) .await?; - shard0 + let mut records = Vec::new(); + for candidate in 0..1_000 { + let record = record_input( + candidate, + "person", + "crm", + &format!("stream-copy-{candidate}"), + vec![( + "email", + &format!("stream-copy-{candidate}@example.com"), + 0, + 10, + )], + ); + if hash_record_to_shard(&config, &record, 2) == 0 { + records.push(record); + } + if records.len() == 2 { + break; + } + } + assert_eq!(records.len(), 2); + router .ingest_records(IngestRecordsRequest { - records: vec![ - record_input( - 0, - "person", - "crm", - "crm_001", - vec![("email", "alice@example.com", 0, 10)], - ), - record_input( - 1, - "person", - "crm", - "crm_002", - vec![("email", "bob@example.com", 0, 10)], - ), - ], + internal_protocol_version: 3, + records, }) .await?; - let mut router = RouterServiceClient::connect(format!("http://{}", router_addr)).await?; let (tx, rx) = tokio::sync::mpsc::channel::(4); let import_task = tokio::spawn(async move { router .import_records_stream(ReceiverStream::new(rx)) .await .map(|response| response.into_inner()) - .map_err(|err| anyhow::anyhow!(err.to_string())) }); let mut export_stream = RouterServiceClient::connect(format!("http://{}", router_addr)) @@ -150,7 +158,7 @@ async fn distributed_rebalance_stream_via_router() -> anyhow::Result<()> { shard_id: 0, start_id: 0, end_id: 0, - limit: 1, + limit: 10, }) .await? .into_inner(); @@ -169,14 +177,18 @@ async fn distributed_rebalance_stream_via_router() -> anyhow::Result<()> { } drop(tx); - let response = import_task.await??; - assert_eq!(response.imported, 2); + let error = import_task + .await? + .expect_err("streaming copy of reserved source identities must fail"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + let mut shard1 = ShardServiceClient::connect(format!("http://{}", shard1_addr)).await?; let range = shard1 - .get_record_id_range(proto::RecordIdRangeRequest {}) + .get_record_id_range(RecordIdRangeRequest {}) .await? .into_inner(); - assert_eq!(range.record_count, 2); + assert!(range.empty); + assert_eq!(range.record_count, 0); Ok(()) } diff --git a/tests/distributed_reliability.rs b/tests/distributed_reliability.rs index 580f958..e074005 100644 --- a/tests/distributed_reliability.rs +++ b/tests/distributed_reliability.rs @@ -7,35 +7,97 @@ //! - Entity resolution consistency after failures use std::net::SocketAddr; -use std::time::Duration; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; use tempfile::tempdir; use tokio::task::JoinHandle; use tokio::time::sleep; use tokio_stream::wrappers::TcpListenerStream; +use tonic::codegen::{http, Service}; +use tonic::server::NamedService; use tonic::transport::Server; use unirust_rs::distributed::proto::{ self, router_service_client::RouterServiceClient, shard_service_client::ShardServiceClient, ApplyOntologyRequest, IngestRecordsRequest, RecordDescriptor, RecordIdentity, RecordInput, StatsRequest, }; -use unirust_rs::distributed::{DistributedOntologyConfig, RouterNode, ShardNode}; +use unirust_rs::distributed::{ + AdaptiveReconciliationConfig, DistributedOntologyConfig, RouterNode, RouterRpcConfig, ShardNode, +}; use unirust_rs::{StreamingTuning, TuningProfile}; mod support; +// These cases each open one or more RocksDB shards. Serialize them so the test +// binary also runs under the default macOS per-process file descriptor limit. +static PERSISTENT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +#[derive(Clone)] +struct FailSetOntology { + inner: S, +} + +impl FailSetOntology { + fn new(inner: S) -> Self { + Self { inner } + } +} + +impl Service> for FailSetOntology +where + S: Service< + http::Request, + Response = http::Response, + Error = std::convert::Infallible, + > + Send, + S::Future: Send + 'static, + B: Send + 'static, +{ + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + if request.uri().path() == "/unirust.ShardService/SetOntology" { + return Box::pin(async { + Ok(tonic::Status::unavailable("injected SetOntology failure").into_http()) + }); + } + Box::pin(self.inner.call(request)) + } +} + +impl NamedService for FailSetOntology +where + S: NamedService, +{ + const NAME: &'static str = S::NAME; +} + async fn spawn_shard( shard_id: u32, config: DistributedOntologyConfig, ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -45,6 +107,62 @@ async fn spawn_shard( Ok((addr, handle)) } +async fn spawn_set_ontology_failing_shard( + shard_id: u32, + config: DistributedOntologyConfig, +) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( + shard_id, + config, + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, + )?; + let service = proto::shard_service_server::ShardServiceServer::new(shard); + let handle = tokio::spawn(async move { + let _data_dir = data_dir; + Server::builder() + .add_service(FailSetOntology::new(service)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, handle)) +} + +async fn spawn_stoppable_shard( + shard_id: u32, + config: DistributedOntologyConfig, +) -> anyhow::Result<(SocketAddr, tokio::sync::oneshot::Sender<()>, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( + shard_id, + config, + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, + )?; + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let handle = tokio::spawn(async move { + let _data_dir = data_dir; + Server::builder() + .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = shutdown_rx.await; + }) + .await + .expect("shard server"); + }); + Ok((addr, shutdown_tx, handle)) +} + async fn spawn_shard_with_data_dir( shard_id: u32, config: DistributedOntologyConfig, @@ -124,6 +242,7 @@ fn record_input( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shard_recovery_after_restart() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let temp_dir = tempdir()?; let data_dir = temp_dir.path().join("shard0"); std::fs::create_dir_all(&data_dir)?; @@ -143,29 +262,29 @@ async fn shard_recovery_after_restart() -> anyhow::Result<()> { }) .await?; - // Ingest records - let records = vec![ - record_input( - 0, - "person", - "crm", - "persist-1", - vec![("email", "persist@example.com", 0, 10)], - ), - record_input( - 1, - "person", - "crm", - "persist-2", - vec![("email", "persist2@example.com", 0, 10)], - ), - ]; + // Cross the partitioned-dispatch threshold. Persistent shards must still route + // these records through the durable store and perform entity resolution. + let records: Vec<_> = (0..128) + .map(|index| { + record_input( + index, + "person", + "crm", + &format!("persist-{index}"), + vec![("email", &format!("entity-{}@example.com", index / 2), 0, 10)], + ) + }) + .collect(); client - .ingest_records(proto::IngestRecordsRequest { records }) + .ingest_records(proto::IngestRecordsRequest { + internal_protocol_version: 3, + records, + }) .await?; let stats = client.get_stats(StatsRequest {}).await?.into_inner(); - assert_eq!(stats.record_count, 2); + assert_eq!(stats.record_count, 128); + assert_eq!(stats.cluster_count, 64); // Stop shard shard_handle.abort(); @@ -181,15 +300,20 @@ async fn shard_recovery_after_restart() -> anyhow::Result<()> { let stats = client.get_stats(StatsRequest {}).await?.into_inner(); assert_eq!( - stats.record_count, 2, + stats.record_count, 128, "Records should persist after restart" ); + assert_eq!( + stats.cluster_count, 64, + "Entity-resolution assignments should persist after restart" + ); Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn router_handles_partial_shard_availability() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -218,6 +342,7 @@ async fn router_handles_partial_shard_availability() -> anyhow::Result<()> { ); let response = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record], }) .await? @@ -240,12 +365,268 @@ async fn router_handles_partial_shard_availability() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reconciliation_fails_when_boundary_metadata_is_incomplete() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let config = support::build_iam_config(); + let (shard0_addr, shutdown_shard0, shard0_handle) = + spawn_stoppable_shard(0, config.clone()).await?; + let (shard1_addr, _shard1_handle) = spawn_shard(1, config.clone()).await?; + let (router_addr, _router_handle) = + spawn_router(vec![shard0_addr, shard1_addr], config).await?; + let mut client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + + shutdown_shard0 + .send(()) + .map_err(|()| anyhow::anyhow!("shard shutdown receiver dropped"))?; + shard0_handle.await?; + + let error = client + .reconcile(proto::ReconcileRequest { + shard_metadata: Vec::new(), + }) + .await + .expect_err("reconciliation must not succeed with a missing shard"); + assert_eq!(error.code(), tonic::Code::Unavailable); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reconciliation_rejects_caller_supplied_boundary_metadata() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let config = support::build_iam_config(); + let (shard_addr, _shard_handle) = spawn_shard(0, config.clone()).await?; + let (router_addr, _router_handle) = spawn_router(vec![shard_addr], config).await?; + let mut client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + + let error = client + .reconcile(proto::ReconcileRequest { + shard_metadata: vec![proto::BoundaryMetadata::default()], + }) + .await + .expect_err("reconciliation must only use metadata fetched from configured shards"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn router_rejects_misordered_shard_endpoints() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let config = support::build_iam_config(); + let (shard_addr, _shard_handle) = spawn_shard(1, config.clone()).await?; + + let error = match RouterNode::connect(vec![format!("http://{shard_addr}")], config).await { + Ok(_) => anyhow::bail!("router accepted shard 1 at endpoint index 0"), + Err(error) => error, + }; + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn router_rejects_shard_count_change_after_initialization() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let config = support::build_iam_config(); + let (shard0_addr, _shard0_handle) = spawn_shard(0, config.clone()).await?; + let _initial_router = + RouterNode::connect(vec![format!("http://{shard0_addr}")], config.clone()).await?; + + let (shard1_addr, _shard1_handle) = spawn_shard(1, config.clone()).await?; + let error = match RouterNode::connect( + vec![ + format!("http://{shard0_addr}"), + format!("http://{shard1_addr}"), + ], + config, + ) + .await + { + Ok(_) => anyhow::bail!("router accepted a shard-count change without relocation"), + Err(error) => error, + }; + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("topology mismatch")); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn router_requires_at_least_one_shard() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let error = match RouterNode::connect(Vec::new(), support::build_iam_config()).await { + Ok(_) => anyhow::bail!("router accepted an empty shard list"), + Err(error) => error, + }; + assert_eq!(error.code(), tonic::Code::InvalidArgument); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn router_startup_times_out_when_shard_never_responds() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let _hung_peer = tokio::spawn(async move { + let Ok((_socket, _peer)) = listener.accept().await else { + return; + }; + std::future::pending::<()>().await; + }); + + let start = Instant::now(); + let result = RouterNode::connect_with_runtime_config( + vec![format!("http://{addr}")], + support::build_iam_config(), + None, + AdaptiveReconciliationConfig::default(), + RouterRpcConfig { + connect_timeout: Duration::from_millis(100), + request_timeout: Duration::from_millis(100), + tcp_keepalive: Duration::from_secs(1), + shard_mtls: None, + }, + ) + .await; + let error = match result { + Ok(_) => anyhow::bail!("router connected to a shard that never answered"), + Err(error) => error, + }; + + assert_eq!(error.code(), tonic::Code::Unavailable); + assert!( + start.elapsed() < Duration::from_secs(2), + "router startup exceeded its configured shard RPC deadline" + ); + Ok(()) +} + +#[test] +fn shard_id_must_fit_global_cluster_id() { + let result = ShardNode::new( + u32::from(u16::MAX) + 1, + support::build_iam_config(), + StreamingTuning::from_profile(TuningProfile::Balanced), + ); + assert!(result.is_err()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn destructive_reset_is_disabled_and_preserves_records_by_default() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let config = support::build_iam_config(); + let (shard_addr, _shard_handle) = spawn_shard(0, config).await?; + let mut client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + + client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 0, + "person", + "crm", + "reset-guard", + vec![("email", "reset-guard@example.com", 0, 10)], + )], + }) + .await?; + + let error = client + .reset(proto::Empty {}) + .await + .expect_err("destructive reset must require explicit server opt-in"); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + + let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(stats.record_count, 1); + assert_eq!(stats.cluster_count, 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn checkpoint_paths_cannot_escape_the_shard_data_directory() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let (shard_addr, _shard_handle) = spawn_shard(0, support::build_iam_config()).await?; + let mut client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + + for unsafe_path in ["/tmp/unirust-checkpoint", "../outside"] { + let error = client + .checkpoint(proto::CheckpointRequest { + path: unsafe_path.to_string(), + checkpoint_protocol_version: unirust_rs::CHECKPOINT_PROTOCOL_VERSION, + shard_count: 1, + finalize: false, + }) + .await + .expect_err("unsafe checkpoint path must be rejected"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + + let response = client + .checkpoint(proto::CheckpointRequest { + path: "operator-snapshot".to_string(), + checkpoint_protocol_version: unirust_rs::CHECKPOINT_PROTOCOL_VERSION, + shard_count: 1, + finalize: false, + }) + .await? + .into_inner(); + assert_eq!(response.paths.len(), 1); + let checkpoint = std::path::Path::new(&response.paths[0]); + assert!(checkpoint.ends_with("checkpoints/operator-snapshot")); + assert!(checkpoint.is_dir()); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn incomplete_ontology_update_blocks_cluster_traffic() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let empty = DistributedOntologyConfig::empty(); + let (shard0_addr, _shard0_handle) = spawn_shard(0, empty.clone()).await?; + let (shard1_addr, _shard1_handle) = spawn_set_ontology_failing_shard(1, empty.clone()).await?; + let (router_addr, _router_handle) = spawn_router(vec![shard0_addr, shard1_addr], empty).await?; + let mut client = RouterServiceClient::connect(format!("http://{router_addr}")).await?; + + let update_error = client + .set_ontology(ApplyOntologyRequest { + config: Some(support::to_proto_config(&support::build_iam_config())), + }) + .await + .expect_err("ontology update must fail when a shard is unavailable"); + assert_eq!(update_error.code(), tonic::Code::Aborted); + assert!(update_error.message().contains("cluster is blocked")); + + let ingest_error = client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record_input( + 0, + "person", + "crm", + "blocked-ingest", + vec![("email", "blocked@example.com", 0, 10)], + )], + }) + .await + .expect_err("traffic must remain blocked after an incomplete rollback"); + assert_eq!(ingest_error.code(), tonic::Code::FailedPrecondition); + assert!(ingest_error.message().contains("cluster is blocked")); + + Ok(()) +} + // Note: entity_resolution_consistent_after_shard_restart test is complex due to // RocksDB lock management. The shard_recovery_after_restart test covers the // core functionality of shard restart and data persistence. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ingest_operations_complete_before_shutdown() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -275,7 +656,10 @@ async fn ingest_operations_complete_before_shutdown() -> anyhow::Result<()> { .collect(); let response = client - .ingest_records(IngestRecordsRequest { records }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records, + }) .await? .into_inner(); @@ -292,6 +676,7 @@ async fn ingest_operations_complete_before_shutdown() -> anyhow::Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cluster_stats_reflect_all_shards() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -322,7 +707,10 @@ async fn cluster_stats_reflect_all_shards() -> anyhow::Result<()> { .collect(); client - .ingest_records(IngestRecordsRequest { records }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records, + }) .await?; // Get stats from router @@ -333,3 +721,77 @@ async fn cluster_stats_reflect_all_shards() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cross_shard_conflicts_survive_shard_restart() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let temp_dir = tempdir()?; + let data_dir = temp_dir.path().join("shard0"); + std::fs::create_dir_all(&data_dir)?; + let config = support::build_iam_config(); + + let (shard_addr, shard_handle) = + spawn_shard_with_data_dir(0, config.clone(), data_dir.clone()).await?; + let mut client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + + let conflict = proto::CrossShardConflict { + identity_key_signature: Some(proto::IdentityKeySignature { + signature: vec![7; 32], + }), + cluster1: Some(proto::GlobalClusterId { + shard_id: 0, + local_id: 11, + version: 1, + }), + cluster2: Some(proto::GlobalClusterId { + shard_id: 1, + local_id: 22, + version: 1, + }), + interval_start: 10, + interval_end: 20, + perspective_hash: 31, + strong_id_hash1: 41, + strong_id_hash2: 42, + }; + let stored = client + .store_cross_shard_conflicts(proto::StoreCrossShardConflictsRequest { + conflicts: vec![conflict.clone()], + }) + .await? + .into_inner(); + assert_eq!(stored.stored_count, 1); + + let duplicate = client + .store_cross_shard_conflicts(proto::StoreCrossShardConflictsRequest { + conflicts: vec![conflict], + }) + .await? + .into_inner(); + assert_eq!(duplicate.stored_count, 0); + + shard_handle.abort(); + drop(client); + sleep(Duration::from_millis(100)).await; + + let (shard_addr, _shard_handle) = spawn_shard_with_data_dir(0, config, data_dir).await?; + let mut client = ShardServiceClient::connect(format!("http://{shard_addr}")).await?; + let conflicts = client + .list_conflicts(proto::ListConflictsRequest { + start: 0, + end: 0, + attribute: String::new(), + }) + .await? + .into_inner() + .conflicts; + + assert_eq!(conflicts.len(), 1); + assert_eq!(conflicts[0].kind, "indirect_cross_shard"); + assert_eq!(conflicts[0].start, 10); + assert_eq!(conflicts[0].end, 20); + let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(stats.cross_shard_conflicts, 1); + + Ok(()) +} diff --git a/tests/distributed_router_admin.rs b/tests/distributed_router_admin.rs index 95cd10c..15f498b 100644 --- a/tests/distributed_router_admin.rs +++ b/tests/distributed_router_admin.rs @@ -1,15 +1,17 @@ use std::net::SocketAddr; +use tempfile::tempdir; use tokio::task::JoinHandle; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; use unirust_rs::distributed::proto::{ - self, router_service_client::RouterServiceClient, shard_service_client::ShardServiceClient, - ApplyOntologyRequest, IngestRecordsRequest, RecordDescriptor, - RecordIdentity as ProtoRecordIdentity, RecordInput, RouterExportRecordsRequest, - RouterImportRecordsRequest, RouterRecordIdRangeRequest, + self, router_service_client::RouterServiceClient, ApplyOntologyRequest, IngestRecordsRequest, + RecordDescriptor, RecordIdentity as ProtoRecordIdentity, RecordInput, + RouterExportRecordsRequest, RouterImportRecordsRequest, RouterRecordIdRangeRequest, +}; +use unirust_rs::distributed::{ + hash_record_to_shard, DistributedOntologyConfig, RouterNode, ShardNode, }; -use unirust_rs::distributed::{DistributedOntologyConfig, RouterNode, ShardNode}; use unirust_rs::{StreamingTuning, TuningProfile}; mod support; @@ -20,12 +22,17 @@ async fn spawn_shard( ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -88,7 +95,7 @@ fn record_input( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn router_admin_proxies_to_shards() -> anyhow::Result<()> { +async fn router_admin_rejects_non_atomic_cross_shard_copy() -> anyhow::Result<()> { let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -105,25 +112,32 @@ async fn router_admin_proxies_to_shards() -> anyhow::Result<()> { }) .await?; - let mut shard0 = ShardServiceClient::connect(format!("http://{}", shard0_addr)).await?; - shard0 + let mut records = Vec::new(); + for candidate in 0..1_000 { + let record = record_input( + candidate, + "person", + "crm", + &format!("admin-copy-{candidate}"), + vec![( + "email", + &format!("admin-copy-{candidate}@example.com"), + 0, + 10, + )], + ); + if hash_record_to_shard(&config, &record, 2) == 0 { + records.push(record); + } + if records.len() == 2 { + break; + } + } + assert_eq!(records.len(), 2); + router .ingest_records(IngestRecordsRequest { - records: vec![ - record_input( - 0, - "person", - "crm", - "crm_001", - vec![("email", "alice@example.com", 0, 10)], - ), - record_input( - 1, - "person", - "crm", - "crm_002", - vec![("email", "bob@example.com", 0, 10)], - ), - ], + internal_protocol_version: 3, + records, }) .await?; @@ -145,18 +159,21 @@ async fn router_admin_proxies_to_shards() -> anyhow::Result<()> { .into_inner(); assert_eq!(export.records.len(), 2); - router + let error = router .import_records(RouterImportRecordsRequest { shard_id: 1, records: export.records, }) - .await?; + .await + .expect_err("copying a reserved source identity to another shard must fail"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); let range = router .get_record_id_range(RouterRecordIdRangeRequest { shard_id: 1 }) .await? .into_inner(); - assert_eq!(range.record_count, 2); + assert!(range.empty); + assert_eq!(range.record_count, 0); Ok(()) } diff --git a/tests/distributed_wal_chaos.rs b/tests/distributed_wal_chaos.rs index 3303905..fee0f1d 100644 --- a/tests/distributed_wal_chaos.rs +++ b/tests/distributed_wal_chaos.rs @@ -167,7 +167,7 @@ async fn wal_replay_skips_duplicate_records() -> anyhow::Result<()> { } #[tokio::test] -async fn wal_replay_quarantines_corrupt_file() -> anyhow::Result<()> { +async fn corrupt_wal_fails_startup_and_preserves_evidence() -> anyhow::Result<()> { let dir = tempdir()?; let data_dir = dir.path().join("data"); std::fs::create_dir_all(&data_dir)?; @@ -175,11 +175,17 @@ async fn wal_replay_quarantines_corrupt_file() -> anyhow::Result<()> { let wal_path = data_dir.join("ingest_wal.bin"); std::fs::write(&wal_path, b"\x00\xff\x00corrupt")?; - let (addr, handle) = spawn_shard_with_data_dir(0, data_dir.clone()).await?; - let mut client = ShardServiceClient::connect(format!("http://{}", addr)).await?; - let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + let result = ShardNode::new_with_data_dir( + 0, + support::build_iam_config(), + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.clone()), + false, + None, + ); - assert_eq!(stats.record_count, 0); + let error = result.err().expect("corrupt WAL must fail shard startup"); + assert!(error.to_string().contains("ingest WAL is corrupt")); assert!(!wal_path.exists()); let corrupt_found = std::fs::read_dir(&data_dir)? @@ -191,7 +197,5 @@ async fn wal_replay_quarantines_corrupt_file() -> anyhow::Result<()> { .starts_with("ingest_wal.bin.corrupt") }); assert!(corrupt_found); - - handle.abort(); Ok(()) } diff --git a/tests/external_backup_restore.rs b/tests/external_backup_restore.rs new file mode 100644 index 0000000..4fd8fc5 --- /dev/null +++ b/tests/external_backup_restore.rs @@ -0,0 +1,722 @@ +use std::net::SocketAddr; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use tempfile::TempDir; +use tokio::task::JoinHandle; +use tokio::time::{sleep, Duration}; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::codegen::{http, Service}; +use tonic::server::NamedService; +use tonic::transport::Server; +use tonic::Request; +use unirust_rs::distributed::proto::{ + self, shard_service_client::ShardServiceClient, CheckpointRequest, IngestRecordsRequest, + RecordDescriptor, RecordIdentity, RecordInput, StatsRequest, +}; +use unirust_rs::distributed::{ + hash_record_to_shard, hash_source_identity_to_shard, DistributedOntologyConfig, + IdentityKeyConfig, RouterNode, ShardNode, +}; +use unirust_rs::{ + read_cluster_checkpoint_manifest, restore_checkpoint, restore_checkpoint_for_shard, + StreamingTuning, TuningProfile, +}; + +static PERSISTENT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +#[derive(Clone)] +struct FailFirstCheckpoint { + inner: S, + fail_next: Arc, +} + +impl Service> for FailFirstCheckpoint +where + S: Service< + http::Request, + Response = http::Response, + Error = std::convert::Infallible, + > + Send, + S::Future: Send + 'static, + B: Send + 'static, +{ + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + if request.uri().path() == "/unirust.ShardService/Checkpoint" + && self.fail_next.swap(false, Ordering::AcqRel) + { + return Box::pin(async { + Ok(tonic::Status::unavailable("injected checkpoint failure").into_http()) + }); + } + Box::pin(self.inner.call(request)) + } +} + +impl NamedService for FailFirstCheckpoint +where + S: NamedService, +{ + const NAME: &'static str = S::NAME; +} + +async fn spawn_shard( + shard_id: u32, + data_dir: PathBuf, + backup_dir: PathBuf, + config: DistributedOntologyConfig, +) -> anyhow::Result<(SocketAddr, ShardNode, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let shard = ShardNode::new_with_storage_paths( + shard_id, + config, + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir), + Some(backup_dir), + false, + None, + )?; + let service_shard = shard.clone(); + let handle = tokio::spawn(async move { + Server::builder() + .add_service(proto::shard_service_server::ShardServiceServer::new( + service_shard, + )) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, shard, handle)) +} + +async fn spawn_fail_first_checkpoint_shard( + shard_id: u32, + data_dir: PathBuf, + backup_dir: PathBuf, + config: DistributedOntologyConfig, +) -> anyhow::Result<(SocketAddr, ShardNode, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let shard = ShardNode::new_with_storage_paths( + shard_id, + config, + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir), + Some(backup_dir), + false, + None, + )?; + let service = proto::shard_service_server::ShardServiceServer::new(shard.clone()); + let service = FailFirstCheckpoint { + inner: service, + fail_next: Arc::new(AtomicBool::new(true)), + }; + let handle = tokio::spawn(async move { + Server::builder() + .add_service(service) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, shard, handle)) +} + +fn config() -> DistributedOntologyConfig { + DistributedOntologyConfig { + identity_keys: vec![IdentityKeyConfig { + name: "email_key".to_string(), + attributes: vec!["email".to_string()], + }], + strong_identifiers: Vec::new(), + constraints: Vec::new(), + } +} + +fn record(index: u32, email: &str) -> RecordInput { + RecordInput { + index, + identity: Some(RecordIdentity { + entity_type: "person".to_string(), + perspective: "crm".to_string(), + uid: "backup-source".to_string(), + }), + descriptors: vec![RecordDescriptor { + attr: "email".to_string(), + value: email.to_string(), + start: 0, + end: 100, + }], + } +} + +async fn stop_shard(shard: ShardNode, handle: JoinHandle<()>) -> anyhow::Result<()> { + shard.shutdown().await?; + handle.abort(); + let _ = handle.await; + drop(shard); + sleep(Duration::from_millis(50)).await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn external_checkpoint_restores_a_lost_shard_volume() -> anyhow::Result<()> { + use proto::router_service_server::RouterService; + + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let data_volume = TempDir::new()?; + let backup_volume = TempDir::new()?; + let original_data = data_volume.path().join("original-shard"); + let replacement_data = data_volume.path().join("replacement-shard"); + let backup_root = backup_volume.path().join("shard-0"); + + let (addr, shard, handle) = + spawn_shard(0, original_data.clone(), backup_root.clone(), config()).await?; + let mut client = ShardServiceClient::connect(format!("http://{addr}")).await?; + client + .ingest_records(IngestRecordsRequest { + records: vec![record(0, "backup@example.com")], + internal_protocol_version: 3, + }) + .await?; + let before = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(before.record_count, 1); + assert_eq!(before.cluster_count, 1); + + let router = RouterNode::connect(vec![format!("http://{addr}")], config()).await?; + let checkpoint = RouterService::checkpoint( + router.as_ref(), + Request::new(CheckpointRequest { + path: "snapshot-a".to_string(), + checkpoint_protocol_version: 0, + shard_count: 0, + finalize: false, + }), + ) + .await? + .into_inner(); + assert_eq!(checkpoint.paths.len(), 1); + assert!(checkpoint.committed); + assert_eq!(checkpoint.generation, "snapshot-a"); + let checkpoint_path = PathBuf::from(&checkpoint.paths[0]); + assert!(checkpoint_path.starts_with(backup_root.canonicalize()?)); + assert!(checkpoint_path.join("CURRENT").is_file()); + + drop(client); + drop(router); + stop_shard(shard, handle).await?; + std::fs::remove_dir_all(&original_data)?; + restore_checkpoint(&checkpoint_path, &replacement_data)?; + + let (addr, restored_shard, restored_handle) = + spawn_shard(0, replacement_data, backup_root, config()).await?; + let mut client = ShardServiceClient::connect(format!("http://{addr}")).await?; + let after = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(after.record_count, 1); + assert_eq!(after.cluster_count, 1); + + client + .ingest_records(IngestRecordsRequest { + records: vec![record(1, "backup@example.com")], + internal_protocol_version: 3, + }) + .await?; + let retry_stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(retry_stats.record_count, 1); + + let error = client + .ingest_records(IngestRecordsRequest { + records: vec![record(2, "changed@example.com")], + internal_protocol_version: 3, + }) + .await + .expect_err("restored immutable source identity must reject a changed payload"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + + drop(client); + stop_shard(restored_shard, restored_handle).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn coordinated_checkpoint_restores_source_reservations_across_shards() -> anyhow::Result<()> { + use proto::router_service_server::RouterService; + + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let data_volume = TempDir::new()?; + let backup_volume = TempDir::new()?; + let cluster_config = config(); + let original_paths = (0..2) + .map(|shard_id| data_volume.path().join(format!("original-{shard_id}"))) + .collect::>(); + let replacement_paths = (0..2) + .map(|shard_id| data_volume.path().join(format!("replacement-{shard_id}"))) + .collect::>(); + let backup_roots = (0..2) + .map(|shard_id| backup_volume.path().join(format!("shard-{shard_id}"))) + .collect::>(); + + let mut source_record = None; + for candidate in 0..1_000 { + let mut candidate_record = record(0, &format!("cluster-backup-{candidate}@example.com")); + candidate_record.identity.as_mut().unwrap().uid = format!("cluster-source-{candidate}"); + let identity = candidate_record.identity.as_ref().unwrap(); + if hash_source_identity_to_shard(identity, 2) + != hash_record_to_shard(&cluster_config, &candidate_record, 2) + { + source_record = Some(candidate_record); + break; + } + } + let source_record = + source_record.expect("test must find distinct reservation owner and target shards"); + + let mut shard_addrs = Vec::new(); + let mut shard_nodes = Vec::new(); + let mut shard_handles = Vec::new(); + for (shard_id, (data_path, backup_root)) in original_paths.iter().zip(&backup_roots).enumerate() + { + let (addr, shard, handle) = spawn_shard( + shard_id as u32, + data_path.clone(), + backup_root.clone(), + cluster_config.clone(), + ) + .await?; + shard_addrs.push(addr); + shard_nodes.push(shard); + shard_handles.push(handle); + } + let router = RouterNode::connect( + shard_addrs + .iter() + .map(|addr| format!("http://{addr}")) + .collect(), + cluster_config.clone(), + ) + .await?; + + RouterService::ingest_records( + router.as_ref(), + Request::new(IngestRecordsRequest { + records: vec![source_record.clone()], + internal_protocol_version: 0, + }), + ) + .await?; + let checkpoint = RouterService::checkpoint( + router.as_ref(), + Request::new(CheckpointRequest { + path: "coordinated".to_string(), + checkpoint_protocol_version: 0, + shard_count: 0, + finalize: false, + }), + ) + .await? + .into_inner(); + assert_eq!(checkpoint.paths.len(), 2); + + drop(router); + for (shard, handle) in shard_nodes.into_iter().zip(shard_handles) { + stop_shard(shard, handle).await?; + } + for path in &original_paths { + std::fs::remove_dir_all(path)?; + } + for (checkpoint_path, replacement_path) in checkpoint.paths.iter().zip(&replacement_paths) { + restore_checkpoint(std::path::Path::new(checkpoint_path), replacement_path)?; + } + + let mut restarted_addrs = Vec::new(); + let mut restarted_nodes = Vec::new(); + let mut restarted_handles = Vec::new(); + for (shard_id, (replacement_path, backup_root)) in + replacement_paths.iter().zip(&backup_roots).enumerate() + { + let (addr, shard, handle) = spawn_shard( + shard_id as u32, + replacement_path.clone(), + backup_root.clone(), + cluster_config.clone(), + ) + .await?; + restarted_addrs.push(addr); + restarted_nodes.push(shard); + restarted_handles.push(handle); + } + let router = RouterNode::connect( + restarted_addrs + .iter() + .map(|addr| format!("http://{addr}")) + .collect(), + cluster_config, + ) + .await?; + let stats = RouterService::get_stats(router.as_ref(), Request::new(StatsRequest {})) + .await? + .into_inner(); + assert_eq!(stats.record_count, 1); + + let mut retry = source_record.clone(); + retry.index = 1; + RouterService::ingest_records( + router.as_ref(), + Request::new(IngestRecordsRequest { + records: vec![retry], + internal_protocol_version: 0, + }), + ) + .await?; + let stats = RouterService::get_stats(router.as_ref(), Request::new(StatsRequest {})) + .await? + .into_inner(); + assert_eq!(stats.record_count, 1); + + let mut changed = source_record; + changed.index = 2; + changed.descriptors[0].value = "changed-after-cluster-restore@example.com".to_string(); + let error = RouterService::ingest_records( + router.as_ref(), + Request::new(IngestRecordsRequest { + records: vec![changed], + internal_protocol_version: 0, + }), + ) + .await + .expect_err("restored cluster reservation must reject a changed payload"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + + drop(router); + for (shard, handle) in restarted_nodes.into_iter().zip(restarted_handles) { + stop_shard(shard, handle).await?; + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn router_rejects_partial_and_mixed_generation_restores() -> anyhow::Result<()> { + use proto::router_service_server::RouterService; + + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let data_volume = TempDir::new()?; + let backup_volume = TempDir::new()?; + let original_paths = (0..2) + .map(|shard_id| data_volume.path().join(format!("original-{shard_id}"))) + .collect::>(); + let replacement_paths = (0..2) + .map(|shard_id| data_volume.path().join(format!("replacement-{shard_id}"))) + .collect::>(); + let backup_roots = (0..2) + .map(|shard_id| backup_volume.path().join(format!("shard-{shard_id}"))) + .collect::>(); + + let mut original_addrs = Vec::new(); + let mut original_nodes = Vec::new(); + let mut original_handles = Vec::new(); + for shard_id in 0..2 { + let (addr, shard, handle) = spawn_shard( + shard_id as u32, + original_paths[shard_id].clone(), + backup_roots[shard_id].clone(), + config(), + ) + .await?; + original_addrs.push(addr); + original_nodes.push(shard); + original_handles.push(handle); + } + let router = RouterNode::connect( + original_addrs + .iter() + .map(|addr| format!("http://{addr}")) + .collect(), + config(), + ) + .await?; + let generation_a = RouterService::checkpoint( + router.as_ref(), + Request::new(CheckpointRequest { + path: "generation-a".to_string(), + checkpoint_protocol_version: 0, + shard_count: 0, + finalize: false, + }), + ) + .await? + .into_inner(); + let generation_b = RouterService::checkpoint( + router.as_ref(), + Request::new(CheckpointRequest { + path: "generation-b".to_string(), + checkpoint_protocol_version: 0, + shard_count: 0, + finalize: false, + }), + ) + .await? + .into_inner(); + drop(router); + for (shard, handle) in original_nodes.into_iter().zip(original_handles) { + stop_shard(shard, handle).await?; + } + + let wrong_shard_path = data_volume.path().join("wrong-shard"); + restore_checkpoint( + std::path::Path::new(&generation_a.paths[0]), + &wrong_shard_path, + )?; + let wrong_shard_error = match ShardNode::new_with_storage_paths( + 1, + config(), + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(wrong_shard_path), + Some(backup_roots[1].clone()), + false, + None, + ) { + Ok(_) => anyhow::bail!("shard accepted a checkpoint belonging to another shard"), + Err(error) => error, + }; + assert!(wrong_shard_error + .to_string() + .contains("restored checkpoint belongs to shard 0, not configured shard 1")); + + restore_checkpoint_for_shard( + std::path::Path::new(&generation_a.paths[0]), + &replacement_paths[0], + Some(0), + )?; + let (restored_addr, restored_node, restored_handle) = spawn_shard( + 0, + replacement_paths[0].clone(), + backup_roots[0].clone(), + config(), + ) + .await?; + let (live_addr, live_node, live_handle) = spawn_shard( + 1, + original_paths[1].clone(), + backup_roots[1].clone(), + config(), + ) + .await?; + let partial_error = match RouterNode::connect( + vec![ + format!("http://{restored_addr}"), + format!("http://{live_addr}"), + ], + config(), + ) + .await + { + Ok(_) => anyhow::bail!("router accepted a mix of restored and unrestored volumes"), + Err(error) => error, + }; + assert_eq!(partial_error.code(), tonic::Code::FailedPrecondition); + assert!(partial_error.message().contains("restored and unrestored")); + stop_shard(restored_node, restored_handle).await?; + stop_shard(live_node, live_handle).await?; + + restore_checkpoint_for_shard( + std::path::Path::new(&generation_b.paths[1]), + &replacement_paths[1], + Some(1), + )?; + let (generation_a_addr, generation_a_node, generation_a_handle) = spawn_shard( + 0, + replacement_paths[0].clone(), + backup_roots[0].clone(), + config(), + ) + .await?; + let (generation_b_addr, generation_b_node, generation_b_handle) = spawn_shard( + 1, + replacement_paths[1].clone(), + backup_roots[1].clone(), + config(), + ) + .await?; + let mixed_error = match RouterNode::connect( + vec![ + format!("http://{generation_a_addr}"), + format!("http://{generation_b_addr}"), + ], + config(), + ) + .await + { + Ok(_) => anyhow::bail!("router accepted shards restored from different generations"), + Err(error) => error, + }; + assert_eq!(mixed_error.code(), tonic::Code::FailedPrecondition); + assert!(mixed_error + .message() + .contains("different checkpoint generations")); + + stop_shard(generation_a_node, generation_a_handle).await?; + stop_shard(generation_b_node, generation_b_handle).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn partial_checkpoint_is_not_restorable_and_same_generation_can_retry() -> anyhow::Result<()> +{ + use proto::router_service_server::RouterService; + + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let data_volume = TempDir::new()?; + let backup_volume = TempDir::new()?; + let backup_roots = (0..2) + .map(|shard_id| backup_volume.path().join(format!("shard-{shard_id}"))) + .collect::>(); + + let (addr0, shard0, handle0) = spawn_shard( + 0, + data_volume.path().join("shard-0"), + backup_roots[0].clone(), + config(), + ) + .await?; + let (addr1, shard1, handle1) = spawn_fail_first_checkpoint_shard( + 1, + data_volume.path().join("shard-1"), + backup_roots[1].clone(), + config(), + ) + .await?; + let router = RouterNode::connect( + vec![format!("http://{addr0}"), format!("http://{addr1}")], + config(), + ) + .await?; + let request = CheckpointRequest { + path: "retryable-generation".to_string(), + checkpoint_protocol_version: 0, + shard_count: 0, + finalize: false, + }; + + let error = RouterService::checkpoint(router.as_ref(), Request::new(request.clone())) + .await + .expect_err("injected second-shard failure must fail the cluster checkpoint"); + assert_eq!(error.code(), tonic::Code::Unavailable); + + let partial = backup_roots[0].join("retryable-generation"); + assert!(partial.join("CURRENT").is_file()); + assert!(read_cluster_checkpoint_manifest(&partial).is_err()); + let partial_restore = data_volume.path().join("partial-restore"); + restore_checkpoint(&partial, &partial_restore) + .expect_err("an uncommitted shard checkpoint must not be restorable"); + assert!(!partial_restore.exists()); + + let completed = RouterService::checkpoint(router.as_ref(), Request::new(request)) + .await? + .into_inner(); + assert!(completed.committed); + assert_eq!(completed.generation, "retryable-generation"); + assert_eq!(completed.paths.len(), 2); + + for (shard_id, checkpoint_path) in completed.paths.iter().enumerate() { + let checkpoint_path = std::path::Path::new(checkpoint_path); + let manifest = read_cluster_checkpoint_manifest(checkpoint_path)?; + assert_eq!(manifest.generation(), "retryable-generation"); + assert_eq!(manifest.shard_id(), shard_id as u32); + assert_eq!(manifest.shard_count(), 2); + restore_checkpoint_for_shard( + checkpoint_path, + &data_volume.path().join(format!("restored-{shard_id}")), + Some(shard_id as u32), + )?; + } + + drop(router); + stop_shard(shard0, handle0).await?; + stop_shard(shard1, handle1).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn checkpoint_scheduler_retries_one_generation_until_commit() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let data_volume = TempDir::new()?; + let backup_volume = TempDir::new()?; + let backup_roots = (0..2) + .map(|shard_id| backup_volume.path().join(format!("shard-{shard_id}"))) + .collect::>(); + let (addr0, shard0, handle0) = spawn_shard( + 0, + data_volume.path().join("scheduled-shard-0"), + backup_roots[0].clone(), + config(), + ) + .await?; + let (addr1, shard1, handle1) = spawn_fail_first_checkpoint_shard( + 1, + data_volume.path().join("scheduled-shard-1"), + backup_roots[1].clone(), + config(), + ) + .await?; + let router = RouterNode::connect( + vec![format!("http://{addr0}"), format!("http://{addr1}")], + config(), + ) + .await?; + let scheduler = router + .clone() + .start_checkpoint_scheduler(Duration::from_millis(50))?; + + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + let committed_generation = loop { + let mut committed_generation = None; + if backup_roots[0].is_dir() { + for entry in std::fs::read_dir(&backup_roots[0])? { + let entry = entry?; + let generation = entry.file_name(); + let other = backup_roots[1].join(&generation); + if read_cluster_checkpoint_manifest(&entry.path()).is_ok() + && read_cluster_checkpoint_manifest(&other).is_ok() + { + committed_generation = generation.to_str().map(str::to_string); + break; + } + } + } + if let Some(generation) = committed_generation { + break generation; + } + if tokio::time::Instant::now() >= deadline { + anyhow::bail!("scheduled checkpoint did not commit before the deadline"); + } + sleep(Duration::from_millis(10)).await; + }; + scheduler.abort(); + let _ = scheduler.await; + + assert!(committed_generation.starts_with("scheduled-")); + let shard0_manifest = + read_cluster_checkpoint_manifest(&backup_roots[0].join(&committed_generation))?; + let shard1_manifest = + read_cluster_checkpoint_manifest(&backup_roots[1].join(&committed_generation))?; + assert_eq!(shard0_manifest.generation(), shard1_manifest.generation()); + assert_eq!(shard0_manifest.shard_count(), 2); + assert_eq!(shard1_manifest.shard_count(), 2); + + drop(router); + stop_shard(shard0, handle0).await?; + stop_shard(shard1, handle1).await?; + Ok(()) +} diff --git a/tests/healthcheck_cli.rs b/tests/healthcheck_cli.rs new file mode 100644 index 0000000..5620188 --- /dev/null +++ b/tests/healthcheck_cli.rs @@ -0,0 +1,177 @@ +use std::net::SocketAddr; +use std::pin::Pin; +use std::process::{Command, Output}; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use tempfile::tempdir; +use tokio::task::JoinHandle; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::codegen::{http, Service}; +use tonic::server::NamedService; +use tonic::transport::Server; +use unirust_rs::distributed::proto; +use unirust_rs::distributed::{DistributedOntologyConfig, RouterNode, ShardNode}; +use unirust_rs::{StreamingTuning, TuningProfile}; + +static PERSISTENT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +#[derive(Clone)] +struct FailHealth { + inner: S, +} + +impl Service> for FailHealth +where + S: Service< + http::Request, + Response = http::Response, + Error = std::convert::Infallible, + > + Send, + S::Future: Send + 'static, + B: Send + 'static, +{ + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + if request.uri().path() == "/unirust.ShardService/HealthCheck" { + return Box::pin(async { + Ok(tonic::Status::failed_precondition("injected unhealthy shard").into_http()) + }); + } + Box::pin(self.inner.call(request)) + } +} + +impl NamedService for FailHealth +where + S: NamedService, +{ + const NAME: &'static str = S::NAME; +} + +async fn run_probe(args: Vec) -> anyhow::Result { + Ok(tokio::task::spawn_blocking(move || { + Command::new(env!("CARGO_BIN_EXE_unirust_healthcheck")) + .args(args) + .output() + }) + .await??) +} + +async fn spawn_healthy_shard() -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( + 0, + DistributedOntologyConfig::empty(), + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, + )?; + let handle = tokio::spawn(async move { + let _data_dir = data_dir; + Server::builder() + .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, handle)) +} + +async fn spawn_unhealthy_shard() -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( + 0, + DistributedOntologyConfig::empty(), + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, + )?; + let service = proto::shard_service_server::ShardServiceServer::new(shard); + let handle = tokio::spawn(async move { + let _data_dir = data_dir; + Server::builder() + .add_service(FailHealth { inner: service }) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("shard server"); + }); + Ok((addr, handle)) +} + +async fn spawn_router( + shard_addr: SocketAddr, +) -> anyhow::Result<(SocketAddr, Arc, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let router = RouterNode::connect( + vec![format!("http://{shard_addr}")], + DistributedOntologyConfig::empty(), + ) + .await?; + let server_router = router.clone(); + let handle = tokio::spawn(async move { + Server::builder() + .add_service(proto::router_service_server::RouterServiceServer::new( + server_router, + )) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("router server"); + }); + Ok((addr, router, handle)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn healthcheck_accepts_healthy_shard_and_router() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let (shard_addr, shard_handle) = spawn_healthy_shard().await?; + let shard_output = run_probe(vec!["--shard".into(), shard_addr.to_string()]).await?; + assert!( + shard_output.status.success(), + "shard probe failed: {}", + String::from_utf8_lossy(&shard_output.stderr) + ); + + let (router_addr, _router, router_handle) = spawn_router(shard_addr).await?; + let router_output = run_probe(vec!["--router".into(), router_addr.to_string()]).await?; + assert!( + router_output.status.success(), + "router probe failed: {}", + String::from_utf8_lossy(&router_output.stderr) + ); + + router_handle.abort(); + shard_handle.abort(); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn healthcheck_rejects_unhealthy_rpc_behind_open_socket() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let (addr, handle) = spawn_unhealthy_shard().await?; + let output = run_probe(vec!["--shard".into(), addr.to_string()]).await?; + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("injected unhealthy shard"), + "unexpected probe error: {}", + String::from_utf8_lossy(&output.stderr) + ); + + handle.abort(); + Ok(()) +} diff --git a/tests/linker_state_recovery.rs b/tests/linker_state_recovery.rs index 8999be6..00c86f6 100644 --- a/tests/linker_state_recovery.rs +++ b/tests/linker_state_recovery.rs @@ -6,9 +6,12 @@ use std::collections::HashMap; use tempfile::tempdir; use unirust_rs::advanced::{ClusterId, GlobalClusterId, LinkerStateConfig}; +use unirust_rs::model::{Descriptor, Record, RecordIdentity}; +use unirust_rs::ontology::{IdentityKey, StrongIdentifier}; use unirust_rs::persistence::linker_encoding; +use unirust_rs::temporal::Interval; use unirust_rs::test_support::{default_ontology, generate_dataset}; -use unirust_rs::{PersistentStore, RecordId, StreamingTuning, Unirust}; +use unirust_rs::{Ontology, PersistentStore, RecordId, StreamingTuning, Unirust}; #[test] fn linker_state_persists_across_restart() -> anyhow::Result<()> { @@ -166,6 +169,187 @@ fn linker_state_with_lru_config_persists() -> anyhow::Result<()> { Ok(()) } +#[test] +fn bounded_linker_configuration_does_not_evict_resolution_state() -> anyhow::Result<()> { + let dir = tempdir()?; + let store = PersistentStore::open(dir.path())?; + let mut ontology = Ontology::new(); + ontology.add_identity_key(IdentityKey::from_names(vec!["email"], "email")); + + let mut tuning = StreamingTuning::balanced(); + tuning.linker_state_config = Some(LinkerStateConfig { + cluster_ids_capacity: 1, + global_ids_capacity: 1, + summaries_capacity: 1, + perspectives_capacity: 1, + dirty_buffer_size: 1, + }); + let mut unirust = Unirust::with_store_and_tuning(ontology, store, tuning); + + fn record(unirust: &mut Unirust, uid: &str, email: &str) -> Record { + let attr = unirust.intern_attr("email"); + let value = unirust.intern_value(email); + Record::new( + RecordId(0), + RecordIdentity::new("person".to_string(), "crm".to_string(), uid.to_string()), + vec![Descriptor::new( + attr, + value, + Interval::new(0, 10).expect("valid interval"), + )], + ) + } + + let first_record = record(&mut unirust, "first", "shared@example.com"); + let first = unirust.stream_records(vec![first_record])?[0].cluster_id; + let unrelated_record = record(&mut unirust, "unrelated", "other@example.com"); + unirust.stream_records(vec![unrelated_record])?; + let related_record = record(&mut unirust, "related", "shared@example.com"); + let related = unirust.stream_records(vec![related_record])?[0].cluster_id; + + assert_eq!( + related, first, + "a cache capacity must never change entity-resolution results" + ); + Ok(()) +} + +#[test] +fn persistent_dsu_restart_rebuilds_cluster_summaries() -> anyhow::Result<()> { + let dir = tempdir()?; + let data_dir = dir.path().join("persistent-dsu"); + std::fs::create_dir_all(&data_dir)?; + + fn ontology() -> Ontology { + let mut ontology = Ontology::new(); + ontology.add_identity_key(IdentityKey::from_names(vec!["email"], "email")); + ontology.add_strong_identifier(StrongIdentifier::from_name("ssn", "ssn")); + ontology + } + + fn record(unirust: &mut Unirust, uid: &str, ssn: Option<&str>) -> Record { + let email_attr = unirust.intern_attr("email"); + let email_value = unirust.intern_value("shared@example.com"); + let mut descriptors = vec![Descriptor::new( + email_attr, + email_value, + Interval::new(0, 10).expect("valid interval"), + )]; + if let Some(ssn) = ssn { + let ssn_attr = unirust.intern_attr("ssn"); + let ssn_value = unirust.intern_value(ssn); + descriptors.push(Descriptor::new( + ssn_attr, + ssn_value, + Interval::new(0, 10).expect("valid interval"), + )); + } + Record::new( + RecordId(0), + RecordIdentity::new("person".to_string(), "crm".to_string(), uid.to_string()), + descriptors, + ) + } + + let original_cluster; + { + let store = PersistentStore::open(&data_dir)?; + let mut tuning = StreamingTuning::billion_scale(); + tuning + .dsu_config + .as_mut() + .expect("persistent DSU config") + .dirty_buffer_size = 1; + let mut unirust = Unirust::with_store_and_tuning(ontology(), store, tuning); + let first = record(&mut unirust, "first", None); + original_cluster = unirust.stream_records(vec![first])?[0].cluster_id; + let second = record(&mut unirust, "second", Some("111-11-1111")); + let second_cluster = unirust.stream_records(vec![second])?[0].cluster_id; + assert_eq!(second_cluster, original_cluster); + unirust.checkpoint_linker_state()?; + } + + { + let store = PersistentStore::open(&data_dir)?; + let mut tuning = StreamingTuning::billion_scale(); + tuning + .dsu_config + .as_mut() + .expect("persistent DSU config") + .dirty_buffer_size = 1; + let mut unirust = Unirust::with_store_and_tuning(ontology(), store, tuning); + unirust.initialize_streaming()?; + + let conflicting = record(&mut unirust, "conflicting", Some("222-22-2222")); + let conflicting_cluster = unirust.stream_records(vec![conflicting])?[0].cluster_id; + assert_ne!( + conflicting_cluster, original_cluster, + "restart recovery must retain strong-ID conflict guards" + ); + assert_eq!(unirust.streaming_cluster_count(), Some(2)); + } + + Ok(()) +} + +#[test] +fn cross_shard_merge_mapping_survives_restart() -> anyhow::Result<()> { + let dir = tempdir()?; + let data_dir = dir.path().join("store"); + std::fs::create_dir_all(&data_dir)?; + let primary = GlobalClusterId::new(0, 10, 0); + let secondary = GlobalClusterId::new(1, 20, 0); + + { + let store = PersistentStore::open(&data_dir)?; + let mut unirust = Unirust::with_store(default_ontology_for_empty_store(), store); + unirust.stream_records(Vec::new())?; + unirust.apply_cross_shard_merge(primary, secondary)?; + assert_eq!(unirust.cross_shard_merge_count(), 1); + } + + { + let store = PersistentStore::open(&data_dir)?; + let mut unirust = Unirust::with_store(default_ontology_for_empty_store(), store); + unirust.stream_records(Vec::new())?; + assert_eq!(unirust.cross_shard_merge_count(), 1); + } + + Ok(()) +} + +#[test] +fn cross_shard_merge_chain_is_persisted_as_direct_canonical_redirects() -> anyhow::Result<()> { + let dir = tempdir()?; + let data_dir = dir.path().join("store"); + std::fs::create_dir_all(&data_dir)?; + let primary = GlobalClusterId::new(0, 10, 0); + let middle = GlobalClusterId::new(1, 20, 0); + let secondary = GlobalClusterId::new(2, 30, 0); + + { + let store = PersistentStore::open(&data_dir)?; + let mut unirust = Unirust::with_store(default_ontology_for_empty_store(), store); + unirust.stream_records(Vec::new())?; + unirust.apply_cross_shard_merge(primary, middle)?; + unirust.apply_cross_shard_merge(middle, secondary)?; + assert_eq!(unirust.resolve_global_cluster_id(secondary), Some(primary)); + } + + { + let store = PersistentStore::open(&data_dir)?; + let mut unirust = Unirust::with_store(default_ontology_for_empty_store(), store); + unirust.stream_records(Vec::new())?; + assert_eq!(unirust.resolve_global_cluster_id(secondary), Some(primary)); + } + + Ok(()) +} + +fn default_ontology_for_empty_store() -> unirust_rs::Ontology { + unirust_rs::Ontology::new() +} + #[test] fn linker_persistence_round_trip_encoding() { // Test the encoding/decoding helpers directly diff --git a/tests/mtls_transport.rs b/tests/mtls_transport.rs new file mode 100644 index 0000000..77629e3 --- /dev/null +++ b/tests/mtls_transport.rs @@ -0,0 +1,294 @@ +use std::fs; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use rcgen::{ + BasicConstraints, CertificateParams, CertifiedIssuer, ExtendedKeyUsagePurpose, IsCa, KeyPair, + KeyUsagePurpose, +}; +use tempfile::{tempdir, TempDir}; +use tokio::task::JoinHandle; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::transport::{Certificate, ClientTlsConfig, Server}; +use unirust_rs::distributed::proto; +use unirust_rs::distributed::{ + AdaptiveReconciliationConfig, DistributedOntologyConfig, RouterNode, RouterRpcConfig, ShardNode, +}; +use unirust_rs::transport_security::{load_client_mtls, load_server_mtls}; +use unirust_rs::{StreamingTuning, TuningProfile}; + +static PERSISTENT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +struct PemIdentity { + cert: String, + key: String, +} + +struct TestPki { + ca: String, + server: PemIdentity, + client: PemIdentity, + rogue_client: PemIdentity, +} + +struct PkiPaths { + _dir: TempDir, + ca: PathBuf, + server_cert: PathBuf, + server_key: PathBuf, + client_cert: PathBuf, + client_key: PathBuf, + rogue_client_cert: PathBuf, + rogue_client_key: PathBuf, +} + +fn certificate_authority() -> anyhow::Result> { + let mut params = CertificateParams::new(Vec::::new())?; + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]; + Ok(CertifiedIssuer::self_signed(params, KeyPair::generate()?)?) +} + +fn leaf_identity( + issuer: &CertifiedIssuer<'static, KeyPair>, + names: Vec, + usage: ExtendedKeyUsagePurpose, +) -> anyhow::Result { + let mut params = CertificateParams::new(names)?; + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + params.extended_key_usages = vec![usage]; + let key = KeyPair::generate()?; + let cert = params.signed_by(&key, issuer)?; + Ok(PemIdentity { + cert: cert.pem(), + key: key.serialize_pem(), + }) +} + +fn test_pki() -> anyhow::Result { + let issuer = certificate_authority()?; + let rogue_issuer = certificate_authority()?; + Ok(TestPki { + ca: issuer.pem(), + server: leaf_identity( + &issuer, + vec!["127.0.0.1".to_string(), "localhost".to_string()], + ExtendedKeyUsagePurpose::ServerAuth, + )?, + client: leaf_identity(&issuer, Vec::new(), ExtendedKeyUsagePurpose::ClientAuth)?, + rogue_client: leaf_identity( + &rogue_issuer, + Vec::new(), + ExtendedKeyUsagePurpose::ClientAuth, + )?, + }) +} + +fn write_file(dir: &Path, name: &str, contents: &str) -> anyhow::Result { + let path = dir.join(name); + fs::write(&path, contents)?; + Ok(path) +} + +fn write_pki(pki: &TestPki) -> anyhow::Result { + let dir = tempdir()?; + let root = dir.path(); + Ok(PkiPaths { + ca: write_file(root, "ca.pem", &pki.ca)?, + server_cert: write_file(root, "server.pem", &pki.server.cert)?, + server_key: write_file(root, "server-key.pem", &pki.server.key)?, + client_cert: write_file(root, "client.pem", &pki.client.cert)?, + client_key: write_file(root, "client-key.pem", &pki.client.key)?, + rogue_client_cert: write_file(root, "rogue-client.pem", &pki.rogue_client.cert)?, + rogue_client_key: write_file(root, "rogue-client-key.pem", &pki.rogue_client.key)?, + _dir: dir, + }) +} + +async fn spawn_mtls_shard( + paths: &PkiPaths, +) -> anyhow::Result<(SocketAddr, ShardNode, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( + 0, + DistributedOntologyConfig::empty(), + StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, + )?; + let server_shard = shard.clone(); + let tls = load_server_mtls( + Some(&paths.server_cert), + Some(&paths.server_key), + Some(&paths.ca), + )? + .expect("complete server mTLS"); + let handle = tokio::spawn(async move { + let _data_dir = data_dir; + Server::builder() + .tls_config(tls) + .expect("valid test TLS") + .add_service(proto::shard_service_server::ShardServiceServer::new( + server_shard, + )) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("mTLS shard server"); + }); + Ok((addr, shard, handle)) +} + +async fn spawn_mtls_router( + router: std::sync::Arc, + paths: &PkiPaths, +) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let tls = load_server_mtls( + Some(&paths.server_cert), + Some(&paths.server_key), + Some(&paths.ca), + )? + .expect("complete server mTLS"); + let handle = tokio::spawn(async move { + Server::builder() + .tls_config(tls) + .expect("valid test TLS") + .add_service(proto::router_service_server::RouterServiceServer::new( + router, + )) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("mTLS router server"); + }); + Ok((addr, handle)) +} + +async fn run_healthcheck(args: Vec) -> anyhow::Result { + Ok(tokio::task::spawn_blocking(move || { + Command::new(env!("CARGO_BIN_EXE_unirust_healthcheck")) + .args(args) + .output() + }) + .await??) +} + +fn probe_args( + service: &str, + addr: SocketAddr, + paths: &PkiPaths, + cert: &Path, + key: &Path, +) -> Vec { + vec![ + service.to_string(), + format!("https://{addr}"), + "--ca-cert".to_string(), + paths.ca.display().to_string(), + "--client-cert".to_string(), + cert.display().to_string(), + "--client-key".to_string(), + key.display().to_string(), + ] +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mutual_tls_authenticates_complete_router_and_shard_path() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; + let pki = test_pki()?; + let paths = write_pki(&pki)?; + let (shard_addr, shard, shard_handle) = spawn_mtls_shard(&paths).await?; + let shard_url = format!("https://{shard_addr}"); + + let missing_tls_error = match RouterNode::connect_with_runtime_config( + vec![shard_url.clone()], + DistributedOntologyConfig::empty(), + None, + AdaptiveReconciliationConfig::default(), + RouterRpcConfig::default(), + ) + .await + { + Ok(_) => anyhow::bail!("router connected to HTTPS shard without trust material"), + Err(error) => error, + }; + assert_eq!(missing_tls_error.code(), tonic::Code::InvalidArgument); + assert!(missing_tls_error + .message() + .contains("explicit router-to-shard")); + + let shard_client_tls = load_client_mtls( + Some(&paths.ca), + Some(&paths.client_cert), + Some(&paths.client_key), + )? + .expect("complete client mTLS"); + let router = RouterNode::connect_with_runtime_config( + vec![shard_url], + DistributedOntologyConfig::empty(), + None, + AdaptiveReconciliationConfig::default(), + RouterRpcConfig { + shard_mtls: Some(shard_client_tls), + ..RouterRpcConfig::default() + }, + ) + .await?; + let (router_addr, router_handle) = spawn_mtls_router(router, &paths).await?; + + let trusted = run_healthcheck(probe_args( + "--router", + router_addr, + &paths, + &paths.client_cert, + &paths.client_key, + )) + .await?; + assert!( + trusted.status.success(), + "trusted mTLS probe failed: {}", + String::from_utf8_lossy(&trusted.stderr) + ); + + let rogue = run_healthcheck(probe_args( + "--router", + router_addr, + &paths, + &paths.rogue_client_cert, + &paths.rogue_client_key, + )) + .await?; + assert!( + !rogue.status.success(), + "router accepted a client certificate signed by an untrusted CA" + ); + + let endpoint = tonic::transport::Endpoint::from_shared(format!("https://{router_addr}"))? + .tls_config( + ClientTlsConfig::new().ca_certificate(Certificate::from_pem(pki.ca.as_bytes())), + )?; + let missing_identity_rejected = match endpoint.connect().await { + Ok(channel) => proto::router_service_client::RouterServiceClient::new(channel) + .health_check(proto::HealthCheckRequest {}) + .await + .is_err(), + Err(_) => true, + }; + assert!( + missing_identity_rejected, + "router accepted a TLS client without a certificate" + ); + + router_handle.abort(); + shard.shutdown().await?; + shard_handle.abort(); + Ok(()) +} diff --git a/tests/process_crash_recovery.rs b/tests/process_crash_recovery.rs new file mode 100644 index 0000000..c02e9d7 --- /dev/null +++ b/tests/process_crash_recovery.rs @@ -0,0 +1,245 @@ +use std::net::{SocketAddr, TcpListener}; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use tempfile::tempdir; +use tokio::time::sleep; +use unirust_rs::distributed::proto::{ + self, shard_service_client::ShardServiceClient, IngestRecordsRequest, QueryDescriptor, + QueryEntitiesRequest, RecordDescriptor, RecordIdentity, RecordInput, StatsRequest, +}; + +mod support; + +struct ShardProcess { + child: Child, +} + +impl ShardProcess { + fn spawn(listen: SocketAddr, data_dir: &Path, ontology_path: &Path) -> anyhow::Result { + let child = Command::new(env!("CARGO_BIN_EXE_unirust_shard")) + .args([ + "--listen", + &listen.to_string(), + "--shard-id", + "0", + "--data-dir", + data_dir + .to_str() + .ok_or_else(|| anyhow::anyhow!("data directory is not valid UTF-8"))?, + "--ontology", + ontology_path + .to_str() + .ok_or_else(|| anyhow::anyhow!("ontology path is not valid UTF-8"))?, + "--profile", + "balanced", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + Ok(Self { child }) + } + + fn kill_and_wait(&mut self) -> anyhow::Result<()> { + if self.child.try_wait()?.is_none() { + self.child.kill()?; + } + let _ = self.child.wait()?; + Ok(()) + } + + #[cfg(unix)] + async fn terminate_and_wait(&mut self) -> anyhow::Result<()> { + if self.child.try_wait()?.is_some() { + return Ok(()); + } + let status = Command::new("kill") + .args(["-TERM", &self.child.id().to_string()]) + .status()?; + if !status.success() { + anyhow::bail!("failed to send SIGTERM to shard process"); + } + for _ in 0..100 { + if self.child.try_wait()?.is_some() { + return Ok(()); + } + sleep(Duration::from_millis(50)).await; + } + anyhow::bail!("shard did not exit after SIGTERM"); + } +} + +impl Drop for ShardProcess { + fn drop(&mut self) { + let _ = self.kill_and_wait(); + } +} + +fn available_addr() -> anyhow::Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + let addr = listener.local_addr()?; + drop(listener); + Ok(addr) +} + +async fn wait_for_shard( + addr: SocketAddr, + process: &mut ShardProcess, +) -> anyhow::Result> { + let endpoint = format!("http://{addr}"); + for _ in 0..100 { + if let Some(status) = process.child.try_wait()? { + anyhow::bail!("shard exited before becoming ready: {status}"); + } + if let Ok(client) = ShardServiceClient::connect(endpoint.clone()).await { + return Ok(client); + } + sleep(Duration::from_millis(50)).await; + } + anyhow::bail!("shard did not become ready at {endpoint}") +} + +fn record(index: u32, uid: String, email: String) -> RecordInput { + RecordInput { + index, + identity: Some(RecordIdentity { + entity_type: "person".to_string(), + perspective: "crm".to_string(), + uid, + }), + descriptors: vec![RecordDescriptor { + attr: "email".to_string(), + value: email, + start: 0, + end: 10, + }], + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn acknowledged_ingest_survives_process_kill_and_restart() -> anyhow::Result<()> { + let temp_dir = tempdir()?; + let data_dir = temp_dir.path().join("shard-data"); + let ontology_path = temp_dir.path().join("ontology.json"); + std::fs::write( + &ontology_path, + serde_json::to_vec(&support::build_iam_config())?, + )?; + + let first_addr = available_addr()?; + let mut process = ShardProcess::spawn(first_addr, &data_dir, &ontology_path)?; + let mut client = wait_for_shard(first_addr, &mut process).await?; + + let records = (0..128) + .map(|index| { + record( + index, + format!("process-kill-{index}"), + format!("entity-{}@example.com", index / 2), + ) + }) + .collect(); + let response = client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records, + }) + .await? + .into_inner(); + assert_eq!(response.assignments.len(), 128); + let original_cluster = response + .assignments + .iter() + .find(|assignment| assignment.index == 0) + .expect("assignment for first record") + .cluster_id; + + let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(stats.record_count, 128); + assert_eq!(stats.cluster_count, 64); + + drop(client); + process.kill_and_wait()?; + + let second_addr = available_addr()?; + let mut restarted = ShardProcess::spawn(second_addr, &data_dir, &ontology_path)?; + let mut client = wait_for_shard(second_addr, &mut restarted).await?; + + let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(stats.record_count, 128); + assert_eq!(stats.cluster_count, 64); + + let query = client + .query_entities(QueryEntitiesRequest { + descriptors: vec![QueryDescriptor { + attr: "email".to_string(), + value: "entity-0@example.com".to_string(), + }], + start: 0, + end: 10, + }) + .await? + .into_inner(); + match query.outcome { + Some(proto::query_entities_response::Outcome::Matches(matches)) => { + assert_eq!(matches.matches.len(), 1); + } + other => anyhow::bail!("expected one recovered entity-resolution match, got {other:?}"), + } + + let linked_after_restart = client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record( + 128, + "process-kill-after-restart".to_string(), + "entity-0@example.com".to_string(), + )], + }) + .await? + .into_inner(); + assert_eq!(linked_after_restart.assignments.len(), 1); + assert_eq!( + linked_after_restart.assignments[0].cluster_id, original_cluster, + "a post-restart record must resolve into the pre-crash entity" + ); + + let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(stats.record_count, 129); + assert_eq!(stats.cluster_count, 64); + + #[cfg(unix)] + { + drop(client); + restarted.terminate_and_wait().await?; + + let third_addr = available_addr()?; + let mut restarted_after_shutdown = + ShardProcess::spawn(third_addr, &data_dir, &ontology_path)?; + let mut client = wait_for_shard(third_addr, &mut restarted_after_shutdown).await?; + let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(stats.record_count, 129); + assert_eq!(stats.cluster_count, 64); + + let linked_after_shutdown = client + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![record( + 129, + "graceful-shutdown-after-restart".to_string(), + "entity-0@example.com".to_string(), + )], + }) + .await? + .into_inner(); + assert_eq!(linked_after_shutdown.assignments.len(), 1); + assert_eq!( + linked_after_shutdown.assignments[0].cluster_id, original_cluster, + "a graceful restart must preserve entity-resolution identity" + ); + } + + Ok(()) +} diff --git a/tests/router_cli_safety.rs b/tests/router_cli_safety.rs new file mode 100644 index 0000000..28694fc --- /dev/null +++ b/tests/router_cli_safety.rs @@ -0,0 +1,68 @@ +use std::process::Command; + +#[test] +fn router_binary_reads_comma_separated_shards_from_environment() { + let output = Command::new(env!("CARGO_BIN_EXE_unirust_router")) + .env_clear() + .env("UNIRUST_ROUTER_SHARDS", "http://[::1,http://127.0.0.1:1") + .output() + .expect("run router binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("invalid URI"), + "environment shard list was not applied: {stderr}" + ); +} + +#[test] +fn router_cli_shards_override_environment() { + let output = Command::new(env!("CARGO_BIN_EXE_unirust_router")) + .env_clear() + .env("UNIRUST_ROUTER_SHARDS", "http://[::1") + .args(["--shards", ""]) + .output() + .expect("run router binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("at least one shard address is required"), + "CLI did not override environment shard list: {stderr}" + ); +} + +#[test] +fn router_binary_rejects_unknown_router_environment_variable() { + let output = Command::new(env!("CARGO_BIN_EXE_unirust_router")) + .env_clear() + .env("UNIRUST_ROUTER_CHECKPOINT_INTERVL_SECS", "60") + .output() + .expect("run router binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains( + "unknown Unirust configuration variable UNIRUST_ROUTER_CHECKPOINT_INTERVL_SECS" + ), + "unexpected stderr: {stderr}" + ); +} + +#[test] +fn router_binary_rejects_partial_internal_mtls_configuration() { + let output = Command::new(env!("CARGO_BIN_EXE_unirust_router")) + .env_clear() + .args(["--shard-tls-ca", "/tmp/ca.pem"]) + .output() + .expect("run router binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("router-to-shard mTLS requires all three certificate paths together"), + "unexpected stderr: {stderr}" + ); +} diff --git a/tests/shard_cli_safety.rs b/tests/shard_cli_safety.rs new file mode 100644 index 0000000..ddbada3 --- /dev/null +++ b/tests/shard_cli_safety.rs @@ -0,0 +1,76 @@ +use std::process::Command; + +use tempfile::tempdir; + +#[test] +fn shard_binary_refuses_implicit_ephemeral_storage() { + let output = Command::new(env!("CARGO_BIN_EXE_unirust_shard")) + .env_clear() + .output() + .expect("run shard binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("persistent shard storage is required"), + "unexpected stderr: {stderr}" + ); +} + +#[test] +fn shard_binary_reads_persistent_path_from_environment() { + let source = tempdir().expect("temporary checkpoint source"); + let destination = source.path().join("replacement"); + let output = Command::new(env!("CARGO_BIN_EXE_unirust_shard")) + .env_clear() + .env("UNIRUST_SHARD_DATA_DIR", &destination) + .args([ + "--restore-from", + source.path().to_str().expect("UTF-8 path"), + ]) + .output() + .expect("run shard binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("not a RocksDB checkpoint"), + "environment data directory was not applied: {stderr}" + ); + assert!( + !stderr.contains("persistent shard storage is required"), + "environment data directory was ignored: {stderr}" + ); +} + +#[test] +fn shard_binary_rejects_unknown_shard_environment_variable() { + let output = Command::new(env!("CARGO_BIN_EXE_unirust_shard")) + .env_clear() + .env("UNIRUST_SHARD_DAT_DIR", "/tmp/unirust-typo") + .output() + .expect("run shard binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("unknown Unirust configuration variable UNIRUST_SHARD_DAT_DIR"), + "unexpected stderr: {stderr}" + ); +} + +#[test] +fn shard_binary_rejects_partial_mtls_configuration() { + let output = Command::new(env!("CARGO_BIN_EXE_unirust_shard")) + .env_clear() + .args(["--tls-cert", "/tmp/server.pem"]) + .output() + .expect("run shard binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("shard mTLS requires all three certificate paths together"), + "unexpected stderr: {stderr}" + ); +} diff --git a/tests/temporal_evolution.rs b/tests/temporal_evolution.rs index b2cdcd4..cc89bfb 100644 --- a/tests/temporal_evolution.rs +++ b/tests/temporal_evolution.rs @@ -14,7 +14,6 @@ use std::collections::HashSet; use std::net::SocketAddr; -#[allow(unused_imports)] use tempfile::tempdir; use tokio::task::JoinHandle; use tokio_stream::wrappers::TcpListenerStream; @@ -29,18 +28,27 @@ use unirust_rs::{StreamingTuning, TuningProfile}; mod support; +// Each scenario starts a persistent cluster. Serialize this integration file so +// concurrent RocksDB instances stay within the process file-descriptor limit. +static PERSISTENT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + async fn spawn_shard( shard_id: u32, config: DistributedOntologyConfig, ) -> anyhow::Result<(SocketAddr, JoinHandle<()>)> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let shard = ShardNode::new( + let data_dir = tempdir()?; + let shard = ShardNode::new_with_data_dir( shard_id, config, StreamingTuning::from_profile(TuningProfile::Balanced), + Some(data_dir.path().to_path_buf()), + false, + None, )?; let handle = tokio::spawn(async move { + let _data_dir = data_dir; Server::builder() .add_service(proto::shard_service_server::ShardServiceServer::new(shard)) .serve_with_incoming(TcpListenerStream::new(listener)) @@ -157,6 +165,7 @@ async fn setup_cluster() -> anyhow::Result<(RouterServiceClient anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; // Setup: Two separate clusters, one for incremental, one for batch let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -220,16 +229,19 @@ async fn incremental_ingestion_equals_batch_ingestion() -> anyhow::Result<()> { // Ingest incrementally client_inc .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record_c1.clone()], }) .await?; client_inc .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record_c2.clone()], }) .await?; client_inc .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record_c3.clone()], }) .await?; @@ -237,6 +249,7 @@ async fn incremental_ingestion_equals_batch_ingestion() -> anyhow::Result<()> { // Batch: Ingest all 3 at once client_batch .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record_c1, record_c2, record_c3], }) .await?; @@ -295,6 +308,7 @@ async fn incremental_ingestion_equals_batch_ingestion() -> anyhow::Result<()> { /// Ingesting C1→C2→C3 should produce same result as C3→C2→C1 or C2→C1→C3. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ingestion_order_independence() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -356,6 +370,7 @@ async fn ingestion_order_independence() -> anyhow::Result<()> { for &idx in order { client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![records[idx].clone()], }) .await?; @@ -396,6 +411,7 @@ async fn ingestion_order_independence() -> anyhow::Result<()> { /// Entity has email in month 1, phone in month 2, address in month 3. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn temporal_descriptor_evolution() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let (mut client,) = setup_cluster().await?; // Entity evolves over time with different descriptors from different source records. @@ -430,13 +446,22 @@ async fn temporal_descriptor_evolution() -> anyhow::Result<()> { // Ingest over time client - .ingest_records(IngestRecordsRequest { records: vec![jan] }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![jan], + }) .await?; client - .ingest_records(IngestRecordsRequest { records: vec![feb] }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![feb], + }) .await?; client - .ingest_records(IngestRecordsRequest { records: vec![mar] }) + .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, + records: vec![mar], + }) .await?; // Query by email - should find the entity @@ -522,6 +547,7 @@ async fn temporal_descriptor_evolution() -> anyhow::Result<()> { /// Entity gets additional contact info from a second source record. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn attribute_value_changes_over_time() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let (mut client,) = setup_cluster().await?; // Two source records that share an identity key (email) and merge. @@ -546,6 +572,7 @@ async fn attribute_value_changes_over_time() -> anyhow::Result<()> { client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![initial, with_secondary], }) .await?; @@ -612,6 +639,7 @@ async fn attribute_value_changes_over_time() -> anyhow::Result<()> { /// and can be queried incrementally. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn multi_perspective_incremental_merge() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let (mut client,) = setup_cluster().await?; // CRM perspective first @@ -624,6 +652,7 @@ async fn multi_perspective_incremental_merge() -> anyhow::Result<()> { ); client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![crm_record], }) .await?; @@ -654,6 +683,7 @@ async fn multi_perspective_incremental_merge() -> anyhow::Result<()> { ); client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![erp_record], }) .await?; @@ -690,6 +720,7 @@ async fn multi_perspective_incremental_merge() -> anyhow::Result<()> { ); client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![hr_record], }) .await?; @@ -720,6 +751,7 @@ async fn multi_perspective_incremental_merge() -> anyhow::Result<()> { /// Test idempotency: re-ingesting the same record should not change state. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn idempotent_re_ingestion() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let (mut client,) = setup_cluster().await?; let record = record_input( @@ -736,6 +768,7 @@ async fn idempotent_re_ingestion() -> anyhow::Result<()> { // First ingestion let resp1 = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record.clone()], }) .await? @@ -745,6 +778,7 @@ async fn idempotent_re_ingestion() -> anyhow::Result<()> { // Second ingestion (identical) let resp2 = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record.clone()], }) .await? @@ -760,6 +794,7 @@ async fn idempotent_re_ingestion() -> anyhow::Result<()> { // Third ingestion let resp3 = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record], }) .await? @@ -788,6 +823,7 @@ async fn idempotent_re_ingestion() -> anyhow::Result<()> { /// Test that partial descriptor overlap correctly extends entity validity. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn overlapping_temporal_ranges_extend_validity() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let (mut client,) = setup_cluster().await?; // First record: email valid Jan-Mar @@ -799,22 +835,25 @@ async fn overlapping_temporal_ranges_extend_validity() -> anyhow::Result<()> { vec![("email", "overlap@example.com", 202401, 202404)], ); - // Second record: same entity, email valid Feb-Apr (overlapping) + // Second source snapshot: same resolved entity, email valid Feb-Apr (overlapping). + // Source record identities are immutable, so each snapshot has its own UID. let feb_apr = record_input( 1, "person", "crm", - "overlap_001", + "overlap_002", vec![("email", "overlap@example.com", 202402, 202405)], ); client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![jan_mar], }) .await?; client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![feb_apr], }) .await?; @@ -835,6 +874,27 @@ async fn overlapping_temporal_ranges_extend_validity() -> anyhow::Result<()> { "entity found across full range" ); + // The extension-only interval proves the second snapshot was persisted. + let extension_response = client + .query_entities(QueryEntitiesRequest { + descriptors: vec![ProtoQueryDescriptor { + attr: "email".to_string(), + value: "overlap@example.com".to_string(), + }], + start: 202404, + end: 202405, + }) + .await? + .into_inner(); + assert_eq!( + count_matches(&extension_response), + 1, + "entity found in extension-only interval" + ); + + let stats = client.get_stats(StatsRequest {}).await?.into_inner(); + assert_eq!(stats.record_count, 2, "both source snapshots persisted"); + Ok(()) } @@ -842,6 +902,7 @@ async fn overlapping_temporal_ranges_extend_validity() -> anyhow::Result<()> { /// Entity email spans full range, but phone has a gap (only in Q1 and Q3). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn temporal_gap_between_records() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let (mut client,) = setup_cluster().await?; // Two source records with the same email (spanning full range for identity key overlap). @@ -871,6 +932,7 @@ async fn temporal_gap_between_records() -> anyhow::Result<()> { client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![q1, q3], }) .await?; @@ -967,6 +1029,7 @@ async fn temporal_gap_between_records() -> anyhow::Result<()> { /// regardless of which record is ingested first. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn same_cluster_regardless_of_insertion_order() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -1021,6 +1084,7 @@ async fn same_cluster_regardless_of_insertion_order() -> anyhow::Result<()> { for &idx in order { let resp = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![records[idx].clone()], }) .await? @@ -1052,6 +1116,7 @@ async fn same_cluster_regardless_of_insertion_order() -> anyhow::Result<()> { /// All records in a batch with same identity key go to same cluster. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn batch_ingestion_consistent_cluster() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -1094,6 +1159,7 @@ async fn batch_ingestion_consistent_cluster() -> anyhow::Result<()> { let batch_resp = batch_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: records.clone(), }) .await? @@ -1120,6 +1186,7 @@ async fn batch_ingestion_consistent_cluster() -> anyhow::Result<()> { for record in &records { let resp = seq_client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![record.clone()], }) .await? @@ -1145,6 +1212,7 @@ async fn batch_ingestion_consistent_cluster() -> anyhow::Result<()> { /// regardless of interleaved insertion order. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn independent_entities_stay_separate_any_order() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let config = support::build_iam_config(); let empty_config = DistributedOntologyConfig::empty(); @@ -1203,6 +1271,7 @@ async fn independent_entities_stay_separate_any_order() -> anyhow::Result<()> { for &idx in order { client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![records[idx].clone()], }) .await?; @@ -1268,6 +1337,7 @@ async fn independent_entities_stay_separate_any_order() -> anyhow::Result<()> { /// Test late-arriving record merges into existing cluster correctly. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn late_arrival_merges_into_existing_cluster() -> anyhow::Result<()> { + let _test_guard = PERSISTENT_TEST_LOCK.lock().await; let (mut client,) = setup_cluster().await?; // First, ingest two records that form a cluster @@ -1288,6 +1358,7 @@ async fn late_arrival_merges_into_existing_cluster() -> anyhow::Result<()> { let resp1 = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![rec1, rec2], }) .await? @@ -1314,6 +1385,7 @@ async fn late_arrival_merges_into_existing_cluster() -> anyhow::Result<()> { let resp2 = client .ingest_records(IngestRecordsRequest { + internal_protocol_version: 3, records: vec![late_rec], }) .await?