Harden durability, backups, and synchronous replication

.github/workflows/ci.yml

@@ -62,3 +62,17 @@ jobs:
         run: |
           bash -n scripts/*.sh
           docker compose -f compose.yaml config --quiet
+
+  container-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build production image
+        run: docker build --file Containerfile --tag unirust-ci .

.github/workflows/release.yml

@@ -26,6 +26,8 @@ jobs:
         run: cargo clippy --locked --all-targets --all-features -- -D warnings
       - name: cargo package
         run: cargo package --locked
+      - name: Build production image
+        run: docker build --file Containerfile --tag unirust-release .
       - name: cargo publish
         run: cargo publish --locked
         env:

Cargo.toml

@@ -7,8 +7,8 @@ description = "High-performance temporal entity resolution engine with conflict
 authors = ["unirust contributors"]
 license = "MIT"
 readme = "README.md"
-repository = "https://github.com/unirust/unirust"
-documentation = "https://docs.rs/unirust"
+repository = "https://github.com/script3r/unirust"
+documentation = "https://docs.rs/unirust-rs"
 keywords = ["entity-resolution", "temporal", "data-deduplication", "conflict-resolution"]
 categories = ["data-structures", "algorithms"]
 

Containerfile

@@ -4,7 +4,7 @@
 #
 # Usage:
 #   podman build -t unirust -f Containerfile .
-#   podman run --rm -p 50061:50061 -v unirust-data:/data unirust shard
+#   podman run --rm -p 50061:50061 -v unirust-data:/data -v unirust-backup:/backup unirust shard
 #   podman run --rm -p 50060:50060 unirust router --shards shard-0:50061
 #
 # Or use with compose.yaml for full cluster deployment
@@ -17,6 +17,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     cmake \
+    libclang-dev \
     protobuf-compiler \
     && rm -rf /var/lib/apt/lists/*
 
@@ -31,6 +32,8 @@ RUN cargo build --release --locked --features test-support \
     --bin unirust_shard \
     --bin unirust_router \
     --bin unirust_healthcheck \
+    --bin unirust_backup \
+    --bin unirust_rebalance \
     --bin unirust_client \
     --bin unirust_loadtest
 
@@ -49,6 +52,8 @@ RUN useradd -m -u 1000 unirust
 COPY --from=builder /app/target/release/unirust_shard /usr/local/bin/
 COPY --from=builder /app/target/release/unirust_router /usr/local/bin/
 COPY --from=builder /app/target/release/unirust_healthcheck /usr/local/bin/
+COPY --from=builder /app/target/release/unirust_backup /usr/local/bin/
+COPY --from=builder /app/target/release/unirust_rebalance /usr/local/bin/
 COPY --from=builder /app/target/release/unirust_client /usr/local/bin/
 COPY --from=builder /app/target/release/unirust_loadtest /usr/local/bin/
 
@@ -82,6 +87,14 @@ case "$1" in
         shift
         exec unirust_loadtest "$@"
         ;;
+    backup)
+        shift
+        exec unirust_backup "$@"
+        ;;
+    rebalance)
+        shift
+        exec unirust_rebalance "$@"
+        ;;
     *)
         exec "$@"
         ;;
@@ -91,4 +104,4 @@ EOF
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
-CMD ["shard", "--shard-id", "0", "--data-dir", "/data"]
+CMD ["shard", "--shard-id", "0", "--data-dir", "/data", "--backup-dir", "/backup"]

compose.yaml

@@ -101,7 +101,7 @@ services:
     volumes:
       - ./examples/loadtest-ontology.json:/etc/unirust/ontology.json:ro
     ports:
-      - "50060:50060"
+      - "127.0.0.1:50060:50060"
     depends_on:
       shard-0:
         condition: service_healthy

examples/cluster.rs

@@ -27,9 +27,9 @@
 //! SHARDS=3 ./scripts/cluster.sh start
 //!
 //! # Option 2: Start manually
-//! ./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /tmp/shard0
-//! ./target/release/unirust_shard --listen 127.0.0.1:50062 --shard-id 1 --data-dir /tmp/shard1
-//! ./target/release/unirust_shard --listen 127.0.0.1:50063 --shard-id 2 --data-dir /tmp/shard2
+//! ./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /tmp/shard0 --backup-dir /tmp/shard0-backup
+//! ./target/release/unirust_shard --listen 127.0.0.1:50062 --shard-id 1 --data-dir /tmp/shard1 --backup-dir /tmp/shard1-backup
+//! ./target/release/unirust_shard --listen 127.0.0.1:50063 --shard-id 2 --data-dir /tmp/shard2 --backup-dir /tmp/shard2-backup
 //! ./target/release/unirust_router --listen 127.0.0.1:50060 \
 //!     --shards 127.0.0.1:50061,127.0.0.1:50062,127.0.0.1:50063
 //! ```
@@ -73,13 +73,13 @@ async fn main() -> anyhow::Result<()> {
             eprintln!("  SHARDS=3 ./scripts/cluster.sh start\n");
             eprintln!("Or manually:");
             eprintln!(
-                "  ./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /tmp/shard0"
+                "  ./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /tmp/shard0 --backup-dir /tmp/shard0-backup"
             );
             eprintln!(
-                "  ./target/release/unirust_shard --listen 127.0.0.1:50062 --shard-id 1 --data-dir /tmp/shard1"
+                "  ./target/release/unirust_shard --listen 127.0.0.1:50062 --shard-id 1 --data-dir /tmp/shard1 --backup-dir /tmp/shard1-backup"
             );
             eprintln!(
-                "  ./target/release/unirust_shard --listen 127.0.0.1:50063 --shard-id 2 --data-dir /tmp/shard2"
+                "  ./target/release/unirust_shard --listen 127.0.0.1:50063 --shard-id 2 --data-dir /tmp/shard2 --backup-dir /tmp/shard2-backup"
             );
             eprintln!("  ./target/release/unirust_router --listen 127.0.0.1:50060 \\");
             eprintln!("      --shards 127.0.0.1:50061,127.0.0.1:50062,127.0.0.1:50063");

examples/unirust.toml

@@ -15,12 +15,28 @@ profile = "high-throughput"
 listen = "0.0.0.0:50061"
 # Shard ID (0-indexed, unique per shard)
 id = 0
-# Data directory for persistence (comment out for in-memory only)
+# Data directory for persistence (`--ephemeral` is required when omitted)
 # data_dir = "/var/lib/unirust/shard-0"
 # Checkpoint root on an independent volume (use a unique path per shard)
 # backup_dir = "/var/backups/unirust/shard-0"
 # Path to ontology configuration (JSON)
 # ontology = "/etc/unirust/ontology.json"
+# Mutually authenticated TLS server credentials (all three are required together)
+# tls_cert = "/etc/unirust/tls/shard-0.crt"
+# tls_key = "/etc/unirust/tls/shard-0.key"
+# tls_client_ca = "/etc/unirust/tls/clients-ca.crt"
+# Primary-only synchronous replica settings. Bootstrap both volumes from the
+# same committed checkpoint before enabling these options.
+# replica = "https://shard-0-replica:50061"
+# replication_token_file = "/etc/unirust/replication/shard-0.token"
+# replica_tls_ca = "/etc/unirust/tls/replicas-ca.crt"
+# replica_tls_cert = "/etc/unirust/tls/shard-0-primary.crt"
+# replica_tls_key = "/etc/unirust/tls/shard-0-primary.key"
+# On the passive process, omit `replica` and set:
+# replica_mode = true
+# Plaintext replication is rejected unless explicitly enabled for isolated
+# development:
+# allow_insecure_replication = false
 # Run repair on startup (recovers from unclean shutdown)
 repair = false
 
@@ -44,6 +60,14 @@ shard_request_timeout_secs = 120
 shard_tcp_keepalive_secs = 30
 # Automatic coordinated checkpoint interval (0 disables)
 # checkpoint_interval_secs = 3600
+# Mutually authenticated TLS for clients connecting to the router
+# tls_cert = "/etc/unirust/tls/router.crt"
+# tls_key = "/etc/unirust/tls/router.key"
+# tls_client_ca = "/etc/unirust/tls/clients-ca.crt"
+# Router identity and trust for https:// shard addresses
+# shard_tls_ca = "/etc/unirust/tls/shards-ca.crt"
+# shard_tls_cert = "/etc/unirust/tls/router-client.crt"
+# shard_tls_key = "/etc/unirust/tls/router-client.key"
 
 # Advanced storage tuning (RocksDB)
 # These settings control the underlying storage engine performance.

proto/unirust.proto

@@ -255,7 +255,16 @@ message IngestRecordsFromUrlRequest {
 
 // --- Config Version ---
 
-message ConfigVersionRequest {}
+message ConfigVersionRequest {
+  bool include_durable_state_digest = 1;
+}
+
+enum ShardRole {
+  SHARD_ROLE_UNSPECIFIED = 0;
+  SHARD_ROLE_STANDALONE = 1;
+  SHARD_ROLE_PRIMARY = 2;
+  SHARD_ROLE_REPLICA = 3;
+}
 
 message ConfigVersionResponse {
   string version = 1;
@@ -267,6 +276,9 @@ message ConfigVersionResponse {
   // Empty/zero for a volume that has never been restored from a checkpoint.
   string restore_generation = 7;
   uint32 restore_shard_count = 8;
+  ShardRole shard_role = 9;
+  // SHA-256 over every logical RocksDB column-family key/value pair.
+  bytes durable_state_digest = 10;
 }
 
 // --- Durable Source Identity Reservation ---
@@ -456,13 +468,20 @@ message BoundaryKeyEntries {
 
 message GetBoundaryMetadataRequest {
   uint64 since_version = 1;
+  // Return metadata only for these signatures. Empty requests return shard
+  // identity/version without materializing the full boundary index.
+  repeated IdentityKeySignature signatures = 2;
 }
 
 message GetBoundaryMetadataResponse {
   BoundaryMetadata metadata = 1;
 }
 
-message GetDirtyBoundaryKeysRequest {}
+message GetDirtyBoundaryKeysRequest {
+  // Exclusive lexicographic cursor over the 32-byte signature.
+  bytes after_signature = 1;
+  uint32 limit = 2;
+}
 
 message DirtyBoundaryKey {
   IdentityKeySignature signature = 1;
@@ -472,6 +491,8 @@ message DirtyBoundaryKey {
 message GetDirtyBoundaryKeysResponse {
   repeated DirtyBoundaryKey dirty_keys = 1;
   uint32 shard_id = 2;
+  bytes next_after_signature = 3;
+  bool has_more = 4;
 }
 
 message ClearDirtyKeysRequest {
@@ -511,6 +532,16 @@ message ApplyMergeResponse {
   string error = 3;
 }
 
+message ApplyMergesRequest {
+  repeated ClusterMerge merges = 1;
+}
+
+message ApplyMergesResponse {
+  bool success = 1;
+  uint64 records_updated = 2;
+  string error = 3;
+}
+
 // --- Cross-Shard Conflict Propagation ---
 
 // A cross-shard conflict detected during reconciliation.
@@ -580,6 +611,7 @@ service ShardService {
   // Cross-shard reconciliation
   rpc GetBoundaryMetadata(GetBoundaryMetadataRequest) returns (GetBoundaryMetadataResponse);
   rpc ApplyMerge(ApplyMergeRequest) returns (ApplyMergeResponse);
+  rpc ApplyMerges(ApplyMergesRequest) returns (ApplyMergesResponse);
   rpc GetDirtyBoundaryKeys(GetDirtyBoundaryKeysRequest) returns (GetDirtyBoundaryKeysResponse);
   rpc ClearDirtyKeys(ClearDirtyKeysRequest) returns (ClearDirtyKeysResponse);
 

scripts/podman_cluster.sh

@@ -9,8 +9,10 @@ ROUTER_PORT="${ROUTER_PORT:-50060}"
 ONTOLOGY="${ONTOLOGY:-$ROOT_DIR/examples/loadtest-ontology.json}"
 CONFIG_VERSION="${CONFIG_VERSION:-}"
 PROFILE="${PROFILE:-high-throughput}"
+CHECKPOINT_INTERVAL_SECS="${CHECKPOINT_INTERVAL_SECS:-3600}"
 CONTAINER_PREFIX="${CONTAINER_PREFIX:-unirust}"
 VOLUME_PREFIX="${VOLUME_PREFIX:-unirust-data}"
+BACKUP_VOLUME_PREFIX="${BACKUP_VOLUME_PREFIX:-unirust-backup}"
 STOP_WAIT_SECS="${STOP_WAIT_SECS:-10}"
 UNIRUST_CONFIRM_RESET="${UNIRUST_CONFIRM_RESET:-0}"
 
@@ -61,11 +63,13 @@ start_cluster() {
       --network "$NETWORK" \
       --restart=unless-stopped \
       -v "${VOLUME_PREFIX}-shard-$i:/data" \
+      -v "${BACKUP_VOLUME_PREFIX}-shard-$i:/backup" \
       "${ontology_mount[@]}" \
       "$IMAGE" \
       shard \
       --shard-id "$i" \
       --data-dir /data \
+      --backup-dir /backup \
       --profile "$PROFILE" \
       "${ontology_args[@]}" \
       "${version_args[@]}"
@@ -90,12 +94,14 @@ start_cluster() {
     "$IMAGE" \
     router \
     --shards "$shard_list" \
+    --checkpoint-interval-secs "$CHECKPOINT_INTERVAL_SECS" \
     "${ontology_args[@]}" \
     "${version_args[@]}"
 
   echo "Cluster started."
   echo "Router listening on localhost:${ROUTER_PORT}"
   echo "Persistent volumes use prefix: ${VOLUME_PREFIX}-shard-"
+  echo "Checkpoint volumes use prefix: ${BACKUP_VOLUME_PREFIX}-shard-"
 }
 
 stop_cluster() {
@@ -125,11 +131,12 @@ reset_cluster() {
   fi
   stop_cluster
   while IFS= read -r volume; do
-    if [[ "$volume" == "${VOLUME_PREFIX}-shard-"* ]]; then
+    if [[ "$volume" == "${VOLUME_PREFIX}-shard-"* || "$volume" == "${BACKUP_VOLUME_PREFIX}-shard-"* ]]; then
       podman volume rm "$volume"
     fi
   done < <(podman volume ls --format "{{.Name}}")
   echo "Deleted persistent volumes with prefix: ${VOLUME_PREFIX}-shard-"
+  echo "Deleted checkpoint volumes with prefix: ${BACKUP_VOLUME_PREFIX}-shard-"
 }
 
 status_cluster() {