Merge #158781

158781: drtprod: 300 node scale test config for 2025-12 r=shailendra-patel a=herkolategan

Created a config for the 300 node scale test Dec 2025.

Release note: None
Epic: None

Co-authored-by: Herko Lategan <herko@cockroachlabs.com>

Author: craig[bot]

pkg/cmd/drtprod/configs/archived/2025_05_drt_scale_300.yaml

@@ -0,0 +1,171 @@
+# Yaml for creating and configuring the drt-scale cluster. This also configures Datadog.
+# Build the roachprod and roachtest binaries (using --cross) before running this script
+environment:
+  ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: 622274581499-compute@developer.gserviceaccount.com
+  ROACHPROD_DNS: drt.crdb.io
+  ROACHPROD_GCE_DNS_DOMAIN: drt.crdb.io
+  ROACHPROD_GCE_DNS_ZONE: drt
+  ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
+  CLUSTER: drt-scale-300
+  WORKLOAD_CLUSTER: workload-scale-300
+  CLUSTER_NODES: 300
+  TOTAL_PARTITIONS: 15
+  WORKLOAD_NODES: 15
+
+dependent_file_locations:
+  - pkg/cmd/drtprod/scripts/setup_datadog_cluster
+  - pkg/cmd/drtprod/scripts/setup_datadog_workload
+  - pkg/cmd/drtprod/scripts/tpcc_init.sh
+  - pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
+  - artifacts/roachtest
+  - artifacts/drtprod
+
+targets:
+  # crdb cluster specs
+  - target_name: $CLUSTER
+    steps:
+      - command: create
+        args:
+          - $CLUSTER
+        flags:
+          clouds: gce
+          gce-managed: true
+          gce-enable-multiple-stores: true
+          gce-zones: "us-central1-a:30,us-central1-b:30,us-central1-c:30"
+          nodes: $CLUSTER_NODES
+          gce-machine-type: n2-standard-16
+          local-ssd: false
+          gce-pd-volume-size: 2048
+          gce-pd-volume-type: pd-ssd
+          gce-pd-volume-count: 2
+          os-volume-size: 100
+          username: drt
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20250112"
+      - command: sync
+        flags:
+          clouds: gce
+      - command: stage
+        args:
+          - $CLUSTER
+          - release
+          - v25.2.0-rc.1 # for libgeos
+      - command: stage
+        args:
+          - $CLUSTER
+          - cockroach
+          - release-25.2.1-rc
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_cluster"
+      - command: start
+        args:
+          - $CLUSTER
+          - "--binary"
+          - "./cockroach"
+        flags:
+          # add flag to set provisioned throughput on each store according to their cloud provider limits
+          enable-fluent-sink: true
+          store-count: 2
+          args: --wal-failover=among-stores
+          restart: false
+          sql-port: 26257
+      - command: run
+        args:
+          - $CLUSTER
+          - --
+          - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING kv.snapshot_rebalance.max_rate='256 MB'"
+  # workload cluster specs
+  - target_name: $WORKLOAD_CLUSTER
+    steps:
+      - command: create
+        args:
+          - $WORKLOAD_CLUSTER
+        flags:
+          clouds: gce
+          gce-zones: "us-central1-a"
+          nodes: $WORKLOAD_NODES
+          gce-machine-type: n2-standard-8
+          os-volume-size: 100
+          username: workload
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20250112"
+        on_rollback:
+          - command: destroy
+            args:
+              - $WORKLOAD_CLUSTER
+      - command: sync
+        flags:
+          clouds: gce
+      - command: stage
+        args:
+          - $WORKLOAD_CLUSTER
+          - cockroach
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - artifacts/roachtest
+          - roachtest-operations
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - artifacts/drtprod
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_workload"
+  - target_name: post_tasks
+    dependent_targets:
+      - $CLUSTER
+      - $WORKLOAD_CLUSTER
+    steps:
+      - script: rm
+        args:
+          - -rf
+          - certs-$CLUSTER
+      - command: get
+        args:
+          - $CLUSTER:1
+          - certs
+          - certs-$CLUSTER
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - certs-$CLUSTER
+          - certs
+      - command: ssh
+        args:
+          - $WORKLOAD_CLUSTER
+          - --
+          - chmod
+          - 600
+          - './certs/*'
+      - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
+        args:
+          - cct_tpcc
+          - false
+        flags:
+          warehouses: 4000000
+          db: cct_tpcc
+      - script: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
+  - target_name: tpcc_run
+    dependent_targets:
+      - $CLUSTER
+      - $WORKLOAD_CLUSTER
+    steps:
+      - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh"
+        args:
+          - cct_tpcc
+          - false
+        flags:
+          db: cct_tpcc
+          warehouses: 4000000
+          active-warehouses: 500000
+          active-workers: 2000
+          conns: 2000
+          max-rate: 2500
+          workers: 500000
+          duration: 12h
+          ramp: 1h
+          wait: 0

…drtprod/configs/drt_scale_300_bench.yaml …rchived/2025_07_drt_scale_300_bench.yamlpkg/cmd/drtprod/configs/drt_scale_300_bench.yaml renamed to pkg/cmd/drtprod/configs/archived/2025_07_drt_scale_300_bench.yaml pkg/cmd/drtprod/configs/drt_scale_300_bench.yaml renamed to pkg/cmd/drtprod/configs/archived/2025_07_drt_scale_300_bench.yaml

@@ -1,5 +1,7 @@
-# Yaml for creating and configuring the drt-scale cluster. This also configures Datadog.
-# Build the roachprod and roachtest binaries (using --cross) before running this script
+# YAML for creating and configuring the drt-scale cluster. This also configures Datadog.
+# Build the drtprod and roachtest binaries (using --cross=linux) before running this script
+#
+# Planned Execution Date: 2025-12
 environment:
   ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: 622274581499-compute@developer.gserviceaccount.com
   ROACHPROD_DNS: drt.crdb.io
@@ -9,20 +11,28 @@ environment:
   CLUSTER: drt-scale-300
   WORKLOAD_CLUSTER: workload-scale-300
   CLUSTER_NODES: 300
-  TOTAL_PARTITIONS: 15
+  RACKS: 300
+  NODES_PER_ZONE: 100
+  TOTAL_PARTITIONS: 300
+  PARTITION_TYPE: partitions
   WORKLOAD_NODES: 15
+  VERSION: v25.4.1 # TBD - update once a custom binary is available (also update stage command)
+  WAREHOUSES: 4000000
 
 dependent_file_locations:
+  - pkg/cmd/drtprod/scripts/setup_dmsetup_disk_staller
   - pkg/cmd/drtprod/scripts/setup_datadog_cluster
   - pkg/cmd/drtprod/scripts/setup_datadog_workload
   - pkg/cmd/drtprod/scripts/tpcc_init.sh
   - pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
+  - pkg/cmd/drtprod/scripts/populate_workload_keys.sh
   - artifacts/roachtest
   - artifacts/drtprod
 
 targets:
   # crdb cluster specs
-  - target_name: $CLUSTER
+  - target_name: $CLUSTER cluster initialisation
+    notify_progress: true
     steps:
       - command: create
         args:
@@ -31,7 +41,7 @@ targets:
           clouds: gce
           gce-managed: true
           gce-enable-multiple-stores: true
-          gce-zones: "us-central1-a:30,us-central1-b:30,us-central1-c:30"
+          gce-zones: "us-central1-a:$NODES_PER_ZONE,us-central1-b:$NODES_PER_ZONE,us-central1-c:$NODES_PER_ZONE"
           nodes: $CLUSTER_NODES
           gce-machine-type: n2-standard-16
           local-ssd: false
@@ -43,36 +53,38 @@ targets:
           lifetime: 8760h
           gce-image: "ubuntu-2204-jammy-v20250112"
       - command: sync
+        skip_notification: true
         flags:
           clouds: gce
+      - script: "pkg/cmd/drtprod/scripts/setup_dmsetup_disk_staller"
+        skip_notification: true
       - command: stage
+        skip_notification: true
         args:
           - $CLUSTER
           - release
-          - v25.2.0-rc.1 # for libgeos
-      - command: stage
-        args:
-          - $CLUSTER
-          - cockroach
-          - release-25.2.1-rc
+          - $VERSION
       - script: "pkg/cmd/drtprod/scripts/setup_datadog_cluster"
+        skip_notification: true
       - command: start
         args:
           - $CLUSTER
           - "--binary"
           - "./cockroach"
+          - "--env" # from defaults
+          - "COCKROACH_TESTING_FORCE_RELEASE_BRANCH=true"
+          - "--env" # from defaults
+          - "COCKROACH_INTERNAL_DISABLE_METAMORPHIC_TESTING=true"
+          - "--env" # for MMA test case scenario
+          - "COCKROACH_ALLOW_MMA=true"
         flags:
           # add flag to set provisioned throughput on each store according to their cloud provider limits
           enable-fluent-sink: true
           store-count: 2
           args: --wal-failover=among-stores
           restart: false
           sql-port: 26257
-      - command: run
-        args:
-          - $CLUSTER
-          - --
-          - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
+          racks: $RACKS
       - command: sql
         args:
           - $CLUSTER:1
@@ -104,7 +116,8 @@ targets:
       - command: stage
         args:
           - $WORKLOAD_CLUSTER
-          - cockroach
+          - release
+          - $VERSION
       - command: put
         args:
           - $WORKLOAD_CLUSTER
@@ -116,42 +129,42 @@ targets:
           - artifacts/drtprod
       - script: "pkg/cmd/drtprod/scripts/setup_datadog_workload"
   - target_name: post_tasks
+    notify_progress: true
     dependent_targets:
-      - $CLUSTER
+      - $CLUSTER cluster initialisation
       - $WORKLOAD_CLUSTER
     steps:
       - script: rm
+        skip_notification: true
         args:
           - -rf
           - certs-$CLUSTER
-      - command: get
+      - command: fetch-certs
+        skip_notification: true
         args:
           - $CLUSTER:1
-          - certs
           - certs-$CLUSTER
       - command: put
+        skip_notification: true
         args:
           - $WORKLOAD_CLUSTER
           - certs-$CLUSTER
           - certs
-      - command: ssh
-        args:
-          - $WORKLOAD_CLUSTER
-          - --
-          - chmod
-          - 600
-          - './certs/*'
       - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
         args:
           - cct_tpcc
           - false
         flags:
-          warehouses: 4000000
+          partitions: $TOTAL_PARTITIONS
+          replicate-static-columns: true
+          partition-strategy: leases
+          warehouses: $WAREHOUSES
           db: cct_tpcc
       - script: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
   - target_name: tpcc_run
+    notify_progress: true
     dependent_targets:
-      - $CLUSTER
+      - $CLUSTER cluster initialisation
       - $WORKLOAD_CLUSTER
     steps:
       - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh"
@@ -160,12 +173,11 @@ targets:
           - false
         flags:
           db: cct_tpcc
-          warehouses: 4000000
-          active-warehouses: 500000
-          active-workers: 2000
-          conns: 2000
-          max-rate: 2500
-          workers: 500000
+          warehouses:  $WAREHOUSES
+          active-warehouses: 266666
+          workers: 266666
+          conns: 1000
+          active-workers: 1000
           duration: 12h
           ramp: 1h
           wait: 0

pkg/cmd/drtprod/configs/drt_scale_300.yaml

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Sets up datadog for the drt clusters.
+# Sets up disk staller support (dmsetup) for /mnt/data1.
 # NOTE - This uses CLUSTER environment variable, if not set the script fails
 
 if [ -z "${CLUSTER}" ]; then