Merge #158515

158515: cli/interactive_tests: wait for liveness replication before node shutdowns r=rafiss a=rafiss

This commit extends the fix from 015b29c to cover all node shutdown operations in test_demo_node_cmds.tcl. Previously, the test only waited for the liveness range to have 5 voting replicas before decommissioning a node, but not before shutdown operations.

Without this check, if the liveness range has fewer than 3 voting replicas when a node is shut down, and that node is one of them, the cluster can lose quorum on the liveness range. This causes subsequent queries to system tables (particularly crdb_internal.kv_node_liveness) to hang, resulting in test timeouts.

The fix adds the liveness range replication check in two places:
1. Before shutting down node 3 (the first shutdown operation)
2. Before shutting down node 6 (after adding a new node)

fixes #158208
Release note: None

Co-authored-by: Rafi Shamim <rafi@cockroachlabs.com>

Author: craig[bot]

pkg/cli/interactive_tests/test_demo_node_cmds.tcl

@@ -36,6 +36,26 @@ send "\\demo restart 2\r"
 eexpect "node 2 is already running"
 eexpect "defaultdb>"
 
+# Wait for the liveness range to have the default 5 voters. If its replication
+# factor is too low, shutting down the node below can cause it to lose quorum
+# and stall queries to system tables (example: #147867).
+set timeout 2
+set stmt "select range_id, array_length(voting_replicas,1) from crdb_internal.ranges where range_id=2;\r"
+send $stmt
+expect {
+    "2 |            5" {
+        puts "\rliveness range has 5 voters"
+    }
+    timeout {
+        puts "\rliveness range does not yet have 5 voters"
+        sleep 2
+        send $stmt
+        exp_continue
+    }
+}
+# Reset timeout back to 45 to match common.tcl.
+set timeout 45
+
 # Shut down a separate node.
 send "\\demo shutdown 3\r"
 eexpect "node 3 has been shutdown"
@@ -148,6 +168,26 @@ eexpect "5 | region=us-west1,az=b"
 eexpect "6 | region=ca-central,zone=a"
 eexpect "defaultdb>"
 
+# Wait for the liveness range to have the default 5 voters. If its replication
+# factor is too low, shutting down the node below can cause it to lose quorum
+# and stall queries to system tables (example: #147867).
+set timeout 2
+set stmt "select range_id, array_length(voting_replicas,1) from crdb_internal.ranges where range_id=2;\r"
+send $stmt
+expect {
+    "2 |            5" {
+        puts "\rliveness range has 5 voters"
+    }
+    timeout {
+        puts "\rliveness range does not yet have 5 voters"
+        sleep 2
+        send $stmt
+        exp_continue
+    }
+}
+# Reset timeout back to 45 to match common.tcl.
+set timeout 45
+
 # Shut down the newly created node.
 send "\\demo shutdown 6\r"
 set timeout 120