Merge #158021 #158262

158021: sql: add `INSPECT` to DRT runs r=bghal a=bghal

This change adds a daily `INSPECT` operation to DRT. It is currently
limited to index consistency checks; more checks are planned.

Fixes: #148942
Epic: CRDB-55075

Release note: None


158262: kvserver: skip `TestAdminScatterWithDrainingNodes` test in DRPC mode r=cthumuluru-crdb a=cthumuluru-crdb

`TestAdminScatterWithDrainingNodes` is failing at random when DRPC is enabled. The issue is specific to DRPC and the failure is reproducible once every 25 runs. The issue is not reproducible with cluster setting `SET CLUSTER SETTING sql.stats.flush.enabled=false` set to false to turn off stats flush.

Re-enable the test once the underlying issue is resolved.

Fixes: #156372
Epic: None
Release note: None

Co-authored-by: Brendan Gerrity <brendan.gerrity@cockroachlabs.com>
Co-authored-by: Chandra Thumuluru <chandra.thumuluru@cockroachlabs.com>

Author: 3 people

pkg/cmd/roachtest/operations/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "disk_stall.go",
         "grant_revoke_all.go",
         "grant_role.go",
+        "inspect.go",
         "license_throttle.go",
         "manual_compaction.go",
         "network_partition.go",
@@ -41,6 +42,7 @@ go_library(
         "//pkg/cmd/roachtest/roachtestutil",
         "//pkg/cmd/roachtest/tests",
         "//pkg/sql/catalog/catpb",
+        "//pkg/sql/lexbase",
         "//pkg/sql/randgen",
         "//pkg/sql/sem/tree",
         "//pkg/sql/types",

pkg/cmd/roachtest/operations/inspect.go

@@ -0,0 +1,51 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package operations
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/operation"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/operations/helpers"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
+	"github.com/cockroachdb/cockroach/pkg/sql/lexbase"
+)
+
+func runInspect(
+	ctx context.Context, o operation.Operation, c cluster.Cluster,
+) registry.OperationCleanup {
+	conn := c.Conn(ctx, o.L(), 1)
+	defer conn.Close()
+
+	_, err := conn.ExecContext(ctx, "SET enable_inspect_command = true;")
+	if err != nil {
+		o.Fatal(err)
+	}
+
+	dbName := helpers.PickRandomDB(ctx, o, conn, []string{} /* excludeDBs */)
+
+	o.Status(fmt.Sprintf("inspecting database %s", dbName))
+	_, err = conn.ExecContext(ctx, fmt.Sprintf("INSPECT DATABASE %s", lexbase.EscapeSQLIdent(dbName)))
+	if err != nil {
+		o.Fatal(err)
+	}
+
+	return nil
+}
+
+func registerInspect(r registry.Registry) {
+	r.AddOperation(registry.OperationSpec{
+		Name:               "inspect/database",
+		Owner:              registry.OwnerSQLFoundations,
+		Timeout:            24 * time.Hour,
+		CompatibleClouds:   registry.AllClouds,
+		CanRunConcurrently: registry.OperationCanRunConcurrently,
+		Run:                runInspect,
+	})
+}

pkg/cmd/roachtest/operations/register.go

@@ -34,4 +34,5 @@ func RegisterOperations(r registry.Registry) {
 	registerSessionVariables(r)
 	registerDebugZip(r)
 	changefeeds.RegisterChangefeeds(r)
+	registerInspect(r)
 }

pkg/kv/kvserver/scatter_test.go

@@ -45,6 +45,11 @@ func TestAdminScatterWithDrainingNodes(t *testing.T) {
 	const drainingNodeID = drainingServerIdx + 1
 	tc := testcluster.StartTestCluster(
 		t, 2, base.TestClusterArgs{
+			ServerArgs: base.TestServerArgs{
+				// TODO(server): test is timing out when DRPC is enabled.
+				// Investigate and re-enable once the issue is resolved.
+				DefaultDRPCOption: base.TestDRPCDisabled,
+			},
 			ReplicationMode: base.ReplicationManual,
 		},
 	)