Merge #157940 #158056 #158080

157940: *: standardize pprof labeling code r=pav-kv,jasonlmfong a=dhartunian

This commit replaces usages of `pprof.Do` with our internal
implementation that wraps the `SetProfilerLabels` utility. This
enables us to have better standardization around profiler label
application and easier tracking of all usages.

A linter is added to enforce this usage.

Epic: CRDB-55080
Part of: CRDB-55923

Release note: None

158056: storage,kv: propagate iterator ReadCategory more broadly r=tbg a=jbowens

These commits expand the propagation of the ReadCategory enum across more of the iterator-constructing codepaths, hoping to fix gaps in categorization observed in #158017.

158080: ui: extend storage dashboard r=sumeerbhola a=jbowens

Add a new graph for the aggregate duration of compactions.

Add a new graph for the compaction scores of each level.

Epic: none
Release note: none

Co-authored-by: David Hartunian <davidh@cockroachlabs.com>
Co-authored-by: Jackson Owens <jackson@cockroachlabs.com>

Author: 3 people

docs/generated/metrics/metrics.yaml

@@ -16234,6 +16234,30 @@ layers:
       unit: NANOSECONDS
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
+    - name: storage.iterator.category-consistency-checker.block-load.bytes
+      exported_name: storage_iterator_category_consistency_checker_block_load_bytes
+      description: Bytes loaded by storage sstable iterators (possibly cached).
+      y_axis_label: Bytes
+      type: COUNTER
+      unit: BYTES
+      aggregation: AVG
+      derivative: NON_NEGATIVE_DERIVATIVE
+    - name: storage.iterator.category-consistency-checker.block-load.cached-bytes
+      exported_name: storage_iterator_category_consistency_checker_block_load_cached_bytes
+      description: Bytes loaded by storage sstable iterators from the block cache
+      y_axis_label: Bytes
+      type: COUNTER
+      unit: BYTES
+      aggregation: AVG
+      derivative: NON_NEGATIVE_DERIVATIVE
+    - name: storage.iterator.category-consistency-checker.block-load.latency-sum
+      exported_name: storage_iterator_category_consistency_checker_block_load_latency_sum
+      description: Cumulative latency for loading bytes not in the block cache, by storage sstable iterators
+      y_axis_label: Latency
+      type: COUNTER
+      unit: NANOSECONDS
+      aggregation: AVG
+      derivative: NON_NEGATIVE_DERIVATIVE
     - name: storage.iterator.category-crdb-unknown.block-load.bytes
       exported_name: storage_iterator_category_crdb_unknown_block_load_bytes
       description: Bytes loaded by storage sstable iterators (possibly cached).

pkg/cli/debug.go

@@ -497,7 +497,7 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error {
 	defer snapshot.Close()
 
 	var results int
-	return rditer.IterateReplicaKeySpans(cmd.Context(), &desc, snapshot, rditer.SelectOpts{
+	return rditer.IterateReplicaKeySpans(cmd.Context(), &desc, snapshot, fs.UnknownReadCategory, rditer.SelectOpts{
 		Ranged: rditer.SelectRangedOptions{
 			SystemKeys: true,
 			LockTable:  true,

pkg/cli/debug_check_store.go

@@ -125,7 +125,7 @@ func worker(ctx context.Context, in checkInput) checkResult {
 		res.err = err
 		return res
 	}
-	ms, err := rditer.ComputeStatsForRange(ctx, desc, eng, claimedMS.LastUpdateNanos)
+	ms, err := rditer.ComputeStatsForRange(ctx, desc, eng, fs.UnknownReadCategory, claimedMS.LastUpdateNanos)
 	if err != nil {
 		res.err = err
 		return res

pkg/crosscluster/logical/BUILD.bazel

@@ -101,6 +101,7 @@ go_library(
         "//pkg/util/log/logcrash",
         "//pkg/util/metamorphic",
         "//pkg/util/metric",
+        "//pkg/util/pprofutil",
         "//pkg/util/protoutil",
         "//pkg/util/randutil",
         "//pkg/util/retry",

pkg/crosscluster/logical/logical_replication_writer_processor.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"hash/fnv"
 	"regexp"
-	"runtime/pprof"
 	"slices"
 	"strings"
 	"time"
@@ -45,6 +44,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/log/logcrash"
+	"github.com/cockroachdb/cockroach/pkg/util/pprofutil"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
 	"github.com/cockroachdb/cockroach/pkg/util/span"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -349,12 +349,12 @@ func (lrw *logicalReplicationWriterProcessor) Start(ctx context.Context) {
 	})
 	lrw.workerGroup.GoCtx(func(ctx context.Context) error {
 		defer close(lrw.checkpointCh)
-		pprof.Do(ctx, pprof.Labels("proc", fmt.Sprintf("%d", lrw.ProcessorID)), func(ctx context.Context) {
+		pprofutil.Do(ctx, func(ctx context.Context) {
 			if err := lrw.consumeEvents(ctx); err != nil {
 				log.Dev.Infof(lrw.Ctx(), "consumer completed. Error: %s", err)
 				lrw.sendError(errors.Wrap(err, "consume events"))
 			}
-		})
+		}, "proc", fmt.Sprintf("%d", lrw.ProcessorID))
 		return nil
 	})
 }

pkg/crosscluster/logical/offline_initial_scan_processor.go

@@ -8,7 +8,6 @@ package logical
 import (
 	"context"
 	"fmt"
-	"runtime/pprof"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/backup"
@@ -31,6 +30,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/log/logcrash"
+	"github.com/cockroachdb/cockroach/pkg/util/pprofutil"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
 	"github.com/cockroachdb/cockroach/pkg/util/span"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -225,7 +225,7 @@ func (o *offlineInitialScanProcessor) Start(ctx context.Context) {
 	o.workerGroup.GoCtx(func(ctx context.Context) error {
 		defer close(o.checkpointCh)
 		defer close(o.rangeStatsCh)
-		pprof.Do(ctx, pprof.Labels("proc", fmt.Sprintf("%d", o.ProcessorID)), func(ctx context.Context) {
+		pprofutil.Do(ctx, func(ctx context.Context) {
 			for event := range o.subscription.Events() {
 				if err := o.handleEvent(ctx, event); err != nil {
 					log.Dev.Infof(o.Ctx(), "consumer completed. Error: %s", err)
@@ -235,7 +235,7 @@ func (o *offlineInitialScanProcessor) Start(ctx context.Context) {
 			if err := o.subscription.Err(); err != nil {
 				o.sendError(errors.Wrap(err, "subscription"))
 			}
-		})
+		}, "proc", fmt.Sprintf("%d", o.ProcessorID))
 		return nil
 	})
 }

pkg/kv/kvclient/rangefeed/BUILD.bazel

@@ -26,6 +26,7 @@ go_library(
         "//pkg/util/limit",
         "//pkg/util/log",
         "//pkg/util/mon",
+        "//pkg/util/pprofutil",
         "//pkg/util/retry",
         "//pkg/util/span",
         "//pkg/util/stop",

pkg/kv/kvclient/rangefeed/rangefeed.go

@@ -8,7 +8,6 @@ package rangefeed
 import (
 	"context"
 	"fmt"
-	"runtime/pprof"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -23,6 +22,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/pprofutil"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/span"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
@@ -241,9 +241,9 @@ func (f *RangeFeed) start(
 		// pprof.Do function does exactly what we do here, but it also results in
 		// pprof.Do function showing up in the stack traces -- so, just set and reset
 		// labels manually.
-		defer pprof.SetGoroutineLabels(ctx)
-		ctx = pprof.WithLabels(ctx, pprof.Labels(append(f.extraPProfLabels, "rangefeed", f.name)...))
-		pprof.SetGoroutineLabels(ctx)
+		ctx, reset := pprofutil.SetProfilerLabels(ctx, append(f.extraPProfLabels, "rangefeed", f.name)...)
+		defer reset()
+
 		if f.invoker != nil {
 			_ = f.invoker(func() error {
 				f.run(ctx, frontier, resumeFromFrontier)

pkg/kv/kvserver/batcheval/cmd_clear_range.go

@@ -189,7 +189,7 @@ func computeStatsDelta(
 	// If we can't use the fast stats path, or race test is enabled, compute stats
 	// across the key span to be cleared.
 	if !entireRange || util.RaceEnabled {
-		computed, err := storage.ComputeStats(ctx, reader, from, to, delta.LastUpdateNanos)
+		computed, err := storage.ComputeStats(ctx, reader, fs.BatchEvalReadCategory, from, to, delta.LastUpdateNanos)
 		if err != nil {
 			return enginepb.MVCCStats{}, err
 		}

pkg/kv/kvserver/batcheval/cmd_delete_range_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
+	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/storage/mvccencoding"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -411,7 +412,7 @@ func computeStats(
 	if len(to) == 0 {
 		to = keys.MaxKey
 	}
-	ms, err := storage.ComputeStats(context.Background(), reader, from, to, nowNanos)
+	ms, err := storage.ComputeStats(t.Context(), reader, fs.UnknownReadCategory, from, to, nowNanos)
 	require.NoError(t, err)
 	return ms
 }