From 9f82ba98e51c7d4d610c432807ee8f2cb5af33e3 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 16 Jul 2026 12:09:13 -0400 Subject: [PATCH] fix(seidb-metrics): use shared fine-grained buckets for memiavl/flatkv/mvcc histograms The memiavl, flatkv, and pebble MVCC latency histograms used the OTel default bucket boundaries (5s, 10s, 25s, ...), which are useless for percentile queries over millisecond-scale per-block operations: every sample lands in the first bucket, so Grafana p95/p99 cannot resolve. Apply the shared smetrics.LatencyBuckets (10us-5min) to the latency histograms, and ByteSizeBuckets/CountBuckets to the mvcc batch-size and iterator histograms, matching the pattern already used by pebbledb/pebble_metrics.go, storev2 rootmulti, and app metrics. Co-authored-by: Cursor --- sei-db/db_engine/pebbledb/mvcc/metrics.go | 12 ++++++++++++ sei-db/state_db/sc/flatkv/metrics.go | 14 ++++++++++++++ sei-db/state_db/sc/memiavl/metrics.go | 9 +++++++++ 3 files changed, 35 insertions(+) diff --git a/sei-db/db_engine/pebbledb/mvcc/metrics.go b/sei-db/db_engine/pebbledb/mvcc/metrics.go index 6c1d6f2297..d417ce441a 100644 --- a/sei-db/db_engine/pebbledb/mvcc/metrics.go +++ b/sei-db/db_engine/pebbledb/mvcc/metrics.go @@ -3,6 +3,8 @@ package mvcc import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" + + smetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" ) var ( @@ -42,31 +44,37 @@ var ( "pebble_get_latency", metric.WithDescription("Time taken to get a key from PebbleDB"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), applyChangesetLatency: must(meter.Float64Histogram( "pebble_apply_changeset_latency", metric.WithDescription("Time taken to apply changeset to PebbleDB"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), applyChangesetAsyncLatency: must(meter.Float64Histogram( "pebble_apply_changeset_async_latency", metric.WithDescription("Time taken to queue changeset for async write"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), pruneLatency: must(meter.Float64Histogram( "pebble_prune_latency", metric.WithDescription("Time taken to prune old versions from PebbleDB"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), importLatency: must(meter.Float64Histogram( "pebble_import_latency", metric.WithDescription("Time taken to import snapshot data to PebbleDB"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), batchWriteLatency: must(meter.Float64Histogram( "pebble_batch_write_latency", metric.WithDescription("Time taken to write a batch to PebbleDB"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), compactionCount: must(meter.Int64Counter( @@ -78,6 +86,7 @@ var ( "pebble_compaction_duration", metric.WithDescription("Duration of compaction operations"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), compactionBytesRead: must(meter.Int64Counter( "pebble_compaction_bytes_read", @@ -99,6 +108,7 @@ var ( "pebble_flush_duration", metric.WithDescription("Duration of memtable flush operations"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), flushBytesWritten: must(meter.Int64Counter( "pebble_flush_bytes_written", @@ -152,6 +162,7 @@ var ( "pebble_batch_size", metric.WithDescription("Size of batches written to PebbleDB"), metric.WithUnit("By"), + metric.WithExplicitBucketBoundaries(smetrics.ByteSizeBuckets...), )), pendingChangesQueueDepth: must(meter.Int64Gauge( "pebble_pending_changes_queue_depth", @@ -162,6 +173,7 @@ var ( "pebble_iterator_iterations", metric.WithDescription("Number of iterations per iterator"), metric.WithUnit("{count}"), + metric.WithExplicitBucketBoundaries(smetrics.CountBuckets...), )), } ) diff --git a/sei-db/state_db/sc/flatkv/metrics.go b/sei-db/state_db/sc/flatkv/metrics.go index f1964763f3..fa7437338d 100644 --- a/sei-db/state_db/sc/flatkv/metrics.go +++ b/sei-db/state_db/sc/flatkv/metrics.go @@ -8,6 +8,8 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + smetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" ) var ( @@ -38,26 +40,31 @@ var ( "flatkv_open_latency", metric.WithDescription("Time taken to open the FlatKV store"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), ApplyChangesetsLatency: must(flatkvMeter.Float64Histogram( "flatkv_apply_changesets_latency", metric.WithDescription("Time taken to apply changesets to FlatKV"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), CommitLatency: must(flatkvMeter.Float64Histogram( "flatkv_commit_latency", metric.WithDescription("Time taken to commit FlatKV changes"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), CommitBatchLatency: must(flatkvMeter.Float64Histogram( "flatkv_commit_batch_latency", metric.WithDescription("Time taken to commit a FlatKV data DB batch"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), BatchReadOldValuesLatency: must(flatkvMeter.Float64Histogram( "flatkv_batch_read_old_values_latency", metric.WithDescription("Time taken to batch read old FlatKV values"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), NumKVPairs: must(flatkvMeter.Int64Counter( "flatkv_num_kv_pairs", @@ -78,6 +85,7 @@ var ( "flatkv_catchup_latency", metric.WithDescription("Time taken to replay FlatKV WAL entries"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), CatchupReplayNumBlocks: must(flatkvMeter.Int64Counter( "flatkv_catchup_replay_num_blocks", @@ -88,11 +96,13 @@ var ( "flatkv_snapshot_write_latency", metric.WithDescription("Time taken to write a FlatKV snapshot"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), SnapshotPruneLatency: must(flatkvMeter.Float64Histogram( "flatkv_snapshot_prune_latency", metric.WithDescription("Time taken to prune FlatKV snapshots"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), SnapshotPruneAttempts: must(flatkvMeter.Int64Counter( "flatkv_snapshot_prune_attempts", @@ -108,11 +118,13 @@ var ( "flatkv_rollback_latency", metric.WithDescription("Time taken to rollback FlatKV state"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), ImportLatency: must(flatkvMeter.Float64Histogram( "flatkv_import_latency", metric.WithDescription("Time taken to import FlatKV snapshot data"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), ImportKVPairs: must(flatkvMeter.Int64Counter( "flatkv_import_kv_pairs", @@ -123,11 +135,13 @@ var ( "flatkv_import_worker_flush_latency", metric.WithDescription("Time taken to flush a FlatKV import worker batch"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), FlushLatency: must(flatkvMeter.Float64Histogram( "flatkv_flush_latency", metric.WithDescription("Time taken to flush a FlatKV data DB"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), } ) diff --git a/sei-db/state_db/sc/memiavl/metrics.go b/sei-db/state_db/sc/memiavl/metrics.go index ee8037e718..fbef64fcbb 100644 --- a/sei-db/state_db/sc/memiavl/metrics.go +++ b/sei-db/state_db/sc/memiavl/metrics.go @@ -3,6 +3,8 @@ package memiavl import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" + + smetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" ) var ( @@ -28,11 +30,13 @@ var ( "memiavl_restart_latency", metric.WithDescription("Time taken to restart the memiavl database"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), SnapshotRewriteLatency: must(meter.Float64Histogram( "memiavl_snapshot_rewrite_latency", metric.WithDescription("Time taken to write to the new memiavl snapshot"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), NumSnapshotRewriteAttempts: must(meter.Int64Counter( "memiavl_num_snapshot_rewrite_attempts", @@ -42,6 +46,7 @@ var ( "memiavl_snapshot_prune_latency", metric.WithDescription("Time taken to prune memiavl snapshot"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), NumSnapshotPruneAttempts: must(meter.Int64Counter( "memiavl_num_snapshot_prune_attempts", @@ -52,11 +57,13 @@ var ( "memiavl_snapshot_catchup_after_rewrite_latency", metric.WithDescription("Time taken to catchup and replay after snapshot rewrite"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), CatchupBeforeReloadLatency: must(meter.Float64Histogram( "memiavl_snapshot_catchup_before_reload_latency", metric.WithDescription("Time taken to catchup and replay before switch to new snapshot"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), CatchupReplayNumBlocks: must(meter.Int64Counter( "memiavl_snapshot_catchup_replay_num_blocks", @@ -71,11 +78,13 @@ var ( "memiavl_commit_latency", metric.WithDescription("Time taken to commit"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), ApplyChangesetLatency: must(meter.Float64Histogram( "memiavl_apply_changeset_latency", metric.WithDescription("Time taken to apply changesets"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), )), NumOfKVPairs: must(meter.Int64Counter( "memiavl_num_of_kv_pairs",