diff --git a/.gitignore b/.gitignore index 30c6efe10c..b8223e9da9 100644 --- a/.gitignore +++ b/.gitignore @@ -67,4 +67,5 @@ sei-db/state_db/bench/cryptosim/data/** sei-db/state_db/bench/cryptosim/bin/ sei-db/state_db/bench/cryptosim/logs/ sei-db/ledger_db/block/blocksim/bin/ +sei-db/seiwal/walsim/bin/ sei-db/db_engine/litt/bin/ diff --git a/sei-db/seiwal/walsim/Makefile b/sei-db/seiwal/walsim/Makefile new file mode 100644 index 0000000000..362f0f78b3 --- /dev/null +++ b/sei-db/seiwal/walsim/Makefile @@ -0,0 +1,14 @@ +BINARY := bin/walsim +LOGGER_BINARY := bin/configure-logger +MODULE_ROOT ?= $(shell git -C "$(CURDIR)" rev-parse --show-toplevel) +CMD_DIR := ./sei-db/seiwal/walsim/cmd + +.PHONY: build +build: + @mkdir -p "$(CURDIR)/bin" + cd "$(MODULE_ROOT)" && go build -o "$(CURDIR)/$(BINARY)" "$(CMD_DIR)/walsim" + cd "$(MODULE_ROOT)" && go build -o "$(CURDIR)/$(LOGGER_BINARY)" "$(CMD_DIR)/configure-logger" + +.PHONY: clean +clean: + rm -f "$(CURDIR)/$(BINARY)" "$(CURDIR)/$(LOGGER_BINARY)" diff --git a/sei-db/seiwal/walsim/cmd/configure-logger/main.go b/sei-db/seiwal/walsim/cmd/configure-logger/main.go new file mode 100644 index 0000000000..ef30685b84 --- /dev/null +++ b/sei-db/seiwal/walsim/cmd/configure-logger/main.go @@ -0,0 +1,52 @@ +// configure-logger reads a walsim config file and prints shell export +// statements that configure seilog's environment variables. Intended to be +// called via eval in a shell script before launching the benchmark binary. +// +// Usage: +// +// eval "$(configure-logger config.json)" +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/seiwal/walsim" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "configure-logger: %v\n", err) + os.Exit(1) + } +} + +func run() error { + if len(os.Args) != 2 { + return fmt.Errorf("usage: configure-logger ") + } + + cfg := walsim.DefaultWalsimConfig() + if err := utils.LoadConfigFromFile(os.Args[1], cfg); err != nil { + return fmt.Errorf("load config: %w", err) + } + + logDir, err := utils.ResolveAndCreateDir(cfg.LogDir) + if err != nil { + return fmt.Errorf("resolve log dir: %w", err) + } + + logFile := filepath.Join(logDir, "walsim.log") + + fmt.Printf("export SEI_LOG_OUTPUT=%s\n", shellQuote(logFile)) + fmt.Printf("export SEI_LOG_LEVEL=%s\n", shellQuote(strings.ToLower(cfg.LogLevel))) + + return nil +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} diff --git a/sei-db/seiwal/walsim/cmd/walsim/main.go b/sei-db/seiwal/walsim/cmd/walsim/main.go new file mode 100644 index 0000000000..ecb644b52d --- /dev/null +++ b/sei-db/seiwal/walsim/cmd/walsim/main.go @@ -0,0 +1,88 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "os" + "os/signal" + + "github.com/sei-protocol/sei-chain/sei-db/common/metrics" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/seiwal/walsim" +) + +func main() { + err := run() + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func run() error { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0]) + os.Exit(1) + } + config := walsim.DefaultWalsimConfig() + if err := utils.LoadConfigFromFile(os.Args[1], config); err != nil { + return err + } + + configString, err := utils.StringifyConfig(config) + if err != nil { + return fmt.Errorf("failed to stringify config: %w", err) + } + fmt.Printf("%s\n", configString) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + reg, shutdown, err := metrics.SetupOtelPrometheus() + if err != nil { + return fmt.Errorf("setup metrics: %w", err) + } + defer func() { + _ = shutdown(context.Background()) + }() + + wsMetrics := walsim.NewWalsimMetrics(ctx, config) + + ws, err := walsim.NewWalSim(ctx, config, wsMetrics) + if err != nil { + return fmt.Errorf("failed to create walsim: %w", err) + } + defer func() { + if err := ws.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Error closing walsim: %v\n", err) + } + }() + + metrics.StartMetricsServer(ctx, reg, config.MetricsAddr) + metrics.StartSystemMetrics(ctx, "walsim", config.BackgroundMetricsScrapeInterval, + []metrics.MonitoredDir{ + {Name: "data_dir", Path: config.DataDir, TrackAvailableSpace: true}, + {Name: "log_dir", Path: config.LogDir}, + }) + + if config.EnableSuspension { + go func() { + scanner := bufio.NewScanner(os.Stdin) + suspended := false + for scanner.Scan() { + if suspended { + ws.Resume() + suspended = false + } else { + ws.Suspend() + suspended = true + } + } + }() + } + + ws.BlockUntilHalted() + + return nil +} diff --git a/sei-db/seiwal/walsim/config/debug.json b/sei-db/seiwal/walsim/config/debug.json new file mode 100644 index 0000000000..fb169c31ac --- /dev/null +++ b/sei-db/seiwal/walsim/config/debug.json @@ -0,0 +1,9 @@ +{ + "Comment": "Configuration for local smoke testing and debugging.", + "DataDir": "~/walsim/data", + "LogDir": "~/walsim/logs", + "CleanDataOnStart": true, + "CleanLogsOnStart": true, + "CleanDataOnExit": true, + "CleanLogsOnExit": true +} diff --git a/sei-db/seiwal/walsim/config/standard.json b/sei-db/seiwal/walsim/config/standard.json new file mode 100644 index 0000000000..24dbd4ea98 --- /dev/null +++ b/sei-db/seiwal/walsim/config/standard.json @@ -0,0 +1,5 @@ +{ + "Comment": "Standard walsim configuration.", + "DataDir": "~/walsim/data", + "LogDir": "~/walsim/logs" +} diff --git a/sei-db/seiwal/walsim/record_generator.go b/sei-db/seiwal/walsim/record_generator.go new file mode 100644 index 0000000000..ff9eda5d5c --- /dev/null +++ b/sei-db/seiwal/walsim/record_generator.go @@ -0,0 +1,50 @@ +package walsim + +import ( + "context" + + crand "github.com/sei-protocol/sei-chain/sei-db/common/rand" +) + +// RecordGenerator asynchronously produces fixed-size opaque records and feeds them into a channel. +// Each record is a zero-copy sub-slice of a pre-generated, immutable CannedRandom buffer, so the +// generator never runs math/rand or allocates payload bytes on the hot path. The generator stops +// when the context is cancelled. +// +// The CannedRandom buffer is never mutated, so the sub-slices remain valid indefinitely; this makes +// it safe for the WAL to retain a record slice and serialize it asynchronously. +type RecordGenerator struct { + ctx context.Context + rand *crand.CannedRandom + recordSize int + recordChan chan []byte +} + +// NewRecordGenerator creates a RecordGenerator and immediately starts its background goroutine. rand +// must not be shared with any other goroutine (this generator owns it). +func NewRecordGenerator( + ctx context.Context, + rng *crand.CannedRandom, + recordSize int, + queueSize int, +) *RecordGenerator { + g := &RecordGenerator{ + ctx: ctx, + rand: rng, + recordSize: recordSize, + recordChan: make(chan []byte, queueSize), + } + go g.mainLoop() + return g +} + +func (g *RecordGenerator) mainLoop() { + for { + record := g.rand.Bytes(g.recordSize) + select { + case <-g.ctx.Done(): + return + case g.recordChan <- record: + } + } +} diff --git a/sei-db/seiwal/walsim/wal_store.go b/sei-db/seiwal/walsim/wal_store.go new file mode 100644 index 0000000000..d79dbfaccb --- /dev/null +++ b/sei-db/seiwal/walsim/wal_store.go @@ -0,0 +1,147 @@ +package walsim + +import ( + "context" + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/seiwal" + "github.com/sei-protocol/sei-chain/sei-db/wal" +) + +// walStore is the minimal WAL surface the benchmark drives. The new seiwal WAL (seiwal.WAL[[]byte]) +// satisfies it directly with no wrapper; the legacy sei-db/wal WAL is adapted by legacyWALShim. +type walStore interface { + // Append schedules a record with the given index and payload. + Append(index uint64, data []byte) error + + // Flush blocks until previously scheduled appends are durable. + Flush() error + + // Bounds reports whether any record is stored and, if so, the lowest and highest stored indices. + Bounds() (ok bool, first uint64, last uint64, err error) + + // PruneBefore requests removal of all records with an index below lowestIndexToKeep. + PruneBefore(lowestIndexToKeep uint64) error + + // Close flushes pending appends and releases resources. + Close() error +} + +// The seiwal WAL implements walStore directly. +var _ walStore = (seiwal.WAL[[]byte])(nil) + +// The legacy shim implements walStore. +var _ walStore = (*legacyWALShim)(nil) + +// openWALStore opens the WAL backend selected by the configuration. +func openWALStore(ctx context.Context, config *WalsimConfig) (walStore, error) { + switch config.Backend { + case "seiwal": + cfg := config.Seiwal + // walsim owns the storage path and metric name. + cfg.Path = config.DataDir + cfg.Name = "walsim" + w, err := seiwal.NewWAL(&cfg) + if err != nil { + return nil, fmt.Errorf("failed to open seiwal WAL: %w", err) + } + return w, nil + case "legacy": + return newLegacyWALShim(ctx, config) + default: + return nil, fmt.Errorf("unknown WAL backend: %q", config.Backend) + } +} + +// legacyWALShim adapts the legacy sei-db/wal WAL to the walStore interface so the benchmark can +// drive it exactly like the new seiwal WAL. It is throwaway code: the legacy WAL is scheduled for +// deletion, and this shim goes with it. +// +// The legacy WAL's TruncateBefore rewrites segment files and is O(segments), so forwarding it on +// every request (as the size-target prune loop wants) would be catastrophic. The shim therefore +// coalesces prune requests, forwarding only every pruneRelaxationFactor-th PruneBefore to the +// underlying TruncateBefore. +type legacyWALShim struct { + inner *wal.WAL[[]byte] + + // Forward a TruncateBefore only once per this many PruneBefore calls. + pruneRelaxationFactor uint64 + + // The number of PruneBefore calls received so far. + pruneRequestCount uint64 +} + +func newLegacyWALShim(ctx context.Context, config *WalsimConfig) (*legacyWALShim, error) { + identity := func(b []byte) ([]byte, error) { return b, nil } + + cfg := config.Legacy + // Force the legacy WAL's own background pruning off so pruning is driven solely through + // PruneBefore (and coalesced below). + cfg.KeepRecent = 0 + cfg.PruneInterval = 0 + + inner, err := wal.NewWAL[[]byte](ctx, identity, identity, config.DataDir, cfg) + if err != nil { + return nil, fmt.Errorf("failed to open legacy WAL: %w", err) + } + + return &legacyWALShim{ + inner: inner, + pruneRelaxationFactor: config.PruneRelaxationFactor, + }, nil +} + +// Append writes data to the legacy WAL. The legacy WAL assigns its own 1-based, contiguous index, +// so the caller-supplied index is ignored; walsim seeds its counter from Bounds, keeping the two +// aligned. +func (s *legacyWALShim) Append(_ uint64, data []byte) error { + if err := s.inner.Write(data); err != nil { + return fmt.Errorf("failed to write to legacy WAL: %w", err) + } + return nil +} + +// Flush is a no-op. The legacy WAL exposes no explicit sync, so durability is governed entirely by +// its FsyncEnabled and batching configuration. +func (s *legacyWALShim) Flush() error { + return nil +} + +// Bounds reports the stored index range. The legacy WAL reports 0/0 for an empty log; real indices +// are 1-based, so a last offset of 0 means empty. +func (s *legacyWALShim) Bounds() (bool, uint64, uint64, error) { + first, err := s.inner.FirstOffset() + if err != nil { + return false, 0, 0, fmt.Errorf("failed to read first offset: %w", err) + } + last, err := s.inner.LastOffset() + if err != nil { + return false, 0, 0, fmt.Errorf("failed to read last offset: %w", err) + } + if last == 0 { + return false, 0, 0, nil + } + return true, first, last, nil +} + +// PruneBefore coalesces prune requests: only every pruneRelaxationFactor-th call reaches the +// underlying TruncateBefore, which is expensive in the legacy WAL. The most recent +// lowestIndexToKeep is used when the forward fires. +func (s *legacyWALShim) PruneBefore(lowestIndexToKeep uint64) error { + s.pruneRequestCount++ + if s.pruneRequestCount%s.pruneRelaxationFactor != 0 { + return nil + } + if err := s.inner.TruncateBefore(lowestIndexToKeep); err != nil { + return fmt.Errorf("failed to truncate legacy WAL: %w", err) + } + return nil +} + +// Close shuts down the legacy WAL. +func (s *legacyWALShim) Close() error { + if err := s.inner.Close(); err != nil { + return fmt.Errorf("failed to close legacy WAL: %w", err) + } + return nil +} diff --git a/sei-db/seiwal/walsim/wal_store_test.go b/sei-db/seiwal/walsim/wal_store_test.go new file mode 100644 index 0000000000..af32de853f --- /dev/null +++ b/sei-db/seiwal/walsim/wal_store_test.go @@ -0,0 +1,83 @@ +package walsim + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +// newTestShim opens a legacy shim backed by a temp dir with synchronous writes (so appends and +// truncations are observable deterministically) and the given prune relaxation factor. +func newTestShim(t *testing.T, pruneRelaxationFactor uint64) *legacyWALShim { + t.Helper() + config := DefaultWalsimConfig() + config.Backend = "legacy" + config.DataDir = t.TempDir() + config.Legacy.WriteBufferSize = 0 // synchronous writes + config.Legacy.WriteBatchSize = 1 // no batching + config.PruneRelaxationFactor = pruneRelaxationFactor + + shim, err := newLegacyWALShim(context.Background(), config) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, shim.Close()) }) + return shim +} + +func appendN(t *testing.T, shim *legacyWALShim, n int) { + t.Helper() + for i := 0; i < n; i++ { + // The index argument is ignored by the legacy shim; the legacy WAL assigns its own. + require.NoError(t, shim.Append(uint64(i+1), []byte("payload"))) + } +} + +func firstIndex(t *testing.T, shim *legacyWALShim) uint64 { + t.Helper() + ok, first, _, err := shim.Bounds() + require.NoError(t, err) + require.True(t, ok) + return first +} + +func TestLegacyShimBoundsEmptyThenPopulated(t *testing.T) { + shim := newTestShim(t, 1) + + ok, _, _, err := shim.Bounds() + require.NoError(t, err) + require.False(t, ok, "a fresh legacy WAL must report no bounds") + + appendN(t, shim, 5) + + ok, first, last, err := shim.Bounds() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first, "legacy WAL indices are 1-based") + require.Equal(t, uint64(5), last) +} + +func TestLegacyShimCoalescesPruneRequests(t *testing.T) { + const relaxation = 3 + shim := newTestShim(t, relaxation) + appendN(t, shim, 10) + + require.Equal(t, uint64(1), firstIndex(t, shim)) + + // The first two prune requests are swallowed; the underlying WAL is untouched. + require.NoError(t, shim.PruneBefore(2)) + require.Equal(t, uint64(1), firstIndex(t, shim)) + require.NoError(t, shim.PruneBefore(3)) + require.Equal(t, uint64(1), firstIndex(t, shim)) + + // The third request (a multiple of the relaxation factor) is forwarded, using its own + // lowestIndexToKeep. + require.NoError(t, shim.PruneBefore(4)) + require.Equal(t, uint64(4), firstIndex(t, shim)) + + // The cycle repeats: two more swallowed, then one forwarded. + require.NoError(t, shim.PruneBefore(5)) + require.NoError(t, shim.PruneBefore(6)) + require.Equal(t, uint64(4), firstIndex(t, shim)) + require.NoError(t, shim.PruneBefore(7)) + require.Equal(t, uint64(7), firstIndex(t, shim)) +} diff --git a/sei-db/seiwal/walsim/walsim.go b/sei-db/seiwal/walsim/walsim.go new file mode 100644 index 0000000000..be01fa4afe --- /dev/null +++ b/sei-db/seiwal/walsim/walsim.go @@ -0,0 +1,365 @@ +package walsim + +import ( + "context" + "fmt" + "os" + "time" + + crand "github.com/sei-protocol/sei-chain/sei-db/common/rand" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "golang.org/x/time/rate" +) + +// WalSim is the benchmark runner for the walsim benchmark. +type WalSim struct { + ctx context.Context + cancel context.CancelFunc + + config *WalsimConfig + + store walStore + generator *RecordGenerator + metrics *WalsimMetrics + + // The index to assign to the next appended record. + nextIndex uint64 + + // The number of records to retain on disk, derived from TargetDiskSizeBytes. 0 disables pruning. + retainedRecords uint64 + + // Console reporting state. + consoleUpdatePeriod time.Duration + lastConsoleUpdateTime time.Time + lastConsoleUpdateRecordCount int64 + startTimestamp time.Time + totalRecordsWritten int64 + totalBytesWritten int64 + highestIndex uint64 + + // A message is sent on this channel when the benchmark is fully stopped. + closeChan chan struct{} + + // Suspend/resume toggle channel. + suspendChan chan bool + + // Enforces a maximum record write rate (if enabled). + rateLimiter *rate.Limiter +} + +// NewWalSim creates a new walsim benchmark runner and starts it. +func NewWalSim( + ctx context.Context, + config *WalsimConfig, + metrics *WalsimMetrics, +) (*WalSim, error) { + + var err error + config.DataDir, err = utils.ResolveAndCreateDir(config.DataDir) + if err != nil { + return nil, fmt.Errorf("failed to resolve data directory: %w", err) + } + config.LogDir, err = utils.ResolveAndCreateDir(config.LogDir) + if err != nil { + return nil, fmt.Errorf("failed to resolve log directory: %w", err) + } + + if config.CleanDataOnStart { + fmt.Printf("CleanDataOnStart is enabled, removing contents of: %s\n", config.DataDir) + if err := removeContents(config.DataDir); err != nil { + return nil, fmt.Errorf("failed to clean data directory: %w", err) + } + } + if config.CleanLogsOnStart { + fmt.Printf("CleanLogsOnStart is enabled, removing contents of: %s\n", config.LogDir) + if err := removeContents(config.LogDir); err != nil { + return nil, fmt.Errorf("failed to clean log directory: %w", err) + } + } + + fmt.Printf("Running walsim benchmark (%s backend) from data directory: %s\n", config.Backend, config.DataDir) + fmt.Printf("Logs are being routed to: %s\n", config.LogDir) + + fmt.Printf("Initializing random number generator.\n") + // Pre-generate a random buffer once; all record data slices into it (zero-copy) so the generator + // never runs math/rand on the hot path. + cannedRand := crand.NewCannedRandom(int(config.RandomDataBufferSizeBytes), config.Seed) //nolint:gosec // buffer size is bounded by config + + ctx, cancel := context.WithCancel(ctx) + + store, err := openWALStore(ctx, config) + if err != nil { + cancel() + return nil, fmt.Errorf("failed to open WAL: %w", err) + } + + // Resume after any existing history instead of colliding with on-disk records. + ok, _, last, err := store.Bounds() + if err != nil { + cancel() + _ = store.Close() + return nil, fmt.Errorf("failed to read WAL bounds: %w", err) + } + var highest uint64 + var nextIndex uint64 = 1 + if ok { + highest = last + nextIndex = last + 1 + fmt.Printf("Resuming from index %d.\n", highest) + } + + var retainedRecords uint64 + if config.TargetDiskSizeBytes > 0 { + retainedRecords = config.TargetDiskSizeBytes / config.RecordSizeBytes + } + + generator := NewRecordGenerator( + ctx, + cannedRand, + int(config.RecordSizeBytes), //nolint:gosec // record size is bounded by config + int(config.StagedRecordQueueSize), //nolint:gosec // queue size is bounded by config + ) + + consoleUpdatePeriod := time.Duration(config.ConsoleUpdateIntervalSeconds * float64(time.Second)) + + var rateLimiter *rate.Limiter + if config.MaxRecordsPerSecond > 0 { + rateLimiter = rate.NewLimiter(rate.Limit(config.MaxRecordsPerSecond), 1) + } + + start := time.Now() + + b := &WalSim{ + ctx: ctx, + cancel: cancel, + config: config, + store: store, + generator: generator, + metrics: metrics, + nextIndex: nextIndex, + retainedRecords: retainedRecords, + consoleUpdatePeriod: consoleUpdatePeriod, + lastConsoleUpdateTime: start, + startTimestamp: start, + highestIndex: highest, + closeChan: make(chan struct{}, 1), + suspendChan: make(chan bool, 1), + rateLimiter: rateLimiter, + } + + go b.run() + return b, nil +} + +// The main loop of the benchmark. +func (b *WalSim) run() { + defer b.teardown() + + var timeoutChan <-chan time.Time + if b.config.MaxRuntimeSeconds > 0 { + timeoutChan = time.After(time.Duration(b.config.MaxRuntimeSeconds) * time.Second) + } + + for { + b.metrics.SetMainThreadPhase("get_record") + + select { + case <-b.ctx.Done(): + b.generateConsoleReport(true) + fmt.Printf("\nBenchmark halted.\n") + return + case isSuspended := <-b.suspendChan: + if isSuspended { + b.suspend() + } + case <-timeoutChan: + fmt.Printf("\nBenchmark timed out after %s.\n", + utils.FormatDuration(time.Since(b.startTimestamp), 1)) + b.cancel() + return + case record := <-b.generator.recordChan: + b.maybeThrottle() + b.handleNextRecord(record) + } + + b.generateConsoleReport(false) + } +} + +func (b *WalSim) maybeThrottle() { + if b.rateLimiter == nil { + return + } + b.metrics.SetMainThreadPhase("throttling") + if err := b.rateLimiter.Wait(b.ctx); err != nil { + return + } +} + +// handleNextRecord appends one record, then applies the periodic flush and size-targeted prune. +func (b *WalSim) handleNextRecord(record []byte) { + b.metrics.SetMainThreadPhase("append") + index := b.nextIndex + if err := b.store.Append(index, record); err != nil { + fmt.Printf("failed to append record %d: %v\n", index, err) + b.cancel() + return + } + b.nextIndex++ + b.totalRecordsWritten++ + b.totalBytesWritten += int64(len(record)) + b.highestIndex = index + b.metrics.ReportRecordWritten(int64(len(record))) + b.metrics.RecordHighestIndex(b.highestIndex) + + // Periodic flush. + if b.config.FlushIntervalRecords > 0 && b.totalRecordsWritten%int64(b.config.FlushIntervalRecords) == 0 { //nolint:gosec + b.metrics.SetMainThreadPhase("flush") + if err := b.store.Flush(); err != nil { + fmt.Printf("failed to flush: %v\n", err) + b.cancel() + return + } + b.metrics.ReportFlush() + } + + // Size-targeted prune: hold the on-disk data as close to TargetDiskSizeBytes as possible by + // keeping the most recent retainedRecords records. + if b.retainedRecords > 0 && b.highestIndex >= b.retainedRecords { + b.metrics.SetMainThreadPhase("prune") + lowestToKeep := b.highestIndex - b.retainedRecords + 1 + if err := b.store.PruneBefore(lowestToKeep); err != nil { + fmt.Printf("failed to prune: %v\n", err) + b.cancel() + return + } + b.metrics.ReportPruneRequest() + b.metrics.RecordLowestIndex(lowestToKeep) + } +} + +func (b *WalSim) suspend() { + // Flush before suspending so state is durable. + if err := b.store.Flush(); err != nil { + fmt.Printf("failed to flush on suspend: %v\n", err) + } + + fmt.Printf("Benchmark suspended.\n") + b.metrics.SetMainThreadPhase("suspended") + + for { + select { + case <-b.ctx.Done(): + return + case suspended := <-b.suspendChan: + if suspended { + break + } + // Reset console metrics on resume. + b.totalRecordsWritten = 0 + b.totalBytesWritten = 0 + b.startTimestamp = time.Now() + fmt.Printf("Benchmark resumed.\n") + return + } + } +} + +func (b *WalSim) teardown() { + fmt.Printf("Flushing and closing WAL.\n") + if err := b.store.Flush(); err != nil { + fmt.Printf("failed to flush during teardown: %v\n", err) + } + if err := b.store.Close(); err != nil { + fmt.Printf("failed to close WAL: %v\n", err) + } + + if b.config.CleanDataOnExit { + fmt.Printf("CleanDataOnExit is enabled, removing contents of: %s\n", b.config.DataDir) + if err := removeContents(b.config.DataDir); err != nil { + fmt.Printf("failed to clean data directory on exit: %v\n", err) + } + } + if b.config.CleanLogsOnExit { + fmt.Printf("CleanLogsOnExit is enabled, removing contents of: %s\n", b.config.LogDir) + if err := removeContents(b.config.LogDir); err != nil { + fmt.Printf("failed to clean log directory on exit: %v\n", err) + } + } + + b.closeChan <- struct{}{} +} + +func (b *WalSim) generateConsoleReport(force bool) { + now := time.Now() + timeSinceLastUpdate := now.Sub(b.lastConsoleUpdateTime) + recordsSinceLastUpdate := b.totalRecordsWritten - b.lastConsoleUpdateRecordCount + + if !force && + timeSinceLastUpdate < b.consoleUpdatePeriod && + recordsSinceLastUpdate < int64(b.config.ConsoleUpdateIntervalRecords) { //nolint:gosec + return + } + + b.lastConsoleUpdateTime = now + b.lastConsoleUpdateRecordCount = b.totalRecordsWritten + + elapsed := now.Sub(b.startTimestamp) + bytesPerSecond := float64(b.totalBytesWritten) / elapsed.Seconds() + recordsPerSecond := float64(b.totalRecordsWritten) / elapsed.Seconds() + + fmt.Printf("%s records in %s | %s written | %s/sec | %s rec/sec \r", + utils.Int64Commas(b.totalRecordsWritten), + utils.FormatDuration(elapsed, 1), + utils.FormatBytes(b.totalBytesWritten), + utils.FormatBytes(int64(bytesPerSecond)), + utils.Int64Commas(int64(recordsPerSecond))) +} + +// BlockUntilHalted blocks until the benchmark has halted. +func (b *WalSim) BlockUntilHalted() { + <-b.closeChan + b.closeChan <- struct{}{} +} + +// Close shuts down the benchmark and releases resources. +func (b *WalSim) Close() error { + b.cancel() + <-b.closeChan + b.closeChan <- struct{}{} + fmt.Printf("Benchmark terminated successfully.\n") + return nil +} + +// Suspend the benchmark. Call Resume() to continue. +func (b *WalSim) Suspend() { + select { + case <-b.ctx.Done(): + case b.suspendChan <- true: + } +} + +// Resume the benchmark after a Suspend(). +func (b *WalSim) Resume() { + select { + case <-b.ctx.Done(): + case b.suspendChan <- false: + } +} + +// removeContents deletes all entries inside dir without removing dir itself. +func removeContents(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, entry := range entries { + if err := os.RemoveAll(fmt.Sprintf("%s/%s", dir, entry.Name())); err != nil { + return err + } + } + return nil +} diff --git a/sei-db/seiwal/walsim/walsim.sh b/sei-db/seiwal/walsim/walsim.sh new file mode 100755 index 0000000000..b3d0b3a25a --- /dev/null +++ b/sei-db/seiwal/walsim/walsim.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Resolve script directory (handles symlinks and relative paths). +SCRIPT_SOURCE="${BASH_SOURCE[0]}" +[[ "$SCRIPT_SOURCE" != /* ]] && SCRIPT_SOURCE="$(pwd)/${SCRIPT_SOURCE#./}" +while [[ -L "$SCRIPT_SOURCE" ]]; do + SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)" + SCRIPT_SOURCE="$(readlink "$SCRIPT_SOURCE")" + [[ "$SCRIPT_SOURCE" != /* ]] && SCRIPT_SOURCE="${SCRIPT_DIR}/${SCRIPT_SOURCE}" +done +SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)" +BINARY="${SCRIPT_DIR}/bin/walsim" + +# Build binaries (no-op if already up to date; Go's build cache handles staleness). +make -C "$SCRIPT_DIR" build + +# Configure seilog env vars from the config file. The config file is the sole +# source of truth -- any pre-existing SEI_LOG_* env vars are overwritten. +if [[ $# -ge 1 && -f "$1" ]]; then + LOGGER_OUTPUT=$("${SCRIPT_DIR}/bin/configure-logger" "$1") || { + echo "configure-logger failed" >&2 + exit 1 + } + eval "$LOGGER_OUTPUT" +fi + +# Run the benchmark. +exec "$BINARY" "$@" diff --git a/sei-db/seiwal/walsim/walsim_config.go b/sei-db/seiwal/walsim/walsim_config.go new file mode 100644 index 0000000000..1a86331a9d --- /dev/null +++ b/sei-db/seiwal/walsim/walsim_config.go @@ -0,0 +1,198 @@ +package walsim + +import ( + "fmt" + "strings" + + "github.com/sei-protocol/sei-chain/sei-db/common/unit" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/seiwal" + "github.com/sei-protocol/sei-chain/sei-db/wal" +) + +var _ utils.Config = (*WalsimConfig)(nil) + +// WalsimConfig is the configuration for the walsim benchmark. +type WalsimConfig struct { + + // The WAL implementation to benchmark. One of "seiwal" (the new WAL) or "legacy" (the old + // sei-db/wal WAL, driven through a throwaway adapter). + Backend string + + // The size in bytes of each record appended to the WAL. Every append writes exactly this many + // bytes of opaque, canned-random data. + RecordSizeBytes uint64 + + // Size in bytes of the pre-generated random buffer used to synthesize record payloads. The buffer + // is filled once at startup and sliced (zero-copy) thereafter, so the generator never runs + // math/rand or allocates payload bytes on the hot path. Must be at least RecordSizeBytes. + RandomDataBufferSizeBytes uint64 + + // The capacity of the queue that holds generated records before they are consumed by the + // benchmark. A larger queue lets the generator run further ahead of the consumer. + StagedRecordQueueSize uint64 + + // How often (in records) to call Flush() on the WAL. 0 means never explicitly flush. + FlushIntervalRecords uint64 + + // The target size in bytes of the on-disk data. Once the retained data would exceed this, the + // benchmark issues prune requests to hold the on-disk size as close to this target as possible. + // 0 disables pruning entirely. + TargetDiskSizeBytes uint64 + + // For the "legacy" backend only: coalesce prune requests, forwarding only every Nth PruneBefore + // to the underlying (expensive) TruncateBefore. Ignored by the "seiwal" backend. Must be at + // least 1. + PruneRelaxationFactor uint64 + + // The configuration for the "seiwal" backend, used only when Backend is "seiwal". Path and Name + // are owned by walsim (Path is taken from DataDir); any values supplied for them here are + // overwritten. + Seiwal seiwal.Config + + // The configuration for the "legacy" backend, used only when Backend is "legacy". The storage + // directory is taken from DataDir, and KeepRecent/PruneInterval are forced off so that walsim + // drives pruning through PruneBefore; any values supplied for them here are overwritten. + Legacy wal.Config + + // The seed to use for the random number generator. Altering this seed for a pre-existing data + // directory is harmless (records are opaque), but keeping it stable makes runs reproducible. + Seed int64 + + // The directory to store the benchmark data. + DataDir string + + // If this many seconds go by without a console update, the benchmark will print a report. + ConsoleUpdateIntervalSeconds float64 + + // If this many records are written without a console update, the benchmark will print a report. + // Prevents console spam when throughput is very high. + ConsoleUpdateIntervalRecords uint64 + + // The amount of time to run the benchmark for, in seconds. If 0, runs until interrupted. + MaxRuntimeSeconds int + + // Address for the Prometheus metrics HTTP server (e.g. ":9090"). If empty, metrics are disabled. + MetricsAddr string + + // If true, pressing Enter in the terminal will toggle suspend/resume of the benchmark. + EnableSuspension bool + + // How often (in seconds) to scrape background metrics (data dir size, uptime). If 0, background + // metrics are disabled. + BackgroundMetricsScrapeInterval int + + // Directory for seilog output files. Supports ~ expansion and relative paths. + LogDir string + + // Log level for seilog output. Valid values: debug, info, warn, error. + LogLevel string + + // If true, delete the contents of DataDir before opening the WAL. + CleanDataOnStart bool + + // If true, delete the contents of LogDir before starting. + CleanLogsOnStart bool + + // If true, delete the contents of DataDir after the benchmark finishes. + CleanDataOnExit bool + + // If true, delete the contents of LogDir after the benchmark finishes. + CleanLogsOnExit bool + + // Throttle record write rate to this many records per second. 0 = disabled (unlimited + // throughput). Useful for testing steady-state performance rather than max-throughput thrashing. + MaxRecordsPerSecond float64 + + // This field is ignored, but allows for a comment to be added to the config file. + Comment string +} + +// DefaultWalsimConfig returns the default configuration for the walsim benchmark. +func DefaultWalsimConfig() *WalsimConfig { + return &WalsimConfig{ + Backend: "seiwal", + RecordSizeBytes: 1 * unit.MB, + RandomDataBufferSizeBytes: unit.GB, + StagedRecordQueueSize: 16, + FlushIntervalRecords: 1, + TargetDiskSizeBytes: 1 * unit.GB, + PruneRelaxationFactor: 100, + // Path is set by walsim from DataDir at open time. + Seiwal: *seiwal.DefaultConfig("", "walsim"), + Legacy: wal.Config{ + WriteBufferSize: 0, + WriteBatchSize: 64, + FsyncEnabled: false, + }, + Seed: 1337, + DataDir: "data", + ConsoleUpdateIntervalSeconds: 1, + ConsoleUpdateIntervalRecords: 10_000, + MaxRuntimeSeconds: 0, + MetricsAddr: ":9090", + EnableSuspension: true, + BackgroundMetricsScrapeInterval: 60, + LogDir: "logs", + LogLevel: "info", + CleanDataOnStart: false, + CleanLogsOnStart: false, + CleanDataOnExit: false, + CleanLogsOnExit: false, + MaxRecordsPerSecond: 0, + } +} + +// Validate checks that the configuration is sane and returns an error if not. +func (c *WalsimConfig) Validate() error { + switch c.Backend { + case "seiwal", "legacy": + default: + return fmt.Errorf("invalid Backend %q, must be one of seiwal, legacy", c.Backend) + } + if c.RecordSizeBytes < 1 { + return fmt.Errorf("RecordSizeBytes must be at least 1 (got %d)", c.RecordSizeBytes) + } + if c.RandomDataBufferSizeBytes < c.RecordSizeBytes { + return fmt.Errorf("RandomDataBufferSizeBytes must be at least RecordSizeBytes (%d) (got %d)", + c.RecordSizeBytes, c.RandomDataBufferSizeBytes) + } + // The random buffer is fed to crand.NewCannedRandom, which requires at least 8 bytes. + if c.RandomDataBufferSizeBytes < 8 { + return fmt.Errorf("RandomDataBufferSizeBytes must be at least 8 (got %d)", c.RandomDataBufferSizeBytes) + } + if c.StagedRecordQueueSize < 1 { + return fmt.Errorf("StagedRecordQueueSize must be at least 1 (got %d)", c.StagedRecordQueueSize) + } + if c.TargetDiskSizeBytes > 0 && c.TargetDiskSizeBytes < c.RecordSizeBytes { + return fmt.Errorf("TargetDiskSizeBytes, when non-zero, must be at least RecordSizeBytes (%d) (got %d)", + c.RecordSizeBytes, c.TargetDiskSizeBytes) + } + if c.PruneRelaxationFactor < 1 { + return fmt.Errorf("PruneRelaxationFactor must be at least 1 (got %d)", c.PruneRelaxationFactor) + } + if c.DataDir == "" { + return fmt.Errorf("DataDir is required") + } + if c.LogDir == "" { + return fmt.Errorf("LogDir is required") + } + if c.ConsoleUpdateIntervalSeconds < 0 { + return fmt.Errorf("ConsoleUpdateIntervalSeconds must be non-negative (got %f)", c.ConsoleUpdateIntervalSeconds) + } + if c.MaxRuntimeSeconds < 0 { + return fmt.Errorf("MaxRuntimeSeconds must be non-negative (got %d)", c.MaxRuntimeSeconds) + } + if c.BackgroundMetricsScrapeInterval < 0 { + return fmt.Errorf("BackgroundMetricsScrapeInterval must be non-negative (got %d)", c.BackgroundMetricsScrapeInterval) + } + if c.MaxRecordsPerSecond < 0 { + return fmt.Errorf("MaxRecordsPerSecond must be non-negative (got %f)", c.MaxRecordsPerSecond) + } + switch strings.ToLower(c.LogLevel) { + case "debug", "info", "warn", "error": + default: + return fmt.Errorf("LogLevel must be one of debug, info, warn, error (got %q)", c.LogLevel) + } + return nil +} diff --git a/sei-db/seiwal/walsim/walsim_metrics.go b/sei-db/seiwal/walsim/walsim_metrics.go new file mode 100644 index 0000000000..c100981e6d --- /dev/null +++ b/sei-db/seiwal/walsim/walsim_metrics.go @@ -0,0 +1,148 @@ +package walsim + +import ( + "context" + + "github.com/sei-protocol/sei-chain/sei-db/common/metrics" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +const walsimMeterName = "walsim" + +// WalsimMetrics holds OpenTelemetry metrics for the walsim benchmark. +type WalsimMetrics struct { + ctx context.Context + + recordsWrittenTotal metric.Int64Counter + bytesWrittenTotal metric.Int64Counter + flushCallsTotal metric.Int64Counter + pruneRequestsTotal metric.Int64Counter + + highestIndex metric.Int64Gauge + lowestIndex metric.Int64Gauge + recordSizeBytes metric.Int64Gauge + + mainThreadPhase *metrics.PhaseTimer +} + +// NewWalsimMetrics creates metrics for the walsim benchmark using the global OTel MeterProvider. The +// caller must configure the MeterProvider before calling this. +func NewWalsimMetrics(ctx context.Context, config *WalsimConfig) *WalsimMetrics { + meter := otel.Meter(walsimMeterName) + + recordsWrittenTotal, _ := meter.Int64Counter( + "walsim_records_written_total", + metric.WithDescription("Total number of records appended to the WAL"), + metric.WithUnit("{count}"), + ) + bytesWrittenTotal, _ := meter.Int64Counter( + "walsim_bytes_written_total", + metric.WithDescription("Total bytes of record payload data appended to the WAL"), + metric.WithUnit("By"), + ) + flushCallsTotal, _ := meter.Int64Counter( + "walsim_flush_calls_total", + metric.WithDescription("Total number of Flush calls"), + metric.WithUnit("{count}"), + ) + pruneRequestsTotal, _ := meter.Int64Counter( + "walsim_prune_requests_total", + metric.WithDescription("Total number of prune requests issued by the benchmark (before any legacy coalescing)"), + metric.WithUnit("{count}"), + ) + + highestIndex, _ := meter.Int64Gauge( + "walsim_highest_index", + metric.WithDescription("Highest record index currently stored in the WAL"), + metric.WithUnit("{index}"), + ) + lowestIndex, _ := meter.Int64Gauge( + "walsim_lowest_index", + metric.WithDescription("Lowest record index the benchmark has requested be retained"), + metric.WithUnit("{index}"), + ) + recordSizeBytes, _ := meter.Int64Gauge( + "walsim_record_size_bytes", + metric.WithDescription("Size in bytes of a single record (constant for a given config)"), + metric.WithUnit("By"), + ) + + mainThreadPhase := metrics.NewPhaseTimer(meter, "walsim_main_thread") + + m := &WalsimMetrics{ + ctx: ctx, + recordsWrittenTotal: recordsWrittenTotal, + bytesWrittenTotal: bytesWrittenTotal, + flushCallsTotal: flushCallsTotal, + pruneRequestsTotal: pruneRequestsTotal, + highestIndex: highestIndex, + lowestIndex: lowestIndex, + recordSizeBytes: recordSizeBytes, + mainThreadPhase: mainThreadPhase, + } + + m.recordRecordSize(config) + return m +} + +func (m *WalsimMetrics) recordRecordSize(config *WalsimConfig) { + if m == nil || m.recordSizeBytes == nil { + return + } + m.recordSizeBytes.Record(context.Background(), int64(config.RecordSizeBytes)) //nolint:gosec // record size is bounded by config +} + +// ReportRecordWritten records a single appended record of the given byte count. +func (m *WalsimMetrics) ReportRecordWritten(byteCount int64) { + if m == nil { + return + } + ctx := context.Background() + if m.recordsWrittenTotal != nil { + m.recordsWrittenTotal.Add(ctx, 1) + } + if m.bytesWrittenTotal != nil { + m.bytesWrittenTotal.Add(ctx, byteCount) + } +} + +// ReportFlush records a Flush call. +func (m *WalsimMetrics) ReportFlush() { + if m == nil || m.flushCallsTotal == nil { + return + } + m.flushCallsTotal.Add(context.Background(), 1) +} + +// ReportPruneRequest records a prune request issued by the benchmark. +func (m *WalsimMetrics) ReportPruneRequest() { + if m == nil || m.pruneRequestsTotal == nil { + return + } + m.pruneRequestsTotal.Add(context.Background(), 1) +} + +// RecordHighestIndex records the highest stored record index as a gauge. +func (m *WalsimMetrics) RecordHighestIndex(index uint64) { + if m == nil || m.highestIndex == nil { + return + } + m.highestIndex.Record(context.Background(), int64(index)) //nolint:gosec // index fits in int64 for any realistic run +} + +// RecordLowestIndex records the lowest retained record index as a gauge. +func (m *WalsimMetrics) RecordLowestIndex(index uint64) { + if m == nil || m.lowestIndex == nil { + return + } + m.lowestIndex.Record(context.Background(), int64(index)) //nolint:gosec // index fits in int64 for any realistic run +} + +// SetMainThreadPhase records a transition of the main loop into the named phase. +func (m *WalsimMetrics) SetMainThreadPhase(phase string) { + if m == nil || m.mainThreadPhase == nil { + return + } + m.mainThreadPhase.SetPhase(phase) +}