diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index f211069b52..0e77dd25e0 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -69,6 +69,7 @@ func TestBlockDB(t *testing.T) { t.Run("PruneQCOnlyThenWriteBlock", func(t *testing.T) { testPruneQCOnlyThenWriteBlock(t, impl.build) }) + t.Run("Status", func(t *testing.T) { testStatus(t, impl.build) }) t.Run("WriteOrderRejected", func(t *testing.T) { testWriteOrderRejected(t, impl.build) }) t.Run("WriteOrderRejectedAfterRestart", func(t *testing.T) { testWriteOrderRejectedAfterRestart(t, impl.build) @@ -130,6 +131,88 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, ok, "empty db should yield no QCs") require.NoError(t, qcIt.Close()) + + tips := db.Status() + require.False(t, tips.LastBlockNumber.IsPresent(), "empty db has no block write tip") + require.False(t, tips.LastQCNext.IsPresent(), "empty db has no QC write tip") +} + +// testStatus asserts Status matches the highest block/QC still present +// (via reverse iterators), including after prune and restart, and that a QC +// written ahead of its blocks advances only LastQCNext. +func testStatus(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, o := openFresh(t, build) + defer func() { _ = db.Close() }() + + require.NoError(t, db.WriteQC(batches[0].first, batches[0].next, batches[0].qc)) + tips := db.Status() + gotQC, ok := tips.LastQCNext.Get() + require.True(t, ok) + require.Equal(t, batches[0].next, gotQC) + require.False(t, tips.LastBlockNumber.IsPresent(), "QC-only store has no block tip") + assertTipsMatchPresent(t, db) + + for i, blk := range batches[0].blocks { + require.NoError(t, db.WriteBlock(batches[0].first+gbn(i), blk)) + } + writeAll(t, db, batches[1:]) + last := batches[len(batches)-1] + tips = db.Status() + gotBlock, ok := tips.LastBlockNumber.Get() + require.True(t, ok) + require.Equal(t, last.next-1, gotBlock) + gotQC, ok = tips.LastQCNext.Get() + require.True(t, ok) + require.Equal(t, last.next, gotQC) + assertTipsMatchPresent(t, db) + + // Prune away an early cohort; the write tip must still equal the highest + // present records (newest cohort is never removed). + require.Greater(t, len(batches), 1) + require.NoError(t, db.PruneBefore(batches[1].first)) + assertTipsMatchPresent(t, db) + tips = db.Status() + gotBlock, ok = tips.LastBlockNumber.Get() + require.True(t, ok) + require.Equal(t, last.next-1, gotBlock, "prune must not move the block write tip") + gotQC, ok = tips.LastQCNext.Get() + require.True(t, ok) + require.Equal(t, last.next, gotQC, "prune must not move the QC write tip") + + db = restart(t, o, db) + assertTipsMatchPresent(t, db) + tips = db.Status() + gotBlock, ok = tips.LastBlockNumber.Get() + require.True(t, ok) + require.Equal(t, last.next-1, gotBlock, "block tip must survive restart") + gotQC, ok = tips.LastQCNext.Get() + require.True(t, ok) + require.Equal(t, last.next, gotQC, "QC tip must survive restart") +} + +// assertTipsMatchPresent checks Status against one reverse-iterator +// step for blocks and QCs (the highest records the public read API still serves). +func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { + t.Helper() + tips := db.Status() + + highest, hasBlock := recoverHighestBlock(t, db) + if n, ok := tips.LastBlockNumber.Get(); ok { + require.True(t, hasBlock, "Status has a block tip but Blocks is empty") + require.Equal(t, highest, n, "LastBlockNumber must be the highest present block") + } else { + require.False(t, hasBlock, "Blocks has data but Status has no block tip") + } + + lastQC, hasQC := recoverLastQC(t, db) + if n, ok := tips.LastQCNext.Get(); ok { + require.True(t, hasQC, "Status has a QC tip but QCs is empty") + require.Equal(t, lastQC.GlobalRange().Next, n, "LastQCNext must be Next of the highest present QC") + } else { + require.False(t, hasQC, "QCs has data but Status has no QC tip") + } } func testReadRoundTrip(t *testing.T, build builder) { diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 2882e7c219..f5fd157d9b 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -328,6 +328,19 @@ func (s *blockDB) Flush() error { return nil } +func (s *blockDB) Status() types.DBStatus { + s.mu.Lock() + defer s.mu.Unlock() + var tips types.DBStatus + if s.hasBlocks { + tips.LastBlockNumber = utils.Some(s.lastBlockNumber) + } + if s.hasQC { + tips.LastQCNext = utils.Some(s.lastQCNext) + } + return tips +} + func (s *blockDB) Blocks(reverse bool) (types.BlockIterator, error) { it, err := s.table.Iterator(reverse) if err != nil { diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 8fb1e97243..930fe01529 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -147,6 +147,19 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { func (s *blockDB) Flush() error { return nil } +func (s *blockDB) Status() types.DBStatus { + s.mu.RLock() + defer s.mu.RUnlock() + var tips types.DBStatus + if s.hasBlocks { + tips.LastBlockNumber = utils.Some(s.lastBlockNumber) + } + if s.hasQC { + tips.LastQCNext = utils.Some(s.lastQCNext) + } + return tips +} + func (s *blockDB) Blocks(reverse bool) (types.BlockIterator, error) { s.mu.RLock() defer s.mu.RUnlock() diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index dc8cbf95d4..733a4c9832 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -183,6 +183,9 @@ type BlockDB interface { // issuance). Flush() error + // Status returns a consistent snapshot of the in-memory write tips (no I/O). + Status() DBStatus + // Blocks returns an iterator over every persisted block not yet // pruned, for startup replay. Intended to be called once at // construction by data.State.NewState. @@ -269,6 +272,14 @@ type BlockDB interface { Close() error } +// DBStatus is the in-memory write tips returned by BlockDB.Status. +type DBStatus struct { + // LastBlockNumber is the highest GlobalBlockNumber accepted by WriteBlock. + LastBlockNumber utils.Option[GlobalBlockNumber] + // LastQCNext is the exclusive upper bound of the last WriteQC. + LastQCNext utils.Option[GlobalBlockNumber] +} + // BlockIterator iterates over persisted blocks in GlobalBlockNumber order — // ascending, or descending if the iterator was created with reverse=true. It // is created via BlockDB.Blocks and captures a snapshot of the blocks present diff --git a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go index 13736e876d..76db17c455 100644 --- a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go +++ b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go @@ -34,6 +34,8 @@ Output is written to the file specified by --output.`, return fmt.Errorf("--output flag is required") } persistentStateDir, _ := cmd.Flags().GetString("persistent-state-dir") + blockDBRetention, _ := cmd.Flags().GetString("blockdb-retention") + blockDBGCPeriod, _ := cmd.Flags().GetString("blockdb-gc-period") var validators []config.AutobahnValidator for _, dir := range args { @@ -97,6 +99,11 @@ Output is written to the file specified by --output.`, if persistentStateDir != "" { cfg.PersistentStateDir = utils.Some(persistentStateDir) } + blockDB, err := buildGenBlockDBConfig(blockDBRetention, blockDBGCPeriod) + if err != nil { + return err + } + cfg.BlockDB = blockDB data, err := json.MarshalIndent(cfg, "", " ") if err != nil { @@ -110,6 +117,35 @@ Output is written to the file specified by --output.`, }, } cmd.Flags().StringP("output", "o", "", "output file path for the autobahn config") - cmd.Flags().String("persistent-state-dir", "data/autobahn", "directory to persist autobahn consensus and data WALs across restarts; relative paths are resolved against the node's --home dir; pass --persistent-state-dir= (empty) to disable persistence and run in-memory only") + cmd.Flags().String("persistent-state-dir", "data/autobahn", "directory to persist autobahn consensus state and BlockDB across restarts; relative paths are resolved against the node's --home dir; pass --persistent-state-dir= (empty) to disable persistence and run in-memory only (memblock)") + // Default 30s: this helper is used by docker/local clusters, not production + // node bring-up. Pass --blockdb-retention= (empty) to omit block_db and keep + // littblock's production default (24h). + cmd.Flags().String("blockdb-retention", "30s", "BlockDB retention TTL written into block_db (default 30s for local/docker); pass empty to omit and keep littblock's 24h default") + cmd.Flags().String("blockdb-gc-period", "", "optional BlockDB GC period (e.g. 10s); omit to keep littblock default") return cmd } + +// buildGenBlockDBConfig builds optional block_db overrides from gen-autobahn-config flags. +// Empty duration strings leave BlockDB as the zero value (omitted from JSON). +func buildGenBlockDBConfig(retention, gcPeriod string) (config.AutobahnBlockDBConfig, error) { + var bdb config.AutobahnBlockDBConfig + if retention != "" { + d, err := time.ParseDuration(retention) + if err != nil { + return config.AutobahnBlockDBConfig{}, fmt.Errorf("--blockdb-retention: %w", err) + } + bdb.Retention = utils.Some(utils.Duration(d)) + } + if gcPeriod != "" { + d, err := time.ParseDuration(gcPeriod) + if err != nil { + return config.AutobahnBlockDBConfig{}, fmt.Errorf("--blockdb-gc-period: %w", err) + } + bdb.GCPeriod = utils.Some(utils.Duration(d)) + } + if err := bdb.Validate(); err != nil { + return config.AutobahnBlockDBConfig{}, fmt.Errorf("block_db: %w", err) + } + return bdb, nil +} diff --git a/sei-tendermint/config/autobahn.go b/sei-tendermint/config/autobahn.go index 6c08e0629c..c042a9afc1 100644 --- a/sei-tendermint/config/autobahn.go +++ b/sei-tendermint/config/autobahn.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -34,6 +35,26 @@ type AutobahnValidator struct { EVMRPC URL `json:"evmrpc"` } +// AutobahnBlockDBConfig holds optional overrides for the LittDB-backed BlockDB +// opened under persistent_state_dir/blockdb. Paths are never taken from here — +// Autobahn always roots BlockDB at /blockdb. +// +// Each field is independently optional. Absent fields keep whatever +// littblock.DefaultConfig currently uses (do not duplicate those values here — +// they live in the littblock / LittDB packages and may change). +// +// Disk impact (most → least useful for bounding usage): Retention, then +// GCPeriod. Segment sizing is intentionally not exposed (engine-internal). +type AutobahnBlockDBConfig struct { + // Retention is the failsafe minimum age before pruned records may be + // reclaimed. Primary knob for worst-case disk after PruneBefore advances. + // Absent ⇒ littblock.DefaultConfig Retention. + Retention utils.Option[utils.Duration] `json:"retention"` + // GCPeriod is how often GC runs once data is eligible (reclaim latency). + // Absent ⇒ littblock.DefaultConfig / LittDB GCPeriod. + GCPeriod utils.Option[utils.Duration] `json:"gc_period"` +} + // AutobahnFileConfig is the JSON structure of the autobahn config file. type AutobahnFileConfig struct { Validators []AutobahnValidator `json:"validators"` @@ -49,6 +70,11 @@ type AutobahnFileConfig struct { // fullnodes serving downstream block-sync are subject to the same // cap). Absent ⇒ DefaultMaxInboundFullnodePeers. Some(0) ⇒ reject all. MaxInboundFullnodePeers utils.Option[uint64] `json:"max_inbound_fullnode_peers"` + // BlockDB optionally overlays AutobahnBlockDBConfig onto littblock.DefaultConfig + // when PersistentStateDir is set. Zero value ⇒ littblock.DefaultConfig unchanged + // (see AutobahnBlockDBConfig for field semantics). Ignored when + // PersistentStateDir is absent (memblock). Omitted from JSON when empty. + BlockDB AutobahnBlockDBConfig `json:"block_db,omitzero"` } // DefaultMaxInboundFullnodePeers is the built-in cap used when @@ -80,5 +106,40 @@ func (fc *AutobahnFileConfig) Validate() error { if fc.DialInterval <= 0 { return errors.New("dial_interval must be > 0") } + if err := fc.BlockDB.Validate(); err != nil { + return fmt.Errorf("block_db: %w", err) + } return nil } + +// Validate checks optional BlockDB overrides. Absent fields are fine. +func (c AutobahnBlockDBConfig) Validate() error { + if r, ok := c.Retention.Get(); ok && r <= 0 { + return errors.New("retention must be > 0 when set") + } + if p, ok := c.GCPeriod.Get(); ok && p <= 0 { + return errors.New("gc_period must be > 0 when set") + } + return nil +} + +// LittBlockConfig returns littblock.DefaultConfig(dir) with this config's +// optional overrides applied. Fsync is always forced on. +func (c AutobahnBlockDBConfig) LittBlockConfig(dir string) (littblock.LittBlockConfig, error) { + if err := c.Validate(); err != nil { + return littblock.LittBlockConfig{}, err + } + cfg, err := littblock.DefaultConfig(dir) + if err != nil { + return littblock.LittBlockConfig{}, fmt.Errorf("littblock.DefaultConfig: %w", err) + } + if r, ok := c.Retention.Get(); ok { + cfg.Retention = r.Duration() + } + if p, ok := c.GCPeriod.Get(); ok { + cfg.Litt.GCPeriod = p.Duration() + } + // NOT SAFE to set false: crash can lose acknowledged BlockDB writes. + cfg.Litt.Fsync = true + return *cfg, nil +} diff --git a/sei-tendermint/config/autobahn_test.go b/sei-tendermint/config/autobahn_test.go index 92bf883f2a..b6e4b815a7 100644 --- a/sei-tendermint/config/autobahn_test.go +++ b/sei-tendermint/config/autobahn_test.go @@ -4,7 +4,9 @@ import ( "encoding/json" "net/url" "testing" + "time" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/stretchr/testify/require" ) @@ -24,3 +26,19 @@ func TestURLJSONReencode(t *testing.T) { require.NotNil(t, got.URL) require.Equal(t, want.String(), got.String()) } + +func TestAutobahnBlockDBConfig_LittBlockConfig(t *testing.T) { + dir := t.TempDir() + const ( + wantRetention = 2 * time.Hour + wantGCPeriod = 3 * time.Second + ) + cfg, err := (AutobahnBlockDBConfig{ + Retention: utils.Some(utils.Duration(wantRetention)), + GCPeriod: utils.Some(utils.Duration(wantGCPeriod)), + }).LittBlockConfig(dir) + require.NoError(t, err) + require.Equal(t, wantRetention, cfg.Retention) + require.Equal(t, wantGCPeriod, cfg.Litt.GCPeriod) + require.True(t, cfg.Litt.Fsync) +} diff --git a/sei-tendermint/internal/autobahn/avail/inner_test.go b/sei-tendermint/internal/autobahn/avail/inner_test.go index afdee71d9a..a6deb0a076 100644 --- a/sei-tendermint/internal/autobahn/avail/inner_test.go +++ b/sei-tendermint/internal/autobahn/avail/inner_test.go @@ -3,6 +3,7 @@ package avail import ( "testing" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" @@ -12,6 +13,10 @@ import ( "github.com/stretchr/testify/require" ) +func newTestDataState(cfg *data.Config) *data.State { + return utils.OrPanic1(data.NewState(cfg, memblock.NewBlockDB())) +} + func TestPruneMismatchedIndices(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) @@ -36,8 +41,8 @@ func TestPruneMismatchedIndices(t *testing.T) { qc1 := makeCommitQC(utils.Some(qc0)) t.Logf("test State.PushAppQC") - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) - state, err := NewState(keys[0], ds, utils.None[string]()) + ds := newTestDataState(&data.Config{Registry: registry}) + state, err := NewState(keys[0], ds, utils.Some(t.TempDir())) require.NoError(t, err) require.Error(t, state.PushAppQC(makeAppQC(qc0, qc0), qc1), "bad range, bad index should fail") require.Error(t, state.PushAppQC(makeAppQC(qc1, qc0), qc1), "good range, bad index should fail") @@ -45,8 +50,8 @@ func TestPruneMismatchedIndices(t *testing.T) { require.NoError(t, state.PushAppQC(makeAppQC(qc1, qc1), qc1), "good range, good index should succeed") t.Logf("test inner.prune") - ds = utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) - state, err = NewState(keys[0], ds, utils.None[string]()) + ds = newTestDataState(&data.Config{Registry: registry}) + state, err = NewState(keys[0], ds, utils.Some(t.TempDir())) require.NoError(t, err) for inner := range state.inner.Lock() { _, err := inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc1, qc0), qc1) diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 3558c4a5ad..3fc5f17c4b 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -93,9 +93,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { committee := registry.LatestEpoch().Committee() if err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - ds := utils.OrPanic1(data.NewState(&data.Config{ - Registry: registry, - }, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) s.SpawnBgNamed("data.State.Run()", func() error { return utils.IgnoreCancel(ds.Run(ctx)) }) @@ -225,7 +223,7 @@ func TestStateRestartFromPersisted(t *testing.T) { var wantNextBlocks map[types.LaneID]types.BlockNumber require.NoError(t, scope.Run(t.Context(), func(ctx context.Context, s scope.Scope) error { - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) s.SpawnBgNamed("data.Run", func() error { return utils.IgnoreCancel(ds.Run(ctx)) }) @@ -300,7 +298,7 @@ func TestStateRestartFromPersisted(t *testing.T) { })) // Phase 2: Restart from the same directory. - ds2 := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds2 := newTestDataState(&data.Config{Registry: registry}) state2, err := NewState(keys[0], ds2, utils.Some(dir)) require.NoError(t, err) @@ -325,10 +323,8 @@ func TestStateMismatchedQCs(t *testing.T) { committee := registry.LatestEpoch().Committee() initialBlock := registry.FirstBlock() - ds := utils.OrPanic1(data.NewState(&data.Config{ - Registry: registry, - }, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) - state, err := NewState(keys[0], ds, utils.None[string]()) + ds := newTestDataState(&data.Config{Registry: registry}) + state, err := NewState(keys[0], ds, utils.Some(t.TempDir())) require.NoError(t, err) // Helper to create a CommitQC for a specific index @@ -382,10 +378,8 @@ func TestPushBlockRejectsBadParentHash(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - ds := utils.OrPanic1(data.NewState(&data.Config{ - Registry: registry, - }, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) - state := utils.OrPanic1(NewState(keys[0], ds, utils.None[string]())) + ds := newTestDataState(&data.Config{Registry: registry}) + state := utils.OrPanic1(NewState(keys[0], ds, utils.Some(t.TempDir()))) // Produce a valid first block on our lane. _, err := state.ProduceLocalBlock(state.NextBlock(keys[0].Public()), types.GenPayload(rng)) @@ -407,10 +401,8 @@ func TestPushBlockRejectsWrongSigner(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - ds := utils.OrPanic1(data.NewState(&data.Config{ - Registry: registry, - }, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) - state := utils.OrPanic1(NewState(keys[0], ds, utils.None[string]())) + ds := newTestDataState(&data.Config{Registry: registry}) + state := utils.OrPanic1(NewState(keys[0], ds, utils.Some(t.TempDir()))) // Create a block on keys[0]'s lane but sign it with keys[1]. lane := keys[0].Public() @@ -424,11 +416,11 @@ func TestPushBlockRejectsWrongSigner(t *testing.T) { func TestNewStateWithPersistence(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - initialBlock := types.GlobalBlockNumber(0) + initialBlock := registry.FirstBlock() t.Run("empty dir loads fresh state", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) state, err := NewState(keys[0], ds, utils.Some(dir)) require.NoError(t, err) @@ -441,7 +433,7 @@ func TestNewStateWithPersistence(t *testing.T) { t.Run("loads persisted AppQC", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) roadIdx := types.RoadIndex(7) globalNum := types.GlobalBlockNumber(50) @@ -482,7 +474,7 @@ func TestNewStateWithPersistence(t *testing.T) { t.Run("loads persisted blocks", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) lane := keys[0].Public() // Persist blocks using BlockPersister. @@ -506,7 +498,7 @@ func TestNewStateWithPersistence(t *testing.T) { t.Run("loads persisted AppQC and blocks together", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) lane := keys[0].Public() roadIdx := types.RoadIndex(2) @@ -559,7 +551,7 @@ func TestNewStateWithPersistence(t *testing.T) { t.Run("loads persisted commitQCs", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) // Persist CommitQCs to disk. cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) @@ -579,14 +571,12 @@ func TestNewStateWithPersistence(t *testing.T) { // All 3 commitQCs should be loaded (no AppQC to skip past). require.Equal(t, types.RoadIndex(0), state.FirstCommitQC()) // LastCommitQC should be set to the last loaded one. - got2, ok2 := state.LastCommitQC().Load().Get() - require.True(t, ok2) - require.NoError(t, utils.TestDiff(qcs[2], got2)) + require.NoError(t, utils.TestDiff(utils.Some(qcs[2]), state.LastCommitQC().Load())) }) t.Run("loads persisted commitQCs with AppQC", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) // Persist AppQC at road index 1. roadIdx := types.RoadIndex(1) @@ -619,9 +609,7 @@ func TestNewStateWithPersistence(t *testing.T) { // inner.prune(appQC@1, commitQC@1) sets commitQCs.first = 1. require.Equal(t, types.RoadIndex(1), state.FirstCommitQC()) - got4, ok4 := state.LastCommitQC().Load().Get() - require.True(t, ok4) - require.NoError(t, utils.TestDiff(qcs[4], got4)) + require.NoError(t, utils.TestDiff(utils.Some(qcs[4]), state.LastCommitQC().Load())) }) t.Run("non-contiguous commitQC files return error", func(t *testing.T) { @@ -661,7 +649,7 @@ func TestNewStateWithPersistence(t *testing.T) { t.Run("anchor past all persisted commitQCs truncates WAL", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) // Build a chain of 10 CommitQCs (indices 0-9). qcs := make([]*types.CommitQC, 10) @@ -695,9 +683,7 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, err) require.Equal(t, types.RoadIndex(9), state.FirstCommitQC()) - got9, ok9 := state.LastCommitQC().Load().Get() - require.True(t, ok9) - require.NoError(t, utils.TestDiff(qcs[9], got9)) + require.NoError(t, utils.TestDiff(utils.Some(qcs[9]), state.LastCommitQC().Load())) got, ok := state.LastAppQC().Get() require.True(t, ok) @@ -706,7 +692,7 @@ func TestNewStateWithPersistence(t *testing.T) { t.Run("anchor past all persisted blocks truncates lane WAL", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) lane := keys[0].Public() // Persist commitQCs 0-9 and blocks 0-2 for one lane. @@ -754,7 +740,7 @@ func TestNewStateWithPersistence(t *testing.T) { t.Run("corrupt AppQC data returns error", func(t *testing.T) { dir := t.TempDir() - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + ds := newTestDataState(&data.Config{Registry: registry}) // Create a throwaway persister to discover the A/B filenames, // then corrupt them so NewState fails on load. @@ -770,49 +756,3 @@ func TestNewStateWithPersistence(t *testing.T) { require.Error(t, err) }) } - -func TestWaitForLaneQCs_OnlyReturnsCommitteeLanes(t *testing.T) { - ctx := t.Context() - rng := utils.TestRng() - - committeeKey := types.GenSecretKey(rng) - committee := utils.OrPanic1(types.NewCommittee(map[types.PublicKey]uint64{ - committeeKey.Public(): 1, - })) - registry := utils.OrPanic1(epoch.NewRegistry(committee, 0, time.Time{})) - ep := registry.LatestEpoch() - - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) - state, err := NewState(committeeKey, ds, utils.None[string]()) - require.NoError(t, err) - - require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - s.SpawnBgNamed("data.Run", func() error { return utils.IgnoreCancel(ds.Run(ctx)) }) - s.SpawnBgNamed("avail.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) - - // Produce and vote on a block for the committee lane. - b, err := state.produceLocalBlock(0, committeeKey, types.GenPayload(rng)) - if err != nil { - return err - } - for _, vote := range makeLaneVotes([]types.SecretKey{committeeKey}, b.Msg().Block().Header()) { - if err := state.PushVote(ctx, vote); err != nil { - return err - } - } - - laneQCs, err := state.WaitForLaneQCs(ctx, ep, utils.None[*types.CommitQC]()) - if err != nil { - return err - } - for lane := range laneQCs { - if !ep.Committee().HasLane(lane) { - return fmt.Errorf("WaitForLaneQCs returned lane %v outside committee", lane) - } - } - if _, ok := laneQCs[committeeKey.Public()]; !ok { - return fmt.Errorf("WaitForLaneQCs missing committee lane") - } - return nil - })) -} diff --git a/sei-tendermint/internal/autobahn/consensus/inner_test.go b/sei-tendermint/internal/autobahn/consensus/inner_test.go index 6625771c79..93d13f84ac 100644 --- a/sei-tendermint/internal/autobahn/consensus/inner_test.go +++ b/sei-tendermint/internal/autobahn/consensus/inner_test.go @@ -3,9 +3,12 @@ package consensus import ( "context" "errors" + "path/filepath" "testing" "time" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" @@ -16,6 +19,18 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" ) +func newTestBlockDB(t *testing.T, dir string) types.BlockDB { + t.Helper() + cfg := utils.OrPanic1(littblock.DefaultConfig(dir)) + db := utils.OrPanic1(littblock.NewBlockDB(cfg)) + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func newTestDataState(registry *epoch.Registry) *data.State { + return utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, memblock.NewBlockDB())) +} + // seedPersistedInner is a test helper that persists a persistedInner using the public API. func seedPersistedInner(dir string, state *persistedInner) { p, _, err := persist.NewPersister[*pb.PersistedInner](utils.Some(dir), innerFile) @@ -921,15 +936,21 @@ func (f failPersister[T]) Persist(T) error { return f.err } func TestRunOutputsPersistErrorPropagates(t *testing.T) { // Verify that a persist error in runOutputs propagates // and terminates the consensus component (instead of panicking). + dir := t.TempDir() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - ds := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + db := newTestBlockDB(t, filepath.Join(dir, "blockdb")) + ds, err := data.NewState(&data.Config{Registry: registry}, db) + if err != nil { + t.Fatalf("data.NewState: %v", err) + } wantErr := errors.New("disk on fire") pers := utils.Some[persist.Persister[*pb.PersistedInner]](failPersister[*pb.PersistedInner]{err: wantErr}) cs, err := newState(&Config{ - Key: keys[0], - ViewTimeout: func(types.View) time.Duration { return time.Hour }, + Key: keys[0], + ViewTimeout: func(types.View) time.Duration { return time.Hour }, + PersistentStateDir: utils.Some(dir), }, ds, pers, utils.None[*pb.PersistedInner]()) require.NoError(t, err) diff --git a/sei-tendermint/internal/autobahn/consensus/persist/fullcommitqcs.go b/sei-tendermint/internal/autobahn/consensus/persist/fullcommitqcs.go deleted file mode 100644 index 13b44b0fd2..0000000000 --- a/sei-tendermint/internal/autobahn/consensus/persist/fullcommitqcs.go +++ /dev/null @@ -1,167 +0,0 @@ -package persist - -import ( - "fmt" - "path/filepath" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -const fullCommitQCsDir = "fullcommitqcs" - -// fullCommitQCState is the mutable state protected by FullCommitQCPersister's mutex. -type fullCommitQCState struct { - iw utils.Option[*indexedWAL[*types.FullCommitQC]] - firstBlock types.GlobalBlockNumber - next types.GlobalBlockNumber // next expected GlobalRange().First == last QC's GlobalRange().Next - loaded []*types.FullCommitQC -} - -func (s *fullCommitQCState) persistQC(qc *types.FullCommitQC) error { - gr := qc.QC().GlobalRange() - if gr.First < s.next { - return nil - } - if gr.First > s.next { - return fmt.Errorf("full commitqc %d out of sequence (next=%d)", gr.First, s.next) - } - if iw, ok := s.iw.Get(); ok { - if err := iw.Write(qc); err != nil { - return fmt.Errorf("persist full commitqc at %d: %w", gr.First, err) - } - } - s.next = gr.Next - return nil -} - -func (s *fullCommitQCState) truncateBefore(n types.GlobalBlockNumber) error { - if n == 0 { - return nil - } - if n > s.next { - s.next = n - } - iw, ok := s.iw.Get() - if !ok || iw.Count() == 0 { - return nil - } - // Remove QCs whose range is fully before n. A QC is stale when - // GlobalRange().Next <= n. In practice the scan visits 0–1 entries - // per prune call because pruning advances one block at a time while - // each QC covers many blocks. - if err := iw.TruncateWhile(func(entry *types.FullCommitQC) bool { - return entry.QC().GlobalRange().Next <= n - }); err != nil { - return fmt.Errorf("truncate full commitqc WAL: %w", err) - } - return nil -} - -// FullCommitQCPersister manages persistence of FullCommitQCs using a WAL. -// Each entry is one FullCommitQC covering a range of global block numbers. -// When stateDir is None, all disk I/O is skipped (no-op mode). -// All public methods are safe for concurrent use. -type FullCommitQCPersister struct { - state utils.Mutex[*fullCommitQCState] -} - -// NewFullCommitQCPersister opens (or creates) a WAL in the fullcommitqcs/ -// subdir and replays all persisted entries. Loaded QCs are available via -// ConsumeLoaded. When stateDir is None, returns a no-op persister. -func NewFullCommitQCPersister(stateDir utils.Option[string], firstBlock types.GlobalBlockNumber) (*FullCommitQCPersister, error) { - sd, ok := stateDir.Get() - if !ok { - return &FullCommitQCPersister{state: utils.NewMutex(&fullCommitQCState{firstBlock: firstBlock, next: firstBlock})}, nil - } - dir := filepath.Join(sd, fullCommitQCsDir) - iw, err := openIndexedWAL(dir, types.FullCommitQCConv) - if err != nil { - return nil, fmt.Errorf("open full commitqc WAL in %s: %w", dir, err) - } - - s := &fullCommitQCState{iw: utils.Some(iw), firstBlock: firstBlock, next: firstBlock} - loaded, err := s.loadAll() - if err != nil { - _ = iw.Close() - return nil, err - } - if len(loaded) > 0 { - s.firstBlock = loaded[0].QC().GlobalRange().First - s.next = loaded[len(loaded)-1].QC().GlobalRange().Next - } - s.loaded = loaded - return &FullCommitQCPersister{ - state: utils.NewMutex(s), - }, nil -} - -// Next returns the next GlobalBlockNumber expected by the persister -// (i.e., the next QC's GlobalRange().First). -func (gp *FullCommitQCPersister) Next() types.GlobalBlockNumber { - for s := range gp.state.Lock() { - return s.next - } - panic("unreachable") -} - -// LoadedFirst returns the first global block number of the first loaded QC, -// or firstBlock if empty. -func (gp *FullCommitQCPersister) LoadedFirst() types.GlobalBlockNumber { - for s := range gp.state.Lock() { - if len(s.loaded) > 0 { - return s.loaded[0].QC().GlobalRange().First - } - return s.firstBlock - } - panic("unreachable") -} - -// ConsumeLoaded returns QCs loaded from the WAL during construction -// and nils the internal slice so the data is not retained. -func (gp *FullCommitQCPersister) ConsumeLoaded() []*types.FullCommitQC { - for s := range gp.state.Lock() { - loaded := s.loaded - s.loaded = nil - return loaded - } - panic("unreachable") -} - -// PersistQC appends a FullCommitQC to the WAL. Duplicates are silently ignored. -// Gaps return an error. -func (gp *FullCommitQCPersister) PersistQC(qc *types.FullCommitQC) error { - for s := range gp.state.Lock() { - return s.persistQC(qc) - } - panic("unreachable") -} - -// TruncateBefore removes all QC entries whose range is fully before n. -func (gp *FullCommitQCPersister) TruncateBefore(n types.GlobalBlockNumber) error { - for s := range gp.state.Lock() { - return s.truncateBefore(n) - } - panic("unreachable") -} - -// Close shuts down the WAL. -func (gp *FullCommitQCPersister) Close() error { - for s := range gp.state.Lock() { - iw, ok := s.iw.Get() - if !ok { - return nil - } - s.iw = utils.None[*indexedWAL[*types.FullCommitQC]]() - return iw.Close() - } - panic("unreachable") -} - -func (s *fullCommitQCState) loadAll() ([]*types.FullCommitQC, error) { - iw, ok := s.iw.Get() - if !ok { - return nil, nil - } - return iw.ReadAll() -} diff --git a/sei-tendermint/internal/autobahn/consensus/persist/fullcommitqcs_test.go b/sei-tendermint/internal/autobahn/consensus/persist/fullcommitqcs_test.go deleted file mode 100644 index 5d21fa5046..0000000000 --- a/sei-tendermint/internal/autobahn/consensus/persist/fullcommitqcs_test.go +++ /dev/null @@ -1,270 +0,0 @@ -package persist - -import ( - "testing" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" -) - -func makeSequentialFullCommitQCs( - rng utils.Rng, - registry *epoch.Registry, - keys []types.SecretKey, - n int, -) []*types.FullCommitQC { - qcs := make([]*types.FullCommitQC, n) - prev := utils.None[*types.CommitQC]() - for i := range n { - vs := types.ViewSpec{CommitQC: prev, Epoch: registry.LatestEpoch()} - committee := vs.Epoch.Committee() - lane := committee.Lanes().At(rng.Intn(committee.Lanes().Len())) - b := types.NewBlock(lane, types.LaneRangeOpt(prev, lane).Next(), types.GenBlockHeaderHash(rng), types.GenPayload(rng)) - lv := types.NewLaneVote(b.Header()) - lvotes := make([]*types.Signed[*types.LaneVote], 0, len(keys)) - for _, k := range keys { - lvotes = append(lvotes, types.Sign(k, lv)) - } - laneQCs := map[types.LaneID]*types.LaneQC{lane: types.NewLaneQC(lvotes)} - cqc := types.BuildCommitQC(vs.Epoch, keys, prev, laneQCs, utils.None[*types.AppQC]()) - qcs[i] = types.NewFullCommitQC(cqc, []*types.BlockHeader{b.Header()}) - prev = utils.Some(qcs[i].QC()) - } - return qcs -} - -func TestNewFullCommitQCPersisterEmptyDir(t *testing.T) { - dir := t.TempDir() - gp, err := NewFullCommitQCPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.NotNil(t, gp) - require.Equal(t, 0, len(gp.ConsumeLoaded())) - require.Equal(t, types.GlobalBlockNumber(0), gp.Next()) - require.NoError(t, gp.Close()) -} - -func TestNewFullCommitQCPersisterNoop(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 5) - - gp, err := NewFullCommitQCPersister(utils.None[string](), registry.FirstBlock()) - require.NoError(t, err) - require.NotNil(t, gp) - require.Equal(t, 0, len(gp.ConsumeLoaded())) - - for _, qc := range qcs { - require.NoError(t, gp.PersistQC(qc)) - } - lastNext := qcs[len(qcs)-1].QC().GlobalRange().Next - require.Equal(t, lastNext, gp.Next()) - - // Truncate past everything in no-op mode advances cursor. - futureN := lastNext + 100 - require.NoError(t, gp.TruncateBefore(futureN)) - require.Equal(t, futureN, gp.Next()) - require.NoError(t, gp.Close()) -} - -func TestFullCommitQCPersisterFirstBlockFromWAL(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 3) - wantFirst := qcs[0].QC().GlobalRange().First - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - for _, qc := range qcs { - require.NoError(t, gp.PersistQC(qc)) - } - require.NoError(t, gp.Close()) - - // Reopen with a wrong firstBlock — WAL should take precedence. - gp2, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()+999) - require.NoError(t, err) - require.Equal(t, wantFirst, gp2.LoadedFirst()) - require.NoError(t, gp2.Close()) -} - -func TestFullCommitQCPersistAndReload(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 5) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - for _, qc := range qcs { - require.NoError(t, gp.PersistQC(qc)) - } - lastNext := qcs[len(qcs)-1].QC().GlobalRange().Next - require.Equal(t, lastNext, gp.Next()) - require.NoError(t, gp.Close()) - - gp2, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - loaded := gp2.ConsumeLoaded() - require.Equal(t, len(qcs), len(loaded)) - for i, lqc := range loaded { - require.Equal(t, qcs[i].QC().GlobalRange().First, lqc.QC().GlobalRange().First) - } - require.Equal(t, lastNext, gp2.Next()) - require.NoError(t, gp2.Close()) -} - -func TestFullCommitQCTruncateAndReload(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 5) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - for _, qc := range qcs { - require.NoError(t, gp.PersistQC(qc)) - } - // Truncate before the third QC's range start, which should remove - // all QCs whose range is fully below that point. - truncPoint := qcs[2].QC().GlobalRange().First - require.NoError(t, gp.TruncateBefore(truncPoint)) - require.NoError(t, gp.Close()) - - gp2, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - loaded := gp2.ConsumeLoaded() - // QCs 0 and 1 should be gone (their ranges are fully before truncPoint). - // QC 2 should be the first one remaining. - require.GreaterOrEqual(t, len(loaded), 1) - require.Equal(t, qcs[2].QC().GlobalRange().First, loaded[0].QC().GlobalRange().First) - require.NoError(t, gp2.Close()) -} - -func TestFullCommitQCTruncateAll(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 3) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - for _, qc := range qcs { - require.NoError(t, gp.PersistQC(qc)) - } - lastNext := qcs[len(qcs)-1].QC().GlobalRange().Next - require.NoError(t, gp.TruncateBefore(lastNext+100)) - require.NoError(t, gp.Close()) - - gp2, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - require.Equal(t, 0, len(gp2.ConsumeLoaded())) - require.NoError(t, gp2.Close()) -} - -func TestFullCommitQCDuplicateIgnored(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 2) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - require.NoError(t, gp.PersistQC(qcs[0])) - require.NoError(t, gp.PersistQC(qcs[0])) // duplicate - require.Equal(t, qcs[0].QC().GlobalRange().Next, gp.Next()) - require.NoError(t, gp.Close()) -} - -func TestFullCommitQCGapError(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 3) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - // Skip qcs[0] and try to persist qcs[1] directly. - err = gp.PersistQC(qcs[1]) - require.Error(t, err) - require.Contains(t, err.Error(), "out of sequence") - require.NoError(t, gp.Close()) -} - -func TestFullCommitQCTruncateBeforeNoop(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 3) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - for _, qc := range qcs { - require.NoError(t, gp.PersistQC(qc)) - } - // TruncateBefore(0) is a no-op. - require.NoError(t, gp.TruncateBefore(0)) - require.NoError(t, gp.Close()) - - gp2, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - require.Equal(t, 3, len(gp2.ConsumeLoaded())) - require.NoError(t, gp2.Close()) -} - -func TestFullCommitQCContinueAfterReload(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 6) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - for _, qc := range qcs[:3] { - require.NoError(t, gp.PersistQC(qc)) - } - require.NoError(t, gp.Close()) - - gp2, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - require.Equal(t, 3, len(gp2.ConsumeLoaded())) - for _, qc := range qcs[3:] { - require.NoError(t, gp2.PersistQC(qc)) - } - require.Equal(t, qcs[5].QC().GlobalRange().Next, gp2.Next()) - require.NoError(t, gp2.Close()) - - gp3, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - require.Equal(t, 6, len(gp3.ConsumeLoaded())) - require.NoError(t, gp3.Close()) -} - -func TestFullCommitQCTruncateMidRange(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qcs := makeSequentialFullCommitQCs(rng, registry, keys, 5) - - gp, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - for _, qc := range qcs { - require.NoError(t, gp.PersistQC(qc)) - } - // Truncate at a point inside the first QC's range. - // The first QC should be kept because its range extends past truncPoint. - gr0 := qcs[0].QC().GlobalRange() - if gr0.Len() > 1 { - midPoint := gr0.First + types.GlobalBlockNumber(gr0.Len()/2) - require.NoError(t, gp.TruncateBefore(midPoint)) - require.NoError(t, gp.Close()) - - gp2, err := NewFullCommitQCPersister(utils.Some(dir), registry.FirstBlock()) - require.NoError(t, err) - require.Equal(t, 5, len(gp2.ConsumeLoaded())) - require.NoError(t, gp2.Close()) - } else { - require.NoError(t, gp.Close()) - } -} diff --git a/sei-tendermint/internal/autobahn/consensus/persist/globalblocks.go b/sei-tendermint/internal/autobahn/consensus/persist/globalblocks.go deleted file mode 100644 index d366a13f28..0000000000 --- a/sei-tendermint/internal/autobahn/consensus/persist/globalblocks.go +++ /dev/null @@ -1,232 +0,0 @@ -package persist - -import ( - "encoding/binary" - "fmt" - "path/filepath" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -const globalBlocksDir = "globalblocks" - -// LoadedGlobalBlock is a block loaded from disk during state restoration. -// Also used as the internal WAL entry type. Block doesn't carry its -// GlobalBlockNumber (that's assigned by the ordering layer), so we embed -// it in each WAL entry to make entries self-describing. -type LoadedGlobalBlock struct { - Number types.GlobalBlockNumber - Block *types.Block -} - -// loadedGlobalBlockCodec serializes LoadedGlobalBlock as [8-byte LE number][proto block]. -type loadedGlobalBlockCodec struct{} - -func (loadedGlobalBlockCodec) Marshal(e LoadedGlobalBlock) []byte { - proto := types.BlockConv.Marshal(e.Block) - out := make([]byte, 8+len(proto)) - binary.LittleEndian.PutUint64(out, uint64(e.Number)) - copy(out[8:], proto) - return out -} - -func (loadedGlobalBlockCodec) Unmarshal(raw []byte) (LoadedGlobalBlock, error) { - if len(raw) < 8 { - return LoadedGlobalBlock{}, fmt.Errorf("global block entry too short: %d bytes", len(raw)) - } - n := types.GlobalBlockNumber(binary.LittleEndian.Uint64(raw[:8])) - block, err := types.BlockConv.Unmarshal(raw[8:]) - if err != nil { - return LoadedGlobalBlock{}, fmt.Errorf("unmarshal block %d: %w", n, err) - } - return LoadedGlobalBlock{Number: n, Block: block}, nil -} - -// globalBlockState is the mutable state protected by GlobalBlockPersister's mutex. -type globalBlockState struct { - iw utils.Option[*indexedWAL[LoadedGlobalBlock]] - firstBlock types.GlobalBlockNumber - next types.GlobalBlockNumber - loaded []LoadedGlobalBlock -} - -func (s *globalBlockState) persistBlock(n types.GlobalBlockNumber, block *types.Block) error { - if n < s.next { - return nil - } - if n > s.next { - return fmt.Errorf("global block %d out of sequence (next=%d)", n, s.next) - } - if iw, ok := s.iw.Get(); ok { - if err := iw.Write(LoadedGlobalBlock{Number: n, Block: block}); err != nil { - return fmt.Errorf("persist global block %d: %w", n, err) - } - } - s.next = n + 1 - return nil -} - -func (s *globalBlockState) truncateBefore(n types.GlobalBlockNumber) error { - if n == 0 { - return nil - } - if n > s.next { - s.next = n - } - iw, ok := s.iw.Get() - if !ok || iw.Count() == 0 { - return nil - } - firstGlobal := s.next - types.GlobalBlockNumber(iw.Count()) - if n <= firstGlobal { - return nil - } - walIdx := iw.FirstIdx() + uint64(n-firstGlobal) - if err := iw.TruncateBefore(walIdx, func(entry LoadedGlobalBlock) error { - if entry.Number != n { - return fmt.Errorf("global block at WAL index %d has number %d, expected %d (index mapping broken)", walIdx, entry.Number, n) - } - return nil - }); err != nil { - return fmt.Errorf("truncate global block WAL before %d: %w", n, err) - } - return nil -} - -// GlobalBlockPersister manages persistence of globally-ordered blocks using a WAL. -// Each entry embeds its GlobalBlockNumber since Block doesn't carry it. -// When stateDir is None, all disk I/O is skipped (no-op mode). -// All public methods are safe for concurrent use. -type GlobalBlockPersister struct { - state utils.Mutex[*globalBlockState] -} - -// NewGlobalBlockPersister opens (or creates) a WAL in the globalblocks/ subdir -// and replays all persisted entries. Loaded blocks are available via -// ConsumeLoaded. When stateDir is None, returns a no-op persister. -func NewGlobalBlockPersister(stateDir utils.Option[string], firstBlock types.GlobalBlockNumber) (*GlobalBlockPersister, error) { - sd, ok := stateDir.Get() - if !ok { - return &GlobalBlockPersister{state: utils.NewMutex(&globalBlockState{firstBlock: firstBlock, next: firstBlock})}, nil - } - dir := filepath.Join(sd, globalBlocksDir) - iw, err := openIndexedWAL(dir, loadedGlobalBlockCodec{}) - if err != nil { - return nil, fmt.Errorf("open global block WAL in %s: %w", dir, err) - } - - s := &globalBlockState{iw: utils.Some(iw), firstBlock: firstBlock, next: firstBlock} - // TODO: avoid loading all blocks on startup; cache only the last N blocks - // (e.g. 1000) in memory instead. - loaded, err := s.loadAll() - if err != nil { - _ = iw.Close() - return nil, err - } - if len(loaded) > 0 { - s.firstBlock = loaded[0].Number - s.next = loaded[len(loaded)-1].Number + 1 - } - s.loaded = loaded - return &GlobalBlockPersister{ - state: utils.NewMutex(s), - }, nil -} - -// Next returns the next GlobalBlockNumber expected by the persister. -func (gp *GlobalBlockPersister) Next() types.GlobalBlockNumber { - for s := range gp.state.Lock() { - return s.next - } - panic("unreachable") -} - -// LoadedFirst returns the first loaded block number, or firstBlock if empty. -func (gp *GlobalBlockPersister) LoadedFirst() types.GlobalBlockNumber { - for s := range gp.state.Lock() { - if len(s.loaded) > 0 { - return s.loaded[0].Number - } - return s.firstBlock - } - panic("unreachable") -} - -// ConsumeLoaded returns blocks loaded from the WAL during construction -// and nils the internal slice so the data is not retained. -func (gp *GlobalBlockPersister) ConsumeLoaded() []LoadedGlobalBlock { - for s := range gp.state.Lock() { - loaded := s.loaded - s.loaded = nil - return loaded - } - panic("unreachable") -} - -// PersistBlock appends a block to the WAL. Duplicates are silently ignored. -// Gaps return an error. -func (gp *GlobalBlockPersister) PersistBlock(n types.GlobalBlockNumber, block *types.Block) error { - for s := range gp.state.Lock() { - return s.persistBlock(n, block) - } - panic("unreachable") -} - -// TruncateBefore removes all entries before n from the WAL. -func (gp *GlobalBlockPersister) TruncateBefore(n types.GlobalBlockNumber) error { - for s := range gp.state.Lock() { - return s.truncateBefore(n) - } - panic("unreachable") -} - -// TruncateAfter removes block entries >= n from the WAL, updates the -// persister cursor, and trims loaded data. Called by DataWAL.reconcile -// to remove blocks persisted without corresponding QCs. -func (gp *GlobalBlockPersister) TruncateAfter(n types.GlobalBlockNumber) error { - for s := range gp.state.Lock() { - iw, ok := s.iw.Get() - if ok && iw.Count() > 0 && n < s.next { - firstGlobal := s.next - types.GlobalBlockNumber(iw.Count()) - walIdx := iw.FirstIdx() - if n > firstGlobal { - walIdx += uint64(n - firstGlobal) - } - if err := iw.TruncateAfter(walIdx); err != nil { - return fmt.Errorf("truncate global block WAL after %d: %w", n, err) - } - s.next = firstGlobal + types.GlobalBlockNumber(iw.Count()) - } - trimmed := s.loaded[:0] - for _, lb := range s.loaded { - if lb.Number < n { - trimmed = append(trimmed, lb) - } - } - s.loaded = trimmed - return nil - } - panic("unreachable") -} - -// Close shuts down the WAL. -func (gp *GlobalBlockPersister) Close() error { - for s := range gp.state.Lock() { - iw, ok := s.iw.Get() - if !ok { - return nil - } - s.iw = utils.None[*indexedWAL[LoadedGlobalBlock]]() - return iw.Close() - } - panic("unreachable") -} - -func (s *globalBlockState) loadAll() ([]LoadedGlobalBlock, error) { - iw, ok := s.iw.Get() - if !ok { - return nil, nil - } - return iw.ReadAll() -} diff --git a/sei-tendermint/internal/autobahn/consensus/persist/globalblocks_test.go b/sei-tendermint/internal/autobahn/consensus/persist/globalblocks_test.go deleted file mode 100644 index 7a16453364..0000000000 --- a/sei-tendermint/internal/autobahn/consensus/persist/globalblocks_test.go +++ /dev/null @@ -1,313 +0,0 @@ -package persist - -import ( - "testing" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" -) - -func makeGlobalBlocks(rng utils.Rng, n int) []*types.Block { - blocks := make([]*types.Block, n) - for i := range blocks { - blocks[i] = types.GenBlock(rng) - } - return blocks -} - -func TestNewGlobalBlockPersisterEmptyDir(t *testing.T) { - dir := t.TempDir() - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.NotNil(t, gp) - require.Equal(t, 0, len(gp.ConsumeLoaded())) - require.Equal(t, types.GlobalBlockNumber(0), gp.Next()) - require.NoError(t, gp.Close()) -} - -func TestNewGlobalBlockPersisterNoop(t *testing.T) { - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 5) - - gp, err := NewGlobalBlockPersister(utils.None[string](), 0) - require.NoError(t, err) - require.NotNil(t, gp) - require.Equal(t, 0, len(gp.ConsumeLoaded())) - - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - require.Equal(t, fb+5, gp.Next()) - - // Truncate in no-op mode. Before the fix, truncateBefore wouldn't - // advance s.next, so subsequent PersistBlock calls at higher numbers - // would fail with "out of sequence". - require.NoError(t, gp.TruncateBefore(fb+10)) - require.Equal(t, fb+10, gp.Next()) - - newBlock := types.GenBlock(rng) - require.NoError(t, gp.PersistBlock(fb+10, newBlock)) - require.Equal(t, fb+11, gp.Next()) - require.NoError(t, gp.Close()) -} - -func TestGlobalBlockPersisterFirstBlockFromWAL(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - const wantFirst = types.GlobalBlockNumber(10) - blocks := makeGlobalBlocks(rng, 3) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), wantFirst) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(wantFirst+types.GlobalBlockNumber(i), b)) - } - require.NoError(t, gp.Close()) - - // Reopen with a wrong firstBlock — WAL should take precedence. - gp2, err := NewGlobalBlockPersister(utils.Some(dir), wantFirst+999) - require.NoError(t, err) - require.Equal(t, wantFirst, gp2.LoadedFirst()) - require.NoError(t, gp2.Close()) -} - -func TestGlobalBlockPersistAndReload(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 5) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - require.Equal(t, fb+5, gp.Next()) - require.NoError(t, gp.Close()) - - gp2, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - loaded := gp2.ConsumeLoaded() - require.Equal(t, 5, len(loaded)) - for i, lb := range loaded { - require.Equal(t, fb+types.GlobalBlockNumber(i), lb.Number) - } - require.Equal(t, fb+5, gp2.Next()) - require.NoError(t, gp2.Close()) -} - -func TestGlobalBlockTruncateAndReload(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 10) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - require.NoError(t, gp.TruncateBefore(fb+5)) - require.NoError(t, gp.Close()) - - gp2, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - loaded := gp2.ConsumeLoaded() - require.Equal(t, 5, len(loaded)) - require.Equal(t, fb+5, loaded[0].Number) - require.Equal(t, fb+9, loaded[4].Number) - require.Equal(t, fb+10, gp2.Next()) - require.NoError(t, gp2.Close()) -} - -func TestGlobalBlockTruncateAll(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 5) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - require.NoError(t, gp.TruncateBefore(fb+10)) - require.Equal(t, fb+10, gp.Next()) - require.NoError(t, gp.Close()) - - gp2, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.Equal(t, 0, len(gp2.ConsumeLoaded())) - require.Equal(t, fb, gp2.Next()) - require.NoError(t, gp2.Close()) -} - -func TestGlobalBlockDuplicateIgnored(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - block := types.GenBlock(rng) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.NoError(t, gp.PersistBlock(fb, block)) - require.NoError(t, gp.PersistBlock(fb, block)) - require.Equal(t, fb+1, gp.Next()) - require.NoError(t, gp.Close()) -} - -func TestGlobalBlockGapError(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - block := types.GenBlock(rng) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - err = gp.PersistBlock(fb+2, block) - require.Error(t, err) - require.Contains(t, err.Error(), "out of sequence") - require.NoError(t, gp.Close()) -} - -func TestGlobalBlockTruncateBeforeNoop(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 5) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - require.NoError(t, gp.TruncateBefore(0)) - require.Equal(t, fb+5, gp.Next()) - require.NoError(t, gp.Close()) - - gp2, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.Equal(t, 5, len(gp2.ConsumeLoaded())) - require.NoError(t, gp2.Close()) -} - -func TestGlobalBlockContinueAfterReload(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 10) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i := range 5 { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), blocks[i])) - } - require.NoError(t, gp.Close()) - - gp2, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.Equal(t, 5, len(gp2.ConsumeLoaded())) - for i := 5; i < 10; i++ { - require.NoError(t, gp2.PersistBlock(fb+types.GlobalBlockNumber(i), blocks[i])) - } - require.Equal(t, fb+10, gp2.Next()) - require.NoError(t, gp2.Close()) - - gp3, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.Equal(t, 10, len(gp3.ConsumeLoaded())) - require.NoError(t, gp3.Close()) -} - -func TestGlobalBlockTruncateAfterMiddle(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 5) - - // First session: persist 5 blocks. - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - require.NoError(t, gp.Close()) - - // Second session: reload, then TruncateAfter trims WAL and loaded. - gp2, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.NoError(t, gp2.TruncateAfter(fb+3)) - require.Equal(t, fb+3, gp2.Next()) - loaded := gp2.ConsumeLoaded() - require.Equal(t, 3, len(loaded)) - require.Equal(t, fb, loaded[0].Number) - require.Equal(t, fb+2, loaded[2].Number) - require.NoError(t, gp2.Close()) - - // Third session: verify WAL persistence. - gp3, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - loaded = gp3.ConsumeLoaded() - require.Equal(t, 3, len(loaded)) - require.Equal(t, fb, loaded[0].Number) - require.Equal(t, fb+2, loaded[2].Number) - require.NoError(t, gp3.Close()) -} - -func TestGlobalBlockTruncateAfterNoop(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 3) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - // TruncateAfter at or past the last block is a no-op. - require.NoError(t, gp.TruncateAfter(fb+11)) - require.Equal(t, fb+3, gp.Next()) - require.NoError(t, gp.Close()) -} - -func TestGlobalBlockTruncateAfterBeforeFirst(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRng() - _, _ = epoch.GenRegistry(rng, 3) - fb := types.GlobalBlockNumber(0) - blocks := makeGlobalBlocks(rng, 5) - - gp, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - for i, b := range blocks { - require.NoError(t, gp.PersistBlock(fb+types.GlobalBlockNumber(i), b)) - } - // Truncate first 2 blocks, leaving [fb+2, fb+5). - require.NoError(t, gp.TruncateBefore(fb+2)) - require.Equal(t, fb+5, gp.Next()) - - // TruncateAfter at or before the first remaining block removes everything. - require.NoError(t, gp.TruncateAfter(fb+2)) - require.Equal(t, fb+2, gp.Next()) // cursor stays at firstGlobal - require.NoError(t, gp.Close()) - - // Reload — should be empty. - gp2, err := NewGlobalBlockPersister(utils.Some(dir), 0) - require.NoError(t, err) - require.Equal(t, 0, len(gp2.ConsumeLoaded())) - require.NoError(t, gp2.Close()) -} diff --git a/sei-tendermint/internal/autobahn/consensus/persist/testonly.go b/sei-tendermint/internal/autobahn/consensus/persist/testonly.go deleted file mode 100644 index 31f852897d..0000000000 --- a/sei-tendermint/internal/autobahn/consensus/persist/testonly.go +++ /dev/null @@ -1,8 +0,0 @@ -package persist - -// SetLoadedForTest overrides loaded block data. Test-only. -func (gp *GlobalBlockPersister) SetLoadedForTest(loaded []LoadedGlobalBlock) { - for s := range gp.state.Lock() { - s.loaded = loaded - } -} diff --git a/sei-tendermint/internal/autobahn/consensus/state_test.go b/sei-tendermint/internal/autobahn/consensus/state_test.go index a4ede7d420..0606b4d20d 100644 --- a/sei-tendermint/internal/autobahn/consensus/state_test.go +++ b/sei-tendermint/internal/autobahn/consensus/state_test.go @@ -19,10 +19,7 @@ import ( // keys[0] is used as the node's signing key. func newTestState(rng utils.Rng) (*State, []types.SecretKey, *epoch.Registry) { registry, keys := epoch.GenRegistry(rng, 3) - dataState := utils.OrPanic1(data.NewState( - &data.Config{Registry: registry}, - utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())), - )) + dataState := newTestDataState(registry) s := utils.OrPanic1(NewState(&Config{ Key: keys[0], ViewTimeout: func(types.View) time.Duration { return time.Hour }, @@ -269,12 +266,7 @@ func TestVoteTimeoutPrepareQC_PersistedRestart(t *testing.T) { PersistentStateDir: utils.Some(dir), } } - makeDataState := func() *data.State { - return utils.OrPanic1(data.NewState( - &data.Config{Registry: registry}, - utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())), - )) - } + makeDataState := func() *data.State { return newTestDataState(registry) } view0 := types.View{Index: 0, Number: 0} pqc0 := makePrepareQC(keys, types.GenProposalForEpoch(rng, registry.LatestEpoch(), view0)) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 6311c20c05..09cff73da2 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -7,7 +7,6 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data/metrics" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -16,18 +15,20 @@ import ( const blocksCacheSize = 4000 -// ErrNotFound is returned when the resource is not found. +// ErrNotFound is returned when the resource is not found / not yet available. var ErrNotFound = errors.New("not found") -// ErrPruned is returned when the resource has been is pruned. -var ErrPruned = errors.New("pruned") +// ErrPruned aliases types.ErrPruned (BlockDB below-watermark). +var ErrPruned = types.ErrPruned + +// ErrBlockGap is returned by NewState when BlockDB blocks are not contiguous. +// That indicates store corruption (or an incomplete write that left a hole). +var ErrBlockGap = errors.New("block gap in BlockDB") // Config is the config for the data State. type Config struct { // Registry is the authoritative source of committee and stake information. Registry *epoch.Registry - // PruneAfter is the duration after which the state prunes executed blocks. - PruneAfter utils.Option[time.Duration] } // StateAPI is the interface of the State for consuming global blocks @@ -41,132 +42,32 @@ type StateAPI interface { var _ StateAPI = (*State)(nil) -// DataWAL groups the WALs used by State for crash recovery. -// Both fields are always non-nil; when stateDir is None they are no-op persisters. -// This is temporary and will go away once we switch to a proper storage solution. -type DataWAL struct { - Blocks *persist.GlobalBlockPersister - CommitQCs *persist.FullCommitQCPersister -} - -// Close shuts down both WALs. -func (dw *DataWAL) Close() error { - return errors.Join(dw.Blocks.Close(), dw.CommitQCs.Close()) -} - -// TruncateBefore removes entries fully before n from both WALs in parallel. -// A crash between the two calls just leaves stale entries in one WAL, -// which are harmless on reload. -func (dw *DataWAL) TruncateBefore(n types.GlobalBlockNumber) error { - return scope.Parallel(func(ps scope.ParallelScope) error { - ps.Spawn(func() error { - if err := dw.Blocks.TruncateBefore(n); err != nil { - return fmt.Errorf("truncate global block WAL: %w", err) - } - return nil - }) - ps.Spawn(func() error { - if err := dw.CommitQCs.TruncateBefore(n); err != nil { - return fmt.Errorf("truncate full commitqc WAL: %w", err) - } - return nil - }) - return nil - }) -} - -// reconcile fixes cursor inconsistencies between the two WALs that can -// result from crashes during parallel persistence or pruning. -// -// The possible WAL states on startup and how they are handled: -// -// Case Blocks QCs Scenario Action -// 1 empty empty Fresh start no-op -// 2 [a,b] empty QCs lost (corruption) error (statesync needed) -// 3 empty [X,Y) Blocks lost (crash) Prefix: fast-forward blocks.next to X -// 4 [a,b] [X,Y) Normal (a=X, bX) Prefix: truncate QCs to a -// 6 [a,b] [X,Y) Prune crash: QCs ahead (a=Y) Tail: truncate blocks to Y -// 8 [a,b] [X,Y) QCs ahead (normal, b fb { - // Blocks exist but QCs WAL is empty — data is corrupted. - return fmt.Errorf("corrupted WAL: blocks exist but QCs WAL is empty; statesync required to recover") - } - if err := dw.Blocks.TruncateAfter(qcNext); err != nil { - return fmt.Errorf("truncate blocks tail: %w", err) - } - // Fix prefix: align both WALs to the later start. - blocksFirst := dw.Blocks.LoadedFirst() - qcsFirst := dw.CommitQCs.LoadedFirst() - reconciled := max(blocksFirst, qcsFirst) - if reconciled > fb { - if err := dw.TruncateBefore(reconciled); err != nil { - return err - } - } - return nil -} - -// NewDataWAL constructs both global-block and global-commitqc WALs. -// When stateDir is None, the returned persisters are no-ops. -func NewDataWAL(stateDir utils.Option[string], firstBlock types.GlobalBlockNumber) (*DataWAL, error) { - blocks, err := persist.NewGlobalBlockPersister(stateDir, firstBlock) - if err != nil { - return nil, fmt.Errorf("global block WAL: %w", err) - } - commitQCs, err := persist.NewFullCommitQCPersister(stateDir, firstBlock) - if err != nil { - _ = blocks.Close() - return nil, fmt.Errorf("full commitqc WAL: %w", err) - } - dw := &DataWAL{ - Blocks: blocks, - CommitQCs: commitQCs, - } - // Reconcile cursor inconsistency: a crash between the two parallel - // TruncateBefore calls can leave one WAL truncated while the other - // still has stale entries. Advance both to the max starting point. - if err := dw.reconcile(firstBlock); err != nil { - _ = dw.Close() - return nil, fmt.Errorf("reconcile WALs: %w", err) - } - return dw, nil -} - -type appProposalWithTimestamp struct { - proposal *types.AppProposal - timestamp time.Time +// blockEntry is a (number, block) pair collected in runPersist batches. +type blockEntry struct { + n types.GlobalBlockNumber + block *types.Block } type inner struct { - qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first,nextQC) - blocks map[types.GlobalBlockNumber]*types.Block // [first,nextBlock) + subset of [nextBlock,nextQC) - // appProposal[n] contains appProposal block >=n. - appProposals map[types.GlobalBlockNumber]appProposalWithTimestamp // [first,nextAppProposal) - - // blockHashes is a hash → height index mirroring blocks. Maintained - // in lockstep with blocks via insertBlock / pruneFirst, so it covers - // exactly the same retain window without a separate prune cursor or - // startup warmup. Powers BlockByHash. - // - // TODO(autobahn): remove once a writer is wired into block execution - // that populates sei-db/ledger_db/block.BlockDB. BlockDB has a built-in - // hash index that survives process restart and lives outside Autobahn's - // RetainHeight pruning, making this in-memory index obsolete. + // qcs/blocks may omit heights below nextAppProposal after eviction + // (durable copies live in BlockDB). Keys are not a dense prefix. + qcs map[types.GlobalBlockNumber]*types.FullCommitQC + blocks map[types.GlobalBlockNumber]*types.Block + // appProposals[n] contains the AppProposal for block n. Entries are + // removed by evictExecuted once a later CommitQC embeds an App that + // certifies them (see evictionBound). + appProposals map[types.GlobalBlockNumber]*types.AppProposal + + // blockHashes is a hash → height index for GlobalBlockByHash. Maintained + // in lockstep with blocks via insertBlock / evictExecuted. Misses fall + // through to blockDB.ReadBlockByHash. blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber - // first <= nextAppProposal <= nextBlockToPersist <= nextBlock <= nextQC + // nextAppProposal <= nextBlockToPersist <= nextBlock <= nextQC // - // This invariant guarantees no race between pruning and persisting: - // blocks are not eligible for pruning until they have an AppProposal - // (first <= nextAppProposal), which requires persistence - // (nextAppProposal <= nextBlockToPersist). - first types.GlobalBlockNumber + // AppProposals require persistence (nextAppProposal <= nextBlockToPersist). + // Executed heights are dropped from memory by evictExecuted; BlockDB prune + // status lives only in the store watermark (see PruneBefore). nextAppProposal types.GlobalBlockNumber nextBlockToPersist types.GlobalBlockNumber nextBlock types.GlobalBlockNumber @@ -174,17 +75,15 @@ type inner struct { } func newInner(firstBlock types.GlobalBlockNumber) *inner { - first := firstBlock return &inner{ qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, blocks: map[types.GlobalBlockNumber]*types.Block{}, - appProposals: map[types.GlobalBlockNumber]appProposalWithTimestamp{}, + appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, - first: first, - nextAppProposal: first, - nextBlockToPersist: first, - nextBlock: first, - nextQC: first, + nextAppProposal: firstBlock, + nextBlockToPersist: firstBlock, + nextBlock: firstBlock, + nextQC: firstBlock, } } @@ -192,7 +91,6 @@ func newInner(firstBlock types.GlobalBlockNumber) *inner { // Used on recovery when the first loaded QC starts past committee.FirstBlock() // (i.e. data before n was pruned in a previous run). func (i *inner) skipTo(n types.GlobalBlockNumber) { - i.first = n i.nextAppProposal = n i.nextBlockToPersist = n i.nextBlock = n @@ -233,12 +131,15 @@ func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error // allows batch insertion (e.g. PushQC inserts multiple blocks, then // advances nextBlock once). func (i *inner) insertBlock(committee *types.Committee, n types.GlobalBlockNumber, block *types.Block) error { - if n < i.first || n >= i.nextQC { - return nil // outside QC range + // Contiguous prefix is done or evicted; only [nextBlock, nextQC) inserts. + if n < i.nextBlock || n >= i.nextQC { + return nil } if _, ok := i.blocks[n]; ok { - return nil // already have it + return nil // already have it (gap fill) } + // n is in [nextBlock, nextQC); QCs are contiguous and eviction stops below + // nextAppProposal <= nextBlock, so qcs[n] is always present. qc := i.qcs[n] storedGR := qc.QC().GlobalRange() want := qc.Headers()[n-storedGR.First].Hash() @@ -265,90 +166,137 @@ func (i *inner) updateNextBlock(m *metrics.Metrics) { } } -func (i *inner) pruneFirst(now time.Time, m *metrics.Metrics) { - b := i.blocks[i.first] - latency := now.Sub(b.Payload().CreatedAt()).Seconds() - m.Blocks.Prune.Observe(latency) - m.Txs.Prune.ObserveWithWeight(latency, uint64(len(b.Payload().Txs()))) - delete(i.appProposals, i.first) - delete(i.blocks, i.first) - delete(i.qcs, i.first) - delete(i.blockHashes, b.Header().Hash()) - i.first += 1 -} - // State of the chain. // Contains blocks in global order and proofs of their finality. type State struct { cfg *Config metrics *metrics.Metrics inner utils.Watch[*inner] - dataWAL *DataWAL -} - -// NewState constructs a new data State. -// dataWAL persists blocks and QCs to WALs for crash recovery and provides -// preloaded data from the previous run. Use NewDataWAL to construct it. -func NewState(cfg *Config, dataWAL *DataWAL) (*State, error) { - inner := newInner(cfg.Registry.FirstBlock()) - // Fast-forward cursors to where data starts. Use blocks as golden: - // per-block pruning may split a QC range, so blocks determine where - // useful data starts. QCs before that are kept for verification but - // don't set inner.first. - blocksFirst := dataWAL.Blocks.LoadedFirst() - qcFirst := dataWAL.CommitQCs.LoadedFirst() - dataFirst := max(blocksFirst, qcFirst) - if dataFirst > cfg.Registry.FirstBlock() { - inner.skipTo(dataFirst) - } - // Restore QCs. insertQC handles partially pruned QCs (range starts - // before inner.first) by skipping the pruned prefix. - for _, qc := range dataWAL.CommitQCs.ConsumeLoaded() { - if err := inner.insertQC(cfg.Registry, qc); err != nil { - return nil, fmt.Errorf("load QC from WAL: %w", err) - } - } - // Restore blocks. Verify contiguity as defense in depth. - expectedBlock := inner.first - for _, lb := range dataWAL.Blocks.ConsumeLoaded() { - if lb.Number < inner.first || lb.Number >= inner.nextQC { - continue // outside QC range (stale from reconcile) - } - if lb.Number != expectedBlock { - return nil, fmt.Errorf("block gap in WAL: expected %d, got %d", expectedBlock, lb.Number) - } - expectedBlock = lb.Number + 1 - qc := inner.qcs[lb.Number] - e, ok := cfg.Registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) - if !ok { - return nil, fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) + blockDB types.BlockDB +} + +// NewState constructs a data State, replaying persisted state from blockDB. +// Use memblock.NewBlockDB() for an in-memory store (testing / no persistent dir). +// The caller owns blockDB and must close it after State.Run returns (nodeImpl +// owns this in production); State never closes it. +// TODO(gprusak): add support for starting execution from non-zero commit QC. +func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { + s := &State{ + cfg: cfg, + metrics: metrics.Get(), + inner: utils.NewWatch(newInner(cfg.Registry.FirstBlock())), + blockDB: blockDB, + } + if err := s.loadFromBlockDB(blockDB); err != nil { + return nil, fmt.Errorf("loadFromBlockDB: %w", err) + } + return s, nil +} + +// loadFromBlockDB replays QCs and blocks from blockDB into s.inner. +// Called from NewState before any goroutines are spawned; the lock is acquired +// only to satisfy the Watch API. +// +// The recovery floor is derived from BlockDB: empty store keeps +// registry.FirstBlock(); otherwise cursors skipTo the first retained QC start. +// Inconsistencies (gaps, block without QC, first-block/QC mismatch, etc.) are +// returned as errors rather than normalized — BlockDB is expected to present a +// consistent iterator view (see littblock watermark / stranding rules). +func (s *State) loadFromBlockDB(blockDB types.BlockDB) error { + for in := range s.inner.Lock() { + // Restore QCs from BlockDB. On the first QC, skipTo its GlobalRange.First + // to advance past any pruned prefix. Subsequent QCs must be consecutive — + // insertQC errors on any gap. + err := func() error { + it, err := blockDB.QCs(false) + if err != nil { + return fmt.Errorf("open QC iterator: %w", err) + } + defer func() { _ = it.Close() }() + for { + ok, err := it.Next() + if err != nil || !ok { + return err + } + qc, err := it.QC() + if err != nil { + return err + } + if len(in.qcs) == 0 { + gr := qc.QC().GlobalRange() + if gr.First < in.nextQC { + return fmt.Errorf("QC in BlockDB predates committee genesis %d: got %d", in.nextQC, gr.First) + } + if gr.First > in.nextQC { + in.skipTo(gr.First) + } + } + if err := in.insertQC(s.cfg.Registry, qc); err != nil { + return fmt.Errorf("load QC from BlockDB: %w", err) + } + } + }() + if err != nil { + return err } - committee := e.Committee() - if err := lb.Block.Verify(committee); err != nil { - return nil, fmt.Errorf("load block %d from WAL: %w", lb.Number, err) + + // Restore blocks from BlockDB. First block must align with first QC start + // (set by the QC pass); a mismatch is corruption / incomplete store. + // After the QC pass with no AppProposals, nextAppProposal is the recovery + // floor (registry.FirstBlock() or first retained QC start). + err = func() error { + it, err := blockDB.Blocks(false) + if err != nil { + return fmt.Errorf("open block iterator: %w", err) + } + defer func() { _ = it.Close() }() + nextExpect := in.nextAppProposal + for { + ok, err := it.Next() + if err != nil || !ok { + return err + } + n := it.Number() + if n >= in.nextQC { + return fmt.Errorf("block %d in BlockDB has no QC coverage (nextQC=%d)", n, in.nextQC) + } + // updateNextBlock only advances nextBlock through contiguous present + // entries, so runPersist always writes [persistedBlock, nextBlock) + // fully populated. A gap here means BlockDB corruption. + if n != nextExpect { + return fmt.Errorf("%w: expected %d, got %d", ErrBlockGap, nextExpect, n) + } + nextExpect++ + blk, err := it.Block() + if err != nil { + return err + } + qc := in.qcs[n] + e, ok := s.cfg.Registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) + if !ok { + return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) + } + committee := e.Committee() + if err := blk.Verify(committee); err != nil { + return fmt.Errorf("verify block %d from BlockDB: %w", n, err) + } + if err := in.insertBlock(committee, n, blk); err != nil { + return fmt.Errorf("insert block %d from BlockDB: %w", n, err) + } + } + }() + if err != nil { + return err } - if err := inner.insertBlock(committee, lb.Number, lb.Block); err != nil { - return nil, fmt.Errorf("load block %d from WAL: %w", lb.Number, err) + + // Advance nextBlock through contiguous loaded blocks. Don't use + // updateNextBlock: stale timestamps would skew metrics. + for ; in.blocks[in.nextBlock] != nil; in.nextBlock++ { } + // Data loaded from BlockDB was already durably persisted. + in.nextBlockToPersist = in.nextBlock } - // Advance nextBlock through contiguous blocks. Don't use - // updateNextBlock: stale timestamps would skew metrics. - for ; inner.blocks[inner.nextBlock] != nil; inner.nextBlock++ { - } - // Data loaded from WALs was already persisted in the previous run. - inner.nextBlockToPersist = inner.nextBlock - // WAL cursor consistency was resolved by DataWAL.reconcile at construction. - // Verify the blocks persister cursor is not behind inner.nextBlock. - if dataWAL.Blocks.Next() < inner.nextBlock { - return nil, fmt.Errorf("blocks WAL cursor %d behind inner.nextBlock %d after reconciliation", - dataWAL.Blocks.Next(), inner.nextBlock) - } - return &State{ - cfg: cfg, - metrics: metrics.Get(), - inner: utils.NewWatch(inner), - dataWAL: dataWAL, - }, nil + return nil } // Registry returns the epoch registry. @@ -433,27 +381,34 @@ func (s *State) QC(ctx context.Context, n types.GlobalBlockNumber) (*types.FullC }); err != nil { return nil, err } - if n < inner.first { - return nil, ErrPruned + if qc, ok := inner.qcs[n]; ok { + return qc, nil } - return inner.qcs[n], nil + // Evicted from memory after persist; fall through to BlockDB. } - panic("unreachable") + return s.qcFromDB(n) } // PushBlock pushes block to the state. -// The QC for n must already be present (guaranteed by PushQC ordering). +// The QC for n must already be present (guaranteed by PushQC ordering), unless +// the height was already executed (n < nextAppProposal) — in that case the +// block is dropped silently. func (s *State) PushBlock(ctx context.Context, n types.GlobalBlockNumber, block *types.Block) error { var epochIdx types.EpochIndex for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextQC }); err != nil { return err } - if n < inner.first { - // Block arrived after pruning; drop silently so the sender keeps delivering future blocks. + // nextAppProposal is the in-memory executed floor: heights below it are + // not needed in RAM (durable in BlockDB / already voted). + if n < inner.nextAppProposal { + return nil + } + qc := inner.qcs[n] + if qc == nil { return nil } - epochIdx = inner.qcs[n].QC().Proposal().EpochIndex() + epochIdx = qc.QC().Proposal().EpochIndex() } ep, ok := s.cfg.Registry.EpochByIndex(epochIdx) if !ok { @@ -464,9 +419,6 @@ func (s *State) PushBlock(ctx context.Context, n types.GlobalBlockNumber, block return fmt.Errorf("block.Verify(): %w", err) } for inner, ctrl := range s.inner.Lock() { - if n < inner.first { - return nil - } if err := inner.insertBlock(ep.Committee(), n, block); err != nil { return err } @@ -485,33 +437,26 @@ func (s *State) NextBlock() types.GlobalBlockNumber { } // GlobalBlockByHash returns the finalized GlobalBlock whose stored header -// hashes to the given value, or None if no such block is currently in the -// retained range. The lookup-and-construct happens under a single lock so -// the returned block matches the looked-up hash atomically — pruning can't -// change which height a hash maps to between the index check and the block -// construction. Tracks the same retain window as Block / GlobalBlock since -// the hash index is maintained in lockstep by insertBlock / pruneFirst. -// -// Returns an error in the signature for forward-compat with the eventual -// switch to sei-db/ledger_db/block.BlockDB.GetBlockByHash. Today's -// in-memory implementation never errors. -// -// TODO(autobahn): when BlockDB is wired, take a ctx parameter and narrow -// the error contract — db-internal errors should surface by shutting down -// the persistence background task (matching how persistence handles errors -// today), so the query path's error stays bounded to context.Canceled. +// hashes to the given value, or None if no such block is currently retained. +// Non-blocking. Falls back to BlockDB when the entry was evicted from memory +// after persist. func (s *State) GlobalBlockByHash(hash types.BlockHeaderHash) (utils.Option[*types.GlobalBlock], error) { for inner := range s.inner.Lock() { n, ok := inner.blockHashes[hash] - if !ok { - return utils.None[*types.GlobalBlock](), nil + if !ok || n < inner.nextAppProposal { + break + } + // Live window: block implies covering QC (same insert/evict lifetime). + if b, ok := inner.blocks[n]; ok { + return utils.Some(assembleGlobalBlock(n, b, inner.qcs[n])), nil } - return utils.Some(inner.globalBlockAt(n)), nil } - panic("unreachable") + return s.globalBlockByHashFromDB(hash) } // Block returns the block with the given global number. +// Waits until the contiguous prefix reaches n (n < nextBlock), then returns +// it from memory or BlockDB. Does not expose gaps ahead of nextBlock. // This function is used for syncing - GlobalBlock can be derived from Block and FullCommitQC, // which have to be fetched upfront anyway. func (s *State) Block(ctx context.Context, n types.GlobalBlockNumber) (*types.Block, error) { @@ -521,37 +466,35 @@ func (s *State) Block(ctx context.Context, n types.GlobalBlockNumber) (*types.Bl }); err != nil { return nil, err } - if n < inner.first { - return nil, ErrPruned + if b, ok := inner.blocks[n]; ok { + return b, nil } - return inner.blocks[n], nil + // Evicted from the contiguous prefix; fall through to BlockDB. } - panic("unreachable") + return s.blockFromDB(n) } // TryBlock returns the block with the given global number. -// Returns ErrPruned if the block has already been pruned. -// Returns ErrNotFound if the block is not available yet. +// Returns ErrNotFound if n is not yet in the contiguous prefix (n >= nextBlock), +// including gap-fills stored above nextBlock — same no-gap contract as Block. +// Returns ErrPruned if BlockDB no longer has an evicted height. +// Evicted-but-still-durable heights (n < nextBlock) load from BlockDB. func (s *State) TryBlock(n types.GlobalBlockNumber) (*types.Block, error) { for inner := range s.inner.Lock() { - if n < inner.first { - return nil, ErrPruned - } - b, ok := inner.blocks[n] - if !ok { + if n >= inner.nextBlock { return nil, ErrNotFound } - return b, nil + if b, ok := inner.blocks[n]; ok { + return b, nil + } + // Evicted from the contiguous prefix; fall through to BlockDB. } - panic("unreachable") + return s.blockFromDB(n) } -// globalBlockAt assembles the GlobalBlock at height n from inner state. -// Caller must have verified n is in [inner.first, inner.nextBlock); n -// outside that range nil-derefs on inner.blocks[n] / inner.qcs[n]. -func (i *inner) globalBlockAt(n types.GlobalBlockNumber) *types.GlobalBlock { - b := i.blocks[n] - qc := i.qcs[n].QC() +// assembleGlobalBlock builds a GlobalBlock from a block and its covering QC. +func assembleGlobalBlock(n types.GlobalBlockNumber, b *types.Block, fqc *types.FullCommitQC) *types.GlobalBlock { + qc := fqc.QC() return &types.GlobalBlock{ GlobalNumber: n, Timestamp: qc.Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), @@ -562,7 +505,9 @@ func (i *inner) globalBlockAt(n types.GlobalBlockNumber) *types.GlobalBlock { } // GlobalBlock returns the block with the given global number. -// Returns ErrPruned if the block has already been pruned. +// Waits until the contiguous prefix reaches n (same no-gap contract as Block). +// Returns ErrPruned if the block has already been pruned from BlockDB. +// Falls back to BlockDB when the entry was evicted from memory after persist. func (s *State) GlobalBlock(ctx context.Context, n types.GlobalBlockNumber) (*types.GlobalBlock, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { @@ -570,12 +515,69 @@ func (s *State) GlobalBlock(ctx context.Context, n types.GlobalBlockNumber) (*ty }); err != nil { return nil, err } - if n < inner.first { - return nil, ErrPruned + if b, ok := inner.blocks[n]; ok { + return assembleGlobalBlock(n, b, inner.qcs[n]), nil } - return inner.globalBlockAt(n), nil + // Evicted from the contiguous prefix; fall through to BlockDB. } - panic("unreachable") + return s.globalBlockFromDB(n) +} + +func (s *State) blockFromDB(n types.GlobalBlockNumber) (*types.Block, error) { + opt, err := s.blockDB.ReadBlockByNumber(n) + if err != nil { + return nil, fmt.Errorf("blockDB.ReadBlockByNumber(%d): %w", n, err) + } + b, ok := opt.Get() + if !ok { + // Caller only falls through for heights below nextBlock (already seen). + // None here means the store no longer has them (pruned/reclaimed). + return nil, ErrPruned + } + return b, nil +} + +func (s *State) qcFromDB(n types.GlobalBlockNumber) (*types.FullCommitQC, error) { + opt, err := s.blockDB.ReadQCByBlockNumber(n) + if err != nil { + return nil, fmt.Errorf("blockDB.ReadQCByBlockNumber(%d): %w", n, err) + } + qc, ok := opt.Get() + if !ok { + return nil, ErrPruned + } + return qc, nil +} + +func (s *State) globalBlockFromDB(n types.GlobalBlockNumber) (*types.GlobalBlock, error) { + b, err := s.blockFromDB(n) + if err != nil { + return nil, err + } + qc, err := s.qcFromDB(n) + if err != nil { + return nil, err + } + return assembleGlobalBlock(n, b, qc), nil +} + +func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Option[*types.GlobalBlock], error) { + opt, err := s.blockDB.ReadBlockByHash(hash) + if err != nil { + return utils.None[*types.GlobalBlock](), fmt.Errorf("blockDB.ReadBlockByHash: %w", err) + } + bn, ok := opt.Get() + if !ok { + return utils.None[*types.GlobalBlock](), nil + } + qc, err := s.qcFromDB(bn.Number) + if err != nil { + if errors.Is(err, ErrPruned) || errors.Is(err, ErrNotFound) { + return utils.None[*types.GlobalBlock](), nil + } + return utils.None[*types.GlobalBlock](), err + } + return utils.Some(assembleGlobalBlock(bn.Number, bn.Block, qc)), nil } // PushAppHash marks blocks up to n as executed. Hash is the execution result. @@ -597,10 +599,6 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash inner.qcs[n].QC().Proposal().EpochIndex(), ) t := time.Now() - apt := appProposalWithTimestamp{ - proposal: proposal, - timestamp: t, - } // TODO(gprusak): this will be problematic on restart, // nextAppProposal should be initiated wrt current application height, // so that we don't iterate over all blocks in storage on startup. @@ -609,7 +607,7 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash latency := t.Sub(b.Payload().CreatedAt()).Seconds() s.metrics.Blocks.Execute.Observe(latency) s.metrics.Txs.Execute.ObserveWithWeight(latency, uint64(len(b.Payload().Txs()))) - inner.appProposals[inner.nextAppProposal] = apt + inner.appProposals[inner.nextAppProposal] = proposal inner.nextAppProposal += 1 } ctrl.Updated() @@ -625,17 +623,18 @@ func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*ty if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppProposal }); err != nil { return nil, err } - if n < inner.first { + ap, ok := inner.appProposals[n] + if !ok { return nil, ErrPruned } - return inner.appProposals[n].proposal, nil + return ap, nil } panic("unreachable") } func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { // TODO(gprusak): decide whether 0 is a good result in this case in general. - if i.first == i.nextAppProposal { + if len(i.appProposals) == 0 { return 0 } n := i.nextAppProposal - 1 @@ -670,168 +669,197 @@ func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n type panic("unreachable") } -// PruneBefore removes blocks, QCs, and AppProposals before retainFrom. -// Blocks at retainFrom and above are kept. Per-block pruning may split -// a QC range; this is handled on recovery (NewState skips partial QC prefixes). +// PruneBefore asks BlockDB to drop data before retainFrom. Only executed +// heights are eligible (capped to nextAppProposal). Memory is not cleared +// here — that is evictExecuted's job after persist/execution. BlockDB enforces +// its own never-empty retention and refuses reads below its watermark. func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { - pruningTime := time.Now() - truncateTo := utils.None[types.GlobalBlockNumber]() - for inner, ctrl := range s.inner.Lock() { - // Can only prune executed blocks (those with AppProposals). - firstToKeep := min(retainFrom, inner.nextAppProposal) - if firstToKeep <= inner.first { - return nil - } - // Keep at least one entry so WALs are never empty on restart. - for inner.first+1 < firstToKeep { - inner.pruneFirst(pruningTime, s.metrics) - } - ctrl.Updated() - truncateTo = utils.Some(inner.first) - } - // Truncate WALs outside the lock to avoid holding it during disk I/O. - if n, ok := truncateTo.Get(); ok { - return s.dataWAL.TruncateBefore(n) + for inner := range s.inner.Lock() { + n := min(retainFrom, inner.nextAppProposal) + return s.blockDB.PruneBefore(n) } return nil } -// runPersist is a background goroutine that persists blocks and QCs to WALs. +// runPersist is a background goroutine that persists blocks and QCs to BlockDB. // It waits for in-memory data to advance past the persistence cursors, then -// writes QCs and blocks in parallel. QCs are persisted up to nextQC (eagerly), -// blocks up to nextBlock. nextBlockToPersist advances to min(persistedQC, persistedBlock) +// writes QCs (first, per the BlockDB contract) and blocks, then flushes once +// per batch. nextBlockToPersist advances to min(nextToPersistQC, nextToPersistBlock) // to unblock PushAppHash only when both are durable. // Errors propagate vertically (kill the component). +// +// Persistence cursors (nextToPersistQC / nextToPersistBlock) seed from +// BlockDB.Status() when present so PushQC-before-Run heights are not skipped. +// When a Status tip is absent, seed from post-load nextBlockToPersist (recovery +// floor), never bare registry.FirstBlock() — a QC-only store can skipTo past +// genesis while LastBlockNumber is still missing. func (s *State) runPersist(ctx context.Context) error { - // Initialize from nextBlockToPersist, not nextQC/nextBlock. PushQC may - // have been called before Run() (race between p2p startup and Run), - // advancing nextQC/nextBlock beyond what's been persisted. Starting - // from nextBlockToPersist ensures we don't skip unpersisted data. - var persistedQC, persistedBlock types.GlobalBlockNumber + tips := s.blockDB.Status() + var nextToPersistQC, nextToPersistBlock, evicted types.GlobalBlockNumber for inner := range s.inner.Lock() { - persistedQC = inner.nextBlockToPersist - persistedBlock = inner.nextBlockToPersist + // After loadFromBlockDB, nextBlockToPersist is the durable recovery tip. + nextToPersistQC = inner.nextBlockToPersist + nextToPersistBlock = inner.nextBlockToPersist + evicted = inner.nextAppProposal + } + if n, ok := tips.LastQCNext.Get(); ok { + nextToPersistQC = n + } + if n, ok := tips.LastBlockNumber.Get(); ok { + nextToPersistBlock = n + 1 } for { - // Wait for unpersisted data and snapshot what needs writing. + // Wait for unpersisted data and/or executable heights to evict, then + // snapshot what needs writing. type batch struct { qcs []*types.FullCommitQC - blocks []persist.LoadedGlobalBlock + blocks []blockEntry qcEnd types.GlobalBlockNumber blockEnd types.GlobalBlockNumber } var b batch + var doWrite bool for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return persistedQC < inner.nextQC || persistedBlock < inner.nextBlock + return persistOrEvictReady(inner, nextToPersistQC, nextToPersistBlock, evicted) }); err != nil { return err } - b.qcEnd = inner.nextQC - b.blockEnd = inner.nextBlock - // Collect deduplicated QCs for [persistedQC, nextQC). - seen := map[types.GlobalBlockNumber]bool{} - for n := persistedQC; n < inner.nextQC; n++ { - qc := inner.qcs[n] - first := qc.QC().GlobalRange().First - if !seen[first] { - seen[first] = true - b.qcs = append(b.qcs, qc) + doWrite = nextToPersistQC < inner.nextQC || nextToPersistBlock < inner.nextBlock + if doWrite { + b.qcEnd = inner.nextQC + b.blockEnd = inner.nextBlock + // Collect deduplicated QCs for [nextToPersistQC, nextQC). + seen := map[types.GlobalBlockNumber]bool{} + for n := nextToPersistQC; n < inner.nextQC; n++ { + qc := inner.qcs[n] + qcFirst := qc.QC().GlobalRange().First + if !seen[qcFirst] { + seen[qcFirst] = true + b.qcs = append(b.qcs, qc) + } + } + // Collect blocks for [nextToPersistBlock, nextBlock). + for n := nextToPersistBlock; n < inner.nextBlock; n++ { + b.blocks = append(b.blocks, blockEntry{n: n, block: inner.blocks[n]}) } - } - // Collect blocks for [persistedBlock, nextBlock). - for n := persistedBlock; n < inner.nextBlock; n++ { - b.blocks = append(b.blocks, persist.LoadedGlobalBlock{ - Number: n, - Block: inner.blocks[n], - }) } } - // Persist QCs and blocks in parallel. - if err := scope.Parallel(func(ps scope.ParallelScope) error { - ps.Spawn(func() error { - for _, qc := range b.qcs { - if err := s.dataWAL.CommitQCs.PersistQC(qc); err != nil { - return fmt.Errorf("persist full commitqc: %w", err) - } + if doWrite { + // Write QCs first (BlockDB contract: QC must precede covered blocks). + for _, qc := range b.qcs { + gr := qc.QC().GlobalRange() + if err := s.blockDB.WriteQC(gr.First, gr.Next, qc); err != nil { + return fmt.Errorf("write QC [%d,%d): %w", gr.First, gr.Next, err) } - return nil - }) - ps.Spawn(func() error { - for _, lb := range b.blocks { - if err := s.dataWAL.Blocks.PersistBlock(lb.Number, lb.Block); err != nil { - return fmt.Errorf("persist global block %d: %w", lb.Number, err) - } + } + for _, lb := range b.blocks { + if err := s.blockDB.WriteBlock(lb.n, lb.block); err != nil { + return fmt.Errorf("write block %d: %w", lb.n, err) } - return nil - }) - return nil - }); err != nil { - return err - } - persistedQC = b.qcEnd - persistedBlock = b.blockEnd - // Advance nextBlockToPersist to where both QCs and blocks are durable. - newToPersist := min(persistedQC, persistedBlock) - for inner, ctrl := range s.inner.Lock() { - if newToPersist > inner.nextBlockToPersist { - inner.nextBlockToPersist = newToPersist - ctrl.Updated() + } + // Flush once per batch before advancing nextBlockToPersist, so that + // PushAppHash only unblocks after data is crash-durable. + if err := s.blockDB.Flush(); err != nil { + return fmt.Errorf("flush BlockDB: %w", err) + } + nextToPersistQC = b.qcEnd + nextToPersistBlock = b.blockEnd + newToPersist := min(nextToPersistQC, nextToPersistBlock) + for inner, ctrl := range s.inner.Lock() { + if newToPersist > inner.nextBlockToPersist { + inner.nextBlockToPersist = newToPersist + ctrl.Updated() + } + evicted = evictExecuted(inner, evicted) + } + } else { + // Restart / catch-up: nothing new to write, but PushAppHash advanced + // nextAppProposal over already-durable recovered heights. + for inner := range s.inner.Lock() { + evicted = evictExecuted(inner, evicted) } } } } -func (s *State) runPruning(ctx context.Context, after time.Duration) error { - pruningTime := time.Now() - for { - truncateTo := utils.None[types.GlobalBlockNumber]() - for inner, ctrl := range s.inner.Lock() { - // Prune blocks old enough. Keep at least one entry. - // Per-block pruning may split QC ranges; handled on recovery. - // TODO: a proper fix would not prune until AppQC exists. - pruned := false - for inner.first+1 < inner.nextAppProposal && pruningTime.Sub(inner.appProposals[inner.first].timestamp) >= after { - inner.pruneFirst(pruningTime, s.metrics) - pruned = true - } - if pruned { - ctrl.Updated() - truncateTo = utils.Some(inner.first) - } - // Wait for at least 2 entries before retrying. Without +1, - // the loop would spin when only one entry remains (kept by - // the +1 guard above). - if err := ctrl.WaitUntil(ctx, func() bool { return inner.first+1 < inner.nextAppProposal }); err != nil { - return err +// persistOrEvictReady reports whether runPersist has either unpersisted data or +// durable executed heights that can be dropped from the in-memory maps. +func persistOrEvictReady( + inner *inner, + nextToPersistQC, nextToPersistBlock, evicted types.GlobalBlockNumber, +) bool { + if nextToPersistQC < inner.nextQC || nextToPersistBlock < inner.nextBlock { + return true + } + return inner.evictionBound() > evicted +} + +// certifiedAppFloor is the GlobalNumber of the AppProposal embedded in the +// latest cached CommitQC, if any. That App is already covered by an AppQC, so +// heights below it no longer need to be served for AppVotes. +func (i *inner) certifiedAppFloor() utils.Option[types.GlobalBlockNumber] { + if len(i.qcs) == 0 { + return utils.None[types.GlobalBlockNumber]() + } + for n := i.nextQC - 1; ; n-- { + if qc, ok := i.qcs[n]; ok { + if app, ok := qc.QC().Proposal().App().Get(); ok { + return utils.Some(app.GlobalNumber()) } - pruningTime = inner.appProposals[inner.first].timestamp.Add(after) + return utils.None[types.GlobalBlockNumber]() } - // Truncate WALs outside the lock to avoid holding it during disk I/O. - if n, ok := truncateTo.Get(); ok { - if err := s.dataWAL.TruncateBefore(n); err != nil { - return err - } + if n == 0 { + return utils.None[types.GlobalBlockNumber]() } - // Wait until the next pruning time. - if err := utils.SleepUntil(ctx, pruningTime); err != nil { - return err + } +} + +// evictionBound is the exclusive end of the range that may be dropped from +// memory. Heights in [evictionBound, nextAppProposal) stay cached. +// +// Without a CommitQC-embedded App, nothing is evicted: AppVotes still need +// AppProposals until an AppQC is certified. With an App at GlobalNumber A, +// the retain floor is min(nextAppProposal, A) (pompon0): keep [A, ...) for +// voting, and never drop nextAppProposal-1 (nextToExecute). +func (i *inner) evictionBound() types.GlobalBlockNumber { + floor, ok := i.certifiedAppFloor().Get() + if !ok { + return 0 // no certified App yet — do not evict + } + bound := min(i.nextAppProposal, floor) + if i.nextAppProposal > 0 { + if sentinel := i.nextAppProposal - 1; bound > sentinel { + bound = sentinel } } + return bound } -// Run runs the background tasks of the data State. -// TODO(gprusak): add support for starting execution from non-zero commit QC. +// evictExecuted drops cached blocks/QCs/AppProposals in [evicted, evictionBound). +// Caller must hold inner's lock. +func evictExecuted(inner *inner, evicted types.GlobalBlockNumber) types.GlobalBlockNumber { + evictBelow := inner.evictionBound() + if evictBelow <= evicted { + return evicted + } + for ; evicted < evictBelow; evicted++ { + if b, ok := inner.blocks[evicted]; ok { + delete(inner.blockHashes, b.Header().Hash()) + delete(inner.blocks, evicted) + } + delete(inner.qcs, evicted) + delete(inner.appProposals, evicted) + } + return evicted +} + +// Run starts the background persistence goroutine. func (s *State) Run(ctx context.Context) error { return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { scope.SpawnNamed("runPersist", func() error { return s.runPersist(ctx) }) - if pruneAfter, ok := s.cfg.PruneAfter.Get(); ok { - scope.SpawnNamed("runPruning", func() error { - return s.runPruning(ctx, pruneAfter) - }) - } return nil }) } diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go new file mode 100644 index 0000000000..718cd55933 --- /dev/null +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -0,0 +1,414 @@ +package data + +import ( + "context" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" +) + +// TestRecoveryEmpty verifies that NewState is a no-op on a fresh BlockDB. +func TestRecoveryEmpty(t *testing.T) { + rng := utils.TestRng() + registry, _ := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + fb := registry.FirstBlock() + + db := newTestBlockDB(t, dir) + state := newTestState(t, &Config{Registry: registry}, db) + require.Equal(t, fb, state.NextBlock()) + for inner := range state.inner.Lock() { + require.Equal(t, fb, inner.nextQC) + require.Equal(t, fb, inner.nextAppProposal) + } +} + +// TestNewStateInMemoryMode verifies that NewState with memblock followed by Run +// works end-to-end: QCs and blocks are accessible without a durable BlockDB dir. +func TestNewStateInMemoryMode(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + + state, err := NewState(&Config{Registry: registry}, memblock.NewBlockDB()) + require.NoError(t, err) + + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + s.SpawnBgNamed("state", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + if err := state.PushQC(ctx, qc1, blocks1); err != nil { + return err + } + // Verify data is accessible (no panic, no error). + gr := qc1.QC().GlobalRange() + for n := gr.First; n < gr.Next; n++ { + if _, err := state.Block(ctx, n); err != nil { + return err + } + } + return nil + })) +} + +// TestRecoveryNormal verifies that NewState fully restores QCs and blocks +// from BlockDB on restart. +func TestRecoveryNormal(t *testing.T) { + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + gr1 := qc1.QC().GlobalRange() + gr2 := qc2.QC().GlobalRange() + + // Session 1: write both QCs and all blocks. + db1 := newTestBlockDB(t, dir) + writeToBlockDB(t, db1, + []*types.FullCommitQC{qc1, qc2}, + [][]*types.Block{blocks1, blocks2}) + require.NoError(t, db1.Close()) + + // Session 2: NewState should recover blocks and QCs. + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + + require.Equal(t, gr2.Next, state2.NextBlock()) + for n := gr1.First; n < gr2.Next; n++ { + got, err := state2.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) + } + for n := gr1.First; n < gr2.Next; n++ { + got, err := state2.QC(t.Context(), n) + require.NoError(t, err) + require.NotNil(t, got) + } + require.NoError(t, db2.Close()) + + // Session 3: verify session 2 did not corrupt BlockDB. + db3 := newTestBlockDB(t, dir) + state3 := newTestState(t, &Config{Registry: registry}, db3) + require.Equal(t, gr2.Next, state3.NextBlock()) +} + +// TestPruningDiscards verifies that PruneBefore advances BlockDB's watermark so +// TryBlock returns ErrPruned for the discarded range, while later blocks stay +// accessible. Memory is cleared by evictExecuted (via Run during pushAppHashes). +func TestPruningDiscards(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + qc3, blocks3 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc2.QC())) + gr1 := qc1.QC().GlobalRange() + gr2 := qc2.QC().GlobalRange() + gr3 := qc3.QC().GlobalRange() + + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + require.NoError(t, state.PushQC(ctx, qc2, blocks2)) + require.NoError(t, state.PushQC(ctx, qc3, blocks3)) + + // Execute all blocks so they are eligible for pruning. + require.NoError(t, pushAppHashesRunning(ctx, state, rng, gr1.First, gr3.Next)) + + // Prune qc1 entirely (keep from qc2 onward). + require.NoError(t, state.PruneBefore(gr2.First)) + + for n := gr1.First; n < gr2.First; n++ { + _, err := state.TryBlock(n) + require.ErrorIs(t, err, ErrPruned) + } + for n := gr2.First; n < gr3.Next; n++ { + got, err := state.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) + } +} + +// TestRecoveryAfterPruning verifies that NewState recovers correctly when +// BlockDB only contains data from a later QC range (as left by pruning + GC). +func TestRecoveryAfterPruning(t *testing.T) { + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, _ := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + qc3, blocks3 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc2.QC())) + gr2 := qc2.QC().GlobalRange() + gr3 := qc3.QC().GlobalRange() + + // Write only qc2 and qc3 — simulating a DB where qc1 was pruned and GC'd. + db1 := newTestBlockDB(t, dir) + writeToBlockDB(t, db1, + []*types.FullCommitQC{qc2, qc3}, + [][]*types.Block{blocks2, blocks3}) + require.NoError(t, db1.Close()) + + // Recovery: first = gr2.First; qc1's range is before first, so ErrPruned. + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + + require.Equal(t, gr3.Next, state2.NextBlock()) + for n := qc1.QC().GlobalRange().First; n < gr2.First; n++ { + _, err := state2.TryBlock(n) + require.ErrorIs(t, err, ErrPruned) + } + for n := gr2.First; n < gr3.Next; n++ { + got, err := state2.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) + } +} + +// TestRecoveryBlocksBehind verifies recovery when QCs cover more range than +// blocks (e.g. a crash during block writes). Blocks up to the crash point are +// available; the rest are re-fetched via PushBlock. +func TestRecoveryBlocksBehind(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + gr1 := qc1.QC().GlobalRange() + gr2 := qc2.QC().GlobalRange() + + // Write both QCs but only qc1's blocks (simulate crash before qc2 blocks). + db1 := newTestBlockDB(t, dir) + require.NoError(t, db1.WriteQC(gr1.First, gr1.Next, qc1)) + require.NoError(t, db1.WriteQC(gr2.First, gr2.Next, qc2)) + for i, n := 0, gr1.First; n < gr1.Next; n++ { + require.NoError(t, db1.WriteBlock(n, blocks1[i])) + i++ + } + require.NoError(t, db1.Flush()) + require.NoError(t, db1.Close()) + + // Recovery: both QCs loaded, but only qc1's blocks. + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + + for n := gr1.First; n < gr1.Next; n++ { + got, err := state2.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) + } + for n := gr2.First; n < gr2.Next; n++ { + _, err := state2.TryBlock(n) + require.ErrorIs(t, err, ErrNotFound) + } + + // Re-push qc2's blocks to fill the gap. + for i, n := 0, gr2.First; n < gr2.Next; n++ { + require.NoError(t, state2.PushBlock(ctx, n, blocks2[i])) + i++ + } + require.Equal(t, gr2.Next, state2.NextBlock()) +} + +// TestRecoveryPartialQCPrefix verifies that a QC covering a wider range than +// the available blocks (block prefix missing) is rejected — we do not normalize +// by advancing first to the first present block. +func TestRecoveryPartialQCPrefix(t *testing.T) { + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + if gr1.Next-gr1.First < 3 { + t.Skip("need at least 3 blocks in QC range to test split") + } + + // Write the QC for the full range, but write blocks only from mid onwards. + mid := gr1.First + (gr1.Next-gr1.First)/2 + db1 := newTestBlockDB(t, dir) + require.NoError(t, db1.WriteQC(gr1.First, gr1.Next, qc1)) + for i, n := 0, gr1.First; n < gr1.Next; n++ { + if n >= mid { + require.NoError(t, db1.WriteBlock(n, blocks1[i])) + } + i++ + } + require.NoError(t, db1.Flush()) + require.NoError(t, db1.Close()) + + _, err := NewState(&Config{Registry: registry}, newTestBlockDB(t, dir)) + require.Error(t, err) +} + +// TestRecoveryAfterPruneNoGC verifies that restarting before async GC reclaims +// pruned entries does not cause NewState to fail. Blocks and QCs share the same +// GC filter in littblock, so below-watermark blocks never survive past their +// corresponding QCs — the first block iterator entry is always >= the recovery +// floor set by the QC pass, so the "block predates first QC start" guard never fires. +func TestRecoveryAfterPruneNoGC(t *testing.T) { + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + gr1 := qc1.QC().GlobalRange() + gr2 := qc2.QC().GlobalRange() + + // Write both QCs and all their blocks to the DB. + cfg1, err := littblock.DefaultConfig(dir) + require.NoError(t, err) + cfg1.Retention = time.Nanosecond + db1, err := littblock.NewBlockDB(cfg1) + require.NoError(t, err) + writeToBlockDB(t, db1, []*types.FullCommitQC{qc1, qc2}, [][]*types.Block{blocks1, blocks2}) + + // Prune qc1's range. GC is NOT called — pruned entries remain on disk. + require.NoError(t, db1.PruneBefore(gr2.First)) + require.NoError(t, db1.Close()) + + // Reopen the same dir without ForceGC — pruned entries may still be present. + cfg2, err := littblock.DefaultConfig(dir) + require.NoError(t, err) + cfg2.Retention = time.Nanosecond + db2, err := littblock.NewBlockDB(cfg2) + require.NoError(t, err) + t.Cleanup(func() { _ = db2.Close() }) + + // NewState must succeed — below-watermark blocks never outlive their QCs + // because blocks and QCs share the same GC filter in littblock. Without GC, + // all entries are still present and recovery treats the DB as unpruned. + // This is the PruneBefore-without-GC path: the watermark advanced but + // physical reclamation has not happened yet, so the DB looks like a + // fresh DB containing all data from qc1 and qc2. + state := newTestState(t, &Config{Registry: registry}, db2) + + // Without GC all data is still present; qc1 and qc2 blocks are accessible. + for n := gr1.First; n < gr2.Next; n++ { + _, err := state.TryBlock(n) + require.NoError(t, err) + } +} + +// TestRecoveryQCsNoBlocks verifies that NewState succeeds when the DB contains +// QCs but no blocks (crash between QC flush and block writes). The state +// cursor sits at the QC start with nextBlock at that floor and no block data. +func TestRecoveryQCsNoBlocks(t *testing.T) { + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, _ := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + + db1 := newTestBlockDB(t, dir) + require.NoError(t, db1.WriteQC(gr1.First, gr1.Next, qc1)) + require.NoError(t, db1.Flush()) + require.NoError(t, db1.Close()) + + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + + require.Equal(t, gr1.First, state2.NextBlock()) + for inner := range state2.inner.Lock() { + require.Equal(t, gr1.Next, inner.nextQC) + require.Equal(t, gr1.First, inner.nextAppProposal) + } + for n := gr1.First; n < gr1.Next; n++ { + _, err := state2.TryBlock(n) + require.ErrorIs(t, err, ErrNotFound) + } +} + +// TestRunPersistSeedsFromRecoveryFloor verifies that runPersist does not walk +// [genesis, recoveryFloor) when Status lacks LastBlockNumber (QC-only store +// whose first QC starts past FirstBlock). Seeding from nextBlockToPersist +// avoids collecting nil block pointers. +func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, _ := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + gr2 := qc2.QC().GlobalRange() + require.Greater(t, gr2.First, registry.FirstBlock(), "need skipTo past genesis") + + // First WriteQC on an empty DB may start past genesis (crash / partial retain). + db1 := newTestBlockDB(t, dir) + require.NoError(t, db1.WriteQC(gr2.First, gr2.Next, qc2)) + require.NoError(t, db1.Flush()) + require.NoError(t, db1.Close()) + + db2 := newTestBlockDB(t, dir) + tips := db2.Status() + require.False(t, tips.LastBlockNumber.IsPresent()) + _, ok := tips.LastQCNext.Get() + require.True(t, ok) + + state := newTestState(t, &Config{Registry: registry}, db2) + require.Equal(t, gr2.First, state.NextBlock()) + + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + s.SpawnBgNamed("state.Run", func() error { + return utils.IgnoreCancel(state.Run(runCtx)) + }) + // Filling blocks must not panic inside runPersist's write loop. + for i, n := 0, gr2.First; n < gr2.Next; n++ { + if err := state.PushBlock(ctx, n, blocks2[i]); err != nil { + return err + } + i++ + } + for n := gr2.First; n < gr2.Next; n++ { + if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { + return err + } + } + return nil + })) +} + +// TestRecoveryBlockGap verifies that NewState returns an error when blocks in +// BlockDB are not contiguous. WriteBlock only enforces strictly-ascending and +// QC coverage, not continuity, so a gap can arise from corruption. +func TestRecoveryBlockGap(t *testing.T) { + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + + // TestCommitQC generates 10 global blocks, so the range is always wide + // enough to skip one block in the middle. + mid := gr1.First + (gr1.Next-gr1.First)/2 + + db1 := newTestBlockDB(t, dir) + require.NoError(t, db1.WriteQC(gr1.First, gr1.Next, qc1)) + for i, n := 0, gr1.First; n < gr1.Next; n++ { + if n != mid { + require.NoError(t, db1.WriteBlock(n, blocks1[i])) + } + i++ + } + require.NoError(t, db1.Flush()) + require.NoError(t, db1.Close()) + + db2 := newTestBlockDB(t, dir) + _, err := NewState(&Config{Registry: registry}, db2) + require.ErrorIs(t, err, ErrBlockGap) +} diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index a25e4d61aa..370bc26438 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -5,14 +5,12 @@ import ( "errors" "fmt" "maps" - "os" - "path/filepath" "testing" "testing/synctest" "time" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" @@ -36,8 +34,8 @@ func newSnapshot() Snapshot { func snapshot(s *State) Snapshot { for inner := range s.inner.Lock() { aps := map[types.GlobalBlockNumber]*types.AppProposal{} - for n, apt := range inner.appProposals { - aps[n] = apt.proposal + for n, ap := range inner.appProposals { + aps[n] = ap } return Snapshot{ QCs: maps.Clone(inner.qcs), @@ -48,14 +46,83 @@ func snapshot(s *State) Snapshot { panic("unreachable") } +// newTestBlockDB opens (or creates) a LittDB-backed BlockDB at dir. +// Retention is set to 1ns so ForceGC reclaims pruned data immediately in tests. +// Errors panic so the helper is safe to call from non-main test goroutines. +func newTestBlockDB(t *testing.T, dir string) types.BlockDB { + t.Helper() + cfg, err := littblock.DefaultConfig(dir) + if err != nil { + panic(fmt.Sprintf("littblock.DefaultConfig: %v", err)) + } + cfg.Retention = time.Nanosecond + db, err := littblock.NewBlockDB(cfg) + if err != nil { + panic(fmt.Sprintf("littblock.NewBlockDB: %v", err)) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +// newTestState constructs a State, replays db, and returns it ready to Run. +// Errors panic so the helper is safe to call from non-main test goroutines. +func newTestState(t testing.TB, cfg *Config, db types.BlockDB) *State { + t.Helper() + s, err := NewState(cfg, db) + if err != nil { + panic(fmt.Sprintf("NewState: %v", err)) + } + return s +} + +// writeToBlockDB writes QC+block pairs sequentially to db and flushes once. +// qcs[i] and blockss[i] must correspond; QCs must be in ascending order. +// Errors panic so the helper is safe to call from non-main test goroutines. +func writeToBlockDB(t *testing.T, db types.BlockDB, qcs []*types.FullCommitQC, blockss [][]*types.Block) { + t.Helper() + for i, qc := range qcs { + gr := qc.QC().GlobalRange() + if err := db.WriteQC(gr.First, gr.Next, qc); err != nil { + panic(fmt.Sprintf("WriteQC: %v", err)) + } + for j, n := 0, gr.First; n < gr.Next; n++ { + if err := db.WriteBlock(n, blockss[i][j]); err != nil { + panic(fmt.Sprintf("WriteBlock: %v", err)) + } + j++ + } + } + if err := db.Flush(); err != nil { + panic(fmt.Sprintf("Flush: %v", err)) + } +} + +// pushAppHashesRunning runs state.Run under scope.Run long enough to accept +// PushAppHash for [first, next), then cancels Run. Prefers scope.Run over a +// raw goroutine so cleanup is structured. +func pushAppHashesRunning(ctx context.Context, state *State, rng utils.Rng, first, next types.GlobalBlockNumber) error { + return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + s.SpawnBgNamed("state.Run", func() error { + return utils.IgnoreCancel(state.Run(runCtx)) + }) + for n := first; n < next; n++ { + if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { + cancel() + return err + } + } + cancel() + return nil + }) +} + func TestState(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) if err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) s.SpawnBgNamed("state.Run()", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) @@ -127,14 +194,12 @@ func TestPushConflictingBadCommitQC(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) committee := registry.LatestEpoch().Committee() - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) // Push a valid QC to advance inner.nextQC. qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) require.NoError(t, state.PushQC(ctx, qc1, blocks1)) - nextQC := qc1.QC().GlobalRange().Next + gr1 := qc1.QC().GlobalRange() // Construct a malicious QC signed by non-committee keys. // It starts from block 0 (stale) but extends beyond nextQC. @@ -145,7 +210,7 @@ func TestPushConflictingBadCommitQC(t *testing.T) { badKeys[i] = types.GenSecretKey(rng) } laneBlocks := map[types.LaneID][]*types.Block{} - maliciousBlocksTotal := int(nextQC-registry.FirstBlock()) + 1 + maliciousBlocksTotal := int(gr1.Len()) + 1 require.LessOrEqual(t, maliciousBlocksTotal, committee.Lanes().Len()*types.MaxLaneRangeInProposal) for i := range maliciousBlocksTotal { lane := committee.Lanes().At(i % committee.Lanes().Len()) @@ -189,8 +254,8 @@ func TestPushConflictingBadCommitQC(t *testing.T) { utils.None[*types.AppQC](), )) malGR := proposal.Proposal().Msg().GlobalRange() - require.Less(t, malGR.First, nextQC, "test setup: malicious gr.First must be < nextQC") - require.Greater(t, malGR.Next, nextQC, "test setup: malicious gr.Next must be > nextQC") + require.Less(t, malGR.First, gr1.Next, "test setup: malicious gr.First must be < nextQC") + require.Greater(t, malGR.Next, gr1.Next, "test setup: malicious gr.Next must be > nextQC") votes := make([]*types.Signed[*types.CommitVote], 0, len(badKeys)) for _, k := range badKeys { @@ -205,7 +270,6 @@ func TestPushConflictingBadCommitQC(t *testing.T) { _ = state.PushQC(ctx, maliciousQC, malBlocks) // Verify state was not corrupted: all previously pushed QCs and blocks are intact. - gr1 := qc1.QC().GlobalRange() for n := gr1.First; n < gr1.Next; n++ { got, err := state.QC(ctx, n) require.NoError(t, err) @@ -237,9 +301,7 @@ func TestPushQCIgnoresBlocksMatchingUnverifiedHeaders(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) // Push qc1 with NO blocks — only the QC is stored. qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) @@ -283,9 +345,7 @@ func TestExecution(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) if err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) s.SpawnBgNamed("state.Run()", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) @@ -327,9 +387,7 @@ func TestPushBlockAcceptsBlockWithQC(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) // Push QC without blocks. qc, blocks := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) @@ -343,26 +401,12 @@ func TestPushBlockAcceptsBlockWithQC(t *testing.T) { require.Equal(t, blocks[0], got) } -// TestGlobalBlockByHash isolates the hash-keyed lookup from the -// consensus-driven harness. We push a single QC + block via the same code -// path the network would (insertBlock writes to inner.blockHashes), then: -// -// - the block's own header hash resolves to Some(*GlobalBlock) with the -// expected height/header/payload — the index points at the right -// block, atomically with the block construction -// - a zero hash and a random hash both resolve to None — distinct -// unknown-hash inputs all read as "not found", no panics -// - err is nil throughout — today's in-memory implementation has no -// failure mode; the error return on GlobalBlockByHash is reserved -// for the future BlockDB-backed path func TestGlobalBlockByHash(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) qc, blocks := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) require.NoError(t, state.PushQC(ctx, qc, blocks)) @@ -395,697 +439,258 @@ func TestGlobalBlockByHash(t *testing.T) { require.False(t, ok, "GlobalBlockByHash(random) returned Some") } -// ── Reconcile tests (grouped by case number) ────────────────────────── - -// TestStateRecoveryBlocksOnly simulates a crash after blocks are written -// TestReconcileCase1Empty verifies that reconcile is a no-op on a fresh -// WAL directory with no data. -func TestReconcileCase1Empty(t *testing.T) { - t.Log("Reconcile case 1: Fresh start (empty/empty)") - rng := utils.TestRng() - registry, _ := epoch.GenRegistry(rng, 3) - dir := t.TempDir() - fb := registry.FirstBlock() - - dw := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - state := utils.OrPanic1(NewState(&Config{Registry: registry}, dw)) - - for inner := range state.inner.Lock() { - require.Equal(t, fb, inner.first) - require.Equal(t, fb, inner.nextQC) - require.Equal(t, fb, inner.nextBlock) - } - require.NoError(t, dw.Close()) -} - -// TestReconcileCase2Corrupted verifies that when blocks are persisted but -// QCs WAL is deleted (corruption), NewDataWAL returns a corruption error. -func TestReconcileCase2Corrupted(t *testing.T) { - t.Log("Reconcile case 2: QCs lost (corruption), returns error") - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - dir := t.TempDir() - - // Persist blocks and QCs normally. - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - gr1 := qc1.QC().GlobalRange() - - dw1 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - require.NoError(t, dw1.CommitQCs.PersistQC(qc1)) - for i, n := 0, gr1.First; n < gr1.Next; n++ { - require.NoError(t, dw1.Blocks.PersistBlock(n, blocks1[i])) - i++ - } - require.NoError(t, dw1.Close()) - - // Simulate corruption: delete QCs WAL. - require.NoError(t, os.RemoveAll(filepath.Join(dir, "fullcommitqcs"))) - - // Reopen should fail — blocks exist but QCs are gone. - _, err := NewDataWAL(utils.Some(dir), registry.FirstBlock()) - require.Error(t, err) - require.Contains(t, err.Error(), "corrupted") -} - -// TestStateRecoveryQCsOnly simulates a crash after QCs are written to the -// WAL but before blocks are written (or after blocks WAL is truncated but -// QCs WAL is not). On recovery, QCs are loaded but no blocks exist. -// The cursor sync in NewState must advance the blocks persister so that -// subsequent PersistBlock calls don't fail with "out of sequence". -func TestReconcileCase3BlocksLost(t *testing.T) { - t.Log("Reconcile case 3: Blocks lost (crash), QCs survive") +// TestPushQCBeforeRunPersistsToBlockDB seeds in-memory QCs/blocks before Run +// (mirroring inbound PushQC after transport start) and asserts runPersist still +// writes them — Status seeding, not inner.nextQC/nextBlock. +func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) dir := t.TempDir() - // First run: populate both WALs. qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr1 := qc1.QC().GlobalRange() - dw1 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - state1 := utils.OrPanic1(NewState(&Config{Registry: registry}, dw1)) - require.NoError(t, state1.PushQC(ctx, qc1, blocks1)) - require.NoError(t, dw1.CommitQCs.PersistQC(qc1)) - for i, n := 0, gr1.First; n < gr1.Next; n++ { - require.NoError(t, dw1.Blocks.PersistBlock(n, blocks1[i])) - i++ - } - require.NoError(t, dw1.Close()) - - // Simulate crash: delete blocks WAL directory, keep QCs WAL. - require.NoError(t, os.RemoveAll(filepath.Join(dir, "globalblocks"))) - - // Second run: only QCs WAL survives. - dw2 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - state2 := utils.OrPanic1(NewState(&Config{Registry: registry}, dw2)) - - // QCs loaded, blocks empty. The state needs blocks re-pushed. - // Without the cursor sync fix, PushBlock here would fail with - // "out of sequence" because the blocks persister cursor is 0 - // but inner.nextBlock was advanced by QC data. - for i, n := 0, gr1.First; n < gr1.Next; n++ { - require.NoError(t, state2.PushBlock(ctx, n, blocks1[i])) - i++ - } + db := newTestBlockDB(t, dir) + state := newTestState(t, &Config{Registry: registry}, db) + // Transport-race window: PushQC before data.Run / runPersist starts. + require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + tips := db.Status() + require.False(t, tips.LastBlockNumber.IsPresent(), "PushQC must not write BlockDB before Run") + require.False(t, tips.LastQCNext.IsPresent()) + + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + s.SpawnBgNamed("state.Run", func() error { + return utils.IgnoreCancel(state.Run(runCtx)) + }) + // PushAppHash waits on nextBlockToPersist, so success implies Flush. + for n := gr1.First; n < gr1.Next; n++ { + if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { + cancel() + return fmt.Errorf("PushAppHash(%d): %w", n, err) + } + } + cancel() + return nil + })) + + tips = db.Status() + gotBlock, ok := tips.LastBlockNumber.Get() + require.True(t, ok) + require.Equal(t, gr1.Next-1, gotBlock) + gotQC, ok := tips.LastQCNext.Get() + require.True(t, ok) + require.Equal(t, gr1.Next, gotQC) + + require.NoError(t, db.Close()) + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + require.Equal(t, gr1.Next, state2.NextBlock()) for n := gr1.First; n < gr1.Next; n++ { got, err := state2.TryBlock(n) - require.NoError(t, err) + require.NoError(t, err, "block %d", n) require.NotNil(t, got) } - - // State should accept the next QC normally. - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - require.NoError(t, state2.PushQC(ctx, qc2, blocks2)) - require.Equal(t, qc2.QC().GlobalRange().Next, state2.NextBlock()) - require.NoError(t, dw2.Close()) } -func TestReconcileCase4Normal(t *testing.T) { - t.Log("Reconcile case 4: Normal (a=X, bX). -// Blocks WAL was truncated further than QCs during a crash between -// the two parallel TruncateBefore calls. -func TestReconcileCase5BlocksAhead(t *testing.T) { - t.Log("Reconcile case 5: Prune crash, blocks ahead (a>X)") - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - dir := t.TempDir() + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + s.SpawnBgNamed("state.Run", func() error { + return utils.IgnoreCancel(state.Run(runCtx)) + }) - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - gr1 := qc1.QC().GlobalRange() - gr2 := qc2.QC().GlobalRange() + require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + for n := gr1.First; n < gr1.Next; n++ { + if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { + return err + } + } - // Persist both QCs and all blocks. - dw := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - require.NoError(t, dw.CommitQCs.PersistQC(qc1)) - require.NoError(t, dw.CommitQCs.PersistQC(qc2)) - allBlocks := append(blocks1, blocks2...) - for i, n := 0, gr1.First; n < gr2.Next; n++ { - require.NoError(t, dw.Blocks.PersistBlock(n, allBlocks[i])) - i++ - } - // Simulate crash: blocks truncated to gr2.First but QCs not. - require.NoError(t, dw.Blocks.TruncateBefore(gr2.First)) - require.NoError(t, dw.Close()) + // No CommitQC.App yet → eviction must not strip AppProposals. + for inner := range state.inner.Lock() { + require.Equal(t, types.GlobalBlockNumber(0), inner.evictionBound(), "no certified App → bound 0") + for n := gr1.First; n < gr1.Next; n++ { + _, ok := inner.appProposals[n] + require.True(t, ok, "AppProposal %d must survive without CommitQC.App", n) + } + } - // Reopen: blocks start at gr2.First, QCs start at gr1.First. - // Reconcile should truncate QCs to match blocks. - dw2 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - state := utils.OrPanic1(NewState(&Config{Registry: registry}, dw2)) + require.NoError(t, state.PushQC(ctx, qc2, blocks2)) + for n := gr2.First; n < gr2.Next; n++ { + if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { + return err + } + } - for inner := range state.inner.Lock() { - require.Equal(t, gr2.First, inner.first) - } - // Blocks in qc2's range should be available. - for n := gr2.First; n < gr2.Next; n++ { - got, err := state.TryBlock(n) - require.NoError(t, err) - require.NotNil(t, got) - } - require.NoError(t, dw2.Close()) + for inner := range state.inner.Lock() { + bound := inner.evictionBound() + require.Equal(t, appFloor, bound, "retain floor is CommitQC.App.GlobalNumber()") + for n := gr1.First; n < bound; n++ { + _, ok := inner.appProposals[n] + require.False(t, ok, "AppProposal %d should be evicted (< App floor)", n) + } + _, ok := inner.appProposals[appFloor] + require.True(t, ok, "AppProposal at App floor %d must remain", appFloor) + } + return nil + })) } -// TestStateRecoverySkipsStaleBlocks verifies that blocks loaded from the WAL -// that fall before the first QC range are not inserted into the state map. -// This can happen when the QCs WAL is pruned but the blocks WAL still has -// older entries (e.g., a crash between the two TruncateBefore calls). -func TestReconcileCase6QCsAhead(t *testing.T) { - t.Log("Reconcile case 6: Prune crash, QCs ahead (a= nextQC are ignored. This can happen when blocks are -// persisted in parallel with QCs and we crash before QCs catch up. -func TestReconcileCase7BlocksPastQCs(t *testing.T) { - t.Log("Reconcile case 7: Persist crash, blocks past QCs (b>=Y)") - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - dir := t.TempDir() - - // Build 2 sequential QCs. - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - gr1 := qc1.QC().GlobalRange() - gr2 := qc2.QC().GlobalRange() - - // Persist only qc1 to QCs WAL but persist ALL blocks (qc1 + qc2) to blocks WAL. - // This simulates blocks being persisted ahead of QCs. - dw1 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - require.NoError(t, dw1.CommitQCs.PersistQC(qc1)) - allBlocks := append(blocks1, blocks2...) - for i, n := 0, gr1.First; n < gr2.Next; n++ { - require.NoError(t, dw1.Blocks.PersistBlock(n, allBlocks[i])) - i++ - } - require.NoError(t, dw1.Close()) + // Incomplete store (QC covers a range but only one block is present) + // must fail NewState — we do not normalize partial QC prefixes. + survivor := gr1.Next - 1 + dirBad := t.TempDir() + dbBad := newTestBlockDB(t, dirBad) + require.NoError(t, dbBad.WriteQC(gr1.First, gr1.Next, qc1)) + require.NoError(t, dbBad.WriteBlock(survivor, blocks1[survivor-gr1.First])) + require.NoError(t, dbBad.Flush()) + require.NoError(t, dbBad.Close()) + _, err := NewState(&Config{Registry: registry}, newTestBlockDB(t, dirBad)) + require.Error(t, err) - // On recovery, only blocks within qc1's range should be loaded. - // Blocks in qc2's range have no QC and should be ignored. - dw2 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - state2 := utils.OrPanic1(NewState(&Config{Registry: registry}, dw2)) + // Consistent post-GC shape: full QC range of blocks. Restart recovers at QC start. + dir := t.TempDir() + db := newTestBlockDB(t, dir) + writeToBlockDB(t, db, []*types.FullCommitQC{qc1}, [][]*types.Block{blocks1}) + require.NoError(t, db.Close()) - // Blocks in qc1's range should be available. + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + require.Equal(t, gr1.Next, state2.NextBlock()) for n := gr1.First; n < gr1.Next; n++ { got, err := state2.TryBlock(n) require.NoError(t, err) require.NotNil(t, got) } - - // Blocks in qc2's range should NOT be in the state (no QC for them). - for n := gr2.First; n < gr2.Next; n++ { - _, err := state2.TryBlock(n) - require.ErrorIs(t, err, ErrNotFound) - } - - // State should still accept qc2 normally. - require.NoError(t, state2.PushQC(t.Context(), qc2, blocks2)) - require.Equal(t, gr2.Next, state2.NextBlock()) - require.NoError(t, dw2.Close()) } -// TestReconcileTruncatesBlocksTail verifies that blocks persisted without -// corresponding QCs are removed during WAL reconciliation. This prevents -// stale blocks from blocking new (different) blocks at the same positions. -func TestReconcileCase7BlocksTail(t *testing.T) { - t.Log("Reconcile case 7: Persist crash, tail truncation with re-push") +// TestPruningWithPartialQCRange verifies BlockDB watermark pruning across QC +// ranges, and that a restart from a consistent BlockDB recovers from the +// retained QC start. BlockDB clamps prune requests to QC First (cohort-atomic +// readability), so a mid-range prune does not refuse heights inside that QC. +// +// PruneBefore is BlockDB-only: heights still retained in RAM for AppVotes +// (at/above CommitQC.App) remain readable via TryBlock even after the store +// watermark advances past them. +func TestPruningWithPartialQCRange(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - dir := t.TempDir() - // Build 2 sequential QCs. qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) gr1 := qc1.QC().GlobalRange() gr2 := qc2.QC().GlobalRange() + app, ok := qc2.QC().Proposal().App().Get() + require.True(t, ok) + appFloor := app.GlobalNumber() - // Persist qc1 to both WALs, but only blocks (not QC) for qc2. - // This simulates a crash during parallel persistence in runPersist. - dw1 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - require.NoError(t, dw1.CommitQCs.PersistQC(qc1)) - allBlocks := append(blocks1, blocks2...) - for i, n := 0, gr1.First; n < gr2.Next; n++ { - require.NoError(t, dw1.Blocks.PersistBlock(n, allBlocks[i])) - i++ - } - require.NoError(t, dw1.Close()) - - // Reopen: reconcile should truncate the blocks tail (qc2's blocks). - dw2 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - - // Blocks persister cursor should now match QCs range. - require.Equal(t, dw2.CommitQCs.Next(), dw2.Blocks.Next()) + state1 := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, state1.PushQC(ctx, qc1, blocks1)) + require.NoError(t, state1.PushQC(ctx, qc2, blocks2)) - state := utils.OrPanic1(NewState(&Config{Registry: registry}, dw2)) + require.NoError(t, pushAppHashesRunning(ctx, state1, rng, gr1.First, gr2.Next)) - // qc1's blocks should be available. - for n := gr1.First; n < gr1.Next; n++ { - got, err := state.TryBlock(n) - require.NoError(t, err) - require.NotNil(t, got) + // Mid-QC prune clamps to gr1.First, so the whole qc1 cohort stays readable. + midQC1 := gr1.First + (gr1.Next-gr1.First)/2 + if midQC1 > gr1.First { + require.NoError(t, state1.PruneBefore(midQC1)) + for n := gr1.First; n < midQC1; n++ { + got, err := state1.TryBlock(n) + require.NoError(t, err, "mid-QC prune must not refuse block %d (clamped to QC First)", n) + require.NotNil(t, got) + } } - // Now push qc2 with its blocks — should succeed because stale blocks - // were removed and the persister cursor was reset. - require.NoError(t, state.PushQC(ctx, qc2, blocks2)) - require.Equal(t, gr2.Next, state.NextBlock()) - require.NoError(t, dw2.Close()) -} - -// TestStateRecoveryBlocksBehindQCs verifies recovery when QCs cover a wider -// range than blocks (e.g. crash during block persistence). Blocks up to -// blocksEnd are available; the rest must be re-fetched via PushBlock. -func TestReconcileCase8BlocksBehind(t *testing.T) { - t.Log("Reconcile case 8: QCs ahead normal (b gr1.First+1 { - state.PruneBefore(midQC1) - for inner := range state.inner.Lock() { - require.Greater(t, inner.first, gr1.First, - "per-block pruning should advance past gr1.First") - } - } - - // Prune past qc1 entirely. - state.PruneBefore(gr2.Next) - - // Restart — recovery uses blocks as golden, skips partial QC prefix. - require.NoError(t, dw.Close()) - dw2 := utils.OrPanic1(NewDataWAL(utils.Some(dir), registry.FirstBlock())) - state2 := utils.OrPanic1(NewState(&Config{Registry: registry}, dw2)) - - // first should be recoverable and blocks should be available. - for inner := range state2.inner.Lock() { - require.GreaterOrEqual(t, inner.first, gr2.First, - "after restart, first should be at or past gr2.First") - require.Less(t, inner.first, gr2.Next, - "at least one block should survive") - } - require.NoError(t, dw2.Close()) -} - -// TestRunPruningEmptyState verifies that runPruning does not panic when -// the state has no QCs (e.g. on first startup before any data arrives). -func TestRunPruningEmptyState(t *testing.T) { - rng := utils.TestRng() - registry, _ := epoch.GenRegistry(rng, 3) - - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - PruneAfter: utils.Some(time.Duration(0)), - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) - - // Run briefly — runPruning should not panic on empty state. - ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) - defer cancel() - _ = state.Run(ctx) // returns context.DeadlineExceeded, that's fine } func TestPushBlockWaitsForQC(t *testing.T) { @@ -1094,9 +699,7 @@ func TestPushBlockWaitsForQC(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - state := utils.OrPanic1(NewState(&Config{ - Registry: registry, - }, utils.OrPanic1(NewDataWAL(utils.None[string](), registry.FirstBlock())))) + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) // Push first QC covering [0, N). qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) @@ -1136,3 +739,39 @@ func TestPushBlockWaitsForQC(t *testing.T) { require.Equal(t, blocks2[0], got) }) } + +// TestTryBlockHidesGapFills verifies the no-gap contract: a block stored above +// nextBlock (gap-fill) is not visible via TryBlock until the contiguous prefix +// catches up. +func TestTryBlockHidesGapFills(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + gr1 := qc1.QC().GlobalRange() + gr2 := qc2.QC().GlobalRange() + require.GreaterOrEqual(t, gr2.Len(), 2) + + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + require.NoError(t, state.PushQC(ctx, qc2, nil)) + require.Equal(t, gr1.Next, state.NextBlock()) + + // Gap-fill the last height of qc2 before earlier qc2 blocks. + last := gr2.Next - 1 + require.NoError(t, state.PushBlock(ctx, last, blocks2[last-gr2.First])) + _, err := state.TryBlock(last) + require.ErrorIs(t, err, ErrNotFound, "gap-fill above nextBlock must stay hidden") + + // Fill contiguous prefix; last becomes visible with the rest. + for i, n := 0, gr2.First; n < gr2.Next; n++ { + require.NoError(t, state.PushBlock(ctx, n, blocks2[i])) + i++ + } + require.Equal(t, gr2.Next, state.NextBlock()) + got, err := state.TryBlock(last) + require.NoError(t, err) + require.Equal(t, blocks2[last-gr2.First], got) +} diff --git a/sei-tendermint/internal/autobahn/producer/mempool_test.go b/sei-tendermint/internal/autobahn/producer/mempool_test.go index 8961595683..f2560b2b4b 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool_test.go +++ b/sei-tendermint/internal/autobahn/producer/mempool_test.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" @@ -231,10 +232,7 @@ func (env *testEnv) Run(ctx context.Context) error { func newTestEnv(rng utils.Rng, cfg *Config, app *proxy.Proxy) *testEnv { registry, keys := epoch.GenRegistry(rng, 1) - dataState := utils.OrPanic1(data.NewState( - &data.Config{Registry: registry}, - utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())), - )) + dataState := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, memblock.NewBlockDB())) consensusState := utils.OrPanic1(consensus.NewState(&consensus.Config{ Key: keys[0], ViewTimeout: func(types.View) time.Duration { return time.Hour }, diff --git a/sei-tendermint/internal/p2p/giga/avail_test.go b/sei-tendermint/internal/p2p/giga/avail_test.go index 038c0abf07..1eecbbe878 100644 --- a/sei-tendermint/internal/p2p/giga/avail_test.go +++ b/sei-tendermint/internal/p2p/giga/avail_test.go @@ -39,8 +39,9 @@ func TestAvailClientServer(t *testing.T) { s.SpawnBg(func() error { return env.Run(ctx) }) t.Log("Spawn a fake unconnected node0 to generate some conflicting blocks and push them to node2.") fakeNode0, err := consensus.NewState(&consensus.Config{ - Key: keys[0], - ViewTimeout: defaultViewTimeout, + Key: keys[0], + ViewTimeout: defaultViewTimeout, + PersistentStateDir: utils.Some(t.TempDir()), }, nodes[0].data) if err != nil { return fmt.Errorf("consensus.NewState(): %w", err) diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index 7d1b7f4a9f..0a3f1f8cfd 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -120,9 +120,14 @@ func (x *Service) runBlockFetcher(ctx context.Context) error { scope.Spawn(func() error { defer release() for first := true; ; first = false { - if _, err := x.data.TryBlock(n); !errors.Is(err, data.ErrNotFound) { + _, err := x.data.TryBlock(n) + if err == nil || errors.Is(err, data.ErrPruned) { + // Have it, or no longer needed (pruned past catch-up). return nil } + if !errors.Is(err, data.ErrNotFound) { + return fmt.Errorf("TryBlock(%d): %w", n, err) + } // Back off between repeated requests for the same block — // avoids hammering a peer that responded with an empty // block (doesn't have it yet). @@ -183,10 +188,15 @@ func (x *Service) serverGetBlock(ctx context.Context, server rpc.Server[API]) er return fmt.Errorf("GetBlockReqConv.Decode(): %w", err) } block, err := x.data.TryBlock(req.GlobalNumber) - resp := utils.None[*types.Block]() - if err == nil { - resp = utils.Some(block) + if err != nil { + // Absent (not yet / pruned) is a successful empty response so the + // peer can retry. Any other error is a store failure — do not + // disguise it as "not available". + if errors.Is(err, data.ErrPruned) || errors.Is(err, data.ErrNotFound) { + return stream.Send(ctx, GetBlockRespConv.Encode(utils.None[*types.Block]())) + } + return fmt.Errorf("TryBlock(%d): %w", req.GlobalNumber, err) } - return stream.Send(ctx, GetBlockRespConv.Encode(resp)) + return stream.Send(ctx, GetBlockRespConv.Encode(utils.Some(block))) }) } diff --git a/sei-tendermint/internal/p2p/giga/data_test.go b/sei-tendermint/internal/p2p/giga/data_test.go index 549a234152..0da59b5ea4 100644 --- a/sei-tendermint/internal/p2p/giga/data_test.go +++ b/sei-tendermint/internal/p2p/giga/data_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" @@ -25,7 +26,11 @@ type testNode struct { func defaultViewTimeout(view types.View) time.Duration { return time.Hour } func newTestNode(registry *epoch.Registry, cfg *consensus.Config) *testNode { - dataState := utils.OrPanic1(data.NewState(&data.Config{Registry: registry}, utils.OrPanic1(data.NewDataWAL(utils.None[string](), registry.FirstBlock())))) + cfg.PersistentStateDir = utils.None[string]() + dataState, err := data.NewState(&data.Config{Registry: registry}, memblock.NewBlockDB()) + if err != nil { + panic(fmt.Sprintf("data.NewState: %v", err)) + } consensusState, err := consensus.NewState(cfg, dataState) if err != nil { panic(fmt.Sprintf("consensus.NewState(): %v", err)) diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index 8ec60a977d..1cc77d14d8 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -32,8 +32,9 @@ type GigaRouterCommonConfig struct { DialInterval time.Duration ValidatorAddrs map[atypes.PublicKey]GigaNodeAddr GenDoc *types.GenesisDoc - // PersistentStateDir is the on-disk root for the data WAL (and the - // validator's consensus persister in a sibling subdir). None ⇒ in-memory. + // PersistentStateDir is the on-disk root for durable state (BlockDB, + // hashvault, and the validator's consensus persister in sibling subdirs). + // If None, persistence is disabled and the node runs fully in-memory. PersistentStateDir utils.Option[string] // App is the ABCI proxy executeBlock drives. NewGigaValidatorRouter // also passes it to producer.NewState so the producer's internal diff --git a/sei-tendermint/internal/p2p/giga_router_common.go b/sei-tendermint/internal/p2p/giga_router_common.go index 9be48343c3..d19f964dae 100644 --- a/sei-tendermint/internal/p2p/giga_router_common.go +++ b/sei-tendermint/internal/p2p/giga_router_common.go @@ -48,19 +48,16 @@ type gigaRouterCommon struct { // by one or two under contention but never over-accepts. inboundFullnodeCount atomic.Int64 inboundFullnodeCap int64 - - // hashVault is the app-hash equivocation guard. runExecute owns its lifecycle: it builds the - // vault on entry (durable under PersistentStateDir, or a no-op when disabled / in-memory) and - // closes it on exit. - hashVault hashvault.HashVault } -// buildDataState validates the common config and constructs the data -// layer (committee, WAL, State) shared by both giga constructors. +// BuildDataState validates the common config, constructs the committee, and +// returns an initialised data.State backed by blockDB. // -// TODO(autobahn): once sei-db/ledger_db/block.BlockDB has a writer wired -// (see BlockByNumber's TODO), the data WAL is redundant. -func buildDataState(cfg *GigaRouterCommonConfig) (*data.State, error) { +// The caller owns blockDB: close it after giga.Run returns, or immediately if +// construction of the GigaRouter fails. data.State never closes it. nodeImpl +// opens BlockDB in setup and closes after Run via SpawnCritical; tests that call +// BuildDataState directly must open and Close blockDB themselves. +func BuildDataState(cfg *GigaRouterCommonConfig, blockDB atypes.BlockDB) (*data.State, error) { if cfg.GenDoc.InitialHeight < 1 { return nil, fmt.Errorf("GenDoc.InitialHeight = %v, want >=1", cfg.GenDoc.InitialHeight) } @@ -83,13 +80,9 @@ func buildDataState(cfg *GigaRouterCommonConfig) (*data.State, error) { if err != nil { return nil, fmt.Errorf("epoch.NewRegistry(): %w", err) } - dataWAL, err := data.NewDataWAL(cfg.PersistentStateDir, registry.FirstBlock()) - if err != nil { - return nil, fmt.Errorf("data.NewDataWAL(): %w", err) - } - dataState, err := data.NewState(&data.Config{Registry: registry}, dataWAL) + dataState, err := data.NewState(&data.Config{Registry: registry}, blockDB) if err != nil { - return nil, fmt.Errorf("data.NewState(): %w", err) + return nil, fmt.Errorf("data.NewState: %w", err) } return dataState, nil } @@ -119,22 +112,14 @@ func (r *gigaRouterCommon) MaxGasEstimatedPerBlock() uint64 { // evmrpc does not read them on the receipt path. If gb.Header is nil // BlockID.Hash also stays empty; if gb.Payload is nil Block.Data.Txs // stays empty (see the malformed-block handling below). -// -// TODO(autobahn): switch this to read from sei-db/ledger_db/block.BlockDB -// once a writer is wired (e.g. from app.FinalizeBlocker or executeBlock). -// Today no production code calls BlockDB.WriteBlock, so Autobahn's in-memory -// data.State is the only place a full block lives — but it's pruned per -// Sei's RetainHeight and exposes only a height index (no GetBlockByHash). -// BlockDB has the right shape (height + hash indexes, async pruning) and -// is the long-term home for this read path. func (r *gigaRouterCommon) BlockByNumber(ctx context.Context, n atypes.GlobalBlockNumber) (*coretypes.ResultBlock, error) { gb, err := r.data.GlobalBlock(ctx, n) if err != nil { // Map Autobahn's pruning sentinel to CometBFT's, so callers // (env.Block, evmrpc, ops tooling) get the same error type they // already handle on the CometBFT path. base is None because the - // active lower bound (data.State.inner.first) is internal to - // data.State; both call sites format through the same helper. + // active lower bound lives in BlockDB's prune watermark (internal + // to the store); both call sites format through the same helper. if errors.Is(err, data.ErrPruned) { return nil, coretypes.WrapErrHeightNotAvailable(utils.Clamp[int64](n), utils.None[int64]()) } @@ -148,15 +133,11 @@ func (r *gigaRouterCommon) BlockByNumber(ctx context.Context, n atypes.GlobalBlo // (same translation as BlockByNumber). Matches CometBFT semantics for // unknown hashes: returns &ResultBlock{Block: nil} with no error. // -// Lookup-and-construct happens under a single data.State lock acquire, so -// the returned block matches the requested hash atomically. Hashes below -// the pruning watermark are not indexed and read as "unknown". Wrong-size -// inputs are rejected at the call site (env.BlockByHash) so this method -// can stay strongly typed on atypes.BlockHeaderHash. -// -// TODO(autobahn): replace this with a direct read from -// sei-db/ledger_db/block.BlockDB.GetBlockByHash once a writer is wired into -// block execution. The data.State-side index can also go away at that point. +// The lookup delegates to data.State.GlobalBlockByHash: an in-memory hash +// index first, then BlockDB for heights evicted after persist. Hashes not yet +// seen or below the prune watermark are read as "unknown". Wrong-size inputs +// are rejected at the call site (env.BlockByHash) so this method can stay +// strongly typed on atypes.BlockHeaderHash. func (r *gigaRouterCommon) BlockByHash(ctx context.Context, hash atypes.BlockHeaderHash) (*coretypes.ResultBlock, error) { opt, err := r.data.GlobalBlockByHash(hash) if err != nil { @@ -214,7 +195,7 @@ func (r *gigaRouterCommon) translateGlobalBlock(gb *atypes.GlobalBlock) *coretyp } } -func (r *gigaRouterCommon) executeBlock(ctx context.Context, b *atypes.GlobalBlock) (*abci.ResponseCommit, error) { +func (r *gigaRouterCommon) executeBlock(ctx context.Context, b *atypes.GlobalBlock, hashVault hashvault.HashVault) (*abci.ResponseCommit, error) { app := r.app hash := b.Header.Hash() var proposerAddress types.Address @@ -256,7 +237,7 @@ func (r *gigaRouterCommon) executeBlock(ctx context.Context, b *atypes.GlobalBlo // before the hash is proposed for AppQC voting via PushAppHash below). On restart the block is // re-executed and the identical hash is re-committed idempotently. A returned error is a benign // shutdown cancellation; genuine faults panic inside the call. See commitAppHashToVault. - if err := commitAppHashToVault(ctx, r.hashVault, b.GlobalNumber, resp.AppHash); err != nil { + if err := commitAppHashToVault(ctx, hashVault, b.GlobalNumber, resp.AppHash); err != nil { return nil, err } @@ -272,10 +253,8 @@ func (r *gigaRouterCommon) executeBlock(ctx context.Context, b *atypes.GlobalBlo // buildHashVault constructs the app-hash equivocation guard runExecute owns. By default it // returns a durable Pebble-backed vault rooted at /hashvault, alongside the -// other Autobahn on-disk state. It returns a no-op vault (no protection) in two cases: the operator -// explicitly set HashVaultDisabledUnsafe — logged loudly as unsafe — or there is no persistent -// state directory (in-memory mode, e.g. tests), where a durable vault would be pointless because -// the data WAL is itself a no-op (see data.NewDataWAL). +// other Autobahn on-disk state. It returns a no-op vault (no protection) when PersistentStateDir +// is unset or when the operator explicitly sets HashVaultDisabledUnsafe — both logged loudly. func buildHashVault(ctx context.Context, cfg *GigaRouterCommonConfig) (hashvault.HashVault, error) { if cfg.HashVaultDisabledUnsafe { logger.Error("################################################################") @@ -288,9 +267,12 @@ func buildHashVault(ctx context.Context, cfg *GigaRouterCommonConfig) (hashvault } dir, ok := cfg.PersistentStateDir.Get() if !ok { - logger.Error("HASHVAULT DISABLED: no persistent state dir (in-memory mode); using no-op vault. " + - "This node has NO app-hash equivocation protection. This is expected only for in-memory " + - "runs such as tests; a production node reaching this path is misconfigured.") + logger.Error("################################################################") + logger.Error("# HASHVAULT DISABLED (PersistentStateDir not set). #") + logger.Error("# This node has NO app-hash equivocation protection and is #") + logger.Error("# running in an UNSAFE configuration. Set persistent_state_dir #") + logger.Error("# in the node config to enable protection. #") + logger.Error("################################################################") return hashvault.NewNoopHashVault(), nil } hvCfg := hashvault.DefaultHashVaultConfig() @@ -342,7 +324,6 @@ func (r *gigaRouterCommon) runExecute(ctx context.Context) error { if err != nil { return fmt.Errorf("buildHashVault(): %w", err) } - r.hashVault = hashVault defer func() { if err := hashVault.Close(context.Background()); err != nil { logger.Error("failed to close hashvault", "err", err) @@ -384,7 +365,7 @@ func (r *gigaRouterCommon) runExecute(ctx context.Context) error { // first executed; if the committed app state has diverged from what the vault recorded (e.g. an // out-of-band rollback/restore), this halts the node instead of externalizing a conflicting // hash. A returned error is a benign shutdown cancellation; genuine faults panic inside. - if err := commitAppHashToVault(ctx, r.hashVault, last, info.LastBlockAppHash); err != nil { + if err := commitAppHashToVault(ctx, hashVault, last, info.LastBlockAppHash); err != nil { return err } // Losing a prefix of appHashes on crash is fine: AppQC is reached @@ -399,7 +380,7 @@ func (r *gigaRouterCommon) runExecute(ctx context.Context) error { if err != nil { return fmt.Errorf("r.data.GlobalBlock(%v): %w", n, err) } - commitResp, err := r.executeBlock(ctx, b) + commitResp, err := r.executeBlock(ctx, b, hashVault) if err != nil { return fmt.Errorf("r.executeBlock(%v): %w", n, err) } @@ -411,7 +392,7 @@ func (r *gigaRouterCommon) runExecute(ctx context.Context) error { return fmt.Errorf("r.data.PruneBefore(%v): %w", pruneBefore, err) } // Align the vault's retention with the data layer's prune boundary. - if err := r.hashVault.Prune(ctx, uint64(pruneBefore)); err != nil { + if err := hashVault.Prune(ctx, uint64(pruneBefore)); err != nil { // A canceled context just means we're shutting down between a successful executeBlock // and this prune; that's benign, not a prune failure, so don't alarm operators. if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { diff --git a/sei-tendermint/internal/p2p/giga_router_fullnode.go b/sei-tendermint/internal/p2p/giga_router_fullnode.go index afd0e1fa3f..890b899845 100644 --- a/sei-tendermint/internal/p2p/giga_router_fullnode.go +++ b/sei-tendermint/internal/p2p/giga_router_fullnode.go @@ -6,6 +6,7 @@ import ( "math/rand/v2" "slices" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/producer" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/giga" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/rpc" @@ -17,11 +18,9 @@ type gigaFullnodeRouter struct { *gigaRouterCommon } -func NewGigaFullnodeRouter(cfg *GigaRouterCommonConfig, key NodeSecretKey) (*gigaFullnodeRouter, error) { - dataState, err := buildDataState(cfg) - if err != nil { - return nil, err - } +// NewGigaFullnodeRouter constructs a fullnode GigaRouter over an already-built +// data.State. The caller owns the BlockDB that backs dataState (see BuildDataState). +func NewGigaFullnodeRouter(cfg *GigaRouterCommonConfig, key NodeSecretKey, dataState *data.State) (*gigaFullnodeRouter, error) { logger.Info("GigaRouter initialized (fullnode)", "validators", len(cfg.ValidatorAddrs), "dial_interval", cfg.DialInterval, "inbound_fullnode_cap", cfg.MaxInboundFullnodePeers) return &gigaFullnodeRouter{ gigaRouterCommon: &gigaRouterCommon{ diff --git a/sei-tendermint/internal/p2p/giga_router_fullnode_test.go b/sei-tendermint/internal/p2p/giga_router_fullnode_test.go index b4f0686c46..4c1c846b51 100644 --- a/sei-tendermint/internal/p2p/giga_router_fullnode_test.go +++ b/sei-tendermint/internal/p2p/giga_router_fullnode_test.go @@ -3,11 +3,13 @@ package p2p import ( "fmt" "net/url" + "path/filepath" "testing" "time" "github.com/ethereum/go-ethereum/common" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -57,17 +59,28 @@ func TestGigaRouter_Fullnode(t *testing.T) { app := newTestApp() proxyApp := proxy.New(app) - // Fullnodes have no validator key and no Producer config. The data WAL - // reads PersistentStateDir = None (in-memory) for this construction- - // shape check; App is required for executeBlock but isn't exercised by - // this test. - router, err := NewGigaFullnodeRouter(&GigaRouterCommonConfig{ + dir := t.TempDir() + // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig + // (zero overrides); p2p can't import config (import cycle). + littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) + require.NoError(t, err) + littCfg.Litt.Fsync = true + blockDB, err := littblock.NewBlockDB(littCfg) + require.NoError(t, err) + t.Cleanup(func() { _ = blockDB.Close() }) + cfg := &GigaRouterCommonConfig{ DialInterval: time.Second, ValidatorAddrs: addrs, - PersistentStateDir: utils.None[string](), + PersistentStateDir: utils.Some(dir), App: proxyApp, GenDoc: genDoc, - }, makeKey(rng)) + } + dataState, err := BuildDataState(cfg, blockDB) + require.NoError(t, err) + + // Fullnodes have no validator key and no Producer config. + // App is required for executeBlock but isn't exercised by this test. + router, err := NewGigaFullnodeRouter(cfg, makeKey(rng), dataState) require.NoError(t, err) // EvmProxy: for every sender, the fullnode router resolves to the diff --git a/sei-tendermint/internal/p2p/giga_router_validator.go b/sei-tendermint/internal/p2p/giga_router_validator.go index 9fe39e6dbf..347ae04015 100644 --- a/sei-tendermint/internal/p2p/giga_router_validator.go +++ b/sei-tendermint/internal/p2p/giga_router_validator.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/producer" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/giga" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/rpc" @@ -25,11 +26,10 @@ type gigaValidatorRouter struct { validatorKey atypes.PublicKey } -func NewGigaValidatorRouter(cfg *GigaValidatorConfig, key NodeSecretKey) (*gigaValidatorRouter, error) { - dataState, err := buildDataState(&cfg.GigaRouterCommonConfig) - if err != nil { - return nil, err - } +// NewGigaValidatorRouter constructs a validator GigaRouter over an already-built +// data.State. The caller owns the BlockDB that backs dataState (see BuildDataState); +// close it if this constructor returns an error. +func NewGigaValidatorRouter(cfg *GigaValidatorConfig, key NodeSecretKey, dataState *data.State) (*gigaValidatorRouter, error) { consensusState, err := consensus.NewState(&consensus.Config{ Key: cfg.ValidatorKey, ViewTimeout: cfg.ViewTimeout, diff --git a/sei-tendermint/internal/p2p/giga_router_validator_test.go b/sei-tendermint/internal/p2p/giga_router_validator_test.go index 1ee7e80f8a..21ada2c68a 100644 --- a/sei-tendermint/internal/p2p/giga_router_validator_test.go +++ b/sei-tendermint/internal/p2p/giga_router_validator_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "path/filepath" "testing" "time" @@ -11,6 +12,7 @@ import ( dbm "github.com/tendermint/tm-db" "golang.org/x/time/rate" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/producer" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/conn" @@ -62,17 +64,29 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { proxyApp := proxy.New(app) // In giga mode the CometBFT handshaker is skipped; the router's // runExecute calls InitChain itself on fresh start. + dir := t.TempDir() + // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig + // (zero overrides); p2p can't import config (import cycle). + littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) + require.NoError(t, err, "littblock.DefaultConfig[%v]", i) + littCfg.Litt.Fsync = true + blockDB, err := littblock.NewBlockDB(littCfg) + require.NoError(t, err, "littblock.NewBlockDB[%v]", i) + t.Cleanup(func() { _ = blockDB.Close() }) + commonCfg := GigaRouterCommonConfig{ + // Aggressive dialing rate to speed up startup. + DialInterval: 100 * time.Millisecond, + ValidatorAddrs: addrs, + PersistentStateDir: utils.Some(dir), + App: proxyApp, + GenDoc: genDoc, + } + dataState, err := BuildDataState(&commonCfg, blockDB) + require.NoError(t, err, "BuildDataState[%v]", i) giga, err := NewGigaValidatorRouter(&GigaValidatorConfig{ - GigaRouterCommonConfig: GigaRouterCommonConfig{ - // Aggressive dialing rate to speed up startup. - DialInterval: 100 * time.Millisecond, - ValidatorAddrs: addrs, - PersistentStateDir: utils.None[string](), - App: proxyApp, - GenDoc: genDoc, - }, - ValidatorKey: cfg.validatorKey, - ViewTimeout: func(atypes.View) time.Duration { return time.Hour }, + GigaRouterCommonConfig: commonCfg, + ValidatorKey: cfg.validatorKey, + ViewTimeout: func(atypes.View) time.Duration { return time.Hour }, Producer: &producer.Config{ MaxGasWantedPerBlock: txGasUsed * maxTxsPerBlock, MaxGasEstimatedPerBlock: txGasUsed * maxTxsPerBlock, @@ -81,7 +95,7 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { BlockInterval: 100 * time.Millisecond, AllowEmptyBlocks: false, }, - }, cfg.nodeKey) + }, cfg.nodeKey, dataState) require.NoError(t, err, "NewGigaValidatorRouter[%v]", i) router, err := NewRouter( cfg.nodeKey, @@ -220,16 +234,29 @@ func TestGigaRouter_EvmProxy(t *testing.T) { } require.NoError(t, genDoc.ValidateAndComplete()) + dir := t.TempDir() + // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig + // (zero overrides); p2p can't import config (import cycle). + littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) + require.NoError(t, err) + littCfg.Litt.Fsync = true + blockDB, err := littblock.NewBlockDB(littCfg) + require.NoError(t, err) + t.Cleanup(func() { _ = blockDB.Close() }) + commonCfg := GigaRouterCommonConfig{ + DialInterval: time.Second, + ValidatorAddrs: addrs, + PersistentStateDir: utils.Some(dir), + App: proxy.New(newTestApp()), + GenDoc: genDoc, + } + dataState, err := BuildDataState(&commonCfg, blockDB) + require.NoError(t, err) + router, err := NewGigaValidatorRouter(&GigaValidatorConfig{ - GigaRouterCommonConfig: GigaRouterCommonConfig{ - DialInterval: time.Second, - ValidatorAddrs: addrs, - PersistentStateDir: utils.None[string](), - App: proxy.New(newTestApp()), - GenDoc: genDoc, - }, - ValidatorKey: validatorKeys[0], - ViewTimeout: func(atypes.View) time.Duration { return time.Second }, + GigaRouterCommonConfig: commonCfg, + ValidatorKey: validatorKeys[0], + ViewTimeout: func(atypes.View) time.Duration { return time.Second }, Producer: &producer.Config{ MaxGasWantedPerBlock: 1, MaxGasEstimatedPerBlock: 1, @@ -237,7 +264,7 @@ func TestGigaRouter_EvmProxy(t *testing.T) { MaxTxsPerSecond: utils.None[uint64](), BlockInterval: time.Second, }, - }, nodeKeys[0]) + }, nodeKeys[0], dataState) require.NoError(t, err) localValidator := validatorKeys[0].Public() diff --git a/sei-tendermint/internal/p2p/routeroptions.go b/sei-tendermint/internal/p2p/routeroptions.go index b7eca55273..2403f42f5e 100644 --- a/sei-tendermint/internal/p2p/routeroptions.go +++ b/sei-tendermint/internal/p2p/routeroptions.go @@ -47,9 +47,10 @@ type RouterOptions struct { PexOnHandshake bool // Giga, if Some, is the already-constructed GigaRouter the Router - // should attach. Setup-side code (node/setup.go) picks the right - // constructor — NewGigaValidatorRouter or NewGigaFullnodeRouter — - // based on whether the node has a local validator key. + // should attach. Setup-side code (node/setup.go) opens BlockDB via + // BuildDataState, then picks NewGigaValidatorRouter or + // NewGigaFullnodeRouter based on cfg.Mode. BlockDB lifecycle is owned + // by nodeImpl, not the Router. Giga utils.Option[GigaRouter] // Local endpoint to listen for p2p connections on. diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index 72ca5e4cd6..fa970409fa 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -8,6 +8,7 @@ import ( _ "net/http/pprof" // nolint: gosec // securely exposed on separate, optional port "slices" "strings" + "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -95,11 +96,13 @@ type nodeImpl struct { consensusPolicy types.ConsensusPolicy // network - router *p2p.Router - giga utils.Option[p2p.GigaRouter] - ServiceRestartCh utils.Option[chan []string] - nodeInfo types.NodeInfo - nodeKey types.NodeKey // our node privkey + router *p2p.Router + giga utils.Option[p2p.GigaRouter] + gigaBlockDB utils.Option[atypes.BlockDB] // owned here; closed after giga.Run (sync.Once) + gigaBlockDBCloseOnce sync.Once + ServiceRestartCh utils.Option[chan []string] + nodeInfo types.NodeInfo + nodeKey types.NodeKey // our node privkey // services eventSinks []indexer.EventSink @@ -129,11 +132,19 @@ func makeNode( tracerProviderOptions []trace.TracerProviderOption, consensusPolicy types.ConsensusPolicy, ) (_ local.NodeService, err error) { - var cancel context.CancelFunc + var ( + cancel context.CancelFunc + node *nodeImpl + ) ctx, cancel = context.WithCancel(ctx) closers := []closer{convertCancelCloser(cancel)} defer func() { if err != nil { + // Close BlockDB on construct failure after it was opened. Must not + // live in shutdownOps (see OnStart comment on SpawnCritical). + if node != nil { + _ = node.closeGigaBlockDB() + } err = combineCloseError(err, makeCloser(closers)) } }() @@ -199,7 +210,7 @@ func makeNode( eventLogOpt = utils.Some(eventLog) } // TODO construct node here: - node := &nodeImpl{ + node = &nodeImpl{ config: cfg, genesisDoc: genDoc, privValidator: privValidator, @@ -236,7 +247,7 @@ func makeNode( if gigaEnabled { gigaValidatorKey = utils.Some(atypes.SecretKeyFromED25519(filePrivval.Key.PrivKey)) } - router, peerCloser, err := createRouter( + router, peerCloser, gigaBlockDB, err := createRouter( node.NodeInfo, nodeKey, gigaValidatorKey, @@ -251,6 +262,13 @@ func makeNode( } node.router = router node.giga = router.Giga() + node.gigaBlockDB = gigaBlockDB + // BlockDB is NOT closed in OnStop: BaseService runs OnStop before + // SpawnCritical (giga.Run) finishes, so closing there would race with + // still-running persist/execute. Close paths: + // - makeNode defer on construct failure + // - OnStart defer if Start fails before giga is spawned + // - SpawnCritical wrapper after giga.Run returns (happy path / cancel) node.rpcEnv.Router = router evReactor, evPool, edbCloser, err := createEvidenceReactor(cfg, dbProvider, @@ -448,16 +466,28 @@ func makeNode( } // OnStart starts the Node. It implements service.Service. -func (n *nodeImpl) OnStart(ctx context.Context) error { +func (n *nodeImpl) OnStart(ctx context.Context) (err error) { + // If Start fails before giga is spawned, BaseService does not call OnStop + // and never cancels SpawnCritical — so BlockDB would otherwise leak. + // When giga has already been spawned, its wrapper closes BlockDB after + // Run observes the service-context cancel issued once OnStart returns. + gigaSpawned := false + defer func() { + if err == nil || gigaSpawned { + return + } + _ = n.closeGigaBlockDB() + }() + // EventBus and IndexerService must be started before the handshake because // we might need to index the txs of the replayed block as this might not have happened // when the node stopped last time (i.e. the node stopped or crashed after it saved the block // but before it indexed the txs) - if err := n.rpcEnv.EventBus.Start(ctx); err != nil { + if err = n.rpcEnv.EventBus.Start(ctx); err != nil { return err } - if err := n.indexerService.Start(ctx); err != nil { + if err = n.indexerService.Start(ctx); err != nil { return err } @@ -467,7 +497,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { if n.shouldHandshake { // Create the handshaker, which calls RequestInfo, sets the AppVersion on the state, // and replays any blocks as necessary to sync tendermint with the app. - if err := consensus.NewHandshaker( + if err = consensus.NewHandshaker( n.stateStore, n.initialState, n.blockStore, n.rpcEnv.EventBus, n.genesisDoc, n.consensusPolicy, ).Handshake(ctx, n.rpcEnv.App); err != nil { return err @@ -547,23 +577,29 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { n.prometheusSrv = utils.Some(n.startPrometheusServer(ctx, n.config.Instrumentation.PrometheusListenAddr)) } + // Start giga before the transport so runPersist is active before inbound + // giga connections (dispatched via the router) can advance inner cursors + // via PushQC. Keep this after the genesis-time wait and EventBus/indexer + // init so Autobahn does not InitChain or produce ahead of genesis. + if giga, ok := n.giga.Get(); ok { + gigaSpawned = true + n.SpawnCritical("giga", func(ctx context.Context) error { + defer func() { _ = n.closeGigaBlockDB() }() + return giga.Run(ctx) + }) + } + // Start the transport. - if err := n.router.Start(ctx); err != nil { + if err = n.router.Start(ctx); err != nil { return err } n.rpcEnv.IsListening = true if m, ok := n.mempool.Get(); ok { n.SpawnCritical("mempool", m.Run) } - // Run the GigaRouter alongside the transport. n.giga is the canonical - // reference; the Router holds a copy only for its own internal use - // (dispatching inbound giga connections). Lifecycle is owned here. - if giga, ok := n.giga.Get(); ok { - n.SpawnCritical("giga", giga.Run) - } for _, reactor := range n.services { - if err := reactor.Start(ctx); err != nil { + if err = reactor.Start(ctx); err != nil { return fmt.Errorf("problem starting service '%T': %w ", reactor, err) } } @@ -572,7 +608,6 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { // Start the RPC server before the P2P server // so we can eg. receive txs for the first block if n.config.RPC.ListenAddress != "" { - var err error n.rpcListeners, err = n.rpcEnv.StartService(ctx, n.config) if err != nil { return err @@ -635,6 +670,21 @@ func (n *nodeImpl) OnStop() { } } +// closeGigaBlockDB closes the Autobahn BlockDB at most once. Safe to call from +// makeNode's failure defer, OnStart's pre-giga failure path, and the giga +// SpawnCritical wrapper. +func (n *nodeImpl) closeGigaBlockDB() error { + var err error + n.gigaBlockDBCloseOnce.Do(func() { + if db, ok := n.gigaBlockDB.Get(); ok { + if err = db.Close(); err != nil { + logger.Error("failed to close Autobahn BlockDB", "err", err) + } + } + }) + return err +} + // startPrometheusServer starts a Prometheus HTTP server, listening for metrics // collectors on addr. func (n *nodeImpl) startPrometheusServer(ctx context.Context, addr string) *http.Server { diff --git a/sei-tendermint/node/seed.go b/sei-tendermint/node/seed.go index 359db8506c..53b44d20ba 100644 --- a/sei-tendermint/node/seed.go +++ b/sei-tendermint/node/seed.go @@ -77,7 +77,7 @@ func makeSeedNode( return nil, err } - router, peerCloser, err := createRouter( + router, peerCloser, _, err := createRouter( func() *types.NodeInfo { return &nodeInfo }, nodeKey, utils.None[atypes.SecretKey](), diff --git a/sei-tendermint/node/setup.go b/sei-tendermint/node/setup.go index 0b81dbe4db..9671843322 100644 --- a/sei-tendermint/node/setup.go +++ b/sei-tendermint/node/setup.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/config" "github.com/sei-protocol/sei-chain/sei-tendermint/crypto" @@ -55,7 +57,7 @@ func makeCloser(cs []closer) closer { errs = append(errs, err.Error()) } } - if len(errs) >= 0 { + if len(errs) > 0 { return errors.New(strings.Join(errs, "; ")) } return nil @@ -275,16 +277,21 @@ func buildValidatorGigaConfig( // catch-up as a fullnode before the operator flips to mode = "validator". // A warning is logged if mode and committee membership disagree so an // operator misconfiguration is visible at startup. +// +// The returned BlockDB is owned by the caller (nodeImpl): open happens here +// before the transport starts, so inbound giga connections see a fully +// replayed data.State. Close after giga.Run returns (or immediately if this +// function / subsequent construction fails). func buildGigaRouter( cfg *config.Config, nodeKey types.NodeKey, validatorKey utils.Option[atypes.SecretKey], app *proxy.Proxy, genDoc *types.GenesisDoc, -) (p2p.GigaRouter, error) { - _, validatorAddrs, err := loadAutobahnCommittee(cfg.AutobahnConfigFile) +) (p2p.GigaRouter, atypes.BlockDB, error) { + fc, validatorAddrs, err := loadAutobahnCommittee(cfg.AutobahnConfigFile) if err != nil { - return nil, err + return nil, nil, err } if valKey, ok := validatorKey.Get(); ok { _, inCommittee := validatorAddrs[valKey.Public()] @@ -298,44 +305,106 @@ func buildGigaRouter( if cfg.Mode == config.ModeValidator { valKey, ok := validatorKey.Get() if !ok { - return nil, fmt.Errorf("autobahn: mode = %q requires a local validator key", cfg.Mode) + return nil, nil, fmt.Errorf("autobahn: mode = %q requires a local validator key", cfg.Mode) } // Remote signers aren't supported on the validator path — // autobahn signs in-process. Fullnodes don't sign and aren't // penalised for having priv-validator.laddr set. if cfg.PrivValidator.ListenAddr != "" { - return nil, fmt.Errorf("autobahn does not support remote validator signers (priv-validator.laddr is set)") + return nil, nil, fmt.Errorf("autobahn does not support remote validator signers (priv-validator.laddr is set)") } valCfg, err := buildValidatorGigaConfig(cfg.AutobahnConfigFile, nodeKey, valKey, app, genDoc) if err != nil { - return nil, fmt.Errorf("buildValidatorGigaConfig: %w", err) + return nil, nil, fmt.Errorf("buildValidatorGigaConfig: %w", err) + } + if err := preparePersistentStateDir(cfg.RootDir, &valCfg.GigaRouterCommonConfig); err != nil { + return nil, nil, err } - rootifyPersistentStateDir(cfg.RootDir, &valCfg.GigaRouterCommonConfig) // The GigaRouter builds and owns the equivocation guard itself; just pass the operator's // enable/disable decision through as plain config. valCfg.HashVaultDisabledUnsafe = cfg.HashVaultDisabledUnsafe logger.Info("Autobahn: starting as validator", "validators", len(valCfg.ValidatorAddrs)) - return p2p.NewGigaValidatorRouter(valCfg, p2p.NodeSecretKey(nodeKey)) + blockDB, err := openBlockDB(&valCfg.GigaRouterCommonConfig, fc.BlockDB) + if err != nil { + return nil, nil, err + } + dataState, err := p2p.BuildDataState(&valCfg.GigaRouterCommonConfig, blockDB) + if err != nil { + _ = blockDB.Close() + return nil, nil, err + } + giga, err := p2p.NewGigaValidatorRouter(valCfg, p2p.NodeSecretKey(nodeKey), dataState) + if err != nil { + _ = blockDB.Close() + return nil, nil, err + } + return giga, blockDB, nil } fnCfg, err := buildFullnodeGigaConfig(cfg.AutobahnConfigFile, app, genDoc) if err != nil { - return nil, fmt.Errorf("buildFullnodeGigaConfig: %w", err) + return nil, nil, fmt.Errorf("buildFullnodeGigaConfig: %w", err) + } + if err := preparePersistentStateDir(cfg.RootDir, fnCfg); err != nil { + return nil, nil, err } - rootifyPersistentStateDir(cfg.RootDir, fnCfg) // The GigaRouter builds and owns the equivocation guard itself; just pass the operator's // enable/disable decision through as plain config. fnCfg.HashVaultDisabledUnsafe = cfg.HashVaultDisabledUnsafe logger.Info("Autobahn: starting as fullnode", "mode", cfg.Mode, "validators", len(validatorAddrs)) - return p2p.NewGigaFullnodeRouter(fnCfg, p2p.NodeSecretKey(nodeKey)) + blockDB, err := openBlockDB(fnCfg, fc.BlockDB) + if err != nil { + return nil, nil, err + } + dataState, err := p2p.BuildDataState(fnCfg, blockDB) + if err != nil { + _ = blockDB.Close() + return nil, nil, err + } + giga, err := p2p.NewGigaFullnodeRouter(fnCfg, p2p.NodeSecretKey(nodeKey), dataState) + if err != nil { + _ = blockDB.Close() + return nil, nil, err + } + return giga, blockDB, nil +} + +// preparePersistentStateDir resolves a relative PersistentStateDir against +// the node's --home dir (mirrors config.go's rootify) and creates it if absent. +// Some("") is treated as None (in-memory / disabled): JSON unmarshals a literal +// empty string as present, which would otherwise Join to rootDir and silently +// enable durable BlockDB + HashVault. +func preparePersistentStateDir(rootDir string, c *p2p.GigaRouterCommonConfig) error { + dir, ok := c.PersistentStateDir.Get() + if !ok || dir == "" { + c.PersistentStateDir = utils.None[string]() + return nil + } + if !filepath.IsAbs(dir) { + dir = filepath.Join(rootDir, dir) + c.PersistentStateDir = utils.Some(dir) + } + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("creating persistent state dir %q: %w", dir, err) + } + return nil } -// rootifyPersistentStateDir resolves a relative PersistentStateDir -// against the node's --home dir (mirrors config.go's rootify). Absolute -// paths pass through; None stays None. -func rootifyPersistentStateDir(rootDir string, c *p2p.GigaRouterCommonConfig) { - if dir, ok := c.PersistentStateDir.Get(); ok && !filepath.IsAbs(dir) { - c.PersistentStateDir = utils.Some(filepath.Join(rootDir, dir)) +// openBlockDB opens littblock when PersistentStateDir is set, memblock otherwise. +// preparePersistentStateDir must have run first so dir is rootified and created. +func openBlockDB(c *p2p.GigaRouterCommonConfig, blockDBCfg config.AutobahnBlockDBConfig) (atypes.BlockDB, error) { + dir, ok := c.PersistentStateDir.Get() + if !ok { + return memblock.NewBlockDB(), nil + } + littCfg, err := blockDBCfg.LittBlockConfig(filepath.Join(dir, "blockdb")) + if err != nil { + return nil, fmt.Errorf("block_db: %w", err) } + blockDB, err := littblock.NewBlockDB(&littCfg) + if err != nil { + return nil, fmt.Errorf("open BlockDB: %w", err) + } + return blockDB, nil } // resolveMaxInboundFullnodePeers: None ⇒ default, Some(0) ⇒ reject all, @@ -393,11 +462,13 @@ func createRouter( app utils.Option[*proxy.Proxy], genDoc *types.GenesisDoc, dbProvider config.DBProvider, -) (*p2p.Router, closer, error) { +) (*p2p.Router, closer, utils.Option[atypes.BlockDB], error) { closer := func() error { return nil } + noneDB := utils.None[atypes.BlockDB]() + gigaBlockDB := noneDB ep, err := p2p.ResolveEndpoint(nodeKey.ID().AddressString(cfg.P2P.ListenAddress)) if err != nil { - return nil, closer, err + return nil, closer, noneDB, err } var privatePeerIDs []types.NodeID for _, id := range tmstrings.SplitAndTrimEmpty(cfg.P2P.PrivatePeerIDs, ",", " ") { @@ -441,7 +512,7 @@ func createRouter( if addr := cfg.P2P.ExternalAddress; addr != "" { nodeAddr, err := p2p.ParseNodeAddress(nodeKey.ID().AddressString(addr)) if err != nil { - return nil, closer, fmt.Errorf("couldn't parse ExternalAddress %q: %w", cfg.P2P.ExternalAddress, err) + return nil, closer, noneDB, fmt.Errorf("couldn't parse ExternalAddress %q: %w", cfg.P2P.ExternalAddress, err) } options.SelfAddress = utils.Some(nodeAddr) } @@ -449,7 +520,7 @@ func createRouter( for _, p := range tmstrings.SplitAndTrimEmpty(cfg.P2P.PersistentPeers, ",", " ") { address, err := p2p.ParseNodeAddress(p) if err != nil { - return nil, closer, fmt.Errorf("invalid peer address %q: %w", p, err) + return nil, closer, noneDB, fmt.Errorf("invalid peer address %q: %w", p, err) } options.PersistentPeers = append(options.PersistentPeers, address) } @@ -457,7 +528,7 @@ func createRouter( for _, p := range tmstrings.SplitAndTrimEmpty(cfg.P2P.BootstrapPeers, ",", " ") { address, err := p2p.ParseNodeAddress(p) if err != nil { - return nil, closer, fmt.Errorf("invalid peer address %q: %w", p, err) + return nil, closer, noneDB, fmt.Errorf("invalid peer address %q: %w", p, err) } options.BootstrapPeers = append(options.BootstrapPeers, address) } @@ -465,7 +536,7 @@ func createRouter( for _, p := range tmstrings.SplitAndTrimEmpty(cfg.P2P.BlockSyncPeers, ",", " ") { address, err := p2p.ParseNodeAddress(p) if err != nil { - return nil, closer, fmt.Errorf("invalid peer address %q: %w", p, err) + return nil, closer, noneDB, fmt.Errorf("invalid peer address %q: %w", p, err) } options.PersistentPeers = append(options.PersistentPeers, address) options.BlockSyncPeers = append(options.BlockSyncPeers, address.NodeID) @@ -480,18 +551,22 @@ func createRouter( logger.Info("Autobahn config enabled", "config_file", cfg.AutobahnConfigFile, "mode", cfg.Mode) proxyApp, ok := app.Get() if !ok { - return nil, closer, fmt.Errorf("autobahn requires app") + return nil, closer, noneDB, fmt.Errorf("autobahn requires app") } - giga, err := buildGigaRouter(cfg, nodeKey, validatorKey, proxyApp, genDoc) + giga, blockDB, err := buildGigaRouter(cfg, nodeKey, validatorKey, proxyApp, genDoc) if err != nil { - return nil, closer, err + return nil, closer, noneDB, err } options.Giga = utils.Some(giga) + gigaBlockDB = utils.Some(blockDB) } peerDB, err := dbProvider(&config.DBContext{ID: "peerstore", Config: cfg}) if err != nil { - return nil, closer, fmt.Errorf("unable to initialize peer store: %w", err) + if db, ok := gigaBlockDB.Get(); ok { + _ = db.Close() + } + return nil, closer, noneDB, fmt.Errorf("unable to initialize peer store: %w", err) } closer = peerDB.Close router, err := p2p.NewRouter( @@ -501,9 +576,12 @@ func createRouter( options, ) if err != nil { - return nil, closer, fmt.Errorf("p2p.NewRouter(): %w", err) + if db, ok := gigaBlockDB.Get(); ok { + _ = db.Close() + } + return nil, closer, noneDB, fmt.Errorf("p2p.NewRouter(): %w", err) } - return router, closer, nil + return router, closer, gigaBlockDB, nil } func makeNodeInfo( diff --git a/sei-tendermint/node/setup_test.go b/sei-tendermint/node/setup_test.go index 375a0098fa..62668485a0 100644 --- a/sei-tendermint/node/setup_test.go +++ b/sei-tendermint/node/setup_test.go @@ -54,7 +54,7 @@ func writeAutobahnConfig(t *testing.T, fc *config.AutobahnFileConfig) string { return path } -func defaultFileConfig(validators []config.AutobahnValidator) *config.AutobahnFileConfig { +func defaultFileConfig(t testing.TB, validators []config.AutobahnValidator) *config.AutobahnFileConfig { return &config.AutobahnFileConfig{ Validators: validators, MaxTxsPerBlock: 5_000, @@ -62,7 +62,7 @@ func defaultFileConfig(validators []config.AutobahnValidator) *config.AutobahnFi AllowEmptyBlocks: false, BlockInterval: utils.Duration(400 * time.Millisecond), ViewTimeout: utils.Duration(1500 * time.Millisecond), - PersistentStateDir: utils.None[string](), + PersistentStateDir: utils.Some(t.TempDir()), DialInterval: utils.Duration(10 * time.Second), } } @@ -84,6 +84,67 @@ func makeTestGigaDeps() (*proxy.Proxy, *types.GenesisDoc) { return app, genDoc } +func TestBuildGigaConfig_NonePersistentStateDir(t *testing.T) { + v1 := makeValidator([]byte("val-seed"), []byte("node-seed"), "localhost:26660") + fc := defaultFileConfig(t, []config.AutobahnValidator{v1}) + fc.PersistentStateDir = utils.None[string]() + cfgFile := writeAutobahnConfig(t, fc) + nodeKey := makeTestNodeKey([]byte("node-seed")) + valKey := makeTestValidatorKey([]byte("val-seed")) + txMempool, genDoc := makeTestGigaDeps() + + result, err := buildValidatorGigaConfig(cfgFile, nodeKey, valKey, txMempool, genDoc) + require.NoError(t, err) + assert.False(t, result.PersistentStateDir.IsPresent()) +} + +func TestBuildGigaConfig_BlockDBOverrides(t *testing.T) { + v1 := makeValidator([]byte("val-seed"), []byte("node-seed"), "localhost:26660") + fc := defaultFileConfig(t, []config.AutobahnValidator{v1}) + fc.PersistentStateDir = utils.Some("data/autobahn") + fc.BlockDB = config.AutobahnBlockDBConfig{ + Retention: utils.Some(utils.Duration(30 * time.Second)), + GCPeriod: utils.Some(utils.Duration(5 * time.Second)), + } + cfgFile := writeAutobahnConfig(t, fc) + nodeKey := makeTestNodeKey([]byte("node-seed")) + valKey := makeTestValidatorKey([]byte("val-seed")) + txMempool, genDoc := makeTestGigaDeps() + + result, err := buildValidatorGigaConfig(cfgFile, nodeKey, valKey, txMempool, genDoc) + require.NoError(t, err) + require.NoError(t, preparePersistentStateDir(t.TempDir(), &result.GigaRouterCommonConfig)) + dir, ok := result.PersistentStateDir.Get() + require.True(t, ok) + littCfg, err := fc.BlockDB.LittBlockConfig(filepath.Join(dir, "blockdb")) + require.NoError(t, err) + require.NotNil(t, littCfg.Litt) + assert.Equal(t, 30*time.Second, littCfg.Retention) + assert.Equal(t, 5*time.Second, littCfg.Litt.GCPeriod) + assert.True(t, littCfg.Litt.Fsync) +} + +func TestBuildGigaConfig_BlockDBOmittedKeepsDefaults(t *testing.T) { + v1 := makeValidator([]byte("val-seed"), []byte("node-seed"), "localhost:26660") + fc := defaultFileConfig(t, []config.AutobahnValidator{v1}) + fc.PersistentStateDir = utils.Some("data/autobahn") + cfgFile := writeAutobahnConfig(t, fc) + nodeKey := makeTestNodeKey([]byte("node-seed")) + valKey := makeTestValidatorKey([]byte("val-seed")) + txMempool, genDoc := makeTestGigaDeps() + + result, err := buildValidatorGigaConfig(cfgFile, nodeKey, valKey, txMempool, genDoc) + require.NoError(t, err) + require.NoError(t, preparePersistentStateDir(t.TempDir(), &result.GigaRouterCommonConfig)) + dir, ok := result.PersistentStateDir.Get() + require.True(t, ok) + littCfg, err := config.AutobahnBlockDBConfig{}.LittBlockConfig(filepath.Join(dir, "blockdb")) + require.NoError(t, err) + require.NotNil(t, littCfg.Litt) + assert.Equal(t, 24*time.Hour, littCfg.Retention) + assert.True(t, littCfg.Litt.Fsync) +} + func TestBuildGigaConfig_EmptyPathErrors(t *testing.T) { nodeKey := makeTestNodeKey([]byte("test-node-key")) valKey := makeTestValidatorKey([]byte("val-seed")) @@ -122,9 +183,7 @@ func TestBuildGigaConfig_EnabledWithValidators(t *testing.T) { assert.Equal(t, 5*time.Second, result.DialInterval) assert.Equal(t, 3*time.Second, result.ViewTimeout(atypes.View{})) - persistDir, ok := result.PersistentStateDir.Get() - require.True(t, ok) - assert.Equal(t, "/tmp/autobahn-state", persistDir) + assert.Equal(t, utils.Some("/tmp/autobahn-state"), result.PersistentStateDir) // Verify the validator key is derived from the validator-key seed, not the node key. expectedValPub := makeTestValidatorKey([]byte("val1-seed")).Public() @@ -145,7 +204,7 @@ func TestBuildGigaConfig_EnabledWithValidators(t *testing.T) { func TestBuildGigaConfig_NoneMaxTxsPerSecond(t *testing.T) { v1 := makeValidator([]byte("val-seed"), []byte("node-seed"), "localhost:26660") - fc := defaultFileConfig([]config.AutobahnValidator{v1}) + fc := defaultFileConfig(t, []config.AutobahnValidator{v1}) cfgFile := writeAutobahnConfig(t, fc) nodeKey := makeTestNodeKey([]byte("node-seed")) valKey := makeTestValidatorKey([]byte("val-seed")) @@ -157,19 +216,6 @@ func TestBuildGigaConfig_NoneMaxTxsPerSecond(t *testing.T) { assert.False(t, result.Producer.MaxTxsPerSecond.IsPresent()) } -func TestBuildGigaConfig_NonePersistentStateDir(t *testing.T) { - v1 := makeValidator([]byte("val-seed"), []byte("node-seed"), "localhost:26660") - fc := defaultFileConfig([]config.AutobahnValidator{v1}) - cfgFile := writeAutobahnConfig(t, fc) - nodeKey := makeTestNodeKey([]byte("node-seed")) - valKey := makeTestValidatorKey([]byte("val-seed")) - txMempool, genDoc := makeTestGigaDeps() - - result, err := buildValidatorGigaConfig(cfgFile, nodeKey, valKey, txMempool, genDoc) - require.NoError(t, err) - assert.False(t, result.PersistentStateDir.IsPresent()) -} - func TestBuildGigaConfig_InvalidConfigFile(t *testing.T) { nodeKey := makeTestNodeKey([]byte("node-seed")) valKey := makeTestValidatorKey([]byte("val-seed")) @@ -188,7 +234,7 @@ func TestBuildGigaConfig_InvalidConfigFile(t *testing.T) { }) t.Run("empty validators", func(t *testing.T) { - fc := defaultFileConfig([]config.AutobahnValidator{}) + fc := defaultFileConfig(t, []config.AutobahnValidator{}) cfgFile := writeAutobahnConfig(t, fc) _, err := buildValidatorGigaConfig(cfgFile, nodeKey, valKey, txMempool, genDoc) assert.Error(t, err) @@ -202,7 +248,7 @@ func TestBuildGigaConfig_GenesisMaxGas(t *testing.T) { nodeKey := makeTestNodeKey([]byte("node-seed")) valKey := makeTestValidatorKey([]byte("val-seed")) v := makeValidator([]byte("val-seed"), []byte("node-seed"), "localhost:26660") - cfgFile := writeAutobahnConfig(t, defaultFileConfig([]config.AutobahnValidator{v})) + cfgFile := writeAutobahnConfig(t, defaultFileConfig(t, []config.AutobahnValidator{v})) t.Run("nil ConsensusParams", func(t *testing.T) { txMempool, genDoc := makeTestGigaDeps() @@ -229,7 +275,7 @@ func TestBuildGigaConfig_GenesisMaxGas(t *testing.T) { func TestBuildGigaConfig_DuplicateValidatorKey(t *testing.T) { v1 := makeValidator([]byte("val-seed"), []byte("node1"), "localhost:26660") v1dup := makeValidator([]byte("val-seed"), []byte("node2"), "localhost:26661") - fc := defaultFileConfig([]config.AutobahnValidator{v1, v1dup}) + fc := defaultFileConfig(t, []config.AutobahnValidator{v1, v1dup}) data, _ := json.Marshal(fc) path := filepath.Join(t.TempDir(), "autobahn.json") os.WriteFile(path, data, 0644) @@ -245,7 +291,7 @@ func TestBuildGigaConfig_DuplicateValidatorKey(t *testing.T) { func TestBuildGigaConfig_DuplicateNodeKey(t *testing.T) { v1 := makeValidator([]byte("val1"), []byte("same-node"), "localhost:26660") v2 := makeValidator([]byte("val2"), []byte("same-node"), "localhost:26661") - fc := defaultFileConfig([]config.AutobahnValidator{v1, v2}) + fc := defaultFileConfig(t, []config.AutobahnValidator{v1, v2}) data, _ := json.Marshal(fc) path := filepath.Join(t.TempDir(), "autobahn.json") os.WriteFile(path, data, 0644) @@ -260,7 +306,7 @@ func TestBuildGigaConfig_DuplicateNodeKey(t *testing.T) { func TestBuildGigaConfig_SelfNotInValidators(t *testing.T) { v1 := makeValidator([]byte("other-val"), []byte("other-node"), "localhost:26660") - cfgFile := writeAutobahnConfig(t, defaultFileConfig([]config.AutobahnValidator{v1})) + cfgFile := writeAutobahnConfig(t, defaultFileConfig(t, []config.AutobahnValidator{v1})) nodeKey := makeTestNodeKey([]byte("my-node")) valKey := makeTestValidatorKey([]byte("my-val")) txMempool, genDoc := makeTestGigaDeps() @@ -273,7 +319,7 @@ func TestBuildGigaConfig_SelfNotInValidators(t *testing.T) { func TestBuildGigaConfig_NodeKeyMismatch(t *testing.T) { // Validator entry has the right val key but wrong node key. v1 := makeValidator([]byte("my-val"), []byte("wrong-node"), "localhost:26660") - cfgFile := writeAutobahnConfig(t, defaultFileConfig([]config.AutobahnValidator{v1})) + cfgFile := writeAutobahnConfig(t, defaultFileConfig(t, []config.AutobahnValidator{v1})) nodeKey := makeTestNodeKey([]byte("my-node")) valKey := makeTestValidatorKey([]byte("my-val")) txMempool, genDoc := makeTestGigaDeps() @@ -282,3 +328,20 @@ func TestBuildGigaConfig_NodeKeyMismatch(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "node key mismatch") } + +func TestMakeCloser_NoErrorsReturnsNil(t *testing.T) { + cl := makeCloser([]closer{ + func() error { return nil }, + func() error { return nil }, + }) + require.NoError(t, cl()) +} + +func TestPreparePersistentStateDir_EmptyStringIsNone(t *testing.T) { + cfg := &p2p.GigaRouterCommonConfig{ + PersistentStateDir: utils.Some(""), + } + require.NoError(t, preparePersistentStateDir(t.TempDir(), cfg)) + _, ok := cfg.PersistentStateDir.Get() + require.False(t, ok, "Some(\"\") must be cleared to None for in-memory mode") +} diff --git a/sei-tendermint/rpc/coretypes/responses.go b/sei-tendermint/rpc/coretypes/responses.go index 48817edc75..f158926a3e 100644 --- a/sei-tendermint/rpc/coretypes/responses.go +++ b/sei-tendermint/rpc/coretypes/responses.go @@ -35,7 +35,7 @@ var ( // BlockStore.Base() in env.getHeight. Pass None when the caller only knows // the height was unavailable but not what the bound is — e.g. the Autobahn // path where data.GlobalBlock returns data.ErrPruned and the bound -// (data.State.inner.first) is internal to data.State. +// (BlockDB's prune watermark) is internal to the store. // // Centralizing this format means both paths produce identical error // strings for the same case modulo the optional base height, so callers