From e04092ddbb58440d659421eb8506d61be6ca4b6a Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 11 May 2026 14:05:47 -0400 Subject: [PATCH 01/41] Add Scylla historical state offload prototype --- app/seidb.go | 16 + app/seidb_test.go | 38 +++ go.mod | 3 + go.sum | 8 + sei-db/config/ss_config.go | 21 ++ sei-db/config/toml.go | 11 + sei-db/config/toml_test.go | 4 + sei-db/state_db/ss/offload/consumer/README.md | 73 +++++ .../cmd/historical-scylla-consumer/main.go | 51 +++ sei-db/state_db/ss/offload/consumer/config.go | 54 ++++ .../consumer/config/example-scylla.json | 21 ++ .../state_db/ss/offload/consumer/consumer.go | 297 ++++++++++++++++++ sei-db/state_db/ss/offload/consumer/kafka.go | 115 +++++++ .../ss/offload/consumer/kafka_test.go | 46 +++ .../ss/offload/consumer/schema/scylla.cql | 46 +++ sei-db/state_db/ss/offload/consumer/scylla.go | 201 ++++++++++++ .../ss/offload/consumer/scylla_test.go | 68 ++++ sei-db/state_db/ss/offload/consumer/sink.go | 26 ++ .../state_db/ss/offload/historical/reader.go | 37 +++ .../state_db/ss/offload/historical/scylla.go | 239 ++++++++++++++ .../ss/offload/historical/scylla_test.go | 95 ++++++ .../state_db/ss/offload/historical/store.go | 96 ++++++ .../ss/offload/historical/store_test.go | 103 ++++++ sei-db/state_db/ss/offload/kafka.go | 5 +- sei-db/state_db/ss/store.go | 47 ++- 25 files changed, 1718 insertions(+), 3 deletions(-) create mode 100644 sei-db/state_db/ss/offload/consumer/README.md create mode 100644 sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go create mode 100644 sei-db/state_db/ss/offload/consumer/config.go create mode 100644 sei-db/state_db/ss/offload/consumer/config/example-scylla.json create mode 100644 sei-db/state_db/ss/offload/consumer/consumer.go create mode 100644 sei-db/state_db/ss/offload/consumer/kafka.go create mode 100644 sei-db/state_db/ss/offload/consumer/kafka_test.go create mode 100644 sei-db/state_db/ss/offload/consumer/schema/scylla.cql create mode 100644 sei-db/state_db/ss/offload/consumer/scylla.go create mode 100644 sei-db/state_db/ss/offload/consumer/scylla_test.go create mode 100644 sei-db/state_db/ss/offload/consumer/sink.go create mode 100644 sei-db/state_db/ss/offload/historical/reader.go create mode 100644 sei-db/state_db/ss/offload/historical/scylla.go create mode 100644 sei-db/state_db/ss/offload/historical/scylla_test.go create mode 100644 sei-db/state_db/ss/offload/historical/store.go create mode 100644 sei-db/state_db/ss/offload/historical/store_test.go diff --git a/app/seidb.go b/app/seidb.go index 4ed31b2d7e..6be013875d 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -45,6 +45,15 @@ const ( FlagEVMSSSplit = "state-store.evm-ss-split" FlagEVMSSSeparateDBs = "state-store.evm-ss-separate-dbs" + // Historical SS offload fallback. + FlagHistoricalOffloadScyllaHosts = "state-store.historical-offload-scylla-hosts" + FlagHistoricalOffloadScyllaKeyspace = "state-store.historical-offload-scylla-keyspace" + FlagHistoricalOffloadScyllaUsername = "state-store.historical-offload-scylla-username" + FlagHistoricalOffloadScyllaPassword = "state-store.historical-offload-scylla-password" + FlagHistoricalOffloadScyllaDatacenter = "state-store.historical-offload-scylla-datacenter" + FlagHistoricalOffloadScyllaConsistency = "state-store.historical-offload-scylla-consistency" + FlagHistoricalOffloadScyllaTimeoutMS = "state-store.historical-offload-scylla-timeout-ms" + // Other configs FlagSnapshotInterval = "state-sync.snapshot-interval" ) @@ -148,6 +157,13 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig { ssConfig.EVMDBDirectory = cast.ToString(appOpts.Get(FlagEVMSSDirectory)) ssConfig.SeparateEVMSubDBs = cast.ToBool(appOpts.Get(FlagEVMSSSeparateDBs)) ssConfig.EVMSplit = cast.ToBool(appOpts.Get(FlagEVMSSSplit)) + ssConfig.HistoricalOffloadScyllaHosts = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaHosts)) + ssConfig.HistoricalOffloadScyllaKeyspace = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaKeyspace)) + ssConfig.HistoricalOffloadScyllaUsername = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaUsername)) + ssConfig.HistoricalOffloadScyllaPassword = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaPassword)) + ssConfig.HistoricalOffloadScyllaDatacenter = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaDatacenter)) + ssConfig.HistoricalOffloadScyllaConsistency = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaConsistency)) + ssConfig.HistoricalOffloadScyllaTimeoutMS = cast.ToInt(appOpts.Get(FlagHistoricalOffloadScyllaTimeoutMS)) return ssConfig } diff --git a/app/seidb_test.go b/app/seidb_test.go index 54b3eb6027..d0a0c88dc1 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -61,6 +61,20 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSSConfig.EVMSplit case FlagEVMSSSeparateDBs: return defaultSSConfig.SeparateEVMSubDBs + case FlagHistoricalOffloadScyllaHosts: + return defaultSSConfig.HistoricalOffloadScyllaHosts + case FlagHistoricalOffloadScyllaKeyspace: + return defaultSSConfig.HistoricalOffloadScyllaKeyspace + case FlagHistoricalOffloadScyllaUsername: + return defaultSSConfig.HistoricalOffloadScyllaUsername + case FlagHistoricalOffloadScyllaPassword: + return defaultSSConfig.HistoricalOffloadScyllaPassword + case FlagHistoricalOffloadScyllaDatacenter: + return defaultSSConfig.HistoricalOffloadScyllaDatacenter + case FlagHistoricalOffloadScyllaConsistency: + return defaultSSConfig.HistoricalOffloadScyllaConsistency + case FlagHistoricalOffloadScyllaTimeoutMS: + return defaultSSConfig.HistoricalOffloadScyllaTimeoutMS } return nil } @@ -114,6 +128,30 @@ func TestParseSSConfigs_EVMFlags(t *testing.T) { assert.True(t, ssConfig.SeparateEVMSubDBs) } +func TestParseSSConfigs_HistoricalScyllaFlags(t *testing.T) { + appOpts := mapAppOpts{ + FlagSSEnable: true, + FlagHistoricalOffloadScyllaHosts: "10.0.0.1:9042,10.0.0.2:9042", + FlagHistoricalOffloadScyllaKeyspace: "sei_history", + FlagHistoricalOffloadScyllaUsername: "sei", + FlagHistoricalOffloadScyllaPassword: "secret", + FlagHistoricalOffloadScyllaDatacenter: "use1", + FlagHistoricalOffloadScyllaConsistency: "local_quorum", + FlagHistoricalOffloadScyllaTimeoutMS: 1500, + FlagSSAsyncWriterBuffer: 0, + } + + ssConfig := parseSSConfigs(appOpts) + assert.True(t, ssConfig.Enable) + assert.Equal(t, "10.0.0.1:9042,10.0.0.2:9042", ssConfig.HistoricalOffloadScyllaHosts) + assert.Equal(t, "sei_history", ssConfig.HistoricalOffloadScyllaKeyspace) + assert.Equal(t, "sei", ssConfig.HistoricalOffloadScyllaUsername) + assert.Equal(t, "secret", ssConfig.HistoricalOffloadScyllaPassword) + assert.Equal(t, "use1", ssConfig.HistoricalOffloadScyllaDatacenter) + assert.Equal(t, "local_quorum", ssConfig.HistoricalOffloadScyllaConsistency) + assert.Equal(t, 1500, ssConfig.HistoricalOffloadScyllaTimeoutMS) +} + func TestParseReceiptConfigs_DefaultsToPebbleWhenUnset(t *testing.T) { receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{}) assert.NoError(t, err) diff --git a/go.mod b/go.mod index db681f7bf4..dae2d00b1a 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.1 github.com/golang/mock v1.7.0-rc.1 github.com/golang/protobuf v1.5.4 + github.com/gocql/gocql v1.7.0 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.7.0 github.com/google/gofuzz v1.2.0 @@ -119,6 +120,7 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.8.0 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -128,6 +130,7 @@ require ( github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8ab76b9ab4..b941802165 100644 --- a/go.sum +++ b/go.sum @@ -736,6 +736,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E= github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.14.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.17.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= @@ -743,6 +744,7 @@ github.com/bits-and-blooms/bitset v1.24.3 h1:Bte86SlO3lwPQqww+7BE9ZuUCKIjfqnG5jt github.com/bits-and-blooms/bitset v1.24.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/btcsuite/btcd v0.23.2 h1:/YOgUp25sdCnP5ho6Hl3s0E438zlX+Kak7E6TgBgoT0= @@ -1122,6 +1124,8 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= +github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -1328,6 +1332,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/guptarohit/asciigraph v0.5.5/go.mod h1:dYl5wwK4gNsnFf9Zp+l06rFiDZ5YtXM6x7SRWZ3KGag= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -2911,6 +2917,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index 3fe94e750d..09fd69117f 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -76,6 +76,27 @@ type StateStoreConfig struct { // When true, data is routed to separate DBs by EVM key family while // preserving the same logical store key and full key encoding inside each DB. SeparateEVMSubDBs bool `mapstructure:"evm-separate-dbs"` + + // HistoricalOffloadScyllaHosts enables ScyllaDB/Cassandra fallback reads + // for versions pruned from local SS when non-empty. Hosts are comma-separated + // host[:port] values. + HistoricalOffloadScyllaHosts string `mapstructure:"historical-offload-scylla-hosts"` + + // HistoricalOffloadScyllaKeyspace is the keyspace containing state_mutations. + HistoricalOffloadScyllaKeyspace string `mapstructure:"historical-offload-scylla-keyspace"` + + // HistoricalOffloadScyllaUsername and Password are optional. + HistoricalOffloadScyllaUsername string `mapstructure:"historical-offload-scylla-username"` + HistoricalOffloadScyllaPassword string `mapstructure:"historical-offload-scylla-password"` + + // HistoricalOffloadScyllaDatacenter enables DC-aware routing when set. + HistoricalOffloadScyllaDatacenter string `mapstructure:"historical-offload-scylla-datacenter"` + + // HistoricalOffloadScyllaConsistency defaults to local_quorum when empty. + HistoricalOffloadScyllaConsistency string `mapstructure:"historical-offload-scylla-consistency"` + + // HistoricalOffloadScyllaTimeoutMS defaults in the Scylla reader when zero. + HistoricalOffloadScyllaTimeoutMS int `mapstructure:"historical-offload-scylla-timeout-ms"` } // DefaultStateStoreConfig returns the default StateStoreConfig diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index eb387cb1d5..68e14fce7a 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -139,6 +139,17 @@ evm-ss-split = {{ .StateStore.EVMSplit }} # When false, all EVM data stays in one DB using the current unified layout. # When true, data is routed to separate DBs while preserving the same evm key prefix format. evm-ss-separate-dbs = {{ .StateStore.SeparateEVMSubDBs }} + +# Optional ScyllaDB/Cassandra historical-state fallback. When hosts are set, +# point reads for versions pruned from local SS fall back to state_mutations in +# the configured keyspace. Iterators still use local SS. +historical-offload-scylla-hosts = "{{ .StateStore.HistoricalOffloadScyllaHosts }}" +historical-offload-scylla-keyspace = "{{ .StateStore.HistoricalOffloadScyllaKeyspace }}" +historical-offload-scylla-username = "{{ .StateStore.HistoricalOffloadScyllaUsername }}" +historical-offload-scylla-password = "{{ .StateStore.HistoricalOffloadScyllaPassword }}" +historical-offload-scylla-datacenter = "{{ .StateStore.HistoricalOffloadScyllaDatacenter }}" +historical-offload-scylla-consistency = "{{ .StateStore.HistoricalOffloadScyllaConsistency }}" +historical-offload-scylla-timeout-ms = {{ .StateStore.HistoricalOffloadScyllaTimeoutMS }} ` // ReceiptStoreConfigTemplate defines the configuration template for receipt-store diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index fd0a51f932..b6bd267faf 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -88,6 +88,10 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, `evm-ss-db-directory = ""`, "Missing evm-ss-db-directory") require.Contains(t, output, `evm-ss-split = false`, "Missing or incorrect evm-ss-split") require.Contains(t, output, "evm-ss-separate-dbs = false", "Missing or incorrect evm-ss-separate-dbs") + require.Contains(t, output, `historical-offload-scylla-hosts = ""`, "Missing historical Scylla hosts") + require.Contains(t, output, `historical-offload-scylla-keyspace = ""`, "Missing historical Scylla keyspace") + require.Contains(t, output, `historical-offload-scylla-consistency = ""`, "Missing historical Scylla consistency") + require.Contains(t, output, "historical-offload-scylla-timeout-ms = 0", "Missing historical Scylla timeout") } // TestReceiptStoreConfigTemplate verifies that all field paths in the receipt-store TOML template diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md new file mode 100644 index 0000000000..9ccc28b29d --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -0,0 +1,73 @@ +# Historical Scylla/Cassandra Offload + +This is a prototype historical-state backend for ScyllaDB or Cassandra. + +The intended shape is narrow: + +- local SS remains the hot store for recent state, writes, imports, pruning, and iterators +- Scylla/Cassandra stores immutable MVCC mutation rows for older history +- reads below local SS retention can fall back to Scylla/Cassandra for `Get` and `Has` + +The table layout is built for point reads by `(store_name, state_key, target_version)`: + +```sql +SELECT version, value, deleted +FROM state_mutations +WHERE store_name = ? AND state_key = ? AND version <= ? +ORDER BY version DESC +LIMIT 1; +``` + +Ordered prefix iteration is intentionally not served from Scylla/Cassandra in this prototype. + +## Schema + +Apply the schema once: + +```bash +cqlsh 127.0.0.1 9042 -f sei-db/state_db/ss/offload/consumer/schema/scylla.cql +``` + +For production, edit the keyspace replication in `schema/scylla.cql` to use +`NetworkTopologyStrategy` with the actual datacenter names and replication +factors before applying it. + +## Consumer + +The consumer reads historical offload changelog messages from Kafka and writes +them into Scylla/Cassandra. Kafka offsets are committed only after the sink +write succeeds. + +```bash +go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer \ + ./sei-db/state_db/ss/offload/consumer/config/example-scylla.json +``` + +The example config is local-dev only. Set real Kafka brokers, Scylla hosts, +keyspace, datacenter, and credentials in your own config. + +## Node Read Fallback + +Enable fallback reads in the node config: + +```toml +[state-store] +historical-offload-scylla-hosts = "10.0.0.1:9042,10.0.0.2:9042" +historical-offload-scylla-keyspace = "sei_history" +historical-offload-scylla-username = "" +historical-offload-scylla-password = "" +historical-offload-scylla-datacenter = "datacenter1" +historical-offload-scylla-consistency = "local_quorum" +historical-offload-scylla-timeout-ms = 2000 +``` + +Fallback activates only for point reads where the requested version is below the +local SS earliest version. Missing rows and tombstones return empty state, same +as local SS. + +## Current Limits + +- No Scylla/Cassandra iterator path. +- No cross-row transaction on ingest; mutation rows are written first and the + version marker is written last, so replay is idempotent after partial failure. +- No automatic schema creation from the binary. diff --git a/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go b/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go new file mode 100644 index 0000000000..65020ca354 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/consumer" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "usage: %s \n", os.Args[0]) + os.Exit(2) + } + + cfg, err := consumer.LoadConfig(os.Args[1]) + if err != nil { + log.Fatalf("load config: %v", err) + } + + sink, err := consumer.NewScyllaSink(cfg.Scylla) + if err != nil { + log.Fatalf("open scylla/cassandra sink: %v", err) + } + defer func() { _ = sink.Close() }() + + reader, err := consumer.NewKafkaReader(cfg.Kafka) + if err != nil { + log.Fatalf("open kafka reader: %v", err) + } + defer func() { _ = reader.Close() }() + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + c := consumer.New(reader, sink, consumer.Options{ + Logf: func(format string, args ...interface{}) { log.Printf(format, args...) }, + Workers: cfg.Workers, + ShardBufferSize: cfg.ShardBufferSize, + MaxBatchRecords: cfg.MaxBatchRecords, + BatchMaxWait: time.Duration(cfg.BatchMaxWaitMS) * time.Millisecond, + }) + if err := c.Run(ctx); err != nil { + log.Fatalf("consumer: %v", err) + } +} diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go new file mode 100644 index 0000000000..f381b2c11f --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -0,0 +1,54 @@ +package consumer + +import ( + "encoding/json" + "fmt" + "os" +) + +type Config struct { + Kafka KafkaReaderConfig + Scylla ScyllaConfig + Workers int + ShardBufferSize int + MaxBatchRecords int + BatchMaxWaitMS int +} + +func (c *Config) Validate() error { + if err := c.Kafka.Validate(); err != nil { + return fmt.Errorf("kafka: %w", err) + } + if err := c.Scylla.Validate(); err != nil { + return fmt.Errorf("scylla: %w", err) + } + if c.Workers < 0 { + return fmt.Errorf("workers must be non-negative") + } + if c.ShardBufferSize < 0 { + return fmt.Errorf("shard buffer size must be non-negative") + } + if c.MaxBatchRecords < 0 { + return fmt.Errorf("max batch records must be non-negative") + } + if c.BatchMaxWaitMS < 0 { + return fmt.Errorf("batch max wait ms must be non-negative") + } + return nil +} + +func LoadConfig(path string) (*Config, error) { + // #nosec G304 -- config path is supplied by the operator on the command line. + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read config: %w", err) + } + cfg := &Config{} + if err := json.Unmarshal(raw, cfg); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + if err := cfg.Validate(); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/sei-db/state_db/ss/offload/consumer/config/example-scylla.json b/sei-db/state_db/ss/offload/consumer/config/example-scylla.json new file mode 100644 index 0000000000..94779c83e9 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/config/example-scylla.json @@ -0,0 +1,21 @@ +{ + "Kafka": { + "Brokers": ["localhost:9092"], + "Topic": "historical-offload", + "GroupID": "historical-scylla", + "StartOffset": "first" + }, + "Scylla": { + "Hosts": ["127.0.0.1:9042"], + "Keyspace": "sei_history", + "Datacenter": "datacenter1", + "Consistency": "local_quorum", + "TimeoutMS": 2000, + "ConnectTimeoutMS": 2000, + "NumConns": 4 + }, + "Workers": 16, + "ShardBufferSize": 128, + "MaxBatchRecords": 16, + "BatchMaxWaitMS": 10 +} diff --git a/sei-db/state_db/ss/offload/consumer/consumer.go b/sei-db/state_db/ss/offload/consumer/consumer.go new file mode 100644 index 0000000000..a0adaaca46 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/consumer.go @@ -0,0 +1,297 @@ +package consumer + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/segmentio/kafka-go" + "golang.org/x/sync/errgroup" +) + +// MessageSource is the subset of *kafka.Reader used by the loop. +type MessageSource interface { + FetchMessage(ctx context.Context) (kafka.Message, error) + CommitMessages(ctx context.Context, msgs ...kafka.Message) error +} + +// Messages are sharded by partition: cross-partition writes parallelize while +// ordering within a partition is preserved. +type Consumer struct { + reader MessageSource + sink Sink + logf func(format string, args ...interface{}) + workers int + shardBuf int + batchSize int + batchWait time.Duration + maxAttempts int + baseBackoff time.Duration + maxBackoff time.Duration +} + +const ( + defaultSinkMaxAttempts = 5 + defaultSinkBaseBackoff = 1 * time.Second + defaultSinkMaxBackoff = 30 * time.Second + defaultWorkers = 16 + defaultShardBuffer = 128 + defaultBatchSize = 16 + defaultBatchMaxWait = 10 * time.Millisecond +) + +// Backpressure: when the sink falls behind, ShardBufferSize fills, the fetcher +// blocks, and Kafka stops being polled. Zero values pick defaults. +type Options struct { + Logf func(format string, args ...interface{}) + SinkMaxAttempts int + SinkBaseBackoff time.Duration + SinkMaxBackoff time.Duration + Workers int + ShardBufferSize int + MaxBatchRecords int + BatchMaxWait time.Duration +} + +func New(reader MessageSource, sink Sink, opts Options) *Consumer { + logf := opts.Logf + if logf == nil { + logf = func(string, ...interface{}) {} + } + maxAttempts := opts.SinkMaxAttempts + if maxAttempts <= 0 { + maxAttempts = defaultSinkMaxAttempts + } + base := opts.SinkBaseBackoff + if base <= 0 { + base = defaultSinkBaseBackoff + } + maxBackoff := opts.SinkMaxBackoff + if maxBackoff <= 0 { + maxBackoff = defaultSinkMaxBackoff + } + workers := opts.Workers + if workers <= 0 { + workers = defaultWorkers + } + shardBuf := opts.ShardBufferSize + if shardBuf <= 0 { + shardBuf = defaultShardBuffer + } + batchSize := opts.MaxBatchRecords + if batchSize <= 0 { + batchSize = defaultBatchSize + } + batchWait := opts.BatchMaxWait + if batchWait <= 0 { + batchWait = defaultBatchMaxWait + } + return &Consumer{ + reader: reader, + sink: sink, + logf: logf, + workers: workers, + shardBuf: shardBuf, + batchSize: batchSize, + batchWait: batchWait, + maxAttempts: maxAttempts, + baseBackoff: base, + maxBackoff: maxBackoff, + } +} + +// Run commits offsets only after the sink persists each message. +func (c *Consumer) Run(ctx context.Context) error { + return c.runParallel(ctx) +} + +func (c *Consumer) runParallel(ctx context.Context) error { + g, gctx := errgroup.WithContext(ctx) + shards := make([]chan kafka.Message, c.workers) + for i := range shards { + shards[i] = make(chan kafka.Message, c.shardBuf) + ch := shards[i] + g.Go(func() error { return c.workerLoop(gctx, ch) }) + } + g.Go(func() error { + defer func() { + for _, ch := range shards { + close(ch) + } + }() + for { + msg, err := c.reader.FetchMessage(gctx) + if err != nil { + if isCancellation(err) { + return nil + } + return fmt.Errorf("fetch kafka message: %w", err) + } + shard := shardFor(msg.Partition, c.workers) + select { + case shards[shard] <- msg: + case <-gctx.Done(): + return nil + } + } + }) + if err := g.Wait(); err != nil && !isCancellation(err) { + return err + } + return nil +} + +func (c *Consumer) workerLoop(ctx context.Context, ch <-chan kafka.Message) error { + for { + select { + case <-ctx.Done(): + return nil + case msg, ok := <-ch: + if !ok { + return nil + } + msgs, ok := c.collectBatch(ctx, ch, msg) + if !ok { + return nil + } + if err := c.processBatch(ctx, msgs); err != nil { + if isCancellation(err) { + return nil + } + return err + } + } + } +} + +func (c *Consumer) collectBatch(ctx context.Context, ch <-chan kafka.Message, first kafka.Message) ([]kafka.Message, bool) { + msgs := make([]kafka.Message, 1, c.batchSize) + msgs[0] = first + if c.batchSize <= 1 { + return msgs, true + } + +drainBuffered: + for len(msgs) < c.batchSize { + select { + case <-ctx.Done(): + return nil, false + case msg, ok := <-ch: + if !ok { + return msgs, true + } + msgs = append(msgs, msg) + default: + break drainBuffered + } + } + if len(msgs) == c.batchSize { + return msgs, true + } + + timer := time.NewTimer(c.batchWait) + defer timer.Stop() + for len(msgs) < c.batchSize { + select { + case <-ctx.Done(): + return nil, false + case msg, ok := <-ch: + if !ok { + return msgs, true + } + msgs = append(msgs, msg) + case <-timer.C: + return msgs, true + } + } + return msgs, true +} + +func (c *Consumer) processBatch(ctx context.Context, msgs []kafka.Message) error { + records := make([]Record, 0, len(msgs)) + var firstVersion, lastVersion int64 + for i, msg := range msgs { + entry, err := DecodeEntry(msg.Value) + if err != nil { + return fmt.Errorf("decode message at offset %d: %w", msg.Offset, err) + } + if i == 0 { + firstVersion = entry.Version + } + lastVersion = entry.Version + records = append(records, Record{ + Topic: msg.Topic, + Partition: msg.Partition, + Offset: msg.Offset, + Entry: entry, + }) + } + start := time.Now() + if err := c.writeBatchWithRetry(ctx, records); err != nil { + return fmt.Errorf("sink write batch first_version=%d last_version=%d: %w", + firstVersion, lastVersion, err) + } + c.logf("wrote records=%d first_version=%d last_version=%d in %s", + len(records), firstVersion, lastVersion, time.Since(start)) + if err := c.reader.CommitMessages(ctx, msgs...); err != nil { + return fmt.Errorf("commit kafka offsets: %w", err) + } + return nil +} + +func (c *Consumer) writeBatchWithRetry(ctx context.Context, records []Record) error { + backoff := c.baseBackoff + var lastErr error + for attempt := 1; attempt <= c.maxAttempts; attempt++ { + err := c.writeRecords(ctx, records) + if err == nil { + return nil + } + lastErr = err + if isCancellation(err) { + return err + } + if attempt == c.maxAttempts { + break + } + c.logf("sink write attempt %d/%d failed: %v; retrying in %s", + attempt, c.maxAttempts, err, backoff) + select { + case <-time.After(backoff): + case <-ctx.Done(): + return ctx.Err() + } + backoff *= 2 + if backoff > c.maxBackoff { + backoff = c.maxBackoff + } + } + return fmt.Errorf("sink write failed after %d attempts: %w", c.maxAttempts, lastErr) +} + +func (c *Consumer) writeRecords(ctx context.Context, records []Record) error { + if len(records) == 0 { + return nil + } + if sink, ok := c.sink.(BatchSink); ok { + return sink.WriteBatch(ctx, records) + } + for _, rec := range records { + if err := c.sink.Write(ctx, rec); err != nil { + return err + } + } + return nil +} + +func shardFor(partition, workers int) int { + if partition < 0 { + partition = -partition + } + return partition % workers +} + +func isCancellation(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} diff --git a/sei-db/state_db/ss/offload/consumer/kafka.go b/sei-db/state_db/ss/offload/consumer/kafka.go new file mode 100644 index 0000000000..1785bb05f8 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/kafka.go @@ -0,0 +1,115 @@ +package consumer + +import ( + "crypto/tls" + "fmt" + "strings" + "time" + + gogoproto "github.com/gogo/protobuf/proto" + "github.com/segmentio/kafka-go" + + dbproto "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload" +) + +// TLS/SASL must match the producer cluster. Commits are synchronous +// (kafka-go's zero CommitInterval) so offsets only advance after the sink +// persists each entry. +type KafkaReaderConfig struct { + Brokers []string + Topic string + GroupID string + ClientID string + Region string + StartOffset string // "first" or "last"; defaults to "first" + MinBytes int + MaxBytes int + MaxWait time.Duration + TLSEnabled bool + SASLMechanism string +} + +func (c *KafkaReaderConfig) ApplyDefaults() { + if c.ClientID == "" { + c.ClientID = "cryptosim-historical-scylla-consumer" + } + if c.StartOffset == "" { + c.StartOffset = "first" + } + if c.MinBytes == 0 { + c.MinBytes = 1 + } + if c.MaxBytes == 0 { + c.MaxBytes = 10 << 20 + } + if c.MaxWait == 0 { + c.MaxWait = 500 * time.Millisecond + } +} + +func (c *KafkaReaderConfig) Validate() error { + if len(c.Brokers) == 0 { + return fmt.Errorf("kafka brokers are required") + } + if c.Topic == "" { + return fmt.Errorf("kafka topic is required") + } + if c.GroupID == "" { + return fmt.Errorf("kafka group id is required") + } + switch strings.ToLower(c.StartOffset) { + case "", "first", "last": + default: + return fmt.Errorf("unsupported kafka start offset %q", c.StartOffset) + } + return nil +} + +func NewKafkaReader(cfg KafkaReaderConfig) (*kafka.Reader, error) { + cfg.ApplyDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + + dialer := &kafka.Dialer{ + ClientID: cfg.ClientID, + Timeout: 10 * time.Second, + } + if cfg.TLSEnabled { + dialer.TLS = &tls.Config{MinVersion: tls.VersionTLS12} + } + mech, err := offload.NewSASLMechanism(offload.KafkaConfig{ + Region: cfg.Region, + TLSEnabled: cfg.TLSEnabled, + SASLMechanism: cfg.SASLMechanism, + }) + if err != nil { + return nil, err + } + dialer.SASLMechanism = mech + + start := kafka.FirstOffset + if strings.EqualFold(cfg.StartOffset, "last") { + start = kafka.LastOffset + } + + return kafka.NewReader(kafka.ReaderConfig{ + Brokers: cfg.Brokers, + Topic: cfg.Topic, + GroupID: cfg.GroupID, + Dialer: dialer, + MinBytes: cfg.MinBytes, + MaxBytes: cfg.MaxBytes, + MaxWait: cfg.MaxWait, + StartOffset: start, + }), nil +} + +func DecodeEntry(payload []byte) (*dbproto.ChangelogEntry, error) { + entry := &dbproto.ChangelogEntry{} + if err := gogoproto.Unmarshal(payload, entry); err != nil { + return nil, fmt.Errorf("decode changelog entry: %w", err) + } + return entry, nil +} diff --git a/sei-db/state_db/ss/offload/consumer/kafka_test.go b/sei-db/state_db/ss/offload/consumer/kafka_test.go new file mode 100644 index 0000000000..7fa6b320e4 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/kafka_test.go @@ -0,0 +1,46 @@ +package consumer + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" +) + +func TestKafkaReaderConfigApplyDefaults(t *testing.T) { + cfg := KafkaReaderConfig{ + Brokers: []string{"localhost:9092"}, + Topic: "historical-offload", + GroupID: "scylla", + } + cfg.ApplyDefaults() + require.Equal(t, "cryptosim-historical-scylla-consumer", cfg.ClientID) + require.Equal(t, "first", cfg.StartOffset) + require.Equal(t, 1, cfg.MinBytes) + require.Equal(t, 10<<20, cfg.MaxBytes) +} + +func TestKafkaReaderConfigValidate(t *testing.T) { + cfg := KafkaReaderConfig{} + require.ErrorContains(t, cfg.Validate(), "brokers") + cfg = KafkaReaderConfig{Brokers: []string{"x"}} + require.ErrorContains(t, cfg.Validate(), "topic") + cfg = KafkaReaderConfig{Brokers: []string{"x"}, Topic: "t"} + require.ErrorContains(t, cfg.Validate(), "group id") + cfg = KafkaReaderConfig{ + Brokers: []string{"x"}, + Topic: "t", + GroupID: "g", + StartOffset: "middle", + } + require.ErrorContains(t, cfg.Validate(), "start offset") +} + +func TestDecodeEntry(t *testing.T) { + entry := &proto.ChangelogEntry{Version: 7} + payload, err := entry.Marshal() + require.NoError(t, err) + got, err := DecodeEntry(payload) + require.NoError(t, err) + require.Equal(t, int64(7), got.Version) +} diff --git a/sei-db/state_db/ss/offload/consumer/schema/scylla.cql b/sei-db/state_db/ss/offload/consumer/schema/scylla.cql new file mode 100644 index 0000000000..ceb7d3cfc2 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/schema/scylla.cql @@ -0,0 +1,46 @@ +-- ScyllaDB/Cassandra schema for Sei historical state offload. +-- Apply once before running historical-scylla-consumer. + +CREATE KEYSPACE IF NOT EXISTS sei_history +WITH replication = { + 'class': 'SimpleStrategy', + 'replication_factor': '1' +}; + +-- For production, replace the keyspace replication with +-- NetworkTopologyStrategy and the real datacenter replication factors. + +USE sei_history; + +-- Version markers are written after all mutation rows for a block version. +-- Buckets avoid a single hot partition while keeping LastVersion bounded to +-- 64 small point reads. +CREATE TABLE IF NOT EXISTS state_versions ( + bucket int, + version bigint, + kafka_topic text, + kafka_partition int, + kafka_offset bigint, + ingested_at timestamp, + PRIMARY KEY ((bucket), version) +) WITH CLUSTERING ORDER BY (version DESC); + +-- Historical point lookup: +-- WHERE store_name = ? AND state_key = ? AND version <= ? +-- ORDER BY version DESC LIMIT 1 +CREATE TABLE IF NOT EXISTS state_mutations ( + store_name text, + state_key blob, + version bigint, + value blob, + deleted boolean, + PRIMARY KEY ((store_name, state_key), version) +) WITH CLUSTERING ORDER BY (version DESC); + +CREATE TABLE IF NOT EXISTS state_tree_upgrades ( + version bigint, + name text, + rename_from text, + deleted boolean, + PRIMARY KEY ((version), name) +); diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go new file mode 100644 index 0000000000..c16744662c --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -0,0 +1,201 @@ +package consumer + +import ( + "context" + "fmt" + "time" + + "github.com/gocql/gocql" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" +) + +type ScyllaConfig struct { + Hosts []string + Keyspace string + Username string + Password string + Datacenter string + Consistency string + TimeoutMS int + ConnectTimeoutMS int + NumConns int +} + +func (c *ScyllaConfig) ApplyDefaults() { + cfg := c.toHistorical() + cfg.ApplyDefaults() + c.Consistency = cfg.Consistency + c.TimeoutMS = int(cfg.Timeout / time.Millisecond) + c.ConnectTimeoutMS = int(cfg.ConnectTimeout / time.Millisecond) + c.NumConns = cfg.NumConns +} + +func (c *ScyllaConfig) Validate() error { + cfg := c.toHistorical() + cfg.ApplyDefaults() + return cfg.Validate() +} + +func (c ScyllaConfig) toHistorical() historical.ScyllaConfig { + return historical.ScyllaConfig{ + Hosts: c.Hosts, + Keyspace: c.Keyspace, + Username: c.Username, + Password: c.Password, + Datacenter: c.Datacenter, + Consistency: c.Consistency, + Timeout: time.Duration(c.TimeoutMS) * time.Millisecond, + ConnectTimeout: time.Duration(c.ConnectTimeoutMS) * time.Millisecond, + NumConns: c.NumConns, + } +} + +type scyllaSink struct { + session *gocql.Session +} + +var _ Sink = (*scyllaSink)(nil) +var _ BatchSink = (*scyllaSink)(nil) + +func NewScyllaSink(cfg ScyllaConfig) (Sink, error) { + session, err := historical.OpenScyllaSession(cfg.toHistorical()) + if err != nil { + return nil, err + } + return &scyllaSink{session: session}, nil +} + +func (s *scyllaSink) Close() error { + s.session.Close() + return nil +} + +func (s *scyllaSink) LastVersion(ctx context.Context) (int64, error) { + var maxVersion int64 + for bucket := 0; bucket < historical.VersionBucketCount; bucket++ { + var version int64 + err := s.session.Query(selectLatestVersionCQL, bucket).WithContext(ctx).Scan(&version) + if err != nil { + if err == gocql.ErrNotFound { + continue + } + return 0, fmt.Errorf("read latest scylla/cassandra version bucket %d: %w", bucket, err) + } + if version > maxVersion { + maxVersion = version + } + } + return maxVersion, nil +} + +func (s *scyllaSink) Write(ctx context.Context, rec Record) error { + return s.WriteBatch(ctx, []Record{rec}) +} + +func (s *scyllaSink) WriteBatch(ctx context.Context, records []Record) error { + for _, rec := range compactRecords(records) { + if err := s.writeRecord(ctx, rec); err != nil { + return err + } + } + return nil +} + +func compactRecords(records []Record) []Record { + for _, rec := range records { + if rec.Entry == nil { + out := make([]Record, 0, len(records)) + for _, rec := range records { + if rec.Entry != nil { + out = append(out, rec) + } + } + return out + } + } + return records +} + +func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { + entry := rec.Entry + if entry == nil { + return nil + } + version := entry.Version + for _, ncs := range entry.Changesets { + for _, pair := range ncs.Changeset.Pairs { + if err := s.writeMutation(ctx, version, ncs.Name, pair); err != nil { + return err + } + } + } + for _, up := range entry.Upgrades { + if err := s.writeUpgrade(ctx, version, up); err != nil { + return err + } + } + if err := s.session.Query(insertVersionCQL, + historical.VersionBucket(version), + version, + rec.Topic, + rec.Partition, + rec.Offset, + time.Now(), + ).WithContext(ctx).Exec(); err != nil { + return fmt.Errorf("insert scylla/cassandra version %d: %w", version, err) + } + return nil +} + +func (s *scyllaSink) writeMutation(ctx context.Context, version int64, storeName string, pair *proto.KVPair) error { + deleted := pair.Delete || pair.Value == nil + value := pair.Value + if deleted { + value = nil + } + if err := s.session.Query(insertMutationCQL, + storeName, + pair.Key, + version, + value, + deleted, + ).WithContext(ctx).Exec(); err != nil { + return fmt.Errorf("insert scylla/cassandra mutation store=%s version=%d: %w", storeName, version, err) + } + return nil +} + +func (s *scyllaSink) writeUpgrade(ctx context.Context, version int64, up *proto.TreeNameUpgrade) error { + if err := s.session.Query(insertUpgradeCQL, + version, + up.Name, + up.RenameFrom, + up.Delete, + ).WithContext(ctx).Exec(); err != nil { + return fmt.Errorf("insert scylla/cassandra tree upgrade version=%d name=%s: %w", version, up.Name, err) + } + return nil +} + +const selectLatestVersionCQL = ` +SELECT version +FROM state_versions +WHERE bucket = ? +LIMIT 1` + +const insertVersionCQL = ` +INSERT INTO state_versions ( + bucket, version, kafka_topic, kafka_partition, kafka_offset, ingested_at +) VALUES (?, ?, ?, ?, ?, ?)` + +const insertMutationCQL = ` +INSERT INTO state_mutations ( + store_name, state_key, version, value, deleted +) VALUES (?, ?, ?, ?, ?)` + +const insertUpgradeCQL = ` +INSERT INTO state_tree_upgrades ( + version, name, rename_from, deleted +) VALUES (?, ?, ?, ?)` diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go new file mode 100644 index 0000000000..b80b409aa8 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -0,0 +1,68 @@ +package consumer + +import ( + "strings" + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" +) + +func TestScyllaConfigValidate(t *testing.T) { + cfg := ScyllaConfig{ + Hosts: []string{"127.0.0.1"}, + Keyspace: "sei_history", + } + require.NoError(t, cfg.Validate()) + + cfg.TimeoutMS = -1 + require.ErrorContains(t, cfg.Validate(), "timeout") +} + +func TestScyllaConfigApplyDefaults(t *testing.T) { + cfg := ScyllaConfig{ + Hosts: []string{"127.0.0.1"}, + Keyspace: "sei_history", + } + cfg.ApplyDefaults() + require.Equal(t, "local_quorum", cfg.Consistency) + require.Equal(t, 2000, cfg.TimeoutMS) + require.Equal(t, 2000, cfg.ConnectTimeoutMS) + require.Equal(t, 4, cfg.NumConns) +} + +func TestCompactRecordsDropsNilEntries(t *testing.T) { + records := compactRecords([]Record{ + {Entry: &proto.ChangelogEntry{Version: 1}}, + {}, + {Entry: &proto.ChangelogEntry{Version: 2}}, + }) + require.Len(t, records, 2) + require.Equal(t, int64(1), records[0].Entry.Version) + require.Equal(t, int64(2), records[1].Entry.Version) +} + +func TestScyllaCQLShape(t *testing.T) { + for _, frag := range []string{ + "INSERT INTO state_mutations", + "store_name", + "state_key", + "version", + "value", + "deleted", + } { + require.Contains(t, insertMutationCQL, frag) + } + for _, frag := range []string{ + "INSERT INTO state_versions", + "bucket", + "version", + "kafka_topic", + "kafka_partition", + "kafka_offset", + "ingested_at", + } { + require.Contains(t, insertVersionCQL, frag) + } + require.True(t, strings.Contains(selectLatestVersionCQL, "LIMIT 1")) +} diff --git a/sei-db/state_db/ss/offload/consumer/sink.go b/sei-db/state_db/ss/offload/consumer/sink.go new file mode 100644 index 0000000000..79da24d0cd --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/sink.go @@ -0,0 +1,26 @@ +package consumer + +import ( + "context" + + dbproto "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// Topic/Partition/Offset are kept alongside Entry so sinks can be idempotent +// across replayed Kafka messages. +type Record struct { + Topic string + Partition int + Offset int64 + Entry *dbproto.ChangelogEntry +} + +type Sink interface { + Write(ctx context.Context, rec Record) error + LastVersion(ctx context.Context) (int64, error) + Close() error +} + +type BatchSink interface { + WriteBatch(ctx context.Context, records []Record) error +} diff --git a/sei-db/state_db/ss/offload/historical/reader.go b/sei-db/state_db/ss/offload/historical/reader.go new file mode 100644 index 0000000000..7dfdfee993 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/reader.go @@ -0,0 +1,37 @@ +// Package historical reads MVCC state from an external historical store. +package historical + +import ( + "context" + "errors" +) + +var ErrNotFound = errors.New("historical state not found") + +// Key is a string so Lookup is usable as a map key. +type Lookup struct { + StoreName string + Key string +} + +// Value is the actual MVCC value that satisfied the lookup. +// Version may be older than the requested target version. +type Value struct { + Bytes []byte + Version int64 +} + +type Reader interface { + // Get returns ErrNotFound if no row exists at or before targetVersion, + // or if the latest such row is a tombstone. + Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) + + // Has skips value transfer and returns false for missing or tombstoned keys. + Has(ctx context.Context, storeName string, key []byte, targetVersion int64) (bool, error) + + // BatchGet returns only found, non-tombstoned lookups. + BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) + + LastVersion(ctx context.Context) (int64, error) + Close() error +} diff --git a/sei-db/state_db/ss/offload/historical/scylla.go b/sei-db/state_db/ss/offload/historical/scylla.go new file mode 100644 index 0000000000..5d48c8a778 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/scylla.go @@ -0,0 +1,239 @@ +package historical + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/gocql/gocql" +) + +const ( + defaultScyllaConsistency = "local_quorum" + defaultScyllaTimeout = 2 * time.Second + defaultScyllaNumConns = 4 + + // VersionBucketCount spreads monotonically increasing block-version markers + // across a bounded set of partitions while keeping LastVersion cheap. + VersionBucketCount = 64 +) + +type ScyllaConfig struct { + Hosts []string + Keyspace string + Username string + Password string + Datacenter string + Consistency string + Timeout time.Duration + ConnectTimeout time.Duration + NumConns int +} + +func (c *ScyllaConfig) ApplyDefaults() { + if c.Consistency == "" { + c.Consistency = defaultScyllaConsistency + } + if c.Timeout == 0 { + c.Timeout = defaultScyllaTimeout + } + if c.ConnectTimeout == 0 { + c.ConnectTimeout = defaultScyllaTimeout + } + if c.NumConns == 0 { + c.NumConns = defaultScyllaNumConns + } +} + +func (c *ScyllaConfig) Validate() error { + if len(c.Hosts) == 0 { + return fmt.Errorf("scylla/cassandra hosts are required") + } + for _, host := range c.Hosts { + if strings.TrimSpace(host) == "" { + return fmt.Errorf("scylla/cassandra hosts must not contain blanks") + } + } + if strings.TrimSpace(c.Keyspace) == "" { + return fmt.Errorf("scylla/cassandra keyspace is required") + } + if c.Password != "" && c.Username == "" { + return fmt.Errorf("scylla/cassandra username is required when password is set") + } + if _, err := parseConsistency(c.Consistency); err != nil { + return err + } + if c.Timeout < 0 { + return fmt.Errorf("scylla/cassandra timeout must be non-negative") + } + if c.ConnectTimeout < 0 { + return fmt.Errorf("scylla/cassandra connect timeout must be non-negative") + } + if c.NumConns < 0 { + return fmt.Errorf("scylla/cassandra num conns must be non-negative") + } + return nil +} + +func NewScyllaReader(cfg ScyllaConfig) (Reader, error) { + session, err := OpenScyllaSession(cfg) + if err != nil { + return nil, err + } + return &scyllaReader{session: session}, nil +} + +func OpenScyllaSession(cfg ScyllaConfig) (*gocql.Session, error) { + cfg.ApplyDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + consistency, err := parseConsistency(cfg.Consistency) + if err != nil { + return nil, err + } + + cluster := gocql.NewCluster(cfg.Hosts...) + cluster.Keyspace = cfg.Keyspace + cluster.Consistency = consistency + cluster.Timeout = cfg.Timeout + cluster.ConnectTimeout = cfg.ConnectTimeout + cluster.NumConns = cfg.NumConns + if cfg.Username != "" { + cluster.Authenticator = gocql.PasswordAuthenticator{ + Username: cfg.Username, + Password: cfg.Password, + } + } + if cfg.Datacenter != "" { + cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy( + gocql.DCAwareRoundRobinPolicy(cfg.Datacenter), + ) + } + + session, err := cluster.CreateSession() + if err != nil { + return nil, fmt.Errorf("open scylla/cassandra session: %w", err) + } + return session, nil +} + +type scyllaReader struct { + session *gocql.Session +} + +var _ Reader = (*scyllaReader)(nil) + +func (r *scyllaReader) Close() error { + r.session.Close() + return nil +} + +func (r *scyllaReader) LastVersion(ctx context.Context) (int64, error) { + var maxVersion int64 + for bucket := 0; bucket < VersionBucketCount; bucket++ { + var version int64 + err := r.session.Query(selectLatestVersionCQL, bucket).WithContext(ctx).Scan(&version) + if err != nil { + if err == gocql.ErrNotFound { + continue + } + return 0, fmt.Errorf("read latest scylla/cassandra version bucket %d: %w", bucket, err) + } + if version > maxVersion { + maxVersion = version + } + } + return maxVersion, nil +} + +func (r *scyllaReader) Has(ctx context.Context, storeName string, key []byte, targetVersion int64) (bool, error) { + var deleted bool + err := r.session.Query(hasLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&deleted) + if err != nil { + if err == gocql.ErrNotFound { + return false, nil + } + return false, fmt.Errorf("scylla/cassandra has lookup: %w", err) + } + return !deleted, nil +} + +func (r *scyllaReader) Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { + var ( + version int64 + bz []byte + deleted bool + ) + err := r.session.Query(getLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&version, &bz, &deleted) + if err != nil { + if err == gocql.ErrNotFound { + return Value{}, ErrNotFound + } + return Value{}, fmt.Errorf("scylla/cassandra get lookup: %w", err) + } + if deleted { + return Value{}, ErrNotFound + } + return Value{Bytes: bz, Version: version}, nil +} + +func (r *scyllaReader) BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) { + out := make(map[Lookup]Value, len(lookups)) + for _, lookup := range lookups { + value, err := r.Get(ctx, lookup.StoreName, []byte(lookup.Key), targetVersion) + if err != nil { + if err == ErrNotFound { + continue + } + return nil, err + } + out[lookup] = value + } + return out, nil +} + +func VersionBucket(version int64) int { + if version < 0 { + version = -version + } + return int(version % VersionBucketCount) +} + +func parseConsistency(name string) (gocql.Consistency, error) { + switch strings.ToLower(strings.TrimSpace(name)) { + case "one": + return gocql.One, nil + case "local_one": + return gocql.LocalOne, nil + case "quorum": + return gocql.Quorum, nil + case "", "local_quorum": + return gocql.LocalQuorum, nil + case "all": + return gocql.All, nil + default: + return gocql.Any, fmt.Errorf("unsupported scylla/cassandra consistency %q", name) + } +} + +const selectLatestVersionCQL = ` +SELECT version +FROM state_versions +WHERE bucket = ? +LIMIT 1` + +const hasLookupCQL = ` +SELECT deleted +FROM state_mutations +WHERE store_name = ? AND state_key = ? AND version <= ? +ORDER BY version DESC +LIMIT 1` + +const getLookupCQL = ` +SELECT version, value, deleted +FROM state_mutations +WHERE store_name = ? AND state_key = ? AND version <= ? +ORDER BY version DESC +LIMIT 1` diff --git a/sei-db/state_db/ss/offload/historical/scylla_test.go b/sei-db/state_db/ss/offload/historical/scylla_test.go new file mode 100644 index 0000000000..8147b44bc4 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/scylla_test.go @@ -0,0 +1,95 @@ +package historical + +import ( + "strings" + "testing" + "time" + + "github.com/gocql/gocql" + "github.com/stretchr/testify/require" +) + +func TestScyllaConfigApplyDefaults(t *testing.T) { + cfg := ScyllaConfig{ + Hosts: []string{"127.0.0.1"}, + Keyspace: "sei_history", + } + cfg.ApplyDefaults() + require.Equal(t, defaultScyllaConsistency, cfg.Consistency) + require.Equal(t, defaultScyllaTimeout, cfg.Timeout) + require.Equal(t, defaultScyllaTimeout, cfg.ConnectTimeout) + require.Equal(t, defaultScyllaNumConns, cfg.NumConns) +} + +func TestScyllaConfigValidate(t *testing.T) { + tests := []struct { + name string + cfg ScyllaConfig + err string + }{ + {"missing hosts", ScyllaConfig{Keyspace: "ks"}, "hosts"}, + {"blank host", ScyllaConfig{Hosts: []string{" "}, Keyspace: "ks"}, "blanks"}, + {"missing keyspace", ScyllaConfig{Hosts: []string{"127.0.0.1"}}, "keyspace"}, + {"password without username", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", Password: "secret"}, "username"}, + {"bad consistency", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", Consistency: "bad"}, "consistency"}, + {"negative timeout", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", Timeout: -time.Second}, "timeout"}, + {"negative connect timeout", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", ConnectTimeout: -time.Second}, "connect"}, + {"negative conns", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", NumConns: -1}, "conns"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.Validate() + require.Error(t, err) + require.Contains(t, err.Error(), tc.err) + }) + } +} + +func TestParseConsistency(t *testing.T) { + tests := []struct { + in string + out gocql.Consistency + }{ + {"", gocql.LocalQuorum}, + {"local_quorum", gocql.LocalQuorum}, + {"LOCAL_ONE", gocql.LocalOne}, + {"one", gocql.One}, + {"quorum", gocql.Quorum}, + {"all", gocql.All}, + } + for _, tc := range tests { + t.Run(tc.in, func(t *testing.T) { + got, err := parseConsistency(tc.in) + require.NoError(t, err) + require.Equal(t, tc.out, got) + }) + } +} + +func TestVersionBucket(t *testing.T) { + require.Equal(t, 0, VersionBucket(0)) + require.Equal(t, 1, VersionBucket(1)) + require.Equal(t, 0, VersionBucket(VersionBucketCount)) + require.Equal(t, 1, VersionBucket(-1)) +} + +func TestPointLookupCQLShape(t *testing.T) { + for _, q := range []string{getLookupCQL, hasLookupCQL} { + for _, frag := range []string{ + "FROM state_mutations", + "store_name = ?", + "state_key = ?", + "version <= ?", + "ORDER BY version DESC", + "LIMIT 1", + } { + require.Contains(t, q, frag) + } + } +} + +func TestLatestVersionCQLShape(t *testing.T) { + require.Contains(t, selectLatestVersionCQL, "FROM state_versions") + require.Contains(t, selectLatestVersionCQL, "bucket = ?") + require.True(t, strings.Contains(selectLatestVersionCQL, "LIMIT 1")) +} diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go new file mode 100644 index 0000000000..9c0dc975f6 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -0,0 +1,96 @@ +package historical + +import ( + "context" + "errors" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// FallbackStateStore routes pruned point reads to the historical reader. +// Iteration and writes stay on the primary state store. +type FallbackStateStore struct { + primary types.StateStore + reader Reader +} + +var _ types.StateStore = (*FallbackStateStore)(nil) + +// NewFallbackStateStore takes ownership of primary and reader for Close. +func NewFallbackStateStore(primary types.StateStore, reader Reader) *FallbackStateStore { + return &FallbackStateStore{primary: primary, reader: reader} +} + +func (s *FallbackStateStore) shouldFallback(version int64) bool { + earliest := s.primary.GetEarliestVersion() + return earliest > 0 && version < earliest +} + +func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([]byte, error) { + if !s.shouldFallback(version) { + return s.primary.Get(storeKey, version, key) + } + v, err := s.reader.Get(context.Background(), storeKey, key, version) + if err != nil { + if errors.Is(err, ErrNotFound) { + return nil, nil + } + return nil, err + } + return v.Bytes, nil +} + +func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bool, error) { + if !s.shouldFallback(version) { + return s.primary.Has(storeKey, version, key) + } + return s.reader.Has(context.Background(), storeKey, key, version) +} + +func (s *FallbackStateStore) Iterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { + return s.primary.Iterator(storeKey, version, start, end) +} + +func (s *FallbackStateStore) ReverseIterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { + return s.primary.ReverseIterator(storeKey, version, start, end) +} + +func (s *FallbackStateStore) RawIterate(storeKey string, fn func([]byte, []byte, int64) bool) (bool, error) { + return s.primary.RawIterate(storeKey, fn) +} + +func (s *FallbackStateStore) GetLatestVersion() int64 { return s.primary.GetLatestVersion() } + +func (s *FallbackStateStore) SetLatestVersion(version int64) error { + return s.primary.SetLatestVersion(version) +} + +func (s *FallbackStateStore) GetEarliestVersion() int64 { return s.primary.GetEarliestVersion() } + +func (s *FallbackStateStore) SetEarliestVersion(version int64, ignoreVersion bool) error { + return s.primary.SetEarliestVersion(version, ignoreVersion) +} + +func (s *FallbackStateStore) ApplyChangesetSync(version int64, changesets []*proto.NamedChangeSet) error { + return s.primary.ApplyChangesetSync(version, changesets) +} + +func (s *FallbackStateStore) ApplyChangesetAsync(version int64, changesets []*proto.NamedChangeSet) error { + return s.primary.ApplyChangesetAsync(version, changesets) +} + +func (s *FallbackStateStore) Prune(version int64) error { return s.primary.Prune(version) } + +func (s *FallbackStateStore) Import(version int64, ch <-chan types.SnapshotNode) error { + return s.primary.Import(version, ch) +} + +func (s *FallbackStateStore) Close() error { + primaryErr := s.primary.Close() + readerErr := s.reader.Close() + if primaryErr != nil { + return primaryErr + } + return readerErr +} diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go new file mode 100644 index 0000000000..fd7696aed1 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -0,0 +1,103 @@ +package historical + +import ( + "context" + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" +) + +type fakeStateStore struct { + earliest int64 + gets int + has int +} + +func (f *fakeStateStore) Get(_ string, _ int64, _ []byte) ([]byte, error) { + f.gets++ + return []byte("primary"), nil +} + +func (f *fakeStateStore) Has(_ string, _ int64, _ []byte) (bool, error) { + f.has++ + return true, nil +} + +func (f *fakeStateStore) Iterator(string, int64, []byte, []byte) (types.DBIterator, error) { + return nil, nil +} + +func (f *fakeStateStore) ReverseIterator(string, int64, []byte, []byte) (types.DBIterator, error) { + return nil, nil +} + +func (f *fakeStateStore) RawIterate(string, func([]byte, []byte, int64) bool) (bool, error) { + return false, nil +} + +func (f *fakeStateStore) GetLatestVersion() int64 { return 0 } +func (f *fakeStateStore) SetLatestVersion(int64) error { return nil } +func (f *fakeStateStore) GetEarliestVersion() int64 { return f.earliest } +func (f *fakeStateStore) SetEarliestVersion(version int64, _ bool) error { + f.earliest = version + return nil +} +func (f *fakeStateStore) ApplyChangesetSync(int64, []*proto.NamedChangeSet) error { return nil } +func (f *fakeStateStore) ApplyChangesetAsync(int64, []*proto.NamedChangeSet) error { return nil } +func (f *fakeStateStore) Prune(int64) error { return nil } +func (f *fakeStateStore) Import(int64, <-chan types.SnapshotNode) error { return nil } +func (f *fakeStateStore) Close() error { return nil } + +type fakeReader struct { + gets int + has int +} + +func (f *fakeReader) Get(context.Context, string, []byte, int64) (Value, error) { + f.gets++ + return Value{Bytes: []byte("historical"), Version: 7}, nil +} + +func (f *fakeReader) Has(context.Context, string, []byte, int64) (bool, error) { + f.has++ + return true, nil +} + +func (f *fakeReader) BatchGet(context.Context, int64, []Lookup) (map[Lookup]Value, error) { + return nil, nil +} + +func (f *fakeReader) LastVersion(context.Context) (int64, error) { return 0, nil } +func (f *fakeReader) Close() error { return nil } + +func TestFallbackStateStoreRoutesPrunedPointReads(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{} + store := NewFallbackStateStore(primary, reader) + + value, err := store.Get("bank", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 0, primary.gets) + require.Equal(t, 1, reader.gets) + + ok, err := store.Has("bank", 7, []byte("k")) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 0, primary.has) + require.Equal(t, 1, reader.has) +} + +func TestFallbackStateStoreKeepsRecentPointReadsOnPrimary(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{} + store := NewFallbackStateStore(primary, reader) + + value, err := store.Get("bank", 10, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("primary"), value) + require.Equal(t, 1, primary.gets) + require.Equal(t, 0, reader.gets) +} diff --git a/sei-db/state_db/ss/offload/kafka.go b/sei-db/state_db/ss/offload/kafka.go index edbe366818..981cb35e02 100644 --- a/sei-db/state_db/ss/offload/kafka.go +++ b/sei-db/state_db/ss/offload/kafka.go @@ -122,7 +122,7 @@ func NewKafkaStream(cfg KafkaConfig) (Stream, error) { } } - mechanism, err := kafkaSASLMechanism(cfg) + mechanism, err := NewSASLMechanism(cfg) if err != nil { return nil, err } @@ -211,7 +211,8 @@ func kafkaCompression(name string) compress.Compression { } } -func kafkaSASLMechanism(cfg KafkaConfig) (sasl.Mechanism, error) { +// NewSASLMechanism is exported so out-of-package consumers share the producer's auth path. +func NewSASLMechanism(cfg KafkaConfig) (sasl.Mechanism, error) { switch strings.ToLower(cfg.SASLMechanism) { case "", kafkaOptionNone: return nil, nil diff --git a/sei-db/state_db/ss/store.go b/sei-db/state_db/ss/store.go index 0fb4b5184e..dc1ce2edc2 100644 --- a/sei-db/state_db/ss/store.go +++ b/sei-db/state_db/ss/store.go @@ -1,9 +1,14 @@ package ss import ( + "fmt" + "strings" + "time" + "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" ) // NewStateStore creates a CompositeStateStore which handles both Cosmos and EVM data. @@ -11,5 +16,45 @@ import ( // files in the backend package. When WriteMode/ReadMode are both cosmos_only (the default), // the EVM stores are not opened and the composite store behaves identically to a plain cosmos state store. func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.StateStore, error) { - return composite.NewCompositeStateStore(ssConfig, homeDir) + primary, err := composite.NewCompositeStateStore(ssConfig, homeDir) + if err != nil { + return nil, err + } + if !scyllaHistoricalOffloadConfigured(ssConfig) { + return primary, nil + } + reader, err := historical.NewScyllaReader(historical.ScyllaConfig{ + Hosts: splitCSV(ssConfig.HistoricalOffloadScyllaHosts), + Keyspace: ssConfig.HistoricalOffloadScyllaKeyspace, + Username: ssConfig.HistoricalOffloadScyllaUsername, + Password: ssConfig.HistoricalOffloadScyllaPassword, + Datacenter: ssConfig.HistoricalOffloadScyllaDatacenter, + Consistency: ssConfig.HistoricalOffloadScyllaConsistency, + Timeout: time.Duration(ssConfig.HistoricalOffloadScyllaTimeoutMS) * time.Millisecond, + }) + if err != nil { + _ = primary.Close() + return nil, fmt.Errorf("open scylla/cassandra historical offload reader: %w", err) + } + return historical.NewFallbackStateStore(primary, reader), nil +} + +func scyllaHistoricalOffloadConfigured(cfg config.StateStoreConfig) bool { + return strings.TrimSpace(cfg.HistoricalOffloadScyllaHosts) != "" || + strings.TrimSpace(cfg.HistoricalOffloadScyllaKeyspace) != "" +} + +func splitCSV(value string) []string { + if strings.TrimSpace(value) == "" { + return nil + } + parts := strings.Split(value, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out } From 32b55d2b0c910d546f054af7c11fdba7fddfd2f3 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 12 May 2026 12:02:48 -0400 Subject: [PATCH 02/41] Avoid retry timer leak in Scylla consumer --- sei-db/state_db/ss/offload/consumer/consumer.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/consumer.go b/sei-db/state_db/ss/offload/consumer/consumer.go index a0adaaca46..a04c42e75e 100644 --- a/sei-db/state_db/ss/offload/consumer/consumer.go +++ b/sei-db/state_db/ss/offload/consumer/consumer.go @@ -257,10 +257,8 @@ func (c *Consumer) writeBatchWithRetry(ctx context.Context, records []Record) er } c.logf("sink write attempt %d/%d failed: %v; retrying in %s", attempt, c.maxAttempts, err, backoff) - select { - case <-time.After(backoff): - case <-ctx.Done(): - return ctx.Err() + if err := sleepWithContext(ctx, backoff); err != nil { + return err } backoff *= 2 if backoff > c.maxBackoff { @@ -270,6 +268,17 @@ func (c *Consumer) writeBatchWithRetry(ctx context.Context, records []Record) er return fmt.Errorf("sink write failed after %d attempts: %w", c.maxAttempts, lastErr) } +func sleepWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + func (c *Consumer) writeRecords(ctx context.Context, records []Record) error { if len(records) == 0 { return nil From 6c6f5890114f4664fdd08e08dba51cfd2d7776ae Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 12 May 2026 12:20:53 -0400 Subject: [PATCH 03/41] Parallelize Scylla mutation ingest --- sei-db/state_db/ss/offload/consumer/README.md | 3 +- .../consumer/config/example-scylla.json | 3 +- sei-db/state_db/ss/offload/consumer/scylla.go | 94 ++++++++++++++----- .../ss/offload/consumer/scylla_test.go | 89 ++++++++++++++++++ 4 files changed, 166 insertions(+), 23 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md index 9ccc28b29d..185920d783 100644 --- a/sei-db/state_db/ss/offload/consumer/README.md +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -36,7 +36,8 @@ factors before applying it. The consumer reads historical offload changelog messages from Kafka and writes them into Scylla/Cassandra. Kafka offsets are committed only after the sink -write succeeds. +write succeeds. Within each block, mutation rows are written with bounded +concurrency and the version marker is written last. ```bash go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer \ diff --git a/sei-db/state_db/ss/offload/consumer/config/example-scylla.json b/sei-db/state_db/ss/offload/consumer/config/example-scylla.json index 94779c83e9..013217af75 100644 --- a/sei-db/state_db/ss/offload/consumer/config/example-scylla.json +++ b/sei-db/state_db/ss/offload/consumer/config/example-scylla.json @@ -12,7 +12,8 @@ "Consistency": "local_quorum", "TimeoutMS": 2000, "ConnectTimeoutMS": 2000, - "NumConns": 4 + "NumConns": 4, + "MutationWorkers": 16 }, "Workers": 16, "ShardBufferSize": 128, diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index c16744662c..ebe0b276eb 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -6,11 +6,14 @@ import ( "time" "github.com/gocql/gocql" + "golang.org/x/sync/errgroup" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" ) +const defaultScyllaMutationWorkers = 16 + type ScyllaConfig struct { Hosts []string Keyspace string @@ -21,6 +24,7 @@ type ScyllaConfig struct { TimeoutMS int ConnectTimeoutMS int NumConns int + MutationWorkers int } func (c *ScyllaConfig) ApplyDefaults() { @@ -30,12 +34,21 @@ func (c *ScyllaConfig) ApplyDefaults() { c.TimeoutMS = int(cfg.Timeout / time.Millisecond) c.ConnectTimeoutMS = int(cfg.ConnectTimeout / time.Millisecond) c.NumConns = cfg.NumConns + if c.MutationWorkers == 0 { + c.MutationWorkers = defaultScyllaMutationWorkers + } } func (c *ScyllaConfig) Validate() error { cfg := c.toHistorical() cfg.ApplyDefaults() - return cfg.Validate() + if err := cfg.Validate(); err != nil { + return err + } + if c.MutationWorkers < 0 { + return fmt.Errorf("scylla/cassandra mutation workers must be non-negative") + } + return nil } func (c ScyllaConfig) toHistorical() historical.ScyllaConfig { @@ -53,22 +66,34 @@ func (c ScyllaConfig) toHistorical() historical.ScyllaConfig { } type scyllaSink struct { - session *gocql.Session + session *gocql.Session + exec scyllaExecFunc + mutationWorkers int } var _ Sink = (*scyllaSink)(nil) var _ BatchSink = (*scyllaSink)(nil) func NewScyllaSink(cfg ScyllaConfig) (Sink, error) { + cfg.ApplyDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } session, err := historical.OpenScyllaSession(cfg.toHistorical()) if err != nil { return nil, err } - return &scyllaSink{session: session}, nil + return &scyllaSink{ + session: session, + exec: sessionExec(session), + mutationWorkers: cfg.MutationWorkers, + }, nil } func (s *scyllaSink) Close() error { - s.session.Close() + if s.session != nil { + s.session.Close() + } return nil } @@ -124,61 +149,88 @@ func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { return nil } version := entry.Version - for _, ncs := range entry.Changesets { - for _, pair := range ncs.Changeset.Pairs { - if err := s.writeMutation(ctx, version, ncs.Name, pair); err != nil { - return err - } - } - } - for _, up := range entry.Upgrades { - if err := s.writeUpgrade(ctx, version, up); err != nil { - return err - } + if err := s.writeRecordRows(ctx, version, entry); err != nil { + return err } - if err := s.session.Query(insertVersionCQL, + if err := s.exec(ctx, insertVersionCQL, historical.VersionBucket(version), version, rec.Topic, rec.Partition, rec.Offset, time.Now(), - ).WithContext(ctx).Exec(); err != nil { + ); err != nil { return fmt.Errorf("insert scylla/cassandra version %d: %w", version, err) } return nil } +func (s *scyllaSink) writeRecordRows(ctx context.Context, version int64, entry *proto.ChangelogEntry) error { + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(s.effectiveMutationWorkers()) + for _, ncs := range entry.Changesets { + storeName := ncs.Name + for _, pair := range ncs.Changeset.Pairs { + pair := pair + g.Go(func() error { + return s.writeMutation(gctx, version, storeName, pair) + }) + } + } + for _, up := range entry.Upgrades { + up := up + g.Go(func() error { + return s.writeUpgrade(gctx, version, up) + }) + } + return g.Wait() +} + +func (s *scyllaSink) effectiveMutationWorkers() int { + if s.mutationWorkers <= 0 { + return defaultScyllaMutationWorkers + } + return s.mutationWorkers +} + func (s *scyllaSink) writeMutation(ctx context.Context, version int64, storeName string, pair *proto.KVPair) error { deleted := pair.Delete || pair.Value == nil value := pair.Value if deleted { value = nil } - if err := s.session.Query(insertMutationCQL, + if err := s.exec(ctx, insertMutationCQL, storeName, pair.Key, version, value, deleted, - ).WithContext(ctx).Exec(); err != nil { + ); err != nil { return fmt.Errorf("insert scylla/cassandra mutation store=%s version=%d: %w", storeName, version, err) } return nil } func (s *scyllaSink) writeUpgrade(ctx context.Context, version int64, up *proto.TreeNameUpgrade) error { - if err := s.session.Query(insertUpgradeCQL, + if err := s.exec(ctx, insertUpgradeCQL, version, up.Name, up.RenameFrom, up.Delete, - ).WithContext(ctx).Exec(); err != nil { + ); err != nil { return fmt.Errorf("insert scylla/cassandra tree upgrade version=%d name=%s: %w", version, up.Name, err) } return nil } +type scyllaExecFunc func(ctx context.Context, stmt string, values ...interface{}) error + +func sessionExec(session *gocql.Session) scyllaExecFunc { + return func(ctx context.Context, stmt string, values ...interface{}) error { + return session.Query(stmt, values...).WithContext(ctx).Exec() + } +} + const selectLatestVersionCQL = ` SELECT version FROM state_versions diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go index b80b409aa8..60db6acf9a 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -1,8 +1,11 @@ package consumer import ( + "context" "strings" + "sync/atomic" "testing" + "time" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/stretchr/testify/require" @@ -17,6 +20,10 @@ func TestScyllaConfigValidate(t *testing.T) { cfg.TimeoutMS = -1 require.ErrorContains(t, cfg.Validate(), "timeout") + + cfg.TimeoutMS = 0 + cfg.MutationWorkers = -1 + require.ErrorContains(t, cfg.Validate(), "mutation workers") } func TestScyllaConfigApplyDefaults(t *testing.T) { @@ -29,6 +36,7 @@ func TestScyllaConfigApplyDefaults(t *testing.T) { require.Equal(t, 2000, cfg.TimeoutMS) require.Equal(t, 2000, cfg.ConnectTimeoutMS) require.Equal(t, 4, cfg.NumConns) + require.Equal(t, 16, cfg.MutationWorkers) } func TestCompactRecordsDropsNilEntries(t *testing.T) { @@ -66,3 +74,84 @@ func TestScyllaCQLShape(t *testing.T) { } require.True(t, strings.Contains(selectLatestVersionCQL, "LIMIT 1")) } + +func TestScyllaSinkWritesRowsConcurrentlyBeforeVersionMarker(t *testing.T) { + rowStarted := make(chan struct{}, 8) + releaseRows := make(chan struct{}) + var activeRows atomic.Int32 + var sawConcurrentRows atomic.Bool + var markerBeforeRowsDone atomic.Bool + var versionMarkers atomic.Int32 + + sink := &scyllaSink{ + mutationWorkers: 2, + exec: func(ctx context.Context, stmt string, _ ...interface{}) error { + if strings.Contains(stmt, "state_versions") { + if activeRows.Load() != 0 { + markerBeforeRowsDone.Store(true) + } + versionMarkers.Add(1) + return nil + } + if activeRows.Add(1) > 1 { + sawConcurrentRows.Store(true) + } + rowStarted <- struct{}{} + select { + case <-releaseRows: + case <-ctx.Done(): + activeRows.Add(-1) + return ctx.Err() + } + activeRows.Add(-1) + return nil + }, + } + entry := &proto.ChangelogEntry{ + Version: 7, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k1"), Value: []byte("v1")}, + {Key: []byte("k2"), Value: []byte("v2")}, + {Key: []byte("k3"), Value: []byte("v3")}, + }}, + }}, + Upgrades: []*proto.TreeNameUpgrade{{Name: "new-store"}}, + } + + errCh := make(chan error, 1) + go func() { + errCh <- sink.writeRecord(context.Background(), Record{Topic: "t", Partition: 1, Offset: 2, Entry: entry}) + }() + + releaseClosed := false + closeRelease := func() { + if !releaseClosed { + close(releaseRows) + releaseClosed = true + } + } + defer closeRelease() + + for i := 0; i < 2; i++ { + select { + case <-rowStarted: + case <-time.After(time.Second): + closeRelease() + t.Fatal("timed out waiting for concurrent row writes") + } + } + require.True(t, sawConcurrentRows.Load()) + require.Equal(t, int32(0), versionMarkers.Load(), "version marker must wait for row writes") + + closeRelease() + select { + case err := <-errCh: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for record write") + } + require.False(t, markerBeforeRowsDone.Load()) + require.Equal(t, int32(1), versionMarkers.Load()) +} From 4fcd2421f4fd638195aea8c949d640509422ca1f Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 12 May 2026 12:56:20 -0400 Subject: [PATCH 04/41] Parallelize Scylla historical batch reads --- .../state_db/ss/offload/historical/scylla.go | 74 +++++++++++++------ .../ss/offload/historical/scylla_test.go | 73 ++++++++++++++++++ 2 files changed, 125 insertions(+), 22 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/scylla.go b/sei-db/state_db/ss/offload/historical/scylla.go index 5d48c8a778..bcb98cebba 100644 --- a/sei-db/state_db/ss/offload/historical/scylla.go +++ b/sei-db/state_db/ss/offload/historical/scylla.go @@ -2,17 +2,21 @@ package historical import ( "context" + "errors" "fmt" "strings" + "sync" "time" "github.com/gocql/gocql" + "golang.org/x/sync/errgroup" ) const ( defaultScyllaConsistency = "local_quorum" defaultScyllaTimeout = 2 * time.Second defaultScyllaNumConns = 4 + defaultScyllaReadWorkers = 16 // VersionBucketCount spreads monotonically increasing block-version markers // across a bounded set of partitions while keeping LastVersion cheap. @@ -81,7 +85,10 @@ func NewScyllaReader(cfg ScyllaConfig) (Reader, error) { if err != nil { return nil, err } - return &scyllaReader{session: session}, nil + return &scyllaReader{ + session: session, + get: sessionGet(session), + }, nil } func OpenScyllaSession(cfg ScyllaConfig) (*gocql.Session, error) { @@ -121,12 +128,15 @@ func OpenScyllaSession(cfg ScyllaConfig) (*gocql.Session, error) { type scyllaReader struct { session *gocql.Session + get scyllaGetFunc } var _ Reader = (*scyllaReader)(nil) func (r *scyllaReader) Close() error { - r.session.Close() + if r.session != nil { + r.session.Close() + } return nil } @@ -161,35 +171,55 @@ func (r *scyllaReader) Has(ctx context.Context, storeName string, key []byte, ta } func (r *scyllaReader) Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { - var ( - version int64 - bz []byte - deleted bool - ) - err := r.session.Query(getLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&version, &bz, &deleted) - if err != nil { - if err == gocql.ErrNotFound { + return r.get(ctx, storeName, key, targetVersion) +} + +func sessionGet(session *gocql.Session) scyllaGetFunc { + return func(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { + var ( + version int64 + bz []byte + deleted bool + ) + err := session.Query(getLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&version, &bz, &deleted) + if err != nil { + if err == gocql.ErrNotFound { + return Value{}, ErrNotFound + } + return Value{}, fmt.Errorf("scylla/cassandra get lookup: %w", err) + } + if deleted { return Value{}, ErrNotFound } - return Value{}, fmt.Errorf("scylla/cassandra get lookup: %w", err) - } - if deleted { - return Value{}, ErrNotFound + return Value{Bytes: bz, Version: version}, nil } - return Value{Bytes: bz, Version: version}, nil } +type scyllaGetFunc func(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) + func (r *scyllaReader) BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) { out := make(map[Lookup]Value, len(lookups)) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(defaultScyllaReadWorkers) + var mu sync.Mutex for _, lookup := range lookups { - value, err := r.Get(ctx, lookup.StoreName, []byte(lookup.Key), targetVersion) - if err != nil { - if err == ErrNotFound { - continue + lookup := lookup + g.Go(func() error { + value, err := r.Get(gctx, lookup.StoreName, []byte(lookup.Key), targetVersion) + if err != nil { + if errors.Is(err, ErrNotFound) { + return nil + } + return err } - return nil, err - } - out[lookup] = value + mu.Lock() + out[lookup] = value + mu.Unlock() + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err } return out, nil } diff --git a/sei-db/state_db/ss/offload/historical/scylla_test.go b/sei-db/state_db/ss/offload/historical/scylla_test.go index 8147b44bc4..2c5f3a8d42 100644 --- a/sei-db/state_db/ss/offload/historical/scylla_test.go +++ b/sei-db/state_db/ss/offload/historical/scylla_test.go @@ -1,7 +1,9 @@ package historical import ( + "context" "strings" + "sync/atomic" "testing" "time" @@ -93,3 +95,74 @@ func TestLatestVersionCQLShape(t *testing.T) { require.Contains(t, selectLatestVersionCQL, "bucket = ?") require.True(t, strings.Contains(selectLatestVersionCQL, "LIMIT 1")) } + +func TestScyllaReaderBatchGetParallelizesLookups(t *testing.T) { + started := make(chan string, 4) + release := make(chan struct{}) + var active atomic.Int32 + var sawConcurrent atomic.Bool + + reader := &scyllaReader{ + get: func(ctx context.Context, _ string, key []byte, targetVersion int64) (Value, error) { + if active.Add(1) > 1 { + sawConcurrent.Store(true) + } + defer active.Add(-1) + keyString := string(key) + started <- keyString + select { + case <-release: + case <-ctx.Done(): + return Value{}, ctx.Err() + } + if keyString == "missing" { + return Value{}, ErrNotFound + } + return Value{Bytes: []byte("value-" + keyString), Version: targetVersion - 1}, nil + }, + } + + errCh := make(chan error, 1) + var got map[Lookup]Value + lookups := []Lookup{ + {StoreName: "bank", Key: "k1"}, + {StoreName: "bank", Key: "missing"}, + {StoreName: "evm", Key: "k2"}, + } + go func() { + var err error + got, err = reader.BatchGet(context.Background(), 10, lookups) + errCh <- err + }() + + releaseClosed := false + closeRelease := func() { + if !releaseClosed { + close(release) + releaseClosed = true + } + } + defer closeRelease() + + for i := 0; i < 2; i++ { + select { + case <-started: + case <-time.After(time.Second): + closeRelease() + t.Fatal("timed out waiting for concurrent lookups") + } + } + require.True(t, sawConcurrent.Load()) + + closeRelease() + select { + case err := <-errCh: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for batch get") + } + require.Len(t, got, 2) + require.Equal(t, []byte("value-k1"), got[lookups[0]].Bytes) + require.Equal(t, []byte("value-k2"), got[lookups[2]].Bytes) + require.NotContains(t, got, lookups[1]) +} From 0fa1aef5c03e79b41b23344e30d5a67a939bf004 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Wed, 13 May 2026 10:08:22 -0400 Subject: [PATCH 05/41] Cache Scylla historical point reads --- .../state_db/ss/offload/historical/store.go | 43 ++++++++++++++++++- .../ss/offload/historical/store_test.go | 21 +++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index 9c0dc975f6..90dc7abdac 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -1,25 +1,43 @@ package historical import ( + "bytes" "context" "errors" + lru "github.com/hashicorp/golang-lru/v2" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" ) +const ( + defaultHistoricalReadCacheEntries = 64 * 1024 + maxHistoricalReadCacheValueBytes = 64 * 1024 +) + +type historicalReadCacheKey struct { + storeKey string + version int64 + key string +} + // FallbackStateStore routes pruned point reads to the historical reader. // Iteration and writes stay on the primary state store. type FallbackStateStore struct { primary types.StateStore reader Reader + cache *lru.Cache[historicalReadCacheKey, []byte] } var _ types.StateStore = (*FallbackStateStore)(nil) // NewFallbackStateStore takes ownership of primary and reader for Close. func NewFallbackStateStore(primary types.StateStore, reader Reader) *FallbackStateStore { - return &FallbackStateStore{primary: primary, reader: reader} + cache, err := lru.New[historicalReadCacheKey, []byte](defaultHistoricalReadCacheEntries) + if err != nil { + panic(err) + } + return &FallbackStateStore{primary: primary, reader: reader, cache: cache} } func (s *FallbackStateStore) shouldFallback(version int64) bool { @@ -31,6 +49,10 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] if !s.shouldFallback(version) { return s.primary.Get(storeKey, version, key) } + cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} + if value, ok := s.getCached(cacheKey); ok { + return value, nil + } v, err := s.reader.Get(context.Background(), storeKey, key, version) if err != nil { if errors.Is(err, ErrNotFound) { @@ -38,9 +60,28 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] } return nil, err } + s.cacheValue(cacheKey, v.Bytes) return v.Bytes, nil } +func (s *FallbackStateStore) getCached(key historicalReadCacheKey) ([]byte, bool) { + if s.cache == nil { + return nil, false + } + value, ok := s.cache.Get(key) + if !ok { + return nil, false + } + return bytes.Clone(value), true +} + +func (s *FallbackStateStore) cacheValue(key historicalReadCacheKey, value []byte) { + if s.cache == nil || value == nil || len(value) > maxHistoricalReadCacheValueBytes { + return + } + s.cache.Add(key, bytes.Clone(value)) +} + func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bool, error) { if !s.shouldFallback(version) { return s.primary.Has(storeKey, version, key) diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go index fd7696aed1..1e3cbfea57 100644 --- a/sei-db/state_db/ss/offload/historical/store_test.go +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -101,3 +101,24 @@ func TestFallbackStateStoreKeepsRecentPointReadsOnPrimary(t *testing.T) { require.Equal(t, 1, primary.gets) require.Equal(t, 0, reader.gets) } + +func TestFallbackStateStoreCachesHistoricalPointReads(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{} + store := NewFallbackStateStore(primary, reader) + + value, err := store.Get("bank", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + value[0] = 'H' + + value, err = store.Get("bank", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + value[0] = 'H' + + value, err = store.Get("bank", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 1, reader.gets) +} From 5ef70bafd2bcc277ecedd37aef8e10082fd7faf2 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Wed, 13 May 2026 10:31:25 -0400 Subject: [PATCH 06/41] Enable token-aware Scylla routing --- sei-db/state_db/ss/offload/historical/scylla.go | 14 +++++++++----- .../ss/offload/historical/scylla_test.go | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/scylla.go b/sei-db/state_db/ss/offload/historical/scylla.go index bcb98cebba..686908c343 100644 --- a/sei-db/state_db/ss/offload/historical/scylla.go +++ b/sei-db/state_db/ss/offload/historical/scylla.go @@ -113,11 +113,7 @@ func OpenScyllaSession(cfg ScyllaConfig) (*gocql.Session, error) { Password: cfg.Password, } } - if cfg.Datacenter != "" { - cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy( - gocql.DCAwareRoundRobinPolicy(cfg.Datacenter), - ) - } + cluster.PoolConfig.HostSelectionPolicy = scyllaHostSelectionPolicy(cfg.Datacenter) session, err := cluster.CreateSession() if err != nil { @@ -126,6 +122,14 @@ func OpenScyllaSession(cfg ScyllaConfig) (*gocql.Session, error) { return session, nil } +func scyllaHostSelectionPolicy(datacenter string) gocql.HostSelectionPolicy { + datacenter = strings.TrimSpace(datacenter) + if datacenter != "" { + return gocql.TokenAwareHostPolicy(gocql.DCAwareRoundRobinPolicy(datacenter)) + } + return gocql.TokenAwareHostPolicy(gocql.RoundRobinHostPolicy()) +} + type scyllaReader struct { session *gocql.Session get scyllaGetFunc diff --git a/sei-db/state_db/ss/offload/historical/scylla_test.go b/sei-db/state_db/ss/offload/historical/scylla_test.go index 2c5f3a8d42..2699cdf3ea 100644 --- a/sei-db/state_db/ss/offload/historical/scylla_test.go +++ b/sei-db/state_db/ss/offload/historical/scylla_test.go @@ -2,6 +2,7 @@ package historical import ( "context" + "reflect" "strings" "sync/atomic" "testing" @@ -68,6 +69,22 @@ func TestParseConsistency(t *testing.T) { } } +func TestScyllaHostSelectionPolicyIsTokenAware(t *testing.T) { + for _, tc := range []struct { + name string + datacenter string + }{ + {"no datacenter", ""}, + {"with datacenter", "dc1"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := scyllaHostSelectionPolicy(tc.datacenter) + require.NotNil(t, policy) + require.Contains(t, reflect.TypeOf(policy).String(), "tokenAware") + }) + } +} + func TestVersionBucket(t *testing.T) { require.Equal(t, 0, VersionBucket(0)) require.Equal(t, 1, VersionBucket(1)) From 37d0086494da7beef1475edfa1e4c79f30a6c604 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Wed, 13 May 2026 15:57:40 -0400 Subject: [PATCH 07/41] Compact duplicate Scylla mutation writes --- sei-db/state_db/ss/offload/consumer/scylla.go | 41 +++++++++++++--- .../ss/offload/consumer/scylla_test.go | 49 +++++++++++++++++++ 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index ebe0b276eb..fe030c6901 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -168,14 +168,11 @@ func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { func (s *scyllaSink) writeRecordRows(ctx context.Context, version int64, entry *proto.ChangelogEntry) error { g, gctx := errgroup.WithContext(ctx) g.SetLimit(s.effectiveMutationWorkers()) - for _, ncs := range entry.Changesets { - storeName := ncs.Name - for _, pair := range ncs.Changeset.Pairs { - pair := pair - g.Go(func() error { - return s.writeMutation(gctx, version, storeName, pair) - }) - } + for _, mutation := range compactMutations(entry) { + mutation := mutation + g.Go(func() error { + return s.writeMutation(gctx, version, mutation.storeName, mutation.pair) + }) } for _, up := range entry.Upgrades { up := up @@ -186,6 +183,34 @@ func (s *scyllaSink) writeRecordRows(ctx context.Context, version int64, entry * return g.Wait() } +type scyllaMutation struct { + storeName string + pair *proto.KVPair +} + +type scyllaMutationKey struct { + storeName string + key string +} + +func compactMutations(entry *proto.ChangelogEntry) []scyllaMutation { + mutations := make([]scyllaMutation, 0) + indexByKey := make(map[scyllaMutationKey]int) + for _, ncs := range entry.Changesets { + storeName := ncs.Name + for _, pair := range ncs.Changeset.Pairs { + key := scyllaMutationKey{storeName: storeName, key: string(pair.Key)} + if idx, ok := indexByKey[key]; ok { + mutations[idx].pair = pair + continue + } + indexByKey[key] = len(mutations) + mutations = append(mutations, scyllaMutation{storeName: storeName, pair: pair}) + } + } + return mutations +} + func (s *scyllaSink) effectiveMutationWorkers() int { if s.mutationWorkers <= 0 { return defaultScyllaMutationWorkers diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go index 60db6acf9a..1c2d3b1645 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -3,6 +3,7 @@ package consumer import ( "context" "strings" + "sync" "sync/atomic" "testing" "time" @@ -155,3 +156,51 @@ func TestScyllaSinkWritesRowsConcurrentlyBeforeVersionMarker(t *testing.T) { require.False(t, markerBeforeRowsDone.Load()) require.Equal(t, int32(1), versionMarkers.Load()) } + +func TestScyllaSinkCompactsDuplicateMutations(t *testing.T) { + type write struct { + value []byte + deleted bool + } + var mu sync.Mutex + writes := make(map[string]write) + sink := &scyllaSink{ + mutationWorkers: 1, + exec: func(_ context.Context, stmt string, values ...interface{}) error { + if !strings.Contains(stmt, "state_mutations") { + return nil + } + storeName := values[0].(string) + key := string(values[1].([]byte)) + value := values[3].([]byte) + deleted := values[4].(bool) + mu.Lock() + writes[storeName+"/"+key] = write{value: value, deleted: deleted} + mu.Unlock() + return nil + }, + } + entry := &proto.ChangelogEntry{ + Version: 9, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k"), Value: []byte("old")}, + {Key: []byte("drop"), Value: []byte("present")}, + {Key: []byte("k"), Value: []byte("new")}, + {Key: []byte("drop"), Delete: true}, + }}, + }, { + Name: "evm", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k"), Value: []byte("separate-store")}, + }}, + }}, + } + + require.NoError(t, sink.writeRecordRows(context.Background(), entry.Version, entry)) + require.Len(t, writes, 3) + require.Equal(t, write{value: []byte("new")}, writes["bank/k"]) + require.Equal(t, write{deleted: true}, writes["bank/drop"]) + require.Equal(t, write{value: []byte("separate-store")}, writes["evm/k"]) +} From 4a14131fc1d9b3e311819937293982e1a150fe48 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Wed, 13 May 2026 16:07:31 -0400 Subject: [PATCH 08/41] Pipeline Scylla batch row writes --- sei-db/state_db/ss/offload/consumer/scylla.go | 50 +++++++- .../ss/offload/consumer/scylla_test.go | 121 ++++++++++++++++++ 2 files changed, 166 insertions(+), 5 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index fe030c6901..bcf061c7b0 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -120,12 +120,14 @@ func (s *scyllaSink) Write(ctx context.Context, rec Record) error { } func (s *scyllaSink) WriteBatch(ctx context.Context, records []Record) error { - for _, rec := range compactRecords(records) { - if err := s.writeRecord(ctx, rec); err != nil { - return err - } + records = compactRecords(records) + if len(records) == 0 { + return nil } - return nil + if len(records) == 1 { + return s.writeRecord(ctx, records[0]) + } + return s.writeRecordsPipelined(ctx, records) } func compactRecords(records []Record) []Record { @@ -152,6 +154,11 @@ func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { if err := s.writeRecordRows(ctx, version, entry); err != nil { return err } + return s.writeVersionMarker(ctx, rec) +} + +func (s *scyllaSink) writeVersionMarker(ctx context.Context, rec Record) error { + version := rec.Entry.Version if err := s.exec(ctx, insertVersionCQL, historical.VersionBucket(version), version, @@ -165,6 +172,39 @@ func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { return nil } +func (s *scyllaSink) writeRecordsPipelined(ctx context.Context, records []Record) error { + rowCtx, cancel := context.WithCancel(ctx) + defer cancel() + g, gctx := errgroup.WithContext(rowCtx) + rowDone := make([]chan error, len(records)) + for i := range records { + rowDone[i] = make(chan error, 1) + i := i + rec := records[i] + g.Go(func() error { + err := s.writeRecordRows(gctx, rec.Entry.Version, rec.Entry) + if err != nil { + err = fmt.Errorf("write scylla/cassandra rows version %d: %w", rec.Entry.Version, err) + } + rowDone[i] <- err + return err + }) + } + for i, rec := range records { + if err := <-rowDone[i]; err != nil { + cancel() + _ = g.Wait() + return err + } + if err := s.writeVersionMarker(ctx, rec); err != nil { + cancel() + _ = g.Wait() + return err + } + } + return g.Wait() +} + func (s *scyllaSink) writeRecordRows(ctx context.Context, version int64, entry *proto.ChangelogEntry) error { g, gctx := errgroup.WithContext(ctx) g.SetLimit(s.effectiveMutationWorkers()) diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go index 1c2d3b1645..8516ed6e74 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -204,3 +204,124 @@ func TestScyllaSinkCompactsDuplicateMutations(t *testing.T) { require.Equal(t, write{deleted: true}, writes["bank/drop"]) require.Equal(t, write{value: []byte("separate-store")}, writes["evm/k"]) } + +func TestScyllaSinkWriteBatchPipelinesRowsAndOrdersMarkers(t *testing.T) { + rowStarted := make(chan int64, 2) + markerWritten := make(chan int64, 2) + releaseRows := map[int64]chan struct{}{ + 1: make(chan struct{}), + 2: make(chan struct{}), + } + var activeRows atomic.Int32 + var sawConcurrentRows atomic.Bool + var mu sync.Mutex + rowsDone := make(map[int64]bool) + var markers []int64 + var markerBeforeRowsDone bool + + sink := &scyllaSink{ + mutationWorkers: 1, + exec: func(ctx context.Context, stmt string, values ...interface{}) error { + switch { + case strings.Contains(stmt, "state_mutations"): + version := values[2].(int64) + if activeRows.Add(1) > 1 { + sawConcurrentRows.Store(true) + } + rowStarted <- version + select { + case <-releaseRows[version]: + case <-ctx.Done(): + activeRows.Add(-1) + return ctx.Err() + } + activeRows.Add(-1) + mu.Lock() + rowsDone[version] = true + mu.Unlock() + return nil + case strings.Contains(stmt, "state_versions"): + version := values[1].(int64) + mu.Lock() + if !rowsDone[version] { + markerBeforeRowsDone = true + } + markers = append(markers, version) + mu.Unlock() + markerWritten <- version + return nil + default: + return nil + } + }, + } + records := []Record{ + { + Topic: "t", + Partition: 0, + Offset: 10, + Entry: &proto.ChangelogEntry{ + Version: 1, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k1"), Value: []byte("v1")}}}, + }}, + }, + }, + { + Topic: "t", + Partition: 0, + Offset: 11, + Entry: &proto.ChangelogEntry{ + Version: 2, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k2"), Value: []byte("v2")}}}, + }}, + }, + }, + } + + errCh := make(chan error, 1) + go func() { + errCh <- sink.WriteBatch(context.Background(), records) + }() + + started := map[int64]bool{} + for len(started) < 2 { + select { + case version := <-rowStarted: + started[version] = true + case <-time.After(time.Second): + t.Fatal("timed out waiting for pipelined row writes") + } + } + require.True(t, sawConcurrentRows.Load()) + + close(releaseRows[2]) + select { + case version := <-markerWritten: + t.Fatalf("marker %d written before earlier record rows completed", version) + case <-time.After(100 * time.Millisecond): + } + + close(releaseRows[1]) + for _, want := range []int64{1, 2} { + select { + case got := <-markerWritten: + require.Equal(t, want, got) + case <-time.After(time.Second): + t.Fatalf("timed out waiting for marker %d", want) + } + } + select { + case err := <-errCh: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for batch write") + } + mu.Lock() + defer mu.Unlock() + require.False(t, markerBeforeRowsDone) + require.Equal(t, []int64{1, 2}, markers) +} From f11f0e813202aa8fe782bb0f77459cf0a45e1180 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 14 May 2026 13:10:18 -0400 Subject: [PATCH 09/41] Add Bigtable historical offload backend --- go.mod | 3 + go.sum | 6 + sei-db/config/ss_config.go | 9 + sei-db/config/toml.go | 10 + sei-db/config/toml_test.go | 4 + sei-db/state_db/ss/offload/consumer/README.md | 58 +- .../state_db/ss/offload/consumer/bigtable.go | 262 +++++++++ .../ss/offload/consumer/bigtable_test.go | 168 ++++++ .../cmd/historical-scylla-consumer/main.go | 4 +- sei-db/state_db/ss/offload/consumer/config.go | 38 +- .../consumer/config/example-bigtable.json | 22 + .../ss/offload/consumer/config_test.go | 33 ++ sei-db/state_db/ss/offload/consumer/scylla.go | 4 + .../ss/offload/historical/bigtable.go | 533 ++++++++++++++++++ .../ss/offload/historical/bigtable_test.go | 93 +++ sei-db/state_db/ss/store.go | 29 +- 16 files changed, 1259 insertions(+), 17 deletions(-) create mode 100644 sei-db/state_db/ss/offload/consumer/bigtable.go create mode 100644 sei-db/state_db/ss/offload/consumer/bigtable_test.go create mode 100644 sei-db/state_db/ss/offload/consumer/config/example-bigtable.json create mode 100644 sei-db/state_db/ss/offload/consumer/config_test.go create mode 100644 sei-db/state_db/ss/offload/historical/bigtable.go create mode 100644 sei-db/state_db/ss/offload/historical/bigtable_test.go diff --git a/go.mod b/go.mod index dae2d00b1a..6012f8f185 100644 --- a/go.mod +++ b/go.mod @@ -141,6 +141,8 @@ require ( ) require ( + cloud.google.com/go/bigtable v1.37.0 + cloud.google.com/go/compute/metadata v0.8.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/DataDog/zstd v1.5.7 // indirect @@ -281,6 +283,7 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/image v0.36.0 golang.org/x/mod v0.32.0 + golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect golang.org/x/term v0.39.0 // indirect golang.org/x/text v0.34.0 // indirect diff --git a/go.sum b/go.sum index b941802165..ad867b4147 100644 --- a/go.sum +++ b/go.sum @@ -125,6 +125,8 @@ cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/Zur cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigtable v1.37.0 h1:Q+x7y04lQ0B+WXp03wc1/FLhFt4CwcQdkwWT0M4Jp3w= +cloud.google.com/go/bigtable v1.37.0/go.mod h1:HXqddP6hduwzrtiTCqZPpj9ij4hGZb4Zy1WF/dT+yaU= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= @@ -175,6 +177,8 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA= +cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= @@ -2332,6 +2336,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5/go.mod h1:UBKtEnL8aqnd+0JHqZ+2qoMDwtuy6cYhhKNoHLBiTQc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index 09fd69117f..ff24924727 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -97,6 +97,15 @@ type StateStoreConfig struct { // HistoricalOffloadScyllaTimeoutMS defaults in the Scylla reader when zero. HistoricalOffloadScyllaTimeoutMS int `mapstructure:"historical-offload-scylla-timeout-ms"` + + // HistoricalOffloadBigtableProjectID enables Bigtable fallback reads when + // project, instance, and table are set. + HistoricalOffloadBigtableProjectID string `mapstructure:"historical-offload-bigtable-project-id"` + HistoricalOffloadBigtableInstance string `mapstructure:"historical-offload-bigtable-instance"` + HistoricalOffloadBigtableTable string `mapstructure:"historical-offload-bigtable-table"` + HistoricalOffloadBigtableFamily string `mapstructure:"historical-offload-bigtable-family"` + HistoricalOffloadBigtableAppProfile string `mapstructure:"historical-offload-bigtable-app-profile"` + HistoricalOffloadBigtableShards int `mapstructure:"historical-offload-bigtable-shards"` } // DefaultStateStoreConfig returns the default StateStoreConfig diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 68e14fce7a..3c8343bd38 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -150,6 +150,16 @@ historical-offload-scylla-password = "{{ .StateStore.HistoricalOffloadScyllaPass historical-offload-scylla-datacenter = "{{ .StateStore.HistoricalOffloadScyllaDatacenter }}" historical-offload-scylla-consistency = "{{ .StateStore.HistoricalOffloadScyllaConsistency }}" historical-offload-scylla-timeout-ms = {{ .StateStore.HistoricalOffloadScyllaTimeoutMS }} + +# Optional Bigtable historical-state fallback. When project, instance, and +# table are set, point reads for versions pruned from local SS fall back to +# Bigtable. Use the same family/shards as the Bigtable consumer. +historical-offload-bigtable-project-id = "{{ .StateStore.HistoricalOffloadBigtableProjectID }}" +historical-offload-bigtable-instance = "{{ .StateStore.HistoricalOffloadBigtableInstance }}" +historical-offload-bigtable-table = "{{ .StateStore.HistoricalOffloadBigtableTable }}" +historical-offload-bigtable-family = "{{ .StateStore.HistoricalOffloadBigtableFamily }}" +historical-offload-bigtable-app-profile = "{{ .StateStore.HistoricalOffloadBigtableAppProfile }}" +historical-offload-bigtable-shards = {{ .StateStore.HistoricalOffloadBigtableShards }} ` // ReceiptStoreConfigTemplate defines the configuration template for receipt-store diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index b6bd267faf..ae7e4515d2 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -92,6 +92,10 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, `historical-offload-scylla-keyspace = ""`, "Missing historical Scylla keyspace") require.Contains(t, output, `historical-offload-scylla-consistency = ""`, "Missing historical Scylla consistency") require.Contains(t, output, "historical-offload-scylla-timeout-ms = 0", "Missing historical Scylla timeout") + require.Contains(t, output, `historical-offload-bigtable-project-id = ""`, "Missing historical Bigtable project") + require.Contains(t, output, `historical-offload-bigtable-instance = ""`, "Missing historical Bigtable instance") + require.Contains(t, output, `historical-offload-bigtable-table = ""`, "Missing historical Bigtable table") + require.Contains(t, output, "historical-offload-bigtable-shards = 0", "Missing historical Bigtable shards") } // TestReceiptStoreConfigTemplate verifies that all field paths in the receipt-store TOML template diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md index 185920d783..7320f3fde4 100644 --- a/sei-db/state_db/ss/offload/consumer/README.md +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -1,14 +1,16 @@ -# Historical Scylla/Cassandra Offload +# Historical State Offload -This is a prototype historical-state backend for ScyllaDB or Cassandra. +This is a prototype historical-state backend for ScyllaDB/Cassandra and +Bigtable. The intended shape is narrow: - local SS remains the hot store for recent state, writes, imports, pruning, and iterators -- Scylla/Cassandra stores immutable MVCC mutation rows for older history -- reads below local SS retention can fall back to Scylla/Cassandra for `Get` and `Has` +- the downstream store keeps immutable MVCC mutation rows for older history +- reads below local SS retention can fall back to the downstream store for `Get` and `Has` -The table layout is built for point reads by `(store_name, state_key, target_version)`: +The Scylla table layout is built for point reads by +`(store_name, state_key, target_version)`: ```sql SELECT version, value, deleted @@ -18,7 +20,17 @@ ORDER BY version DESC LIMIT 1; ``` -Ordered prefix iteration is intentionally not served from Scylla/Cassandra in this prototype. +Bigtable uses salted row keys with an inverted height suffix: + +```text +m | shard(store,key) | store_name | state_key | inverted_height +``` + +Reads scan from `inverted(target_height)` and stop after the first row, giving +the latest write at or before the requested height. + +Ordered prefix iteration is intentionally not served from the offload store in +this prototype. ## Schema @@ -35,17 +47,27 @@ factors before applying it. ## Consumer The consumer reads historical offload changelog messages from Kafka and writes -them into Scylla/Cassandra. Kafka offsets are committed only after the sink -write succeeds. Within each block, mutation rows are written with bounded -concurrency and the version marker is written last. +them into the configured backend. Kafka offsets are committed only after the +sink write succeeds. Mutation rows are written with bounded concurrency and the +version marker is written last. ```bash go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer \ ./sei-db/state_db/ss/offload/consumer/config/example-scylla.json ``` -The example config is local-dev only. Set real Kafka brokers, Scylla hosts, -keyspace, datacenter, and credentials in your own config. +For Bigtable: + +```bash +cbt -project my-gcp-project -instance sei-history createtable state_mutations +cbt -project my-gcp-project -instance sei-history createfamily state_mutations state + +go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer \ + ./sei-db/state_db/ss/offload/consumer/config/example-bigtable.json +``` + +The example configs are local/dev placeholders. Set real Kafka brokers and +backend credentials/config in your own config. ## Node Read Fallback @@ -62,13 +84,25 @@ historical-offload-scylla-consistency = "local_quorum" historical-offload-scylla-timeout-ms = 2000 ``` +Or Bigtable: + +```toml +[state-store] +historical-offload-bigtable-project-id = "my-gcp-project" +historical-offload-bigtable-instance = "sei-history" +historical-offload-bigtable-table = "state_mutations" +historical-offload-bigtable-family = "state" +historical-offload-bigtable-app-profile = "" +historical-offload-bigtable-shards = 256 +``` + Fallback activates only for point reads where the requested version is below the local SS earliest version. Missing rows and tombstones return empty state, same as local SS. ## Current Limits -- No Scylla/Cassandra iterator path. +- No offload iterator path. - No cross-row transaction on ingest; mutation rows are written first and the version marker is written last, so replay is idempotent after partial failure. - No automatic schema creation from the binary. diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go new file mode 100644 index 0000000000..811786dcda --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -0,0 +1,262 @@ +package consumer + +import ( + "context" + "fmt" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" +) + +const defaultBigtableMutationWorkers = 16 + +type BigtableConfig struct { + ProjectID string + InstanceID string + Table string + Family string + AppProfile string + Shards int + MutationWorkers int +} + +func (c *BigtableConfig) ApplyDefaults() { + cfg := c.toHistorical() + cfg.ApplyDefaults() + c.Family = cfg.Family + c.Shards = cfg.Shards + if c.MutationWorkers == 0 { + c.MutationWorkers = defaultBigtableMutationWorkers + } +} + +func (c BigtableConfig) Configured() bool { + return c.toHistorical().Configured() +} + +func (c *BigtableConfig) Validate() error { + cfg := c.toHistorical() + cfg.ApplyDefaults() + if err := cfg.Validate(); err != nil { + return err + } + if c.MutationWorkers < 0 { + return fmt.Errorf("bigtable mutation workers must be non-negative") + } + return nil +} + +func (c BigtableConfig) toHistorical() historical.BigtableConfig { + return historical.BigtableConfig{ + ProjectID: c.ProjectID, + InstanceID: c.InstanceID, + Table: c.Table, + Family: c.Family, + AppProfile: c.AppProfile, + Shards: c.Shards, + } +} + +type bigtableSink struct { + client *historical.BigtableClient + applyBulk historical.BigtableApplyBulkFunc + readRows historical.BigtableReadRowsFunc + family string + shards int + mutationWorkers int +} + +var _ Sink = (*bigtableSink)(nil) +var _ BatchSink = (*bigtableSink)(nil) + +func NewBigtableSink(cfg BigtableConfig) (Sink, error) { + cfg.ApplyDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + ctx := context.Background() + client, err := historical.OpenBigtableClient(ctx, cfg.toHistorical()) + if err != nil { + return nil, err + } + return &bigtableSink{ + client: client, + applyBulk: client.ApplyBulk, + readRows: client.ReadRows, + family: cfg.Family, + shards: cfg.Shards, + mutationWorkers: cfg.MutationWorkers, + }, nil +} + +func (s *bigtableSink) Close() error { + if s.client != nil { + return s.client.Close() + } + return nil +} + +func (s *bigtableSink) LastVersion(ctx context.Context) (int64, error) { + return historical.BigtableLastVersion(ctx, s.readRows) +} + +func (s *bigtableSink) Write(ctx context.Context, rec Record) error { + return s.WriteBatch(ctx, []Record{rec}) +} + +func (s *bigtableSink) WriteBatch(ctx context.Context, records []Record) error { + records = compactRecords(records) + if len(records) == 0 { + return nil + } + if len(records) == 1 { + return s.writeRecord(ctx, records[0]) + } + return s.writeRecordsPipelined(ctx, records) +} + +func (s *bigtableSink) writeRecord(ctx context.Context, rec Record) error { + if rec.Entry == nil { + return nil + } + if err := s.writeRecordRows(ctx, rec.Entry.Version, rec.Entry); err != nil { + return err + } + return s.writeVersionMarker(ctx, rec) +} + +func (s *bigtableSink) writeRecordsPipelined(ctx context.Context, records []Record) error { + rowCtx, cancel := context.WithCancel(ctx) + defer cancel() + g, gctx := errgroup.WithContext(rowCtx) + g.SetLimit(s.effectiveMutationWorkers()) + rowDone := make([]chan error, len(records)) + for i := range records { + rowDone[i] = make(chan error, 1) + i := i + rec := records[i] + g.Go(func() error { + err := s.writeRecordRows(gctx, rec.Entry.Version, rec.Entry) + if err != nil { + err = fmt.Errorf("write bigtable rows version %d: %w", rec.Entry.Version, err) + } + rowDone[i] <- err + return err + }) + } + for i, rec := range records { + if err := <-rowDone[i]; err != nil { + cancel() + _ = g.Wait() + return err + } + if err := s.writeVersionMarker(ctx, rec); err != nil { + cancel() + _ = g.Wait() + return err + } + } + return g.Wait() +} + +func (s *bigtableSink) writeRecordRows(ctx context.Context, version int64, entry *proto.ChangelogEntry) error { + rows := s.recordRowMutations(version, entry) + if len(rows) == 0 { + return nil + } + errs, err := s.applyBulk(ctx, rows) + return bigtableBulkError(rows, errs, err) +} + +func (s *bigtableSink) recordRowMutations(version int64, entry *proto.ChangelogEntry) []historical.BigtableRowMutation { + rows := make([]historical.BigtableRowMutation, 0) + for _, mutation := range compactMutations(entry) { + rows = append(rows, s.mutationRow(version, mutation.storeName, mutation.pair)) + } + for _, up := range entry.Upgrades { + rows = append(rows, s.upgradeRow(version, up)) + } + return rows +} + +func (s *bigtableSink) mutationRow(version int64, storeName string, pair *proto.KVPair) historical.BigtableRowMutation { + ts := historical.BigtableTimestamp(version) + deleted := pair.Delete || pair.Value == nil + row := historical.BigtableRowMutation{ + RowKey: historical.BigtableMutationRowKey(storeName, pair.Key, version, s.shards), + } + if !deleted { + row.SetCells = append(row.SetCells, historical.BigtableSetCell{ + Family: s.family, + Qualifier: historical.BigtableValueColumn, + TimestampMicros: ts, + Value: pair.Value, + }) + } + row.SetCells = append(row.SetCells, historical.BigtableSetCell{ + Family: s.family, + Qualifier: historical.BigtableDeletedColumn, + TimestampMicros: ts, + Value: boolByte(deleted), + }) + return row +} + +func (s *bigtableSink) upgradeRow(version int64, up *proto.TreeNameUpgrade) historical.BigtableRowMutation { + ts := historical.BigtableTimestamp(version) + return historical.BigtableRowMutation{ + RowKey: historical.BigtableUpgradeRowKey(version, up.Name), + SetCells: []historical.BigtableSetCell{ + {Family: s.family, Qualifier: "rename_from", TimestampMicros: ts, Value: []byte(up.RenameFrom)}, + {Family: s.family, Qualifier: historical.BigtableDeletedColumn, TimestampMicros: ts, Value: boolByte(up.Delete)}, + }, + } +} + +func (s *bigtableSink) writeVersionMarker(ctx context.Context, rec Record) error { + version := rec.Entry.Version + ts := historical.BigtableTimestamp(version) + row := historical.BigtableRowMutation{ + RowKey: historical.BigtableVersionRowKey(version), + SetCells: []historical.BigtableSetCell{ + {Family: s.family, Qualifier: "topic", TimestampMicros: ts, Value: []byte(rec.Topic)}, + {Family: s.family, Qualifier: "partition", TimestampMicros: ts, Value: []byte(fmt.Sprintf("%d", rec.Partition))}, + {Family: s.family, Qualifier: "offset", TimestampMicros: ts, Value: []byte(fmt.Sprintf("%d", rec.Offset))}, + {Family: s.family, Qualifier: "ingested_at_unix_nano", TimestampMicros: ts, Value: []byte(fmt.Sprintf("%d", time.Now().UnixNano()))}, + }, + } + errs, err := s.applyBulk(ctx, []historical.BigtableRowMutation{row}) + if err := bigtableBulkError([]historical.BigtableRowMutation{row}, errs, err); err != nil { + return fmt.Errorf("insert bigtable version %d: %w", version, err) + } + return nil +} + +func (s *bigtableSink) effectiveMutationWorkers() int { + if s.mutationWorkers <= 0 { + return defaultBigtableMutationWorkers + } + return s.mutationWorkers +} + +func bigtableBulkError(rows []historical.BigtableRowMutation, errs []error, err error) error { + if err != nil { + return err + } + for i, rowErr := range errs { + if rowErr != nil { + return fmt.Errorf("row %q: %w", rows[i].RowKey, rowErr) + } + } + return nil +} + +func boolByte(v bool) []byte { + if v { + return []byte{1} + } + return []byte{0} +} diff --git a/sei-db/state_db/ss/offload/consumer/bigtable_test.go b/sei-db/state_db/ss/offload/consumer/bigtable_test.go new file mode 100644 index 0000000000..d8903798b3 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/bigtable_test.go @@ -0,0 +1,168 @@ +package consumer + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" + "github.com/stretchr/testify/require" +) + +func TestBigtableConfigApplyDefaults(t *testing.T) { + cfg := BigtableConfig{ + ProjectID: "project", + InstanceID: "instance", + Table: "state", + } + cfg.ApplyDefaults() + require.Equal(t, historical.DefaultBigtableFamily, cfg.Family) + require.Equal(t, historical.DefaultBigtableShards, cfg.Shards) + require.Equal(t, defaultBigtableMutationWorkers, cfg.MutationWorkers) + require.NoError(t, cfg.Validate()) +} + +func TestBigtableSinkWritesMutationRowsAndVersionMarker(t *testing.T) { + var rows []string + sink := &bigtableSink{ + family: historical.DefaultBigtableFamily, + shards: historical.DefaultBigtableShards, + applyBulk: func(_ context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { + for _, mutation := range mutations { + rows = append(rows, mutation.RowKey) + } + return make([]error, len(mutations)), nil + }, + } + entry := &proto.ChangelogEntry{ + Version: 7, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k1"), Value: []byte("old")}, + {Key: []byte("k1"), Value: []byte("new")}, + {Key: []byte("drop"), Delete: true}, + }}, + }}, + Upgrades: []*proto.TreeNameUpgrade{{Name: "new-store"}}, + } + + require.NoError(t, sink.Write(context.Background(), Record{Topic: "t", Partition: 1, Offset: 2, Entry: entry})) + require.Len(t, rows, 4) + require.Equal(t, historical.BigtableMutationRowKey("bank", []byte("k1"), 7, historical.DefaultBigtableShards), rows[0]) + require.Equal(t, historical.BigtableMutationRowKey("bank", []byte("drop"), 7, historical.DefaultBigtableShards), rows[1]) + require.Equal(t, historical.BigtableUpgradeRowKey(7, "new-store"), rows[2]) + require.Equal(t, historical.BigtableVersionRowKey(7), rows[3]) +} + +func TestBigtableSinkWriteBatchPipelinesRowsAndOrdersMarkers(t *testing.T) { + rowStarted := make(chan int64, 2) + markerWritten := make(chan int64, 2) + releaseRows := map[int64]chan struct{}{ + 1: make(chan struct{}), + 2: make(chan struct{}), + } + var activeRows atomic.Int32 + var sawConcurrentRows atomic.Bool + var mu sync.Mutex + rowsDone := make(map[int64]bool) + var markers []int64 + var markerBeforeRowsDone bool + + sink := &bigtableSink{ + family: historical.DefaultBigtableFamily, + shards: historical.DefaultBigtableShards, + mutationWorkers: 2, + applyBulk: func(ctx context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { + version, ok := historical.BigtableVersionFromRowKey(mutations[0].RowKey) + require.True(t, ok) + if mutations[0].RowKey == historical.BigtableVersionRowKey(version) { + mu.Lock() + if !rowsDone[version] { + markerBeforeRowsDone = true + } + markers = append(markers, version) + mu.Unlock() + markerWritten <- version + return make([]error, len(mutations)), nil + } + if activeRows.Add(1) > 1 { + sawConcurrentRows.Store(true) + } + rowStarted <- version + select { + case <-releaseRows[version]: + case <-ctx.Done(): + activeRows.Add(-1) + return nil, ctx.Err() + } + activeRows.Add(-1) + mu.Lock() + rowsDone[version] = true + mu.Unlock() + return make([]error, len(mutations)), nil + }, + } + records := []Record{ + {Entry: &proto.ChangelogEntry{ + Version: 1, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k1"), Value: []byte("v1")}}}, + }}, + }}, + {Entry: &proto.ChangelogEntry{ + Version: 2, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k2"), Value: []byte("v2")}}}, + }}, + }}, + } + + errCh := make(chan error, 1) + go func() { + errCh <- sink.WriteBatch(context.Background(), records) + }() + + started := map[int64]bool{} + for len(started) < 2 { + select { + case version := <-rowStarted: + started[version] = true + case <-time.After(time.Second): + t.Fatal("timed out waiting for pipelined row writes") + } + } + require.True(t, sawConcurrentRows.Load()) + + close(releaseRows[2]) + select { + case version := <-markerWritten: + t.Fatalf("marker %d written before earlier record rows completed", version) + case <-time.After(100 * time.Millisecond): + } + + close(releaseRows[1]) + for _, want := range []int64{1, 2} { + select { + case got := <-markerWritten: + require.Equal(t, want, got) + case <-time.After(time.Second): + t.Fatalf("timed out waiting for marker %d", want) + } + } + select { + case err := <-errCh: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for batch write") + } + mu.Lock() + defer mu.Unlock() + require.False(t, markerBeforeRowsDone) + require.Equal(t, []int64{1, 2}, markers) +} diff --git a/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go b/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go index 65020ca354..60dc2fc241 100644 --- a/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go +++ b/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go @@ -23,9 +23,9 @@ func main() { log.Fatalf("load config: %v", err) } - sink, err := consumer.NewScyllaSink(cfg.Scylla) + sink, err := consumer.NewSinkFromConfig(*cfg) if err != nil { - log.Fatalf("open scylla/cassandra sink: %v", err) + log.Fatalf("open %s sink: %v", cfg.BackendName(), err) } defer func() { _ = sink.Close() }() diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go index f381b2c11f..eceb801fb0 100644 --- a/sei-db/state_db/ss/offload/consumer/config.go +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -4,11 +4,14 @@ import ( "encoding/json" "fmt" "os" + "strings" ) type Config struct { + Backend string Kafka KafkaReaderConfig Scylla ScyllaConfig + Bigtable BigtableConfig Workers int ShardBufferSize int MaxBatchRecords int @@ -19,8 +22,17 @@ func (c *Config) Validate() error { if err := c.Kafka.Validate(); err != nil { return fmt.Errorf("kafka: %w", err) } - if err := c.Scylla.Validate(); err != nil { - return fmt.Errorf("scylla: %w", err) + switch c.BackendName() { + case "scylla": + if err := c.Scylla.Validate(); err != nil { + return fmt.Errorf("scylla: %w", err) + } + case "bigtable": + if err := c.Bigtable.Validate(); err != nil { + return fmt.Errorf("bigtable: %w", err) + } + default: + return fmt.Errorf("unsupported backend %q", c.Backend) } if c.Workers < 0 { return fmt.Errorf("workers must be non-negative") @@ -37,6 +49,28 @@ func (c *Config) Validate() error { return nil } +func (c *Config) BackendName() string { + backend := strings.ToLower(strings.TrimSpace(c.Backend)) + if backend != "" { + return backend + } + if c.Bigtable.Configured() && !c.Scylla.Configured() { + return "bigtable" + } + return "scylla" +} + +func NewSinkFromConfig(cfg Config) (Sink, error) { + switch cfg.BackendName() { + case "scylla": + return NewScyllaSink(cfg.Scylla) + case "bigtable": + return NewBigtableSink(cfg.Bigtable) + default: + return nil, fmt.Errorf("unsupported backend %q", cfg.Backend) + } +} + func LoadConfig(path string) (*Config, error) { // #nosec G304 -- config path is supplied by the operator on the command line. raw, err := os.ReadFile(path) diff --git a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json new file mode 100644 index 0000000000..8af5152b35 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json @@ -0,0 +1,22 @@ +{ + "Backend": "bigtable", + "Kafka": { + "Brokers": ["localhost:9092"], + "Topic": "historical-offload", + "GroupID": "historical-bigtable", + "StartOffset": "first" + }, + "Bigtable": { + "ProjectID": "my-gcp-project", + "InstanceID": "sei-history", + "Table": "state_mutations", + "Family": "state", + "AppProfile": "", + "Shards": 256, + "MutationWorkers": 16 + }, + "Workers": 16, + "ShardBufferSize": 128, + "MaxBatchRecords": 16, + "BatchMaxWaitMS": 10 +} diff --git a/sei-db/state_db/ss/offload/consumer/config_test.go b/sei-db/state_db/ss/offload/consumer/config_test.go new file mode 100644 index 0000000000..6e335bd3cd --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/config_test.go @@ -0,0 +1,33 @@ +package consumer + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfigBackendName(t *testing.T) { + require.Equal(t, "scylla", (&Config{}).BackendName()) + require.Equal(t, "bigtable", (&Config{Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) + require.Equal(t, "scylla", (&Config{Backend: "Scylla", Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) +} + +func TestConfigValidateBigtable(t *testing.T) { + cfg := Config{ + Backend: "bigtable", + Kafka: KafkaReaderConfig{ + Brokers: []string{"localhost:9092"}, + Topic: "historical-offload", + GroupID: "historical-bigtable", + }, + Bigtable: BigtableConfig{ + ProjectID: "project", + InstanceID: "instance", + Table: "state", + }, + } + require.NoError(t, cfg.Validate()) + + cfg.Bigtable.Table = "" + require.ErrorContains(t, cfg.Validate(), "bigtable") +} diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index bcf061c7b0..babec955e7 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -27,6 +27,10 @@ type ScyllaConfig struct { MutationWorkers int } +func (c ScyllaConfig) Configured() bool { + return len(c.Hosts) != 0 || c.Keyspace != "" +} + func (c *ScyllaConfig) ApplyDefaults() { cfg := c.toHistorical() cfg.ApplyDefaults() diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go new file mode 100644 index 0000000000..c1f075e4ed --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -0,0 +1,533 @@ +package historical + +import ( + "context" + "encoding/binary" + "fmt" + "hash/fnv" + "io" + "regexp" + "strings" + "sync" + "time" + + "cloud.google.com/go/bigtable/apiv2/bigtablepb" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" +) + +const ( + DefaultBigtableFamily = "state" + DefaultBigtableShards = 256 + + defaultBigtableReadWorkers = 16 + + bigtableMutationPrefix = byte('m') + bigtableVersionPrefix = byte('v') + bigtableUpgradePrefix = byte('u') + + BigtableValueColumn = "value" + BigtableDeletedColumn = "deleted" +) + +const bigtableEndpoint = "bigtable.googleapis.com:443" + +type BigtableClient struct { + conn *grpc.ClientConn + data bigtablepb.BigtableClient + tableName string + appProfile string +} + +type BigtableCell struct { + Family string + Qualifier string + Value []byte +} + +type BigtableRow struct { + Key string + Cells []BigtableCell +} + +type BigtableSetCell struct { + Family string + Qualifier string + TimestampMicros int64 + Value []byte +} + +type BigtableRowMutation struct { + RowKey string + SetCells []BigtableSetCell +} + +type BigtableConfig struct { + ProjectID string + InstanceID string + Table string + Family string + AppProfile string + Shards int +} + +func (c *BigtableConfig) ApplyDefaults() { + if c.Family == "" { + c.Family = DefaultBigtableFamily + } + if c.Shards == 0 { + c.Shards = DefaultBigtableShards + } +} + +func (c BigtableConfig) Configured() bool { + return strings.TrimSpace(c.ProjectID) != "" || + strings.TrimSpace(c.InstanceID) != "" || + strings.TrimSpace(c.Table) != "" +} + +func (c *BigtableConfig) Validate() error { + if strings.TrimSpace(c.ProjectID) == "" { + return fmt.Errorf("bigtable project id is required") + } + if strings.TrimSpace(c.InstanceID) == "" { + return fmt.Errorf("bigtable instance id is required") + } + if strings.TrimSpace(c.Table) == "" { + return fmt.Errorf("bigtable table is required") + } + if strings.TrimSpace(c.Family) == "" { + return fmt.Errorf("bigtable family is required") + } + if c.Shards < 0 || c.Shards > 65535 { + return fmt.Errorf("bigtable shards must be between 1 and 65535") + } + return nil +} + +func NewBigtableReader(cfg BigtableConfig) (Reader, error) { + ctx := context.Background() + client, err := OpenBigtableClient(ctx, cfg) + if err != nil { + return nil, err + } + cfg.ApplyDefaults() + return &bigtableReader{ + client: client, + readRows: client.ReadRows, + family: cfg.Family, + shards: cfg.Shards, + }, nil +} + +func OpenBigtableClient(ctx context.Context, cfg BigtableConfig) (*BigtableClient, error) { + cfg.ApplyDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + perRPC, err := oauth.NewApplicationDefault(ctx, "https://www.googleapis.com/auth/bigtable.data", "https://www.googleapis.com/auth/cloud-platform") + if err != nil { + return nil, fmt.Errorf("bigtable auth: %w", err) + } + conn, err := grpc.DialContext(ctx, bigtableEndpoint, + grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), + grpc.WithPerRPCCredentials(perRPC), + ) + if err != nil { + return nil, fmt.Errorf("open bigtable connection: %w", err) + } + return &BigtableClient{ + conn: conn, + data: bigtablepb.NewBigtableClient(conn), + tableName: bigtableTableName(cfg.ProjectID, cfg.InstanceID, cfg.Table), + appProfile: cfg.AppProfile, + }, nil +} + +type BigtableReadRowsFunc func(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error + +type BigtableApplyBulkFunc func(ctx context.Context, rows []BigtableRowMutation) ([]error, error) + +func (c *BigtableClient) Close() error { + if c.conn == nil { + return nil + } + return c.conn.Close() +} + +func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error { + req := &bigtablepb.ReadRowsRequest{ + TableName: c.tableName, + AppProfileId: c.appProfile, + Rows: &bigtablepb.RowSet{RowRanges: []*bigtablepb.RowRange{{ + StartKey: &bigtablepb.RowRange_StartKeyClosed{StartKeyClosed: startKey}, + EndKey: &bigtablepb.RowRange_EndKeyOpen{EndKeyOpen: endKey}, + }}}, + RowsLimit: limit, + } + if len(endKey) == 0 { + req.Rows.RowRanges[0].EndKey = nil + } + if family != "" { + req.Filter = &bigtablepb.RowFilter{ + Filter: &bigtablepb.RowFilter_FamilyNameRegexFilter{FamilyNameRegexFilter: regexp.QuoteMeta(family)}, + } + } + stream, err := c.data.ReadRows(ctx, req) + if err != nil { + return err + } + var builder bigtableRowBuilder + for { + resp, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + for _, chunk := range resp.Chunks { + row, committed, err := builder.add(chunk) + if err != nil { + return err + } + if committed && !f(row) { + return nil + } + } + } +} + +func (c *BigtableClient) ApplyBulk(ctx context.Context, rows []BigtableRowMutation) ([]error, error) { + if len(rows) == 0 { + return nil, nil + } + entries := make([]*bigtablepb.MutateRowsRequest_Entry, 0, len(rows)) + for _, row := range rows { + entry := &bigtablepb.MutateRowsRequest_Entry{RowKey: []byte(row.RowKey)} + for _, cell := range row.SetCells { + entry.Mutations = append(entry.Mutations, &bigtablepb.Mutation{ + Mutation: &bigtablepb.Mutation_SetCell_{SetCell: &bigtablepb.Mutation_SetCell{ + FamilyName: cell.Family, + ColumnQualifier: []byte(cell.Qualifier), + TimestampMicros: cell.TimestampMicros, + Value: cell.Value, + }}, + }) + } + entries = append(entries, entry) + } + stream, err := c.data.MutateRows(ctx, &bigtablepb.MutateRowsRequest{ + TableName: c.tableName, + AppProfileId: c.appProfile, + Entries: entries, + }) + if err != nil { + return nil, err + } + rowErrs := make([]error, len(rows)) + seen := make([]bool, len(rows)) + for { + resp, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return rowErrs, err + } + for _, entry := range resp.Entries { + if entry.Index < 0 || int(entry.Index) >= len(rowErrs) { + return rowErrs, fmt.Errorf("bigtable returned invalid mutation index %d", entry.Index) + } + idx := int(entry.Index) + seen[idx] = true + if st := entry.Status; st != nil && st.Code != 0 { + rowErrs[idx] = fmt.Errorf("bigtable status %d: %s", st.Code, st.Message) + } + } + } + for i := range seen { + if !seen[i] { + rowErrs[i] = fmt.Errorf("bigtable missing mutation result") + } + } + return rowErrs, nil +} + +type bigtableReader struct { + client *BigtableClient + readRows BigtableReadRowsFunc + family string + shards int +} + +var _ Reader = (*bigtableReader)(nil) + +func (r *bigtableReader) Close() error { + if r.client != nil { + return r.client.Close() + } + return nil +} + +func (r *bigtableReader) LastVersion(ctx context.Context) (int64, error) { + return BigtableLastVersion(ctx, r.readRows) +} + +func (r *bigtableReader) Has(ctx context.Context, storeName string, key []byte, targetVersion int64) (bool, error) { + _, err := r.Get(ctx, storeName, key, targetVersion) + if err != nil { + if err == ErrNotFound { + return false, nil + } + return false, err + } + return true, nil +} + +func (r *bigtableReader) Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { + prefix := []byte(BigtableMutationRowPrefix(storeName, key, r.shards)) + start := []byte(BigtableMutationRowKey(storeName, key, targetVersion, r.shards)) + var row BigtableRow + err := r.readRows(ctx, start, bigtablePrefixEnd(prefix), 1, r.family, func(r BigtableRow) bool { + row = r + return false + }) + if err != nil { + return Value{}, fmt.Errorf("bigtable get lookup: %w", err) + } + if row.Key == "" { + return Value{}, ErrNotFound + } + return BigtableValueFromRow(row, r.family) +} + +func (r *bigtableReader) BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) { + out := make(map[Lookup]Value, len(lookups)) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(defaultBigtableReadWorkers) + var mu sync.Mutex + for _, lookup := range lookups { + lookup := lookup + g.Go(func() error { + value, err := r.Get(gctx, lookup.StoreName, []byte(lookup.Key), targetVersion) + if err != nil { + if err == ErrNotFound { + return nil + } + return err + } + mu.Lock() + out[lookup] = value + mu.Unlock() + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return out, nil +} + +func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (int64, error) { + var maxVersion int64 + for bucket := 0; bucket < VersionBucketCount; bucket++ { + prefix := bigtableVersionRowPrefix(bucket) + err := readRows(ctx, prefix, bigtablePrefixEnd(prefix), 1, "", func(row BigtableRow) bool { + version, ok := BigtableVersionFromRowKey(row.Key) + if ok && version > maxVersion { + maxVersion = version + } + return false + }) + if err != nil { + return 0, fmt.Errorf("read latest bigtable version bucket %d: %w", bucket, err) + } + } + return maxVersion, nil +} + +func BigtableValueFromRow(row BigtableRow, family string) (Value, error) { + version, ok := BigtableVersionFromRowKey(row.Key) + if !ok { + return Value{}, fmt.Errorf("invalid bigtable mutation row key") + } + var value []byte + deleted := false + for _, cell := range row.Cells { + if cell.Family != family { + continue + } + switch cell.Qualifier { + case BigtableValueColumn: + value = append([]byte(nil), cell.Value...) + case BigtableDeletedColumn: + deleted = len(cell.Value) > 0 && cell.Value[0] == 1 + } + } + if deleted || value == nil { + return Value{}, ErrNotFound + } + return Value{Bytes: value, Version: version}, nil +} + +func BigtableMutationRowPrefix(storeName string, key []byte, shards int) string { + shards = normalizeBigtableShards(shards) + shard := bigtableShard(storeName, key, shards) + prefix := make([]byte, 1+2+2+len(storeName)+4+len(key)) + prefix[0] = bigtableMutationPrefix + binary.BigEndian.PutUint16(prefix[1:], uint16(shard)) + binary.BigEndian.PutUint16(prefix[3:], uint16(len(storeName))) + copy(prefix[5:], storeName) + keyOffset := 5 + len(storeName) + binary.BigEndian.PutUint32(prefix[keyOffset:], uint32(len(key))) + copy(prefix[keyOffset+4:], key) + return string(prefix) +} + +func BigtableMutationRowKey(storeName string, key []byte, version int64, shards int) string { + prefix := []byte(BigtableMutationRowPrefix(storeName, key, shards)) + return string(append(prefix, bigtableInvertedVersion(version)...)) +} + +func BigtableVersionRowKey(version int64) string { + prefix := bigtableVersionRowPrefix(VersionBucket(version)) + return string(append(prefix, bigtableInvertedVersion(version)...)) +} + +func BigtableUpgradeRowKey(version int64, name string) string { + key := make([]byte, 1+8+2+len(name)) + key[0] = bigtableUpgradePrefix + copy(key[1:], bigtableInvertedVersion(version)) + binary.BigEndian.PutUint16(key[9:], uint16(len(name))) + copy(key[11:], name) + return string(key) +} + +func BigtableVersionFromRowKey(rowKey string) (int64, bool) { + key := []byte(rowKey) + switch { + case len(key) >= 1+2+8 && key[0] == bigtableVersionPrefix: + return bigtableDecodeInvertedVersion(key[3:11]), true + case len(key) >= 8 && key[0] == bigtableMutationPrefix: + return bigtableDecodeInvertedVersion(key[len(key)-8:]), true + default: + return 0, false + } +} + +func BigtableTimestamp(version int64) int64 { + return version * int64(time.Millisecond/time.Microsecond) +} + +type bigtableRowBuilder struct { + key []byte + family string + qualifier string + value []byte + inCell bool + cells []BigtableCell +} + +func (b *bigtableRowBuilder) add(chunk *bigtablepb.ReadRowsResponse_CellChunk) (BigtableRow, bool, error) { + if chunk.GetResetRow() { + b.reset() + return BigtableRow{}, false, nil + } + if len(chunk.RowKey) != 0 { + b.key = append(b.key[:0], chunk.RowKey...) + b.family = "" + b.qualifier = "" + b.value = nil + b.inCell = false + b.cells = b.cells[:0] + } + if chunk.FamilyName != nil { + b.family = chunk.FamilyName.Value + } + if chunk.Qualifier != nil { + b.qualifier = string(chunk.Qualifier.Value) + b.value = b.value[:0] + b.inCell = true + } + b.value = append(b.value, chunk.Value...) + if b.inCell && chunk.ValueSize == 0 { + b.cells = append(b.cells, BigtableCell{ + Family: b.family, + Qualifier: b.qualifier, + Value: append([]byte(nil), b.value...), + }) + b.value = nil + b.inCell = false + } + if !chunk.GetCommitRow() { + return BigtableRow{}, false, nil + } + if len(b.key) == 0 { + return BigtableRow{}, false, fmt.Errorf("bigtable committed row without key") + } + row := BigtableRow{ + Key: string(append([]byte(nil), b.key...)), + Cells: append([]BigtableCell(nil), b.cells...), + } + b.reset() + return row, true, nil +} + +func (b *bigtableRowBuilder) reset() { + b.key = nil + b.family = "" + b.qualifier = "" + b.value = nil + b.inCell = false + b.cells = nil +} + +func bigtableTableName(projectID, instanceID, table string) string { + return fmt.Sprintf("projects/%s/instances/%s/tables/%s", projectID, instanceID, table) +} + +func bigtableVersionRowPrefix(bucket int) []byte { + prefix := make([]byte, 1+2) + prefix[0] = bigtableVersionPrefix + binary.BigEndian.PutUint16(prefix[1:], uint16(bucket)) + return prefix +} + +func bigtablePrefixEnd(prefix []byte) []byte { + end := append([]byte(nil), prefix...) + for i := len(end) - 1; i >= 0; i-- { + if end[i] != 0xff { + end[i]++ + return end[:i+1] + } + } + return nil +} + +func bigtableInvertedVersion(version int64) []byte { + out := make([]byte, 8) + binary.BigEndian.PutUint64(out, ^uint64(version)) + return out +} + +func bigtableDecodeInvertedVersion(encoded []byte) int64 { + return int64(^binary.BigEndian.Uint64(encoded)) +} + +func bigtableShard(storeName string, key []byte, shards int) int { + h := fnv.New32a() + _, _ = h.Write([]byte(storeName)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write(key) + return int(h.Sum32() % uint32(shards)) +} + +func normalizeBigtableShards(shards int) int { + if shards <= 0 { + return DefaultBigtableShards + } + return shards +} diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go new file mode 100644 index 0000000000..2c78f2f228 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -0,0 +1,93 @@ +package historical + +import ( + "context" + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBigtableConfigDefaultsAndValidate(t *testing.T) { + cfg := BigtableConfig{ + ProjectID: "project", + InstanceID: "instance", + Table: "state", + } + cfg.ApplyDefaults() + require.Equal(t, DefaultBigtableFamily, cfg.Family) + require.Equal(t, DefaultBigtableShards, cfg.Shards) + require.NoError(t, cfg.Validate()) + + missingProject := BigtableConfig{InstanceID: "i", Table: "t", Family: "f"} + missingInstance := BigtableConfig{ProjectID: "p", Table: "t", Family: "f"} + missingTable := BigtableConfig{ProjectID: "p", InstanceID: "i", Family: "f"} + require.ErrorContains(t, missingProject.Validate(), "project") + require.ErrorContains(t, missingInstance.Validate(), "instance") + require.ErrorContains(t, missingTable.Validate(), "table") +} + +func TestBigtableMutationRowKeyOrdersLatestVersionFirst(t *testing.T) { + key40 := BigtableMutationRowKey("bank", []byte("k1"), 40, 256) + key60 := BigtableMutationRowKey("bank", []byte("k1"), 60, 256) + key80 := BigtableMutationRowKey("bank", []byte("k1"), 80, 256) + keys := []string{key40, key80, key60} + sort.Strings(keys) + require.Equal(t, []string{key80, key60, key40}, keys) + + version, ok := BigtableVersionFromRowKey(key60) + require.True(t, ok) + require.Equal(t, int64(60), version) + require.NotEqual(t, + BigtableMutationRowPrefix("bank", []byte("k"), 256), + BigtableMutationRowPrefix("bank", []byte("k1"), 256), + ) +} + +func TestBigtableValueFromRow(t *testing.T) { + rowKey := BigtableMutationRowKey("bank", []byte("k"), 7, 256) + row := BigtableRow{ + Key: rowKey, + Cells: []BigtableCell{ + {Family: DefaultBigtableFamily, Qualifier: BigtableValueColumn, Value: []byte("value")}, + {Family: DefaultBigtableFamily, Qualifier: BigtableDeletedColumn, Value: []byte{0}}, + }, + } + value, err := BigtableValueFromRow(row, DefaultBigtableFamily) + require.NoError(t, err) + require.Equal(t, []byte("value"), value.Bytes) + require.Equal(t, int64(7), value.Version) + value.Bytes[0] = 'V' + require.Equal(t, []byte("value"), row.Cells[0].Value) + + row.Cells[1].Value = []byte{1} + _, err = BigtableValueFromRow(row, DefaultBigtableFamily) + require.ErrorIs(t, err, ErrNotFound) +} + +func TestBigtableReaderGetUsesMVCCRange(t *testing.T) { + wantRow := BigtableMutationRowKey("bank", []byte("k"), 40, 256) + reader := &bigtableReader{ + family: DefaultBigtableFamily, + shards: 256, + readRows: func(_ context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error { + require.Equal(t, []byte(BigtableMutationRowKey("bank", []byte("k"), 60, 256)), startKey) + require.NotEmpty(t, endKey) + require.Equal(t, int64(1), limit) + require.Equal(t, DefaultBigtableFamily, family) + f(BigtableRow{ + Key: wantRow, + Cells: []BigtableCell{ + {Family: DefaultBigtableFamily, Qualifier: BigtableValueColumn, Value: []byte("v40")}, + {Family: DefaultBigtableFamily, Qualifier: BigtableDeletedColumn, Value: []byte{0}}, + }, + }) + return nil + }, + } + + value, err := reader.Get(context.Background(), "bank", []byte("k"), 60) + require.NoError(t, err) + require.Equal(t, []byte("v40"), value.Bytes) + require.Equal(t, int64(40), value.Version) +} diff --git a/sei-db/state_db/ss/store.go b/sei-db/state_db/ss/store.go index dc1ce2edc2..e102af34c9 100644 --- a/sei-db/state_db/ss/store.go +++ b/sei-db/state_db/ss/store.go @@ -20,9 +20,30 @@ func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.Stat if err != nil { return nil, err } - if !scyllaHistoricalOffloadConfigured(ssConfig) { + scyllaConfigured := scyllaHistoricalOffloadConfigured(ssConfig) + bigtableConfigured := bigtableHistoricalOffloadConfigured(ssConfig) + if scyllaConfigured && bigtableConfigured { + _ = primary.Close() + return nil, fmt.Errorf("only one historical offload fallback can be configured") + } + if !scyllaConfigured && !bigtableConfigured { return primary, nil } + if bigtableConfigured { + reader, err := historical.NewBigtableReader(historical.BigtableConfig{ + ProjectID: ssConfig.HistoricalOffloadBigtableProjectID, + InstanceID: ssConfig.HistoricalOffloadBigtableInstance, + Table: ssConfig.HistoricalOffloadBigtableTable, + Family: ssConfig.HistoricalOffloadBigtableFamily, + AppProfile: ssConfig.HistoricalOffloadBigtableAppProfile, + Shards: ssConfig.HistoricalOffloadBigtableShards, + }) + if err != nil { + _ = primary.Close() + return nil, fmt.Errorf("open bigtable historical offload reader: %w", err) + } + return historical.NewFallbackStateStore(primary, reader), nil + } reader, err := historical.NewScyllaReader(historical.ScyllaConfig{ Hosts: splitCSV(ssConfig.HistoricalOffloadScyllaHosts), Keyspace: ssConfig.HistoricalOffloadScyllaKeyspace, @@ -44,6 +65,12 @@ func scyllaHistoricalOffloadConfigured(cfg config.StateStoreConfig) bool { strings.TrimSpace(cfg.HistoricalOffloadScyllaKeyspace) != "" } +func bigtableHistoricalOffloadConfigured(cfg config.StateStoreConfig) bool { + return strings.TrimSpace(cfg.HistoricalOffloadBigtableProjectID) != "" || + strings.TrimSpace(cfg.HistoricalOffloadBigtableInstance) != "" || + strings.TrimSpace(cfg.HistoricalOffloadBigtableTable) != "" +} + func splitCSV(value string) []string { if strings.TrimSpace(value) == "" { return nil From d3035ecfdd2eb64e3ab8d74305f90a512a6c1c07 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 14 May 2026 14:01:27 -0400 Subject: [PATCH 10/41] Simplify Bigtable offload batching --- sei-db/state_db/ss/offload/consumer/README.md | 3 +- .../state_db/ss/offload/consumer/bigtable.go | 134 +++--------------- .../ss/offload/consumer/bigtable_test.go | 123 ++++------------ sei-db/state_db/ss/offload/consumer/config.go | 4 +- .../consumer/config/example-bigtable.json | 3 +- 5 files changed, 51 insertions(+), 216 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md index 7320f3fde4..f716835cfb 100644 --- a/sei-db/state_db/ss/offload/consumer/README.md +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -48,8 +48,7 @@ factors before applying it. The consumer reads historical offload changelog messages from Kafka and writes them into the configured backend. Kafka offsets are committed only after the -sink write succeeds. Mutation rows are written with bounded concurrency and the -version marker is written last. +sink write succeeds. Mutation rows are written before the version marker. ```bash go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer \ diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index 811786dcda..2bff1be8cc 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -5,68 +5,18 @@ import ( "fmt" "time" - "golang.org/x/sync/errgroup" - "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" ) -const defaultBigtableMutationWorkers = 16 - -type BigtableConfig struct { - ProjectID string - InstanceID string - Table string - Family string - AppProfile string - Shards int - MutationWorkers int -} - -func (c *BigtableConfig) ApplyDefaults() { - cfg := c.toHistorical() - cfg.ApplyDefaults() - c.Family = cfg.Family - c.Shards = cfg.Shards - if c.MutationWorkers == 0 { - c.MutationWorkers = defaultBigtableMutationWorkers - } -} - -func (c BigtableConfig) Configured() bool { - return c.toHistorical().Configured() -} - -func (c *BigtableConfig) Validate() error { - cfg := c.toHistorical() - cfg.ApplyDefaults() - if err := cfg.Validate(); err != nil { - return err - } - if c.MutationWorkers < 0 { - return fmt.Errorf("bigtable mutation workers must be non-negative") - } - return nil -} - -func (c BigtableConfig) toHistorical() historical.BigtableConfig { - return historical.BigtableConfig{ - ProjectID: c.ProjectID, - InstanceID: c.InstanceID, - Table: c.Table, - Family: c.Family, - AppProfile: c.AppProfile, - Shards: c.Shards, - } -} +type BigtableConfig = historical.BigtableConfig type bigtableSink struct { - client *historical.BigtableClient - applyBulk historical.BigtableApplyBulkFunc - readRows historical.BigtableReadRowsFunc - family string - shards int - mutationWorkers int + client *historical.BigtableClient + applyBulk historical.BigtableApplyBulkFunc + readRows historical.BigtableReadRowsFunc + family string + shards int } var _ Sink = (*bigtableSink)(nil) @@ -78,17 +28,16 @@ func NewBigtableSink(cfg BigtableConfig) (Sink, error) { return nil, err } ctx := context.Background() - client, err := historical.OpenBigtableClient(ctx, cfg.toHistorical()) + client, err := historical.OpenBigtableClient(ctx, cfg) if err != nil { return nil, err } return &bigtableSink{ - client: client, - applyBulk: client.ApplyBulk, - readRows: client.ReadRows, - family: cfg.Family, - shards: cfg.Shards, - mutationWorkers: cfg.MutationWorkers, + client: client, + applyBulk: client.ApplyBulk, + readRows: client.ReadRows, + family: cfg.Family, + shards: cfg.Shards, }, nil } @@ -112,58 +61,22 @@ func (s *bigtableSink) WriteBatch(ctx context.Context, records []Record) error { if len(records) == 0 { return nil } - if len(records) == 1 { - return s.writeRecord(ctx, records[0]) - } - return s.writeRecordsPipelined(ctx, records) -} - -func (s *bigtableSink) writeRecord(ctx context.Context, rec Record) error { - if rec.Entry == nil { - return nil - } - if err := s.writeRecordRows(ctx, rec.Entry.Version, rec.Entry); err != nil { + if err := s.writeRecordRows(ctx, records); err != nil { return err } - return s.writeVersionMarker(ctx, rec) -} - -func (s *bigtableSink) writeRecordsPipelined(ctx context.Context, records []Record) error { - rowCtx, cancel := context.WithCancel(ctx) - defer cancel() - g, gctx := errgroup.WithContext(rowCtx) - g.SetLimit(s.effectiveMutationWorkers()) - rowDone := make([]chan error, len(records)) - for i := range records { - rowDone[i] = make(chan error, 1) - i := i - rec := records[i] - g.Go(func() error { - err := s.writeRecordRows(gctx, rec.Entry.Version, rec.Entry) - if err != nil { - err = fmt.Errorf("write bigtable rows version %d: %w", rec.Entry.Version, err) - } - rowDone[i] <- err - return err - }) - } - for i, rec := range records { - if err := <-rowDone[i]; err != nil { - cancel() - _ = g.Wait() - return err - } + for _, rec := range records { if err := s.writeVersionMarker(ctx, rec); err != nil { - cancel() - _ = g.Wait() return err } } - return g.Wait() + return nil } -func (s *bigtableSink) writeRecordRows(ctx context.Context, version int64, entry *proto.ChangelogEntry) error { - rows := s.recordRowMutations(version, entry) +func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) error { + var rows []historical.BigtableRowMutation + for _, rec := range records { + rows = append(rows, s.recordRowMutations(rec.Entry.Version, rec.Entry)...) + } if len(rows) == 0 { return nil } @@ -235,13 +148,6 @@ func (s *bigtableSink) writeVersionMarker(ctx context.Context, rec Record) error return nil } -func (s *bigtableSink) effectiveMutationWorkers() int { - if s.mutationWorkers <= 0 { - return defaultBigtableMutationWorkers - } - return s.mutationWorkers -} - func bigtableBulkError(rows []historical.BigtableRowMutation, errs []error, err error) error { if err != nil { return err diff --git a/sei-db/state_db/ss/offload/consumer/bigtable_test.go b/sei-db/state_db/ss/offload/consumer/bigtable_test.go index d8903798b3..aa084fc9e4 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable_test.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable_test.go @@ -2,29 +2,13 @@ package consumer import ( "context" - "sync" - "sync/atomic" "testing" - "time" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" "github.com/stretchr/testify/require" ) -func TestBigtableConfigApplyDefaults(t *testing.T) { - cfg := BigtableConfig{ - ProjectID: "project", - InstanceID: "instance", - Table: "state", - } - cfg.ApplyDefaults() - require.Equal(t, historical.DefaultBigtableFamily, cfg.Family) - require.Equal(t, historical.DefaultBigtableShards, cfg.Shards) - require.Equal(t, defaultBigtableMutationWorkers, cfg.MutationWorkers) - require.NoError(t, cfg.Validate()) -} - func TestBigtableSinkWritesMutationRowsAndVersionMarker(t *testing.T) { var rows []string sink := &bigtableSink{ @@ -58,51 +42,27 @@ func TestBigtableSinkWritesMutationRowsAndVersionMarker(t *testing.T) { require.Equal(t, historical.BigtableVersionRowKey(7), rows[3]) } -func TestBigtableSinkWriteBatchPipelinesRowsAndOrdersMarkers(t *testing.T) { - rowStarted := make(chan int64, 2) - markerWritten := make(chan int64, 2) - releaseRows := map[int64]chan struct{}{ - 1: make(chan struct{}), - 2: make(chan struct{}), - } - var activeRows atomic.Int32 - var sawConcurrentRows atomic.Bool - var mu sync.Mutex - rowsDone := make(map[int64]bool) - var markers []int64 +func TestBigtableSinkWriteBatchWritesRowsBeforeMarkers(t *testing.T) { + var rowVersions []int64 + var markerVersions []int64 var markerBeforeRowsDone bool sink := &bigtableSink{ - family: historical.DefaultBigtableFamily, - shards: historical.DefaultBigtableShards, - mutationWorkers: 2, - applyBulk: func(ctx context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { - version, ok := historical.BigtableVersionFromRowKey(mutations[0].RowKey) - require.True(t, ok) - if mutations[0].RowKey == historical.BigtableVersionRowKey(version) { - mu.Lock() - if !rowsDone[version] { - markerBeforeRowsDone = true - } - markers = append(markers, version) - mu.Unlock() - markerWritten <- version - return make([]error, len(mutations)), nil - } - if activeRows.Add(1) > 1 { - sawConcurrentRows.Store(true) + family: historical.DefaultBigtableFamily, + shards: historical.DefaultBigtableShards, + applyBulk: func(_ context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { + isMarkerBatch := len(mutations) == 1 && mutations[0].RowKey == historical.BigtableVersionRowKey(mustBigtableVersion(t, mutations[0].RowKey)) + if isMarkerBatch && len(rowVersions) != 2 { + markerBeforeRowsDone = true } - rowStarted <- version - select { - case <-releaseRows[version]: - case <-ctx.Done(): - activeRows.Add(-1) - return nil, ctx.Err() + for _, mutation := range mutations { + version := mustBigtableVersion(t, mutation.RowKey) + if mutation.RowKey == historical.BigtableVersionRowKey(version) { + markerVersions = append(markerVersions, version) + } else { + rowVersions = append(rowVersions, version) + } } - activeRows.Add(-1) - mu.Lock() - rowsDone[version] = true - mu.Unlock() return make([]error, len(mutations)), nil }, } @@ -123,46 +83,15 @@ func TestBigtableSinkWriteBatchPipelinesRowsAndOrdersMarkers(t *testing.T) { }}, } - errCh := make(chan error, 1) - go func() { - errCh <- sink.WriteBatch(context.Background(), records) - }() - - started := map[int64]bool{} - for len(started) < 2 { - select { - case version := <-rowStarted: - started[version] = true - case <-time.After(time.Second): - t.Fatal("timed out waiting for pipelined row writes") - } - } - require.True(t, sawConcurrentRows.Load()) - - close(releaseRows[2]) - select { - case version := <-markerWritten: - t.Fatalf("marker %d written before earlier record rows completed", version) - case <-time.After(100 * time.Millisecond): - } - - close(releaseRows[1]) - for _, want := range []int64{1, 2} { - select { - case got := <-markerWritten: - require.Equal(t, want, got) - case <-time.After(time.Second): - t.Fatalf("timed out waiting for marker %d", want) - } - } - select { - case err := <-errCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("timed out waiting for batch write") - } - mu.Lock() - defer mu.Unlock() + require.NoError(t, sink.WriteBatch(context.Background(), records)) require.False(t, markerBeforeRowsDone) - require.Equal(t, []int64{1, 2}, markers) + require.Equal(t, []int64{1, 2}, rowVersions) + require.Equal(t, []int64{1, 2}, markerVersions) +} + +func mustBigtableVersion(t *testing.T, rowKey string) int64 { + t.Helper() + version, ok := historical.BigtableVersionFromRowKey(rowKey) + require.True(t, ok) + return version } diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go index eceb801fb0..806cfae31e 100644 --- a/sei-db/state_db/ss/offload/consumer/config.go +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -28,7 +28,9 @@ func (c *Config) Validate() error { return fmt.Errorf("scylla: %w", err) } case "bigtable": - if err := c.Bigtable.Validate(); err != nil { + bigtable := c.Bigtable + bigtable.ApplyDefaults() + if err := bigtable.Validate(); err != nil { return fmt.Errorf("bigtable: %w", err) } default: diff --git a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json index 8af5152b35..32e9250678 100644 --- a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json +++ b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json @@ -12,8 +12,7 @@ "Table": "state_mutations", "Family": "state", "AppProfile": "", - "Shards": 256, - "MutationWorkers": 16 + "Shards": 256 }, "Workers": 16, "ShardBufferSize": 128, From 9890f78d393ac5637cc662e24582ac66636219d1 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 14 May 2026 15:02:32 -0400 Subject: [PATCH 11/41] Fix Bigtable lint findings --- .../state_db/ss/offload/consumer/bigtable.go | 7 +- sei-db/state_db/ss/offload/consumer/config.go | 17 +++-- .../ss/offload/consumer/config_test.go | 10 +-- .../ss/offload/historical/bigtable.go | 72 +++++++++++++++---- 4 files changed, 79 insertions(+), 27 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index 2bff1be8cc..409c758421 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -73,7 +73,7 @@ func (s *bigtableSink) WriteBatch(ctx context.Context, records []Record) error { } func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) error { - var rows []historical.BigtableRowMutation + rows := make([]historical.BigtableRowMutation, 0, len(records)) for _, rec := range records { rows = append(rows, s.recordRowMutations(rec.Entry.Version, rec.Entry)...) } @@ -85,8 +85,9 @@ func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) er } func (s *bigtableSink) recordRowMutations(version int64, entry *proto.ChangelogEntry) []historical.BigtableRowMutation { - rows := make([]historical.BigtableRowMutation, 0) - for _, mutation := range compactMutations(entry) { + mutations := compactMutations(entry) + rows := make([]historical.BigtableRowMutation, 0, len(mutations)+len(entry.Upgrades)) + for _, mutation := range mutations { rows = append(rows, s.mutationRow(version, mutation.storeName, mutation.pair)) } for _, up := range entry.Upgrades { diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go index 806cfae31e..98be03feac 100644 --- a/sei-db/state_db/ss/offload/consumer/config.go +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -7,6 +7,11 @@ import ( "strings" ) +const ( + backendScylla = "scylla" + backendBigtable = "bigtable" +) + type Config struct { Backend string Kafka KafkaReaderConfig @@ -23,11 +28,11 @@ func (c *Config) Validate() error { return fmt.Errorf("kafka: %w", err) } switch c.BackendName() { - case "scylla": + case backendScylla: if err := c.Scylla.Validate(); err != nil { return fmt.Errorf("scylla: %w", err) } - case "bigtable": + case backendBigtable: bigtable := c.Bigtable bigtable.ApplyDefaults() if err := bigtable.Validate(); err != nil { @@ -57,16 +62,16 @@ func (c *Config) BackendName() string { return backend } if c.Bigtable.Configured() && !c.Scylla.Configured() { - return "bigtable" + return backendBigtable } - return "scylla" + return backendScylla } func NewSinkFromConfig(cfg Config) (Sink, error) { switch cfg.BackendName() { - case "scylla": + case backendScylla: return NewScyllaSink(cfg.Scylla) - case "bigtable": + case backendBigtable: return NewBigtableSink(cfg.Bigtable) default: return nil, fmt.Errorf("unsupported backend %q", cfg.Backend) diff --git a/sei-db/state_db/ss/offload/consumer/config_test.go b/sei-db/state_db/ss/offload/consumer/config_test.go index 6e335bd3cd..5ca9753998 100644 --- a/sei-db/state_db/ss/offload/consumer/config_test.go +++ b/sei-db/state_db/ss/offload/consumer/config_test.go @@ -7,14 +7,14 @@ import ( ) func TestConfigBackendName(t *testing.T) { - require.Equal(t, "scylla", (&Config{}).BackendName()) - require.Equal(t, "bigtable", (&Config{Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) - require.Equal(t, "scylla", (&Config{Backend: "Scylla", Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) + require.Equal(t, backendScylla, (&Config{}).BackendName()) + require.Equal(t, backendBigtable, (&Config{Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) + require.Equal(t, backendScylla, (&Config{Backend: "Scylla", Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) } func TestConfigValidateBigtable(t *testing.T) { cfg := Config{ - Backend: "bigtable", + Backend: backendBigtable, Kafka: KafkaReaderConfig{ Brokers: []string{"localhost:9092"}, Topic: "historical-offload", @@ -29,5 +29,5 @@ func TestConfigValidateBigtable(t *testing.T) { require.NoError(t, cfg.Validate()) cfg.Bigtable.Table = "" - require.ErrorContains(t, cfg.Validate(), "bigtable") + require.ErrorContains(t, cfg.Validate(), backendBigtable) } diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index c1f075e4ed..39c8a56047 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -32,7 +32,13 @@ const ( BigtableDeletedColumn = "deleted" ) -const bigtableEndpoint = "bigtable.googleapis.com:443" +const ( + bigtableEndpoint = "bigtable.googleapis.com:443" + + maxUint16Int = 1<<16 - 1 + maxUint32Int = 1<<32 - 1 + maxInt64Uint64 = 1<<63 - 1 +) type BigtableClient struct { conn *grpc.ClientConn @@ -378,11 +384,11 @@ func BigtableMutationRowPrefix(storeName string, key []byte, shards int) string shard := bigtableShard(storeName, key, shards) prefix := make([]byte, 1+2+2+len(storeName)+4+len(key)) prefix[0] = bigtableMutationPrefix - binary.BigEndian.PutUint16(prefix[1:], uint16(shard)) - binary.BigEndian.PutUint16(prefix[3:], uint16(len(storeName))) + binary.BigEndian.PutUint16(prefix[1:], shard) + binary.BigEndian.PutUint16(prefix[3:], uint16FromBoundedInt(len(storeName))) copy(prefix[5:], storeName) keyOffset := 5 + len(storeName) - binary.BigEndian.PutUint32(prefix[keyOffset:], uint32(len(key))) + binary.BigEndian.PutUint32(prefix[keyOffset:], uint32FromBoundedInt(len(key))) copy(prefix[keyOffset+4:], key) return string(prefix) } @@ -401,7 +407,7 @@ func BigtableUpgradeRowKey(version int64, name string) string { key := make([]byte, 1+8+2+len(name)) key[0] = bigtableUpgradePrefix copy(key[1:], bigtableInvertedVersion(version)) - binary.BigEndian.PutUint16(key[9:], uint16(len(name))) + binary.BigEndian.PutUint16(key[9:], uint16FromBoundedInt(len(name))) copy(key[11:], name) return string(key) } @@ -410,9 +416,9 @@ func BigtableVersionFromRowKey(rowKey string) (int64, bool) { key := []byte(rowKey) switch { case len(key) >= 1+2+8 && key[0] == bigtableVersionPrefix: - return bigtableDecodeInvertedVersion(key[3:11]), true + return bigtableDecodeInvertedVersion(key[3:11]) case len(key) >= 8 && key[0] == bigtableMutationPrefix: - return bigtableDecodeInvertedVersion(key[len(key)-8:]), true + return bigtableDecodeInvertedVersion(key[len(key)-8:]) default: return 0, false } @@ -492,7 +498,7 @@ func bigtableTableName(projectID, instanceID, table string) string { func bigtableVersionRowPrefix(bucket int) []byte { prefix := make([]byte, 1+2) prefix[0] = bigtableVersionPrefix - binary.BigEndian.PutUint16(prefix[1:], uint16(bucket)) + binary.BigEndian.PutUint16(prefix[1:], uint16FromBoundedInt(bucket)) return prefix } @@ -509,25 +515,65 @@ func bigtablePrefixEnd(prefix []byte) []byte { func bigtableInvertedVersion(version int64) []byte { out := make([]byte, 8) - binary.BigEndian.PutUint64(out, ^uint64(version)) + binary.BigEndian.PutUint64(out, ^uint64FromNonNegativeInt64(version)) return out } -func bigtableDecodeInvertedVersion(encoded []byte) int64 { - return int64(^binary.BigEndian.Uint64(encoded)) +func bigtableDecodeInvertedVersion(encoded []byte) (int64, bool) { + version := ^binary.BigEndian.Uint64(encoded) + if version > maxInt64Uint64 { + return 0, false + } + // #nosec G115 -- version is checked above to fit in int64. + return int64(version), true } -func bigtableShard(storeName string, key []byte, shards int) int { +func bigtableShard(storeName string, key []byte, shards int) uint16 { h := fnv.New32a() _, _ = h.Write([]byte(storeName)) _, _ = h.Write([]byte{0}) _, _ = h.Write(key) - return int(h.Sum32() % uint32(shards)) + return uint16FromBoundedUint32(h.Sum32() % uint32FromBoundedInt(shards)) } func normalizeBigtableShards(shards int) int { if shards <= 0 { return DefaultBigtableShards } + if shards > maxUint16Int { + return maxUint16Int + } return shards } + +func uint16FromBoundedInt(value int) uint16 { + if value < 0 || value > maxUint16Int { + panic(fmt.Sprintf("bigtable value %d exceeds uint16", value)) + } + // #nosec G115 -- value is checked above to fit in uint16. + return uint16(value) +} + +func uint32FromBoundedInt(value int) uint32 { + if value < 0 || value > maxUint32Int { + panic(fmt.Sprintf("bigtable value %d exceeds uint32", value)) + } + // #nosec G115 -- value is checked above to fit in uint32. + return uint32(value) +} + +func uint16FromBoundedUint32(value uint32) uint16 { + if value > maxUint16Int { + panic(fmt.Sprintf("bigtable value %d exceeds uint16", value)) + } + // #nosec G115 -- value is checked above to fit in uint16. + return uint16(value) +} + +func uint64FromNonNegativeInt64(value int64) uint64 { + if value < 0 { + panic(fmt.Sprintf("bigtable version %d is negative", value)) + } + // #nosec G115 -- value is checked above to be non-negative. + return uint64(value) +} From 66f8c9f6fd5c29d3c9cda54e2d25017946e40000 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 14 May 2026 17:21:51 -0400 Subject: [PATCH 12/41] Remove unused historical BatchGet interface --- .../ss/offload/historical/bigtable.go | 31 ------------------- .../state_db/ss/offload/historical/reader.go | 3 -- .../ss/offload/historical/store_test.go | 4 --- 3 files changed, 38 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 39c8a56047..45fb431f1f 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -8,11 +8,9 @@ import ( "io" "regexp" "strings" - "sync" "time" "cloud.google.com/go/bigtable/apiv2/bigtablepb" - "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/oauth" @@ -22,8 +20,6 @@ const ( DefaultBigtableFamily = "state" DefaultBigtableShards = 256 - defaultBigtableReadWorkers = 16 - bigtableMutationPrefix = byte('m') bigtableVersionPrefix = byte('v') bigtableUpgradePrefix = byte('u') @@ -310,33 +306,6 @@ func (r *bigtableReader) Get(ctx context.Context, storeName string, key []byte, return BigtableValueFromRow(row, r.family) } -func (r *bigtableReader) BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) { - out := make(map[Lookup]Value, len(lookups)) - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(defaultBigtableReadWorkers) - var mu sync.Mutex - for _, lookup := range lookups { - lookup := lookup - g.Go(func() error { - value, err := r.Get(gctx, lookup.StoreName, []byte(lookup.Key), targetVersion) - if err != nil { - if err == ErrNotFound { - return nil - } - return err - } - mu.Lock() - out[lookup] = value - mu.Unlock() - return nil - }) - } - if err := g.Wait(); err != nil { - return nil, err - } - return out, nil -} - func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (int64, error) { var maxVersion int64 for bucket := 0; bucket < VersionBucketCount; bucket++ { diff --git a/sei-db/state_db/ss/offload/historical/reader.go b/sei-db/state_db/ss/offload/historical/reader.go index 7dfdfee993..b4bea9414d 100644 --- a/sei-db/state_db/ss/offload/historical/reader.go +++ b/sei-db/state_db/ss/offload/historical/reader.go @@ -29,9 +29,6 @@ type Reader interface { // Has skips value transfer and returns false for missing or tombstoned keys. Has(ctx context.Context, storeName string, key []byte, targetVersion int64) (bool, error) - // BatchGet returns only found, non-tombstoned lookups. - BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) - LastVersion(ctx context.Context) (int64, error) Close() error } diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go index 1e3cbfea57..a234586649 100644 --- a/sei-db/state_db/ss/offload/historical/store_test.go +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -65,10 +65,6 @@ func (f *fakeReader) Has(context.Context, string, []byte, int64) (bool, error) { return true, nil } -func (f *fakeReader) BatchGet(context.Context, int64, []Lookup) (map[Lookup]Value, error) { - return nil, nil -} - func (f *fakeReader) LastVersion(context.Context) (int64, error) { return 0, nil } func (f *fakeReader) Close() error { return nil } From ecdf5cc749a6bf36af59ef8585b7cb85da54a671 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 15 May 2026 13:19:34 -0400 Subject: [PATCH 13/41] Reduce Bigtable write allocation overhead --- .../state_db/ss/offload/consumer/bigtable.go | 62 +++++++++++-------- .../ss/offload/historical/bigtable.go | 5 +- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index 409c758421..13e4513ca1 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -3,6 +3,7 @@ package consumer import ( "context" "fmt" + "strconv" "time" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -73,9 +74,9 @@ func (s *bigtableSink) WriteBatch(ctx context.Context, records []Record) error { } func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) error { - rows := make([]historical.BigtableRowMutation, 0, len(records)) + rows := make([]historical.BigtableRowMutation, 0, bigtableRowMutationCount(records)) for _, rec := range records { - rows = append(rows, s.recordRowMutations(rec.Entry.Version, rec.Entry)...) + rows = s.appendRecordRowMutations(rows, rec.Entry.Version, rec.Entry) } if len(rows) == 0 { return nil @@ -84,9 +85,8 @@ func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) er return bigtableBulkError(rows, errs, err) } -func (s *bigtableSink) recordRowMutations(version int64, entry *proto.ChangelogEntry) []historical.BigtableRowMutation { +func (s *bigtableSink) appendRecordRowMutations(rows []historical.BigtableRowMutation, version int64, entry *proto.ChangelogEntry) []historical.BigtableRowMutation { mutations := compactMutations(entry) - rows := make([]historical.BigtableRowMutation, 0, len(mutations)+len(entry.Upgrades)) for _, mutation := range mutations { rows = append(rows, s.mutationRow(version, mutation.storeName, mutation.pair)) } @@ -99,24 +99,25 @@ func (s *bigtableSink) recordRowMutations(version int64, entry *proto.ChangelogE func (s *bigtableSink) mutationRow(version int64, storeName string, pair *proto.KVPair) historical.BigtableRowMutation { ts := historical.BigtableTimestamp(version) deleted := pair.Delete || pair.Value == nil - row := historical.BigtableRowMutation{ - RowKey: historical.BigtableMutationRowKey(storeName, pair.Key, version, s.shards), - } - if !deleted { - row.SetCells = append(row.SetCells, historical.BigtableSetCell{ - Family: s.family, - Qualifier: historical.BigtableValueColumn, - TimestampMicros: ts, - Value: pair.Value, - }) - } - row.SetCells = append(row.SetCells, historical.BigtableSetCell{ - Family: s.family, - Qualifier: historical.BigtableDeletedColumn, - TimestampMicros: ts, - Value: boolByte(deleted), - }) - return row + rowKey := historical.BigtableMutationRowKey(storeName, pair.Key, version, s.shards) + if deleted { + return historical.BigtableRowMutation{ + RowKey: rowKey, + SetCells: []historical.BigtableSetCell{{ + Family: s.family, + Qualifier: historical.BigtableDeletedColumn, + TimestampMicros: ts, + Value: boolByte(true), + }}, + } + } + return historical.BigtableRowMutation{ + RowKey: rowKey, + SetCells: []historical.BigtableSetCell{ + {Family: s.family, Qualifier: historical.BigtableValueColumn, TimestampMicros: ts, Value: pair.Value}, + {Family: s.family, Qualifier: historical.BigtableDeletedColumn, TimestampMicros: ts, Value: boolByte(false)}, + }, + } } func (s *bigtableSink) upgradeRow(version int64, up *proto.TreeNameUpgrade) historical.BigtableRowMutation { @@ -137,9 +138,9 @@ func (s *bigtableSink) writeVersionMarker(ctx context.Context, rec Record) error RowKey: historical.BigtableVersionRowKey(version), SetCells: []historical.BigtableSetCell{ {Family: s.family, Qualifier: "topic", TimestampMicros: ts, Value: []byte(rec.Topic)}, - {Family: s.family, Qualifier: "partition", TimestampMicros: ts, Value: []byte(fmt.Sprintf("%d", rec.Partition))}, - {Family: s.family, Qualifier: "offset", TimestampMicros: ts, Value: []byte(fmt.Sprintf("%d", rec.Offset))}, - {Family: s.family, Qualifier: "ingested_at_unix_nano", TimestampMicros: ts, Value: []byte(fmt.Sprintf("%d", time.Now().UnixNano()))}, + {Family: s.family, Qualifier: "partition", TimestampMicros: ts, Value: []byte(strconv.Itoa(rec.Partition))}, + {Family: s.family, Qualifier: "offset", TimestampMicros: ts, Value: []byte(strconv.FormatInt(rec.Offset, 10))}, + {Family: s.family, Qualifier: "ingested_at_unix_nano", TimestampMicros: ts, Value: []byte(strconv.FormatInt(time.Now().UnixNano(), 10))}, }, } errs, err := s.applyBulk(ctx, []historical.BigtableRowMutation{row}) @@ -149,6 +150,17 @@ func (s *bigtableSink) writeVersionMarker(ctx context.Context, rec Record) error return nil } +func bigtableRowMutationCount(records []Record) int { + total := 0 + for _, rec := range records { + for _, changeset := range rec.Entry.Changesets { + total += len(changeset.Changeset.Pairs) + } + total += len(rec.Entry.Upgrades) + } + return total +} + func bigtableBulkError(rows []historical.BigtableRowMutation, errs []error, err error) error { if err != nil { return err diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 45fb431f1f..0820def8ee 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -208,7 +208,10 @@ func (c *BigtableClient) ApplyBulk(ctx context.Context, rows []BigtableRowMutati } entries := make([]*bigtablepb.MutateRowsRequest_Entry, 0, len(rows)) for _, row := range rows { - entry := &bigtablepb.MutateRowsRequest_Entry{RowKey: []byte(row.RowKey)} + entry := &bigtablepb.MutateRowsRequest_Entry{ + RowKey: []byte(row.RowKey), + Mutations: make([]*bigtablepb.Mutation, 0, len(row.SetCells)), + } for _, cell := range row.SetCells { entry.Mutations = append(entry.Mutations, &bigtablepb.Mutation{ Mutation: &bigtablepb.Mutation_SetCell_{SetCell: &bigtablepb.Mutation_SetCell{ From 35f7143d6a6afe1ec66c153a5d34c24f1d33a903 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 15 May 2026 17:23:04 -0400 Subject: [PATCH 14/41] Tune Bigtable consumer batch defaults --- sei-db/state_db/ss/offload/consumer/config.go | 16 +++++++++++++ .../consumer/config/example-bigtable.json | 4 ++-- .../ss/offload/consumer/config_test.go | 23 +++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go index 98be03feac..038d8f7486 100644 --- a/sei-db/state_db/ss/offload/consumer/config.go +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -10,6 +10,9 @@ import ( const ( backendScylla = "scylla" backendBigtable = "bigtable" + + defaultBigtableMaxBatchRecords = 128 + defaultBigtableBatchMaxWaitMS = 25 ) type Config struct { @@ -67,6 +70,18 @@ func (c *Config) BackendName() string { return backendScylla } +func (c *Config) applyBackendDefaults() { + if c.BackendName() != backendBigtable { + return + } + if c.MaxBatchRecords == 0 { + c.MaxBatchRecords = defaultBigtableMaxBatchRecords + } + if c.BatchMaxWaitMS == 0 { + c.BatchMaxWaitMS = defaultBigtableBatchMaxWaitMS + } +} + func NewSinkFromConfig(cfg Config) (Sink, error) { switch cfg.BackendName() { case backendScylla: @@ -88,6 +103,7 @@ func LoadConfig(path string) (*Config, error) { if err := json.Unmarshal(raw, cfg); err != nil { return nil, fmt.Errorf("parse config: %w", err) } + cfg.applyBackendDefaults() if err := cfg.Validate(); err != nil { return nil, err } diff --git a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json index 32e9250678..0259f3e4df 100644 --- a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json +++ b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json @@ -16,6 +16,6 @@ }, "Workers": 16, "ShardBufferSize": 128, - "MaxBatchRecords": 16, - "BatchMaxWaitMS": 10 + "MaxBatchRecords": 128, + "BatchMaxWaitMS": 25 } diff --git a/sei-db/state_db/ss/offload/consumer/config_test.go b/sei-db/state_db/ss/offload/consumer/config_test.go index 5ca9753998..0ac2e8ac86 100644 --- a/sei-db/state_db/ss/offload/consumer/config_test.go +++ b/sei-db/state_db/ss/offload/consumer/config_test.go @@ -31,3 +31,26 @@ func TestConfigValidateBigtable(t *testing.T) { cfg.Bigtable.Table = "" require.ErrorContains(t, cfg.Validate(), backendBigtable) } + +func TestConfigApplyBackendDefaultsBigtable(t *testing.T) { + cfg := Config{Backend: backendBigtable} + cfg.applyBackendDefaults() + require.Equal(t, defaultBigtableMaxBatchRecords, cfg.MaxBatchRecords) + require.Equal(t, defaultBigtableBatchMaxWaitMS, cfg.BatchMaxWaitMS) + + cfg = Config{ + Backend: backendBigtable, + MaxBatchRecords: 32, + BatchMaxWaitMS: 5, + } + cfg.applyBackendDefaults() + require.Equal(t, 32, cfg.MaxBatchRecords) + require.Equal(t, 5, cfg.BatchMaxWaitMS) +} + +func TestConfigApplyBackendDefaultsScylla(t *testing.T) { + cfg := Config{Backend: backendScylla} + cfg.applyBackendDefaults() + require.Zero(t, cfg.MaxBatchRecords) + require.Zero(t, cfg.BatchMaxWaitMS) +} From 429ee4514ed0de63da2dca393df77b882b634bf4 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 18 May 2026 11:05:19 -0400 Subject: [PATCH 15/41] Batch Bigtable version marker writes --- .../state_db/ss/offload/consumer/bigtable.go | 39 +++++++++---------- .../ss/offload/consumer/bigtable_test.go | 37 +++++++----------- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index 13e4513ca1..a77c02ad8d 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -65,12 +65,7 @@ func (s *bigtableSink) WriteBatch(ctx context.Context, records []Record) error { if err := s.writeRecordRows(ctx, records); err != nil { return err } - for _, rec := range records { - if err := s.writeVersionMarker(ctx, rec); err != nil { - return err - } - } - return nil + return s.writeVersionMarkers(ctx, records) } func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) error { @@ -131,21 +126,25 @@ func (s *bigtableSink) upgradeRow(version int64, up *proto.TreeNameUpgrade) hist } } -func (s *bigtableSink) writeVersionMarker(ctx context.Context, rec Record) error { - version := rec.Entry.Version - ts := historical.BigtableTimestamp(version) - row := historical.BigtableRowMutation{ - RowKey: historical.BigtableVersionRowKey(version), - SetCells: []historical.BigtableSetCell{ - {Family: s.family, Qualifier: "topic", TimestampMicros: ts, Value: []byte(rec.Topic)}, - {Family: s.family, Qualifier: "partition", TimestampMicros: ts, Value: []byte(strconv.Itoa(rec.Partition))}, - {Family: s.family, Qualifier: "offset", TimestampMicros: ts, Value: []byte(strconv.FormatInt(rec.Offset, 10))}, - {Family: s.family, Qualifier: "ingested_at_unix_nano", TimestampMicros: ts, Value: []byte(strconv.FormatInt(time.Now().UnixNano(), 10))}, - }, +func (s *bigtableSink) writeVersionMarkers(ctx context.Context, records []Record) error { + rows := make([]historical.BigtableRowMutation, 0, len(records)) + ingestedAt := []byte(strconv.FormatInt(time.Now().UnixNano(), 10)) + for _, rec := range records { + version := rec.Entry.Version + ts := historical.BigtableTimestamp(version) + rows = append(rows, historical.BigtableRowMutation{ + RowKey: historical.BigtableVersionRowKey(version), + SetCells: []historical.BigtableSetCell{ + {Family: s.family, Qualifier: "topic", TimestampMicros: ts, Value: []byte(rec.Topic)}, + {Family: s.family, Qualifier: "partition", TimestampMicros: ts, Value: []byte(strconv.Itoa(rec.Partition))}, + {Family: s.family, Qualifier: "offset", TimestampMicros: ts, Value: []byte(strconv.FormatInt(rec.Offset, 10))}, + {Family: s.family, Qualifier: "ingested_at_unix_nano", TimestampMicros: ts, Value: ingestedAt}, + }, + }) } - errs, err := s.applyBulk(ctx, []historical.BigtableRowMutation{row}) - if err := bigtableBulkError([]historical.BigtableRowMutation{row}, errs, err); err != nil { - return fmt.Errorf("insert bigtable version %d: %w", version, err) + errs, err := s.applyBulk(ctx, rows) + if err := bigtableBulkError(rows, errs, err); err != nil { + return fmt.Errorf("insert bigtable version markers: %w", err) } return nil } diff --git a/sei-db/state_db/ss/offload/consumer/bigtable_test.go b/sei-db/state_db/ss/offload/consumer/bigtable_test.go index aa084fc9e4..ef2d1146c4 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable_test.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable_test.go @@ -43,26 +43,17 @@ func TestBigtableSinkWritesMutationRowsAndVersionMarker(t *testing.T) { } func TestBigtableSinkWriteBatchWritesRowsBeforeMarkers(t *testing.T) { - var rowVersions []int64 - var markerVersions []int64 - var markerBeforeRowsDone bool + var calls [][]string sink := &bigtableSink{ family: historical.DefaultBigtableFamily, shards: historical.DefaultBigtableShards, applyBulk: func(_ context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { - isMarkerBatch := len(mutations) == 1 && mutations[0].RowKey == historical.BigtableVersionRowKey(mustBigtableVersion(t, mutations[0].RowKey)) - if isMarkerBatch && len(rowVersions) != 2 { - markerBeforeRowsDone = true - } + call := make([]string, 0, len(mutations)) for _, mutation := range mutations { - version := mustBigtableVersion(t, mutation.RowKey) - if mutation.RowKey == historical.BigtableVersionRowKey(version) { - markerVersions = append(markerVersions, version) - } else { - rowVersions = append(rowVersions, version) - } + call = append(call, mutation.RowKey) } + calls = append(calls, call) return make([]error, len(mutations)), nil }, } @@ -84,14 +75,14 @@ func TestBigtableSinkWriteBatchWritesRowsBeforeMarkers(t *testing.T) { } require.NoError(t, sink.WriteBatch(context.Background(), records)) - require.False(t, markerBeforeRowsDone) - require.Equal(t, []int64{1, 2}, rowVersions) - require.Equal(t, []int64{1, 2}, markerVersions) -} - -func mustBigtableVersion(t *testing.T, rowKey string) int64 { - t.Helper() - version, ok := historical.BigtableVersionFromRowKey(rowKey) - require.True(t, ok) - return version + require.Equal(t, [][]string{ + { + historical.BigtableMutationRowKey("bank", []byte("k1"), 1, historical.DefaultBigtableShards), + historical.BigtableMutationRowKey("bank", []byte("k2"), 2, historical.DefaultBigtableShards), + }, + { + historical.BigtableVersionRowKey(1), + historical.BigtableVersionRowKey(2), + }, + }, calls) } From a406321317208fe50110ee310c304bafcd6fd61d Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 18 May 2026 13:57:38 -0400 Subject: [PATCH 16/41] Parallelize Bigtable latest version reads --- .../ss/offload/historical/bigtable.go | 37 ++++++++++++++----- .../ss/offload/historical/bigtable_test.go | 32 ++++++++++++++++ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 0820def8ee..37cb8faaf7 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -8,9 +8,11 @@ import ( "io" "regexp" "strings" + "sync" "time" "cloud.google.com/go/bigtable/apiv2/bigtablepb" + "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/oauth" @@ -31,6 +33,8 @@ const ( const ( bigtableEndpoint = "bigtable.googleapis.com:443" + defaultBigtableReadWorkers = 16 + maxUint16Int = 1<<16 - 1 maxUint32Int = 1<<32 - 1 maxInt64Uint64 = 1<<63 - 1 @@ -311,18 +315,33 @@ func (r *bigtableReader) Get(ctx context.Context, storeName string, key []byte, func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (int64, error) { var maxVersion int64 + var mu sync.Mutex + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(defaultBigtableReadWorkers) for bucket := 0; bucket < VersionBucketCount; bucket++ { - prefix := bigtableVersionRowPrefix(bucket) - err := readRows(ctx, prefix, bigtablePrefixEnd(prefix), 1, "", func(row BigtableRow) bool { - version, ok := BigtableVersionFromRowKey(row.Key) - if ok && version > maxVersion { - maxVersion = version + bucket := bucket + g.Go(func() error { + prefix := bigtableVersionRowPrefix(bucket) + var bucketVersion int64 + err := readRows(gctx, prefix, bigtablePrefixEnd(prefix), 1, "", func(row BigtableRow) bool { + if version, ok := BigtableVersionFromRowKey(row.Key); ok { + bucketVersion = version + } + return false + }) + if err != nil { + return fmt.Errorf("read latest bigtable version bucket %d: %w", bucket, err) + } + mu.Lock() + if bucketVersion > maxVersion { + maxVersion = bucketVersion } - return false + mu.Unlock() + return nil }) - if err != nil { - return 0, fmt.Errorf("read latest bigtable version bucket %d: %w", bucket, err) - } + } + if err := g.Wait(); err != nil { + return 0, err } return maxVersion, nil } diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go index 2c78f2f228..2f08e0b7bc 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable_test.go +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -2,7 +2,9 @@ package historical import ( "context" + "fmt" "sort" + "sync" "testing" "github.com/stretchr/testify/require" @@ -91,3 +93,33 @@ func TestBigtableReaderGetUsesMVCCRange(t *testing.T) { require.Equal(t, []byte("v40"), value.Bytes) require.Equal(t, int64(40), value.Version) } + +func TestBigtableLastVersionScansBuckets(t *testing.T) { + versions := map[int]int64{ + 3: 42, + 9: 70, + } + seen := make(map[int]struct{}, VersionBucketCount) + var mu sync.Mutex + + got, err := BigtableLastVersion(context.Background(), func(_ context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error { + if len(startKey) != 3 || startKey[0] != bigtableVersionPrefix { + return fmt.Errorf("unexpected start key %q", startKey) + } + if len(endKey) == 0 || limit != 1 || family != "" { + return fmt.Errorf("unexpected scan params") + } + bucket := int(startKey[1])<<8 | int(startKey[2]) + mu.Lock() + seen[bucket] = struct{}{} + version := versions[bucket] + mu.Unlock() + if version > 0 { + f(BigtableRow{Key: BigtableVersionRowKey(version)}) + } + return nil + }) + require.NoError(t, err) + require.Equal(t, int64(70), got) + require.Len(t, seen, VersionBucketCount) +} From 1a9736acc5bb8c1a46d4859600be5c8abb1bf073 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 26 May 2026 14:29:07 -0400 Subject: [PATCH 17/41] Optimize Bigtable historical reads --- .../ss/offload/historical/bigtable.go | 115 ++++++++++++++---- .../ss/offload/historical/bigtable_test.go | 35 +++++- 2 files changed, 126 insertions(+), 24 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 37cb8faaf7..8cff2d5a4e 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -152,7 +152,7 @@ func OpenBigtableClient(ctx context.Context, cfg BigtableConfig) (*BigtableClien }, nil } -type BigtableReadRowsFunc func(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error +type BigtableReadRowsFunc func(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error type BigtableApplyBulkFunc func(ctx context.Context, rows []BigtableRowMutation) ([]error, error) @@ -163,7 +163,7 @@ func (c *BigtableClient) Close() error { return c.conn.Close() } -func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error { +func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { req := &bigtablepb.ReadRowsRequest{ TableName: c.tableName, AppProfileId: c.appProfile, @@ -176,11 +176,7 @@ func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, if len(endKey) == 0 { req.Rows.RowRanges[0].EndKey = nil } - if family != "" { - req.Filter = &bigtablepb.RowFilter{ - Filter: &bigtablepb.RowFilter_FamilyNameRegexFilter{FamilyNameRegexFilter: regexp.QuoteMeta(family)}, - } - } + req.Filter = bigtableReadFilter(family, qualifiers...) stream, err := c.data.ReadRows(ctx, req) if err != nil { return err @@ -286,24 +282,34 @@ func (r *bigtableReader) LastVersion(ctx context.Context) (int64, error) { } func (r *bigtableReader) Has(ctx context.Context, storeName string, key []byte, targetVersion int64) (bool, error) { - _, err := r.Get(ctx, storeName, key, targetVersion) + prefix := bigtableMutationRowPrefixBytes(storeName, key, r.shards) + start := bigtableMutationRowKeyBytes(storeName, key, targetVersion, r.shards) + var row BigtableRow + err := r.readRows(ctx, start, bigtablePrefixEnd(prefix), 1, r.family, func(r BigtableRow) bool { + row = r + return false + }, BigtableDeletedColumn) + if err != nil { + return false, fmt.Errorf("bigtable has lookup: %w", err) + } + if row.Key == "" { + return false, nil + } + deleted, err := bigtableDeletedFromRow(row, r.family) if err != nil { - if err == ErrNotFound { - return false, nil - } return false, err } - return true, nil + return !deleted, nil } func (r *bigtableReader) Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { - prefix := []byte(BigtableMutationRowPrefix(storeName, key, r.shards)) - start := []byte(BigtableMutationRowKey(storeName, key, targetVersion, r.shards)) + prefix := bigtableMutationRowPrefixBytes(storeName, key, r.shards) + start := bigtableMutationRowKeyBytes(storeName, key, targetVersion, r.shards) var row BigtableRow err := r.readRows(ctx, start, bigtablePrefixEnd(prefix), 1, r.family, func(r BigtableRow) bool { row = r return false - }) + }, BigtableValueColumn, BigtableDeletedColumn) if err != nil { return Value{}, fmt.Errorf("bigtable get lookup: %w", err) } @@ -359,7 +365,7 @@ func BigtableValueFromRow(row BigtableRow, family string) (Value, error) { } switch cell.Qualifier { case BigtableValueColumn: - value = append([]byte(nil), cell.Value...) + value = cell.Value case BigtableDeletedColumn: deleted = len(cell.Value) > 0 && cell.Value[0] == 1 } @@ -367,13 +373,30 @@ func BigtableValueFromRow(row BigtableRow, family string) (Value, error) { if deleted || value == nil { return Value{}, ErrNotFound } - return Value{Bytes: value, Version: version}, nil + return Value{Bytes: append([]byte(nil), value...), Version: version}, nil +} + +func bigtableDeletedFromRow(row BigtableRow, family string) (bool, error) { + if _, ok := BigtableVersionFromRowKey(row.Key); !ok { + return false, fmt.Errorf("invalid bigtable mutation row key") + } + for _, cell := range row.Cells { + if cell.Family == family && cell.Qualifier == BigtableDeletedColumn { + return len(cell.Value) > 0 && cell.Value[0] == 1, nil + } + } + return false, nil } func BigtableMutationRowPrefix(storeName string, key []byte, shards int) string { + return string(bigtableMutationRowPrefixBytes(storeName, key, shards)) +} + +func bigtableMutationRowPrefixBytes(storeName string, key []byte, shards int) []byte { shards = normalizeBigtableShards(shards) shard := bigtableShard(storeName, key, shards) - prefix := make([]byte, 1+2+2+len(storeName)+4+len(key)) + prefixLen := 1 + 2 + 2 + len(storeName) + 4 + len(key) + prefix := make([]byte, prefixLen, prefixLen+8) prefix[0] = bigtableMutationPrefix binary.BigEndian.PutUint16(prefix[1:], shard) binary.BigEndian.PutUint16(prefix[3:], uint16FromBoundedInt(len(storeName))) @@ -381,12 +404,16 @@ func BigtableMutationRowPrefix(storeName string, key []byte, shards int) string keyOffset := 5 + len(storeName) binary.BigEndian.PutUint32(prefix[keyOffset:], uint32FromBoundedInt(len(key))) copy(prefix[keyOffset+4:], key) - return string(prefix) + return prefix } func BigtableMutationRowKey(storeName string, key []byte, version int64, shards int) string { - prefix := []byte(BigtableMutationRowPrefix(storeName, key, shards)) - return string(append(prefix, bigtableInvertedVersion(version)...)) + return string(bigtableMutationRowKeyBytes(storeName, key, version, shards)) +} + +func bigtableMutationRowKeyBytes(storeName string, key []byte, version int64, shards int) []byte { + prefix := bigtableMutationRowPrefixBytes(storeName, key, shards) + return append(prefix, bigtableInvertedVersion(version)...) } func BigtableVersionRowKey(version int64) string { @@ -486,6 +513,52 @@ func bigtableTableName(projectID, instanceID, table string) string { return fmt.Sprintf("projects/%s/instances/%s/tables/%s", projectID, instanceID, table) } +func bigtableReadFilter(family string, qualifiers ...string) *bigtablepb.RowFilter { + filters := make([]*bigtablepb.RowFilter, 0, 2) + if family != "" { + filters = append(filters, &bigtablepb.RowFilter{ + Filter: &bigtablepb.RowFilter_FamilyNameRegexFilter{FamilyNameRegexFilter: regexp.QuoteMeta(family)}, + }) + } + if qualifierFilter := bigtableQualifierFilter(qualifiers...); qualifierFilter != nil { + filters = append(filters, qualifierFilter) + } + switch len(filters) { + case 0: + return nil + case 1: + return filters[0] + default: + return &bigtablepb.RowFilter{ + Filter: &bigtablepb.RowFilter_Chain_{Chain: &bigtablepb.RowFilter_Chain{Filters: filters}}, + } + } +} + +func bigtableQualifierFilter(qualifiers ...string) *bigtablepb.RowFilter { + filters := make([]*bigtablepb.RowFilter, 0, len(qualifiers)) + for _, qualifier := range qualifiers { + if qualifier == "" { + continue + } + filters = append(filters, &bigtablepb.RowFilter{ + Filter: &bigtablepb.RowFilter_ColumnQualifierRegexFilter{ + ColumnQualifierRegexFilter: []byte(regexp.QuoteMeta(qualifier)), + }, + }) + } + switch len(filters) { + case 0: + return nil + case 1: + return filters[0] + default: + return &bigtablepb.RowFilter{ + Filter: &bigtablepb.RowFilter_Interleave_{Interleave: &bigtablepb.RowFilter_Interleave{Filters: filters}}, + } + } +} + func bigtableVersionRowPrefix(bucket int) []byte { prefix := make([]byte, 1+2) prefix[0] = bigtableVersionPrefix diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go index 2f08e0b7bc..ef8dafff8a 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable_test.go +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -72,11 +72,12 @@ func TestBigtableReaderGetUsesMVCCRange(t *testing.T) { reader := &bigtableReader{ family: DefaultBigtableFamily, shards: 256, - readRows: func(_ context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error { + readRows: func(_ context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { require.Equal(t, []byte(BigtableMutationRowKey("bank", []byte("k"), 60, 256)), startKey) require.NotEmpty(t, endKey) require.Equal(t, int64(1), limit) require.Equal(t, DefaultBigtableFamily, family) + require.Equal(t, []string{BigtableValueColumn, BigtableDeletedColumn}, qualifiers) f(BigtableRow{ Key: wantRow, Cells: []BigtableCell{ @@ -94,6 +95,34 @@ func TestBigtableReaderGetUsesMVCCRange(t *testing.T) { require.Equal(t, int64(40), value.Version) } +func TestBigtableReaderHasOnlyReadsDeletedColumn(t *testing.T) { + wantRow := BigtableMutationRowKey("bank", []byte("k"), 40, 256) + reader := &bigtableReader{ + family: DefaultBigtableFamily, + shards: 256, + readRows: func(_ context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { + require.Equal(t, []byte(BigtableMutationRowKey("bank", []byte("k"), 60, 256)), startKey) + require.NotEmpty(t, endKey) + require.Equal(t, int64(1), limit) + require.Equal(t, DefaultBigtableFamily, family) + require.Equal(t, []string{BigtableDeletedColumn}, qualifiers) + f(BigtableRow{ + Key: wantRow, + Cells: []BigtableCell{{ + Family: DefaultBigtableFamily, + Qualifier: BigtableDeletedColumn, + Value: []byte{0}, + }}, + }) + return nil + }, + } + + ok, err := reader.Has(context.Background(), "bank", []byte("k"), 60) + require.NoError(t, err) + require.True(t, ok) +} + func TestBigtableLastVersionScansBuckets(t *testing.T) { versions := map[int]int64{ 3: 42, @@ -102,11 +131,11 @@ func TestBigtableLastVersionScansBuckets(t *testing.T) { seen := make(map[int]struct{}, VersionBucketCount) var mu sync.Mutex - got, err := BigtableLastVersion(context.Background(), func(_ context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool) error { + got, err := BigtableLastVersion(context.Background(), func(_ context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { if len(startKey) != 3 || startKey[0] != bigtableVersionPrefix { return fmt.Errorf("unexpected start key %q", startKey) } - if len(endKey) == 0 || limit != 1 || family != "" { + if len(endKey) == 0 || limit != 1 || family != "" || len(qualifiers) != 0 { return fmt.Errorf("unexpected scan params") } bucket := int(startKey[1])<<8 | int(startKey[2]) From 571a9434b4415f7b0bdc422c626c5acd28f67a07 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Wed, 27 May 2026 16:52:00 -0400 Subject: [PATCH 18/41] Cache historical fallback misses --- .../state_db/ss/offload/historical/store.go | 72 ++++++++++++++--- .../ss/offload/historical/store_test.go | 81 ++++++++++++++++++- 2 files changed, 141 insertions(+), 12 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index 90dc7abdac..756eb4ed19 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -21,19 +21,25 @@ type historicalReadCacheKey struct { key string } +type historicalReadCacheValue struct { + value []byte + found bool + valueKnown bool +} + // FallbackStateStore routes pruned point reads to the historical reader. // Iteration and writes stay on the primary state store. type FallbackStateStore struct { primary types.StateStore reader Reader - cache *lru.Cache[historicalReadCacheKey, []byte] + cache *lru.Cache[historicalReadCacheKey, historicalReadCacheValue] } var _ types.StateStore = (*FallbackStateStore)(nil) // NewFallbackStateStore takes ownership of primary and reader for Close. func NewFallbackStateStore(primary types.StateStore, reader Reader) *FallbackStateStore { - cache, err := lru.New[historicalReadCacheKey, []byte](defaultHistoricalReadCacheEntries) + cache, err := lru.New[historicalReadCacheKey, historicalReadCacheValue](defaultHistoricalReadCacheEntries) if err != nil { panic(err) } @@ -50,12 +56,16 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] return s.primary.Get(storeKey, version, key) } cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} - if value, ok := s.getCached(cacheKey); ok { + if value, found, ok := s.getCachedValue(cacheKey); ok { + if !found { + return nil, nil + } return value, nil } v, err := s.reader.Get(context.Background(), storeKey, key, version) if err != nil { if errors.Is(err, ErrNotFound) { + s.cacheMiss(cacheKey) return nil, nil } return nil, err @@ -64,29 +74,73 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] return v.Bytes, nil } -func (s *FallbackStateStore) getCached(key historicalReadCacheKey) ([]byte, bool) { +func (s *FallbackStateStore) getCachedValue(key historicalReadCacheKey) ([]byte, bool, bool) { + if s.cache == nil { + return nil, false, false + } + value, ok := s.cache.Get(key) + if !ok { + return nil, false, false + } + if !value.found { + return nil, false, true + } + if !value.valueKnown { + return nil, false, false + } + return bytes.Clone(value.value), true, true +} + +func (s *FallbackStateStore) getCachedHas(key historicalReadCacheKey) (bool, bool) { if s.cache == nil { - return nil, false + return false, false } value, ok := s.cache.Get(key) if !ok { - return nil, false + return false, false } - return bytes.Clone(value), true + return value.found, true } func (s *FallbackStateStore) cacheValue(key historicalReadCacheKey, value []byte) { if s.cache == nil || value == nil || len(value) > maxHistoricalReadCacheValueBytes { return } - s.cache.Add(key, bytes.Clone(value)) + s.cache.Add(key, historicalReadCacheValue{value: bytes.Clone(value), found: true, valueKnown: true}) +} + +func (s *FallbackStateStore) cacheMiss(key historicalReadCacheKey) { + if s.cache == nil { + return + } + s.cache.Add(key, historicalReadCacheValue{valueKnown: true}) +} + +func (s *FallbackStateStore) cacheHas(key historicalReadCacheKey) { + if s.cache == nil { + return + } + s.cache.Add(key, historicalReadCacheValue{found: true}) } func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bool, error) { if !s.shouldFallback(version) { return s.primary.Has(storeKey, version, key) } - return s.reader.Has(context.Background(), storeKey, key, version) + cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} + if found, ok := s.getCachedHas(cacheKey); ok { + return found, nil + } + found, err := s.reader.Has(context.Background(), storeKey, key, version) + if err != nil { + return false, err + } + if found { + s.cacheHas(cacheKey) + } else { + s.cacheMiss(cacheKey) + } + return found, nil } func (s *FallbackStateStore) Iterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go index a234586649..ba64ff93cf 100644 --- a/sei-db/state_db/ss/offload/historical/store_test.go +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -2,6 +2,7 @@ package historical import ( "context" + "errors" "testing" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" @@ -51,17 +52,26 @@ func (f *fakeStateStore) Import(int64, <-chan types.SnapshotNode) error func (f *fakeStateStore) Close() error { return nil } type fakeReader struct { - gets int - has int + gets int + has int + getErr error + hasResult bool + hasSet bool } func (f *fakeReader) Get(context.Context, string, []byte, int64) (Value, error) { f.gets++ + if f.getErr != nil { + return Value{}, f.getErr + } return Value{Bytes: []byte("historical"), Version: 7}, nil } func (f *fakeReader) Has(context.Context, string, []byte, int64) (bool, error) { f.has++ + if f.hasSet { + return f.hasResult, nil + } return true, nil } @@ -83,7 +93,7 @@ func TestFallbackStateStoreRoutesPrunedPointReads(t *testing.T) { require.NoError(t, err) require.True(t, ok) require.Equal(t, 0, primary.has) - require.Equal(t, 1, reader.has) + require.Equal(t, 0, reader.has) } func TestFallbackStateStoreKeepsRecentPointReadsOnPrimary(t *testing.T) { @@ -118,3 +128,68 @@ func TestFallbackStateStoreCachesHistoricalPointReads(t *testing.T) { require.Equal(t, []byte("historical"), value) require.Equal(t, 1, reader.gets) } + +func TestFallbackStateStoreCachesHistoricalMisses(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{getErr: ErrNotFound, hasSet: true} + store := NewFallbackStateStore(primary, reader) + + value, err := store.Get("bank", 7, []byte("missing")) + require.NoError(t, err) + require.Nil(t, value) + + value, err = store.Get("bank", 7, []byte("missing")) + require.NoError(t, err) + require.Nil(t, value) + + ok, err := store.Has("bank", 7, []byte("missing")) + require.NoError(t, err) + require.False(t, ok) + require.Equal(t, 1, reader.gets) + require.Equal(t, 0, reader.has) +} + +func TestFallbackStateStoreCachesHistoricalHasResults(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{hasResult: true, hasSet: true} + store := NewFallbackStateStore(primary, reader) + + ok, err := store.Has("bank", 7, []byte("k")) + require.NoError(t, err) + require.True(t, ok) + + ok, err = store.Has("bank", 7, []byte("k")) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 1, reader.has) +} + +func TestFallbackStateStoreDoesNotUseHasOnlyCacheForGet(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{hasResult: true, hasSet: true} + store := NewFallbackStateStore(primary, reader) + + ok, err := store.Has("bank", 7, []byte("k")) + require.NoError(t, err) + require.True(t, ok) + + value, err := store.Get("bank", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 1, reader.gets) +} + +func TestFallbackStateStoreDoesNotCacheHistoricalErrors(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{getErr: errors.New("boom")} + store := NewFallbackStateStore(primary, reader) + + _, err := store.Get("bank", 7, []byte("k")) + require.Error(t, err) + + reader.getErr = nil + value, err := store.Get("bank", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 2, reader.gets) +} From b3e0b81a73234b3b625085ed94353d430dd733b5 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 28 May 2026 10:42:52 -0400 Subject: [PATCH 19/41] Simplify Bigtable offload helpers --- .../state_db/ss/offload/consumer/compact.go | 57 +++++++++++++++++++ sei-db/state_db/ss/offload/consumer/scylla.go | 43 -------------- .../ss/offload/historical/bigtable.go | 16 +++--- .../state_db/ss/offload/historical/scylla.go | 19 ++++--- 4 files changed, 77 insertions(+), 58 deletions(-) create mode 100644 sei-db/state_db/ss/offload/consumer/compact.go diff --git a/sei-db/state_db/ss/offload/consumer/compact.go b/sei-db/state_db/ss/offload/consumer/compact.go new file mode 100644 index 0000000000..5903eae73a --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/compact.go @@ -0,0 +1,57 @@ +package consumer + +import "github.com/sei-protocol/sei-chain/sei-db/proto" + +type stateMutation struct { + storeName string + pair *proto.KVPair +} + +type stateMutationKey struct { + storeName string + key string +} + +func compactRecords(records []Record) []Record { + for _, rec := range records { + if rec.Entry == nil { + out := make([]Record, 0, len(records)) + for _, rec := range records { + if rec.Entry != nil { + out = append(out, rec) + } + } + return out + } + } + return records +} + +func compactMutations(entry *proto.ChangelogEntry) []stateMutation { + if entry == nil { + return nil + } + mutations := make([]stateMutation, 0, entryMutationCapacity(entry)) + indexByKey := make(map[stateMutationKey]int, cap(mutations)) + for _, ncs := range entry.Changesets { + storeName := ncs.Name + for _, pair := range ncs.Changeset.Pairs { + key := stateMutationKey{storeName: storeName, key: string(pair.Key)} + if idx, ok := indexByKey[key]; ok { + mutations[idx].pair = pair + continue + } + indexByKey[key] = len(mutations) + mutations = append(mutations, stateMutation{storeName: storeName, pair: pair}) + } + } + return mutations +} + +func entryMutationCapacity(entry *proto.ChangelogEntry) int { + total := 0 + for _, changeset := range entry.Changesets { + total += len(changeset.Changeset.Pairs) + } + return total +} diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index babec955e7..3bb712c710 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -134,21 +134,6 @@ func (s *scyllaSink) WriteBatch(ctx context.Context, records []Record) error { return s.writeRecordsPipelined(ctx, records) } -func compactRecords(records []Record) []Record { - for _, rec := range records { - if rec.Entry == nil { - out := make([]Record, 0, len(records)) - for _, rec := range records { - if rec.Entry != nil { - out = append(out, rec) - } - } - return out - } - } - return records -} - func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { entry := rec.Entry if entry == nil { @@ -227,34 +212,6 @@ func (s *scyllaSink) writeRecordRows(ctx context.Context, version int64, entry * return g.Wait() } -type scyllaMutation struct { - storeName string - pair *proto.KVPair -} - -type scyllaMutationKey struct { - storeName string - key string -} - -func compactMutations(entry *proto.ChangelogEntry) []scyllaMutation { - mutations := make([]scyllaMutation, 0) - indexByKey := make(map[scyllaMutationKey]int) - for _, ncs := range entry.Changesets { - storeName := ncs.Name - for _, pair := range ncs.Changeset.Pairs { - key := scyllaMutationKey{storeName: storeName, key: string(pair.Key)} - if idx, ok := indexByKey[key]; ok { - mutations[idx].pair = pair - continue - } - indexByKey[key] = len(mutations) - mutations = append(mutations, scyllaMutation{storeName: storeName, pair: pair}) - } - } - return mutations -} - func (s *scyllaSink) effectiveMutationWorkers() int { if s.mutationWorkers <= 0 { return defaultScyllaMutationWorkers diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 8cff2d5a4e..2bf7ada248 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -8,7 +8,6 @@ import ( "io" "regexp" "strings" - "sync" "time" "cloud.google.com/go/bigtable/apiv2/bigtablepb" @@ -320,8 +319,7 @@ func (r *bigtableReader) Get(ctx context.Context, storeName string, key []byte, } func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (int64, error) { - var maxVersion int64 - var mu sync.Mutex + versions := make([]int64, VersionBucketCount) g, gctx := errgroup.WithContext(ctx) g.SetLimit(defaultBigtableReadWorkers) for bucket := 0; bucket < VersionBucketCount; bucket++ { @@ -338,17 +336,19 @@ func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (in if err != nil { return fmt.Errorf("read latest bigtable version bucket %d: %w", bucket, err) } - mu.Lock() - if bucketVersion > maxVersion { - maxVersion = bucketVersion - } - mu.Unlock() + versions[bucket] = bucketVersion return nil }) } if err := g.Wait(); err != nil { return 0, err } + var maxVersion int64 + for _, version := range versions { + if version > maxVersion { + maxVersion = version + } + } return maxVersion, nil } diff --git a/sei-db/state_db/ss/offload/historical/scylla.go b/sei-db/state_db/ss/offload/historical/scylla.go index 686908c343..65a02d4c5d 100644 --- a/sei-db/state_db/ss/offload/historical/scylla.go +++ b/sei-db/state_db/ss/offload/historical/scylla.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "strings" - "sync" "time" "github.com/gocql/gocql" @@ -202,11 +201,12 @@ func sessionGet(session *gocql.Session) scyllaGetFunc { type scyllaGetFunc func(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) func (r *scyllaReader) BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) { - out := make(map[Lookup]Value, len(lookups)) g, gctx := errgroup.WithContext(ctx) g.SetLimit(defaultScyllaReadWorkers) - var mu sync.Mutex - for _, lookup := range lookups { + values := make([]Value, len(lookups)) + found := make([]bool, len(lookups)) + for i, lookup := range lookups { + i := i lookup := lookup g.Go(func() error { value, err := r.Get(gctx, lookup.StoreName, []byte(lookup.Key), targetVersion) @@ -216,15 +216,20 @@ func (r *scyllaReader) BatchGet(ctx context.Context, targetVersion int64, lookup } return err } - mu.Lock() - out[lookup] = value - mu.Unlock() + values[i] = value + found[i] = true return nil }) } if err := g.Wait(); err != nil { return nil, err } + out := make(map[Lookup]Value, len(lookups)) + for i, lookup := range lookups { + if found[i] { + out[lookup] = values[i] + } + } return out, nil } From 4e37dd67be2b8e7f0235c39161ae624730590da6 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 1 Jun 2026 12:42:17 -0400 Subject: [PATCH 20/41] Prepare Bigtable offload for review --- app/seidb.go | 26 +++++-- app/seidb_test.go | 34 +++++++++ go.mod | 2 +- go.sum | 2 + sei-db/state_db/ss/offload/consumer/README.md | 4 +- .../state_db/ss/offload/consumer/bigtable.go | 3 + .../ss/offload/consumer/bigtable_test.go | 19 +++++ .../main.go | 0 sei-db/state_db/ss/offload/consumer/kafka.go | 2 +- .../ss/offload/consumer/kafka_test.go | 2 +- .../ss/offload/consumer/schema/scylla.cql | 2 +- sei-db/state_db/ss/offload/consumer/scylla.go | 4 +- .../ss/offload/consumer/scylla_test.go | 72 +++++++++++++++++++ .../ss/offload/historical/bigtable.go | 2 +- .../ss/offload/historical/bigtable_test.go | 2 + 15 files changed, 160 insertions(+), 16 deletions(-) rename sei-db/state_db/ss/offload/consumer/cmd/{historical-scylla-consumer => historical-offload-consumer}/main.go (100%) diff --git a/app/seidb.go b/app/seidb.go index 6be013875d..8b163da4ed 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -46,13 +46,19 @@ const ( FlagEVMSSSeparateDBs = "state-store.evm-ss-separate-dbs" // Historical SS offload fallback. - FlagHistoricalOffloadScyllaHosts = "state-store.historical-offload-scylla-hosts" - FlagHistoricalOffloadScyllaKeyspace = "state-store.historical-offload-scylla-keyspace" - FlagHistoricalOffloadScyllaUsername = "state-store.historical-offload-scylla-username" - FlagHistoricalOffloadScyllaPassword = "state-store.historical-offload-scylla-password" - FlagHistoricalOffloadScyllaDatacenter = "state-store.historical-offload-scylla-datacenter" - FlagHistoricalOffloadScyllaConsistency = "state-store.historical-offload-scylla-consistency" - FlagHistoricalOffloadScyllaTimeoutMS = "state-store.historical-offload-scylla-timeout-ms" + FlagHistoricalOffloadScyllaHosts = "state-store.historical-offload-scylla-hosts" + FlagHistoricalOffloadScyllaKeyspace = "state-store.historical-offload-scylla-keyspace" + FlagHistoricalOffloadScyllaUsername = "state-store.historical-offload-scylla-username" + FlagHistoricalOffloadScyllaPassword = "state-store.historical-offload-scylla-password" + FlagHistoricalOffloadScyllaDatacenter = "state-store.historical-offload-scylla-datacenter" + FlagHistoricalOffloadScyllaConsistency = "state-store.historical-offload-scylla-consistency" + FlagHistoricalOffloadScyllaTimeoutMS = "state-store.historical-offload-scylla-timeout-ms" + FlagHistoricalOffloadBigtableProjectID = "state-store.historical-offload-bigtable-project-id" + FlagHistoricalOffloadBigtableInstance = "state-store.historical-offload-bigtable-instance" + FlagHistoricalOffloadBigtableTable = "state-store.historical-offload-bigtable-table" + FlagHistoricalOffloadBigtableFamily = "state-store.historical-offload-bigtable-family" + FlagHistoricalOffloadBigtableAppProfile = "state-store.historical-offload-bigtable-app-profile" + FlagHistoricalOffloadBigtableShards = "state-store.historical-offload-bigtable-shards" // Other configs FlagSnapshotInterval = "state-sync.snapshot-interval" @@ -164,6 +170,12 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig { ssConfig.HistoricalOffloadScyllaDatacenter = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaDatacenter)) ssConfig.HistoricalOffloadScyllaConsistency = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaConsistency)) ssConfig.HistoricalOffloadScyllaTimeoutMS = cast.ToInt(appOpts.Get(FlagHistoricalOffloadScyllaTimeoutMS)) + ssConfig.HistoricalOffloadBigtableProjectID = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableProjectID)) + ssConfig.HistoricalOffloadBigtableInstance = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableInstance)) + ssConfig.HistoricalOffloadBigtableTable = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableTable)) + ssConfig.HistoricalOffloadBigtableFamily = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableFamily)) + ssConfig.HistoricalOffloadBigtableAppProfile = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableAppProfile)) + ssConfig.HistoricalOffloadBigtableShards = cast.ToInt(appOpts.Get(FlagHistoricalOffloadBigtableShards)) return ssConfig } diff --git a/app/seidb_test.go b/app/seidb_test.go index d0a0c88dc1..7e19ddf0b4 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -75,6 +75,18 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSSConfig.HistoricalOffloadScyllaConsistency case FlagHistoricalOffloadScyllaTimeoutMS: return defaultSSConfig.HistoricalOffloadScyllaTimeoutMS + case FlagHistoricalOffloadBigtableProjectID: + return defaultSSConfig.HistoricalOffloadBigtableProjectID + case FlagHistoricalOffloadBigtableInstance: + return defaultSSConfig.HistoricalOffloadBigtableInstance + case FlagHistoricalOffloadBigtableTable: + return defaultSSConfig.HistoricalOffloadBigtableTable + case FlagHistoricalOffloadBigtableFamily: + return defaultSSConfig.HistoricalOffloadBigtableFamily + case FlagHistoricalOffloadBigtableAppProfile: + return defaultSSConfig.HistoricalOffloadBigtableAppProfile + case FlagHistoricalOffloadBigtableShards: + return defaultSSConfig.HistoricalOffloadBigtableShards } return nil } @@ -152,6 +164,28 @@ func TestParseSSConfigs_HistoricalScyllaFlags(t *testing.T) { assert.Equal(t, 1500, ssConfig.HistoricalOffloadScyllaTimeoutMS) } +func TestParseSSConfigs_HistoricalBigtableFlags(t *testing.T) { + appOpts := mapAppOpts{ + FlagSSEnable: true, + FlagHistoricalOffloadBigtableProjectID: "sei-project", + FlagHistoricalOffloadBigtableInstance: "sei-history", + FlagHistoricalOffloadBigtableTable: "state_mutations", + FlagHistoricalOffloadBigtableFamily: "state", + FlagHistoricalOffloadBigtableAppProfile: "historical", + FlagHistoricalOffloadBigtableShards: 512, + FlagSSAsyncWriterBuffer: 0, + } + + ssConfig := parseSSConfigs(appOpts) + assert.True(t, ssConfig.Enable) + assert.Equal(t, "sei-project", ssConfig.HistoricalOffloadBigtableProjectID) + assert.Equal(t, "sei-history", ssConfig.HistoricalOffloadBigtableInstance) + assert.Equal(t, "state_mutations", ssConfig.HistoricalOffloadBigtableTable) + assert.Equal(t, "state", ssConfig.HistoricalOffloadBigtableFamily) + assert.Equal(t, "historical", ssConfig.HistoricalOffloadBigtableAppProfile) + assert.Equal(t, 512, ssConfig.HistoricalOffloadBigtableShards) +} + func TestParseReceiptConfigs_DefaultsToPebbleWhenUnset(t *testing.T) { receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{}) assert.NoError(t, err) diff --git a/go.mod b/go.mod index 6012f8f185..df3d3731ae 100644 --- a/go.mod +++ b/go.mod @@ -29,13 +29,13 @@ require ( github.com/fortytw2/leaktest v1.3.0 github.com/go-git/go-git/v5 v5.17.2 github.com/go-kit/kit v0.13.0 + github.com/gocql/gocql v1.7.0 github.com/gofrs/flock v0.13.0 github.com/gogo/gateway v1.1.0 github.com/gogo/protobuf v1.3.3 github.com/golang-jwt/jwt/v4 v4.5.1 github.com/golang/mock v1.7.0-rc.1 github.com/golang/protobuf v1.5.4 - github.com/gocql/gocql v1.7.0 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.7.0 github.com/google/gofuzz v1.2.0 diff --git a/go.sum b/go.sum index ad867b4147..2f48c3f9ca 100644 --- a/go.sum +++ b/go.sum @@ -740,6 +740,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E= github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.14.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= @@ -748,6 +749,7 @@ github.com/bits-and-blooms/bitset v1.24.3 h1:Bte86SlO3lwPQqww+7BE9ZuUCKIjfqnG5jt github.com/bits-and-blooms/bitset v1.24.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md index f716835cfb..82df3149bc 100644 --- a/sei-db/state_db/ss/offload/consumer/README.md +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -51,7 +51,7 @@ them into the configured backend. Kafka offsets are committed only after the sink write succeeds. Mutation rows are written before the version marker. ```bash -go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer \ +go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer \ ./sei-db/state_db/ss/offload/consumer/config/example-scylla.json ``` @@ -61,7 +61,7 @@ For Bigtable: cbt -project my-gcp-project -instance sei-history createtable state_mutations cbt -project my-gcp-project -instance sei-history createfamily state_mutations state -go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer \ +go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer \ ./sei-db/state_db/ss/offload/consumer/config/example-bigtable.json ``` diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index a77c02ad8d..ca05e6d7e5 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -164,6 +164,9 @@ func bigtableBulkError(rows []historical.BigtableRowMutation, errs []error, err if err != nil { return err } + if len(errs) != len(rows) { + return fmt.Errorf("bigtable returned %d mutation results for %d rows", len(errs), len(rows)) + } for i, rowErr := range errs { if rowErr != nil { return fmt.Errorf("row %q: %w", rows[i].RowKey, rowErr) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable_test.go b/sei-db/state_db/ss/offload/consumer/bigtable_test.go index ef2d1146c4..a4794c69df 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable_test.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable_test.go @@ -2,6 +2,7 @@ package consumer import ( "context" + "errors" "testing" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -86,3 +87,21 @@ func TestBigtableSinkWriteBatchWritesRowsBeforeMarkers(t *testing.T) { }, }, calls) } + +func TestBigtableBulkErrorValidatesMutationResultCount(t *testing.T) { + rows := []historical.BigtableRowMutation{{RowKey: "row-1"}, {RowKey: "row-2"}} + + err := bigtableBulkError(rows, []error{nil}, nil) + + require.ErrorContains(t, err, "mutation results") +} + +func TestBigtableBulkErrorWrapsRowError(t *testing.T) { + rowErr := errors.New("failed") + rows := []historical.BigtableRowMutation{{RowKey: "row-1"}} + + err := bigtableBulkError(rows, []error{rowErr}, nil) + + require.ErrorIs(t, err, rowErr) + require.ErrorContains(t, err, "row-1") +} diff --git a/sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go b/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go similarity index 100% rename from sei-db/state_db/ss/offload/consumer/cmd/historical-scylla-consumer/main.go rename to sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go diff --git a/sei-db/state_db/ss/offload/consumer/kafka.go b/sei-db/state_db/ss/offload/consumer/kafka.go index 1785bb05f8..cff52d4d78 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka.go +++ b/sei-db/state_db/ss/offload/consumer/kafka.go @@ -32,7 +32,7 @@ type KafkaReaderConfig struct { func (c *KafkaReaderConfig) ApplyDefaults() { if c.ClientID == "" { - c.ClientID = "cryptosim-historical-scylla-consumer" + c.ClientID = "cryptosim-historical-offload-consumer" } if c.StartOffset == "" { c.StartOffset = "first" diff --git a/sei-db/state_db/ss/offload/consumer/kafka_test.go b/sei-db/state_db/ss/offload/consumer/kafka_test.go index 7fa6b320e4..e714b7dd60 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka_test.go +++ b/sei-db/state_db/ss/offload/consumer/kafka_test.go @@ -14,7 +14,7 @@ func TestKafkaReaderConfigApplyDefaults(t *testing.T) { GroupID: "scylla", } cfg.ApplyDefaults() - require.Equal(t, "cryptosim-historical-scylla-consumer", cfg.ClientID) + require.Equal(t, "cryptosim-historical-offload-consumer", cfg.ClientID) require.Equal(t, "first", cfg.StartOffset) require.Equal(t, 1, cfg.MinBytes) require.Equal(t, 10<<20, cfg.MaxBytes) diff --git a/sei-db/state_db/ss/offload/consumer/schema/scylla.cql b/sei-db/state_db/ss/offload/consumer/schema/scylla.cql index ceb7d3cfc2..aa26df4c97 100644 --- a/sei-db/state_db/ss/offload/consumer/schema/scylla.cql +++ b/sei-db/state_db/ss/offload/consumer/schema/scylla.cql @@ -1,5 +1,5 @@ -- ScyllaDB/Cassandra schema for Sei historical state offload. --- Apply once before running historical-scylla-consumer. +-- Apply once before running historical-offload-consumer. CREATE KEYSPACE IF NOT EXISTS sei_history WITH replication = { diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index 3bb712c710..149aba2693 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -164,14 +164,14 @@ func (s *scyllaSink) writeVersionMarker(ctx context.Context, rec Record) error { func (s *scyllaSink) writeRecordsPipelined(ctx context.Context, records []Record) error { rowCtx, cancel := context.WithCancel(ctx) defer cancel() - g, gctx := errgroup.WithContext(rowCtx) + var g errgroup.Group rowDone := make([]chan error, len(records)) for i := range records { rowDone[i] = make(chan error, 1) i := i rec := records[i] g.Go(func() error { - err := s.writeRecordRows(gctx, rec.Entry.Version, rec.Entry) + err := s.writeRecordRows(rowCtx, rec.Entry.Version, rec.Entry) if err != nil { err = fmt.Errorf("write scylla/cassandra rows version %d: %w", rec.Entry.Version, err) } diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go index 8516ed6e74..f10e672347 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -2,6 +2,7 @@ package consumer import ( "context" + "errors" "strings" "sync" "sync/atomic" @@ -325,3 +326,74 @@ func TestScyllaSinkWriteBatchPipelinesRowsAndOrdersMarkers(t *testing.T) { require.False(t, markerBeforeRowsDone) require.Equal(t, []int64{1, 2}, markers) } + +func TestScyllaSinkWriteBatchReturnsRowErrorAfterLaterRowFailure(t *testing.T) { + rowErr := errors.New("row write failed") + rowStarted := make(chan int64, 2) + releaseFirst := make(chan struct{}) + + sink := &scyllaSink{ + mutationWorkers: 1, + exec: func(ctx context.Context, stmt string, values ...interface{}) error { + if !strings.Contains(stmt, "state_mutations") { + return nil + } + version := values[2].(int64) + rowStarted <- version + if version == 2 { + return rowErr + } + select { + case <-releaseFirst: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }, + } + records := []Record{ + { + Entry: &proto.ChangelogEntry{ + Version: 1, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k1"), Value: []byte("v1")}}}, + }}, + }, + }, + { + Entry: &proto.ChangelogEntry{ + Version: 2, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k2"), Value: []byte("v2")}}}, + }}, + }, + }, + } + + errCh := make(chan error, 1) + go func() { + errCh <- sink.WriteBatch(context.Background(), records) + }() + + started := map[int64]bool{} + for len(started) < 2 { + select { + case version := <-rowStarted: + started[version] = true + case <-time.After(time.Second): + close(releaseFirst) + t.Fatal("timed out waiting for row writes") + } + } + close(releaseFirst) + + select { + case err := <-errCh: + require.ErrorIs(t, err, rowErr) + require.NotErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("timed out waiting for batch write") + } +} diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 2bf7ada248..1a691c110c 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -106,7 +106,7 @@ func (c *BigtableConfig) Validate() error { if strings.TrimSpace(c.Family) == "" { return fmt.Errorf("bigtable family is required") } - if c.Shards < 0 || c.Shards > 65535 { + if c.Shards <= 0 || c.Shards > maxUint16Int { return fmt.Errorf("bigtable shards must be between 1 and 65535") } return nil diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go index ef8dafff8a..acaf815579 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable_test.go +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -24,9 +24,11 @@ func TestBigtableConfigDefaultsAndValidate(t *testing.T) { missingProject := BigtableConfig{InstanceID: "i", Table: "t", Family: "f"} missingInstance := BigtableConfig{ProjectID: "p", Table: "t", Family: "f"} missingTable := BigtableConfig{ProjectID: "p", InstanceID: "i", Family: "f"} + missingShards := BigtableConfig{ProjectID: "p", InstanceID: "i", Table: "t", Family: "f"} require.ErrorContains(t, missingProject.Validate(), "project") require.ErrorContains(t, missingInstance.Validate(), "instance") require.ErrorContains(t, missingTable.Validate(), "table") + require.ErrorContains(t, missingShards.Validate(), "shards") } func TestBigtableMutationRowKeyOrdersLatestVersionFirst(t *testing.T) { From c564e9eb61a77d211831e11fd8f77fffe8fb9e2f Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 1 Jun 2026 12:49:07 -0400 Subject: [PATCH 21/41] Align historical fallback with main iterator type --- sei-db/state_db/ss/offload/historical/store.go | 5 +++-- sei-db/state_db/ss/offload/historical/store_test.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index 756eb4ed19..4a3bfddd47 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -8,6 +8,7 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" + dbm "github.com/tendermint/tm-db" ) const ( @@ -143,11 +144,11 @@ func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bo return found, nil } -func (s *FallbackStateStore) Iterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { +func (s *FallbackStateStore) Iterator(storeKey string, version int64, start, end []byte) (dbm.Iterator, error) { return s.primary.Iterator(storeKey, version, start, end) } -func (s *FallbackStateStore) ReverseIterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { +func (s *FallbackStateStore) ReverseIterator(storeKey string, version int64, start, end []byte) (dbm.Iterator, error) { return s.primary.ReverseIterator(storeKey, version, start, end) } diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go index ba64ff93cf..430eb92b18 100644 --- a/sei-db/state_db/ss/offload/historical/store_test.go +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -8,6 +8,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" ) type fakeStateStore struct { @@ -26,11 +27,11 @@ func (f *fakeStateStore) Has(_ string, _ int64, _ []byte) (bool, error) { return true, nil } -func (f *fakeStateStore) Iterator(string, int64, []byte, []byte) (types.DBIterator, error) { +func (f *fakeStateStore) Iterator(string, int64, []byte, []byte) (dbm.Iterator, error) { return nil, nil } -func (f *fakeStateStore) ReverseIterator(string, int64, []byte, []byte) (types.DBIterator, error) { +func (f *fakeStateStore) ReverseIterator(string, int64, []byte, []byte) (dbm.Iterator, error) { return nil, nil } From 292f5014e6b2af9bfb5d7333498a12a58340880c Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 1 Jun 2026 12:53:29 -0400 Subject: [PATCH 22/41] Refresh PR checks after labeling From e2b358fd9fd0db52420760c811ff68961728a50d Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 1 Jun 2026 14:31:43 -0400 Subject: [PATCH 23/41] Parallelize Bigtable mutation batches by locality --- .../state_db/ss/offload/consumer/bigtable.go | 99 ++++++++-- .../ss/offload/consumer/bigtable_test.go | 173 +++++++++++++++++- 2 files changed, 252 insertions(+), 20 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index ca05e6d7e5..32f2750c6d 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -3,21 +3,30 @@ package consumer import ( "context" "fmt" + "sort" "strconv" "time" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" + "golang.org/x/sync/errgroup" ) type BigtableConfig = historical.BigtableConfig +const ( + defaultBigtableMutationChunkRows = 1024 + defaultBigtableMutationChunkConcurrency = 8 +) + type bigtableSink struct { - client *historical.BigtableClient - applyBulk historical.BigtableApplyBulkFunc - readRows historical.BigtableReadRowsFunc - family string - shards int + client *historical.BigtableClient + applyBulk historical.BigtableApplyBulkFunc + readRows historical.BigtableReadRowsFunc + family string + shards int + bulkChunkRows int + bulkChunkWorkers int } var _ Sink = (*bigtableSink)(nil) @@ -34,11 +43,13 @@ func NewBigtableSink(cfg BigtableConfig) (Sink, error) { return nil, err } return &bigtableSink{ - client: client, - applyBulk: client.ApplyBulk, - readRows: client.ReadRows, - family: cfg.Family, - shards: cfg.Shards, + client: client, + applyBulk: client.ApplyBulk, + readRows: client.ReadRows, + family: cfg.Family, + shards: cfg.Shards, + bulkChunkRows: defaultBigtableMutationChunkRows, + bulkChunkWorkers: defaultBigtableMutationChunkConcurrency, }, nil } @@ -76,8 +87,35 @@ func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) er if len(rows) == 0 { return nil } - errs, err := s.applyBulk(ctx, rows) - return bigtableBulkError(rows, errs, err) + return s.applyRecordRowMutations(ctx, rows) +} + +func (s *bigtableSink) applyRecordRowMutations(ctx context.Context, rows []historical.BigtableRowMutation) error { + chunks := bigtableRowMutationChunks(rows, s.effectiveBulkChunkRows()) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(s.effectiveBulkChunkWorkers()) + for _, chunk := range chunks { + chunk := chunk + g.Go(func() error { + errs, err := s.applyBulk(gctx, chunk) + return bigtableBulkError(chunk, errs, err) + }) + } + return g.Wait() +} + +func (s *bigtableSink) effectiveBulkChunkRows() int { + if s.bulkChunkRows <= 0 { + return defaultBigtableMutationChunkRows + } + return s.bulkChunkRows +} + +func (s *bigtableSink) effectiveBulkChunkWorkers() int { + if s.bulkChunkWorkers <= 0 { + return defaultBigtableMutationChunkConcurrency + } + return s.bulkChunkWorkers } func (s *bigtableSink) appendRecordRowMutations(rows []historical.BigtableRowMutation, version int64, entry *proto.ChangelogEntry) []historical.BigtableRowMutation { @@ -160,6 +198,43 @@ func bigtableRowMutationCount(records []Record) int { return total } +func bigtableRowMutationChunks(rows []historical.BigtableRowMutation, maxRows int) [][]historical.BigtableRowMutation { + if len(rows) == 0 { + return nil + } + if maxRows <= 0 { + maxRows = len(rows) + } + sort.Slice(rows, func(i, j int) bool { + return rows[i].RowKey < rows[j].RowKey + }) + + chunks := make([][]historical.BigtableRowMutation, 0, (len(rows)+maxRows-1)/maxRows) + start := 0 + startLocality := bigtableRowLocality(rows[0].RowKey) + for i := 1; i < len(rows); i++ { + locality := bigtableRowLocality(rows[i].RowKey) + if i-start >= maxRows || locality != startLocality { + chunks = append(chunks, rows[start:i]) + start = i + startLocality = locality + } + } + return append(chunks, rows[start:]) +} + +func bigtableRowLocality(rowKey string) string { + // Mutation row keys are m|shard|store|key|version; keep chunks inside one + // shard prefix so separate chunks can hit separate Bigtable tablets. + if len(rowKey) >= 3 && rowKey[0] == 'm' { + return rowKey[:3] + } + if len(rowKey) > 0 { + return rowKey[:1] + } + return rowKey +} + func bigtableBulkError(rows []historical.BigtableRowMutation, errs []error, err error) error { if err != nil { return err diff --git a/sei-db/state_db/ss/offload/consumer/bigtable_test.go b/sei-db/state_db/ss/offload/consumer/bigtable_test.go index a4794c69df..7b82296389 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable_test.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" @@ -37,9 +38,11 @@ func TestBigtableSinkWritesMutationRowsAndVersionMarker(t *testing.T) { require.NoError(t, sink.Write(context.Background(), Record{Topic: "t", Partition: 1, Offset: 2, Entry: entry})) require.Len(t, rows, 4) - require.Equal(t, historical.BigtableMutationRowKey("bank", []byte("k1"), 7, historical.DefaultBigtableShards), rows[0]) - require.Equal(t, historical.BigtableMutationRowKey("bank", []byte("drop"), 7, historical.DefaultBigtableShards), rows[1]) - require.Equal(t, historical.BigtableUpgradeRowKey(7, "new-store"), rows[2]) + require.ElementsMatch(t, []string{ + historical.BigtableMutationRowKey("bank", []byte("k1"), 7, historical.DefaultBigtableShards), + historical.BigtableMutationRowKey("bank", []byte("drop"), 7, historical.DefaultBigtableShards), + historical.BigtableUpgradeRowKey(7, "new-store"), + }, rows[:3]) require.Equal(t, historical.BigtableVersionRowKey(7), rows[3]) } @@ -76,16 +79,146 @@ func TestBigtableSinkWriteBatchWritesRowsBeforeMarkers(t *testing.T) { } require.NoError(t, sink.WriteBatch(context.Background(), records)) + require.GreaterOrEqual(t, len(calls), 2) + require.ElementsMatch(t, []string{ + historical.BigtableMutationRowKey("bank", []byte("k1"), 1, historical.DefaultBigtableShards), + historical.BigtableMutationRowKey("bank", []byte("k2"), 2, historical.DefaultBigtableShards), + }, flattenCalls(calls[:len(calls)-1])) + require.Equal(t, []string{ + historical.BigtableVersionRowKey(1), + historical.BigtableVersionRowKey(2), + }, calls[len(calls)-1]) +} + +func TestBigtableRowMutationChunksSortsAndGroupsByLocality(t *testing.T) { + rows := []historical.BigtableRowMutation{ + {RowKey: string([]byte{'u', 'z'})}, + {RowKey: string([]byte{'m', 0, 2, 'b'})}, + {RowKey: string([]byte{'m', 0, 1, 'c'})}, + {RowKey: string([]byte{'m', 0, 1, 'a'})}, + {RowKey: string([]byte{'m', 0, 1, 'b'})}, + {RowKey: string([]byte{'m', 0, 2, 'a'})}, + } + + chunks := bigtableRowMutationChunks(rows, 2) + require.Equal(t, [][]string{ { - historical.BigtableMutationRowKey("bank", []byte("k1"), 1, historical.DefaultBigtableShards), - historical.BigtableMutationRowKey("bank", []byte("k2"), 2, historical.DefaultBigtableShards), + string([]byte{'m', 0, 1, 'a'}), + string([]byte{'m', 0, 1, 'b'}), + }, + { + string([]byte{'m', 0, 1, 'c'}), }, { - historical.BigtableVersionRowKey(1), - historical.BigtableVersionRowKey(2), + string([]byte{'m', 0, 2, 'a'}), + string([]byte{'m', 0, 2, 'b'}), + }, + { + string([]byte{'u', 'z'}), + }, + }, chunkRowKeys(chunks)) +} + +func TestBigtableSinkWriteBatchChunksRecordRowsBeforeMarkers(t *testing.T) { + var calls [][]string + sink := &bigtableSink{ + family: historical.DefaultBigtableFamily, + shards: 2, + bulkChunkRows: 1, + bulkChunkWorkers: 1, + applyBulk: func(_ context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { + call := make([]string, 0, len(mutations)) + for _, mutation := range mutations { + call = append(call, mutation.RowKey) + } + calls = append(calls, call) + return make([]error, len(mutations)), nil + }, + } + records := []Record{ + {Entry: &proto.ChangelogEntry{ + Version: 1, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k1"), Value: []byte("v1")}, + {Key: []byte("k2"), Value: []byte("v2")}, + }}, + }}, + Upgrades: []*proto.TreeNameUpgrade{{Name: "new-store"}}, + }}, + {Entry: &proto.ChangelogEntry{ + Version: 2, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k3"), Value: []byte("v3")}, + }}, + }}, + }}, + } + + require.NoError(t, sink.WriteBatch(context.Background(), records)) + + require.Greater(t, len(calls), 2) + markerCall := calls[len(calls)-1] + require.Equal(t, []string{ + historical.BigtableVersionRowKey(1), + historical.BigtableVersionRowKey(2), + }, markerCall) + for _, call := range calls[:len(calls)-1] { + require.Len(t, call, 1) + require.NotEqual(t, markerCall, call) + } + require.ElementsMatch(t, []string{ + historical.BigtableMutationRowKey("bank", []byte("k1"), 1, 2), + historical.BigtableMutationRowKey("bank", []byte("k2"), 1, 2), + historical.BigtableUpgradeRowKey(1, "new-store"), + historical.BigtableMutationRowKey("bank", []byte("k3"), 2, 2), + }, flattenCalls(calls[:len(calls)-1])) +} + +func TestBigtableSinkAppliesRecordChunksConcurrently(t *testing.T) { + started := make(chan struct{}, 2) + release := make(chan struct{}) + sink := &bigtableSink{ + bulkChunkRows: 1, + bulkChunkWorkers: 2, + applyBulk: func(ctx context.Context, rows []historical.BigtableRowMutation) ([]error, error) { + select { + case started <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + select { + case <-release: + return make([]error, len(rows)), nil + case <-ctx.Done(): + return nil, ctx.Err() + } }, - }, calls) + } + rows := []historical.BigtableRowMutation{ + {RowKey: string([]byte{'m', 0, 1, 'a'})}, + {RowKey: string([]byte{'m', 0, 2, 'a'})}, + } + + errCh := make(chan error, 1) + go func() { + errCh <- sink.applyRecordRowMutations(context.Background(), rows) + }() + + for i := 0; i < 2; i++ { + select { + case <-started: + case <-time.After(time.Second): + close(release) + require.FailNow(t, "timed out waiting for concurrent bigtable chunks") + } + } + close(release) + require.NoError(t, <-errCh) } func TestBigtableBulkErrorValidatesMutationResultCount(t *testing.T) { @@ -105,3 +238,27 @@ func TestBigtableBulkErrorWrapsRowError(t *testing.T) { require.ErrorIs(t, err, rowErr) require.ErrorContains(t, err, "row-1") } + +func chunkRowKeys(chunks [][]historical.BigtableRowMutation) [][]string { + out := make([][]string, 0, len(chunks)) + for _, chunk := range chunks { + out = append(out, flattenRowMutations(chunk)) + } + return out +} + +func flattenCalls(calls [][]string) []string { + var out []string + for _, call := range calls { + out = append(out, call...) + } + return out +} + +func flattenRowMutations(rows []historical.BigtableRowMutation) []string { + out := make([]string, 0, len(rows)) + for _, row := range rows { + out = append(out, row.RowKey) + } + return out +} From 2435876aec8d5b1ef76042cdd78ab8b2950a6eaf Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 1 Jun 2026 14:37:08 -0400 Subject: [PATCH 24/41] Fix Bigtable chunking tests under race detector --- sei-db/state_db/ss/offload/consumer/bigtable_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable_test.go b/sei-db/state_db/ss/offload/consumer/bigtable_test.go index 7b82296389..a01bff0800 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable_test.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable_test.go @@ -14,8 +14,9 @@ import ( func TestBigtableSinkWritesMutationRowsAndVersionMarker(t *testing.T) { var rows []string sink := &bigtableSink{ - family: historical.DefaultBigtableFamily, - shards: historical.DefaultBigtableShards, + family: historical.DefaultBigtableFamily, + shards: historical.DefaultBigtableShards, + bulkChunkWorkers: 1, applyBulk: func(_ context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { for _, mutation := range mutations { rows = append(rows, mutation.RowKey) @@ -50,8 +51,9 @@ func TestBigtableSinkWriteBatchWritesRowsBeforeMarkers(t *testing.T) { var calls [][]string sink := &bigtableSink{ - family: historical.DefaultBigtableFamily, - shards: historical.DefaultBigtableShards, + family: historical.DefaultBigtableFamily, + shards: historical.DefaultBigtableShards, + bulkChunkWorkers: 1, applyBulk: func(_ context.Context, mutations []historical.BigtableRowMutation) ([]error, error) { call := make([]string, 0, len(mutations)) for _, mutation := range mutations { From 3b0a3e63b3908fc600b9e562c4545c7368523792 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 2 Jul 2026 10:41:41 -0400 Subject: [PATCH 25/41] Measure Bigtable read/write cost for historical state Adds OpenTelemetry instrumentation at the Bigtable RPC boundary so we can track how Bigtable cost scales with historical query and ingestion load. Captures the three cost drivers Bigtable bills on, with a single bounded "table" attribute (flat storage as request volume grows): - request count: bigtable_{read,mutate}_latency_seconds _count (free) - rows touched: bigtable_rows_read_total / bigtable_rows_mutated_total - bytes moved: bigtable_bytes_read_total / bigtable_bytes_written_total ReadRows records rows/bytes returned and latency; ApplyBulk records rows/ bytes written and latency. Both record on failure too, since a failed RPC still costs. Meter "seidb_bigtable" exports through the existing Prometheus pipeline. Nil-safe so it is a no-op when no MeterProvider is configured. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ss/offload/historical/bigtable.go | 24 +++- .../state_db/ss/offload/historical/metrics.go | 135 ++++++++++++++++++ .../ss/offload/historical/metrics_test.go | 51 +++++++ 3 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 sei-db/state_db/ss/offload/historical/metrics.go create mode 100644 sei-db/state_db/ss/offload/historical/metrics_test.go diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 1a691c110c..c4c29d0294 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -44,6 +44,9 @@ type BigtableClient struct { data bigtablepb.BigtableClient tableName string appProfile string + // table is the short table name used as the metric attribute. + table string + metrics *bigtableMetrics } type BigtableCell struct { @@ -148,6 +151,8 @@ func OpenBigtableClient(ctx context.Context, cfg BigtableConfig) (*BigtableClien data: bigtablepb.NewBigtableClient(conn), tableName: bigtableTableName(cfg.ProjectID, cfg.InstanceID, cfg.Table), appProfile: cfg.AppProfile, + table: cfg.Table, + metrics: newBigtableMetrics(), }, nil } @@ -163,6 +168,11 @@ func (c *BigtableClient) Close() error { } func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { + start := time.Now() + var rowsRead, bytesRead int64 + defer func() { + c.metrics.recordRead(ctx, c.table, time.Since(start), rowsRead, bytesRead) + }() req := &bigtablepb.ReadRowsRequest{ TableName: c.tableName, AppProfileId: c.appProfile, @@ -194,8 +204,12 @@ func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, if err != nil { return err } - if committed && !f(row) { - return nil + if committed { + rowsRead++ + bytesRead += bigtableRowSize(row) + if !f(row) { + return nil + } } } } @@ -205,8 +219,14 @@ func (c *BigtableClient) ApplyBulk(ctx context.Context, rows []BigtableRowMutati if len(rows) == 0 { return nil, nil } + start := time.Now() + var bytesWritten int64 + defer func() { + c.metrics.recordWrite(ctx, c.table, time.Since(start), int64(len(rows)), bytesWritten) + }() entries := make([]*bigtablepb.MutateRowsRequest_Entry, 0, len(rows)) for _, row := range rows { + bytesWritten += bigtableMutationSize(row) entry := &bigtablepb.MutateRowsRequest_Entry{ RowKey: []byte(row.RowKey), Mutations: make([]*bigtablepb.Mutation, 0, len(row.SetCells)), diff --git a/sei-db/state_db/ss/offload/historical/metrics.go b/sei-db/state_db/ss/offload/historical/metrics.go new file mode 100644 index 0000000000..a542121fe1 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/metrics.go @@ -0,0 +1,135 @@ +package historical + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// bigtableMeterName is the OpenTelemetry meter used for Bigtable-backed +// historical-state access. It exports through the same process-wide +// MeterProvider (Prometheus exporter) as the rest of SeiDB. +const bigtableMeterName = "seidb_bigtable" + +// bigtableMetrics measures the cost of Bigtable-backed historical state so it +// is easy to see how read and write volume scales with historical query and +// ingestion load. The three cost drivers Bigtable bills on — request count, +// rows touched, and bytes transferred — are each captured directly: +// +// - request count: the *_latency_seconds histogram's _count series (free); +// - rows touched: bigtable_rows_read_total / bigtable_rows_mutated_total; +// - bytes moved: bigtable_bytes_read_total / bigtable_bytes_written_total. +// +// All series carry only a single low-cardinality "table" attribute, so the +// storage footprint stays flat as request volume grows. +type bigtableMetrics struct { + readLatency metric.Float64Histogram + rowsRead metric.Int64Counter + bytesRead metric.Int64Counter + writeLatency metric.Float64Histogram + rowsMutated metric.Int64Counter + bytesWritten metric.Int64Counter +} + +func newBigtableMetrics() *bigtableMetrics { + meter := otel.Meter(bigtableMeterName) + // Buckets span single-digit-ms cache-warm reads to multi-second slow RPCs. + latencyBuckets := metric.WithExplicitBucketBoundaries( + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, + ) + readLatency, _ := meter.Float64Histogram( + "bigtable_read_latency_seconds", + metric.WithDescription("Latency of Bigtable ReadRows RPCs for historical state; the _count series doubles as the read request count"), + metric.WithUnit("s"), + latencyBuckets, + ) + rowsRead, _ := meter.Int64Counter( + "bigtable_rows_read_total", + metric.WithDescription("Rows returned by Bigtable ReadRows RPCs for historical state"), + metric.WithUnit("{row}"), + ) + bytesRead, _ := meter.Int64Counter( + "bigtable_bytes_read_total", + metric.WithDescription("Bytes (row keys + cell values) returned by Bigtable reads for historical state"), + metric.WithUnit("By"), + ) + writeLatency, _ := meter.Float64Histogram( + "bigtable_mutate_latency_seconds", + metric.WithDescription("Latency of Bigtable MutateRows RPCs for historical-state ingestion; the _count series doubles as the mutate request count"), + metric.WithUnit("s"), + latencyBuckets, + ) + rowsMutated, _ := meter.Int64Counter( + "bigtable_rows_mutated_total", + metric.WithDescription("Rows written by Bigtable MutateRows RPCs for historical-state ingestion"), + metric.WithUnit("{row}"), + ) + bytesWritten, _ := meter.Int64Counter( + "bigtable_bytes_written_total", + metric.WithDescription("Bytes (row keys + cell values) written by Bigtable mutations for historical state"), + metric.WithUnit("By"), + ) + return &bigtableMetrics{ + readLatency: readLatency, + rowsRead: rowsRead, + bytesRead: bytesRead, + writeLatency: writeLatency, + rowsMutated: rowsMutated, + bytesWritten: bytesWritten, + } +} + +// recordRead reports one ReadRows RPC: its latency plus the rows and bytes it +// returned. It records even when the RPC fails, since a failed read still costs. +func (m *bigtableMetrics) recordRead(ctx context.Context, table string, latency time.Duration, rows, bytes int64) { + if m == nil { + return + } + attrs := metric.WithAttributes(attribute.String("table", table)) + m.readLatency.Record(ctx, latency.Seconds(), attrs) + if rows > 0 { + m.rowsRead.Add(ctx, rows, attrs) + } + if bytes > 0 { + m.bytesRead.Add(ctx, bytes, attrs) + } +} + +// recordWrite reports one MutateRows RPC: its latency plus the rows and bytes +// it wrote (attempted). It records even on failure, since the request still costs. +func (m *bigtableMetrics) recordWrite(ctx context.Context, table string, latency time.Duration, rows, bytes int64) { + if m == nil { + return + } + attrs := metric.WithAttributes(attribute.String("table", table)) + m.writeLatency.Record(ctx, latency.Seconds(), attrs) + if rows > 0 { + m.rowsMutated.Add(ctx, rows, attrs) + } + if bytes > 0 { + m.bytesWritten.Add(ctx, bytes, attrs) + } +} + +// bigtableRowSize estimates the transferred size of a read row: the row key +// plus each returned cell's qualifier and value. +func bigtableRowSize(row BigtableRow) int64 { + size := int64(len(row.Key)) + for _, cell := range row.Cells { + size += int64(len(cell.Qualifier)) + int64(len(cell.Value)) + } + return size +} + +// bigtableMutationSize estimates the written size of a mutation row: the row +// key plus each set cell's qualifier and value. +func bigtableMutationSize(row BigtableRowMutation) int64 { + size := int64(len(row.RowKey)) + for _, cell := range row.SetCells { + size += int64(len(cell.Qualifier)) + int64(len(cell.Value)) + } + return size +} diff --git a/sei-db/state_db/ss/offload/historical/metrics_test.go b/sei-db/state_db/ss/offload/historical/metrics_test.go new file mode 100644 index 0000000000..d35cfbf608 --- /dev/null +++ b/sei-db/state_db/ss/offload/historical/metrics_test.go @@ -0,0 +1,51 @@ +package historical + +import ( + "context" + "testing" + "time" +) + +func TestBigtableRowSize(t *testing.T) { + row := BigtableRow{ + Key: "abc", // 3 + Cells: []BigtableCell{ + {Qualifier: "value", Value: []byte("hello")}, // 5 + 5 + {Qualifier: "deleted", Value: []byte{0x1}}, // 7 + 1 + }, + } + if got, want := bigtableRowSize(row), int64(3+5+5+7+1); got != want { + t.Fatalf("bigtableRowSize() = %d, want %d", got, want) + } + if got := bigtableRowSize(BigtableRow{}); got != 0 { + t.Fatalf("bigtableRowSize(empty) = %d, want 0", got) + } +} + +func TestBigtableMutationSize(t *testing.T) { + row := BigtableRowMutation{ + RowKey: "row1", // 4 + SetCells: []BigtableSetCell{ + {Qualifier: "value", Value: []byte("data")}, // 5 + 4 + }, + } + if got, want := bigtableMutationSize(row), int64(4+5+4); got != want { + t.Fatalf("bigtableMutationSize() = %d, want %d", got, want) + } +} + +// A nil *bigtableMetrics must be a safe no-op so callers never need nil checks +// and metrics stay zero-overhead when no MeterProvider records them. +func TestBigtableMetricsNilSafe(t *testing.T) { + var m *bigtableMetrics + m.recordRead(context.Background(), "state", time.Millisecond, 2, 128) + m.recordWrite(context.Background(), "state", time.Millisecond, 4, 256) +} + +func TestBigtableMetricsRecordNoPanic(t *testing.T) { + m := newBigtableMetrics() + m.recordRead(context.Background(), "state", 5*time.Millisecond, 3, 512) + m.recordWrite(context.Background(), "state", 7*time.Millisecond, 10, 4096) + // Zero rows/bytes must still record latency without panicking. + m.recordRead(context.Background(), "state", time.Millisecond, 0, 0) +} From 87a9d5f63731edc717a3d6e11cbac26d0b1f9191 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 7 Jul 2026 13:55:19 -0400 Subject: [PATCH 26/41] Export consumer Prometheus metrics so Bigtable cost is scrapeable The Kafka -> Bigtable consumer records backend cost via the shared historical.BigtableClient (ApplyBulk/ReadRows are already instrumented with bigtable_rows_mutated_total, bigtable_bytes_written_total, bigtable_mutate_latency_seconds, etc.), but the consumer binary never installed a MeterProvider or served /metrics, so those counters went to the no-op provider and could not be scraped -- the dominant ingestion cost at 150k TPS was unmeasurable. Add a MetricsAddr config field and, when set, install the Prometheus MeterProvider before opening the sink (so its meters bind to the real exporter) and serve /metrics. Empty MetricsAddr keeps the endpoint off. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cmd/historical-offload-consumer/main.go | 25 ++++++++++++++++--- sei-db/state_db/ss/offload/consumer/config.go | 5 ++++ .../consumer/config/example-bigtable.json | 3 ++- .../ss/offload/consumer/config_test.go | 16 ++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go b/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go index 60dc2fc241..d949de6706 100644 --- a/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go +++ b/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go @@ -9,6 +9,7 @@ import ( "syscall" "time" + commonmetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/consumer" ) @@ -23,6 +24,27 @@ func main() { log.Fatalf("load config: %v", err) } + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + // Install the Prometheus MeterProvider before opening the sink so the + // backend cost metrics it registers (e.g. bigtable_rows_mutated_total, + // bigtable_bytes_written_total, bigtable_mutate_latency_seconds) bind to a + // real exporter and can be scraped at MetricsAddr/metrics. + if cfg.MetricsAddr != "" { + reg, shutdown, err := commonmetrics.SetupOtelPrometheus() + if err != nil { + log.Fatalf("setup metrics: %v", err) + } + defer func() { + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + _ = shutdown(shutdownCtx) + }() + commonmetrics.StartMetricsServer(ctx, reg, cfg.MetricsAddr) + log.Printf("serving consumer metrics at %s/metrics", cfg.MetricsAddr) + } + sink, err := consumer.NewSinkFromConfig(*cfg) if err != nil { log.Fatalf("open %s sink: %v", cfg.BackendName(), err) @@ -35,9 +57,6 @@ func main() { } defer func() { _ = reader.Close() }() - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer cancel() - c := consumer.New(reader, sink, consumer.Options{ Logf: func(format string, args ...interface{}) { log.Printf(format, args...) }, Workers: cfg.Workers, diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go index 038d8f7486..ad33801d1f 100644 --- a/sei-db/state_db/ss/offload/consumer/config.go +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -24,6 +24,11 @@ type Config struct { ShardBufferSize int MaxBatchRecords int BatchMaxWaitMS int + // MetricsAddr, when set (e.g. ":9092"), serves Prometheus metrics at + // /metrics so the backend cost counters (bigtable_rows_mutated_total, + // bigtable_bytes_written_total, bigtable_mutate_latency_seconds, ...) can be + // scraped. Empty disables the endpoint. + MetricsAddr string } func (c *Config) Validate() error { diff --git a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json index 0259f3e4df..ff58c10240 100644 --- a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json +++ b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json @@ -17,5 +17,6 @@ "Workers": 16, "ShardBufferSize": 128, "MaxBatchRecords": 128, - "BatchMaxWaitMS": 25 + "BatchMaxWaitMS": 25, + "MetricsAddr": ":9092" } diff --git a/sei-db/state_db/ss/offload/consumer/config_test.go b/sei-db/state_db/ss/offload/consumer/config_test.go index 0ac2e8ac86..1c2782cffd 100644 --- a/sei-db/state_db/ss/offload/consumer/config_test.go +++ b/sei-db/state_db/ss/offload/consumer/config_test.go @@ -1,6 +1,8 @@ package consumer import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -54,3 +56,17 @@ func TestConfigApplyBackendDefaultsScylla(t *testing.T) { require.Zero(t, cfg.MaxBatchRecords) require.Zero(t, cfg.BatchMaxWaitMS) } + +func TestLoadConfigMetricsAddr(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + require.NoError(t, os.WriteFile(path, []byte(`{ + "Backend": "bigtable", + "Kafka": {"Brokers": ["localhost:9092"], "Topic": "historical-offload", "GroupID": "g"}, + "Bigtable": {"ProjectID": "p", "InstanceID": "i", "Table": "t"}, + "MetricsAddr": ":9092" + }`), 0o600)) + + cfg, err := LoadConfig(path) + require.NoError(t, err) + require.Equal(t, ":9092", cfg.MetricsAddr) +} From 9a4a36675d123a3914df943e3e8f97b8ae046f82 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 7 Jul 2026 14:11:17 -0400 Subject: [PATCH 27/41] Add consumer throughput and Kafka lag metrics Backend cost counters answer "what does Bigtable cost" but not "can the consumer keep up." Add three metrics on meter seidb_offload_consumer, exported through the same /metrics endpoint: - consumer_records_processed_total: records written + committed (throughput) - consumer_sink_write_latency_seconds: batch write latency incl. retries - consumer_kafka_lag: max (high watermark - offset - 1) over the batch Recorded once per batch after the Kafka commit succeeds, so throughput reflects durably-persisted records. Lag is derived from each message's HighWaterMark (no extra Kafka calls); a batch with no watermark leaves the gauge untouched. Nil-safe. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../state_db/ss/offload/consumer/consumer.go | 22 +++++- .../state_db/ss/offload/consumer/metrics.go | 68 +++++++++++++++++++ .../ss/offload/consumer/metrics_test.go | 36 ++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 sei-db/state_db/ss/offload/consumer/metrics.go create mode 100644 sei-db/state_db/ss/offload/consumer/metrics_test.go diff --git a/sei-db/state_db/ss/offload/consumer/consumer.go b/sei-db/state_db/ss/offload/consumer/consumer.go index a04c42e75e..2d83d14897 100644 --- a/sei-db/state_db/ss/offload/consumer/consumer.go +++ b/sei-db/state_db/ss/offload/consumer/consumer.go @@ -29,6 +29,7 @@ type Consumer struct { maxAttempts int baseBackoff time.Duration maxBackoff time.Duration + metrics *consumerMetrics } const ( @@ -98,6 +99,7 @@ func New(reader MessageSource, sink Sink, opts Options) *Consumer { maxAttempts: maxAttempts, baseBackoff: base, maxBackoff: maxBackoff, + metrics: newConsumerMetrics(), } } @@ -232,14 +234,32 @@ func (c *Consumer) processBatch(ctx context.Context, msgs []kafka.Message) error return fmt.Errorf("sink write batch first_version=%d last_version=%d: %w", firstVersion, lastVersion, err) } + writeLatency := time.Since(start) c.logf("wrote records=%d first_version=%d last_version=%d in %s", - len(records), firstVersion, lastVersion, time.Since(start)) + len(records), firstVersion, lastVersion, writeLatency) if err := c.reader.CommitMessages(ctx, msgs...); err != nil { return fmt.Errorf("commit kafka offsets: %w", err) } + c.metrics.recordBatch(ctx, int64(len(records)), batchLag(msgs), writeLatency) return nil } +// batchLag reports how far behind the partition head the batch left the +// consumer: the max over messages of (high watermark - offset - 1). Returns -1 +// when no message carries a watermark so the gauge is left untouched. +func batchLag(msgs []kafka.Message) int64 { + lag := int64(-1) + for _, msg := range msgs { + if msg.HighWaterMark <= 0 { + continue + } + if l := msg.HighWaterMark - msg.Offset - 1; l > lag { + lag = l + } + } + return lag +} + func (c *Consumer) writeBatchWithRetry(ctx context.Context, records []Record) error { backoff := c.baseBackoff var lastErr error diff --git a/sei-db/state_db/ss/offload/consumer/metrics.go b/sei-db/state_db/ss/offload/consumer/metrics.go new file mode 100644 index 0000000000..f7806f76f0 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/metrics.go @@ -0,0 +1,68 @@ +package consumer + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// consumerMeterName is the OpenTelemetry meter for the Kafka -> backend offload +// consumer. It exports through the same Prometheus pipeline as the sinks, so +// consumer throughput and lag sit alongside the backend cost counters. +const consumerMeterName = "seidb_offload_consumer" + +// consumerMetrics answers the two questions a cost benchmark needs beyond the +// backend's own counters: how fast is the consumer draining Kafka, and is it +// keeping up. Throughput plus lag tell you whether the backend (not the +// consumer) is the bottleneck. One consumer runs per process, so no attributes +// are needed — Prometheus job/instance labels separate backends. +type consumerMetrics struct { + recordsProcessed metric.Int64Counter + sinkWriteLatency metric.Float64Histogram + kafkaLag metric.Int64Gauge +} + +func newConsumerMetrics() *consumerMetrics { + meter := otel.Meter(consumerMeterName) + recordsProcessed, _ := meter.Int64Counter( + "consumer_records_processed_total", + metric.WithDescription("Changelog records written to the backend and committed to Kafka"), + metric.WithUnit("{record}"), + ) + sinkWriteLatency, _ := meter.Float64Histogram( + "consumer_sink_write_latency_seconds", + metric.WithDescription("Latency of a batch sink write including retries; the _count series is the batch count"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries( + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, + ), + ) + kafkaLag, _ := meter.Int64Gauge( + "consumer_kafka_lag", + metric.WithDescription("Messages between the last processed offset and the partition high watermark (max across the batch)"), + metric.WithUnit("{message}"), + ) + return &consumerMetrics{ + recordsProcessed: recordsProcessed, + sinkWriteLatency: sinkWriteLatency, + kafkaLag: kafkaLag, + } +} + +// recordBatch reports one successfully written-and-committed batch: the number +// of records drained, how long the sink write took, and how far behind Kafka +// the batch left the consumer. +func (m *consumerMetrics) recordBatch(ctx context.Context, records int64, lag int64, writeLatency time.Duration) { + if m == nil { + return + } + if records > 0 { + m.recordsProcessed.Add(ctx, records) + } + m.sinkWriteLatency.Record(ctx, writeLatency.Seconds()) + if lag >= 0 { + m.kafkaLag.Record(ctx, lag) + } +} diff --git a/sei-db/state_db/ss/offload/consumer/metrics_test.go b/sei-db/state_db/ss/offload/consumer/metrics_test.go new file mode 100644 index 0000000000..7b77d3b4dc --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/metrics_test.go @@ -0,0 +1,36 @@ +package consumer + +import ( + "context" + "testing" + "time" + + "github.com/segmentio/kafka-go" + "github.com/stretchr/testify/require" +) + +func TestBatchLag(t *testing.T) { + // Max over the batch of (HighWaterMark - Offset - 1); the head message + // (offset == HWM-1) has lag 0. + msgs := []kafka.Message{ + {Offset: 10, HighWaterMark: 100}, // 89 + {Offset: 98, HighWaterMark: 100}, // 1 + {Offset: 99, HighWaterMark: 100}, // 0 (head) + } + require.Equal(t, int64(89), batchLag(msgs)) + + // No watermark information -> sentinel -1 so the gauge is left untouched. + require.Equal(t, int64(-1), batchLag([]kafka.Message{{Offset: 5}})) + require.Equal(t, int64(-1), batchLag(nil)) +} + +// recordBatch must be a safe no-op on a nil receiver and must not panic when +// the global MeterProvider is the default no-op provider. +func TestConsumerMetricsRecordNoPanic(t *testing.T) { + var nilM *consumerMetrics + nilM.recordBatch(context.Background(), 5, 3, time.Millisecond) + + m := newConsumerMetrics() + m.recordBatch(context.Background(), 128, 0, 12*time.Millisecond) + m.recordBatch(context.Background(), 0, -1, time.Millisecond) // zero records, unknown lag +} From d40b29fc6aee375dbc4f6acc53c712389a09efd6 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 14 Jul 2026 14:48:19 -0400 Subject: [PATCH 28/41] Harden historical fallback reads Route the pruned-read fallback decision through the store that will serve the read (EVM vs cosmos earliest can diverge when an EVM prune fails), bound backend point reads with a 10s deadline since types.StateStore has no context to inject one, expire cached backend misses after a TTL so consumer catch-up becomes visible, and join primary+reader Close errors. Co-Authored-By: Claude Fable 5 --- sei-db/state_db/ss/composite/store.go | 11 +++ .../state_db/ss/offload/historical/store.go | 84 +++++++++++++++---- .../ss/offload/historical/store_test.go | 67 +++++++++++++++ 3 files changed, 145 insertions(+), 17 deletions(-) diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index 2d4102a486..7714f3485d 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -219,6 +219,17 @@ func (s *CompositeStateStore) GetEarliestVersion() int64 { return s.cosmosStore.GetEarliestVersion() } +// GetEarliestVersionForKey reports the earliest version still held locally by +// the store that serves storeKey. Cosmos and EVM prune independently (an EVM +// prune failure is only logged), so pruned-read fallback must consult the +// horizon of the store that will actually serve the read. +func (s *CompositeStateStore) GetEarliestVersionForKey(storeKey string) int64 { + if s.evmRouted(storeKey) { + return s.evmStore.GetEarliestVersion() + } + return s.cosmosStore.GetEarliestVersion() +} + func (s *CompositeStateStore) Close() error { s.closeOnce.Do(func() { if s.pruningManager != nil { diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index 4a3bfddd47..5913cf1a1d 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "time" lru "github.com/hashicorp/golang-lru/v2" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" @@ -14,6 +15,17 @@ import ( const ( defaultHistoricalReadCacheEntries = 64 * 1024 maxHistoricalReadCacheValueBytes = 64 * 1024 + + // historicalReadTimeout bounds one backend point read. types.StateStore has + // no context parameter, so the deadline must be injected here; without it a + // silently dropped connection can park an RPC goroutine in a backend read + // until the OS TCP timeout. + historicalReadTimeout = 10 * time.Second + + // historicalMissCacheTTL bounds how long a backend miss is trusted. Hits are + // immutable (a version's value never changes once written) but a miss can + // flip to a hit once the offload consumer catches up. + historicalMissCacheTTL = time.Minute ) type historicalReadCacheKey struct { @@ -26,14 +38,25 @@ type historicalReadCacheValue struct { value []byte found bool valueKnown bool + // missExpiresAt is set only on miss entries; found entries never expire. + missExpiresAt time.Time +} + +// PerKeyEarliestVersioner is implemented by primaries whose stores prune +// independently per store key (e.g. the composite store's cosmos and EVM +// backends), so the fallback horizon can be checked against the store that +// will actually serve the read. +type PerKeyEarliestVersioner interface { + GetEarliestVersionForKey(storeKey string) int64 } // FallbackStateStore routes pruned point reads to the historical reader. // Iteration and writes stay on the primary state store. type FallbackStateStore struct { - primary types.StateStore - reader Reader - cache *lru.Cache[historicalReadCacheKey, historicalReadCacheValue] + primary types.StateStore + perKeyEarliest PerKeyEarliestVersioner + reader Reader + cache *lru.Cache[historicalReadCacheKey, historicalReadCacheValue] } var _ types.StateStore = (*FallbackStateStore)(nil) @@ -44,16 +67,28 @@ func NewFallbackStateStore(primary types.StateStore, reader Reader) *FallbackSta if err != nil { panic(err) } - return &FallbackStateStore{primary: primary, reader: reader, cache: cache} + perKeyEarliest, _ := primary.(PerKeyEarliestVersioner) + return &FallbackStateStore{primary: primary, perKeyEarliest: perKeyEarliest, reader: reader, cache: cache} } -func (s *FallbackStateStore) shouldFallback(version int64) bool { - earliest := s.primary.GetEarliestVersion() +func (s *FallbackStateStore) shouldFallback(storeKey string, version int64) bool { + earliest := s.earliestVersionFor(storeKey) return earliest > 0 && version < earliest } +func (s *FallbackStateStore) earliestVersionFor(storeKey string) int64 { + if s.perKeyEarliest != nil { + return s.perKeyEarliest.GetEarliestVersionForKey(storeKey) + } + return s.primary.GetEarliestVersion() +} + +func (s *FallbackStateStore) readContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), historicalReadTimeout) +} + func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([]byte, error) { - if !s.shouldFallback(version) { + if !s.shouldFallback(storeKey, version) { return s.primary.Get(storeKey, version, key) } cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} @@ -63,7 +98,9 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] } return value, nil } - v, err := s.reader.Get(context.Background(), storeKey, key, version) + ctx, cancel := s.readContext() + defer cancel() + v, err := s.reader.Get(ctx, storeKey, key, version) if err != nil { if errors.Is(err, ErrNotFound) { s.cacheMiss(cacheKey) @@ -84,6 +121,9 @@ func (s *FallbackStateStore) getCachedValue(key historicalReadCacheKey) ([]byte, return nil, false, false } if !value.found { + if s.missExpired(key, value) { + return nil, false, false + } return nil, false, true } if !value.valueKnown { @@ -100,9 +140,22 @@ func (s *FallbackStateStore) getCachedHas(key historicalReadCacheKey) (bool, boo if !ok { return false, false } + if !value.found && s.missExpired(key, value) { + return false, false + } return value.found, true } +// missExpired evicts and reports miss entries whose TTL has lapsed so a +// backend that has since ingested the version gets re-queried. +func (s *FallbackStateStore) missExpired(key historicalReadCacheKey, value historicalReadCacheValue) bool { + if value.missExpiresAt.IsZero() || time.Now().Before(value.missExpiresAt) { + return false + } + s.cache.Remove(key) + return true +} + func (s *FallbackStateStore) cacheValue(key historicalReadCacheKey, value []byte) { if s.cache == nil || value == nil || len(value) > maxHistoricalReadCacheValueBytes { return @@ -114,7 +167,7 @@ func (s *FallbackStateStore) cacheMiss(key historicalReadCacheKey) { if s.cache == nil { return } - s.cache.Add(key, historicalReadCacheValue{valueKnown: true}) + s.cache.Add(key, historicalReadCacheValue{valueKnown: true, missExpiresAt: time.Now().Add(historicalMissCacheTTL)}) } func (s *FallbackStateStore) cacheHas(key historicalReadCacheKey) { @@ -125,14 +178,16 @@ func (s *FallbackStateStore) cacheHas(key historicalReadCacheKey) { } func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bool, error) { - if !s.shouldFallback(version) { + if !s.shouldFallback(storeKey, version) { return s.primary.Has(storeKey, version, key) } cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} if found, ok := s.getCachedHas(cacheKey); ok { return found, nil } - found, err := s.reader.Has(context.Background(), storeKey, key, version) + ctx, cancel := s.readContext() + defer cancel() + found, err := s.reader.Has(ctx, storeKey, key, version) if err != nil { return false, err } @@ -183,10 +238,5 @@ func (s *FallbackStateStore) Import(version int64, ch <-chan types.SnapshotNode) } func (s *FallbackStateStore) Close() error { - primaryErr := s.primary.Close() - readerErr := s.reader.Close() - if primaryErr != nil { - return primaryErr - } - return readerErr + return errors.Join(s.primary.Close(), s.reader.Close()) } diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go index 430eb92b18..c7ab77ccc4 100644 --- a/sei-db/state_db/ss/offload/historical/store_test.go +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -180,6 +181,72 @@ func TestFallbackStateStoreDoesNotUseHasOnlyCacheForGet(t *testing.T) { require.Equal(t, 1, reader.gets) } +type fakePerKeyStateStore struct { + fakeStateStore + perKey map[string]int64 +} + +func (f *fakePerKeyStateStore) GetEarliestVersionForKey(storeKey string) int64 { + if v, ok := f.perKey[storeKey]; ok { + return v + } + return f.earliest +} + +func TestFallbackStateStoreUsesPerKeyEarliestVersion(t *testing.T) { + primary := &fakePerKeyStateStore{ + fakeStateStore: fakeStateStore{earliest: 10}, + perKey: map[string]int64{"evm": 5}, + } + reader := &fakeReader{} + store := NewFallbackStateStore(primary, reader) + + // The EVM store still holds version 7 locally even though the cosmos store + // pruned to 10; the read must stay on the primary. + value, err := store.Get("evm", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("primary"), value) + require.Equal(t, 1, primary.gets) + require.Equal(t, 0, reader.gets) + + // Other store keys fall back below the cosmos horizon as before. + value, err = store.Get("bank", 7, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 1, reader.gets) + + ok, err := store.Has("evm", 7, []byte("k")) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 1, primary.has) + require.Equal(t, 0, reader.has) +} + +func TestFallbackStateStoreExpiresCachedMisses(t *testing.T) { + primary := &fakeStateStore{earliest: 10} + reader := &fakeReader{getErr: ErrNotFound} + store := NewFallbackStateStore(primary, reader) + + value, err := store.Get("bank", 7, []byte("missing")) + require.NoError(t, err) + require.Nil(t, value) + require.Equal(t, 1, reader.gets) + + // Backdate the cached miss to simulate the TTL lapsing after the offload + // consumer catches up. + cacheKey := historicalReadCacheKey{storeKey: "bank", version: 7, key: "missing"} + entry, ok := store.cache.Get(cacheKey) + require.True(t, ok) + entry.missExpiresAt = time.Now().Add(-time.Second) + store.cache.Add(cacheKey, entry) + + reader.getErr = nil + value, err = store.Get("bank", 7, []byte("missing")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 2, reader.gets) +} + func TestFallbackStateStoreDoesNotCacheHistoricalErrors(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{getErr: errors.New("boom")} From af6f3f977df921faed65791bb70a6b73185b9be3 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 14 Jul 2026 14:48:27 -0400 Subject: [PATCH 29/41] Tighten offload consumer validation Reject configs that populate both Scylla and Bigtable stanzas without an explicit Backend (matching the node-side reader), mirror the producer's SASL/TLS/region checks in the Kafka reader config so misconfiguration fails at load instead of on first fetch, and move the example metrics port off the Kafka broker port. Co-Authored-By: Claude Fable 5 --- sei-db/state_db/ss/offload/consumer/config.go | 8 ++++++- .../consumer/config/example-bigtable.json | 2 +- .../ss/offload/consumer/config_test.go | 16 ++++++++++++++ sei-db/state_db/ss/offload/consumer/kafka.go | 14 +++++++++++++ .../ss/offload/consumer/kafka_test.go | 21 +++++++++++++++++++ 5 files changed, 59 insertions(+), 2 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go index ad33801d1f..bb4d3903b8 100644 --- a/sei-db/state_db/ss/offload/consumer/config.go +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -24,7 +24,7 @@ type Config struct { ShardBufferSize int MaxBatchRecords int BatchMaxWaitMS int - // MetricsAddr, when set (e.g. ":9092"), serves Prometheus metrics at + // MetricsAddr, when set (e.g. ":2112"), serves Prometheus metrics at // /metrics so the backend cost counters (bigtable_rows_mutated_total, // bigtable_bytes_written_total, bigtable_mutate_latency_seconds, ...) can be // scraped. Empty disables the endpoint. @@ -35,6 +35,12 @@ func (c *Config) Validate() error { if err := c.Kafka.Validate(); err != nil { return fmt.Errorf("kafka: %w", err) } + // Match the node-side reader, which refuses to guess between two configured + // backends; a consumer silently defaulting to Scylla here could ingest into + // a different store than the node reads from. + if strings.TrimSpace(c.Backend) == "" && c.Scylla.Configured() && c.Bigtable.Configured() { + return fmt.Errorf("both scylla and bigtable are configured; set Backend to pick one") + } switch c.BackendName() { case backendScylla: if err := c.Scylla.Validate(); err != nil { diff --git a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json index ff58c10240..588111679e 100644 --- a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json +++ b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json @@ -18,5 +18,5 @@ "ShardBufferSize": 128, "MaxBatchRecords": 128, "BatchMaxWaitMS": 25, - "MetricsAddr": ":9092" + "MetricsAddr": ":2112" } diff --git a/sei-db/state_db/ss/offload/consumer/config_test.go b/sei-db/state_db/ss/offload/consumer/config_test.go index 1c2782cffd..0f3c7507fe 100644 --- a/sei-db/state_db/ss/offload/consumer/config_test.go +++ b/sei-db/state_db/ss/offload/consumer/config_test.go @@ -34,6 +34,22 @@ func TestConfigValidateBigtable(t *testing.T) { require.ErrorContains(t, cfg.Validate(), backendBigtable) } +func TestConfigValidateRejectsAmbiguousDualBackends(t *testing.T) { + cfg := Config{ + Kafka: KafkaReaderConfig{ + Brokers: []string{"localhost:9092"}, + Topic: "historical-offload", + GroupID: "g", + }, + Scylla: ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks"}, + Bigtable: BigtableConfig{ProjectID: "p", InstanceID: "i", Table: "t"}, + } + require.ErrorContains(t, cfg.Validate(), "both scylla and bigtable") + + cfg.Backend = backendBigtable + require.NoError(t, cfg.Validate()) +} + func TestConfigApplyBackendDefaultsBigtable(t *testing.T) { cfg := Config{Backend: backendBigtable} cfg.applyBackendDefaults() diff --git a/sei-db/state_db/ss/offload/consumer/kafka.go b/sei-db/state_db/ss/offload/consumer/kafka.go index cff52d4d78..3eca641d1b 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka.go +++ b/sei-db/state_db/ss/offload/consumer/kafka.go @@ -63,6 +63,20 @@ func (c *KafkaReaderConfig) Validate() error { default: return fmt.Errorf("unsupported kafka start offset %q", c.StartOffset) } + // Mirror the producer-side SASL checks so a misconfigured consumer fails at + // load time instead of with an obscure handshake error on first fetch. + switch strings.ToLower(c.SASLMechanism) { + case "", "none": + case "aws-msk-iam": + if !c.TLSEnabled { + return fmt.Errorf("kafka tls must be enabled for aws-msk-iam") + } + if c.Region == "" { + return fmt.Errorf("kafka region is required for aws-msk-iam") + } + default: + return fmt.Errorf("unsupported kafka sasl mechanism %q", c.SASLMechanism) + } return nil } diff --git a/sei-db/state_db/ss/offload/consumer/kafka_test.go b/sei-db/state_db/ss/offload/consumer/kafka_test.go index e714b7dd60..e7abda40f1 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka_test.go +++ b/sei-db/state_db/ss/offload/consumer/kafka_test.go @@ -36,6 +36,27 @@ func TestKafkaReaderConfigValidate(t *testing.T) { require.ErrorContains(t, cfg.Validate(), "start offset") } +func TestKafkaReaderConfigValidateSASL(t *testing.T) { + base := KafkaReaderConfig{Brokers: []string{"x"}, Topic: "t", GroupID: "g"} + + cfg := base + cfg.SASLMechanism = "aws-msk-iam" + require.ErrorContains(t, cfg.Validate(), "tls") + + cfg.TLSEnabled = true + require.ErrorContains(t, cfg.Validate(), "region") + + cfg.Region = "us-east-1" + require.NoError(t, cfg.Validate()) + + cfg = base + cfg.SASLMechanism = "plain" + require.ErrorContains(t, cfg.Validate(), "sasl mechanism") + + cfg.SASLMechanism = "none" + require.NoError(t, cfg.Validate()) +} + func TestDecodeEntry(t *testing.T) { entry := &proto.ChangelogEntry{Version: 7} payload, err := entry.Marshal() From c3eac8dee89d955cb42a910a11931bdc4e73ed74 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 14 Jul 2026 14:48:38 -0400 Subject: [PATCH 30/41] Optimize backend readers and dedupe Scylla helpers Bigtable: cancel the ReadRows stream on early callback exit, compute the point-lookup row prefix once per Get/Has, preallocate the row-builder value buffer from the advertised split-cell size, drop the redundant value copy (the builder already allocates per cell), and validate config before dialing. Scylla: parallelize the version-bucket LastVersion scan and share it with the consumer sink, remove the unused BatchGet/Lookup surface, and bound pipelined batch writes to recordWorkers*mutationWorkers in-flight CQL queries with a semaphore that keeps markers overlapping row writes. Co-Authored-By: Claude Fable 5 --- .../state_db/ss/offload/consumer/bigtable.go | 4 + sei-db/state_db/ss/offload/consumer/scylla.go | 42 ++++--- .../ss/offload/consumer/scylla_test.go | 1 - .../ss/offload/historical/bigtable.go | 45 ++++++-- .../ss/offload/historical/bigtable_test.go | 36 +++++- .../state_db/ss/offload/historical/reader.go | 6 - .../state_db/ss/offload/historical/scylla.go | 109 +++++++----------- .../ss/offload/historical/scylla_test.go | 73 ------------ 8 files changed, 134 insertions(+), 182 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index 32f2750c6d..76f572ff39 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -129,6 +129,10 @@ func (s *bigtableSink) appendRecordRowMutations(rows []historical.BigtableRowMut return rows } +// mutationRow writes value+deleted cells for live pairs but only a deleted +// cell for tombstones, saving a cell per delete. Readers must therefore check +// the deleted column before trusting any value cell — a replayed live write +// followed by a tombstone leaves both cells on the row. func (s *bigtableSink) mutationRow(version int64, storeName string, pair *proto.KVPair) historical.BigtableRowMutation { ts := historical.BigtableTimestamp(version) deleted := pair.Delete || pair.Value == nil diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index 149aba2693..6f0568d8e8 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -12,7 +12,14 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" ) -const defaultScyllaMutationWorkers = 16 +const ( + defaultScyllaMutationWorkers = 16 + + // defaultScyllaRecordWorkers bounds how many records write their rows + // concurrently, so peak in-flight CQL writes stay at + // recordWorkers*mutationWorkers instead of len(batch)*mutationWorkers. + defaultScyllaRecordWorkers = 4 +) type ScyllaConfig struct { Hosts []string @@ -102,21 +109,7 @@ func (s *scyllaSink) Close() error { } func (s *scyllaSink) LastVersion(ctx context.Context) (int64, error) { - var maxVersion int64 - for bucket := 0; bucket < historical.VersionBucketCount; bucket++ { - var version int64 - err := s.session.Query(selectLatestVersionCQL, bucket).WithContext(ctx).Scan(&version) - if err != nil { - if err == gocql.ErrNotFound { - continue - } - return 0, fmt.Errorf("read latest scylla/cassandra version bucket %d: %w", bucket, err) - } - if version > maxVersion { - maxVersion = version - } - } - return maxVersion, nil + return historical.ScyllaLastVersion(ctx, s.session) } func (s *scyllaSink) Write(ctx context.Context, rec Record) error { @@ -165,12 +158,23 @@ func (s *scyllaSink) writeRecordsPipelined(ctx context.Context, records []Record rowCtx, cancel := context.WithCancel(ctx) defer cancel() var g errgroup.Group + // A semaphore rather than errgroup.SetLimit keeps goroutine launching + // non-blocking, so the ordered version-marker loop below overlaps with row + // writes from the first record onward. + sem := make(chan struct{}, defaultScyllaRecordWorkers) rowDone := make([]chan error, len(records)) for i := range records { rowDone[i] = make(chan error, 1) i := i rec := records[i] g.Go(func() error { + select { + case sem <- struct{}{}: + case <-rowCtx.Done(): + rowDone[i] <- rowCtx.Err() + return rowCtx.Err() + } + defer func() { <-sem }() err := s.writeRecordRows(rowCtx, rec.Entry.Version, rec.Entry) if err != nil { err = fmt.Errorf("write scylla/cassandra rows version %d: %w", rec.Entry.Version, err) @@ -257,12 +261,6 @@ func sessionExec(session *gocql.Session) scyllaExecFunc { } } -const selectLatestVersionCQL = ` -SELECT version -FROM state_versions -WHERE bucket = ? -LIMIT 1` - const insertVersionCQL = ` INSERT INTO state_versions ( bucket, version, kafka_topic, kafka_partition, kafka_offset, ingested_at diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go index f10e672347..3de2660c81 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -74,7 +74,6 @@ func TestScyllaCQLShape(t *testing.T) { } { require.Contains(t, insertVersionCQL, frag) } - require.True(t, strings.Contains(selectLatestVersionCQL, "LIMIT 1")) } func TestScyllaSinkWritesRowsConcurrentlyBeforeVersionMarker(t *testing.T) { diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index c4c29d0294..86584c7d7d 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -116,12 +116,14 @@ func (c *BigtableConfig) Validate() error { } func NewBigtableReader(cfg BigtableConfig) (Reader, error) { - ctx := context.Background() - client, err := OpenBigtableClient(ctx, cfg) + cfg.ApplyDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + client, err := OpenBigtableClient(context.Background(), cfg) if err != nil { return nil, err } - cfg.ApplyDefaults() return &bigtableReader{ client: client, readRows: client.ReadRows, @@ -186,7 +188,11 @@ func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, req.Rows.RowRanges[0].EndKey = nil } req.Filter = bigtableReadFilter(family, qualifiers...) - stream, err := c.data.ReadRows(ctx, req) + // Cancel on return so an early callback exit tears down the server stream + // instead of leaving it to the connection finalizer. + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := c.data.ReadRows(streamCtx, req) if err != nil { return err } @@ -301,10 +307,9 @@ func (r *bigtableReader) LastVersion(ctx context.Context) (int64, error) { } func (r *bigtableReader) Has(ctx context.Context, storeName string, key []byte, targetVersion int64) (bool, error) { - prefix := bigtableMutationRowPrefixBytes(storeName, key, r.shards) - start := bigtableMutationRowKeyBytes(storeName, key, targetVersion, r.shards) + start, end := bigtableMutationRowRange(storeName, key, targetVersion, r.shards) var row BigtableRow - err := r.readRows(ctx, start, bigtablePrefixEnd(prefix), 1, r.family, func(r BigtableRow) bool { + err := r.readRows(ctx, start, end, 1, r.family, func(r BigtableRow) bool { row = r return false }, BigtableDeletedColumn) @@ -322,10 +327,9 @@ func (r *bigtableReader) Has(ctx context.Context, storeName string, key []byte, } func (r *bigtableReader) Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { - prefix := bigtableMutationRowPrefixBytes(storeName, key, r.shards) - start := bigtableMutationRowKeyBytes(storeName, key, targetVersion, r.shards) + start, end := bigtableMutationRowRange(storeName, key, targetVersion, r.shards) var row BigtableRow - err := r.readRows(ctx, start, bigtablePrefixEnd(prefix), 1, r.family, func(r BigtableRow) bool { + err := r.readRows(ctx, start, end, 1, r.family, func(r BigtableRow) bool { row = r return false }, BigtableValueColumn, BigtableDeletedColumn) @@ -372,6 +376,8 @@ func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (in return maxVersion, nil } +// BigtableValueFromRow interprets a mutation row. The returned Value aliases +// the row's cell buffer, which the row builder allocates per cell. func BigtableValueFromRow(row BigtableRow, family string) (Value, error) { version, ok := BigtableVersionFromRowKey(row.Key) if !ok { @@ -393,7 +399,7 @@ func BigtableValueFromRow(row BigtableRow, family string) (Value, error) { if deleted || value == nil { return Value{}, ErrNotFound } - return Value{Bytes: append([]byte(nil), value...), Version: version}, nil + return Value{Bytes: value, Version: version}, nil } func bigtableDeletedFromRow(row BigtableRow, family string) (bool, error) { @@ -436,6 +442,16 @@ func bigtableMutationRowKeyBytes(storeName string, key []byte, version int64, sh return append(prefix, bigtableInvertedVersion(version)...) } +// bigtableMutationRowRange returns the start key of a point lookup at +// targetVersion and the exclusive end of the key's version range, hashing and +// encoding the shared row prefix only once. +func bigtableMutationRowRange(storeName string, key []byte, targetVersion int64, shards int) (start, end []byte) { + prefix := bigtableMutationRowPrefixBytes(storeName, key, shards) + end = bigtablePrefixEnd(prefix) + start = append(prefix, bigtableInvertedVersion(targetVersion)...) + return start, end +} + func BigtableVersionRowKey(version int64) string { prefix := bigtableVersionRowPrefix(VersionBucket(version)) return string(append(prefix, bigtableInvertedVersion(version)...)) @@ -496,6 +512,13 @@ func (b *bigtableRowBuilder) add(chunk *bigtablepb.ReadRowsResponse_CellChunk) ( b.value = b.value[:0] b.inCell = true } + // A non-final chunk of a split cell advertises the total value size; grow + // the buffer once instead of re-growing on every continuation chunk. + if size := int(chunk.ValueSize); size > cap(b.value) { + grown := make([]byte, len(b.value), size) + copy(grown, b.value) + b.value = grown + } b.value = append(b.value, chunk.Value...) if b.inCell && chunk.ValueSize == 0 { b.cells = append(b.cells, BigtableCell{ diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go index acaf815579..dfd336b7bd 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable_test.go +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -7,7 +7,9 @@ import ( "sync" "testing" + "cloud.google.com/go/bigtable/apiv2/bigtablepb" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" ) func TestBigtableConfigDefaultsAndValidate(t *testing.T) { @@ -61,8 +63,6 @@ func TestBigtableValueFromRow(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("value"), value.Bytes) require.Equal(t, int64(7), value.Version) - value.Bytes[0] = 'V' - require.Equal(t, []byte("value"), row.Cells[0].Value) row.Cells[1].Value = []byte{1} _, err = BigtableValueFromRow(row, DefaultBigtableFamily) @@ -125,6 +125,38 @@ func TestBigtableReaderHasOnlyReadsDeletedColumn(t *testing.T) { require.True(t, ok) } +// A cell split across chunks carries the total size in every chunk except the +// last, whose ValueSize is zero; only then may the cell be committed. +func TestBigtableRowBuilderAssemblesSplitCells(t *testing.T) { + var b bigtableRowBuilder + + row, committed, err := b.add(&bigtablepb.ReadRowsResponse_CellChunk{ + RowKey: []byte("rk"), + FamilyName: wrapperspb.String(DefaultBigtableFamily), + Qualifier: wrapperspb.Bytes([]byte(BigtableValueColumn)), + Value: []byte("hel"), + ValueSize: 8, + }) + require.NoError(t, err) + require.False(t, committed) + require.Empty(t, row.Key) + + _, committed, err = b.add(&bigtablepb.ReadRowsResponse_CellChunk{Value: []byte("lo "), ValueSize: 8}) + require.NoError(t, err) + require.False(t, committed) + + row, committed, err = b.add(&bigtablepb.ReadRowsResponse_CellChunk{ + Value: []byte("bt"), + RowStatus: &bigtablepb.ReadRowsResponse_CellChunk_CommitRow{CommitRow: true}, + }) + require.NoError(t, err) + require.True(t, committed) + require.Equal(t, "rk", row.Key) + require.Len(t, row.Cells, 1) + require.Equal(t, BigtableValueColumn, row.Cells[0].Qualifier) + require.Equal(t, []byte("hello bt"), row.Cells[0].Value) +} + func TestBigtableLastVersionScansBuckets(t *testing.T) { versions := map[int]int64{ 3: 42, diff --git a/sei-db/state_db/ss/offload/historical/reader.go b/sei-db/state_db/ss/offload/historical/reader.go index b4bea9414d..45fb25dd10 100644 --- a/sei-db/state_db/ss/offload/historical/reader.go +++ b/sei-db/state_db/ss/offload/historical/reader.go @@ -8,12 +8,6 @@ import ( var ErrNotFound = errors.New("historical state not found") -// Key is a string so Lookup is usable as a map key. -type Lookup struct { - StoreName string - Key string -} - // Value is the actual MVCC value that satisfied the lookup. // Version may be older than the requested target version. type Value struct { diff --git a/sei-db/state_db/ss/offload/historical/scylla.go b/sei-db/state_db/ss/offload/historical/scylla.go index 65a02d4c5d..9855b658db 100644 --- a/sei-db/state_db/ss/offload/historical/scylla.go +++ b/sei-db/state_db/ss/offload/historical/scylla.go @@ -84,10 +84,7 @@ func NewScyllaReader(cfg ScyllaConfig) (Reader, error) { if err != nil { return nil, err } - return &scyllaReader{ - session: session, - get: sessionGet(session), - }, nil + return &scyllaReader{session: session}, nil } func OpenScyllaSession(cfg ScyllaConfig) (*gocql.Session, error) { @@ -131,7 +128,6 @@ func scyllaHostSelectionPolicy(datacenter string) gocql.HostSelectionPolicy { type scyllaReader struct { session *gocql.Session - get scyllaGetFunc } var _ Reader = (*scyllaReader)(nil) @@ -144,16 +140,36 @@ func (r *scyllaReader) Close() error { } func (r *scyllaReader) LastVersion(ctx context.Context) (int64, error) { - var maxVersion int64 + return ScyllaLastVersion(ctx, r.session) +} + +// ScyllaLastVersion scans the version-marker buckets in parallel and returns +// the highest ingested version. Shared by the node-side reader and the offload +// consumer sink. +func ScyllaLastVersion(ctx context.Context, session *gocql.Session) (int64, error) { + versions := make([]int64, VersionBucketCount) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(defaultScyllaReadWorkers) for bucket := 0; bucket < VersionBucketCount; bucket++ { - var version int64 - err := r.session.Query(selectLatestVersionCQL, bucket).WithContext(ctx).Scan(&version) - if err != nil { - if err == gocql.ErrNotFound { - continue + bucket := bucket + g.Go(func() error { + var version int64 + err := session.Query(selectLatestVersionCQL, bucket).WithContext(gctx).Scan(&version) + if err != nil { + if errors.Is(err, gocql.ErrNotFound) { + return nil + } + return fmt.Errorf("read latest scylla/cassandra version bucket %d: %w", bucket, err) } - return 0, fmt.Errorf("read latest scylla/cassandra version bucket %d: %w", bucket, err) - } + versions[bucket] = version + return nil + }) + } + if err := g.Wait(); err != nil { + return 0, err + } + var maxVersion int64 + for _, version := range versions { if version > maxVersion { maxVersion = version } @@ -165,7 +181,7 @@ func (r *scyllaReader) Has(ctx context.Context, storeName string, key []byte, ta var deleted bool err := r.session.Query(hasLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&deleted) if err != nil { - if err == gocql.ErrNotFound { + if errors.Is(err, gocql.ErrNotFound) { return false, nil } return false, fmt.Errorf("scylla/cassandra has lookup: %w", err) @@ -174,63 +190,22 @@ func (r *scyllaReader) Has(ctx context.Context, storeName string, key []byte, ta } func (r *scyllaReader) Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { - return r.get(ctx, storeName, key, targetVersion) -} - -func sessionGet(session *gocql.Session) scyllaGetFunc { - return func(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { - var ( - version int64 - bz []byte - deleted bool - ) - err := session.Query(getLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&version, &bz, &deleted) - if err != nil { - if err == gocql.ErrNotFound { - return Value{}, ErrNotFound - } - return Value{}, fmt.Errorf("scylla/cassandra get lookup: %w", err) - } - if deleted { + var ( + version int64 + bz []byte + deleted bool + ) + err := r.session.Query(getLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&version, &bz, &deleted) + if err != nil { + if errors.Is(err, gocql.ErrNotFound) { return Value{}, ErrNotFound } - return Value{Bytes: bz, Version: version}, nil + return Value{}, fmt.Errorf("scylla/cassandra get lookup: %w", err) } -} - -type scyllaGetFunc func(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) - -func (r *scyllaReader) BatchGet(ctx context.Context, targetVersion int64, lookups []Lookup) (map[Lookup]Value, error) { - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(defaultScyllaReadWorkers) - values := make([]Value, len(lookups)) - found := make([]bool, len(lookups)) - for i, lookup := range lookups { - i := i - lookup := lookup - g.Go(func() error { - value, err := r.Get(gctx, lookup.StoreName, []byte(lookup.Key), targetVersion) - if err != nil { - if errors.Is(err, ErrNotFound) { - return nil - } - return err - } - values[i] = value - found[i] = true - return nil - }) - } - if err := g.Wait(); err != nil { - return nil, err - } - out := make(map[Lookup]Value, len(lookups)) - for i, lookup := range lookups { - if found[i] { - out[lookup] = values[i] - } + if deleted { + return Value{}, ErrNotFound } - return out, nil + return Value{Bytes: bz, Version: version}, nil } func VersionBucket(version int64) int { diff --git a/sei-db/state_db/ss/offload/historical/scylla_test.go b/sei-db/state_db/ss/offload/historical/scylla_test.go index 2699cdf3ea..567dc5f960 100644 --- a/sei-db/state_db/ss/offload/historical/scylla_test.go +++ b/sei-db/state_db/ss/offload/historical/scylla_test.go @@ -1,10 +1,8 @@ package historical import ( - "context" "reflect" "strings" - "sync/atomic" "testing" "time" @@ -112,74 +110,3 @@ func TestLatestVersionCQLShape(t *testing.T) { require.Contains(t, selectLatestVersionCQL, "bucket = ?") require.True(t, strings.Contains(selectLatestVersionCQL, "LIMIT 1")) } - -func TestScyllaReaderBatchGetParallelizesLookups(t *testing.T) { - started := make(chan string, 4) - release := make(chan struct{}) - var active atomic.Int32 - var sawConcurrent atomic.Bool - - reader := &scyllaReader{ - get: func(ctx context.Context, _ string, key []byte, targetVersion int64) (Value, error) { - if active.Add(1) > 1 { - sawConcurrent.Store(true) - } - defer active.Add(-1) - keyString := string(key) - started <- keyString - select { - case <-release: - case <-ctx.Done(): - return Value{}, ctx.Err() - } - if keyString == "missing" { - return Value{}, ErrNotFound - } - return Value{Bytes: []byte("value-" + keyString), Version: targetVersion - 1}, nil - }, - } - - errCh := make(chan error, 1) - var got map[Lookup]Value - lookups := []Lookup{ - {StoreName: "bank", Key: "k1"}, - {StoreName: "bank", Key: "missing"}, - {StoreName: "evm", Key: "k2"}, - } - go func() { - var err error - got, err = reader.BatchGet(context.Background(), 10, lookups) - errCh <- err - }() - - releaseClosed := false - closeRelease := func() { - if !releaseClosed { - close(release) - releaseClosed = true - } - } - defer closeRelease() - - for i := 0; i < 2; i++ { - select { - case <-started: - case <-time.After(time.Second): - closeRelease() - t.Fatal("timed out waiting for concurrent lookups") - } - } - require.True(t, sawConcurrent.Load()) - - closeRelease() - select { - case err := <-errCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("timed out waiting for batch get") - } - require.Len(t, got, 2) - require.Equal(t, []byte("value-k1"), got[lookups[0]].Bytes) - require.Equal(t, []byte("value-k2"), got[lookups[2]].Bytes) - require.NotContains(t, got, lookups[1]) -} From 024c3a8698be09168b476a1fc6e9deaad7248121 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 16 Jul 2026 13:34:31 -0400 Subject: [PATCH 31/41] Collapse redelivered versions in consumer batches Kafka at-least-once delivery can put the same changelog version in one batch twice; compactRecords now keeps only the latest record per version so sinks write each version's rows once (no duplicate Bigtable mutations to pay for) and version markers carry the newest offset. Co-Authored-By: Claude Fable 5 --- .../state_db/ss/offload/consumer/compact.go | 22 ++++++++++++------- .../ss/offload/consumer/scylla_test.go | 16 ++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/compact.go b/sei-db/state_db/ss/offload/consumer/compact.go index 5903eae73a..9f0495472e 100644 --- a/sei-db/state_db/ss/offload/consumer/compact.go +++ b/sei-db/state_db/ss/offload/consumer/compact.go @@ -12,19 +12,25 @@ type stateMutationKey struct { key string } +// compactRecords drops records without entries and collapses records that +// share a changelog version (Kafka at-least-once redelivery) onto the latest +// one, so sinks write each version's rows once and version markers carry the +// newest offset. Order is preserved by the first occurrence of each version. func compactRecords(records []Record) []Record { + indexByVersion := make(map[int64]int, len(records)) + out := make([]Record, 0, len(records)) for _, rec := range records { if rec.Entry == nil { - out := make([]Record, 0, len(records)) - for _, rec := range records { - if rec.Entry != nil { - out = append(out, rec) - } - } - return out + continue + } + if idx, ok := indexByVersion[rec.Entry.Version]; ok { + out[idx] = rec + continue } + indexByVersion[rec.Entry.Version] = len(out) + out = append(out, rec) } - return records + return out } func compactMutations(entry *proto.ChangelogEntry) []stateMutation { diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go index 3de2660c81..7f6584b51b 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -52,6 +52,22 @@ func TestCompactRecordsDropsNilEntries(t *testing.T) { require.Equal(t, int64(2), records[1].Entry.Version) } +func TestCompactRecordsCollapsesRedeliveredVersions(t *testing.T) { + records := compactRecords([]Record{ + {Offset: 10, Entry: &proto.ChangelogEntry{Version: 1}}, + {Offset: 11, Entry: &proto.ChangelogEntry{Version: 2}}, + {Offset: 12, Entry: &proto.ChangelogEntry{Version: 1}}, + {Offset: 13, Entry: &proto.ChangelogEntry{Version: 3}}, + }) + require.Len(t, records, 3) + // Version order is preserved; the redelivered version keeps its slot but + // carries the newest offset for the version marker. + require.Equal(t, int64(1), records[0].Entry.Version) + require.Equal(t, int64(12), records[0].Offset) + require.Equal(t, int64(2), records[1].Entry.Version) + require.Equal(t, int64(3), records[2].Entry.Version) +} + func TestScyllaCQLShape(t *testing.T) { for _, frag := range []string{ "INSERT INTO state_mutations", From ab50f39795b7ec1cf33ef2143571c52c1ea102a6 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Thu, 16 Jul 2026 13:34:31 -0400 Subject: [PATCH 32/41] Count fallback read outcomes Add a historical_fallback_reads_total counter (op x outcome: cache_hit, backend_hit, backend_miss, error) so the pruned-read path shows how much load the LRU absorbs and how many reads actually reach the backend, complementing the Bigtable cost metrics. Co-Authored-By: Claude Fable 5 --- .../state_db/ss/offload/historical/metrics.go | 39 +++++++++++++++++++ .../ss/offload/historical/metrics_test.go | 18 +++++++++ .../state_db/ss/offload/historical/store.go | 17 +++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/sei-db/state_db/ss/offload/historical/metrics.go b/sei-db/state_db/ss/offload/historical/metrics.go index a542121fe1..e6cc0a90a7 100644 --- a/sei-db/state_db/ss/offload/historical/metrics.go +++ b/sei-db/state_db/ss/offload/historical/metrics.go @@ -114,6 +114,45 @@ func (m *bigtableMetrics) recordWrite(ctx context.Context, table string, latency } } +// fallbackMeterName is the OpenTelemetry meter for the node-side pruned-read +// fallback path, exported through the same process-wide MeterProvider. +const fallbackMeterName = "seidb_historical_fallback" + +const ( + fallbackOutcomeCacheHit = "cache_hit" + fallbackOutcomeBackendHit = "backend_hit" + fallbackOutcomeBackendMiss = "backend_miss" + fallbackOutcomeError = "error" +) + +// fallbackMetrics counts pruned point reads by operation (get/has) and outcome +// so operators can see how much load the read cache absorbs and how many reads +// actually reach the historical backend. Attributes are a closed set, so +// cardinality stays flat. +type fallbackMetrics struct { + reads metric.Int64Counter +} + +func newFallbackMetrics() *fallbackMetrics { + meter := otel.Meter(fallbackMeterName) + reads, _ := meter.Int64Counter( + "historical_fallback_reads_total", + metric.WithDescription("Pruned point reads routed to the historical fallback, by operation and outcome"), + metric.WithUnit("{read}"), + ) + return &fallbackMetrics{reads: reads} +} + +func (m *fallbackMetrics) recordRead(op, outcome string) { + if m == nil { + return + } + m.reads.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("op", op), + attribute.String("outcome", outcome), + )) +} + // bigtableRowSize estimates the transferred size of a read row: the row key // plus each returned cell's qualifier and value. func bigtableRowSize(row BigtableRow) int64 { diff --git a/sei-db/state_db/ss/offload/historical/metrics_test.go b/sei-db/state_db/ss/offload/historical/metrics_test.go index d35cfbf608..9a4eaa8e7d 100644 --- a/sei-db/state_db/ss/offload/historical/metrics_test.go +++ b/sei-db/state_db/ss/offload/historical/metrics_test.go @@ -49,3 +49,21 @@ func TestBigtableMetricsRecordNoPanic(t *testing.T) { // Zero rows/bytes must still record latency without panicking. m.recordRead(context.Background(), "state", time.Millisecond, 0, 0) } + +func TestFallbackMetricsNilSafe(t *testing.T) { + var m *fallbackMetrics + m.recordRead("get", fallbackOutcomeCacheHit) +} + +func TestFallbackMetricsRecordNoPanic(t *testing.T) { + m := newFallbackMetrics() + for _, outcome := range []string{ + fallbackOutcomeCacheHit, + fallbackOutcomeBackendHit, + fallbackOutcomeBackendMiss, + fallbackOutcomeError, + } { + m.recordRead("get", outcome) + m.recordRead("has", outcome) + } +} diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index 5913cf1a1d..3759dcb860 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -57,6 +57,7 @@ type FallbackStateStore struct { perKeyEarliest PerKeyEarliestVersioner reader Reader cache *lru.Cache[historicalReadCacheKey, historicalReadCacheValue] + metrics *fallbackMetrics } var _ types.StateStore = (*FallbackStateStore)(nil) @@ -68,7 +69,13 @@ func NewFallbackStateStore(primary types.StateStore, reader Reader) *FallbackSta panic(err) } perKeyEarliest, _ := primary.(PerKeyEarliestVersioner) - return &FallbackStateStore{primary: primary, perKeyEarliest: perKeyEarliest, reader: reader, cache: cache} + return &FallbackStateStore{ + primary: primary, + perKeyEarliest: perKeyEarliest, + reader: reader, + cache: cache, + metrics: newFallbackMetrics(), + } } func (s *FallbackStateStore) shouldFallback(storeKey string, version int64) bool { @@ -93,6 +100,7 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] } cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} if value, found, ok := s.getCachedValue(cacheKey); ok { + s.metrics.recordRead("get", fallbackOutcomeCacheHit) if !found { return nil, nil } @@ -103,11 +111,14 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] v, err := s.reader.Get(ctx, storeKey, key, version) if err != nil { if errors.Is(err, ErrNotFound) { + s.metrics.recordRead("get", fallbackOutcomeBackendMiss) s.cacheMiss(cacheKey) return nil, nil } + s.metrics.recordRead("get", fallbackOutcomeError) return nil, err } + s.metrics.recordRead("get", fallbackOutcomeBackendHit) s.cacheValue(cacheKey, v.Bytes) return v.Bytes, nil } @@ -183,17 +194,21 @@ func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bo } cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} if found, ok := s.getCachedHas(cacheKey); ok { + s.metrics.recordRead("has", fallbackOutcomeCacheHit) return found, nil } ctx, cancel := s.readContext() defer cancel() found, err := s.reader.Has(ctx, storeKey, key, version) if err != nil { + s.metrics.recordRead("has", fallbackOutcomeError) return false, err } if found { + s.metrics.recordRead("has", fallbackOutcomeBackendHit) s.cacheHas(cacheKey) } else { + s.metrics.recordRead("has", fallbackOutcomeBackendMiss) s.cacheMiss(cacheKey) } return found, nil From 3a2d4d83a555535590a5deb46d5b5075311465dc Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 09:48:48 -0400 Subject: [PATCH 33/41] Gate fallback reads on declared and observed backend coverage Two gaps made the fallback unsafe to rely on: the EVM RPC watermark rejects any height below GetEarliestVersion before a read can reach the store, and a backend that is lagging or was never backfilled silently answers pruned reads with empty state. Add historical-offload-earliest-version: when set, the operator-declared coverage floor becomes the advertised earliest version so height gates admit heights the fallback can serve, while reads below the floor stay local. At read time, cache the backend's last ingested version (from the version markers, refreshed at most every 30s) and error when a requested version is ahead of it instead of returning empty state. Also bound the read cache at 32k entries x 8KiB and never fall back for negative versions. Co-Authored-By: Claude Fable 5 --- app/seidb.go | 2 + app/seidb_test.go | 2 + sei-db/config/ss_config.go | 6 ++ sei-db/config/toml.go | 6 ++ sei-db/config/toml_test.go | 1 + .../state_db/ss/offload/historical/metrics.go | 9 +- .../state_db/ss/offload/historical/store.go | 90 ++++++++++++++++++- .../ss/offload/historical/store_test.go | 85 +++++++++++++++--- sei-db/state_db/ss/store.go | 7 +- 9 files changed, 187 insertions(+), 21 deletions(-) diff --git a/app/seidb.go b/app/seidb.go index 809bfeac52..a18b228ad7 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -68,6 +68,7 @@ const ( FlagHistoricalOffloadBigtableFamily = "state-store.historical-offload-bigtable-family" FlagHistoricalOffloadBigtableAppProfile = "state-store.historical-offload-bigtable-app-profile" FlagHistoricalOffloadBigtableShards = "state-store.historical-offload-bigtable-shards" + FlagHistoricalOffloadEarliestVersion = "state-store.historical-offload-earliest-version" ) var GigaKeys = []string{"evm", "bank"} @@ -236,5 +237,6 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig { ssConfig.HistoricalOffloadBigtableFamily = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableFamily)) ssConfig.HistoricalOffloadBigtableAppProfile = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableAppProfile)) ssConfig.HistoricalOffloadBigtableShards = cast.ToInt(appOpts.Get(FlagHistoricalOffloadBigtableShards)) + ssConfig.HistoricalOffloadEarliestVersion = cast.ToInt64(appOpts.Get(FlagHistoricalOffloadEarliestVersion)) return ssConfig } diff --git a/app/seidb_test.go b/app/seidb_test.go index f17c4f5ea1..da91bb413e 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -263,6 +263,7 @@ func TestParseSSConfigs_HistoricalBigtableFlags(t *testing.T) { FlagHistoricalOffloadBigtableFamily: "state", FlagHistoricalOffloadBigtableAppProfile: "historical", FlagHistoricalOffloadBigtableShards: 512, + FlagHistoricalOffloadEarliestVersion: int64(1234), FlagSSAsyncWriterBuffer: 0, } @@ -274,6 +275,7 @@ func TestParseSSConfigs_HistoricalBigtableFlags(t *testing.T) { assert.Equal(t, "state", ssConfig.HistoricalOffloadBigtableFamily) assert.Equal(t, "historical", ssConfig.HistoricalOffloadBigtableAppProfile) assert.Equal(t, 512, ssConfig.HistoricalOffloadBigtableShards) + assert.Equal(t, int64(1234), ssConfig.HistoricalOffloadEarliestVersion) } func TestParseSSConfigs_ReadWriteMetrics(t *testing.T) { diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index 8a754b1f7e..b0bc38c746 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -110,6 +110,12 @@ type StateStoreConfig struct { HistoricalOffloadBigtableFamily string `mapstructure:"historical-offload-bigtable-family"` HistoricalOffloadBigtableAppProfile string `mapstructure:"historical-offload-bigtable-app-profile"` HistoricalOffloadBigtableShards int `mapstructure:"historical-offload-bigtable-shards"` + + // HistoricalOffloadEarliestVersion is the operator-declared earliest version + // (inclusive) fully ingested into the historical backend. When > 0 it is + // advertised as the store's earliest version so height gates admit pruned + // reads; reads below it stay on the primary. + HistoricalOffloadEarliestVersion int64 `mapstructure:"historical-offload-earliest-version"` } // DefaultStateStoreConfig returns the default StateStoreConfig diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 480bcf2452..caa5df504d 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -175,6 +175,12 @@ historical-offload-bigtable-table = "{{ .StateStore.HistoricalOffloadBigtableTab historical-offload-bigtable-family = "{{ .StateStore.HistoricalOffloadBigtableFamily }}" historical-offload-bigtable-app-profile = "{{ .StateStore.HistoricalOffloadBigtableAppProfile }}" historical-offload-bigtable-shards = {{ .StateStore.HistoricalOffloadBigtableShards }} + +# Earliest version (inclusive) fully ingested into the historical backend. +# When > 0 it becomes the node's advertised earliest version, so RPC height +# gates admit pruned heights the fallback can serve; point reads below it stay +# local. Leave 0 until backfill/ingestion coverage reaches the desired height. +historical-offload-earliest-version = {{ .StateStore.HistoricalOffloadEarliestVersion }} ` // ReceiptStoreConfigTemplate defines the configuration template for receipt-store diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index a4198199ab..c1594de05c 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -107,6 +107,7 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, `historical-offload-bigtable-instance = ""`, "Missing historical Bigtable instance") require.Contains(t, output, `historical-offload-bigtable-table = ""`, "Missing historical Bigtable table") require.Contains(t, output, "historical-offload-bigtable-shards = 0", "Missing historical Bigtable shards") + require.Contains(t, output, "historical-offload-earliest-version = 0", "Missing historical earliest version") } // TestReceiptStoreConfigTemplate verifies that all field paths in the receipt-store TOML template diff --git a/sei-db/state_db/ss/offload/historical/metrics.go b/sei-db/state_db/ss/offload/historical/metrics.go index e6cc0a90a7..145728beec 100644 --- a/sei-db/state_db/ss/offload/historical/metrics.go +++ b/sei-db/state_db/ss/offload/historical/metrics.go @@ -119,10 +119,11 @@ func (m *bigtableMetrics) recordWrite(ctx context.Context, table string, latency const fallbackMeterName = "seidb_historical_fallback" const ( - fallbackOutcomeCacheHit = "cache_hit" - fallbackOutcomeBackendHit = "backend_hit" - fallbackOutcomeBackendMiss = "backend_miss" - fallbackOutcomeError = "error" + fallbackOutcomeCacheHit = "cache_hit" + fallbackOutcomeBackendHit = "backend_hit" + fallbackOutcomeBackendMiss = "backend_miss" + fallbackOutcomeBackendBehind = "backend_behind" + fallbackOutcomeError = "error" ) // fallbackMetrics counts pruned point reads by operation (get/has) and outcome diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index 3759dcb860..f9b07af9d2 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -4,6 +4,9 @@ import ( "bytes" "context" "errors" + "fmt" + "sync" + "sync/atomic" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -13,8 +16,11 @@ import ( ) const ( - defaultHistoricalReadCacheEntries = 64 * 1024 - maxHistoricalReadCacheValueBytes = 64 * 1024 + // Cache entries and the per-value cap bound worst-case memory at + // entries*cap (256 MiB) even when an external caller crafts distinct + // large-value historical queries. + defaultHistoricalReadCacheEntries = 32 * 1024 + maxHistoricalReadCacheValueBytes = 8 * 1024 // historicalReadTimeout bounds one backend point read. types.StateStore has // no context parameter, so the deadline must be injected here; without it a @@ -26,6 +32,10 @@ const ( // immutable (a version's value never changes once written) but a miss can // flip to a hit once the offload consumer catches up. historicalMissCacheTTL = time.Minute + + // backendVersionRecheckInterval rate-limits LastVersion refreshes when a + // requested version is ahead of the backend's cached ingestion watermark. + backendVersionRecheckInterval = 30 * time.Second ) type historicalReadCacheKey struct { @@ -50,6 +60,17 @@ type PerKeyEarliestVersioner interface { GetEarliestVersionForKey(storeKey string) int64 } +// FallbackOptions tunes FallbackStateStore behavior. +type FallbackOptions struct { + // EarliestVersion is the operator-declared earliest version (inclusive) + // fully ingested into the historical backend. When set (> 0) it becomes the + // store's advertised earliest version, so height gates such as the EVM RPC + // watermark admit pruned heights that the fallback can serve; reads below + // it stay on the primary. When zero, the advertised earliest remains the + // local prune horizon and height gates keep rejecting pruned heights. + EarliestVersion int64 +} + // FallbackStateStore routes pruned point reads to the historical reader. // Iteration and writes stay on the primary state store. type FallbackStateStore struct { @@ -58,12 +79,19 @@ type FallbackStateStore struct { reader Reader cache *lru.Cache[historicalReadCacheKey, historicalReadCacheValue] metrics *fallbackMetrics + coverageFloor int64 + + // backendLast caches the backend's last ingested version; it is refreshed + // at most every backendVersionRecheckInterval when a read runs ahead of it. + backendLast atomic.Int64 + backendCheckedAt atomic.Int64 + backendMu sync.Mutex } var _ types.StateStore = (*FallbackStateStore)(nil) // NewFallbackStateStore takes ownership of primary and reader for Close. -func NewFallbackStateStore(primary types.StateStore, reader Reader) *FallbackStateStore { +func NewFallbackStateStore(primary types.StateStore, reader Reader, opts FallbackOptions) *FallbackStateStore { cache, err := lru.New[historicalReadCacheKey, historicalReadCacheValue](defaultHistoricalReadCacheEntries) if err != nil { panic(err) @@ -75,14 +103,48 @@ func NewFallbackStateStore(primary types.StateStore, reader Reader) *FallbackSta reader: reader, cache: cache, metrics: newFallbackMetrics(), + coverageFloor: opts.EarliestVersion, } } func (s *FallbackStateStore) shouldFallback(storeKey string, version int64) bool { + if version < 0 || (s.coverageFloor > 0 && version < s.coverageFloor) { + return false + } earliest := s.earliestVersionFor(storeKey) return earliest > 0 && version < earliest } +// ensureBackendCoverage refuses fallback reads above the backend's last +// ingested version so consumer lag surfaces as an error instead of silently +// empty state (which would also poison the miss cache). +func (s *FallbackStateStore) ensureBackendCoverage(ctx context.Context, version int64) error { + if version <= s.backendLast.Load() { + return nil + } + s.backendMu.Lock() + defer s.backendMu.Unlock() + last := s.backendLast.Load() + if version <= last { + return nil + } + if time.Now().UnixNano()-s.backendCheckedAt.Load() >= backendVersionRecheckInterval.Nanoseconds() { + refreshed, err := s.reader.LastVersion(ctx) + if err != nil { + return fmt.Errorf("check historical backend coverage: %w", err) + } + s.backendCheckedAt.Store(time.Now().UnixNano()) + if refreshed > last { + last = refreshed + s.backendLast.Store(refreshed) + } + } + if version > last { + return fmt.Errorf("historical backend has ingested up to version %d, behind requested version %d", last, version) + } + return nil +} + func (s *FallbackStateStore) earliestVersionFor(storeKey string) int64 { if s.perKeyEarliest != nil { return s.perKeyEarliest.GetEarliestVersionForKey(storeKey) @@ -108,6 +170,10 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] } ctx, cancel := s.readContext() defer cancel() + if err := s.ensureBackendCoverage(ctx, version); err != nil { + s.metrics.recordRead("get", fallbackOutcomeBackendBehind) + return nil, err + } v, err := s.reader.Get(ctx, storeKey, key, version) if err != nil { if errors.Is(err, ErrNotFound) { @@ -199,6 +265,10 @@ func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bo } ctx, cancel := s.readContext() defer cancel() + if err := s.ensureBackendCoverage(ctx, version); err != nil { + s.metrics.recordRead("has", fallbackOutcomeBackendBehind) + return false, err + } found, err := s.reader.Has(ctx, storeKey, key, version) if err != nil { s.metrics.recordRead("has", fallbackOutcomeError) @@ -232,7 +302,19 @@ func (s *FallbackStateStore) SetLatestVersion(version int64) error { return s.primary.SetLatestVersion(version) } -func (s *FallbackStateStore) GetEarliestVersion() int64 { return s.primary.GetEarliestVersion() } +// GetEarliestVersion advertises the earliest queryable version. With a +// configured coverage floor the fallback serves point reads below the local +// prune horizon, so height gates (e.g. the EVM RPC watermark) must see the +// floor rather than the local horizon or every historical query is rejected +// before it reaches the store. Iterators do not use the fallback; range +// queries between the floor and the local horizon see only local data. +func (s *FallbackStateStore) GetEarliestVersion() int64 { + local := s.primary.GetEarliestVersion() + if s.coverageFloor > 0 && local > 0 && s.coverageFloor < local { + return s.coverageFloor + } + return local +} func (s *FallbackStateStore) SetEarliestVersion(version int64, ignoreVersion bool) error { return s.primary.SetEarliestVersion(version, ignoreVersion) diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go index c7ab77ccc4..181c5053ed 100644 --- a/sei-db/state_db/ss/offload/historical/store_test.go +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -59,6 +59,10 @@ type fakeReader struct { getErr error hasResult bool hasSet bool + // lastVersion overrides the reported ingestion watermark; zero means + // "fully caught up" so coverage gating stays out of unrelated tests. + lastVersion int64 + lastVersions int } func (f *fakeReader) Get(context.Context, string, []byte, int64) (Value, error) { @@ -77,13 +81,19 @@ func (f *fakeReader) Has(context.Context, string, []byte, int64) (bool, error) { return true, nil } -func (f *fakeReader) LastVersion(context.Context) (int64, error) { return 0, nil } -func (f *fakeReader) Close() error { return nil } +func (f *fakeReader) LastVersion(context.Context) (int64, error) { + f.lastVersions++ + if f.lastVersion != 0 { + return f.lastVersion, nil + } + return 1 << 62, nil +} +func (f *fakeReader) Close() error { return nil } func TestFallbackStateStoreRoutesPrunedPointReads(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) value, err := store.Get("bank", 7, []byte("k")) require.NoError(t, err) @@ -101,7 +111,7 @@ func TestFallbackStateStoreRoutesPrunedPointReads(t *testing.T) { func TestFallbackStateStoreKeepsRecentPointReadsOnPrimary(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) value, err := store.Get("bank", 10, []byte("k")) require.NoError(t, err) @@ -113,7 +123,7 @@ func TestFallbackStateStoreKeepsRecentPointReadsOnPrimary(t *testing.T) { func TestFallbackStateStoreCachesHistoricalPointReads(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) value, err := store.Get("bank", 7, []byte("k")) require.NoError(t, err) @@ -134,7 +144,7 @@ func TestFallbackStateStoreCachesHistoricalPointReads(t *testing.T) { func TestFallbackStateStoreCachesHistoricalMisses(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{getErr: ErrNotFound, hasSet: true} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) value, err := store.Get("bank", 7, []byte("missing")) require.NoError(t, err) @@ -154,7 +164,7 @@ func TestFallbackStateStoreCachesHistoricalMisses(t *testing.T) { func TestFallbackStateStoreCachesHistoricalHasResults(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{hasResult: true, hasSet: true} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) ok, err := store.Has("bank", 7, []byte("k")) require.NoError(t, err) @@ -169,7 +179,7 @@ func TestFallbackStateStoreCachesHistoricalHasResults(t *testing.T) { func TestFallbackStateStoreDoesNotUseHasOnlyCacheForGet(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{hasResult: true, hasSet: true} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) ok, err := store.Has("bank", 7, []byte("k")) require.NoError(t, err) @@ -199,7 +209,7 @@ func TestFallbackStateStoreUsesPerKeyEarliestVersion(t *testing.T) { perKey: map[string]int64{"evm": 5}, } reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) // The EVM store still holds version 7 locally even though the cosmos store // pruned to 10; the read must stay on the primary. @@ -222,10 +232,63 @@ func TestFallbackStateStoreUsesPerKeyEarliestVersion(t *testing.T) { require.Equal(t, 0, reader.has) } +func TestFallbackStateStoreCoverageFloor(t *testing.T) { + primary := &fakeStateStore{earliest: 100} + reader := &fakeReader{} + store := NewFallbackStateStore(primary, reader, FallbackOptions{EarliestVersion: 50}) + + // The floor becomes the advertised earliest so height gates admit pruned + // heights the fallback can serve. + require.Equal(t, int64(50), store.GetEarliestVersion()) + + // In [floor, local earliest): fallback serves the read. + value, err := store.Get("bank", 60, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 1, reader.gets) + + // Below the floor: the backend has no coverage; stay on the primary. + value, err = store.Get("bank", 49, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("primary"), value) + require.Equal(t, 1, reader.gets) + + // Negative versions never fall back. + _, err = store.Get("bank", -1, []byte("k")) + require.NoError(t, err) + require.Equal(t, 1, reader.gets) +} + +func TestFallbackStateStoreRejectsReadsBeyondBackendCoverage(t *testing.T) { + primary := &fakeStateStore{earliest: 100} + reader := &fakeReader{lastVersion: 40} + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + + // The backend has only ingested up to 40; a pruned read at 90 must error + // instead of returning silently-empty state. + _, err := store.Get("bank", 90, []byte("k")) + require.ErrorContains(t, err, "behind requested version") + require.Equal(t, 0, reader.gets) + + _, err = store.Has("bank", 90, []byte("k")) + require.ErrorContains(t, err, "behind requested version") + require.Equal(t, 0, reader.has) + + // The watermark refresh is rate-limited: the second miss must not trigger + // another LastVersion call. + require.Equal(t, 1, reader.lastVersions) + + // Reads at or below the ingested watermark are served. + value, err := store.Get("bank", 40, []byte("k")) + require.NoError(t, err) + require.Equal(t, []byte("historical"), value) + require.Equal(t, 1, reader.gets) +} + func TestFallbackStateStoreExpiresCachedMisses(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{getErr: ErrNotFound} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) value, err := store.Get("bank", 7, []byte("missing")) require.NoError(t, err) @@ -250,7 +313,7 @@ func TestFallbackStateStoreExpiresCachedMisses(t *testing.T) { func TestFallbackStateStoreDoesNotCacheHistoricalErrors(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{getErr: errors.New("boom")} - store := NewFallbackStateStore(primary, reader) + store := NewFallbackStateStore(primary, reader, FallbackOptions{}) _, err := store.Get("bank", 7, []byte("k")) require.Error(t, err) diff --git a/sei-db/state_db/ss/store.go b/sei-db/state_db/ss/store.go index e102af34c9..fa2c25add4 100644 --- a/sei-db/state_db/ss/store.go +++ b/sei-db/state_db/ss/store.go @@ -29,6 +29,9 @@ func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.Stat if !scyllaConfigured && !bigtableConfigured { return primary, nil } + fallbackOpts := historical.FallbackOptions{ + EarliestVersion: ssConfig.HistoricalOffloadEarliestVersion, + } if bigtableConfigured { reader, err := historical.NewBigtableReader(historical.BigtableConfig{ ProjectID: ssConfig.HistoricalOffloadBigtableProjectID, @@ -42,7 +45,7 @@ func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.Stat _ = primary.Close() return nil, fmt.Errorf("open bigtable historical offload reader: %w", err) } - return historical.NewFallbackStateStore(primary, reader), nil + return historical.NewFallbackStateStore(primary, reader, fallbackOpts), nil } reader, err := historical.NewScyllaReader(historical.ScyllaConfig{ Hosts: splitCSV(ssConfig.HistoricalOffloadScyllaHosts), @@ -57,7 +60,7 @@ func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.Stat _ = primary.Close() return nil, fmt.Errorf("open scylla/cassandra historical offload reader: %w", err) } - return historical.NewFallbackStateStore(primary, reader), nil + return historical.NewFallbackStateStore(primary, reader, fallbackOpts), nil } func scyllaHistoricalOffloadConfigured(cfg config.StateStoreConfig) bool { From d44d8166fcd34859a321ad701144129758085530 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 09:49:00 -0400 Subject: [PATCH 34/41] Support SASL/PLAIN for Google Cloud Managed Kafka Google Cloud Managed Service for Apache Kafka authenticates with TLS + SASL/PLAIN (service-account email and base64 key); the offload producer and consumer previously supported only aws-msk-iam, so they could not connect to the managed cluster. Add plain mechanism with username and password to the shared SASL path, validate credentials at config load, document the GCP connection shape, and drop the cryptosim-era consumer client-id default. Co-Authored-By: Claude Fable 5 --- sei-db/state_db/ss/offload/consumer/README.md | 43 ++++++++++++++++++- .../consumer/config/example-bigtable.json | 8 +++- sei-db/state_db/ss/offload/consumer/kafka.go | 33 +++++++++----- .../ss/offload/consumer/kafka_test.go | 12 +++++- sei-db/state_db/ss/offload/kafka.go | 38 +++++++++++----- 5 files changed, 107 insertions(+), 27 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md index 82df3149bc..e17f8b6c55 100644 --- a/sei-db/state_db/ss/offload/consumer/README.md +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -68,6 +68,19 @@ go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer \ The example configs are local/dev placeholders. Set real Kafka brokers and backend credentials/config in your own config. +For Google Cloud Managed Service for Apache Kafka, connect with TLS plus +SASL/PLAIN using service-account credentials: + +```json +"Kafka": { + "Brokers": ["bootstrap.CLUSTER.REGION.managedkafka.PROJECT.cloud.goog:9092"], + "TLSEnabled": true, + "SASLMechanism": "plain", + "Username": "kafka-client@PROJECT.iam.gserviceaccount.com", + "Password": "" +} +``` + ## Node Read Fallback Enable fallback reads in the node config: @@ -97,11 +110,37 @@ historical-offload-bigtable-shards = 256 Fallback activates only for point reads where the requested version is below the local SS earliest version. Missing rows and tombstones return empty state, same -as local SS. +as local SS. Reads ahead of the backend's last ingested version (consumer lag) +return an error rather than empty state. + +To open RPC height gates for pruned heights, declare how far back the backend +has full coverage (the version at which ingestion/backfill started): + +```toml +[state-store] +historical-offload-earliest-version = 123456789 +``` + +When set (> 0), the node advertises this as its earliest version so height +checks admit heights the fallback can serve; point reads below it stay on the +local store. Leave it 0 until the backend actually covers the target range — +heights the backend never ingested would otherwise read as empty state. + +## Operational preconditions + +Before enabling the node read fallback in production: + +- The backend table/keyspace exists with the same family/shards (Bigtable) or + schema (Scylla) as the consumer wrote with. +- The consumer has been ingesting continuously; check + `consumer_kafka_lag` / `bigtable_rows_mutated_total`. +- `historical-offload-earliest-version` is set to the true coverage floor. ## Current Limits -- No offload iterator path. +- No offload iterator path; range queries between the coverage floor and the + local prune horizon see only local data. - No cross-row transaction on ingest; mutation rows are written first and the version marker is written last, so replay is idempotent after partial failure. - No automatic schema creation from the binary. +- No backfill tooling; coverage starts when ingestion starts. diff --git a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json index 588111679e..27c2640d29 100644 --- a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json +++ b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json @@ -1,10 +1,14 @@ { "Backend": "bigtable", "Kafka": { - "Brokers": ["localhost:9092"], + "Brokers": ["bootstrap.my-cluster.us-east1.managedkafka.my-gcp-project.cloud.goog:9092"], "Topic": "historical-offload", "GroupID": "historical-bigtable", - "StartOffset": "first" + "StartOffset": "first", + "TLSEnabled": true, + "SASLMechanism": "plain", + "Username": "kafka-client@my-gcp-project.iam.gserviceaccount.com", + "Password": "" }, "Bigtable": { "ProjectID": "my-gcp-project", diff --git a/sei-db/state_db/ss/offload/consumer/kafka.go b/sei-db/state_db/ss/offload/consumer/kafka.go index 3eca641d1b..49263dcc21 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka.go +++ b/sei-db/state_db/ss/offload/consumer/kafka.go @@ -17,22 +17,27 @@ import ( // (kafka-go's zero CommitInterval) so offsets only advance after the sink // persists each entry. type KafkaReaderConfig struct { - Brokers []string - Topic string - GroupID string - ClientID string - Region string - StartOffset string // "first" or "last"; defaults to "first" - MinBytes int - MaxBytes int - MaxWait time.Duration - TLSEnabled bool + Brokers []string + Topic string + GroupID string + ClientID string + Region string + StartOffset string // "first" or "last"; defaults to "first" + MinBytes int + MaxBytes int + MaxWait time.Duration + TLSEnabled bool + // SASLMechanism selects broker auth: "none", "plain" (username/password, + // e.g. Google Cloud Managed Kafka service-account credentials), or + // "aws-msk-iam". SASLMechanism string + Username string + Password string } func (c *KafkaReaderConfig) ApplyDefaults() { if c.ClientID == "" { - c.ClientID = "cryptosim-historical-offload-consumer" + c.ClientID = "sei-historical-offload-consumer" } if c.StartOffset == "" { c.StartOffset = "first" @@ -67,6 +72,10 @@ func (c *KafkaReaderConfig) Validate() error { // load time instead of with an obscure handshake error on first fetch. switch strings.ToLower(c.SASLMechanism) { case "", "none": + case "plain": + if c.Username == "" || c.Password == "" { + return fmt.Errorf("kafka username and password are required for sasl plain") + } case "aws-msk-iam": if !c.TLSEnabled { return fmt.Errorf("kafka tls must be enabled for aws-msk-iam") @@ -97,6 +106,8 @@ func NewKafkaReader(cfg KafkaReaderConfig) (*kafka.Reader, error) { Region: cfg.Region, TLSEnabled: cfg.TLSEnabled, SASLMechanism: cfg.SASLMechanism, + Username: cfg.Username, + Password: cfg.Password, }) if err != nil { return nil, err diff --git a/sei-db/state_db/ss/offload/consumer/kafka_test.go b/sei-db/state_db/ss/offload/consumer/kafka_test.go index e7abda40f1..f6122ffb51 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka_test.go +++ b/sei-db/state_db/ss/offload/consumer/kafka_test.go @@ -14,7 +14,7 @@ func TestKafkaReaderConfigApplyDefaults(t *testing.T) { GroupID: "scylla", } cfg.ApplyDefaults() - require.Equal(t, "cryptosim-historical-offload-consumer", cfg.ClientID) + require.Equal(t, "sei-historical-offload-consumer", cfg.ClientID) require.Equal(t, "first", cfg.StartOffset) require.Equal(t, 1, cfg.MinBytes) require.Equal(t, 10<<20, cfg.MaxBytes) @@ -49,10 +49,20 @@ func TestKafkaReaderConfigValidateSASL(t *testing.T) { cfg.Region = "us-east-1" require.NoError(t, cfg.Validate()) + // SASL/PLAIN (e.g. Google Cloud Managed Kafka) needs credentials. cfg = base cfg.SASLMechanism = "plain" + require.ErrorContains(t, cfg.Validate(), "username and password") + + cfg.Username = "svc@project.iam.gserviceaccount.com" + cfg.Password = "base64-encoded-key" + require.NoError(t, cfg.Validate()) + + cfg = base + cfg.SASLMechanism = "scram" require.ErrorContains(t, cfg.Validate(), "sasl mechanism") + cfg = base cfg.SASLMechanism = "none" require.NoError(t, cfg.Validate()) } diff --git a/sei-db/state_db/ss/offload/kafka.go b/sei-db/state_db/ss/offload/kafka.go index 981cb35e02..b870177959 100644 --- a/sei-db/state_db/ss/offload/kafka.go +++ b/sei-db/state_db/ss/offload/kafka.go @@ -12,6 +12,7 @@ import ( "github.com/segmentio/kafka-go" "github.com/segmentio/kafka-go/compress" "github.com/segmentio/kafka-go/sasl" + "github.com/segmentio/kafka-go/sasl/plain" dbproto "github.com/sei-protocol/sei-chain/sei-db/proto" ) @@ -19,18 +20,23 @@ import ( const kafkaOptionNone = "none" type KafkaConfig struct { - Brokers []string - Topic string - ClientID string - Region string - Async bool - RequiredAcks string - Compression string - BatchSize int - BatchTimeout time.Duration - BatchBytes int - TLSEnabled bool + Brokers []string + Topic string + ClientID string + Region string + Async bool + RequiredAcks string + Compression string + BatchSize int + BatchTimeout time.Duration + BatchBytes int + TLSEnabled bool + // SASLMechanism selects broker auth: "none", "plain" (username/password, + // e.g. Google Cloud Managed Kafka service-account credentials), or + // "aws-msk-iam". SASLMechanism string + Username string + Password string } func (c *KafkaConfig) ApplyDefaults() { @@ -86,6 +92,11 @@ func (c *KafkaConfig) Validate() error { switch strings.ToLower(c.SASLMechanism) { case "", kafkaOptionNone: return nil + case "plain": + if c.Username == "" || c.Password == "" { + return fmt.Errorf("kafka username and password are required for sasl plain") + } + return nil case "aws-msk-iam": if !c.TLSEnabled { return fmt.Errorf("kafka tls must be enabled for aws-msk-iam") @@ -216,6 +227,11 @@ func NewSASLMechanism(cfg KafkaConfig) (sasl.Mechanism, error) { switch strings.ToLower(cfg.SASLMechanism) { case "", kafkaOptionNone: return nil, nil + case "plain": + if cfg.Username == "" || cfg.Password == "" { + return nil, fmt.Errorf("kafka username and password are required for sasl plain") + } + return plain.Mechanism{Username: cfg.Username, Password: cfg.Password}, nil case "aws-msk-iam": return newAWSMSKIAMMechanism(cfg) default: From 284d1c169ac89b089ce150a317f3e12618d319ca Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 09:49:00 -0400 Subject: [PATCH 35/41] Retry transient Bigtable read failures and keep connections alive The node-side reader surfaced any mid-stream UNAVAILABLE (tablet moves, GFE restarts) straight to the RPC caller and a silently dropped connection was only detected by the OS TCP timeout. Retry reads up to 3 times with backoff while no row has been delivered, add gRPC keepalive to the Bigtable connection, and document why this package speaks the data protocol directly: the official Bigtable client requires a newer google.golang.org/grpc than the repo-wide v1.57 replace pin allows. Co-Authored-By: Claude Fable 5 --- .../ss/offload/historical/bigtable.go | 55 +++++++++++++++++++ .../ss/offload/historical/bigtable_test.go | 47 ++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 86584c7d7d..ca2eb17e91 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -13,8 +13,11 @@ import ( "cloud.google.com/go/bigtable/apiv2/bigtablepb" "golang.org/x/sync/errgroup" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" ) const ( @@ -34,11 +37,21 @@ const ( defaultBigtableReadWorkers = 16 + // Transient stream failures (tablet moves, GFE restarts) surface as + // UNAVAILABLE; retry reads a bounded number of times with backoff. + bigtableReadAttempts = 3 + bigtableReadRetryBackoff = 100 * time.Millisecond + maxUint16Int = 1<<16 - 1 maxUint32Int = 1<<32 - 1 maxInt64Uint64 = 1<<63 - 1 ) +// BigtableClient speaks the Bigtable data protocol directly over gRPC. The +// official cloud.google.com/go/bigtable client requires a newer +// google.golang.org/grpc than the repo-wide v1.57 replace pin allows, so this +// package carries its own thin transport: ReadRows chunk assembly, MutateRows +// result accounting, bounded read retries, and per-RPC cost metrics. type BigtableClient struct { conn *grpc.ClientConn data bigtablepb.BigtableClient @@ -144,6 +157,12 @@ func OpenBigtableClient(ctx context.Context, cfg BigtableConfig) (*BigtableClien conn, err := grpc.DialContext(ctx, bigtableEndpoint, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), grpc.WithPerRPCCredentials(perRPC), + // Detect silently dropped connections instead of hanging reads on the + // OS TCP timeout. + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 30 * time.Second, + Timeout: 10 * time.Second, + }), ) if err != nil { return nil, fmt.Errorf("open bigtable connection: %w", err) @@ -169,7 +188,43 @@ func (c *BigtableClient) Close() error { return c.conn.Close() } +// ReadRows retries transient stream failures as long as no row has been +// delivered to the callback yet, which covers every limit-1 point read and +// version-bucket scan in this package. func (c *BigtableClient) ReadRows(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { + return readRowsWithRetry(ctx, c.readRowsOnce, startKey, endKey, limit, family, f, qualifiers...) +} + +func readRowsWithRetry(ctx context.Context, once BigtableReadRowsFunc, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { + backoff := bigtableReadRetryBackoff + var delivered bool + for attempt := 1; ; attempt++ { + err := once(ctx, startKey, endKey, limit, family, func(row BigtableRow) bool { + delivered = true + return f(row) + }, qualifiers...) + if err == nil || delivered || attempt >= bigtableReadAttempts || status.Code(err) != codes.Unavailable { + return err + } + if sleepErr := sleepWithContext(ctx, backoff); sleepErr != nil { + return err + } + backoff *= 2 + } +} + +func sleepWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (c *BigtableClient) readRowsOnce(ctx context.Context, startKey, endKey []byte, limit int64, family string, f func(BigtableRow) bool, qualifiers ...string) error { start := time.Now() var rowsRead, bytesRead int64 defer func() { diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go index dfd336b7bd..0d48d429ba 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable_test.go +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -9,6 +9,8 @@ import ( "cloud.google.com/go/bigtable/apiv2/bigtablepb" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/wrapperspb" ) @@ -157,6 +159,51 @@ func TestBigtableRowBuilderAssemblesSplitCells(t *testing.T) { require.Equal(t, []byte("hello bt"), row.Cells[0].Value) } +func TestReadRowsWithRetryRetriesUnavailable(t *testing.T) { + attempts := 0 + once := func(_ context.Context, _, _ []byte, _ int64, _ string, f func(BigtableRow) bool, _ ...string) error { + attempts++ + if attempts < 3 { + return status.Error(codes.Unavailable, "tablet moved") + } + f(BigtableRow{Key: "rk"}) + return nil + } + var got []string + err := readRowsWithRetry(context.Background(), once, []byte("a"), []byte("b"), 1, "state", func(row BigtableRow) bool { + got = append(got, row.Key) + return false + }) + require.NoError(t, err) + require.Equal(t, 3, attempts) + require.Equal(t, []string{"rk"}, got) +} + +func TestReadRowsWithRetryStopsAfterDelivery(t *testing.T) { + // Once a row reached the callback, a retry would re-deliver it; the error + // must surface instead. + attempts := 0 + once := func(_ context.Context, _, _ []byte, _ int64, _ string, f func(BigtableRow) bool, _ ...string) error { + attempts++ + f(BigtableRow{Key: "rk"}) + return status.Error(codes.Unavailable, "mid-stream drop") + } + err := readRowsWithRetry(context.Background(), once, nil, nil, 0, "", func(BigtableRow) bool { return true }) + require.Error(t, err) + require.Equal(t, 1, attempts) +} + +func TestReadRowsWithRetryDoesNotRetryNonRetryable(t *testing.T) { + attempts := 0 + once := func(_ context.Context, _, _ []byte, _ int64, _ string, _ func(BigtableRow) bool, _ ...string) error { + attempts++ + return status.Error(codes.NotFound, "table missing") + } + err := readRowsWithRetry(context.Background(), once, nil, nil, 0, "", func(BigtableRow) bool { return true }) + require.Error(t, err) + require.Equal(t, 1, attempts) +} + func TestBigtableLastVersionScansBuckets(t *testing.T) { versions := map[int]int64{ 3: 42, From 9fc34be9a618a8e416abd34f3aaf7abff7fcb664 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 10:01:28 -0400 Subject: [PATCH 36/41] Distinguish backend-behind from backend-error in fallback metrics A LastVersion RPC failure during the coverage check was counted as backend_behind, overstating lag during backend outages; classify it by a sentinel error instead. Co-Authored-By: Claude Fable 5 --- sei-db/state_db/ss/offload/historical/store.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index f9b07af9d2..be8c7e4030 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -115,6 +115,10 @@ func (s *FallbackStateStore) shouldFallback(storeKey string, version int64) bool return earliest > 0 && version < earliest } +// errBackendBehind marks reads refused because the backend has not ingested +// the requested version yet, distinguishing lag from backend failures. +var errBackendBehind = errors.New("historical backend behind requested version") + // ensureBackendCoverage refuses fallback reads above the backend's last // ingested version so consumer lag surfaces as an error instead of silently // empty state (which would also poison the miss cache). @@ -140,11 +144,18 @@ func (s *FallbackStateStore) ensureBackendCoverage(ctx context.Context, version } } if version > last { - return fmt.Errorf("historical backend has ingested up to version %d, behind requested version %d", last, version) + return fmt.Errorf("%w: ingested up to version %d, requested %d", errBackendBehind, last, version) } return nil } +func coverageOutcome(err error) string { + if errors.Is(err, errBackendBehind) { + return fallbackOutcomeBackendBehind + } + return fallbackOutcomeError +} + func (s *FallbackStateStore) earliestVersionFor(storeKey string) int64 { if s.perKeyEarliest != nil { return s.perKeyEarliest.GetEarliestVersionForKey(storeKey) @@ -171,7 +182,7 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] ctx, cancel := s.readContext() defer cancel() if err := s.ensureBackendCoverage(ctx, version); err != nil { - s.metrics.recordRead("get", fallbackOutcomeBackendBehind) + s.metrics.recordRead("get", coverageOutcome(err)) return nil, err } v, err := s.reader.Get(ctx, storeKey, key, version) @@ -266,7 +277,7 @@ func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bo ctx, cancel := s.readContext() defer cancel() if err := s.ensureBackendCoverage(ctx, version); err != nil { - s.metrics.recordRead("has", fallbackOutcomeBackendBehind) + s.metrics.recordRead("has", coverageOutcome(err)) return false, err } found, err := s.reader.Has(ctx, storeKey, key, version) From 2fa06ee787426d8454c7fa84668233b6a722a076 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 10:59:52 -0400 Subject: [PATCH 37/41] Trim dead knobs and indirection after style audit Remove config surface and defensive code nothing uses: the consumer's unreachable sink-retry Options, re-defaulting accessors on both sinks, nil guards on a cache the constructor always sets, the version parameter every caller derived from entry.Version, the one-field FallbackOptions struct, and a fragile gocql type-name test. Consolidate SASL validation into one shared offload.ValidateSASL, move backend Configured checks onto the config structs, unexport bigtable helpers with no external callers, de-stutter historical package names, and cut comments that restated code. Co-Authored-By: Claude Fable 5 --- .../state_db/ss/offload/consumer/bigtable.go | 34 ++------ .../cmd/historical-offload-consumer/main.go | 4 +- .../state_db/ss/offload/consumer/consumer.go | 86 +++++++------------ sei-db/state_db/ss/offload/consumer/kafka.go | 36 +++----- .../state_db/ss/offload/consumer/metrics.go | 8 +- sei-db/state_db/ss/offload/consumer/scylla.go | 26 ++---- .../ss/offload/consumer/scylla_test.go | 2 +- .../ss/offload/historical/bigtable.go | 18 ++-- .../ss/offload/historical/bigtable_test.go | 10 +-- .../state_db/ss/offload/historical/scylla.go | 4 + .../ss/offload/historical/scylla_test.go | 20 +---- .../state_db/ss/offload/historical/store.go | 78 +++++++---------- .../ss/offload/historical/store_test.go | 24 +++--- sei-db/state_db/ss/offload/kafka.go | 26 +++--- sei-db/state_db/ss/store.go | 62 ++++++------- 15 files changed, 159 insertions(+), 279 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index 76f572ff39..fd98a791a7 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -82,7 +82,7 @@ func (s *bigtableSink) WriteBatch(ctx context.Context, records []Record) error { func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) error { rows := make([]historical.BigtableRowMutation, 0, bigtableRowMutationCount(records)) for _, rec := range records { - rows = s.appendRecordRowMutations(rows, rec.Entry.Version, rec.Entry) + rows = s.appendRecordRowMutations(rows, rec.Entry) } if len(rows) == 0 { return nil @@ -91,9 +91,9 @@ func (s *bigtableSink) writeRecordRows(ctx context.Context, records []Record) er } func (s *bigtableSink) applyRecordRowMutations(ctx context.Context, rows []historical.BigtableRowMutation) error { - chunks := bigtableRowMutationChunks(rows, s.effectiveBulkChunkRows()) + chunks := bigtableRowMutationChunks(rows, s.bulkChunkRows) g, gctx := errgroup.WithContext(ctx) - g.SetLimit(s.effectiveBulkChunkWorkers()) + g.SetLimit(s.bulkChunkWorkers) for _, chunk := range chunks { chunk := chunk g.Go(func() error { @@ -104,27 +104,12 @@ func (s *bigtableSink) applyRecordRowMutations(ctx context.Context, rows []histo return g.Wait() } -func (s *bigtableSink) effectiveBulkChunkRows() int { - if s.bulkChunkRows <= 0 { - return defaultBigtableMutationChunkRows - } - return s.bulkChunkRows -} - -func (s *bigtableSink) effectiveBulkChunkWorkers() int { - if s.bulkChunkWorkers <= 0 { - return defaultBigtableMutationChunkConcurrency - } - return s.bulkChunkWorkers -} - -func (s *bigtableSink) appendRecordRowMutations(rows []historical.BigtableRowMutation, version int64, entry *proto.ChangelogEntry) []historical.BigtableRowMutation { - mutations := compactMutations(entry) - for _, mutation := range mutations { - rows = append(rows, s.mutationRow(version, mutation.storeName, mutation.pair)) +func (s *bigtableSink) appendRecordRowMutations(rows []historical.BigtableRowMutation, entry *proto.ChangelogEntry) []historical.BigtableRowMutation { + for _, mutation := range compactMutations(entry) { + rows = append(rows, s.mutationRow(entry.Version, mutation.storeName, mutation.pair)) } for _, up := range entry.Upgrades { - rows = append(rows, s.upgradeRow(version, up)) + rows = append(rows, s.upgradeRow(entry.Version, up)) } return rows } @@ -194,10 +179,7 @@ func (s *bigtableSink) writeVersionMarkers(ctx context.Context, records []Record func bigtableRowMutationCount(records []Record) int { total := 0 for _, rec := range records { - for _, changeset := range rec.Entry.Changesets { - total += len(changeset.Changeset.Pairs) - } - total += len(rec.Entry.Upgrades) + total += entryMutationCapacity(rec.Entry) + len(rec.Entry.Upgrades) } return total } diff --git a/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go b/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go index d949de6706..53700ad6a0 100644 --- a/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go +++ b/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go @@ -28,9 +28,7 @@ func main() { defer cancel() // Install the Prometheus MeterProvider before opening the sink so the - // backend cost metrics it registers (e.g. bigtable_rows_mutated_total, - // bigtable_bytes_written_total, bigtable_mutate_latency_seconds) bind to a - // real exporter and can be scraped at MetricsAddr/metrics. + // backend cost metrics it registers bind to a real exporter. if cfg.MetricsAddr != "" { reg, shutdown, err := commonmetrics.SetupOtelPrometheus() if err != nil { diff --git a/sei-db/state_db/ss/offload/consumer/consumer.go b/sei-db/state_db/ss/offload/consumer/consumer.go index 2d83d14897..345c8749b5 100644 --- a/sei-db/state_db/ss/offload/consumer/consumer.go +++ b/sei-db/state_db/ss/offload/consumer/consumer.go @@ -19,36 +19,31 @@ type MessageSource interface { // Messages are sharded by partition: cross-partition writes parallelize while // ordering within a partition is preserved. type Consumer struct { - reader MessageSource - sink Sink - logf func(format string, args ...interface{}) - workers int - shardBuf int - batchSize int - batchWait time.Duration - maxAttempts int - baseBackoff time.Duration - maxBackoff time.Duration - metrics *consumerMetrics + reader MessageSource + sink Sink + logf func(format string, args ...interface{}) + workers int + shardBuf int + batchSize int + batchWait time.Duration + metrics *consumerMetrics } const ( - defaultSinkMaxAttempts = 5 - defaultSinkBaseBackoff = 1 * time.Second - defaultSinkMaxBackoff = 30 * time.Second - defaultWorkers = 16 - defaultShardBuffer = 128 - defaultBatchSize = 16 - defaultBatchMaxWait = 10 * time.Millisecond + sinkMaxAttempts = 5 + sinkBaseBackoff = 1 * time.Second + sinkMaxBackoff = 30 * time.Second + + defaultWorkers = 16 + defaultShardBuffer = 128 + defaultBatchSize = 16 + defaultBatchMaxWait = 10 * time.Millisecond ) // Backpressure: when the sink falls behind, ShardBufferSize fills, the fetcher // blocks, and Kafka stops being polled. Zero values pick defaults. type Options struct { Logf func(format string, args ...interface{}) - SinkMaxAttempts int - SinkBaseBackoff time.Duration - SinkMaxBackoff time.Duration Workers int ShardBufferSize int MaxBatchRecords int @@ -60,18 +55,6 @@ func New(reader MessageSource, sink Sink, opts Options) *Consumer { if logf == nil { logf = func(string, ...interface{}) {} } - maxAttempts := opts.SinkMaxAttempts - if maxAttempts <= 0 { - maxAttempts = defaultSinkMaxAttempts - } - base := opts.SinkBaseBackoff - if base <= 0 { - base = defaultSinkBaseBackoff - } - maxBackoff := opts.SinkMaxBackoff - if maxBackoff <= 0 { - maxBackoff = defaultSinkMaxBackoff - } workers := opts.Workers if workers <= 0 { workers = defaultWorkers @@ -89,26 +72,19 @@ func New(reader MessageSource, sink Sink, opts Options) *Consumer { batchWait = defaultBatchMaxWait } return &Consumer{ - reader: reader, - sink: sink, - logf: logf, - workers: workers, - shardBuf: shardBuf, - batchSize: batchSize, - batchWait: batchWait, - maxAttempts: maxAttempts, - baseBackoff: base, - maxBackoff: maxBackoff, - metrics: newConsumerMetrics(), + reader: reader, + sink: sink, + logf: logf, + workers: workers, + shardBuf: shardBuf, + batchSize: batchSize, + batchWait: batchWait, + metrics: newConsumerMetrics(), } } // Run commits offsets only after the sink persists each message. func (c *Consumer) Run(ctx context.Context) error { - return c.runParallel(ctx) -} - -func (c *Consumer) runParallel(ctx context.Context) error { g, gctx := errgroup.WithContext(ctx) shards := make([]chan kafka.Message, c.workers) for i := range shards { @@ -261,9 +237,9 @@ func batchLag(msgs []kafka.Message) int64 { } func (c *Consumer) writeBatchWithRetry(ctx context.Context, records []Record) error { - backoff := c.baseBackoff + backoff := sinkBaseBackoff var lastErr error - for attempt := 1; attempt <= c.maxAttempts; attempt++ { + for attempt := 1; attempt <= sinkMaxAttempts; attempt++ { err := c.writeRecords(ctx, records) if err == nil { return nil @@ -272,20 +248,20 @@ func (c *Consumer) writeBatchWithRetry(ctx context.Context, records []Record) er if isCancellation(err) { return err } - if attempt == c.maxAttempts { + if attempt == sinkMaxAttempts { break } c.logf("sink write attempt %d/%d failed: %v; retrying in %s", - attempt, c.maxAttempts, err, backoff) + attempt, sinkMaxAttempts, err, backoff) if err := sleepWithContext(ctx, backoff); err != nil { return err } backoff *= 2 - if backoff > c.maxBackoff { - backoff = c.maxBackoff + if backoff > sinkMaxBackoff { + backoff = sinkMaxBackoff } } - return fmt.Errorf("sink write failed after %d attempts: %w", c.maxAttempts, lastErr) + return fmt.Errorf("sink write failed after %d attempts: %w", sinkMaxAttempts, lastErr) } func sleepWithContext(ctx context.Context, d time.Duration) error { diff --git a/sei-db/state_db/ss/offload/consumer/kafka.go b/sei-db/state_db/ss/offload/consumer/kafka.go index 49263dcc21..894fbea0ce 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka.go +++ b/sei-db/state_db/ss/offload/consumer/kafka.go @@ -68,25 +68,17 @@ func (c *KafkaReaderConfig) Validate() error { default: return fmt.Errorf("unsupported kafka start offset %q", c.StartOffset) } - // Mirror the producer-side SASL checks so a misconfigured consumer fails at - // load time instead of with an obscure handshake error on first fetch. - switch strings.ToLower(c.SASLMechanism) { - case "", "none": - case "plain": - if c.Username == "" || c.Password == "" { - return fmt.Errorf("kafka username and password are required for sasl plain") - } - case "aws-msk-iam": - if !c.TLSEnabled { - return fmt.Errorf("kafka tls must be enabled for aws-msk-iam") - } - if c.Region == "" { - return fmt.Errorf("kafka region is required for aws-msk-iam") - } - default: - return fmt.Errorf("unsupported kafka sasl mechanism %q", c.SASLMechanism) + return offload.ValidateSASL(c.saslConfig()) +} + +func (c KafkaReaderConfig) saslConfig() offload.KafkaConfig { + return offload.KafkaConfig{ + Region: c.Region, + TLSEnabled: c.TLSEnabled, + SASLMechanism: c.SASLMechanism, + Username: c.Username, + Password: c.Password, } - return nil } func NewKafkaReader(cfg KafkaReaderConfig) (*kafka.Reader, error) { @@ -102,13 +94,7 @@ func NewKafkaReader(cfg KafkaReaderConfig) (*kafka.Reader, error) { if cfg.TLSEnabled { dialer.TLS = &tls.Config{MinVersion: tls.VersionTLS12} } - mech, err := offload.NewSASLMechanism(offload.KafkaConfig{ - Region: cfg.Region, - TLSEnabled: cfg.TLSEnabled, - SASLMechanism: cfg.SASLMechanism, - Username: cfg.Username, - Password: cfg.Password, - }) + mech, err := offload.NewSASLMechanism(cfg.saslConfig()) if err != nil { return nil, err } diff --git a/sei-db/state_db/ss/offload/consumer/metrics.go b/sei-db/state_db/ss/offload/consumer/metrics.go index f7806f76f0..afcca00702 100644 --- a/sei-db/state_db/ss/offload/consumer/metrics.go +++ b/sei-db/state_db/ss/offload/consumer/metrics.go @@ -13,11 +13,9 @@ import ( // consumer throughput and lag sit alongside the backend cost counters. const consumerMeterName = "seidb_offload_consumer" -// consumerMetrics answers the two questions a cost benchmark needs beyond the -// backend's own counters: how fast is the consumer draining Kafka, and is it -// keeping up. Throughput plus lag tell you whether the backend (not the -// consumer) is the bottleneck. One consumer runs per process, so no attributes -// are needed — Prometheus job/instance labels separate backends. +// consumerMetrics tracks Kafka drain throughput and lag. One consumer runs per +// process, so no attributes are needed — Prometheus job/instance labels +// separate backends. type consumerMetrics struct { recordsProcessed metric.Int64Counter sinkWriteLatency metric.Float64Histogram diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go index 6f0568d8e8..84b521120e 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ b/sei-db/state_db/ss/offload/consumer/scylla.go @@ -35,7 +35,7 @@ type ScyllaConfig struct { } func (c ScyllaConfig) Configured() bool { - return len(c.Hosts) != 0 || c.Keyspace != "" + return c.toHistorical().Configured() } func (c *ScyllaConfig) ApplyDefaults() { @@ -128,12 +128,7 @@ func (s *scyllaSink) WriteBatch(ctx context.Context, records []Record) error { } func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { - entry := rec.Entry - if entry == nil { - return nil - } - version := entry.Version - if err := s.writeRecordRows(ctx, version, entry); err != nil { + if err := s.writeRecordRows(ctx, rec.Entry); err != nil { return err } return s.writeVersionMarker(ctx, rec) @@ -175,7 +170,7 @@ func (s *scyllaSink) writeRecordsPipelined(ctx context.Context, records []Record return rowCtx.Err() } defer func() { <-sem }() - err := s.writeRecordRows(rowCtx, rec.Entry.Version, rec.Entry) + err := s.writeRecordRows(rowCtx, rec.Entry) if err != nil { err = fmt.Errorf("write scylla/cassandra rows version %d: %w", rec.Entry.Version, err) } @@ -198,31 +193,24 @@ func (s *scyllaSink) writeRecordsPipelined(ctx context.Context, records []Record return g.Wait() } -func (s *scyllaSink) writeRecordRows(ctx context.Context, version int64, entry *proto.ChangelogEntry) error { +func (s *scyllaSink) writeRecordRows(ctx context.Context, entry *proto.ChangelogEntry) error { g, gctx := errgroup.WithContext(ctx) - g.SetLimit(s.effectiveMutationWorkers()) + g.SetLimit(s.mutationWorkers) for _, mutation := range compactMutations(entry) { mutation := mutation g.Go(func() error { - return s.writeMutation(gctx, version, mutation.storeName, mutation.pair) + return s.writeMutation(gctx, entry.Version, mutation.storeName, mutation.pair) }) } for _, up := range entry.Upgrades { up := up g.Go(func() error { - return s.writeUpgrade(gctx, version, up) + return s.writeUpgrade(gctx, entry.Version, up) }) } return g.Wait() } -func (s *scyllaSink) effectiveMutationWorkers() int { - if s.mutationWorkers <= 0 { - return defaultScyllaMutationWorkers - } - return s.mutationWorkers -} - func (s *scyllaSink) writeMutation(ctx context.Context, version int64, storeName string, pair *proto.KVPair) error { deleted := pair.Delete || pair.Value == nil value := pair.Value diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go index 7f6584b51b..3a48844c6a 100644 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ b/sei-db/state_db/ss/offload/consumer/scylla_test.go @@ -214,7 +214,7 @@ func TestScyllaSinkCompactsDuplicateMutations(t *testing.T) { }}, } - require.NoError(t, sink.writeRecordRows(context.Background(), entry.Version, entry)) + require.NoError(t, sink.writeRecordRows(context.Background(), entry)) require.Len(t, writes, 3) require.Equal(t, write{value: []byte("new")}, writes["bank/k"]) require.Equal(t, write{deleted: true}, writes["bank/drop"]) diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index ca2eb17e91..845bd98ff2 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -394,7 +394,7 @@ func (r *bigtableReader) Get(ctx context.Context, storeName string, key []byte, if row.Key == "" { return Value{}, ErrNotFound } - return BigtableValueFromRow(row, r.family) + return bigtableValueFromRow(row, r.family) } func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (int64, error) { @@ -407,7 +407,7 @@ func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (in prefix := bigtableVersionRowPrefix(bucket) var bucketVersion int64 err := readRows(gctx, prefix, bigtablePrefixEnd(prefix), 1, "", func(row BigtableRow) bool { - if version, ok := BigtableVersionFromRowKey(row.Key); ok { + if version, ok := bigtableVersionFromRowKey(row.Key); ok { bucketVersion = version } return false @@ -431,10 +431,10 @@ func BigtableLastVersion(ctx context.Context, readRows BigtableReadRowsFunc) (in return maxVersion, nil } -// BigtableValueFromRow interprets a mutation row. The returned Value aliases +// bigtableValueFromRow interprets a mutation row. The returned Value aliases // the row's cell buffer, which the row builder allocates per cell. -func BigtableValueFromRow(row BigtableRow, family string) (Value, error) { - version, ok := BigtableVersionFromRowKey(row.Key) +func bigtableValueFromRow(row BigtableRow, family string) (Value, error) { + version, ok := bigtableVersionFromRowKey(row.Key) if !ok { return Value{}, fmt.Errorf("invalid bigtable mutation row key") } @@ -458,7 +458,7 @@ func BigtableValueFromRow(row BigtableRow, family string) (Value, error) { } func bigtableDeletedFromRow(row BigtableRow, family string) (bool, error) { - if _, ok := BigtableVersionFromRowKey(row.Key); !ok { + if _, ok := bigtableVersionFromRowKey(row.Key); !ok { return false, fmt.Errorf("invalid bigtable mutation row key") } for _, cell := range row.Cells { @@ -469,10 +469,6 @@ func bigtableDeletedFromRow(row BigtableRow, family string) (bool, error) { return false, nil } -func BigtableMutationRowPrefix(storeName string, key []byte, shards int) string { - return string(bigtableMutationRowPrefixBytes(storeName, key, shards)) -} - func bigtableMutationRowPrefixBytes(storeName string, key []byte, shards int) []byte { shards = normalizeBigtableShards(shards) shard := bigtableShard(storeName, key, shards) @@ -521,7 +517,7 @@ func BigtableUpgradeRowKey(version int64, name string) string { return string(key) } -func BigtableVersionFromRowKey(rowKey string) (int64, bool) { +func bigtableVersionFromRowKey(rowKey string) (int64, bool) { key := []byte(rowKey) switch { case len(key) >= 1+2+8 && key[0] == bigtableVersionPrefix: diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go index 0d48d429ba..e82c0d4541 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable_test.go +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -43,12 +43,12 @@ func TestBigtableMutationRowKeyOrdersLatestVersionFirst(t *testing.T) { sort.Strings(keys) require.Equal(t, []string{key80, key60, key40}, keys) - version, ok := BigtableVersionFromRowKey(key60) + version, ok := bigtableVersionFromRowKey(key60) require.True(t, ok) require.Equal(t, int64(60), version) require.NotEqual(t, - BigtableMutationRowPrefix("bank", []byte("k"), 256), - BigtableMutationRowPrefix("bank", []byte("k1"), 256), + bigtableMutationRowPrefixBytes("bank", []byte("k"), 256), + bigtableMutationRowPrefixBytes("bank", []byte("k1"), 256), ) } @@ -61,13 +61,13 @@ func TestBigtableValueFromRow(t *testing.T) { {Family: DefaultBigtableFamily, Qualifier: BigtableDeletedColumn, Value: []byte{0}}, }, } - value, err := BigtableValueFromRow(row, DefaultBigtableFamily) + value, err := bigtableValueFromRow(row, DefaultBigtableFamily) require.NoError(t, err) require.Equal(t, []byte("value"), value.Bytes) require.Equal(t, int64(7), value.Version) row.Cells[1].Value = []byte{1} - _, err = BigtableValueFromRow(row, DefaultBigtableFamily) + _, err = bigtableValueFromRow(row, DefaultBigtableFamily) require.ErrorIs(t, err, ErrNotFound) } diff --git a/sei-db/state_db/ss/offload/historical/scylla.go b/sei-db/state_db/ss/offload/historical/scylla.go index 9855b658db..562cd5c192 100644 --- a/sei-db/state_db/ss/offload/historical/scylla.go +++ b/sei-db/state_db/ss/offload/historical/scylla.go @@ -49,6 +49,10 @@ func (c *ScyllaConfig) ApplyDefaults() { } } +func (c ScyllaConfig) Configured() bool { + return len(c.Hosts) != 0 || strings.TrimSpace(c.Keyspace) != "" +} + func (c *ScyllaConfig) Validate() error { if len(c.Hosts) == 0 { return fmt.Errorf("scylla/cassandra hosts are required") diff --git a/sei-db/state_db/ss/offload/historical/scylla_test.go b/sei-db/state_db/ss/offload/historical/scylla_test.go index 567dc5f960..9e483877c7 100644 --- a/sei-db/state_db/ss/offload/historical/scylla_test.go +++ b/sei-db/state_db/ss/offload/historical/scylla_test.go @@ -1,8 +1,6 @@ package historical import ( - "reflect" - "strings" "testing" "time" @@ -67,22 +65,6 @@ func TestParseConsistency(t *testing.T) { } } -func TestScyllaHostSelectionPolicyIsTokenAware(t *testing.T) { - for _, tc := range []struct { - name string - datacenter string - }{ - {"no datacenter", ""}, - {"with datacenter", "dc1"}, - } { - t.Run(tc.name, func(t *testing.T) { - policy := scyllaHostSelectionPolicy(tc.datacenter) - require.NotNil(t, policy) - require.Contains(t, reflect.TypeOf(policy).String(), "tokenAware") - }) - } -} - func TestVersionBucket(t *testing.T) { require.Equal(t, 0, VersionBucket(0)) require.Equal(t, 1, VersionBucket(1)) @@ -108,5 +90,5 @@ func TestPointLookupCQLShape(t *testing.T) { func TestLatestVersionCQLShape(t *testing.T) { require.Contains(t, selectLatestVersionCQL, "FROM state_versions") require.Contains(t, selectLatestVersionCQL, "bucket = ?") - require.True(t, strings.Contains(selectLatestVersionCQL, "LIMIT 1")) + require.Contains(t, selectLatestVersionCQL, "LIMIT 1") } diff --git a/sei-db/state_db/ss/offload/historical/store.go b/sei-db/state_db/ss/offload/historical/store.go index be8c7e4030..b74452bd37 100644 --- a/sei-db/state_db/ss/offload/historical/store.go +++ b/sei-db/state_db/ss/offload/historical/store.go @@ -19,32 +19,32 @@ const ( // Cache entries and the per-value cap bound worst-case memory at // entries*cap (256 MiB) even when an external caller crafts distinct // large-value historical queries. - defaultHistoricalReadCacheEntries = 32 * 1024 - maxHistoricalReadCacheValueBytes = 8 * 1024 + defaultReadCacheEntries = 32 * 1024 + maxReadCacheValueBytes = 8 * 1024 - // historicalReadTimeout bounds one backend point read. types.StateStore has + // readTimeout bounds one backend point read. types.StateStore has // no context parameter, so the deadline must be injected here; without it a // silently dropped connection can park an RPC goroutine in a backend read // until the OS TCP timeout. - historicalReadTimeout = 10 * time.Second + readTimeout = 10 * time.Second - // historicalMissCacheTTL bounds how long a backend miss is trusted. Hits are + // missCacheTTL bounds how long a backend miss is trusted. Hits are // immutable (a version's value never changes once written) but a miss can // flip to a hit once the offload consumer catches up. - historicalMissCacheTTL = time.Minute + missCacheTTL = time.Minute // backendVersionRecheckInterval rate-limits LastVersion refreshes when a // requested version is ahead of the backend's cached ingestion watermark. backendVersionRecheckInterval = 30 * time.Second ) -type historicalReadCacheKey struct { +type readCacheKey struct { storeKey string version int64 key string } -type historicalReadCacheValue struct { +type readCacheValue struct { value []byte found bool valueKnown bool @@ -60,24 +60,13 @@ type PerKeyEarliestVersioner interface { GetEarliestVersionForKey(storeKey string) int64 } -// FallbackOptions tunes FallbackStateStore behavior. -type FallbackOptions struct { - // EarliestVersion is the operator-declared earliest version (inclusive) - // fully ingested into the historical backend. When set (> 0) it becomes the - // store's advertised earliest version, so height gates such as the EVM RPC - // watermark admit pruned heights that the fallback can serve; reads below - // it stay on the primary. When zero, the advertised earliest remains the - // local prune horizon and height gates keep rejecting pruned heights. - EarliestVersion int64 -} - // FallbackStateStore routes pruned point reads to the historical reader. // Iteration and writes stay on the primary state store. type FallbackStateStore struct { primary types.StateStore perKeyEarliest PerKeyEarliestVersioner reader Reader - cache *lru.Cache[historicalReadCacheKey, historicalReadCacheValue] + cache *lru.Cache[readCacheKey, readCacheValue] metrics *fallbackMetrics coverageFloor int64 @@ -91,8 +80,13 @@ type FallbackStateStore struct { var _ types.StateStore = (*FallbackStateStore)(nil) // NewFallbackStateStore takes ownership of primary and reader for Close. -func NewFallbackStateStore(primary types.StateStore, reader Reader, opts FallbackOptions) *FallbackStateStore { - cache, err := lru.New[historicalReadCacheKey, historicalReadCacheValue](defaultHistoricalReadCacheEntries) +// coverageFloor is the operator-declared earliest version (inclusive) fully +// ingested into the backend: when > 0 it becomes the store's advertised +// earliest version, so height gates such as the EVM RPC watermark admit +// pruned heights the fallback can serve, while reads below it stay on the +// primary. When zero, the advertised earliest remains the local prune horizon. +func NewFallbackStateStore(primary types.StateStore, reader Reader, coverageFloor int64) *FallbackStateStore { + cache, err := lru.New[readCacheKey, readCacheValue](defaultReadCacheEntries) if err != nil { panic(err) } @@ -103,7 +97,7 @@ func NewFallbackStateStore(primary types.StateStore, reader Reader, opts Fallbac reader: reader, cache: cache, metrics: newFallbackMetrics(), - coverageFloor: opts.EarliestVersion, + coverageFloor: coverageFloor, } } @@ -164,14 +158,14 @@ func (s *FallbackStateStore) earliestVersionFor(storeKey string) int64 { } func (s *FallbackStateStore) readContext() (context.Context, context.CancelFunc) { - return context.WithTimeout(context.Background(), historicalReadTimeout) + return context.WithTimeout(context.Background(), readTimeout) } func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([]byte, error) { if !s.shouldFallback(storeKey, version) { return s.primary.Get(storeKey, version, key) } - cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} + cacheKey := readCacheKey{storeKey: storeKey, version: version, key: string(key)} if value, found, ok := s.getCachedValue(cacheKey); ok { s.metrics.recordRead("get", fallbackOutcomeCacheHit) if !found { @@ -200,10 +194,7 @@ func (s *FallbackStateStore) Get(storeKey string, version int64, key []byte) ([] return v.Bytes, nil } -func (s *FallbackStateStore) getCachedValue(key historicalReadCacheKey) ([]byte, bool, bool) { - if s.cache == nil { - return nil, false, false - } +func (s *FallbackStateStore) getCachedValue(key readCacheKey) ([]byte, bool, bool) { value, ok := s.cache.Get(key) if !ok { return nil, false, false @@ -220,10 +211,7 @@ func (s *FallbackStateStore) getCachedValue(key historicalReadCacheKey) ([]byte, return bytes.Clone(value.value), true, true } -func (s *FallbackStateStore) getCachedHas(key historicalReadCacheKey) (bool, bool) { - if s.cache == nil { - return false, false - } +func (s *FallbackStateStore) getCachedHas(key readCacheKey) (bool, bool) { value, ok := s.cache.Get(key) if !ok { return false, false @@ -236,7 +224,7 @@ func (s *FallbackStateStore) getCachedHas(key historicalReadCacheKey) (bool, boo // missExpired evicts and reports miss entries whose TTL has lapsed so a // backend that has since ingested the version gets re-queried. -func (s *FallbackStateStore) missExpired(key historicalReadCacheKey, value historicalReadCacheValue) bool { +func (s *FallbackStateStore) missExpired(key readCacheKey, value readCacheValue) bool { if value.missExpiresAt.IsZero() || time.Now().Before(value.missExpiresAt) { return false } @@ -244,32 +232,26 @@ func (s *FallbackStateStore) missExpired(key historicalReadCacheKey, value histo return true } -func (s *FallbackStateStore) cacheValue(key historicalReadCacheKey, value []byte) { - if s.cache == nil || value == nil || len(value) > maxHistoricalReadCacheValueBytes { +func (s *FallbackStateStore) cacheValue(key readCacheKey, value []byte) { + if value == nil || len(value) > maxReadCacheValueBytes { return } - s.cache.Add(key, historicalReadCacheValue{value: bytes.Clone(value), found: true, valueKnown: true}) + s.cache.Add(key, readCacheValue{value: bytes.Clone(value), found: true, valueKnown: true}) } -func (s *FallbackStateStore) cacheMiss(key historicalReadCacheKey) { - if s.cache == nil { - return - } - s.cache.Add(key, historicalReadCacheValue{valueKnown: true, missExpiresAt: time.Now().Add(historicalMissCacheTTL)}) +func (s *FallbackStateStore) cacheMiss(key readCacheKey) { + s.cache.Add(key, readCacheValue{valueKnown: true, missExpiresAt: time.Now().Add(missCacheTTL)}) } -func (s *FallbackStateStore) cacheHas(key historicalReadCacheKey) { - if s.cache == nil { - return - } - s.cache.Add(key, historicalReadCacheValue{found: true}) +func (s *FallbackStateStore) cacheHas(key readCacheKey) { + s.cache.Add(key, readCacheValue{found: true}) } func (s *FallbackStateStore) Has(storeKey string, version int64, key []byte) (bool, error) { if !s.shouldFallback(storeKey, version) { return s.primary.Has(storeKey, version, key) } - cacheKey := historicalReadCacheKey{storeKey: storeKey, version: version, key: string(key)} + cacheKey := readCacheKey{storeKey: storeKey, version: version, key: string(key)} if found, ok := s.getCachedHas(cacheKey); ok { s.metrics.recordRead("has", fallbackOutcomeCacheHit) return found, nil diff --git a/sei-db/state_db/ss/offload/historical/store_test.go b/sei-db/state_db/ss/offload/historical/store_test.go index 181c5053ed..d57f7f0b28 100644 --- a/sei-db/state_db/ss/offload/historical/store_test.go +++ b/sei-db/state_db/ss/offload/historical/store_test.go @@ -93,7 +93,7 @@ func (f *fakeReader) Close() error { return nil } func TestFallbackStateStoreRoutesPrunedPointReads(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) value, err := store.Get("bank", 7, []byte("k")) require.NoError(t, err) @@ -111,7 +111,7 @@ func TestFallbackStateStoreRoutesPrunedPointReads(t *testing.T) { func TestFallbackStateStoreKeepsRecentPointReadsOnPrimary(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) value, err := store.Get("bank", 10, []byte("k")) require.NoError(t, err) @@ -123,7 +123,7 @@ func TestFallbackStateStoreKeepsRecentPointReadsOnPrimary(t *testing.T) { func TestFallbackStateStoreCachesHistoricalPointReads(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) value, err := store.Get("bank", 7, []byte("k")) require.NoError(t, err) @@ -144,7 +144,7 @@ func TestFallbackStateStoreCachesHistoricalPointReads(t *testing.T) { func TestFallbackStateStoreCachesHistoricalMisses(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{getErr: ErrNotFound, hasSet: true} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) value, err := store.Get("bank", 7, []byte("missing")) require.NoError(t, err) @@ -164,7 +164,7 @@ func TestFallbackStateStoreCachesHistoricalMisses(t *testing.T) { func TestFallbackStateStoreCachesHistoricalHasResults(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{hasResult: true, hasSet: true} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) ok, err := store.Has("bank", 7, []byte("k")) require.NoError(t, err) @@ -179,7 +179,7 @@ func TestFallbackStateStoreCachesHistoricalHasResults(t *testing.T) { func TestFallbackStateStoreDoesNotUseHasOnlyCacheForGet(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{hasResult: true, hasSet: true} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) ok, err := store.Has("bank", 7, []byte("k")) require.NoError(t, err) @@ -209,7 +209,7 @@ func TestFallbackStateStoreUsesPerKeyEarliestVersion(t *testing.T) { perKey: map[string]int64{"evm": 5}, } reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) // The EVM store still holds version 7 locally even though the cosmos store // pruned to 10; the read must stay on the primary. @@ -235,7 +235,7 @@ func TestFallbackStateStoreUsesPerKeyEarliestVersion(t *testing.T) { func TestFallbackStateStoreCoverageFloor(t *testing.T) { primary := &fakeStateStore{earliest: 100} reader := &fakeReader{} - store := NewFallbackStateStore(primary, reader, FallbackOptions{EarliestVersion: 50}) + store := NewFallbackStateStore(primary, reader, 50) // The floor becomes the advertised earliest so height gates admit pruned // heights the fallback can serve. @@ -262,7 +262,7 @@ func TestFallbackStateStoreCoverageFloor(t *testing.T) { func TestFallbackStateStoreRejectsReadsBeyondBackendCoverage(t *testing.T) { primary := &fakeStateStore{earliest: 100} reader := &fakeReader{lastVersion: 40} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) // The backend has only ingested up to 40; a pruned read at 90 must error // instead of returning silently-empty state. @@ -288,7 +288,7 @@ func TestFallbackStateStoreRejectsReadsBeyondBackendCoverage(t *testing.T) { func TestFallbackStateStoreExpiresCachedMisses(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{getErr: ErrNotFound} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) value, err := store.Get("bank", 7, []byte("missing")) require.NoError(t, err) @@ -297,7 +297,7 @@ func TestFallbackStateStoreExpiresCachedMisses(t *testing.T) { // Backdate the cached miss to simulate the TTL lapsing after the offload // consumer catches up. - cacheKey := historicalReadCacheKey{storeKey: "bank", version: 7, key: "missing"} + cacheKey := readCacheKey{storeKey: "bank", version: 7, key: "missing"} entry, ok := store.cache.Get(cacheKey) require.True(t, ok) entry.missExpiresAt = time.Now().Add(-time.Second) @@ -313,7 +313,7 @@ func TestFallbackStateStoreExpiresCachedMisses(t *testing.T) { func TestFallbackStateStoreDoesNotCacheHistoricalErrors(t *testing.T) { primary := &fakeStateStore{earliest: 10} reader := &fakeReader{getErr: errors.New("boom")} - store := NewFallbackStateStore(primary, reader, FallbackOptions{}) + store := NewFallbackStateStore(primary, reader, 0) _, err := store.Get("bank", 7, []byte("k")) require.Error(t, err) diff --git a/sei-db/state_db/ss/offload/kafka.go b/sei-db/state_db/ss/offload/kafka.go index b870177959..8bce216c6a 100644 --- a/sei-db/state_db/ss/offload/kafka.go +++ b/sei-db/state_db/ss/offload/kafka.go @@ -89,25 +89,29 @@ func (c *KafkaConfig) Validate() error { return fmt.Errorf("unsupported kafka compression %q", c.Compression) } - switch strings.ToLower(c.SASLMechanism) { + return ValidateSASL(*c) +} + +// ValidateSASL checks the SASL mechanism and the credentials it requires. +// Shared by the producer and the offload consumer configs. +func ValidateSASL(cfg KafkaConfig) error { + switch strings.ToLower(cfg.SASLMechanism) { case "", kafkaOptionNone: - return nil case "plain": - if c.Username == "" || c.Password == "" { + if cfg.Username == "" || cfg.Password == "" { return fmt.Errorf("kafka username and password are required for sasl plain") } - return nil case "aws-msk-iam": - if !c.TLSEnabled { + if !cfg.TLSEnabled { return fmt.Errorf("kafka tls must be enabled for aws-msk-iam") } - if c.Region == "" { + if cfg.Region == "" { return fmt.Errorf("kafka region is required for aws-msk-iam") } - return nil default: - return fmt.Errorf("unsupported kafka sasl mechanism %q", c.SASLMechanism) + return fmt.Errorf("unsupported kafka sasl mechanism %q", cfg.SASLMechanism) } + return nil } type kafkaStream struct { @@ -222,15 +226,13 @@ func kafkaCompression(name string) compress.Compression { } } -// NewSASLMechanism is exported so out-of-package consumers share the producer's auth path. +// NewSASLMechanism returns the SASL mechanism for cfg, which must already +// have passed ValidateSASL. func NewSASLMechanism(cfg KafkaConfig) (sasl.Mechanism, error) { switch strings.ToLower(cfg.SASLMechanism) { case "", kafkaOptionNone: return nil, nil case "plain": - if cfg.Username == "" || cfg.Password == "" { - return nil, fmt.Errorf("kafka username and password are required for sasl plain") - } return plain.Mechanism{Username: cfg.Username, Password: cfg.Password}, nil case "aws-msk-iam": return newAWSMSKIAMMechanism(cfg) diff --git a/sei-db/state_db/ss/store.go b/sei-db/state_db/ss/store.go index fa2c25add4..6d79cf0000 100644 --- a/sei-db/state_db/ss/store.go +++ b/sei-db/state_db/ss/store.go @@ -20,58 +20,44 @@ func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.Stat if err != nil { return nil, err } - scyllaConfigured := scyllaHistoricalOffloadConfigured(ssConfig) - bigtableConfigured := bigtableHistoricalOffloadConfigured(ssConfig) - if scyllaConfigured && bigtableConfigured { + scyllaCfg := historical.ScyllaConfig{ + Hosts: splitCSV(ssConfig.HistoricalOffloadScyllaHosts), + Keyspace: ssConfig.HistoricalOffloadScyllaKeyspace, + Username: ssConfig.HistoricalOffloadScyllaUsername, + Password: ssConfig.HistoricalOffloadScyllaPassword, + Datacenter: ssConfig.HistoricalOffloadScyllaDatacenter, + Consistency: ssConfig.HistoricalOffloadScyllaConsistency, + Timeout: time.Duration(ssConfig.HistoricalOffloadScyllaTimeoutMS) * time.Millisecond, + } + bigtableCfg := historical.BigtableConfig{ + ProjectID: ssConfig.HistoricalOffloadBigtableProjectID, + InstanceID: ssConfig.HistoricalOffloadBigtableInstance, + Table: ssConfig.HistoricalOffloadBigtableTable, + Family: ssConfig.HistoricalOffloadBigtableFamily, + AppProfile: ssConfig.HistoricalOffloadBigtableAppProfile, + Shards: ssConfig.HistoricalOffloadBigtableShards, + } + if scyllaCfg.Configured() && bigtableCfg.Configured() { _ = primary.Close() return nil, fmt.Errorf("only one historical offload fallback can be configured") } - if !scyllaConfigured && !bigtableConfigured { + if !scyllaCfg.Configured() && !bigtableCfg.Configured() { return primary, nil } - fallbackOpts := historical.FallbackOptions{ - EarliestVersion: ssConfig.HistoricalOffloadEarliestVersion, - } - if bigtableConfigured { - reader, err := historical.NewBigtableReader(historical.BigtableConfig{ - ProjectID: ssConfig.HistoricalOffloadBigtableProjectID, - InstanceID: ssConfig.HistoricalOffloadBigtableInstance, - Table: ssConfig.HistoricalOffloadBigtableTable, - Family: ssConfig.HistoricalOffloadBigtableFamily, - AppProfile: ssConfig.HistoricalOffloadBigtableAppProfile, - Shards: ssConfig.HistoricalOffloadBigtableShards, - }) + if bigtableCfg.Configured() { + reader, err := historical.NewBigtableReader(bigtableCfg) if err != nil { _ = primary.Close() return nil, fmt.Errorf("open bigtable historical offload reader: %w", err) } - return historical.NewFallbackStateStore(primary, reader, fallbackOpts), nil + return historical.NewFallbackStateStore(primary, reader, ssConfig.HistoricalOffloadEarliestVersion), nil } - reader, err := historical.NewScyllaReader(historical.ScyllaConfig{ - Hosts: splitCSV(ssConfig.HistoricalOffloadScyllaHosts), - Keyspace: ssConfig.HistoricalOffloadScyllaKeyspace, - Username: ssConfig.HistoricalOffloadScyllaUsername, - Password: ssConfig.HistoricalOffloadScyllaPassword, - Datacenter: ssConfig.HistoricalOffloadScyllaDatacenter, - Consistency: ssConfig.HistoricalOffloadScyllaConsistency, - Timeout: time.Duration(ssConfig.HistoricalOffloadScyllaTimeoutMS) * time.Millisecond, - }) + reader, err := historical.NewScyllaReader(scyllaCfg) if err != nil { _ = primary.Close() return nil, fmt.Errorf("open scylla/cassandra historical offload reader: %w", err) } - return historical.NewFallbackStateStore(primary, reader, fallbackOpts), nil -} - -func scyllaHistoricalOffloadConfigured(cfg config.StateStoreConfig) bool { - return strings.TrimSpace(cfg.HistoricalOffloadScyllaHosts) != "" || - strings.TrimSpace(cfg.HistoricalOffloadScyllaKeyspace) != "" -} - -func bigtableHistoricalOffloadConfigured(cfg config.StateStoreConfig) bool { - return strings.TrimSpace(cfg.HistoricalOffloadBigtableProjectID) != "" || - strings.TrimSpace(cfg.HistoricalOffloadBigtableInstance) != "" || - strings.TrimSpace(cfg.HistoricalOffloadBigtableTable) != "" + return historical.NewFallbackStateStore(primary, reader, ssConfig.HistoricalOffloadEarliestVersion), nil } func splitCSV(value string) []string { From 44c2b26ef06f0986f923db22b96f46340db330dc Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 11:04:13 -0400 Subject: [PATCH 38/41] Fail loudly on garbage Scylla hosts config Restore the pre-cleanup behavior where a hosts value that parses to no usable endpoints (e.g. only commas) reaches reader validation and fails startup instead of silently running without the fallback. Also cover the earliest-version flag in the app-opts test switch. Co-Authored-By: Claude Fable 5 --- app/seidb_test.go | 2 ++ sei-db/state_db/ss/store.go | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/seidb_test.go b/app/seidb_test.go index da91bb413e..5ec478cf9a 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -91,6 +91,8 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSSConfig.HistoricalOffloadBigtableAppProfile case FlagHistoricalOffloadBigtableShards: return defaultSSConfig.HistoricalOffloadBigtableShards + case FlagHistoricalOffloadEarliestVersion: + return defaultSSConfig.HistoricalOffloadEarliestVersion } return nil } diff --git a/sei-db/state_db/ss/store.go b/sei-db/state_db/ss/store.go index 6d79cf0000..68611e51e7 100644 --- a/sei-db/state_db/ss/store.go +++ b/sei-db/state_db/ss/store.go @@ -37,11 +37,15 @@ func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.Stat AppProfile: ssConfig.HistoricalOffloadBigtableAppProfile, Shards: ssConfig.HistoricalOffloadBigtableShards, } - if scyllaCfg.Configured() && bigtableCfg.Configured() { + // Scylla's configured check uses the raw hosts string so a garbage value + // (e.g. only commas) fails reader validation loudly instead of silently + // running without the fallback. + scyllaConfigured := strings.TrimSpace(ssConfig.HistoricalOffloadScyllaHosts) != "" || scyllaCfg.Configured() + if scyllaConfigured && bigtableCfg.Configured() { _ = primary.Close() return nil, fmt.Errorf("only one historical offload fallback can be configured") } - if !scyllaCfg.Configured() && !bigtableCfg.Configured() { + if !scyllaConfigured && !bigtableCfg.Configured() { return primary, nil } if bigtableCfg.Configured() { From cbfb4508f3583fc4c17a0d0556a0626f5f6c24b7 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 16:44:23 -0400 Subject: [PATCH 39/41] Remove the Scylla backend; the offload is Bigtable-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scylla was the original prototype backend; production only ever ran Bigtable via Google Cloud Managed Kafka. Delete the Scylla reader, sink, schema, example config, node config plumbing, and the gocql dependency. The shared version-bucket helpers move to bigtable.go (values unchanged — the marker row encoding is live production data), the consumer config loses its backend selector, and compactRecords/compactMutations coverage moves to a dedicated compact_test.go. Co-Authored-By: Claude Fable 5 --- app/seidb.go | 14 - app/seidb_test.go | 38 -- go.mod | 3 - go.sum | 10 - sei-db/config/ss_config.go | 21 - sei-db/config/toml.go | 14 +- sei-db/config/toml_test.go | 4 - sei-db/state_db/ss/offload/consumer/README.md | 76 +--- .../cmd/historical-offload-consumer/main.go | 4 +- .../ss/offload/consumer/compact_test.go | 65 +++ sei-db/state_db/ss/offload/consumer/config.go | 66 +-- .../consumer/config/example-bigtable.json | 1 - .../consumer/config/example-scylla.json | 22 - .../ss/offload/consumer/config_test.go | 60 +-- .../ss/offload/consumer/kafka_test.go | 2 +- .../ss/offload/consumer/schema/scylla.cql | 46 -- sei-db/state_db/ss/offload/consumer/scylla.go | 265 ----------- .../ss/offload/consumer/scylla_test.go | 414 ------------------ .../ss/offload/historical/bigtable.go | 12 + .../state_db/ss/offload/historical/scylla.go | 257 ----------- .../ss/offload/historical/scylla_test.go | 94 ---- sei-db/state_db/ss/store.go | 50 +-- 22 files changed, 124 insertions(+), 1414 deletions(-) create mode 100644 sei-db/state_db/ss/offload/consumer/compact_test.go delete mode 100644 sei-db/state_db/ss/offload/consumer/config/example-scylla.json delete mode 100644 sei-db/state_db/ss/offload/consumer/schema/scylla.cql delete mode 100644 sei-db/state_db/ss/offload/consumer/scylla.go delete mode 100644 sei-db/state_db/ss/offload/consumer/scylla_test.go delete mode 100644 sei-db/state_db/ss/offload/historical/scylla.go delete mode 100644 sei-db/state_db/ss/offload/historical/scylla_test.go diff --git a/app/seidb.go b/app/seidb.go index a18b228ad7..a159c0cde5 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -55,13 +55,6 @@ const ( FlagEVMSSSeparateDBs = "state-store.evm-ss-separate-dbs" // Historical SS offload fallback. - FlagHistoricalOffloadScyllaHosts = "state-store.historical-offload-scylla-hosts" - FlagHistoricalOffloadScyllaKeyspace = "state-store.historical-offload-scylla-keyspace" - FlagHistoricalOffloadScyllaUsername = "state-store.historical-offload-scylla-username" - FlagHistoricalOffloadScyllaPassword = "state-store.historical-offload-scylla-password" - FlagHistoricalOffloadScyllaDatacenter = "state-store.historical-offload-scylla-datacenter" - FlagHistoricalOffloadScyllaConsistency = "state-store.historical-offload-scylla-consistency" - FlagHistoricalOffloadScyllaTimeoutMS = "state-store.historical-offload-scylla-timeout-ms" FlagHistoricalOffloadBigtableProjectID = "state-store.historical-offload-bigtable-project-id" FlagHistoricalOffloadBigtableInstance = "state-store.historical-offload-bigtable-instance" FlagHistoricalOffloadBigtableTable = "state-store.historical-offload-bigtable-table" @@ -224,13 +217,6 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig { ssConfig.EVMDBDirectory = cast.ToString(appOpts.Get(FlagEVMSSDirectory)) ssConfig.SeparateEVMSubDBs = cast.ToBool(appOpts.Get(FlagEVMSSSeparateDBs)) ssConfig.EVMSplit = cast.ToBool(appOpts.Get(FlagEVMSSSplit)) - ssConfig.HistoricalOffloadScyllaHosts = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaHosts)) - ssConfig.HistoricalOffloadScyllaKeyspace = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaKeyspace)) - ssConfig.HistoricalOffloadScyllaUsername = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaUsername)) - ssConfig.HistoricalOffloadScyllaPassword = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaPassword)) - ssConfig.HistoricalOffloadScyllaDatacenter = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaDatacenter)) - ssConfig.HistoricalOffloadScyllaConsistency = cast.ToString(appOpts.Get(FlagHistoricalOffloadScyllaConsistency)) - ssConfig.HistoricalOffloadScyllaTimeoutMS = cast.ToInt(appOpts.Get(FlagHistoricalOffloadScyllaTimeoutMS)) ssConfig.HistoricalOffloadBigtableProjectID = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableProjectID)) ssConfig.HistoricalOffloadBigtableInstance = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableInstance)) ssConfig.HistoricalOffloadBigtableTable = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableTable)) diff --git a/app/seidb_test.go b/app/seidb_test.go index 5ec478cf9a..09b9252b98 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -65,20 +65,6 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSSConfig.EVMSplit case FlagEVMSSSeparateDBs: return defaultSSConfig.SeparateEVMSubDBs - case FlagHistoricalOffloadScyllaHosts: - return defaultSSConfig.HistoricalOffloadScyllaHosts - case FlagHistoricalOffloadScyllaKeyspace: - return defaultSSConfig.HistoricalOffloadScyllaKeyspace - case FlagHistoricalOffloadScyllaUsername: - return defaultSSConfig.HistoricalOffloadScyllaUsername - case FlagHistoricalOffloadScyllaPassword: - return defaultSSConfig.HistoricalOffloadScyllaPassword - case FlagHistoricalOffloadScyllaDatacenter: - return defaultSSConfig.HistoricalOffloadScyllaDatacenter - case FlagHistoricalOffloadScyllaConsistency: - return defaultSSConfig.HistoricalOffloadScyllaConsistency - case FlagHistoricalOffloadScyllaTimeoutMS: - return defaultSSConfig.HistoricalOffloadScyllaTimeoutMS case FlagHistoricalOffloadBigtableProjectID: return defaultSSConfig.HistoricalOffloadBigtableProjectID case FlagHistoricalOffloadBigtableInstance: @@ -232,30 +218,6 @@ func TestParseSSConfigs_EVMFlags(t *testing.T) { assert.True(t, ssConfig.SeparateEVMSubDBs) } -func TestParseSSConfigs_HistoricalScyllaFlags(t *testing.T) { - appOpts := mapAppOpts{ - FlagSSEnable: true, - FlagHistoricalOffloadScyllaHosts: "10.0.0.1:9042,10.0.0.2:9042", - FlagHistoricalOffloadScyllaKeyspace: "sei_history", - FlagHistoricalOffloadScyllaUsername: "sei", - FlagHistoricalOffloadScyllaPassword: "secret", - FlagHistoricalOffloadScyllaDatacenter: "use1", - FlagHistoricalOffloadScyllaConsistency: "local_quorum", - FlagHistoricalOffloadScyllaTimeoutMS: 1500, - FlagSSAsyncWriterBuffer: 0, - } - - ssConfig := parseSSConfigs(appOpts) - assert.True(t, ssConfig.Enable) - assert.Equal(t, "10.0.0.1:9042,10.0.0.2:9042", ssConfig.HistoricalOffloadScyllaHosts) - assert.Equal(t, "sei_history", ssConfig.HistoricalOffloadScyllaKeyspace) - assert.Equal(t, "sei", ssConfig.HistoricalOffloadScyllaUsername) - assert.Equal(t, "secret", ssConfig.HistoricalOffloadScyllaPassword) - assert.Equal(t, "use1", ssConfig.HistoricalOffloadScyllaDatacenter) - assert.Equal(t, "local_quorum", ssConfig.HistoricalOffloadScyllaConsistency) - assert.Equal(t, 1500, ssConfig.HistoricalOffloadScyllaTimeoutMS) -} - func TestParseSSConfigs_HistoricalBigtableFlags(t *testing.T) { appOpts := mapAppOpts{ FlagSSEnable: true, diff --git a/go.mod b/go.mod index 1959616b78..3136e045f3 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/ethereum/go-ethereum v1.16.8 github.com/fortytw2/leaktest v1.3.0 github.com/go-git/go-git/v5 v5.17.2 - github.com/gocql/gocql v1.7.0 github.com/gofrs/flock v0.13.0 github.com/gogo/gateway v1.1.0 github.com/gogo/protobuf v1.3.3 @@ -118,7 +117,6 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.8.0 // indirect - github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -130,7 +128,6 @@ require ( github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index 549335535c..3aa7bf9eb9 100644 --- a/go.sum +++ b/go.sum @@ -733,8 +733,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E= github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.14.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.17.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= @@ -742,8 +740,6 @@ github.com/bits-and-blooms/bitset v1.24.3 h1:Bte86SlO3lwPQqww+7BE9ZuUCKIjfqnG5jt github.com/bits-and-blooms/bitset v1.24.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/btcsuite/btcd v0.23.2 h1:/YOgUp25sdCnP5ho6Hl3s0E438zlX+Kak7E6TgBgoT0= @@ -1101,8 +1097,6 @@ github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= -github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -1309,8 +1303,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/guptarohit/asciigraph v0.5.5/go.mod h1:dYl5wwK4gNsnFf9Zp+l06rFiDZ5YtXM6x7SRWZ3KGag= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -2878,8 +2870,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index b0bc38c746..8270f6cd2b 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -81,27 +81,6 @@ type StateStoreConfig struct { // preserving the same logical store key and full key encoding inside each DB. SeparateEVMSubDBs bool `mapstructure:"evm-separate-dbs"` - // HistoricalOffloadScyllaHosts enables ScyllaDB/Cassandra fallback reads - // for versions pruned from local SS when non-empty. Hosts are comma-separated - // host[:port] values. - HistoricalOffloadScyllaHosts string `mapstructure:"historical-offload-scylla-hosts"` - - // HistoricalOffloadScyllaKeyspace is the keyspace containing state_mutations. - HistoricalOffloadScyllaKeyspace string `mapstructure:"historical-offload-scylla-keyspace"` - - // HistoricalOffloadScyllaUsername and Password are optional. - HistoricalOffloadScyllaUsername string `mapstructure:"historical-offload-scylla-username"` - HistoricalOffloadScyllaPassword string `mapstructure:"historical-offload-scylla-password"` - - // HistoricalOffloadScyllaDatacenter enables DC-aware routing when set. - HistoricalOffloadScyllaDatacenter string `mapstructure:"historical-offload-scylla-datacenter"` - - // HistoricalOffloadScyllaConsistency defaults to local_quorum when empty. - HistoricalOffloadScyllaConsistency string `mapstructure:"historical-offload-scylla-consistency"` - - // HistoricalOffloadScyllaTimeoutMS defaults in the Scylla reader when zero. - HistoricalOffloadScyllaTimeoutMS int `mapstructure:"historical-offload-scylla-timeout-ms"` - // HistoricalOffloadBigtableProjectID enables Bigtable fallback reads when // project, instance, and table are set. HistoricalOffloadBigtableProjectID string `mapstructure:"historical-offload-bigtable-project-id"` diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index caa5df504d..37a2b0579d 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -155,20 +155,10 @@ evm-ss-split = {{ .StateStore.EVMSplit }} # When true, data is routed to separate DBs while preserving the same evm key prefix format. evm-ss-separate-dbs = {{ .StateStore.SeparateEVMSubDBs }} -# Optional ScyllaDB/Cassandra historical-state fallback. When hosts are set, -# point reads for versions pruned from local SS fall back to state_mutations in -# the configured keyspace. Iterators still use local SS. -historical-offload-scylla-hosts = "{{ .StateStore.HistoricalOffloadScyllaHosts }}" -historical-offload-scylla-keyspace = "{{ .StateStore.HistoricalOffloadScyllaKeyspace }}" -historical-offload-scylla-username = "{{ .StateStore.HistoricalOffloadScyllaUsername }}" -historical-offload-scylla-password = "{{ .StateStore.HistoricalOffloadScyllaPassword }}" -historical-offload-scylla-datacenter = "{{ .StateStore.HistoricalOffloadScyllaDatacenter }}" -historical-offload-scylla-consistency = "{{ .StateStore.HistoricalOffloadScyllaConsistency }}" -historical-offload-scylla-timeout-ms = {{ .StateStore.HistoricalOffloadScyllaTimeoutMS }} - # Optional Bigtable historical-state fallback. When project, instance, and # table are set, point reads for versions pruned from local SS fall back to -# Bigtable. Use the same family/shards as the Bigtable consumer. +# Bigtable. Iterators still use local SS. Use the same family/shards as the +# Bigtable consumer. historical-offload-bigtable-project-id = "{{ .StateStore.HistoricalOffloadBigtableProjectID }}" historical-offload-bigtable-instance = "{{ .StateStore.HistoricalOffloadBigtableInstance }}" historical-offload-bigtable-table = "{{ .StateStore.HistoricalOffloadBigtableTable }}" diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index c1594de05c..8fa681d5a2 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -99,10 +99,6 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, `evm-ss-db-directory = ""`, "Missing evm-ss-db-directory") require.Contains(t, output, `evm-ss-split = false`, "Missing or incorrect evm-ss-split") require.Contains(t, output, "evm-ss-separate-dbs = false", "Missing or incorrect evm-ss-separate-dbs") - require.Contains(t, output, `historical-offload-scylla-hosts = ""`, "Missing historical Scylla hosts") - require.Contains(t, output, `historical-offload-scylla-keyspace = ""`, "Missing historical Scylla keyspace") - require.Contains(t, output, `historical-offload-scylla-consistency = ""`, "Missing historical Scylla consistency") - require.Contains(t, output, "historical-offload-scylla-timeout-ms = 0", "Missing historical Scylla timeout") require.Contains(t, output, `historical-offload-bigtable-project-id = ""`, "Missing historical Bigtable project") require.Contains(t, output, `historical-offload-bigtable-instance = ""`, "Missing historical Bigtable instance") require.Contains(t, output, `historical-offload-bigtable-table = ""`, "Missing historical Bigtable table") diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md index e17f8b6c55..7be12406d1 100644 --- a/sei-db/state_db/ss/offload/consumer/README.md +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -1,61 +1,27 @@ -# Historical State Offload +# Historical State Offload (Bigtable) -This is a prototype historical-state backend for ScyllaDB/Cassandra and -Bigtable. - -The intended shape is narrow: +Bigtable holds immutable MVCC mutation rows for history that local SS has +pruned. The shape is narrow: - local SS remains the hot store for recent state, writes, imports, pruning, and iterators -- the downstream store keeps immutable MVCC mutation rows for older history -- reads below local SS retention can fall back to the downstream store for `Get` and `Has` - -The Scylla table layout is built for point reads by -`(store_name, state_key, target_version)`: - -```sql -SELECT version, value, deleted -FROM state_mutations -WHERE store_name = ? AND state_key = ? AND version <= ? -ORDER BY version DESC -LIMIT 1; -``` +- Bigtable keeps immutable MVCC mutation rows for older history +- reads below local SS retention can fall back to Bigtable for `Get` and `Has` -Bigtable uses salted row keys with an inverted height suffix: +Row keys are salted with an inverted height suffix: ```text m | shard(store,key) | store_name | state_key | inverted_height ``` Reads scan from `inverted(target_height)` and stop after the first row, giving -the latest write at or before the requested height. - -Ordered prefix iteration is intentionally not served from the offload store in -this prototype. - -## Schema - -Apply the schema once: - -```bash -cqlsh 127.0.0.1 9042 -f sei-db/state_db/ss/offload/consumer/schema/scylla.cql -``` - -For production, edit the keyspace replication in `schema/scylla.cql` to use -`NetworkTopologyStrategy` with the actual datacenter names and replication -factors before applying it. +the latest write at or before the requested height. Ordered prefix iteration is +intentionally not served from the offload store. ## Consumer The consumer reads historical offload changelog messages from Kafka and writes -them into the configured backend. Kafka offsets are committed only after the -sink write succeeds. Mutation rows are written before the version marker. - -```bash -go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer \ - ./sei-db/state_db/ss/offload/consumer/config/example-scylla.json -``` - -For Bigtable: +them into Bigtable. Kafka offsets are committed only after the sink write +succeeds. Mutation rows are written before the version marker. ```bash cbt -project my-gcp-project -instance sei-history createtable state_mutations @@ -65,8 +31,8 @@ go run ./sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer \ ./sei-db/state_db/ss/offload/consumer/config/example-bigtable.json ``` -The example configs are local/dev placeholders. Set real Kafka brokers and -backend credentials/config in your own config. +The example config is a local/dev placeholder. Set real Kafka brokers and +Bigtable credentials/config in your own config. For Google Cloud Managed Service for Apache Kafka, connect with TLS plus SASL/PLAIN using service-account credentials: @@ -85,19 +51,6 @@ SASL/PLAIN using service-account credentials: Enable fallback reads in the node config: -```toml -[state-store] -historical-offload-scylla-hosts = "10.0.0.1:9042,10.0.0.2:9042" -historical-offload-scylla-keyspace = "sei_history" -historical-offload-scylla-username = "" -historical-offload-scylla-password = "" -historical-offload-scylla-datacenter = "datacenter1" -historical-offload-scylla-consistency = "local_quorum" -historical-offload-scylla-timeout-ms = 2000 -``` - -Or Bigtable: - ```toml [state-store] historical-offload-bigtable-project-id = "my-gcp-project" @@ -130,8 +83,7 @@ heights the backend never ingested would otherwise read as empty state. Before enabling the node read fallback in production: -- The backend table/keyspace exists with the same family/shards (Bigtable) or - schema (Scylla) as the consumer wrote with. +- The Bigtable table exists with the same family/shards the consumer wrote with. - The consumer has been ingesting continuously; check `consumer_kafka_lag` / `bigtable_rows_mutated_total`. - `historical-offload-earliest-version` is set to the true coverage floor. @@ -142,5 +94,5 @@ Before enabling the node read fallback in production: local prune horizon see only local data. - No cross-row transaction on ingest; mutation rows are written first and the version marker is written last, so replay is idempotent after partial failure. -- No automatic schema creation from the binary. +- No automatic table creation from the binary. - No backfill tooling; coverage starts when ingestion starts. diff --git a/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go b/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go index 53700ad6a0..1fc43636ce 100644 --- a/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go +++ b/sei-db/state_db/ss/offload/consumer/cmd/historical-offload-consumer/main.go @@ -43,9 +43,9 @@ func main() { log.Printf("serving consumer metrics at %s/metrics", cfg.MetricsAddr) } - sink, err := consumer.NewSinkFromConfig(*cfg) + sink, err := consumer.NewBigtableSink(cfg.Bigtable) if err != nil { - log.Fatalf("open %s sink: %v", cfg.BackendName(), err) + log.Fatalf("open bigtable sink: %v", err) } defer func() { _ = sink.Close() }() diff --git a/sei-db/state_db/ss/offload/consumer/compact_test.go b/sei-db/state_db/ss/offload/consumer/compact_test.go new file mode 100644 index 0000000000..919de6f9e0 --- /dev/null +++ b/sei-db/state_db/ss/offload/consumer/compact_test.go @@ -0,0 +1,65 @@ +package consumer + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" +) + +func TestCompactRecordsDropsNilEntries(t *testing.T) { + records := compactRecords([]Record{ + {Entry: &proto.ChangelogEntry{Version: 1}}, + {}, + {Entry: &proto.ChangelogEntry{Version: 2}}, + }) + require.Len(t, records, 2) + require.Equal(t, int64(1), records[0].Entry.Version) + require.Equal(t, int64(2), records[1].Entry.Version) +} + +func TestCompactRecordsCollapsesRedeliveredVersions(t *testing.T) { + records := compactRecords([]Record{ + {Offset: 10, Entry: &proto.ChangelogEntry{Version: 1}}, + {Offset: 11, Entry: &proto.ChangelogEntry{Version: 2}}, + {Offset: 12, Entry: &proto.ChangelogEntry{Version: 1}}, + {Offset: 13, Entry: &proto.ChangelogEntry{Version: 3}}, + }) + require.Len(t, records, 3) + // Version order is preserved; the redelivered version keeps its slot but + // carries the newest offset for the version marker. + require.Equal(t, int64(1), records[0].Entry.Version) + require.Equal(t, int64(12), records[0].Offset) + require.Equal(t, int64(2), records[1].Entry.Version) + require.Equal(t, int64(3), records[2].Entry.Version) +} + +func TestCompactMutationsKeepsLastWritePerKey(t *testing.T) { + entry := &proto.ChangelogEntry{ + Version: 9, + Changesets: []*proto.NamedChangeSet{{ + Name: "bank", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k"), Value: []byte("old")}, + {Key: []byte("drop"), Value: []byte("present")}, + {Key: []byte("k"), Value: []byte("new")}, + {Key: []byte("drop"), Delete: true}, + }}, + }, { + Name: "evm", + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k"), Value: []byte("separate-store")}, + }}, + }}, + } + + mutations := compactMutations(entry) + require.Len(t, mutations, 3) + byKey := map[string]*proto.KVPair{} + for _, m := range mutations { + byKey[m.storeName+"/"+string(m.pair.Key)] = m.pair + } + require.Equal(t, []byte("new"), byKey["bank/k"].Value) + require.True(t, byKey["bank/drop"].Delete) + require.Equal(t, []byte("separate-store"), byKey["evm/k"].Value) +} diff --git a/sei-db/state_db/ss/offload/consumer/config.go b/sei-db/state_db/ss/offload/consumer/config.go index bb4d3903b8..18889f6bf1 100644 --- a/sei-db/state_db/ss/offload/consumer/config.go +++ b/sei-db/state_db/ss/offload/consumer/config.go @@ -4,21 +4,15 @@ import ( "encoding/json" "fmt" "os" - "strings" ) const ( - backendScylla = "scylla" - backendBigtable = "bigtable" - - defaultBigtableMaxBatchRecords = 128 - defaultBigtableBatchMaxWaitMS = 25 + defaultMaxBatchRecords = 128 + defaultBatchMaxWaitMS = 25 ) type Config struct { - Backend string Kafka KafkaReaderConfig - Scylla ScyllaConfig Bigtable BigtableConfig Workers int ShardBufferSize int @@ -35,25 +29,10 @@ func (c *Config) Validate() error { if err := c.Kafka.Validate(); err != nil { return fmt.Errorf("kafka: %w", err) } - // Match the node-side reader, which refuses to guess between two configured - // backends; a consumer silently defaulting to Scylla here could ingest into - // a different store than the node reads from. - if strings.TrimSpace(c.Backend) == "" && c.Scylla.Configured() && c.Bigtable.Configured() { - return fmt.Errorf("both scylla and bigtable are configured; set Backend to pick one") - } - switch c.BackendName() { - case backendScylla: - if err := c.Scylla.Validate(); err != nil { - return fmt.Errorf("scylla: %w", err) - } - case backendBigtable: - bigtable := c.Bigtable - bigtable.ApplyDefaults() - if err := bigtable.Validate(); err != nil { - return fmt.Errorf("bigtable: %w", err) - } - default: - return fmt.Errorf("unsupported backend %q", c.Backend) + bigtable := c.Bigtable + bigtable.ApplyDefaults() + if err := bigtable.Validate(); err != nil { + return fmt.Errorf("bigtable: %w", err) } if c.Workers < 0 { return fmt.Errorf("workers must be non-negative") @@ -70,37 +49,12 @@ func (c *Config) Validate() error { return nil } -func (c *Config) BackendName() string { - backend := strings.ToLower(strings.TrimSpace(c.Backend)) - if backend != "" { - return backend - } - if c.Bigtable.Configured() && !c.Scylla.Configured() { - return backendBigtable - } - return backendScylla -} - -func (c *Config) applyBackendDefaults() { - if c.BackendName() != backendBigtable { - return - } +func (c *Config) applyDefaults() { if c.MaxBatchRecords == 0 { - c.MaxBatchRecords = defaultBigtableMaxBatchRecords + c.MaxBatchRecords = defaultMaxBatchRecords } if c.BatchMaxWaitMS == 0 { - c.BatchMaxWaitMS = defaultBigtableBatchMaxWaitMS - } -} - -func NewSinkFromConfig(cfg Config) (Sink, error) { - switch cfg.BackendName() { - case backendScylla: - return NewScyllaSink(cfg.Scylla) - case backendBigtable: - return NewBigtableSink(cfg.Bigtable) - default: - return nil, fmt.Errorf("unsupported backend %q", cfg.Backend) + c.BatchMaxWaitMS = defaultBatchMaxWaitMS } } @@ -114,7 +68,7 @@ func LoadConfig(path string) (*Config, error) { if err := json.Unmarshal(raw, cfg); err != nil { return nil, fmt.Errorf("parse config: %w", err) } - cfg.applyBackendDefaults() + cfg.applyDefaults() if err := cfg.Validate(); err != nil { return nil, err } diff --git a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json index 27c2640d29..47ba1f0d33 100644 --- a/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json +++ b/sei-db/state_db/ss/offload/consumer/config/example-bigtable.json @@ -1,5 +1,4 @@ { - "Backend": "bigtable", "Kafka": { "Brokers": ["bootstrap.my-cluster.us-east1.managedkafka.my-gcp-project.cloud.goog:9092"], "Topic": "historical-offload", diff --git a/sei-db/state_db/ss/offload/consumer/config/example-scylla.json b/sei-db/state_db/ss/offload/consumer/config/example-scylla.json deleted file mode 100644 index 013217af75..0000000000 --- a/sei-db/state_db/ss/offload/consumer/config/example-scylla.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "Kafka": { - "Brokers": ["localhost:9092"], - "Topic": "historical-offload", - "GroupID": "historical-scylla", - "StartOffset": "first" - }, - "Scylla": { - "Hosts": ["127.0.0.1:9042"], - "Keyspace": "sei_history", - "Datacenter": "datacenter1", - "Consistency": "local_quorum", - "TimeoutMS": 2000, - "ConnectTimeoutMS": 2000, - "NumConns": 4, - "MutationWorkers": 16 - }, - "Workers": 16, - "ShardBufferSize": 128, - "MaxBatchRecords": 16, - "BatchMaxWaitMS": 10 -} diff --git a/sei-db/state_db/ss/offload/consumer/config_test.go b/sei-db/state_db/ss/offload/consumer/config_test.go index 0f3c7507fe..9cd6f65d3f 100644 --- a/sei-db/state_db/ss/offload/consumer/config_test.go +++ b/sei-db/state_db/ss/offload/consumer/config_test.go @@ -8,15 +8,8 @@ import ( "github.com/stretchr/testify/require" ) -func TestConfigBackendName(t *testing.T) { - require.Equal(t, backendScylla, (&Config{}).BackendName()) - require.Equal(t, backendBigtable, (&Config{Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) - require.Equal(t, backendScylla, (&Config{Backend: "Scylla", Bigtable: BigtableConfig{ProjectID: "p"}}).BackendName()) -} - -func TestConfigValidateBigtable(t *testing.T) { +func TestConfigValidate(t *testing.T) { cfg := Config{ - Backend: backendBigtable, Kafka: KafkaReaderConfig{ Brokers: []string{"localhost:9092"}, Topic: "historical-offload", @@ -31,58 +24,31 @@ func TestConfigValidateBigtable(t *testing.T) { require.NoError(t, cfg.Validate()) cfg.Bigtable.Table = "" - require.ErrorContains(t, cfg.Validate(), backendBigtable) + require.ErrorContains(t, cfg.Validate(), "bigtable") } -func TestConfigValidateRejectsAmbiguousDualBackends(t *testing.T) { - cfg := Config{ - Kafka: KafkaReaderConfig{ - Brokers: []string{"localhost:9092"}, - Topic: "historical-offload", - GroupID: "g", - }, - Scylla: ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks"}, - Bigtable: BigtableConfig{ProjectID: "p", InstanceID: "i", Table: "t"}, - } - require.ErrorContains(t, cfg.Validate(), "both scylla and bigtable") +func TestConfigApplyDefaults(t *testing.T) { + cfg := Config{} + cfg.applyDefaults() + require.Equal(t, defaultMaxBatchRecords, cfg.MaxBatchRecords) + require.Equal(t, defaultBatchMaxWaitMS, cfg.BatchMaxWaitMS) - cfg.Backend = backendBigtable - require.NoError(t, cfg.Validate()) -} - -func TestConfigApplyBackendDefaultsBigtable(t *testing.T) { - cfg := Config{Backend: backendBigtable} - cfg.applyBackendDefaults() - require.Equal(t, defaultBigtableMaxBatchRecords, cfg.MaxBatchRecords) - require.Equal(t, defaultBigtableBatchMaxWaitMS, cfg.BatchMaxWaitMS) - - cfg = Config{ - Backend: backendBigtable, - MaxBatchRecords: 32, - BatchMaxWaitMS: 5, - } - cfg.applyBackendDefaults() + cfg = Config{MaxBatchRecords: 32, BatchMaxWaitMS: 5} + cfg.applyDefaults() require.Equal(t, 32, cfg.MaxBatchRecords) require.Equal(t, 5, cfg.BatchMaxWaitMS) } -func TestConfigApplyBackendDefaultsScylla(t *testing.T) { - cfg := Config{Backend: backendScylla} - cfg.applyBackendDefaults() - require.Zero(t, cfg.MaxBatchRecords) - require.Zero(t, cfg.BatchMaxWaitMS) -} - -func TestLoadConfigMetricsAddr(t *testing.T) { +func TestLoadConfig(t *testing.T) { path := filepath.Join(t.TempDir(), "config.json") require.NoError(t, os.WriteFile(path, []byte(`{ - "Backend": "bigtable", "Kafka": {"Brokers": ["localhost:9092"], "Topic": "historical-offload", "GroupID": "g"}, "Bigtable": {"ProjectID": "p", "InstanceID": "i", "Table": "t"}, - "MetricsAddr": ":9092" + "MetricsAddr": ":2112" }`), 0o600)) cfg, err := LoadConfig(path) require.NoError(t, err) - require.Equal(t, ":9092", cfg.MetricsAddr) + require.Equal(t, ":2112", cfg.MetricsAddr) + require.Equal(t, defaultMaxBatchRecords, cfg.MaxBatchRecords) } diff --git a/sei-db/state_db/ss/offload/consumer/kafka_test.go b/sei-db/state_db/ss/offload/consumer/kafka_test.go index f6122ffb51..57056e9951 100644 --- a/sei-db/state_db/ss/offload/consumer/kafka_test.go +++ b/sei-db/state_db/ss/offload/consumer/kafka_test.go @@ -11,7 +11,7 @@ func TestKafkaReaderConfigApplyDefaults(t *testing.T) { cfg := KafkaReaderConfig{ Brokers: []string{"localhost:9092"}, Topic: "historical-offload", - GroupID: "scylla", + GroupID: "bigtable", } cfg.ApplyDefaults() require.Equal(t, "sei-historical-offload-consumer", cfg.ClientID) diff --git a/sei-db/state_db/ss/offload/consumer/schema/scylla.cql b/sei-db/state_db/ss/offload/consumer/schema/scylla.cql deleted file mode 100644 index aa26df4c97..0000000000 --- a/sei-db/state_db/ss/offload/consumer/schema/scylla.cql +++ /dev/null @@ -1,46 +0,0 @@ --- ScyllaDB/Cassandra schema for Sei historical state offload. --- Apply once before running historical-offload-consumer. - -CREATE KEYSPACE IF NOT EXISTS sei_history -WITH replication = { - 'class': 'SimpleStrategy', - 'replication_factor': '1' -}; - --- For production, replace the keyspace replication with --- NetworkTopologyStrategy and the real datacenter replication factors. - -USE sei_history; - --- Version markers are written after all mutation rows for a block version. --- Buckets avoid a single hot partition while keeping LastVersion bounded to --- 64 small point reads. -CREATE TABLE IF NOT EXISTS state_versions ( - bucket int, - version bigint, - kafka_topic text, - kafka_partition int, - kafka_offset bigint, - ingested_at timestamp, - PRIMARY KEY ((bucket), version) -) WITH CLUSTERING ORDER BY (version DESC); - --- Historical point lookup: --- WHERE store_name = ? AND state_key = ? AND version <= ? --- ORDER BY version DESC LIMIT 1 -CREATE TABLE IF NOT EXISTS state_mutations ( - store_name text, - state_key blob, - version bigint, - value blob, - deleted boolean, - PRIMARY KEY ((store_name, state_key), version) -) WITH CLUSTERING ORDER BY (version DESC); - -CREATE TABLE IF NOT EXISTS state_tree_upgrades ( - version bigint, - name text, - rename_from text, - deleted boolean, - PRIMARY KEY ((version), name) -); diff --git a/sei-db/state_db/ss/offload/consumer/scylla.go b/sei-db/state_db/ss/offload/consumer/scylla.go deleted file mode 100644 index 84b521120e..0000000000 --- a/sei-db/state_db/ss/offload/consumer/scylla.go +++ /dev/null @@ -1,265 +0,0 @@ -package consumer - -import ( - "context" - "fmt" - "time" - - "github.com/gocql/gocql" - "golang.org/x/sync/errgroup" - - "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/offload/historical" -) - -const ( - defaultScyllaMutationWorkers = 16 - - // defaultScyllaRecordWorkers bounds how many records write their rows - // concurrently, so peak in-flight CQL writes stay at - // recordWorkers*mutationWorkers instead of len(batch)*mutationWorkers. - defaultScyllaRecordWorkers = 4 -) - -type ScyllaConfig struct { - Hosts []string - Keyspace string - Username string - Password string - Datacenter string - Consistency string - TimeoutMS int - ConnectTimeoutMS int - NumConns int - MutationWorkers int -} - -func (c ScyllaConfig) Configured() bool { - return c.toHistorical().Configured() -} - -func (c *ScyllaConfig) ApplyDefaults() { - cfg := c.toHistorical() - cfg.ApplyDefaults() - c.Consistency = cfg.Consistency - c.TimeoutMS = int(cfg.Timeout / time.Millisecond) - c.ConnectTimeoutMS = int(cfg.ConnectTimeout / time.Millisecond) - c.NumConns = cfg.NumConns - if c.MutationWorkers == 0 { - c.MutationWorkers = defaultScyllaMutationWorkers - } -} - -func (c *ScyllaConfig) Validate() error { - cfg := c.toHistorical() - cfg.ApplyDefaults() - if err := cfg.Validate(); err != nil { - return err - } - if c.MutationWorkers < 0 { - return fmt.Errorf("scylla/cassandra mutation workers must be non-negative") - } - return nil -} - -func (c ScyllaConfig) toHistorical() historical.ScyllaConfig { - return historical.ScyllaConfig{ - Hosts: c.Hosts, - Keyspace: c.Keyspace, - Username: c.Username, - Password: c.Password, - Datacenter: c.Datacenter, - Consistency: c.Consistency, - Timeout: time.Duration(c.TimeoutMS) * time.Millisecond, - ConnectTimeout: time.Duration(c.ConnectTimeoutMS) * time.Millisecond, - NumConns: c.NumConns, - } -} - -type scyllaSink struct { - session *gocql.Session - exec scyllaExecFunc - mutationWorkers int -} - -var _ Sink = (*scyllaSink)(nil) -var _ BatchSink = (*scyllaSink)(nil) - -func NewScyllaSink(cfg ScyllaConfig) (Sink, error) { - cfg.ApplyDefaults() - if err := cfg.Validate(); err != nil { - return nil, err - } - session, err := historical.OpenScyllaSession(cfg.toHistorical()) - if err != nil { - return nil, err - } - return &scyllaSink{ - session: session, - exec: sessionExec(session), - mutationWorkers: cfg.MutationWorkers, - }, nil -} - -func (s *scyllaSink) Close() error { - if s.session != nil { - s.session.Close() - } - return nil -} - -func (s *scyllaSink) LastVersion(ctx context.Context) (int64, error) { - return historical.ScyllaLastVersion(ctx, s.session) -} - -func (s *scyllaSink) Write(ctx context.Context, rec Record) error { - return s.WriteBatch(ctx, []Record{rec}) -} - -func (s *scyllaSink) WriteBatch(ctx context.Context, records []Record) error { - records = compactRecords(records) - if len(records) == 0 { - return nil - } - if len(records) == 1 { - return s.writeRecord(ctx, records[0]) - } - return s.writeRecordsPipelined(ctx, records) -} - -func (s *scyllaSink) writeRecord(ctx context.Context, rec Record) error { - if err := s.writeRecordRows(ctx, rec.Entry); err != nil { - return err - } - return s.writeVersionMarker(ctx, rec) -} - -func (s *scyllaSink) writeVersionMarker(ctx context.Context, rec Record) error { - version := rec.Entry.Version - if err := s.exec(ctx, insertVersionCQL, - historical.VersionBucket(version), - version, - rec.Topic, - rec.Partition, - rec.Offset, - time.Now(), - ); err != nil { - return fmt.Errorf("insert scylla/cassandra version %d: %w", version, err) - } - return nil -} - -func (s *scyllaSink) writeRecordsPipelined(ctx context.Context, records []Record) error { - rowCtx, cancel := context.WithCancel(ctx) - defer cancel() - var g errgroup.Group - // A semaphore rather than errgroup.SetLimit keeps goroutine launching - // non-blocking, so the ordered version-marker loop below overlaps with row - // writes from the first record onward. - sem := make(chan struct{}, defaultScyllaRecordWorkers) - rowDone := make([]chan error, len(records)) - for i := range records { - rowDone[i] = make(chan error, 1) - i := i - rec := records[i] - g.Go(func() error { - select { - case sem <- struct{}{}: - case <-rowCtx.Done(): - rowDone[i] <- rowCtx.Err() - return rowCtx.Err() - } - defer func() { <-sem }() - err := s.writeRecordRows(rowCtx, rec.Entry) - if err != nil { - err = fmt.Errorf("write scylla/cassandra rows version %d: %w", rec.Entry.Version, err) - } - rowDone[i] <- err - return err - }) - } - for i, rec := range records { - if err := <-rowDone[i]; err != nil { - cancel() - _ = g.Wait() - return err - } - if err := s.writeVersionMarker(ctx, rec); err != nil { - cancel() - _ = g.Wait() - return err - } - } - return g.Wait() -} - -func (s *scyllaSink) writeRecordRows(ctx context.Context, entry *proto.ChangelogEntry) error { - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(s.mutationWorkers) - for _, mutation := range compactMutations(entry) { - mutation := mutation - g.Go(func() error { - return s.writeMutation(gctx, entry.Version, mutation.storeName, mutation.pair) - }) - } - for _, up := range entry.Upgrades { - up := up - g.Go(func() error { - return s.writeUpgrade(gctx, entry.Version, up) - }) - } - return g.Wait() -} - -func (s *scyllaSink) writeMutation(ctx context.Context, version int64, storeName string, pair *proto.KVPair) error { - deleted := pair.Delete || pair.Value == nil - value := pair.Value - if deleted { - value = nil - } - if err := s.exec(ctx, insertMutationCQL, - storeName, - pair.Key, - version, - value, - deleted, - ); err != nil { - return fmt.Errorf("insert scylla/cassandra mutation store=%s version=%d: %w", storeName, version, err) - } - return nil -} - -func (s *scyllaSink) writeUpgrade(ctx context.Context, version int64, up *proto.TreeNameUpgrade) error { - if err := s.exec(ctx, insertUpgradeCQL, - version, - up.Name, - up.RenameFrom, - up.Delete, - ); err != nil { - return fmt.Errorf("insert scylla/cassandra tree upgrade version=%d name=%s: %w", version, up.Name, err) - } - return nil -} - -type scyllaExecFunc func(ctx context.Context, stmt string, values ...interface{}) error - -func sessionExec(session *gocql.Session) scyllaExecFunc { - return func(ctx context.Context, stmt string, values ...interface{}) error { - return session.Query(stmt, values...).WithContext(ctx).Exec() - } -} - -const insertVersionCQL = ` -INSERT INTO state_versions ( - bucket, version, kafka_topic, kafka_partition, kafka_offset, ingested_at -) VALUES (?, ?, ?, ?, ?, ?)` - -const insertMutationCQL = ` -INSERT INTO state_mutations ( - store_name, state_key, version, value, deleted -) VALUES (?, ?, ?, ?, ?)` - -const insertUpgradeCQL = ` -INSERT INTO state_tree_upgrades ( - version, name, rename_from, deleted -) VALUES (?, ?, ?, ?)` diff --git a/sei-db/state_db/ss/offload/consumer/scylla_test.go b/sei-db/state_db/ss/offload/consumer/scylla_test.go deleted file mode 100644 index 3a48844c6a..0000000000 --- a/sei-db/state_db/ss/offload/consumer/scylla_test.go +++ /dev/null @@ -1,414 +0,0 @@ -package consumer - -import ( - "context" - "errors" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/stretchr/testify/require" -) - -func TestScyllaConfigValidate(t *testing.T) { - cfg := ScyllaConfig{ - Hosts: []string{"127.0.0.1"}, - Keyspace: "sei_history", - } - require.NoError(t, cfg.Validate()) - - cfg.TimeoutMS = -1 - require.ErrorContains(t, cfg.Validate(), "timeout") - - cfg.TimeoutMS = 0 - cfg.MutationWorkers = -1 - require.ErrorContains(t, cfg.Validate(), "mutation workers") -} - -func TestScyllaConfigApplyDefaults(t *testing.T) { - cfg := ScyllaConfig{ - Hosts: []string{"127.0.0.1"}, - Keyspace: "sei_history", - } - cfg.ApplyDefaults() - require.Equal(t, "local_quorum", cfg.Consistency) - require.Equal(t, 2000, cfg.TimeoutMS) - require.Equal(t, 2000, cfg.ConnectTimeoutMS) - require.Equal(t, 4, cfg.NumConns) - require.Equal(t, 16, cfg.MutationWorkers) -} - -func TestCompactRecordsDropsNilEntries(t *testing.T) { - records := compactRecords([]Record{ - {Entry: &proto.ChangelogEntry{Version: 1}}, - {}, - {Entry: &proto.ChangelogEntry{Version: 2}}, - }) - require.Len(t, records, 2) - require.Equal(t, int64(1), records[0].Entry.Version) - require.Equal(t, int64(2), records[1].Entry.Version) -} - -func TestCompactRecordsCollapsesRedeliveredVersions(t *testing.T) { - records := compactRecords([]Record{ - {Offset: 10, Entry: &proto.ChangelogEntry{Version: 1}}, - {Offset: 11, Entry: &proto.ChangelogEntry{Version: 2}}, - {Offset: 12, Entry: &proto.ChangelogEntry{Version: 1}}, - {Offset: 13, Entry: &proto.ChangelogEntry{Version: 3}}, - }) - require.Len(t, records, 3) - // Version order is preserved; the redelivered version keeps its slot but - // carries the newest offset for the version marker. - require.Equal(t, int64(1), records[0].Entry.Version) - require.Equal(t, int64(12), records[0].Offset) - require.Equal(t, int64(2), records[1].Entry.Version) - require.Equal(t, int64(3), records[2].Entry.Version) -} - -func TestScyllaCQLShape(t *testing.T) { - for _, frag := range []string{ - "INSERT INTO state_mutations", - "store_name", - "state_key", - "version", - "value", - "deleted", - } { - require.Contains(t, insertMutationCQL, frag) - } - for _, frag := range []string{ - "INSERT INTO state_versions", - "bucket", - "version", - "kafka_topic", - "kafka_partition", - "kafka_offset", - "ingested_at", - } { - require.Contains(t, insertVersionCQL, frag) - } -} - -func TestScyllaSinkWritesRowsConcurrentlyBeforeVersionMarker(t *testing.T) { - rowStarted := make(chan struct{}, 8) - releaseRows := make(chan struct{}) - var activeRows atomic.Int32 - var sawConcurrentRows atomic.Bool - var markerBeforeRowsDone atomic.Bool - var versionMarkers atomic.Int32 - - sink := &scyllaSink{ - mutationWorkers: 2, - exec: func(ctx context.Context, stmt string, _ ...interface{}) error { - if strings.Contains(stmt, "state_versions") { - if activeRows.Load() != 0 { - markerBeforeRowsDone.Store(true) - } - versionMarkers.Add(1) - return nil - } - if activeRows.Add(1) > 1 { - sawConcurrentRows.Store(true) - } - rowStarted <- struct{}{} - select { - case <-releaseRows: - case <-ctx.Done(): - activeRows.Add(-1) - return ctx.Err() - } - activeRows.Add(-1) - return nil - }, - } - entry := &proto.ChangelogEntry{ - Version: 7, - Changesets: []*proto.NamedChangeSet{{ - Name: "bank", - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - {Key: []byte("k1"), Value: []byte("v1")}, - {Key: []byte("k2"), Value: []byte("v2")}, - {Key: []byte("k3"), Value: []byte("v3")}, - }}, - }}, - Upgrades: []*proto.TreeNameUpgrade{{Name: "new-store"}}, - } - - errCh := make(chan error, 1) - go func() { - errCh <- sink.writeRecord(context.Background(), Record{Topic: "t", Partition: 1, Offset: 2, Entry: entry}) - }() - - releaseClosed := false - closeRelease := func() { - if !releaseClosed { - close(releaseRows) - releaseClosed = true - } - } - defer closeRelease() - - for i := 0; i < 2; i++ { - select { - case <-rowStarted: - case <-time.After(time.Second): - closeRelease() - t.Fatal("timed out waiting for concurrent row writes") - } - } - require.True(t, sawConcurrentRows.Load()) - require.Equal(t, int32(0), versionMarkers.Load(), "version marker must wait for row writes") - - closeRelease() - select { - case err := <-errCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("timed out waiting for record write") - } - require.False(t, markerBeforeRowsDone.Load()) - require.Equal(t, int32(1), versionMarkers.Load()) -} - -func TestScyllaSinkCompactsDuplicateMutations(t *testing.T) { - type write struct { - value []byte - deleted bool - } - var mu sync.Mutex - writes := make(map[string]write) - sink := &scyllaSink{ - mutationWorkers: 1, - exec: func(_ context.Context, stmt string, values ...interface{}) error { - if !strings.Contains(stmt, "state_mutations") { - return nil - } - storeName := values[0].(string) - key := string(values[1].([]byte)) - value := values[3].([]byte) - deleted := values[4].(bool) - mu.Lock() - writes[storeName+"/"+key] = write{value: value, deleted: deleted} - mu.Unlock() - return nil - }, - } - entry := &proto.ChangelogEntry{ - Version: 9, - Changesets: []*proto.NamedChangeSet{{ - Name: "bank", - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - {Key: []byte("k"), Value: []byte("old")}, - {Key: []byte("drop"), Value: []byte("present")}, - {Key: []byte("k"), Value: []byte("new")}, - {Key: []byte("drop"), Delete: true}, - }}, - }, { - Name: "evm", - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - {Key: []byte("k"), Value: []byte("separate-store")}, - }}, - }}, - } - - require.NoError(t, sink.writeRecordRows(context.Background(), entry)) - require.Len(t, writes, 3) - require.Equal(t, write{value: []byte("new")}, writes["bank/k"]) - require.Equal(t, write{deleted: true}, writes["bank/drop"]) - require.Equal(t, write{value: []byte("separate-store")}, writes["evm/k"]) -} - -func TestScyllaSinkWriteBatchPipelinesRowsAndOrdersMarkers(t *testing.T) { - rowStarted := make(chan int64, 2) - markerWritten := make(chan int64, 2) - releaseRows := map[int64]chan struct{}{ - 1: make(chan struct{}), - 2: make(chan struct{}), - } - var activeRows atomic.Int32 - var sawConcurrentRows atomic.Bool - var mu sync.Mutex - rowsDone := make(map[int64]bool) - var markers []int64 - var markerBeforeRowsDone bool - - sink := &scyllaSink{ - mutationWorkers: 1, - exec: func(ctx context.Context, stmt string, values ...interface{}) error { - switch { - case strings.Contains(stmt, "state_mutations"): - version := values[2].(int64) - if activeRows.Add(1) > 1 { - sawConcurrentRows.Store(true) - } - rowStarted <- version - select { - case <-releaseRows[version]: - case <-ctx.Done(): - activeRows.Add(-1) - return ctx.Err() - } - activeRows.Add(-1) - mu.Lock() - rowsDone[version] = true - mu.Unlock() - return nil - case strings.Contains(stmt, "state_versions"): - version := values[1].(int64) - mu.Lock() - if !rowsDone[version] { - markerBeforeRowsDone = true - } - markers = append(markers, version) - mu.Unlock() - markerWritten <- version - return nil - default: - return nil - } - }, - } - records := []Record{ - { - Topic: "t", - Partition: 0, - Offset: 10, - Entry: &proto.ChangelogEntry{ - Version: 1, - Changesets: []*proto.NamedChangeSet{{ - Name: "bank", - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k1"), Value: []byte("v1")}}}, - }}, - }, - }, - { - Topic: "t", - Partition: 0, - Offset: 11, - Entry: &proto.ChangelogEntry{ - Version: 2, - Changesets: []*proto.NamedChangeSet{{ - Name: "bank", - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k2"), Value: []byte("v2")}}}, - }}, - }, - }, - } - - errCh := make(chan error, 1) - go func() { - errCh <- sink.WriteBatch(context.Background(), records) - }() - - started := map[int64]bool{} - for len(started) < 2 { - select { - case version := <-rowStarted: - started[version] = true - case <-time.After(time.Second): - t.Fatal("timed out waiting for pipelined row writes") - } - } - require.True(t, sawConcurrentRows.Load()) - - close(releaseRows[2]) - select { - case version := <-markerWritten: - t.Fatalf("marker %d written before earlier record rows completed", version) - case <-time.After(100 * time.Millisecond): - } - - close(releaseRows[1]) - for _, want := range []int64{1, 2} { - select { - case got := <-markerWritten: - require.Equal(t, want, got) - case <-time.After(time.Second): - t.Fatalf("timed out waiting for marker %d", want) - } - } - select { - case err := <-errCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("timed out waiting for batch write") - } - mu.Lock() - defer mu.Unlock() - require.False(t, markerBeforeRowsDone) - require.Equal(t, []int64{1, 2}, markers) -} - -func TestScyllaSinkWriteBatchReturnsRowErrorAfterLaterRowFailure(t *testing.T) { - rowErr := errors.New("row write failed") - rowStarted := make(chan int64, 2) - releaseFirst := make(chan struct{}) - - sink := &scyllaSink{ - mutationWorkers: 1, - exec: func(ctx context.Context, stmt string, values ...interface{}) error { - if !strings.Contains(stmt, "state_mutations") { - return nil - } - version := values[2].(int64) - rowStarted <- version - if version == 2 { - return rowErr - } - select { - case <-releaseFirst: - return nil - case <-ctx.Done(): - return ctx.Err() - } - }, - } - records := []Record{ - { - Entry: &proto.ChangelogEntry{ - Version: 1, - Changesets: []*proto.NamedChangeSet{{ - Name: "bank", - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k1"), Value: []byte("v1")}}}, - }}, - }, - }, - { - Entry: &proto.ChangelogEntry{ - Version: 2, - Changesets: []*proto.NamedChangeSet{{ - Name: "bank", - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: []byte("k2"), Value: []byte("v2")}}}, - }}, - }, - }, - } - - errCh := make(chan error, 1) - go func() { - errCh <- sink.WriteBatch(context.Background(), records) - }() - - started := map[int64]bool{} - for len(started) < 2 { - select { - case version := <-rowStarted: - started[version] = true - case <-time.After(time.Second): - close(releaseFirst) - t.Fatal("timed out waiting for row writes") - } - } - close(releaseFirst) - - select { - case err := <-errCh: - require.ErrorIs(t, err, rowErr) - require.NotErrorIs(t, err, context.Canceled) - case <-time.After(time.Second): - t.Fatal("timed out waiting for batch write") - } -} diff --git a/sei-db/state_db/ss/offload/historical/bigtable.go b/sei-db/state_db/ss/offload/historical/bigtable.go index 845bd98ff2..edb85510fa 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable.go +++ b/sei-db/state_db/ss/offload/historical/bigtable.go @@ -30,8 +30,20 @@ const ( BigtableValueColumn = "value" BigtableDeletedColumn = "deleted" + + // VersionBucketCount spreads monotonically increasing block-version markers + // across a bounded set of row prefixes while keeping LastVersion cheap. + VersionBucketCount = 64 ) +// VersionBucket maps a version to its marker bucket. +func VersionBucket(version int64) int { + if version < 0 { + version = -version + } + return int(version % VersionBucketCount) +} + const ( bigtableEndpoint = "bigtable.googleapis.com:443" diff --git a/sei-db/state_db/ss/offload/historical/scylla.go b/sei-db/state_db/ss/offload/historical/scylla.go deleted file mode 100644 index 562cd5c192..0000000000 --- a/sei-db/state_db/ss/offload/historical/scylla.go +++ /dev/null @@ -1,257 +0,0 @@ -package historical - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/gocql/gocql" - "golang.org/x/sync/errgroup" -) - -const ( - defaultScyllaConsistency = "local_quorum" - defaultScyllaTimeout = 2 * time.Second - defaultScyllaNumConns = 4 - defaultScyllaReadWorkers = 16 - - // VersionBucketCount spreads monotonically increasing block-version markers - // across a bounded set of partitions while keeping LastVersion cheap. - VersionBucketCount = 64 -) - -type ScyllaConfig struct { - Hosts []string - Keyspace string - Username string - Password string - Datacenter string - Consistency string - Timeout time.Duration - ConnectTimeout time.Duration - NumConns int -} - -func (c *ScyllaConfig) ApplyDefaults() { - if c.Consistency == "" { - c.Consistency = defaultScyllaConsistency - } - if c.Timeout == 0 { - c.Timeout = defaultScyllaTimeout - } - if c.ConnectTimeout == 0 { - c.ConnectTimeout = defaultScyllaTimeout - } - if c.NumConns == 0 { - c.NumConns = defaultScyllaNumConns - } -} - -func (c ScyllaConfig) Configured() bool { - return len(c.Hosts) != 0 || strings.TrimSpace(c.Keyspace) != "" -} - -func (c *ScyllaConfig) Validate() error { - if len(c.Hosts) == 0 { - return fmt.Errorf("scylla/cassandra hosts are required") - } - for _, host := range c.Hosts { - if strings.TrimSpace(host) == "" { - return fmt.Errorf("scylla/cassandra hosts must not contain blanks") - } - } - if strings.TrimSpace(c.Keyspace) == "" { - return fmt.Errorf("scylla/cassandra keyspace is required") - } - if c.Password != "" && c.Username == "" { - return fmt.Errorf("scylla/cassandra username is required when password is set") - } - if _, err := parseConsistency(c.Consistency); err != nil { - return err - } - if c.Timeout < 0 { - return fmt.Errorf("scylla/cassandra timeout must be non-negative") - } - if c.ConnectTimeout < 0 { - return fmt.Errorf("scylla/cassandra connect timeout must be non-negative") - } - if c.NumConns < 0 { - return fmt.Errorf("scylla/cassandra num conns must be non-negative") - } - return nil -} - -func NewScyllaReader(cfg ScyllaConfig) (Reader, error) { - session, err := OpenScyllaSession(cfg) - if err != nil { - return nil, err - } - return &scyllaReader{session: session}, nil -} - -func OpenScyllaSession(cfg ScyllaConfig) (*gocql.Session, error) { - cfg.ApplyDefaults() - if err := cfg.Validate(); err != nil { - return nil, err - } - consistency, err := parseConsistency(cfg.Consistency) - if err != nil { - return nil, err - } - - cluster := gocql.NewCluster(cfg.Hosts...) - cluster.Keyspace = cfg.Keyspace - cluster.Consistency = consistency - cluster.Timeout = cfg.Timeout - cluster.ConnectTimeout = cfg.ConnectTimeout - cluster.NumConns = cfg.NumConns - if cfg.Username != "" { - cluster.Authenticator = gocql.PasswordAuthenticator{ - Username: cfg.Username, - Password: cfg.Password, - } - } - cluster.PoolConfig.HostSelectionPolicy = scyllaHostSelectionPolicy(cfg.Datacenter) - - session, err := cluster.CreateSession() - if err != nil { - return nil, fmt.Errorf("open scylla/cassandra session: %w", err) - } - return session, nil -} - -func scyllaHostSelectionPolicy(datacenter string) gocql.HostSelectionPolicy { - datacenter = strings.TrimSpace(datacenter) - if datacenter != "" { - return gocql.TokenAwareHostPolicy(gocql.DCAwareRoundRobinPolicy(datacenter)) - } - return gocql.TokenAwareHostPolicy(gocql.RoundRobinHostPolicy()) -} - -type scyllaReader struct { - session *gocql.Session -} - -var _ Reader = (*scyllaReader)(nil) - -func (r *scyllaReader) Close() error { - if r.session != nil { - r.session.Close() - } - return nil -} - -func (r *scyllaReader) LastVersion(ctx context.Context) (int64, error) { - return ScyllaLastVersion(ctx, r.session) -} - -// ScyllaLastVersion scans the version-marker buckets in parallel and returns -// the highest ingested version. Shared by the node-side reader and the offload -// consumer sink. -func ScyllaLastVersion(ctx context.Context, session *gocql.Session) (int64, error) { - versions := make([]int64, VersionBucketCount) - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(defaultScyllaReadWorkers) - for bucket := 0; bucket < VersionBucketCount; bucket++ { - bucket := bucket - g.Go(func() error { - var version int64 - err := session.Query(selectLatestVersionCQL, bucket).WithContext(gctx).Scan(&version) - if err != nil { - if errors.Is(err, gocql.ErrNotFound) { - return nil - } - return fmt.Errorf("read latest scylla/cassandra version bucket %d: %w", bucket, err) - } - versions[bucket] = version - return nil - }) - } - if err := g.Wait(); err != nil { - return 0, err - } - var maxVersion int64 - for _, version := range versions { - if version > maxVersion { - maxVersion = version - } - } - return maxVersion, nil -} - -func (r *scyllaReader) Has(ctx context.Context, storeName string, key []byte, targetVersion int64) (bool, error) { - var deleted bool - err := r.session.Query(hasLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&deleted) - if err != nil { - if errors.Is(err, gocql.ErrNotFound) { - return false, nil - } - return false, fmt.Errorf("scylla/cassandra has lookup: %w", err) - } - return !deleted, nil -} - -func (r *scyllaReader) Get(ctx context.Context, storeName string, key []byte, targetVersion int64) (Value, error) { - var ( - version int64 - bz []byte - deleted bool - ) - err := r.session.Query(getLookupCQL, storeName, key, targetVersion).WithContext(ctx).Scan(&version, &bz, &deleted) - if err != nil { - if errors.Is(err, gocql.ErrNotFound) { - return Value{}, ErrNotFound - } - return Value{}, fmt.Errorf("scylla/cassandra get lookup: %w", err) - } - if deleted { - return Value{}, ErrNotFound - } - return Value{Bytes: bz, Version: version}, nil -} - -func VersionBucket(version int64) int { - if version < 0 { - version = -version - } - return int(version % VersionBucketCount) -} - -func parseConsistency(name string) (gocql.Consistency, error) { - switch strings.ToLower(strings.TrimSpace(name)) { - case "one": - return gocql.One, nil - case "local_one": - return gocql.LocalOne, nil - case "quorum": - return gocql.Quorum, nil - case "", "local_quorum": - return gocql.LocalQuorum, nil - case "all": - return gocql.All, nil - default: - return gocql.Any, fmt.Errorf("unsupported scylla/cassandra consistency %q", name) - } -} - -const selectLatestVersionCQL = ` -SELECT version -FROM state_versions -WHERE bucket = ? -LIMIT 1` - -const hasLookupCQL = ` -SELECT deleted -FROM state_mutations -WHERE store_name = ? AND state_key = ? AND version <= ? -ORDER BY version DESC -LIMIT 1` - -const getLookupCQL = ` -SELECT version, value, deleted -FROM state_mutations -WHERE store_name = ? AND state_key = ? AND version <= ? -ORDER BY version DESC -LIMIT 1` diff --git a/sei-db/state_db/ss/offload/historical/scylla_test.go b/sei-db/state_db/ss/offload/historical/scylla_test.go deleted file mode 100644 index 9e483877c7..0000000000 --- a/sei-db/state_db/ss/offload/historical/scylla_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package historical - -import ( - "testing" - "time" - - "github.com/gocql/gocql" - "github.com/stretchr/testify/require" -) - -func TestScyllaConfigApplyDefaults(t *testing.T) { - cfg := ScyllaConfig{ - Hosts: []string{"127.0.0.1"}, - Keyspace: "sei_history", - } - cfg.ApplyDefaults() - require.Equal(t, defaultScyllaConsistency, cfg.Consistency) - require.Equal(t, defaultScyllaTimeout, cfg.Timeout) - require.Equal(t, defaultScyllaTimeout, cfg.ConnectTimeout) - require.Equal(t, defaultScyllaNumConns, cfg.NumConns) -} - -func TestScyllaConfigValidate(t *testing.T) { - tests := []struct { - name string - cfg ScyllaConfig - err string - }{ - {"missing hosts", ScyllaConfig{Keyspace: "ks"}, "hosts"}, - {"blank host", ScyllaConfig{Hosts: []string{" "}, Keyspace: "ks"}, "blanks"}, - {"missing keyspace", ScyllaConfig{Hosts: []string{"127.0.0.1"}}, "keyspace"}, - {"password without username", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", Password: "secret"}, "username"}, - {"bad consistency", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", Consistency: "bad"}, "consistency"}, - {"negative timeout", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", Timeout: -time.Second}, "timeout"}, - {"negative connect timeout", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", ConnectTimeout: -time.Second}, "connect"}, - {"negative conns", ScyllaConfig{Hosts: []string{"127.0.0.1"}, Keyspace: "ks", NumConns: -1}, "conns"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.cfg.Validate() - require.Error(t, err) - require.Contains(t, err.Error(), tc.err) - }) - } -} - -func TestParseConsistency(t *testing.T) { - tests := []struct { - in string - out gocql.Consistency - }{ - {"", gocql.LocalQuorum}, - {"local_quorum", gocql.LocalQuorum}, - {"LOCAL_ONE", gocql.LocalOne}, - {"one", gocql.One}, - {"quorum", gocql.Quorum}, - {"all", gocql.All}, - } - for _, tc := range tests { - t.Run(tc.in, func(t *testing.T) { - got, err := parseConsistency(tc.in) - require.NoError(t, err) - require.Equal(t, tc.out, got) - }) - } -} - -func TestVersionBucket(t *testing.T) { - require.Equal(t, 0, VersionBucket(0)) - require.Equal(t, 1, VersionBucket(1)) - require.Equal(t, 0, VersionBucket(VersionBucketCount)) - require.Equal(t, 1, VersionBucket(-1)) -} - -func TestPointLookupCQLShape(t *testing.T) { - for _, q := range []string{getLookupCQL, hasLookupCQL} { - for _, frag := range []string{ - "FROM state_mutations", - "store_name = ?", - "state_key = ?", - "version <= ?", - "ORDER BY version DESC", - "LIMIT 1", - } { - require.Contains(t, q, frag) - } - } -} - -func TestLatestVersionCQLShape(t *testing.T) { - require.Contains(t, selectLatestVersionCQL, "FROM state_versions") - require.Contains(t, selectLatestVersionCQL, "bucket = ?") - require.Contains(t, selectLatestVersionCQL, "LIMIT 1") -} diff --git a/sei-db/state_db/ss/store.go b/sei-db/state_db/ss/store.go index 68611e51e7..e0c379d6f6 100644 --- a/sei-db/state_db/ss/store.go +++ b/sei-db/state_db/ss/store.go @@ -2,8 +2,6 @@ package ss import ( "fmt" - "strings" - "time" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" @@ -15,20 +13,13 @@ import ( // The backend (pebbledb or rocksdb) is resolved at compile time via build-tag-gated // files in the backend package. When WriteMode/ReadMode are both cosmos_only (the default), // the EVM stores are not opened and the composite store behaves identically to a plain cosmos state store. +// When the Bigtable historical offload is configured, the store is wrapped so +// pruned point reads fall back to Bigtable. func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.StateStore, error) { primary, err := composite.NewCompositeStateStore(ssConfig, homeDir) if err != nil { return nil, err } - scyllaCfg := historical.ScyllaConfig{ - Hosts: splitCSV(ssConfig.HistoricalOffloadScyllaHosts), - Keyspace: ssConfig.HistoricalOffloadScyllaKeyspace, - Username: ssConfig.HistoricalOffloadScyllaUsername, - Password: ssConfig.HistoricalOffloadScyllaPassword, - Datacenter: ssConfig.HistoricalOffloadScyllaDatacenter, - Consistency: ssConfig.HistoricalOffloadScyllaConsistency, - Timeout: time.Duration(ssConfig.HistoricalOffloadScyllaTimeoutMS) * time.Millisecond, - } bigtableCfg := historical.BigtableConfig{ ProjectID: ssConfig.HistoricalOffloadBigtableProjectID, InstanceID: ssConfig.HistoricalOffloadBigtableInstance, @@ -37,44 +28,13 @@ func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.Stat AppProfile: ssConfig.HistoricalOffloadBigtableAppProfile, Shards: ssConfig.HistoricalOffloadBigtableShards, } - // Scylla's configured check uses the raw hosts string so a garbage value - // (e.g. only commas) fails reader validation loudly instead of silently - // running without the fallback. - scyllaConfigured := strings.TrimSpace(ssConfig.HistoricalOffloadScyllaHosts) != "" || scyllaCfg.Configured() - if scyllaConfigured && bigtableCfg.Configured() { - _ = primary.Close() - return nil, fmt.Errorf("only one historical offload fallback can be configured") - } - if !scyllaConfigured && !bigtableCfg.Configured() { + if !bigtableCfg.Configured() { return primary, nil } - if bigtableCfg.Configured() { - reader, err := historical.NewBigtableReader(bigtableCfg) - if err != nil { - _ = primary.Close() - return nil, fmt.Errorf("open bigtable historical offload reader: %w", err) - } - return historical.NewFallbackStateStore(primary, reader, ssConfig.HistoricalOffloadEarliestVersion), nil - } - reader, err := historical.NewScyllaReader(scyllaCfg) + reader, err := historical.NewBigtableReader(bigtableCfg) if err != nil { _ = primary.Close() - return nil, fmt.Errorf("open scylla/cassandra historical offload reader: %w", err) + return nil, fmt.Errorf("open bigtable historical offload reader: %w", err) } return historical.NewFallbackStateStore(primary, reader, ssConfig.HistoricalOffloadEarliestVersion), nil } - -func splitCSV(value string) []string { - if strings.TrimSpace(value) == "" { - return nil - } - parts := strings.Split(value, ",") - out := make([]string, 0, len(parts)) - for _, part := range parts { - part = strings.TrimSpace(part) - if part != "" { - out = append(out, part) - } - } - return out -} From 66b3d0edb3b4954833f809f73ee4657e82a1d7c7 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Fri, 17 Jul 2026 16:52:15 -0400 Subject: [PATCH 40/41] Collapse the sink to a single WriteBatch interface With one backend the Sink/BatchSink split, the per-record Write fallback, and the uncalled sink-side LastVersion were dead generality; the consumer now writes batches directly. Add a direct VersionBucket test since the bucket encoding is live production data. Co-Authored-By: Claude Fable 5 --- sei-db/state_db/ss/offload/consumer/bigtable.go | 11 ----------- .../ss/offload/consumer/bigtable_test.go | 2 +- sei-db/state_db/ss/offload/consumer/consumer.go | 17 +---------------- sei-db/state_db/ss/offload/consumer/sink.go | 7 +------ .../ss/offload/historical/bigtable_test.go | 7 +++++++ 5 files changed, 10 insertions(+), 34 deletions(-) diff --git a/sei-db/state_db/ss/offload/consumer/bigtable.go b/sei-db/state_db/ss/offload/consumer/bigtable.go index fd98a791a7..0e70ce46a3 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable.go @@ -22,7 +22,6 @@ const ( type bigtableSink struct { client *historical.BigtableClient applyBulk historical.BigtableApplyBulkFunc - readRows historical.BigtableReadRowsFunc family string shards int bulkChunkRows int @@ -30,7 +29,6 @@ type bigtableSink struct { } var _ Sink = (*bigtableSink)(nil) -var _ BatchSink = (*bigtableSink)(nil) func NewBigtableSink(cfg BigtableConfig) (Sink, error) { cfg.ApplyDefaults() @@ -45,7 +43,6 @@ func NewBigtableSink(cfg BigtableConfig) (Sink, error) { return &bigtableSink{ client: client, applyBulk: client.ApplyBulk, - readRows: client.ReadRows, family: cfg.Family, shards: cfg.Shards, bulkChunkRows: defaultBigtableMutationChunkRows, @@ -60,14 +57,6 @@ func (s *bigtableSink) Close() error { return nil } -func (s *bigtableSink) LastVersion(ctx context.Context) (int64, error) { - return historical.BigtableLastVersion(ctx, s.readRows) -} - -func (s *bigtableSink) Write(ctx context.Context, rec Record) error { - return s.WriteBatch(ctx, []Record{rec}) -} - func (s *bigtableSink) WriteBatch(ctx context.Context, records []Record) error { records = compactRecords(records) if len(records) == 0 { diff --git a/sei-db/state_db/ss/offload/consumer/bigtable_test.go b/sei-db/state_db/ss/offload/consumer/bigtable_test.go index a01bff0800..63bae0d7b2 100644 --- a/sei-db/state_db/ss/offload/consumer/bigtable_test.go +++ b/sei-db/state_db/ss/offload/consumer/bigtable_test.go @@ -37,7 +37,7 @@ func TestBigtableSinkWritesMutationRowsAndVersionMarker(t *testing.T) { Upgrades: []*proto.TreeNameUpgrade{{Name: "new-store"}}, } - require.NoError(t, sink.Write(context.Background(), Record{Topic: "t", Partition: 1, Offset: 2, Entry: entry})) + require.NoError(t, sink.WriteBatch(context.Background(), []Record{{Topic: "t", Partition: 1, Offset: 2, Entry: entry}})) require.Len(t, rows, 4) require.ElementsMatch(t, []string{ historical.BigtableMutationRowKey("bank", []byte("k1"), 7, historical.DefaultBigtableShards), diff --git a/sei-db/state_db/ss/offload/consumer/consumer.go b/sei-db/state_db/ss/offload/consumer/consumer.go index 345c8749b5..d021cd9c6b 100644 --- a/sei-db/state_db/ss/offload/consumer/consumer.go +++ b/sei-db/state_db/ss/offload/consumer/consumer.go @@ -240,7 +240,7 @@ func (c *Consumer) writeBatchWithRetry(ctx context.Context, records []Record) er backoff := sinkBaseBackoff var lastErr error for attempt := 1; attempt <= sinkMaxAttempts; attempt++ { - err := c.writeRecords(ctx, records) + err := c.sink.WriteBatch(ctx, records) if err == nil { return nil } @@ -275,21 +275,6 @@ func sleepWithContext(ctx context.Context, d time.Duration) error { } } -func (c *Consumer) writeRecords(ctx context.Context, records []Record) error { - if len(records) == 0 { - return nil - } - if sink, ok := c.sink.(BatchSink); ok { - return sink.WriteBatch(ctx, records) - } - for _, rec := range records { - if err := c.sink.Write(ctx, rec); err != nil { - return err - } - } - return nil -} - func shardFor(partition, workers int) int { if partition < 0 { partition = -partition diff --git a/sei-db/state_db/ss/offload/consumer/sink.go b/sei-db/state_db/ss/offload/consumer/sink.go index 79da24d0cd..deef46c336 100644 --- a/sei-db/state_db/ss/offload/consumer/sink.go +++ b/sei-db/state_db/ss/offload/consumer/sink.go @@ -16,11 +16,6 @@ type Record struct { } type Sink interface { - Write(ctx context.Context, rec Record) error - LastVersion(ctx context.Context) (int64, error) - Close() error -} - -type BatchSink interface { WriteBatch(ctx context.Context, records []Record) error + Close() error } diff --git a/sei-db/state_db/ss/offload/historical/bigtable_test.go b/sei-db/state_db/ss/offload/historical/bigtable_test.go index e82c0d4541..e5a93e9291 100644 --- a/sei-db/state_db/ss/offload/historical/bigtable_test.go +++ b/sei-db/state_db/ss/offload/historical/bigtable_test.go @@ -204,6 +204,13 @@ func TestReadRowsWithRetryDoesNotRetryNonRetryable(t *testing.T) { require.Equal(t, 1, attempts) } +func TestVersionBucket(t *testing.T) { + require.Equal(t, 0, VersionBucket(0)) + require.Equal(t, 1, VersionBucket(1)) + require.Equal(t, 0, VersionBucket(VersionBucketCount)) + require.Equal(t, 1, VersionBucket(-1)) +} + func TestBigtableLastVersionScansBuckets(t *testing.T) { versions := map[int]int64{ 3: 42, From 8cb49c1eabcf11b1a9f3147baa36a375a8ebb817 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Tue, 21 Jul 2026 14:04:00 -0400 Subject: [PATCH 41/41] Gate historical offload behind an explicit backend config The Kafka+Bigtable historical offload is Giga-only and must not be exposed or enabled on current mainnet, so make the pluggable-backend design explicit: - state-store.historical-offload-backend selects the fallback: empty (default) disables it entirely, "bigtable" enables the Bigtable reader, and anything else fails NewStateStore loudly. A partially filled Bigtable config now errors via Validate instead of being silently treated as unconfigured. - The historical-offload keys are removed from the generated state-store TOML template; they still parse when added to app.toml manually, and the consumer README documents that. Co-Authored-By: Claude Fable 5 --- app/seidb.go | 2 ++ app/seidb_test.go | 4 +++ sei-db/config/ss_config.go | 9 +++-- sei-db/config/toml.go | 17 --------- sei-db/config/toml_test.go | 9 +++-- sei-db/state_db/ss/offload/consumer/README.md | 4 +++ sei-db/state_db/ss/store.go | 36 ++++++++++--------- sei-db/state_db/ss/store_test.go | 12 +++++++ 8 files changed, 53 insertions(+), 40 deletions(-) diff --git a/app/seidb.go b/app/seidb.go index a159c0cde5..b00d389416 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -55,6 +55,7 @@ const ( FlagEVMSSSeparateDBs = "state-store.evm-ss-separate-dbs" // Historical SS offload fallback. + FlagHistoricalOffloadBackend = "state-store.historical-offload-backend" FlagHistoricalOffloadBigtableProjectID = "state-store.historical-offload-bigtable-project-id" FlagHistoricalOffloadBigtableInstance = "state-store.historical-offload-bigtable-instance" FlagHistoricalOffloadBigtableTable = "state-store.historical-offload-bigtable-table" @@ -217,6 +218,7 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig { ssConfig.EVMDBDirectory = cast.ToString(appOpts.Get(FlagEVMSSDirectory)) ssConfig.SeparateEVMSubDBs = cast.ToBool(appOpts.Get(FlagEVMSSSeparateDBs)) ssConfig.EVMSplit = cast.ToBool(appOpts.Get(FlagEVMSSSplit)) + ssConfig.HistoricalOffloadBackend = cast.ToString(appOpts.Get(FlagHistoricalOffloadBackend)) ssConfig.HistoricalOffloadBigtableProjectID = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableProjectID)) ssConfig.HistoricalOffloadBigtableInstance = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableInstance)) ssConfig.HistoricalOffloadBigtableTable = cast.ToString(appOpts.Get(FlagHistoricalOffloadBigtableTable)) diff --git a/app/seidb_test.go b/app/seidb_test.go index 09b9252b98..b9a3f5c010 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -65,6 +65,8 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSSConfig.EVMSplit case FlagEVMSSSeparateDBs: return defaultSSConfig.SeparateEVMSubDBs + case FlagHistoricalOffloadBackend: + return defaultSSConfig.HistoricalOffloadBackend case FlagHistoricalOffloadBigtableProjectID: return defaultSSConfig.HistoricalOffloadBigtableProjectID case FlagHistoricalOffloadBigtableInstance: @@ -221,6 +223,7 @@ func TestParseSSConfigs_EVMFlags(t *testing.T) { func TestParseSSConfigs_HistoricalBigtableFlags(t *testing.T) { appOpts := mapAppOpts{ FlagSSEnable: true, + FlagHistoricalOffloadBackend: "bigtable", FlagHistoricalOffloadBigtableProjectID: "sei-project", FlagHistoricalOffloadBigtableInstance: "sei-history", FlagHistoricalOffloadBigtableTable: "state_mutations", @@ -233,6 +236,7 @@ func TestParseSSConfigs_HistoricalBigtableFlags(t *testing.T) { ssConfig := parseSSConfigs(appOpts) assert.True(t, ssConfig.Enable) + assert.Equal(t, "bigtable", ssConfig.HistoricalOffloadBackend) assert.Equal(t, "sei-project", ssConfig.HistoricalOffloadBigtableProjectID) assert.Equal(t, "sei-history", ssConfig.HistoricalOffloadBigtableInstance) assert.Equal(t, "state_mutations", ssConfig.HistoricalOffloadBigtableTable) diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index 8270f6cd2b..820406e2f1 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -81,8 +81,13 @@ type StateStoreConfig struct { // preserving the same logical store key and full key encoding inside each DB. SeparateEVMSubDBs bool `mapstructure:"evm-separate-dbs"` - // HistoricalOffloadBigtableProjectID enables Bigtable fallback reads when - // project, instance, and table are set. + // HistoricalOffloadBackend selects the historical offload read fallback. + // Empty (the default) disables the fallback entirely; "bigtable" enables + // the Bigtable reader configured by the connection fields below. + HistoricalOffloadBackend string `mapstructure:"historical-offload-backend"` + + // HistoricalOffloadBigtable* are the Bigtable connection parameters used + // when HistoricalOffloadBackend is "bigtable". HistoricalOffloadBigtableProjectID string `mapstructure:"historical-offload-bigtable-project-id"` HistoricalOffloadBigtableInstance string `mapstructure:"historical-offload-bigtable-instance"` HistoricalOffloadBigtableTable string `mapstructure:"historical-offload-bigtable-table"` diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 37a2b0579d..d7e782bb89 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -154,23 +154,6 @@ evm-ss-split = {{ .StateStore.EVMSplit }} # When false, all EVM data stays in one DB using the current unified layout. # When true, data is routed to separate DBs while preserving the same evm key prefix format. evm-ss-separate-dbs = {{ .StateStore.SeparateEVMSubDBs }} - -# Optional Bigtable historical-state fallback. When project, instance, and -# table are set, point reads for versions pruned from local SS fall back to -# Bigtable. Iterators still use local SS. Use the same family/shards as the -# Bigtable consumer. -historical-offload-bigtable-project-id = "{{ .StateStore.HistoricalOffloadBigtableProjectID }}" -historical-offload-bigtable-instance = "{{ .StateStore.HistoricalOffloadBigtableInstance }}" -historical-offload-bigtable-table = "{{ .StateStore.HistoricalOffloadBigtableTable }}" -historical-offload-bigtable-family = "{{ .StateStore.HistoricalOffloadBigtableFamily }}" -historical-offload-bigtable-app-profile = "{{ .StateStore.HistoricalOffloadBigtableAppProfile }}" -historical-offload-bigtable-shards = {{ .StateStore.HistoricalOffloadBigtableShards }} - -# Earliest version (inclusive) fully ingested into the historical backend. -# When > 0 it becomes the node's advertised earliest version, so RPC height -# gates admit pruned heights the fallback can serve; point reads below it stay -# local. Leave 0 until backfill/ingestion coverage reaches the desired height. -historical-offload-earliest-version = {{ .StateStore.HistoricalOffloadEarliestVersion }} ` // ReceiptStoreConfigTemplate defines the configuration template for receipt-store diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index 8fa681d5a2..81522f947a 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -99,11 +99,10 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, `evm-ss-db-directory = ""`, "Missing evm-ss-db-directory") require.Contains(t, output, `evm-ss-split = false`, "Missing or incorrect evm-ss-split") require.Contains(t, output, "evm-ss-separate-dbs = false", "Missing or incorrect evm-ss-separate-dbs") - require.Contains(t, output, `historical-offload-bigtable-project-id = ""`, "Missing historical Bigtable project") - require.Contains(t, output, `historical-offload-bigtable-instance = ""`, "Missing historical Bigtable instance") - require.Contains(t, output, `historical-offload-bigtable-table = ""`, "Missing historical Bigtable table") - require.Contains(t, output, "historical-offload-bigtable-shards = 0", "Missing historical Bigtable shards") - require.Contains(t, output, "historical-offload-earliest-version = 0", "Missing historical earliest version") + // The historical-offload keys are deliberately absent from the generated + // default config (internal/Giga-only for now); they still parse when an + // operator adds them to app.toml manually. + require.NotContains(t, output, "historical-offload", "historical-offload keys must not be in the default template") } // TestReceiptStoreConfigTemplate verifies that all field paths in the receipt-store TOML template diff --git a/sei-db/state_db/ss/offload/consumer/README.md b/sei-db/state_db/ss/offload/consumer/README.md index 7be12406d1..5a3435ed60 100644 --- a/sei-db/state_db/ss/offload/consumer/README.md +++ b/sei-db/state_db/ss/offload/consumer/README.md @@ -53,6 +53,7 @@ Enable fallback reads in the node config: ```toml [state-store] +historical-offload-backend = "bigtable" historical-offload-bigtable-project-id = "my-gcp-project" historical-offload-bigtable-instance = "sei-history" historical-offload-bigtable-table = "state_mutations" @@ -61,6 +62,9 @@ historical-offload-bigtable-app-profile = "" historical-offload-bigtable-shards = 256 ``` +These keys are deliberately not part of the generated default config +(internal/Giga-only for now) and must be added to `app.toml` manually. + Fallback activates only for point reads where the requested version is below the local SS earliest version. Missing rows and tombstones return empty state, same as local SS. Reads ahead of the backend's last ingested version (consumer lag) diff --git a/sei-db/state_db/ss/store.go b/sei-db/state_db/ss/store.go index da20013c33..e59ad4a1d8 100644 --- a/sei-db/state_db/ss/store.go +++ b/sei-db/state_db/ss/store.go @@ -14,28 +14,32 @@ import ( // The backend (pebbledb or rocksdb) is resolved at compile time via build-tag-gated // files in the backend package. When WriteMode/ReadMode are both cosmos_only (the default), // the EVM stores are not opened and the composite store behaves identically to a plain cosmos state store. -// When the Bigtable historical offload is configured, the store is wrapped so -// pruned point reads fall back to Bigtable. +// When historical-offload-backend selects a backend (currently "bigtable"), +// the store is wrapped so pruned point reads fall back to it. func NewStateStore(homeDir string, ssConfig config.StateStoreConfig) (types.StateStore, error) { primary, err := composite.NewCompositeStateStore(ssConfig, homeDir) if err != nil { return nil, err } - bigtableCfg := bigtable.Config{ - ProjectID: ssConfig.HistoricalOffloadBigtableProjectID, - InstanceID: ssConfig.HistoricalOffloadBigtableInstance, - Table: ssConfig.HistoricalOffloadBigtableTable, - Family: ssConfig.HistoricalOffloadBigtableFamily, - AppProfile: ssConfig.HistoricalOffloadBigtableAppProfile, - Shards: ssConfig.HistoricalOffloadBigtableShards, - } - if !bigtableCfg.Configured() { + switch ssConfig.HistoricalOffloadBackend { + case "": return primary, nil - } - reader, err := historical.NewBigtableReader(bigtableCfg) - if err != nil { + case "bigtable": + reader, err := historical.NewBigtableReader(bigtable.Config{ + ProjectID: ssConfig.HistoricalOffloadBigtableProjectID, + InstanceID: ssConfig.HistoricalOffloadBigtableInstance, + Table: ssConfig.HistoricalOffloadBigtableTable, + Family: ssConfig.HistoricalOffloadBigtableFamily, + AppProfile: ssConfig.HistoricalOffloadBigtableAppProfile, + Shards: ssConfig.HistoricalOffloadBigtableShards, + }) + if err != nil { + _ = primary.Close() + return nil, fmt.Errorf("open bigtable historical offload reader: %w", err) + } + return historical.NewFallbackStateStore(primary, reader, ssConfig.HistoricalOffloadEarliestVersion), nil + default: _ = primary.Close() - return nil, fmt.Errorf("open bigtable historical offload reader: %w", err) + return nil, fmt.Errorf("unsupported historical offload backend %q", ssConfig.HistoricalOffloadBackend) } - return historical.NewFallbackStateStore(primary, reader, ssConfig.HistoricalOffloadEarliestVersion), nil } diff --git a/sei-db/state_db/ss/store_test.go b/sei-db/state_db/ss/store_test.go index f6e7686c48..5ab0526358 100644 --- a/sei-db/state_db/ss/store_test.go +++ b/sei-db/state_db/ss/store_test.go @@ -50,3 +50,15 @@ func TestNewStateStore(t *testing.T) { require.Equal(t, fmt.Sprintf("value%d", i), string(value)) } } + +func TestNewStateStoreRejectsUnsupportedHistoricalOffloadBackend(t *testing.T) { + ssConfig := config.StateStoreConfig{ + Backend: config.PebbleDBBackend, + AsyncWriteBuffer: 100, + KeepRecent: 500, + HistoricalOffloadBackend: "scylla", + } + + _, err := NewStateStore(filepath.Join(t.TempDir(), "pebbledb"), ssConfig) + require.ErrorContains(t, err, `unsupported historical offload backend "scylla"`) +}