From 2a2ca0794e666f44212bae7287a4dc01a4326319 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 13 Aug 2026 16:31:55 -0400 Subject: [PATCH 1/8] feat(seidb): add exact-version state store snapshots Add opt-in Pebble checkpoints for State Store with ordered queue barriers, crash-safe publication, retention, metrics, and SC-aligned cadence. Preserve honest per-database version bounds across snapshot and prune races, including split EVM stores. Co-authored-by: Cursor --- app/config_fuzz_test.go | 39 +- app/seidb.go | 7 + app/testdata/state-store.golden | 4 + app/testdata/state-store.keys.golden | 1 + sei-cosmos/server/config/config.go | 8 + .../config/testdata/server_config.golden | 4 + sei-cosmos/storev2/rootmulti/store.go | 34 + sei-cosmos/storev2/rootmulti/store_test.go | 51 + sei-db/common/utils/path.go | 7 + sei-db/config/sc_config.go | 29 + sei-db/config/ss_config.go | 52 + sei-db/config/ss_config_test.go | 106 ++ sei-db/config/toml.go | 14 + sei-db/config/toml_test.go | 1 + sei-db/db_engine/pebbledb/mvcc/db.go | 157 ++- .../db_engine/pebbledb/mvcc/db_ascending.go | 27 +- sei-db/db_engine/pebbledb/mvcc/db_test.go | 60 +- sei-db/db_engine/pebbledb/mvcc/prune_test.go | 88 ++ sei-db/db_engine/types/types.go | 24 + sei-db/management/checkpoint_scheduler.go | 59 ++ sei-db/state_db/sc/composite/store.go | 9 +- sei-db/state_db/ss/composite/recovery_test.go | 24 +- sei-db/state_db/ss/composite/snapshot.go | 737 ++++++++++++++ .../state_db/ss/composite/snapshot_metrics.go | 94 ++ sei-db/state_db/ss/composite/snapshot_test.go | 949 ++++++++++++++++++ sei-db/state_db/ss/composite/store.go | 72 +- sei-db/state_db/ss/cosmos/store.go | 22 + sei-db/state_db/ss/evm/db_test.go | 22 + sei-db/state_db/ss/evm/store.go | 108 +- 29 files changed, 2733 insertions(+), 76 deletions(-) create mode 100644 sei-db/config/ss_config_test.go create mode 100644 sei-db/management/checkpoint_scheduler.go create mode 100644 sei-db/state_db/ss/composite/snapshot.go create mode 100644 sei-db/state_db/ss/composite/snapshot_metrics.go create mode 100644 sei-db/state_db/ss/composite/snapshot_test.go diff --git a/app/config_fuzz_test.go b/app/config_fuzz_test.go index fd735504b9..1b8c7617ce 100644 --- a/app/config_fuzz_test.go +++ b/app/config_fuzz_test.go @@ -23,11 +23,12 @@ import ( // // - parseSCConfigs guards almost every read with `if v := opts.Get(k); v != nil`, // so a key absent from an older app.toml keeps its non-zero in-code default. -// - parseSSConfigs guards nothing. Every read is a bare cast of a possibly-nil -// value, so an absent key resolves to the zero value and overwrites the -// default. ss-keep-recent becomes 0 (keep everything, unbounded disk growth), +// - parseSSConfigs leaves every legacy read unguarded. Each is a bare cast of a +// possibly-nil value, so an absent key resolves to the zero value and overwrites +// the default. ss-keep-recent becomes 0 (keep everything, unbounded disk growth), // ss-async-write-buffer becomes 0 (synchronous writes), ss-backend becomes "" -// and ss-enable becomes false. +// and ss-enable becomes false. ss-snapshot-enable is the one guarded read, so an +// app.toml written before SS snapshots existed keeps the in-code default. // // Neither reader returns an error, so nothing about the second case is visible at // boot. It is recorded here as behavior rather than reported as a defect: the @@ -84,8 +85,8 @@ var scKeys = []configtest.KeySpec{ }, } -// ssKeys is the [state-store] read-site manifest. Every row is unguarded and -// unchecked — the section has no presence checks at all. +// ssKeys is the [state-store] read-site manifest. Every row is unchecked; +// SnapshotEnable is guarded while the legacy rows remain unguarded. // // StateStoreConfig also carries KeepLastVersion and UseDefaultComparer, which are // absent here because parseSSConfigs reads neither: they hold their in-code @@ -112,6 +113,10 @@ var ssKeys = []configtest.KeySpec{ {Key: FlagSSImportNumWorkers, Path: "ImportNumWorkers", Cast: configtest.CastInt, Unguarded: true}, {Key: FlagSSDirectory, Path: "DBDirectory", Cast: configtest.CastString, Unguarded: true}, {Key: FlagSSReadWriteMetrics, Path: "EnableReadWriteMetrics", Cast: configtest.CastBool, Unguarded: true}, + { + Key: FlagSSSnapshotEnable, Path: "SnapshotEnable", Cast: configtest.CastBool, + Why: "guarded so app.toml files created before SS snapshots keep the default-off rollout", + }, {Key: FlagEVMSSDirectory, Path: "EVMDBDirectory", Cast: configtest.CastString, Unguarded: true}, {Key: FlagEVMSSSeparateDBs, Path: "SeparateEVMSubDBs", Cast: configtest.CastBool, Unguarded: true}, {Key: FlagEVMSSSplit, Path: "EVMSplit", Cast: configtest.CastBool, Unguarded: true}, @@ -253,9 +258,8 @@ func FuzzParseSCConfigs(f *testing.F) { } // FuzzParseSSConfigs drives every [state-store] key through arbitrary raw values. -// Because the whole section is unguarded, the property being pinned for a nil -// value is the clobber itself: the resolved field must equal the cast's zero, not -// the in-code default. +// For legacy unguarded rows, a nil value must clobber the field to the cast's +// zero. Guarded rows such as SnapshotEnable must retain their in-code default. func FuzzParseSSConfigs(f *testing.F) { seeds := configtest.NewSeeds(f, fuzzing.ConfigValue) @@ -277,7 +281,8 @@ func FuzzParseSSConfigs(f *testing.F) { seeds.AddRow(uint(3), fuzzing.KindInt64, "", int64(200000), false) seeds.AddRow(uint(3), fuzzing.KindNil, "", int64(0), false) // nil clobbers KeepRecent to 0 seeds.AddRow(uint(6), fuzzing.KindString, "/var/lib/sei/ss", int64(0), false) - seeds.AddRow(uint(10), fuzzing.KindBoolString, "", int64(0), true) + seeds.AddRow(uint(8), fuzzing.KindBool, "", int64(0), true) // explicit snapshot opt-in; the default is off + seeds.AddRow(uint(11), fuzzing.KindBoolString, "", int64(0), true) // The clobber cuts both ways for the four rows below. Because the section is unguarded, // an absent key resolves them to their cast's zero, and so does the malformed seed on an @@ -287,7 +292,7 @@ func FuzzParseSSConfigs(f *testing.F) { seeds.AddRow(uint(4), fuzzing.KindInt64, "", int64(1800), false) // prune every 30 min rather than the default 600s seeds.AddRow(uint(5), fuzzing.KindInt64, "", int64(4), false) // four import workers rather than the default 1 seeds.AddRow(uint(7), fuzzing.KindBool, "", int64(0), true) // pebbledb read/write metrics on; the default is off - seeds.AddRow(uint(9), fuzzing.KindBool, "", int64(0), true) // EVM state in its own sub-DBs; the default is shared + seeds.AddRow(uint(10), fuzzing.KindBool, "", int64(0), true) // EVM state in its own sub-DBs; the default is shared configtest.CheckEveryRowHasADiscriminatingSeed(f, "state-store", readSS, ssKeys, seeds) @@ -520,8 +525,9 @@ func TestParseSCConfigsAbsentBaseline(t *testing.T) { // TestParseSSConfigsAbsentBaselineIsZeroClobbered records the clobber in full: an // app.toml with no [state-store] section resolves to a config in which every -// operator-visible knob has been overwritten with a zero value, including the two -// that change the node's disk behavior without any log line. +// unguarded operator-visible knob has been overwritten with a zero value, including +// the two that change the node's disk behavior without any log line. SnapshotEnable +// is the one field that survives, because its read is guarded. func TestParseSSConfigsAbsentBaselineIsZeroClobbered(t *testing.T) { got := parseSSConfigs(configtest.AppOpts{}) @@ -673,6 +679,13 @@ func TestManifestNamesEveryField(t *testing.T) { // manager would otherwise try to map a key onto. "KeepLastVersion", "UseDefaultComparer", + // The three below are unreachable for a different reason, and the distinction is the + // point: they are tagged mapstructure:"-" so no key can bind them even in principle, + // and AlignSSSnapshotWithSC derives all three at runtime from the state-commit cadence. + // ss-snapshot-enable is the only SS-side knob, and it has a row of its own above. + "SnapshotInterval", + "SnapshotKeepRecent", + "SnapshotMinTimeInterval", ) }) t.Run("light_invariance", func(t *testing.T) { diff --git a/app/seidb.go b/app/seidb.go index 9307af2bb0..716583de37 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -48,6 +48,7 @@ const ( FlagSSPruneInterval = "state-store.ss-prune-interval" FlagSSImportNumWorkers = "state-store.ss-import-num-workers" FlagSSReadWriteMetrics = "state-store.ss-enable-read-write-metrics" + FlagSSSnapshotEnable = "state-store.ss-snapshot-enable" // EVM SS optimization (embedded in SS config, controlled via write/read mode) FlagEVMSSDirectory = "state-store.evm-ss-db-directory" @@ -204,6 +205,12 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig { ssConfig.DBDirectory = cast.ToString(appOpts.Get(FlagSSDirectory)) ssConfig.EnableReadWriteMetrics = cast.ToBool(appOpts.Get(FlagSSReadWriteMetrics)) + // An absent key is an app.toml rendered before SS snapshots existed. Keep + // the in-code default (off) rather than relying on a nil cast. + if v := appOpts.Get(FlagSSSnapshotEnable); v != nil { + ssConfig.SnapshotEnable = cast.ToBool(v) + } + // EVM optimization fields (embedded in SS config) ssConfig.EVMDBDirectory = cast.ToString(appOpts.Get(FlagEVMSSDirectory)) ssConfig.SeparateEVMSubDBs = cast.ToBool(appOpts.Get(FlagEVMSSSeparateDBs)) diff --git a/app/testdata/state-store.golden b/app/testdata/state-store.golden index c60c8b49e2..57d01bd92f 100644 --- a/app/testdata/state-store.golden +++ b/app/testdata/state-store.golden @@ -8,6 +8,10 @@ ImportNumWorkers = int(1) EnableReadWriteMetrics = bool(false) KeepLastVersion = bool(true) UseDefaultComparer = bool(false) +SnapshotEnable = bool(false) +SnapshotInterval = int64(0) +SnapshotKeepRecent = int(0) +SnapshotMinTimeInterval = time.Duration(0s) EVMSplit = bool(false) EVMDBDirectory = string("") SeparateEVMSubDBs = bool(false) diff --git a/app/testdata/state-store.keys.golden b/app/testdata/state-store.keys.golden index fc808ad460..a96612730c 100644 --- a/app/testdata/state-store.keys.golden +++ b/app/testdata/state-store.keys.golden @@ -6,6 +6,7 @@ "state-store.ss-import-num-workers" "state-store.ss-db-directory" "state-store.ss-enable-read-write-metrics" +"state-store.ss-snapshot-enable" "state-store.evm-ss-db-directory" "state-store.evm-ss-separate-dbs" "state-store.evm-ss-split" diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index c46a296125..7bf8b4d70a 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -508,6 +508,13 @@ func GetConfig(v *viper.Viper) (Config, error) { memIAVLConfig.SnapshotPrefetchThreshold = v.GetFloat64("state-commit.sc-snapshot-prefetch-threshold") } + // Absent key means an app.toml rendered before SS snapshots existed, which + // should keep the in-code default (off) rather than rely on viper's zero. + ssSnapshotEnable := config.DefaultStateStoreConfig().SnapshotEnable + if v.IsSet("state-store.ss-snapshot-enable") { + ssSnapshotEnable = v.GetBool("state-store.ss-snapshot-enable") + } + // Apply the in-code default when the key is absent so that nodes upgrading // with an older app.toml (which lacks this key) are still bounded rather // than running with unlimited connections. @@ -636,6 +643,7 @@ func GetConfig(v *viper.Viper) (Config, error) { EnableReadWriteMetrics: v.GetBool( "state-store.ss-enable-read-write-metrics", ), + SnapshotEnable: ssSnapshotEnable, EVMSplit: v.GetBool("state-store.evm-ss-split"), EVMDBDirectory: v.GetString("state-store.evm-ss-db-directory"), SeparateEVMSubDBs: v.GetBool("state-store.evm-ss-separate-dbs"), diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index a8b4777fa6..442b08b5d9 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -140,6 +140,10 @@ StateStore.ImportNumWorkers = int(1) StateStore.EnableReadWriteMetrics = bool(false) StateStore.KeepLastVersion = bool(true) StateStore.UseDefaultComparer = bool(false) +StateStore.SnapshotEnable = bool(false) +StateStore.SnapshotInterval = int64(0) +StateStore.SnapshotKeepRecent = int(0) +StateStore.SnapshotMinTimeInterval = time.Duration(0s) StateStore.EVMSplit = bool(false) StateStore.EVMDBDirectory = string("") StateStore.SeparateEVMSubDBs = bool(false) diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 43b8387347..2f04ff75f6 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -37,6 +37,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss" + sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" dbm "github.com/tendermint/tm-db" ) @@ -48,10 +49,24 @@ var ( _ types.Queryable = (*Store)(nil) ) +// stateStoreSnapshotScheduler is the commit path's half of the SS snapshot +// contract: flush tells the state store which version it has just finished +// enqueueing, and the store decides whether that version is a boundary. +type stateStoreSnapshotScheduler interface { + ScheduleSnapshot(version int64) +} + +// ss.NewStateStore returns the interface, so the capability is resolved by type +// assertion at startup. This pins the only implementation, so wrapping the state +// store without carrying the method through fails the build here rather than +// silently ending SS snapshots at runtime. +var _ stateStoreSnapshotScheduler = (*sscomposite.CompositeStateStore)(nil) + type Store struct { mtx sync.RWMutex scStore sctypes.Committer ssStore seidbtypes.StateStore + ssSnapshots stateStoreSnapshotScheduler lastCommitInfo *types.CommitInfo storesParams map[types.StoreKey]storeParams storeKeys map[string]types.StoreKey @@ -136,6 +151,7 @@ func NewStore( scDir: scDir, } if ssConfig.Enable { + config.AlignSSSnapshotWithSC(scConfig, &ssConfig) ssStore, err := ss.NewStateStore(homeDir, ssConfig) if err != nil { panic(err) @@ -150,6 +166,16 @@ func NewStore( panic("Enabling SS store without state sync could cause data corruption") } store.ssStore = ssStore + scheduler, ok := ssStore.(stateStoreSnapshotScheduler) + if !ok { + // Unreachable while CompositeStateStore is the only implementation, + // which the assertion above pins. Log rather than drop silently, so + // a wrapper that loses the method is visible as a boot line instead + // of as snapshots that never appear. + logger.Error("state store does not schedule snapshots; SS snapshots are disabled", + "type", fmt.Sprintf("%T", ssStore)) + } + store.ssSnapshots = scheduler } return store @@ -255,6 +281,14 @@ func (rs *Store) flush() error { telemetry.SetGauge(float32(currentVersion), "storeV2", "ss", "version") } } + // Both branches above have finished handing currentVersion to SS and have + // enqueued nothing above it, which is what makes an SS snapshot label exact. + // Triggering here rather than inside either branch keeps populated and empty + // blocks on one path. A repeat within the same block (flush runs twice, and + // the second pass sees an empty changeset) is ignored by the state store. + if rs.ssSnapshots != nil { + rs.ssSnapshots.ScheduleSnapshot(currentVersion) + } return rs.scStore.ApplyChangeSets(changeSets) } diff --git a/sei-cosmos/storev2/rootmulti/store_test.go b/sei-cosmos/storev2/rootmulti/store_test.go index 90b7aa98c9..90c298eb97 100644 --- a/sei-cosmos/storev2/rootmulti/store_test.go +++ b/sei-cosmos/storev2/rootmulti/store_test.go @@ -3,6 +3,7 @@ package rootmulti import ( "context" "fmt" + "path/filepath" "sync" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/state" "github.com/sei-protocol/sei-chain/sei-db/config" + sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/stretchr/testify/require" "golang.org/x/time/rate" @@ -135,6 +137,55 @@ func TestSCSS_WriteAndHistoricalRead(t *testing.T) { require.Equal(t, valV1, resp.Value) } +// flush owns the SS snapshot trigger for every block, so a boundary must be +// scheduled whether or not the block carried changesets. The composite package +// cannot pin this: its tests call ScheduleSnapshot themselves, so a regression +// in either branch of flush is invisible there. +func TestFlushSchedulesSSSnapshotAtABoundary(t *testing.T) { + for _, tc := range []struct { + name string + writeAtBlock int64 + }{ + {name: "boundary block is populated", writeAtBlock: 2}, + {name: "boundary block is empty", writeAtBlock: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + scCfg := config.DefaultStateCommitConfig() + scCfg.Enable = true + scCfg.MemIAVLConfig.AsyncCommitBuffer = 0 + // SS mirrors the SC cadence, so this is what puts the SS boundary at 2. + scCfg.MemIAVLConfig.SnapshotInterval = 2 + scCfg.MemIAVLConfig.SnapshotKeepRecent = 1 + + ssCfg := config.DefaultStateStoreConfig() + ssCfg.Enable = true + ssCfg.SnapshotEnable = true + + store := NewStore(home, scCfg, ssCfg, []string{}) + defer func() { _ = store.Close() }() + require.NotNil(t, store.ssSnapshots, "SS snapshot capability was not resolved") + + key := types.NewKVStoreKey("bank") + store.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + require.NoError(t, store.LoadLatestVersion()) + + for block := int64(1); block <= 2; block++ { + if block == tc.writeAtBlock { + store.GetStoreByName("bank").(types.KVStore).Set([]byte("k"), []byte("v")) + } + require.Equal(t, block, store.Commit(true).Version) + } + + root := filepath.Join(home, "data", "state_store", sscomposite.SnapshotsDirName) + require.Eventually(t, func() bool { + versions, err := sscomposite.ListSnapshotVersions(root) + return err == nil && len(versions) == 1 && versions[0] == 2 + }, 10*time.Second, 20*time.Millisecond, "boundary did not produce an SS snapshot") + }) + } +} + // TestCacheMultiStoreWithVersion_OnlyUsesSSStores verifies that CacheMultiStoreWithVersion // serves SS stores when enabled, and falls back to SC when SS is disabled, for // height=0 (latest) and explicit latest height. diff --git a/sei-db/common/utils/path.go b/sei-db/common/utils/path.go index d8a03fe841..d073091bad 100644 --- a/sei-db/common/utils/path.go +++ b/sei-db/common/utils/path.go @@ -7,6 +7,8 @@ import ( "strings" ) +const StateStoreSnapshotsDirName = "snapshots" + // DirExists returns true if path exists and is a directory. func DirExists(path string) bool { info, err := os.Stat(path) @@ -63,6 +65,11 @@ func GetEVMStateStorePath(homePath string, backend string) string { return filepath.Join(homePath, "data", "state_store", "evm", backend) } +// GetStateStoreSnapshotsPath returns the path for online state-store snapshots. +func GetStateStoreSnapshotsPath(homePath string) string { + return filepath.Join(homePath, "data", "state_store", StateStoreSnapshotsDirName) +} + // GetReceiptStorePath returns the path for the receipt store. // New nodes use data/ledger/receipt/{backend}; existing nodes with // data/receipt.db continue using the legacy path for backward compatibility. diff --git a/sei-db/config/sc_config.go b/sei-db/config/sc_config.go index 48ec3635ba..6b3bda0a54 100644 --- a/sei-db/config/sc_config.go +++ b/sei-db/config/sc_config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "time" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" @@ -16,6 +17,34 @@ const ( legacySCWriteModeCosmosOnly = "cosmos_only" ) +// EffectiveMemIAVLSnapshotCadence resolves memIAVL's snapshot cadence the way +// Options.FillDefaults resolves it at OpenDB, so a caller mirroring the cadence +// onto another backend sees the values memIAVL will actually run with rather +// than the raw config. A zero means "unset" here, not "disabled": memIAVL heals +// it to the default, so mirroring the raw zero would silently disable snapshots +// on the mirroring backend. +func EffectiveMemIAVLSnapshotCadence(cfg memiavl.Config) (interval, keepRecent uint32) { + interval = cfg.SnapshotInterval + if interval == 0 { + interval = memiavl.DefaultSnapshotInterval + } + keepRecent = cfg.SnapshotKeepRecent + if keepRecent == 0 { + keepRecent = memiavl.DefaultSnapshotKeepRecent + } + return interval, keepRecent +} + +// EffectiveMemIAVLSnapshotMinTimeInterval resolves the minimum wall-clock +// interval the same way memIAVL Options.FillDefaults does. +func EffectiveMemIAVLSnapshotMinTimeInterval(cfg memiavl.Config) time.Duration { + seconds := cfg.SnapshotMinTimeInterval + if seconds == 0 { + seconds = memiavl.DefaultSnapshotMinTimeInterval + } + return time.Duration(seconds) * time.Second +} + // StateCommitConfig defines configuration for the state commit (SC) layer. type StateCommitConfig struct { // Enable defines if the state-commit (SeiDB) should be enabled. diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index a3a89b898e..567faf9138 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -1,5 +1,7 @@ package config +import "time" + // DBBackend defines the SS DB backend. type DBBackend string @@ -63,6 +65,38 @@ type StateStoreConfig struct { // defaults to false (use MVCCComparer for backwards compatibility) UseDefaultComparer bool `mapstructure:"use-default-comparer"` + // SnapshotEnable controls whether the state store takes periodic online + // snapshots. Snapshots are Pebble checkpoints (hardlink trees), so the + // backend must be pebbledb and every SS database must be able to hardlink + // into the snapshot root. Startup fails on either rather than running + // without snapshots. A custom Cosmos SS directory moves the snapshot root + // beside that directory, which keeps the link inside one filesystem. + // + // Taking a snapshot occupies each backend's SS apply goroutine for the WAL + // flush, the filesystem sync, and the checkpoint. No data is copied up + // front, but a queue that fills during that window applies write + // backpressure. + // + // Each retained snapshot pins the SSTs it references and prevents compaction + // from reclaiming them. Steady-state disk overhead is therefore the + // compaction churn accumulated over SnapshotInterval blocks, per retained + // snapshot — significant on a multi-TB state store. Managed snapshots are + // rollback restore points, not an archive format. They have no lease in this + // release, so node-external tools must not resolve a snapshot path and open it + // later without first adding a hold mechanism. Attempts, skips, outcomes, + // duration, in-flight state, height, count, and apparent bytes are exported as + // ss_snapshot_* metrics. + // defaults to false + SnapshotEnable bool `mapstructure:"snapshot-enable"` + + // SnapshotInterval, SnapshotKeepRecent, and SnapshotMinTimeInterval are + // mirrored from the state-commit snapshot settings at runtime by + // AlignSSSnapshotWithSC. They are intentionally not exposed in app.toml; + // SnapshotEnable is the only SS-side knob. + SnapshotInterval int64 `mapstructure:"-"` + SnapshotKeepRecent int `mapstructure:"-"` + SnapshotMinTimeInterval time.Duration `mapstructure:"-"` + // --- EVM optimization fields --- // EVMSplit controls whether EVM data is routed to a dedicated SS backend. @@ -93,7 +127,25 @@ func DefaultStateStoreConfig() StateStoreConfig { ImportNumWorkers: DefaultSSImportWorkers, KeepLastVersion: true, UseDefaultComparer: false, + SnapshotEnable: false, EVMSplit: false, SeparateEVMSubDBs: false, } } + +// AlignSSSnapshotWithSC mirrors the state-commit interval, minimum time +// interval, and retention settings onto the state store. SC and SS apply their +// in-flight gates independently, so this does not promise identical retained +// heights. When SS snapshots are disabled the cadence is zeroed. +func AlignSSSnapshotWithSC(scConfig StateCommitConfig, ssConfig *StateStoreConfig) { + if !ssConfig.SnapshotEnable { + ssConfig.SnapshotInterval = 0 + ssConfig.SnapshotKeepRecent = 0 + ssConfig.SnapshotMinTimeInterval = 0 + return + } + interval, keepRecent := EffectiveMemIAVLSnapshotCadence(scConfig.MemIAVLConfig) + ssConfig.SnapshotInterval = int64(interval) + ssConfig.SnapshotKeepRecent = int(keepRecent) + ssConfig.SnapshotMinTimeInterval = EffectiveMemIAVLSnapshotMinTimeInterval(scConfig.MemIAVLConfig) +} diff --git a/sei-db/config/ss_config_test.go b/sei-db/config/ss_config_test.go new file mode 100644 index 0000000000..543f50ba0c --- /dev/null +++ b/sei-db/config/ss_config_test.go @@ -0,0 +1,106 @@ +package config + +import ( + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" + "github.com/stretchr/testify/require" +) + +func TestAlignSSSnapshotWithSC(t *testing.T) { + scConfig := DefaultStateCommitConfig() + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + + scConfig.MemIAVLConfig.SnapshotInterval = 123 + scConfig.MemIAVLConfig.SnapshotKeepRecent = 4 + scConfig.MemIAVLConfig.SnapshotMinTimeInterval = 17 + + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + require.Equal(t, int64(123), ssConfig.SnapshotInterval) + require.Equal(t, 4, ssConfig.SnapshotKeepRecent) + require.Equal(t, 17*time.Second, ssConfig.SnapshotMinTimeInterval) +} + +func TestAlignSSSnapshotWithSCHealsZeroToSCDefaults(t *testing.T) { + scConfig := DefaultStateCommitConfig() + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + + scConfig.MemIAVLConfig.SnapshotInterval = 0 + scConfig.MemIAVLConfig.SnapshotKeepRecent = 0 + + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + require.Equal(t, int64(memiavl.DefaultSnapshotInterval), ssConfig.SnapshotInterval) + require.Equal(t, memiavl.DefaultSnapshotKeepRecent, ssConfig.SnapshotKeepRecent) + require.Equal( + t, + time.Duration(memiavl.DefaultSnapshotMinTimeInterval)*time.Second, + ssConfig.SnapshotMinTimeInterval, + ) +} + +func TestDefaultStateStoreConfigDisablesSnapshots(t *testing.T) { + require.False(t, DefaultStateStoreConfig().SnapshotEnable, + "snapshots require an explicit ss-snapshot-enable opt-in") +} + +// A zero cadence is what the snapshot manager reads as "do not run", so the +// off switch has to zero it rather than mirror SC's. +func TestAlignSSSnapshotWithSCZeroesCadenceWhenDisabled(t *testing.T) { + scConfig := DefaultStateCommitConfig() + scConfig.MemIAVLConfig.SnapshotInterval = 123 + scConfig.MemIAVLConfig.SnapshotKeepRecent = 4 + scConfig.MemIAVLConfig.SnapshotMinTimeInterval = 17 + + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = false + + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + require.Zero(t, ssConfig.SnapshotInterval) + require.Zero(t, ssConfig.SnapshotKeepRecent) + require.Zero(t, ssConfig.SnapshotMinTimeInterval) +} + +// FlatKV and SS both mirror memIAVL's cadence, and they must resolve it +// identically or the two backends drift onto different snapshot heights. +func TestAlignSSSnapshotMatchesEffectiveMemIAVLCadence(t *testing.T) { + for _, tc := range []struct { + name string + interval, keepRecent uint32 + minTime uint32 + wantInterval int64 + wantMinTime time.Duration + }{ + { + name: "explicit", interval: 500, keepRecent: 3, minTime: 45, + wantInterval: 500, wantMinTime: 45 * time.Second, + }, + { + name: "zero heals to default", interval: 0, keepRecent: 0, + wantInterval: memiavl.DefaultSnapshotInterval, + wantMinTime: time.Duration(memiavl.DefaultSnapshotMinTimeInterval) * time.Second, + }, + } { + t.Run(tc.name, func(t *testing.T) { + scConfig := DefaultStateCommitConfig() + scConfig.MemIAVLConfig.SnapshotInterval = tc.interval + scConfig.MemIAVLConfig.SnapshotKeepRecent = tc.keepRecent + scConfig.MemIAVLConfig.SnapshotMinTimeInterval = tc.minTime + + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + wantInterval, wantKeepRecent := EffectiveMemIAVLSnapshotCadence(scConfig.MemIAVLConfig) + require.Equal(t, int64(wantInterval), ssConfig.SnapshotInterval) + require.Equal(t, int(wantKeepRecent), ssConfig.SnapshotKeepRecent) + require.Equal(t, tc.wantInterval, ssConfig.SnapshotInterval) + require.Equal(t, tc.wantMinTime, ssConfig.SnapshotMinTimeInterval) + }) + } +} diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index d7e782bb89..6847fd3650 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -140,6 +140,20 @@ ss-import-num-workers = {{ .StateStore.ImportNumWorkers }} # Applies when ss-backend = "pebbledb". Default: false. ss-enable-read-write-metrics = {{ .StateStore.EnableReadWriteMetrics }} +# SnapshotEnable turns on periodic online state-store snapshots. The cadence is +# not configurable here: it mirrors the state-commit snapshot settings. +# Two configurations fail startup rather than run without snapshots: an +# ss-backend other than "pebbledb", and SS databases that cannot hardlink into +# the snapshot root, which needs every SS database and that root on one +# filesystem. +# Each retained snapshot pins the SST files it references against compaction, so +# budget the write churn of one snapshot interval per retained snapshot. This is +# substantial on a multi-TB state store. +# Snapshot directories are rollback restore points, not an archive format. +# They have no lease, so do not build tools that resolve one and open it later. +# Cost and progress are exported as ss_snapshot_* metrics. Default: false. +ss-snapshot-enable = {{ .StateStore.SnapshotEnable }} + # EVMDBDirectory defines the directory for the optional EVM state-store DB(s). # If unset, defaults to /data/evm_ss when EVM SS is enabled. evm-ss-db-directory = "{{ .StateStore.EVMDBDirectory }}" diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index 5ffb274af6..a07fbb05fd 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -97,6 +97,7 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, "ss-prune-interval =", "Missing ss-prune-interval") require.Contains(t, output, "ss-import-num-workers =", "Missing ss-import-num-workers") require.Contains(t, output, "ss-enable-read-write-metrics = false", "Missing state-store read/write metrics flag") + require.Contains(t, output, "ss-snapshot-enable = false", "Missing or incorrect ss-snapshot-enable") require.Contains(t, output, `evm-ss-db-directory = ""`, "Missing evm-ss-db-directory") require.Contains(t, output, `evm-ss-split = false`, "Missing or incorrect evm-ss-split") require.Contains(t, output, "evm-ss-separate-dbs = false", "Missing or incorrect evm-ss-separate-dbs") diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index 114659be57..d4ea4c2f94 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -71,7 +71,8 @@ type Database struct { asyncWriteWG sync.WaitGroup config config.StateStoreConfig // Earliest version for db after pruning - earliestVersion atomic.Int64 + earliestVersion atomic.Int64 + earliestVersionMu sync.Mutex // Latest version for db latestVersion atomic.Int64 // descending indicates whether this DB uses descending-version MVCC @@ -85,6 +86,12 @@ type Database struct { // Used in pruning to skip over stores that have not been updated recently storeKeyDirty sync.Map + // pruneIncomplete records that a pass raised the earliest-version marker and + // then failed before it finished deleting. The next pass cannot use that + // marker as its skip baseline: rows below it are still on disk, and a store + // that has gone idle since would be skipped for as long as it stays idle. + pruneIncomplete atomic.Bool + // Changelog used to support async write streamHandler wal.ChangelogWAL @@ -101,12 +108,12 @@ type VersionedChangesets struct { Version int64 Changesets []*proto.NamedChangeSet Done chan struct{} // non-nil for barrier: closed when this entry is processed + // AtDrain, when non-nil, is run by the apply goroutine in queue order + // instead of applying a changeset. See ScheduleAtDrain. + AtDrain func() } -func OpenDB(dataDir string, config config.StateStoreConfig) (types.StateStore, error) { - cache := pebble.NewCache(1024 * 1024 * 32) - defer cache.Unref() - +func newPebbleOptions(config config.StateStoreConfig, cache *pebble.Cache) *pebble.Options { // Select comparer based on config. Note: UseDefaultComparer is NOT backwards compatible // with existing databases created with MVCCComparer - Pebble will refuse to open due to // comparer name mismatch. Only use UseDefaultComparer for NEW databases. @@ -162,6 +169,14 @@ func OpenDB(dataDir string, config config.StateStoreConfig) (types.StateStore, e //TODO: add a new config and check if readonly = true to support readonly mode + return opts +} + +func OpenDB(dataDir string, config config.StateStoreConfig) (types.StateStore, error) { + cache := pebble.NewCache(1024 * 1024 * 32) + defer cache.Unref() + + opts := newPebbleOptions(config, cache) db, err := pebble.Open(dataDir, opts) if err != nil { return nil, fmt.Errorf("failed to open PebbleDB: %w", err) @@ -278,6 +293,51 @@ func (db *Database) PebbleMetrics() *pebble.Metrics { return db.storage.Metrics() } +// Checkpoint writes a point-in-time snapshot of the database into destDir +// (which must not exist yet). Pebble implements this with hardlinks to +// already-fsynced SSTs plus a flushed WAL. SS schedules it on the apply +// goroutine at an ordered queue boundary, so that backend cannot apply more +// changes until the WAL flush, filesystem sync, and checkpoint creation finish. +// Satisfies types.Checkpointable. +func (db *Database) Checkpoint(destDir string) error { + if err := db.storage.Checkpoint(destDir, pebble.WithFlushedWAL()); err != nil { + return fmt.Errorf("pebble checkpoint to %q: %w", destDir, err) + } + return nil +} + +// SetCheckpointVersion writes the logical latest version into a completed +// checkpoint without changing the live database marker. +func (db *Database) SetCheckpointVersion(destDir string, version int64) error { + if version < 0 { + return fmt.Errorf("version must be non-negative") + } + + opts := newPebbleOptions(db.config, nil) + opts.DisableAutomaticCompactions = true + checkpoint, err := pebble.Open(destDir, opts) + if err != nil { + return fmt.Errorf("open checkpoint %q to set markers: %w", destDir, err) + } + + // Converted here, where the non-negative check above is in view. + setErr := setCheckpointMarker(checkpoint, latestVersionKey, uint64(version)) + closeErr := checkpoint.Close() + if setErr != nil { + setErr = fmt.Errorf("set checkpoint version %d: %w", version, setErr) + } + if closeErr != nil { + closeErr = fmt.Errorf("close checkpoint after setting version: %w", closeErr) + } + return errors.Join(setErr, closeErr) +} + +func setCheckpointMarker(checkpoint *pebble.DB, key string, version uint64) error { + var marker [VersionSize]byte + binary.LittleEndian.PutUint64(marker[:], version) + return checkpoint.Set([]byte(key), marker[:], pebble.Sync) +} + func (db *Database) SetLatestVersion(version int64) error { if version < 0 { return fmt.Errorf("version must be non-negative") @@ -337,21 +397,21 @@ func (db *Database) SetEarliestVersion(version int64, ignoreVersion bool) error if version < 0 { return fmt.Errorf("version must be non-negative") } + db.earliestVersionMu.Lock() + defer db.earliestVersionMu.Unlock() + earliestVersion := db.earliestVersion.Load() - if version > earliestVersion || ignoreVersion { - swapped := db.earliestVersion.CompareAndSwap(earliestVersion, version) - if swapped { - var ts [VersionSize]byte - binary.LittleEndian.PutUint64(ts[:], uint64(version)) - err := db.storage.Set([]byte(earliestVersionKey), ts[:], defaultWriteOpts) - if err == nil { - db.operationMetrics.AddWrite(1) - } - return err - } else { - return fmt.Errorf("failed to set earliest version to: %d", version) - } + if version <= earliestVersion && !ignoreVersion { + return nil + } + + var ts [VersionSize]byte + binary.LittleEndian.PutUint64(ts[:], uint64(version)) + if err := db.storage.Set([]byte(earliestVersionKey), ts[:], defaultWriteOpts); err != nil { + return err } + db.earliestVersion.Store(version) + db.operationMetrics.AddWrite(1) return nil } @@ -359,6 +419,18 @@ func (db *Database) GetEarliestVersion() int64 { return db.earliestVersion.Load() } +// advanceEarliestVersion raises the earliest-version marker to target for a +// prune pass that has not deleted anything yet. +// +// SetEarliestVersion serializes competing writers and changes the in-memory +// marker only after Pebble accepts the metadata write. A persistence failure is +// therefore returned with both markers unchanged. Deleting history under a +// marker that only moved in memory would advertise, after a restart, versions +// the pass has already dropped. +func (db *Database) advanceEarliestVersion(target int64) error { + return db.SetEarliestVersion(target, false) +} + // Retrieves earliest version from db, if not found, return 0 func retrieveEarliestVersion(db *pebble.DB) (int64, error) { return retrieveVersionKey(db, earliestVersionKey) @@ -490,6 +562,10 @@ func (db *Database) ApplyChangesetAsync(version int64, changesets []*proto.Named func (db *Database) writeAsyncInBackground() { defer db.asyncWriteWG.Done() for nextChange := range db.pendingChanges { + if nextChange.AtDrain != nil { + nextChange.AtDrain() + continue + } if nextChange.Done != nil { close(nextChange.Done) continue @@ -508,6 +584,19 @@ func (db *Database) WaitForPendingWrites() { <-done } +// ScheduleAtDrain runs fn on the apply goroutine at the point in the queue +// where every changeset enqueued before this call has been applied and none +// enqueued after it has. Unlike WaitForPendingWrites it does not block the +// caller, which is what lets a caller capture the DB at an exact version +// without stalling the block it is committing: the version is pinned by fn's +// position in the queue rather than by when it runs. +// +// fn runs on the writer, so it must not enqueue more work on this DB (that +// deadlocks once the buffer fills) and must not panic. +func (db *Database) ScheduleAtDrain(fn func()) { + db.pendingChanges <- VersionedChangesets{AtDrain: fn} +} + // Prune dispatches between descending- and ascending-mode implementations // depending on the on-disk encoding detected at open time. func (db *Database) Prune(version int64) error { @@ -612,6 +701,11 @@ func (db *Database) getDescending(storeKey string, targetVersion int64, key []by // NOTE: There is a rare case when a module's keys are skipped during pruning even though // it has been updated. This occurs when that module's keys are updated in between pruning runs, the node after is restarted. // This is not a large issue given the next time that module is updated, it will be properly pruned thereafter. +// NOTE: the marker is raised before the deletes, so a pass that fails partway +// leaves rows below the marker on disk. pruneIncomplete makes the next pass in +// the same process rescan every store to reach them. A crash inside that window +// loses the flag, and those rows stay on disk — unreachable by any read, since +// the marker bounds reads too — until the store is written to again. func (db *Database) pruneDescending(version int64) (_err error) { // Defensive check: ensure database is not closed if db.storage == nil { @@ -630,6 +724,22 @@ func (db *Database) pruneDescending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version + skipBelow := db.GetEarliestVersion() + if err := db.advanceEarliestVersion(earliestVersion); err != nil { + return err + } + if db.pruneIncomplete.Load() { + // A previous pass raised the marker and then stopped short of its + // deletes, so the marker no longer bounds what is on disk. Scan every + // store to reach the rows it left behind. + skipBelow = 0 + } + db.pruneIncomplete.Store(true) + defer func() { + if _err == nil { + db.pruneIncomplete.Store(false) + } + }() itr, err := db.storage.NewIter(nil) if err != nil { @@ -676,8 +786,12 @@ func (db *Database) pruneDescending(version int64) (_err error) { prevStore = storeKey updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) - // Skip a store's keys if version it was last updated is less than last prune height - if !ok || (typeOk && versionUpdated < db.GetEarliestVersion()) { + // The marker is advanced before deletes so checkpoints never claim + // history that the prune has already dropped. skipBelow is the marker + // as it stood before this pass raised it; comparing against the raised + // value would skip every store whose latest update is at or below the + // prune height. + if !ok || (typeOk && versionUpdated < skipBelow) { itr.SeekGE(storePrefix(storeKey + "0")) continue } @@ -748,9 +862,6 @@ func (db *Database) pruneDescending(version int64) (_err error) { } db.operationMetrics.AddRead(scanReads) - if err := db.SetEarliestVersion(earliestVersion, false); err != nil { - return err - } return db.compactPrunedRange(firstDeletedKey, lastDeletedKey) } diff --git a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go index 4075f9eea1..6932d06fd2 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go @@ -108,6 +108,22 @@ func (db *Database) pruneAscending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version + skipBelow := db.GetEarliestVersion() + if err := db.advanceEarliestVersion(earliestVersion); err != nil { + return err + } + if db.pruneIncomplete.Load() { + // A previous pass raised the marker and then stopped short of its + // deletes, so the marker no longer bounds what is on disk. Scan every + // store to reach the rows it left behind. + skipBelow = 0 + } + db.pruneIncomplete.Store(true) + defer func() { + if _err == nil { + db.pruneIncomplete.Store(false) + } + }() itr, err := db.storage.NewIter(nil) if err != nil { @@ -154,8 +170,12 @@ func (db *Database) pruneAscending(version int64) (_err error) { prevStore = storeKey updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) - // Skip a store's keys if version it was last updated is less than last prune height - if !ok || (typeOk && versionUpdated < db.GetEarliestVersion()) { + // The marker is advanced before deletes so checkpoints never claim + // history that the prune has already dropped. skipBelow is the marker + // as it stood before this pass raised it; comparing against the raised + // value would skip every store whose latest update is at or below the + // prune height. + if !ok || (typeOk && versionUpdated < skipBelow) { itr.SeekGE(storePrefix(storeKey + "0")) continue } @@ -224,9 +244,6 @@ func (db *Database) pruneAscending(version int64) (_err error) { } db.operationMetrics.AddRead(scanReads) - if err := db.SetEarliestVersion(earliestVersion, false); err != nil { - return err - } return db.compactPrunedRange(firstDeletedKey, lastDeletedKey) } diff --git a/sei-db/db_engine/pebbledb/mvcc/db_test.go b/sei-db/db_engine/pebbledb/mvcc/db_test.go index b923188718..ea914203e5 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_test.go @@ -1,13 +1,17 @@ package mvcc import ( + "encoding/binary" + "path/filepath" "testing" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/test" + sstest "github.com/sei-protocol/sei-chain/sei-db/db_engine/test" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" ) func TestStorageTestSuite(t *testing.T) { @@ -46,3 +50,57 @@ func TestStorageTestSuiteDefaultComparer(t *testing.T) { suite.Run(t, s) } + +func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + + store, err := OpenDB(filepath.Join(t.TempDir(), "live"), cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + require.NoError(t, store.SetLatestVersion(10)) + require.NoError(t, store.SetEarliestVersion(4, false)) + + dest := filepath.Join(t.TempDir(), "snapshot") + done := make(chan error, 1) + management.ScheduleCheckpoint(store, dest, nil, func(err error) { + done <- err + }) + require.NoError(t, <-done) + // The caller stamps only the label. Earliest is inherited from the + // checkpointed DB because prune advances it before deleting history. + require.NoError(t, management.SetCheckpointVersion(store, dest, 5)) + + require.Equal(t, int64(10), store.GetLatestVersion()) + require.Equal(t, int64(4), store.GetEarliestVersion()) + for key, want := range map[string]uint64{latestVersionKey: 10, earliestVersionKey: 4} { + marker, closer, err := store.(*Database).storage.Get([]byte(key)) + require.NoError(t, err) + require.Equal(t, want, binary.LittleEndian.Uint64(marker), "live %s changed", key) + require.NoError(t, closer.Close()) + } + + checkpoint, err := OpenDB(dest, cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, checkpoint.Close()) }) + require.Equal(t, int64(5), checkpoint.GetLatestVersion()) + require.Equal(t, int64(4), checkpoint.GetEarliestVersion()) +} + +func TestScheduledCheckpointCanBeCanceledAtBarrier(t *testing.T) { + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + + store, err := OpenDB(filepath.Join(t.TempDir(), "live"), cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + dest := filepath.Join(t.TempDir(), "snapshot") + done := make(chan error, 1) + management.ScheduleCheckpoint(store, dest, func() bool { return false }, func(err error) { + done <- err + }) + + require.ErrorIs(t, <-done, management.ErrCheckpointCanceled) + require.NoDirExists(t, dest) +} diff --git a/sei-db/db_engine/pebbledb/mvcc/prune_test.go b/sei-db/db_engine/pebbledb/mvcc/prune_test.go index 6334760534..c7909c426c 100644 --- a/sei-db/db_engine/pebbledb/mvcc/prune_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/prune_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/cockroachdb/pebble/v2" + "github.com/cockroachdb/pebble/v2/vfs" "github.com/stretchr/testify/require" "github.com/sei-protocol/sei-chain/sei-db/config" @@ -119,4 +120,91 @@ func TestPruneDescendingOrder_DeletesOldVersions(t *testing.T) { require.ElementsMatch(t, []int64{140}, rawVersionsForKey(t, db, store, k2)) }) + t.Run("idle store still prunes against previous earliest marker", func(t *testing.T) { + db := newTestDB(t, true) + + applyVersion(t, db, store, 50, key, []byte("v50")) + applyVersion(t, db, store, 100, key, []byte("v100")) + + require.NoError(t, db.Prune(150)) + + versions := rawVersionsForKey(t, db, store, key) + require.ElementsMatch(t, []int64{100}, versions, + "prune must not use the just-advanced marker to skip this store") + }) + +} + +func TestPruneAdvancesEarliestBeforeDeletingHistory(t *testing.T) { + db := newTestDB(t, true) + + require.NoError(t, db.storage.Set([]byte("invalid-mvcc-key"), []byte("value"), defaultWriteOpts)) + + err := db.Prune(10) + require.Error(t, err) + require.Equal(t, int64(11), db.GetEarliestVersion(), + "earliest marker must advance before a later prune failure") +} + +// TestPruneAfterFailedPassRescansIdleStores covers the other half of raising the +// marker first: the pass that follows a failure cannot use that marker as its +// skip baseline. store1 goes idle at version 100, below the raised marker, so +// skipping it would leave v50 on disk with no read able to reach it. +func TestPruneAfterFailedPassRescansIdleStores(t *testing.T) { + const store = "store1" + key := []byte("k") + db := newTestDB(t, true) + + applyVersion(t, db, store, 50, key, []byte("v50")) + applyVersion(t, db, store, 100, key, []byte("v100")) + + // "invalid-mvcc-key" sorts ahead of every "s/k:" store key, so the pass + // fails after raising the marker and before deleting anything. + badKey := []byte("invalid-mvcc-key") + require.NoError(t, db.storage.Set(badKey, []byte("value"), defaultWriteOpts)) + require.Error(t, db.Prune(150)) + require.Equal(t, int64(151), db.GetEarliestVersion()) + require.ElementsMatch(t, []int64{50, 100}, rawVersionsForKey(t, db, store, key), + "the failed pass must not have deleted anything") + + require.NoError(t, db.storage.Delete(badKey, defaultWriteOpts)) + require.NoError(t, db.Prune(150)) + + require.ElementsMatch(t, []int64{100}, rawVersionsForKey(t, db, store, key), + "the pass after a failure must rescan a store the raised marker would skip") +} + +// TestAdvanceEarliestVersionAcceptsAHigherMarker pins the outcome a prune pass +// sees when another writer moves the marker past its target. Raising the marker +// now runs ahead of the deletes, so reporting that as a failure would cost the +// whole pass rather than just the marker write. +func TestAdvanceEarliestVersionAcceptsAHigherMarker(t *testing.T) { + db := newTestDB(t, true) + + require.NoError(t, db.SetEarliestVersion(200, false)) + require.NoError(t, db.advanceEarliestVersion(151)) + require.Equal(t, int64(200), db.GetEarliestVersion(), + "the target must not lower a marker another writer raised past it") +} + +// TestAdvanceEarliestVersionReturnsPersistenceFailure pins that Pebble must +// accept the metadata write before the in-memory marker moves. Otherwise a +// later call with the same target would see the target in memory, return nil, +// and let pruning delete history under a marker that was never persisted. +func TestAdvanceEarliestVersionReturnsPersistenceFailure(t *testing.T) { + fs := vfs.NewMem() + storage, err := pebble.Open("db", &pebble.Options{FS: fs}) + require.NoError(t, err) + require.NoError(t, storage.Close()) + + storage, err = pebble.Open("db", &pebble.Options{FS: fs, ReadOnly: true}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, storage.Close()) }) + + db := &Database{storage: storage} + err = db.advanceEarliestVersion(151) + + require.ErrorIs(t, err, pebble.ErrReadOnly) + require.Zero(t, db.GetEarliestVersion(), + "a failed metadata write must not move the in-memory marker") } diff --git a/sei-db/db_engine/types/types.go b/sei-db/db_engine/types/types.go index 00096bf691..24856dc539 100644 --- a/sei-db/db_engine/types/types.go +++ b/sei-db/db_engine/types/types.go @@ -120,6 +120,30 @@ type Checkpointable interface { Checkpoint(destDir string) error } +// CheckpointVersionSetter writes the logical latest version into a completed +// checkpoint without changing the live database. +// +// The latest marker has to be stamped because a checkpoint is a copy of a +// database whose marker may already have moved on. The earliest marker is +// inherited from the checkpoint; pruning advances it before deleting history, so +// every checkpoint boundary either sees the old marker with old data or the new +// marker with data that is at least as deep as advertised. +type CheckpointVersionSetter interface { + SetCheckpointVersion(destDir string, version int64) error +} + +// DrainBarrier is an optional capability for engines that apply changesets from +// an async queue. It lets a caller place work at an exact point in the write +// order without waiting for the queue to drain. +type DrainBarrier interface { + ScheduleAtDrain(fn func()) +} + +// The three interfaces above are engine capabilities. Deciding when a checkpoint +// runs, and what version it is labeled with, is coordination rather than engine +// behavior and lives in sei-db/management: CheckpointScheduler, +// ScheduleCheckpoint, SetCheckpointVersion and ErrCheckpointCanceled. + // --------------------------------------------------------------------------- // SS DB layer // --------------------------------------------------------------------------- diff --git a/sei-db/management/checkpoint_scheduler.go b/sei-db/management/checkpoint_scheduler.go new file mode 100644 index 0000000000..8e68bb92b9 --- /dev/null +++ b/sei-db/management/checkpoint_scheduler.go @@ -0,0 +1,59 @@ +// Package management holds the coordination layer above the DB engines: work +// that decides when an engine-level operation runs, rather than how the engine +// performs it. +package management + +import ( + "errors" + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" +) + +// CheckpointScheduler coordinates checkpoints for stores with in-flight writes. +// +// The engine-side capabilities this builds on — types.Checkpointable, +// types.DrainBarrier and types.CheckpointVersionSetter — stay with the engines +// that implement them. What lives here is the decision of when a checkpoint runs +// and what version it is labeled with. +type CheckpointScheduler interface { + SupportsCheckpoint() bool + ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) + SetCheckpointVersion(destDir string, version int64) error +} + +// ErrCheckpointCanceled reports that a queued checkpoint was canceled before +// it started. +var ErrCheckpointCanceled = errors.New("state store checkpoint canceled") + +// ScheduleCheckpoint checkpoints an engine after all writes already enqueued +// on it have been applied. +func ScheduleCheckpoint(db types.StateStore, destDir string, shouldRun func() bool, done func(error)) { + cp, ok := db.(types.Checkpointable) + if !ok { + done(fmt.Errorf("state store backend %T does not support checkpoints", db)) + return + } + barrier, ok := db.(types.DrainBarrier) + if !ok { + done(fmt.Errorf("state store backend %T does not support ordered checkpoint barriers", db)) + return + } + barrier.ScheduleAtDrain(func() { + if shouldRun != nil && !shouldRun() { + done(ErrCheckpointCanceled) + return + } + done(cp.Checkpoint(destDir)) + }) +} + +// SetCheckpointVersion makes a completed checkpoint self-describing without +// changing the live database. +func SetCheckpointVersion(db types.StateStore, destDir string, version int64) error { + setter, ok := db.(types.CheckpointVersionSetter) + if !ok { + return fmt.Errorf("state store backend %T cannot set checkpoint version", db) + } + return setter.SetCheckpointVersion(destDir, version) +} diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 0bca128e2d..c82d7a0ad6 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -247,14 +247,7 @@ func NewCompositeCommitStore( // different default. Note that mirroring a raw 0 is never correct here (0 means // "disable auto-snapshots" for FlatKV), which is why the zero is resolved first. func alignFlatKVSnapshotWithMemIAVL(cfg *config.StateCommitConfig) { - interval := cfg.MemIAVLConfig.SnapshotInterval - if interval == 0 { - interval = memiavl.DefaultSnapshotInterval - } - keepRecent := cfg.MemIAVLConfig.SnapshotKeepRecent - if keepRecent == 0 { - keepRecent = memiavl.DefaultSnapshotKeepRecent - } + interval, keepRecent := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig) cfg.FlatKVConfig.SnapshotInterval = interval cfg.FlatKVConfig.SnapshotKeepRecent = keepRecent } diff --git a/sei-db/state_db/ss/composite/recovery_test.go b/sei-db/state_db/ss/composite/recovery_test.go index 9688b3afac..926e5d4ef7 100644 --- a/sei-db/state_db/ss/composite/recovery_test.go +++ b/sei-db/state_db/ss/composite/recovery_test.go @@ -78,23 +78,35 @@ func TestEVMSSPreRecoveryAfterStateSync(t *testing.T) { require.Contains(t, err.Error(), "EVM SS is empty") } -// TestEVMSSPostRecoveryEarliestMismatch: diverging earliest versions must abort startup. +// TestEVMSSPostRecoveryEarliestMismatch: diverging earliest versions are allowed +// because the composite reports the highest member floor. func TestEVMSSPostRecoveryEarliestMismatch(t *testing.T) { cosmos := &fakeStateStore{latest: 100, earliest: 50} evm := &fakeStateStore{latest: 100, earliest: 75} cs := newCompositeStateStoreWithStores(cosmos, evm, config.StateStoreConfig{EVMSplit: true}) - err := cs.validateEVMSSPostRecovery() - require.Error(t, err) - require.Contains(t, err.Error(), "earliest version") + cs.validateEVMSSPostRecovery() // Matching earliest → pass. evm.earliest = 50 - require.NoError(t, cs.validateEVMSSPostRecovery()) + cs.validateEVMSSPostRecovery() // Both zero → pass (fresh DBs). cosmos.earliest = 0 evm.earliest = 0 - require.NoError(t, cs.validateEVMSSPostRecovery()) + cs.validateEVMSSPostRecovery() +} + +func TestCompositeGetEarliestVersionReportsHighestMemberFloor(t *testing.T) { + cosmos := &fakeStateStore{latest: 100, earliest: 50} + evm := &fakeStateStore{latest: 100, earliest: 75} + cs := newCompositeStateStoreWithStores(cosmos, evm, config.StateStoreConfig{EVMSplit: true}) + require.Equal(t, int64(75), cs.GetEarliestVersion()) + + cosmos.earliest = 90 + require.Equal(t, int64(90), cs.GetEarliestVersion()) + + cs.evmStore = nil + require.Equal(t, int64(90), cs.GetEarliestVersion()) } // fakeStateStore stubs latest/earliest for validator tests. diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go new file mode 100644 index 0000000000..cd9be00f4a --- /dev/null +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -0,0 +1,737 @@ +package composite + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/management" +) + +// Online state-store snapshots. Every SnapshotInterval blocks the store takes a +// Pebble checkpoint of each backend while the node keeps producing blocks. +// Checkpoints are hardlink trees, so they do not copy database contents, but +// each one occupies its backend's apply goroutine for the full checkpoint +// operation. Writes continue to enter the bounded queue, but a full queue +// applies backpressure until the checkpoint finishes. The result is an +// immutable, crash-consistent image of the query store. +// +// These snapshots are an input to SS rollback, not an export format. The +// intended restore model matches SC FlatKV: restore from an SS snapshot, then +// replay the state WAL forward to the target height. State sync imports the SC +// snapshot stream and rebuilds SS from that stream; it does not consume these +// SS snapshot directories. +// +// On-disk layout under the snapshot root. By default the root is +// /data/state_store/snapshots. A custom Cosmos SS directory moves it +// to the sibling -snapshots directory so Pebble can use hardlinks. +// +// snapshots/ +// current -> snapshot-NNNNN (symlink to newest snapshot) +// snapshot-NNNNN/ (immutable; NNNNN = label version) +// cosmos// (Pebble checkpoint of Cosmos SS) +// evm// (Pebble checkpoint of EVM SS, if split) +// / (when EVM sub-DBs are separate) +// +// Snapshots are eligible at the same interval boundaries and minimum time +// cadence as state commit. This composite implementation uses one trigger and +// one current link for all member stores, so its member snapshots share a label. +// That same label is a property of this layout, not a rollback requirement: +// rollback can replay the state WAL from each store's own nearest snapshot. For +// every accepted SS snapshot, the label is exact: it is the version the write +// path had just handed to the backends when the snapshot was requested. Placing +// a barrier in each backend's apply queue — rather than sampling what the +// backends had applied — makes that label exact without the request having to +// wait. See requestSnapshot. +// +// The barrier orders only the async block-commit queues. Import, recovery, +// pruning, and direct version-marker writes bypass those queues and must not +// call ScheduleSnapshot. The rootmulti commit path owns the trigger for every +// block, populated or empty, and is the only caller of ScheduleSnapshot. +// +// Pruning is the one writer nothing orders a snapshot against. A checkpoint can +// capture a partially applied prune — the same state a crash mid-prune leaves on +// the live DB. This is safe for a snapshot because pruning advances each DB's +// earliest marker before deleting history, so the checkpoint never claims a +// range the DB has already dropped. Reopening a snapshot with different member +// floors is allowed; the composite reports the highest floor any member carries. +// +// SS rollback is not implemented in this feature. When it is added, it should +// use these snapshots the same way SC FlatKV does: restore from a snapshot +// boundary, then replay the state WAL forward. Until then, rolling back or +// state-syncing to a lower height in a reused home directory leaves two stale +// facts behind: lastRequested still carries the old high-water mark, so repeated +// boundaries can be skipped, and already published snapshot-NNNNN directories +// keep labels from the abandoned chain. Clear the snapshot root by hand in that +// case. +// +// Managed snapshot directories have no lease because they are not a node-external +// consumption API. Retention may remove any snapshot that rollback does not need. +// If a future tool opens or copies these directories directly, it must first add +// a lease or other hold mechanism. +// +// This file is the layer the planned per-SS restructure has to move. The +// lifecycle here — layout, retention, the current symlink, staging and +// publication, restart recovery — is reachable only as a method on +// *CompositeStateStore, and startSnapshotManager requires a checkpointable +// Cosmos store, so an EVM-only store cannot use it as written. The agreed +// direction is for each SS to own its own snapshot root, current link, creation, +// and retention behind gc.PrunableStore, mirroring SC FlatKV. Composite mode can +// then fan out to Cosmos SS and EVM SS, while Giga can use EVM SS directly. That +// also removes the second retention path this file adds: prune here is +// count-based and has no ExternalPruning stand-down, so pointing +// StorageGarbageCollector at SS before then would give a store two independent +// pruners. The shape waits on rollback not because GetRollbackFloor is unknown; +// SC FlatKV already defines that floor. It waits because gc.PrunableStore must +// not report a floor above what the store can actually restore to. The current +// link semantics should change with that work too: this implementation points to +// the newest published snapshot, while FlatKV's current link points to the +// active snapshot that open/rollback clones and replays from. +const ( + // SnapshotsDirName is the directory under data/state_store that holds + // online snapshots. + SnapshotsDirName = utils.StateStoreSnapshotsDirName + + snapshotPrefix = "snapshot-" + // snapshotDirLen is "snapshot-" + 20-digit zero-padded version. + snapshotDirLen = len(snapshotPrefix) + 20 + + snapshotCurrentLink = "current" + snapshotCurrentTmpLink = "current-tmp" + snapshotTmpPrefix = "tmp-" + snapshotSizeFile = ".apparent-size" +) + +// SnapshotDirName returns the directory name for a snapshot labeled with the +// given version. +func SnapshotDirName(version int64) string { + return fmt.Sprintf("%s%020d", snapshotPrefix, version) +} + +// ParseSnapshotVersion parses a snapshot directory name; ok is false for +// anything that is not a snapshot-<20 digits> name. +func ParseSnapshotVersion(name string) (version int64, ok bool) { + if !strings.HasPrefix(name, snapshotPrefix) || len(name) != snapshotDirLen { + return 0, false + } + v, err := strconv.ParseInt(name[len(snapshotPrefix):], 10, 64) + if err != nil || v < 0 { + return 0, false + } + return v, true +} + +// ListSnapshotVersions returns the labels of all snapshots under root in +// ascending order. A missing root is not an error (no snapshots yet). +func ListSnapshotVersions(root string) ([]int64, error) { + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read snapshots dir %q: %w", root, err) + } + var versions []int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if v, ok := ParseSnapshotVersion(entry.Name()); ok { + versions = append(versions, v) + } + } + slices.Sort(versions) + return versions, nil +} + +// snapshotManager owns the snapshots directory and the one-at-a-time discipline +// for filling it. It has no goroutine of its own: snapshots are requested from +// the write path and completed on the backends' apply goroutines. +type snapshotManager struct { + root string + backend string + interval int64 + keepRecent int + minTime time.Duration + + cosmosScheduler management.CheckpointScheduler + evmScheduler management.CheckpointScheduler + snapshotSizes map[int64]int64 + + mu sync.Mutex + // lastRequested is the newest label already requested or on disk, so a + // boundary is not snapshotted twice across a restart or a re-sent version. + lastRequested int64 + lastRequestAt time.Time + inFlight bool + stopped bool + // scheduling closes the gap between accepting a request and enqueueing its + // barriers. Close waits for it before closing backend queues. + scheduling sync.WaitGroup + // publishing tracks the goroutine finishing the accepted snapshot off. + publishing sync.WaitGroup + + // publishMu serializes the publish step, which reads and rewrites the + // shared directory (the current link, and pruning). + publishMu sync.Mutex + lastPublished int64 +} + +type checkpointTarget struct { + store management.CheckpointScheduler + dest string +} + +// startSnapshotManager wires the manager into the composite store. Snapshot +// enablement is fail-closed: every backend must support checkpoints, and every +// live DB must be able to hardlink into root. Pebble otherwise silently falls +// back to copying SSTs across filesystems while its apply worker is blocked. +func (s *CompositeStateStore) startSnapshotManager(root string, sourceDirs []string) error { + if s.config.SnapshotInterval <= 0 { + return nil + } + cosmosScheduler, ok := s.cosmosStore.(management.CheckpointScheduler) + if !ok || !cosmosScheduler.SupportsCheckpoint() { + return fmt.Errorf("cosmos backend %q does not support checkpoints", s.config.Backend) + } + var evmScheduler management.CheckpointScheduler + if s.evmStore != nil { + evmScheduler, ok = s.evmStore.(management.CheckpointScheduler) + if !ok || !evmScheduler.SupportsCheckpoint() { + return fmt.Errorf("EVM backend %q does not support checkpoints", s.config.Backend) + } + } + if err := verifySnapshotHardlinks(root, sourceDirs); err != nil { + return err + } + m := &snapshotManager{ + root: root, + backend: s.config.Backend, + interval: s.config.SnapshotInterval, + keepRecent: s.config.SnapshotKeepRecent, + minTime: s.config.SnapshotMinTimeInterval, + cosmosScheduler: cosmosScheduler, + evmScheduler: evmScheduler, + snapshotSizes: map[int64]int64{}, + } + m.lastRequested = m.newestSnapshotVersion() + m.lastPublished = m.lastRequested + m.lastRequestAt = m.snapshotModTime(m.lastRequested) + m.removeStaleTmpDirs() + m.prune() + if m.lastPublished > 0 { + if err := m.updateCurrentLink(SnapshotDirName(m.lastPublished)); err != nil { + logger.Error("failed to restore state store snapshot current link", + "version", m.lastPublished, "error", err) + } + snapshotMetrics.CurrentHeight.Record(context.Background(), m.lastPublished) + } + s.snapshotMgr = m + logger.Info("state store snapshotting enabled", + "root", root, + "interval", m.interval, + "minTimeInterval", m.minTime, + "keepRecent", m.keepRecent, + ) + return nil +} + +func verifySnapshotHardlinks(root string, sourceDirs []string) error { + if err := os.MkdirAll(root, 0o750); err != nil { + return fmt.Errorf("create snapshot root %q: %w", root, err) + } + for _, sourceDir := range sourceDirs { + probe, err := os.CreateTemp(sourceDir, ".ss-snapshot-link-probe-*") + if err != nil { + return fmt.Errorf("create hardlink probe in state store %q: %w", sourceDir, err) + } + source := probe.Name() + if err := probe.Close(); err != nil { + _ = os.Remove(source) + return fmt.Errorf("close hardlink probe in state store %q: %w", sourceDir, err) + } + target := filepath.Join(root, filepath.Base(source)) + if err := os.Link(source, target); err != nil { + _ = os.Remove(source) + return fmt.Errorf( + "state store %q cannot hardlink snapshots into %q; place all SS databases and the snapshot root on one filesystem: %w", + sourceDir, + root, + err, + ) + } + if err := os.Remove(source); err != nil { + _ = os.Remove(target) + return fmt.Errorf("remove hardlink probe %q: %w", source, err) + } + if err := os.Remove(target); err != nil { + return fmt.Errorf("remove hardlink probe %q: %w", target, err) + } + } + return nil +} + +// stop prevents further snapshots, waits for accepted requests to enqueue their +// barriers, and then waits for active publication. Queued barriers are canceled +// before they start when backend close drains their queues. +func (m *snapshotManager) stop() { + m.mu.Lock() + m.stopped = true + m.mu.Unlock() + m.scheduling.Wait() + m.publishing.Wait() +} + +func (m *snapshotManager) isRunning() bool { + m.mu.Lock() + defer m.mu.Unlock() + return !m.stopped +} + +// maybeSnapshot takes a snapshot when version lands on an interval boundary. +// It is called from the write path for every version, so the common case is the +// modulo test and nothing else. +func (m *snapshotManager) maybeSnapshot(version int64) { + if m == nil || version <= 0 || m.interval <= 0 || version%m.interval != 0 { + return + } + now := time.Now() + m.mu.Lock() + previous := m.lastRequested + previousRequestAt := m.lastRequestAt + var skipReason string + accepted := false + switch { + case m.stopped || version <= m.lastRequested: + // A repeated commit-path call is expected and is not a skipped attempt. + case m.inFlight: + skipReason = "in_flight" + case !m.lastRequestAt.IsZero() && now.Sub(m.lastRequestAt) < m.minTime: + skipReason = "minimum_time_interval" + default: + m.lastRequested = version + m.lastRequestAt = now + m.inFlight = true + m.scheduling.Add(1) + recordSnapshotInFlight(1) + accepted = true + } + m.mu.Unlock() + if !accepted { + if skipReason != "" { + recordSnapshotSkipped(skipReason) + // A skipped boundary is the reason a snapshot an operator expected is + // not on disk, so name the gate rather than leaving only a metric. + logger.Info("skipping state store snapshot", "version", version, "reason", skipReason) + } + return + } + defer m.scheduling.Done() + start := time.Now() + recordSnapshotAttempt() + if err := m.requestSnapshot(version, start); err != nil { + recordSnapshotCompletion(start, "failure") + m.mu.Lock() + if m.lastRequested == version { + m.lastRequested = previous + m.lastRequestAt = previousRequestAt + m.inFlight = false + recordSnapshotInFlight(0) + } + m.mu.Unlock() + logger.Error("state store snapshot failed", "version", version, "error", err) + } +} + +func (m *snapshotManager) finishSnapshot() { + m.mu.Lock() + m.inFlight = false + recordSnapshotInFlight(0) + m.mu.Unlock() +} + +// requestSnapshot asks every backend to checkpoint itself into a staging +// directory and publishes the result once they all have. +// +// The label is exact because of when this runs: the caller has just enqueued +// version on the backends and has not enqueued anything above it, so a barrier +// placed in each apply queue now captures that backend with everything up to +// version applied and nothing after it. The backends reach their barriers +// independently and at different wall-clock times, and the caller waits for none +// of the checkpointing — enqueueing a barrier costs what enqueueing a changeset +// costs. The caller does wait for the staging directories below: one Stat, one +// RemoveAll and one MkdirAll per target, on the commit path and ahead of the SC +// apply. +func (m *snapshotManager) requestSnapshot(version int64, start time.Time) error { + name := SnapshotDirName(version) + finalDir := filepath.Join(m.root, name) + if _, err := os.Stat(finalDir); err == nil { + return fmt.Errorf("snapshot dir %q already exists", finalDir) + } else if !os.IsNotExist(err) { + return fmt.Errorf("inspect snapshot dir %q: %w", finalDir, err) + } + tmpDir := filepath.Join(m.root, snapshotTmpPrefix+name) + if err := os.RemoveAll(tmpDir); err != nil { + return fmt.Errorf("clear stale snapshot tmp dir: %w", err) + } + + targets := []checkpointTarget{ + {m.cosmosScheduler, filepath.Join(tmpDir, "cosmos", m.backend)}, + } + if m.evmScheduler != nil { + targets = append(targets, checkpointTarget{ + m.evmScheduler, + filepath.Join(tmpDir, "evm", m.backend), + }) + } + for _, target := range targets { + if err := os.MkdirAll(filepath.Dir(target.dest), 0o750); err != nil { + _ = os.RemoveAll(tmpDir) + return fmt.Errorf("create snapshot dir: %w", err) + } + } + + var ( + mu sync.Mutex + remaining = len(targets) + firstErr error + ) + // Set up before scheduling because callbacks can complete while the loop is + // still scheduling the remaining targets. + for _, target := range targets { + target.store.ScheduleCheckpoint(target.dest, m.isRunning, func(err error) { + mu.Lock() + if err != nil && firstErr == nil { + firstErr = err + } + remaining-- + last, outcome := remaining == 0, firstErr + mu.Unlock() + if !last { + return + } + m.startPublish(version, tmpDir, finalDir, targets, outcome, start) + }) + } + return nil +} + +// startPublish hands a finished set of checkpoints off to a goroutine. It runs +// on whichever backend's apply goroutine finished last, so it must not do the +// work itself: publishing renames directories and prunes old snapshots, and a +// writer stalled on that is a writer not applying blocks. +func (m *snapshotManager) startPublish( + version int64, + tmpDir, finalDir string, + targets []checkpointTarget, + checkpointErr error, + start time.Time, +) { + // Taken under the same lock stop uses, so no goroutine is registered after + // stop has started waiting. + m.mu.Lock() + if m.stopped { + m.mu.Unlock() + _ = os.RemoveAll(tmpDir) + recordSnapshotCompletion(start, "canceled") + m.finishSnapshot() + return + } + m.publishing.Add(1) + m.mu.Unlock() + + go func() { + defer m.publishing.Done() + defer m.finishSnapshot() + if checkpointErr != nil { + if errors.Is(checkpointErr, management.ErrCheckpointCanceled) { + recordSnapshotCompletion(start, "canceled") + } else { + recordSnapshotCompletion(start, "failure") + logger.Error("state store snapshot failed", "version", version, "error", checkpointErr) + } + _ = os.RemoveAll(tmpDir) + return + } + for _, target := range targets { + if err := target.store.SetCheckpointVersion(target.dest, version); err != nil { + recordSnapshotCompletion(start, "failure") + logger.Error("failed to set state store snapshot version", + "version", version, "dir", target.dest, "error", err) + _ = os.RemoveAll(tmpDir) + return + } + } + if m.publish(version, tmpDir, finalDir, start) { + recordSnapshotCompletion(start, "success") + } else { + recordSnapshotCompletion(start, "failure") + } + }() +} + +// publish moves a finished checkpoint into place and reports whether the whole +// publication succeeded. Retention runs either way. +// +// A boundary that fails anywhere past the barrier is given up on, and this is +// deliberate. maybeSnapshot restores lastRequested when requestSnapshot fails, +// because that failure happens before any barrier is enqueued and the boundary +// was never claimed. Once the barriers are out, the version they captured is +// the only image of that boundary there will ever be: the write path has moved +// on, so re-running the attempt would checkpoint a later state under the older +// label, which is the one thing the label is supposed to rule out. Recovery is +// therefore the next boundary rather than a retry of this one, at the cost of +// one snapshot interval of coverage. The error log and the outcome="failure" +// counter are the signal. +func (m *snapshotManager) publish(version int64, tmpDir, finalDir string, start time.Time) bool { + apparentBytes, sizeErr := snapshotDirApparentBytes(tmpDir) + if sizeErr != nil { + logger.Error("failed to measure state store snapshot", "dir", tmpDir, "error", sizeErr) + } else if err := writeSnapshotSize(tmpDir, apparentBytes); err != nil { + logger.Error("failed to persist state store snapshot size", "dir", tmpDir, "error", err) + sizeErr = err + } + + m.publishMu.Lock() + defer m.publishMu.Unlock() + defer m.prune() + + if err := os.Rename(tmpDir, finalDir); err != nil { + logger.Error("failed to finalize state store snapshot", "version", version, "error", err) + _ = os.RemoveAll(tmpDir) + return false + } + if err := syncDir(m.root); err != nil { + logger.Error("failed to persist state store snapshot publication", + "version", version, "dir", finalDir, "error", err) + return false + } + if sizeErr == nil { + if m.snapshotSizes == nil { + m.snapshotSizes = map[int64]int64{} + } + m.snapshotSizes[version] = apparentBytes + } + logger.Info("state store snapshot created", + "version", version, "dir", finalDir, "took", time.Since(start).String()) + + // Snapshots can finish out of order, so only move the link forward. + if version > m.lastPublished { + if err := m.updateCurrentLink(SnapshotDirName(version)); err != nil { + // The snapshot itself is intact and discoverable by name; only the + // convenience symlink is stale. The link is part of the publication + // contract, so record this attempt as a failure. + logger.Error("failed to update state store snapshot current link", + "version", version, "error", err) + return false + } + m.lastPublished = version + } + snapshotMetrics.CurrentHeight.Record(context.Background(), m.lastPublished) + return true +} + +func (m *snapshotManager) newestSnapshotVersion() int64 { + versions, err := ListSnapshotVersions(m.root) + if err != nil { + logger.Error("failed to list state store snapshots", "error", err) + return 0 + } + if len(versions) == 0 { + return 0 + } + return versions[len(versions)-1] +} + +func (m *snapshotManager) snapshotModTime(version int64) time.Time { + if version <= 0 { + return time.Time{} + } + info, err := os.Stat(filepath.Join(m.root, SnapshotDirName(version))) + if err != nil { + logger.Error("failed to read state store snapshot modification time", + "version", version, "error", err) + return time.Time{} + } + return info.ModTime() +} + +// removeStaleTmpDirs clears staging directories left behind by a crash or a +// shutdown that landed mid-snapshot. They are named after the snapshot they +// were staging, so they would otherwise sit there until that exact boundary +// came round again. +func (m *snapshotManager) removeStaleTmpDirs() { + tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) + if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) { + logger.Error("failed to remove stale state store snapshot link", "path", tmpLink, "error", err) + } + + entries, err := os.ReadDir(m.root) + if err != nil { + if !os.IsNotExist(err) { + logger.Error("failed to scan state store snapshots dir", "error", err) + } + return + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), snapshotTmpPrefix) { + continue + } + dir := filepath.Join(m.root, entry.Name()) + if err := os.RemoveAll(dir); err != nil { + logger.Error("failed to remove stale snapshot tmp dir", "dir", dir, "error", err) + continue + } + logger.Info("removed stale state store snapshot tmp dir", "dir", dir) + } +} + +// updateCurrentLink atomically points the current symlink at name. +func (m *snapshotManager) updateCurrentLink(name string) error { + tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) + _ = os.Remove(tmpLink) + if err := os.Symlink(name, tmpLink); err != nil { + return fmt.Errorf("create snapshot current symlink: %w", err) + } + if err := os.Rename(tmpLink, filepath.Join(m.root, snapshotCurrentLink)); err != nil { + return fmt.Errorf("swap snapshot current symlink: %w", err) + } + return syncDir(m.root) +} + +func syncDir(path string) error { + // #nosec G304 -- path is an internal database or snapshot directory, not request input. + dir, err := os.Open(path) + if err != nil { + return fmt.Errorf("open directory %q for sync: %w", path, err) + } + syncErr := dir.Sync() + closeErr := dir.Close() + if syncErr != nil { + syncErr = fmt.Errorf("sync directory %q: %w", path, syncErr) + } + if closeErr != nil { + closeErr = fmt.Errorf("close directory %q after sync: %w", path, closeErr) + } + return errors.Join(syncErr, closeErr) +} + +// prune removes all but the newest 1+keepRecent snapshots. +func (m *snapshotManager) prune() { + versions, err := ListSnapshotVersions(m.root) + if err != nil { + logger.Error("failed to list state store snapshots for pruning", "error", err) + return + } + defer m.recordRetentionMetrics() + keep := 1 + m.keepRecent + if len(versions) <= keep { + return + } + for _, v := range versions[:len(versions)-keep] { + dir := filepath.Join(m.root, SnapshotDirName(v)) + if err := os.RemoveAll(dir); err != nil { + logger.Error("failed to prune state store snapshot", "dir", dir, "error", err) + continue + } + logger.Info("pruned state store snapshot", "dir", dir) + } +} + +func (m *snapshotManager) recordRetentionMetrics() { + versions, err := ListSnapshotVersions(m.root) + if err != nil { + logger.Error("failed to list state store snapshots for metrics", "error", err) + return + } + snapshotMetrics.RetainedCount.Record(context.Background(), int64(len(versions))) + + if m.snapshotSizes == nil { + m.snapshotSizes = map[int64]int64{} + } + retained := make(map[int64]struct{}, len(versions)) + var apparentBytes int64 + for _, version := range versions { + retained[version] = struct{}{} + if size, ok := m.snapshotSizes[version]; ok { + apparentBytes += size + continue + } + dir := filepath.Join(m.root, SnapshotDirName(version)) + size, err := readSnapshotSize(dir) + if err != nil { + if !os.IsNotExist(err) { + logger.Error("failed to read state store snapshot size", "dir", dir, "error", err) + } + continue + } + m.snapshotSizes[version] = size + apparentBytes += size + } + for version := range m.snapshotSizes { + if _, ok := retained[version]; !ok { + delete(m.snapshotSizes, version) + } + } + snapshotMetrics.ApparentBytes.Record(context.Background(), apparentBytes) +} + +func snapshotDirApparentBytes(dir string) (int64, error) { + var apparentBytes int64 + err := filepath.WalkDir(dir, func(_ string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.Type().IsRegular() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + apparentBytes += info.Size() + return nil + }) + return apparentBytes, err +} + +func writeSnapshotSize(dir string, size int64) error { + path := filepath.Join(dir, snapshotSizeFile) + // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, writeErr := fmt.Fprintf(file, "%d\n", size) + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(writeErr, syncErr, closeErr); err != nil { + return err + } + return syncDir(dir) +} + +func readSnapshotSize(dir string) (int64, error) { + // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. + data, err := os.ReadFile(filepath.Join(dir, snapshotSizeFile)) + if err != nil { + return 0, err + } + size, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse snapshot size in %q: %w", dir, err) + } + if size < 0 { + return 0, fmt.Errorf("snapshot size in %q must be non-negative", dir) + } + return size, nil +} diff --git a/sei-db/state_db/ss/composite/snapshot_metrics.go b/sei-db/state_db/ss/composite/snapshot_metrics.go new file mode 100644 index 0000000000..d908055f01 --- /dev/null +++ b/sei-db/state_db/ss/composite/snapshot_metrics.go @@ -0,0 +1,94 @@ +package composite + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + commonmetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" +) + +var snapshotMeter = otel.Meter("seidb_ss_snapshot") + +var snapshotMetrics = struct { + Attempts metric.Int64Counter + Skipped metric.Int64Counter + Completions metric.Int64Counter + Duration metric.Float64Histogram + InFlight metric.Int64Gauge + CurrentHeight metric.Int64Gauge + RetainedCount metric.Int64Gauge + ApparentBytes metric.Int64Gauge +}{ + Attempts: must(snapshotMeter.Int64Counter( + "ss_snapshot_attempts", + metric.WithDescription("Number of state-store snapshot attempts"), + metric.WithUnit("{count}"), + )), + Skipped: must(snapshotMeter.Int64Counter( + "ss_snapshot_skipped", + metric.WithDescription("Number of state-store snapshot boundaries skipped by a scheduling gate"), + metric.WithUnit("{count}"), + )), + Completions: must(snapshotMeter.Int64Counter( + "ss_snapshot_completions", + metric.WithDescription("Number of completed state-store snapshot attempts"), + metric.WithUnit("{count}"), + )), + Duration: must(snapshotMeter.Float64Histogram( + "ss_snapshot_duration", + metric.WithDescription("Time from a state-store snapshot request to completion"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(commonmetrics.LongLatencyBuckets...), + )), + InFlight: must(snapshotMeter.Int64Gauge( + "ss_snapshot_in_flight", + metric.WithDescription("Whether one state-store snapshot is currently in flight"), + )), + CurrentHeight: must(snapshotMeter.Int64Gauge( + "ss_snapshot_current_height", + metric.WithDescription("Height of the newest published state-store snapshot"), + )), + RetainedCount: must(snapshotMeter.Int64Gauge( + "ss_snapshot_retained_count", + metric.WithDescription("Number of retained state-store snapshots"), + metric.WithUnit("{count}"), + )), + ApparentBytes: must(snapshotMeter.Int64Gauge( + "ss_snapshot_retained_apparent_bytes", + metric.WithDescription("Apparent bytes referenced by retained state-store snapshots; hardlinks can share physical blocks"), + metric.WithUnit("By"), + )), +} + +func must[V any](instrument V, err error) V { + if err != nil { + panic(err) + } + return instrument +} + +func recordSnapshotAttempt() { + snapshotMetrics.Attempts.Add(context.Background(), 1) +} + +func recordSnapshotSkipped(reason string) { + snapshotMetrics.Skipped.Add( + context.Background(), + 1, + metric.WithAttributes(attribute.String("reason", reason)), + ) +} + +func recordSnapshotInFlight(value int64) { + snapshotMetrics.InFlight.Record(context.Background(), value) +} + +func recordSnapshotCompletion(start time.Time, outcome string) { + attrs := metric.WithAttributes(attribute.String("outcome", outcome)) + snapshotMetrics.Completions.Add(context.Background(), 1, attrs) + snapshotMetrics.Duration.Record(context.Background(), time.Since(start).Seconds(), attrs) +} diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go new file mode 100644 index 0000000000..565a9fa45a --- /dev/null +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -0,0 +1,949 @@ +package composite + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/cosmos" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" + "github.com/stretchr/testify/require" +) + +type noCheckpointStateStore struct { + types.StateStore +} + +type noBarrierStateStore struct { + types.StateStore +} + +func (*noBarrierStateStore) Checkpoint(string) error { + return nil +} + +func (*noBarrierStateStore) SetCheckpointVersion(string, int64) error { + return nil +} + +type controlledSnapshotScheduler struct { + pending chan func() + entered chan struct{} + checkpointCalls int +} + +func (*controlledSnapshotScheduler) SupportsCheckpoint() bool { + return true +} + +func (s *controlledSnapshotScheduler) ScheduleCheckpoint( + destDir string, + shouldRun func() bool, + done func(error), +) { + if s.entered != nil { + close(s.entered) + } + s.pending <- func() { + if !shouldRun() { + done(management.ErrCheckpointCanceled) + return + } + s.checkpointCalls++ + _ = os.MkdirAll(destDir, 0o750) + done(nil) + } +} + +func (*controlledSnapshotScheduler) SetCheckpointVersion(string, int64) error { + return nil +} + +func bankChangeset(key, value string) []*proto.NamedChangeSet { + return []*proto.NamedChangeSet{ + { + Name: "bank", + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: []byte(key), Value: []byte(value)}}, + }, + }, + } +} + +// evmStorageKey builds a key in the EVM storage family (0x03 prefix), which +// routes to the storage sub-DB when sub-DBs are separate. +func evmStorageKey() []byte { + return append([]byte{0x03}, make([]byte, 20+32)...) +} + +// setupSnapshotStore opens a store with snapshotting on at a small interval so +// tests can cross boundaries cheaply. It returns the store and its snapshots +// root. +func setupSnapshotStore(t *testing.T, interval int64, keepRecent int, separateEVMSubDBs bool) (*CompositeStateStore, string) { + t.Helper() + dir := t.TempDir() + store, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 100, + KeepRecent: 100000, + EVMSplit: true, + SeparateEVMSubDBs: separateEVMSubDBs, + EVMDBDirectory: filepath.Join(dir, "evm_ss"), + SnapshotEnable: true, + SnapshotInterval: interval, + SnapshotKeepRecent: keepRecent, + }, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + require.NotNil(t, store.snapshotMgr) + return store, filepath.Join(dir, "data", "state_store", SnapshotsDirName) +} + +type pendingWaiter interface { + WaitForPendingWrites() +} + +// settle waits until every snapshot requested so far has been published and its +// pruning finished. Snapshot barriers sit in the backends' apply queues, so +// draining those queues is what guarantees the barriers ran. +func settle(t *testing.T, store *CompositeStateStore) { + t.Helper() + if w, ok := store.cosmosStore.(pendingWaiter); ok { + w.WaitForPendingWrites() + } + if w, ok := store.evmStore.(pendingWaiter); ok { + w.WaitForPendingWrites() + } + store.snapshotMgr.publishing.Wait() +} + +// commitBlock is what rootmulti.flush does for a populated block: enqueue the +// changesets, then hand the version to the snapshot manager. ApplyChangesetAsync +// alone schedules nothing, so tests that expect a snapshot must come through +// here. +func commitBlock(t *testing.T, store *CompositeStateStore, version int64, changesets []*proto.NamedChangeSet) { + t.Helper() + require.NoError(t, store.ApplyChangesetAsync(version, changesets)) + store.ScheduleSnapshot(version) +} + +func writeBlock(t *testing.T, store *CompositeStateStore, version int64) { + t.Helper() + commitBlock(t, store, version, []*proto.NamedChangeSet{ + { + Name: "bank", + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: []byte("balance"), Value: []byte{byte(version)}}}, + }, + }, + { + Name: evm.EVMStoreKey, + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: evmStorageKey(), Value: []byte{byte(version)}}}, + }, + }, + }) +} + +// The snapshot manager keys off the mirrored cadence, so the ss-snapshot-enable +// switch has to reach it as a zero interval and leave no manager running. +func TestSnapshotManagerRespectsSnapshotEnable(t *testing.T) { + for _, tc := range []struct { + name string + enable bool + wantRunning bool + }{ + {name: "enabled", enable: true, wantRunning: true}, + {name: "disabled", enable: false, wantRunning: false}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + ssConfig := config.DefaultStateStoreConfig() + ssConfig.SnapshotEnable = tc.enable + config.AlignSSSnapshotWithSC(config.DefaultStateCommitConfig(), &ssConfig) + + store, err := NewCompositeStateStore(ssConfig, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + if tc.wantRunning { + require.NotNil(t, store.snapshotMgr, "explicit opt-in starts snapshotting") + require.Positive(t, ssConfig.SnapshotInterval) + } else { + require.Nil(t, store.snapshotMgr) + require.Zero(t, ssConfig.SnapshotInterval) + } + }) + } +} + +func TestCustomStateStoreDirectoryMovesSnapshotRootBesideDatabase(t *testing.T) { + home := t.TempDir() + customDB := filepath.Join(t.TempDir(), "cosmos-state") + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + cfg.DBDirectory = customDB + cfg.SnapshotEnable = true + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1 + + store, err := NewCompositeStateStore(cfg, home) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + require.Equal(t, customDB+"-"+SnapshotsDirName, store.snapshotMgr.root) +} + +func TestSnapshotHardlinkPreflightCleansProbeFiles(t *testing.T) { + source := t.TempDir() + root := t.TempDir() + require.NoError(t, verifySnapshotHardlinks(root, []string{source})) + + sourceEntries, err := os.ReadDir(source) + require.NoError(t, err) + require.Empty(t, sourceEntries) + rootEntries, err := os.ReadDir(root) + require.NoError(t, err) + require.Empty(t, rootEntries) +} + +func TestSnapshotHardlinkPreflightRejectsCrossFilesystem(t *testing.T) { + root, err := os.MkdirTemp("/dev/shm", "ss-snapshot-test-*") + if err != nil { + t.Skipf("no separate /dev/shm filesystem: %v", err) + } + t.Cleanup(func() { require.NoError(t, os.RemoveAll(root)) }) + + err = verifySnapshotHardlinks(root, []string{t.TempDir()}) + if err == nil { + t.Skip("temporary directory and /dev/shm use the same filesystem") + } + require.ErrorContains(t, err, "cannot hardlink snapshots") +} + +func TestSnapshotManagerRejectsUnsupportedBackend(t *testing.T) { + store := &CompositeStateStore{ + cosmosStore: cosmos.NewCosmosStateStore(&noCheckpointStateStore{}), + config: config.StateStoreConfig{ + Backend: config.RocksDBBackend, + SnapshotInterval: 10, + }, + } + + err := store.startSnapshotManager(t.TempDir(), nil) + require.ErrorContains(t, err, "does not support checkpoints") + require.Nil(t, store.snapshotMgr) +} + +func TestSnapshotManagerRejectsBackendWithoutBarrier(t *testing.T) { + store := &CompositeStateStore{ + cosmosStore: cosmos.NewCosmosStateStore(&noBarrierStateStore{}), + config: config.StateStoreConfig{ + Backend: config.PebbleDBBackend, + SnapshotInterval: 10, + }, + } + + err := store.startSnapshotManager(t.TempDir(), nil) + require.ErrorContains(t, err, "does not support checkpoints") + require.Nil(t, store.snapshotMgr) +} + +func TestSnapshotStopCancelsQueuedCheckpoint(t *testing.T) { + scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + manager.maybeSnapshot(5) + manager.stop() + (<-scheduler.pending)() + + require.Zero(t, scheduler.checkpointCalls) + versions, err := ListSnapshotVersions(manager.root) + require.NoError(t, err) + require.Empty(t, versions) +} + +func TestSnapshotManagerAllowsOnlyOneInFlightSnapshot(t *testing.T) { + scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 2)} + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + manager.maybeSnapshot(5) + manager.maybeSnapshot(10) + require.Len(t, scheduler.pending, 1, "a second boundary must not enqueue while one snapshot is active") + + (<-scheduler.pending)() + manager.publishing.Wait() + require.False(t, manager.inFlight) + require.Equal(t, int64(5), manager.lastRequested) +} + +func TestSnapshotManagerAppliesMinimumTimeInterval(t *testing.T) { + scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 2)} + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + minTime: time.Hour, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + manager.maybeSnapshot(5) + (<-scheduler.pending)() + manager.publishing.Wait() + + manager.maybeSnapshot(10) + require.Empty(t, scheduler.pending, "a rapid boundary must be skipped") + + manager.mu.Lock() + manager.lastRequestAt = time.Now().Add(-2 * time.Hour) + manager.mu.Unlock() + manager.maybeSnapshot(10) + require.Len(t, scheduler.pending, 1) + (<-scheduler.pending)() + manager.publishing.Wait() +} + +func TestSnapshotStopWaitsForBarrierScheduling(t *testing.T) { + scheduler := &controlledSnapshotScheduler{ + pending: make(chan func()), + entered: make(chan struct{}), + } + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + requestDone := make(chan struct{}) + go func() { + manager.maybeSnapshot(5) + close(requestDone) + }() + <-scheduler.entered + + stopDone := make(chan struct{}) + go func() { + manager.stop() + close(stopDone) + }() + require.Never(t, func() bool { + select { + case <-stopDone: + return true + default: + return false + } + }, 50*time.Millisecond, 5*time.Millisecond) + + callback := <-scheduler.pending + <-requestDone + <-stopDone + callback() + require.False(t, manager.inFlight) +} + +// Snapshot labels are the interval boundaries themselves, not whatever version +// the store happened to be at when some background pass noticed. That is the +// property the in-queue barrier buys. It keeps each accepted SS snapshot's +// contents aligned with its own label even when SC independently skips that +// boundary. +func TestSnapshotTakenAtExactIntervalBoundaries(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, false) + + for v := int64(1); v <= 12; v++ { + writeBlock(t, store, v) + if v%5 == 0 { + settle(t, store) + } + } + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5, 10}, versions, + "snapshots must land on interval boundaries and nowhere else") + + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(10), target) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + apparentBytes, err := readSnapshotSize(snapDir) + require.NoError(t, err) + require.Positive(t, apparentBytes) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: config.PebbleDBBackend, + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", config.PebbleDBBackend), + EVMDBDirectory: filepath.Join(snapDir, "evm", config.PebbleDBBackend), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, int64(10), reopened.GetLatestVersion()) + cosmosValue, err := reopened.Get("bank", 12, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{10}, cosmosValue, "snapshot 10 must exclude Cosmos writes 11 and 12") + evmValue, err := reopened.Get(evm.EVMStoreKey, 12, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{10}, evmValue, "snapshot 10 must exclude EVM writes 11 and 12") +} + +func TestSnapshotTakenAtExactIntervalBoundaryWithoutEVMSplit(t *testing.T) { + dir := t.TempDir() + ssConfig := config.DefaultStateStoreConfig() + ssConfig.Backend = config.PebbleDBBackend + ssConfig.AsyncWriteBuffer = 100 + ssConfig.KeepRecent = 100000 + ssConfig.EVMSplit = false + ssConfig.SnapshotEnable = true + ssConfig.SnapshotInterval = 5 + ssConfig.SnapshotKeepRecent = 1 + + store, err := NewCompositeStateStore(ssConfig, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + require.NotNil(t, store.snapshotMgr) + + for version := int64(1); version <= 5; version++ { + commitBlock(t, store, version, bankChangeset("balance", "value")) + } + settle(t, store) + + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5}, versions) +} + +// A snapshot must be a complete image of every version at or below its label, +// reopenable as a store in its own right. +func TestSnapshotReopensWithEveryVersionBelowLabel(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + for v := int64(1); v <= 10; v++ { + writeBlock(t, store, v) + } + settle(t, store) + + const label = int64(10) + snapDir := filepath.Join(root, SnapshotDirName(label)) + require.DirExists(t, snapDir) + + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, label, reopened.GetLatestVersion(), + "the label is the version the snapshot was requested at") + for v := int64(1); v <= label; v++ { + val, err := reopened.Get("bank", v, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{byte(v)}, val, "cosmos version %d missing from snapshot", v) + val, err = reopened.Get(evm.EVMStoreKey, v, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{byte(v)}, val, "evm version %d missing from snapshot", v) + } +} + +// The property the barrier exists for: the label stays exact while the write +// path keeps going. Nothing is drained between block 10 and blocks 11 and 12, so +// the checkpoint runs with later versions already queued behind the barrier — the +// case a post-hoc "snapshot what has been applied" scheme would get wrong. +func TestSnapshotExcludesVersionsWrittenAfterTheBoundary(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + const label = int64(10) + for v := int64(1); v <= label; v++ { + writeBlock(t, store, v) + } + for v := label + 1; v <= label+2; v++ { + writeBlock(t, store, v) + } + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(label)) + require.DirExists(t, snapDir) + + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, label, reopened.GetLatestVersion()) + // Reading above the label returns the label's value rather than 11 or 12, + // which is what "excluded" means for an MVCC store: the later writes are not + // in this image at any version. + for _, above := range []int64{label + 1, label + 2} { + val, err := reopened.Get("bank", above, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{byte(label)}, val, + "cosmos read at %d saw a write from after the boundary", above) + val, err = reopened.Get(evm.EVMStoreKey, above, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{byte(label)}, val, + "evm read at %d saw a write from after the boundary", above) + } + + // The live store keeps them, so the snapshot dropped them rather than the + // writes never landing. + val, err := store.Get("bank", label+2, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{byte(label + 2)}, val) +} + +// A snapshot inherits each database's earliest marker. The composite is allowed +// to reopen with different member floors because it reports the highest one. +func TestSnapshotInheritsPerStoreEarliestMarkers(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + for v := int64(1); v <= 9; v++ { + writeBlock(t, store, v) + } + settle(t, store) + require.NoError(t, store.cosmosStore.SetEarliestVersion(2, false)) + require.NoError(t, store.evmStore.SetEarliestVersion(5, false)) + require.Equal(t, int64(2), store.cosmosStore.GetEarliestVersion()) + require.Equal(t, int64(5), store.evmStore.GetEarliestVersion()) + + writeBlock(t, store, 10) + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + require.DirExists(t, snapDir) + + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err, "a snapshot with different member floors must reopen") + defer reopened.Close() + + require.Equal(t, int64(2), reopened.cosmosStore.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.evmStore.GetEarliestVersion()) +} + +func TestSnapshotInheritsEarliestMarkerAfterPrune(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + for v := int64(1); v <= 9; v++ { + writeBlock(t, store, v) + } + settle(t, store) + require.NoError(t, store.Prune(4)) + + writeBlock(t, store, 10) + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, int64(5), reopened.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.cosmosStore.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.evmStore.GetEarliestVersion()) +} + +// With separate sub-DBs the latest label has to reach every one of them, not +// just the sub-DBs that took writes, or the snapshot is not self-describing at +// its exact boundary. +func TestSnapshotSetsLatestVersionEveryEVMSubDB(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, true) + + for v := int64(1); v <= 9; v++ { + writeBlock(t, store, v) + } + + writeBlock(t, store, 10) + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + SeparateEVMSubDBs: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + + require.Equal(t, int64(10), reopened.evmStore.GetLatestVersion()) + require.NoError(t, reopened.Close()) + + evmRoot := filepath.Join(snapDir, "evm", "pebbledb") + for _, storeType := range evm.AllEVMStoreTypes() { + subDir := filepath.Join(evmRoot, evm.StoreTypeName(storeType)) + subDB, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + UseDefaultComparer: true, + DBDirectory: subDir, + }, t.TempDir()) + require.NoError(t, err) + require.Equal(t, int64(10), subDB.GetLatestVersion(), "sub-DB %s latest marker", evm.StoreTypeName(storeType)) + require.NoError(t, subDB.Close()) + } +} + +// The reason the barrier has to be a message in every queue rather than a wait: +// a block that only touches storage keys is enqueued only on the storage sub-DB, +// so the idle sub-DBs never observe that version and no amount of waiting would +// tell them it passed. Every sub-DB must still be captured. +func TestSnapshotCapturesIdleEVMSubDBs(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, true) + + // Storage keys only: codehash, code and misc sub-DBs stay idle throughout. + for v := int64(1); v <= 5; v++ { + commitBlock(t, store, v, []*proto.NamedChangeSet{ + { + Name: evm.EVMStoreKey, + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: evmStorageKey(), Value: []byte{byte(v)}}}, + }, + }, + }) + } + settle(t, store) + + evmRoot := filepath.Join(root, SnapshotDirName(5), "evm", "pebbledb") + for _, storeType := range evm.AllEVMStoreTypes() { + name := evm.StoreTypeName(storeType) + subDir := filepath.Join(evmRoot, name) + require.DirExists(t, subDir, "sub-DB %s missing from snapshot", name) + // A checkpoint always carries a manifest. An empty directory would mean + // the barrier never reached that sub-DB. + manifests, err := filepath.Glob(filepath.Join(subDir, "MANIFEST-*")) + require.NoError(t, err) + require.NotEmpty(t, manifests, "sub-DB %s was not checkpointed", name) + } + + // The storage sub-DB is the one that actually took writes, and it must be + // readable at every version up to the label. + storage, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + // EVM sub-DBs are opened with the plain byte comparer. + UseDefaultComparer: true, + DBDirectory: filepath.Join(evmRoot, evm.StoreTypeName(evm.StoreStorage)), + }, t.TempDir()) + require.NoError(t, err) + defer storage.Close() + + for v := int64(1); v <= 5; v++ { + val, err := storage.Get(evm.EVMStoreKey, v, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{byte(v)}, val, "evm storage version %d missing from snapshot", v) + } +} + +// An interval boundary that happens to be an empty block arrives through +// SetLatestVersion rather than the changeset path, and must still snapshot — +// otherwise a quiet chain skips whole intervals. +func TestSnapshotTakenOnEmptyBoundaryBlock(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, false) + + for v := int64(1); v <= 4; v++ { + writeBlock(t, store, v) + } + // Block 5 is empty: marker only, nothing enqueued. + require.NoError(t, store.SetLatestVersion(5)) + store.ScheduleSnapshot(5) + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5}, versions) + + // Every data version below the label is inside the snapshot, and the + // checkpoint marker advances to the empty block's version. + snapDir := filepath.Join(root, SnapshotDirName(5)) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, int64(5), reopened.GetLatestVersion()) + val, err := reopened.Get("bank", 4, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{4}, val) +} + +func TestSetLatestVersionDoesNotSnapshotDuringImport(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, false) + nodes := make(chan types.SnapshotNode) + importDone := make(chan error, 1) + go func() { + importDone <- store.Import(5, nodes) + }() + closed := false + t.Cleanup(func() { + if !closed { + close(nodes) + <-importDone + } + }) + + nodes <- types.SnapshotNode{StoreKey: "bank", Key: []byte("balance"), Value: []byte{5}} + require.NoError(t, store.SetLatestVersion(5)) + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Empty(t, versions, "direct restore metadata writes must not trigger a snapshot") + + close(nodes) + closed = true + require.NoError(t, <-importDone) +} + +// TestSnapshotPrune verifies retention: with keepRecent=1, only the newest two +// snapshots survive and current tracks the newest. +func TestSnapshotPrune(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 1, false) + + for v := int64(1); v <= 15; v++ { + writeBlock(t, store, v) + if v%5 == 0 { + settle(t, store) + } + } + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{10, 15}, versions) + + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(15), target) +} + +func TestSnapshotManagerResumesFromNewestSnapshot(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + cfg.AsyncWriteBuffer = 100 + cfg.KeepRecent = 100000 + cfg.SnapshotEnable = true + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1 + cfg.SnapshotMinTimeInterval = time.Hour + + store, err := NewCompositeStateStore(cfg, dir) + require.NoError(t, err) + for version := int64(1); version <= 5; version++ { + commitBlock(t, store, version, bankChangeset("balance", "value")) + } + settle(t, store) + + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + snapshotDir := filepath.Join(root, SnapshotDirName(5)) + before, err := os.Stat(snapshotDir) + require.NoError(t, err) + require.NoError(t, store.Close()) + require.NoError(t, os.Remove(filepath.Join(root, snapshotCurrentLink))) + + reopened, err := NewCompositeStateStore(cfg, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + require.Equal(t, int64(5), reopened.snapshotMgr.lastRequested) + require.WithinDuration(t, before.ModTime(), reopened.snapshotMgr.lastRequestAt, time.Second) + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(5), target) + + reopened.ScheduleSnapshot(5) + settle(t, reopened) + after, err := os.Stat(snapshotDir) + require.NoError(t, err) + require.True(t, os.SameFile(before, after), "restart must not replace an existing boundary snapshot") + + for version := int64(6); version <= 10; version++ { + commitBlock(t, reopened, version, bankChangeset("balance", "value")) + } + settle(t, reopened) + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5}, versions, "restart must preserve the minimum-time gate") +} + +func TestOutOfOrderPublishDoesNotMoveCurrentBackward(t *testing.T) { + root := t.TempDir() + manager := &snapshotManager{root: root, keepRecent: 5} + + publish := func(version int64) { + tmpDir := filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(version)) + require.NoError(t, os.MkdirAll(tmpDir, 0o750)) + manager.publish(version, tmpDir, filepath.Join(root, SnapshotDirName(version)), time.Now()) + } + publish(10) + publish(5) + + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(10), target) +} + +func TestSnapshotManagerPrunesExistingSnapshotsAtStartup(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + for _, version := range []int64{5, 10, 15} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + cfg.SnapshotEnable = true + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1 + store, err := NewCompositeStateStore(cfg, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{10, 15}, versions) +} + +func TestFailedPublishStillEnforcesRetention(t *testing.T) { + root := t.TempDir() + for _, version := range []int64{5, 10, 15} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + manager := &snapshotManager{root: root, keepRecent: 1} + + published := manager.publish( + 20, + filepath.Join(root, "missing-staging-dir"), + filepath.Join(root, SnapshotDirName(20)), + time.Now(), + ) + require.False(t, published) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{10, 15}, versions) +} + +func TestRetentionMetricsCacheSnapshotSizes(t *testing.T) { + root := t.TempDir() + snapshotDir := filepath.Join(root, SnapshotDirName(5)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o750)) + dataFile := filepath.Join(snapshotDir, "data.sst") + require.NoError(t, os.WriteFile(dataFile, []byte("one"), 0o600)) + require.NoError(t, writeSnapshotSize(snapshotDir, 3)) + + manager := &snapshotManager{root: root} + manager.recordRetentionMetrics() + require.Equal(t, int64(3), manager.snapshotSizes[5]) + + // Published snapshots are immutable, so later metric records reuse the + // cached total rather than walking every retained hardlink tree again. + require.NoError(t, os.WriteFile(dataFile, []byte("a longer value"), 0o600)) + manager.recordRetentionMetrics() + require.Equal(t, int64(3), manager.snapshotSizes[5]) + + require.NoError(t, os.RemoveAll(snapshotDir)) + manager.recordRetentionMetrics() + require.NotContains(t, manager.snapshotSizes, int64(5)) +} + +func TestSnapshotRequestReturnsUnexpectedStatError(t *testing.T) { + root := t.TempDir() + name := SnapshotDirName(5) + require.NoError(t, os.Symlink(name, filepath.Join(root, name))) + + manager := &snapshotManager{root: root, backend: config.PebbleDBBackend} + err := manager.requestSnapshot(5, time.Now()) + require.ErrorContains(t, err, "inspect snapshot dir") +} + +// A crash mid-snapshot leaves a staging directory named after the boundary it +// was staging, which would otherwise sit there until that exact boundary came +// round again. +func TestStaleSnapshotTmpDirRemovedAtStartup(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + stale := filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(40)) + require.NoError(t, os.MkdirAll(filepath.Join(stale, "cosmos"), 0o750)) + tmpLink := filepath.Join(root, snapshotCurrentTmpLink) + require.NoError(t, os.Symlink(filepath.Base(stale), tmpLink)) + + ssConfig := config.DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + config.AlignSSSnapshotWithSC(config.DefaultStateCommitConfig(), &ssConfig) + store, err := NewCompositeStateStore(ssConfig, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + require.NoDirExists(t, stale) + _, err = os.Lstat(tmpLink) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestParseSnapshotVersion(t *testing.T) { + v, ok := ParseSnapshotVersion(SnapshotDirName(219140000)) + require.True(t, ok) + require.Equal(t, int64(219140000), v) + + for _, bad := range []string{"snapshot-", "snapshot-123", "current", "tmp-snapshot-00000000000000000010", "snapshot-0000000000000000001x"} { + _, ok := ParseSnapshotVersion(bad) + require.False(t, ok, "expected %q to be rejected", bad) + } +} diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index d88c77647e..70326b70da 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "os" + "path/filepath" "sync" dbm "github.com/tendermint/tm-db" @@ -34,6 +35,7 @@ type CompositeStateStore struct { cosmosStore types.StateStore // CosmosStateStore wrapping MVCC DB evmStore types.StateStore // EVMStateStore wrapping sub MVCC DBs (nil if disabled) pruningManager *pruning.Manager + snapshotMgr *snapshotManager config config.StateStoreConfig closeOnce sync.Once closeErr error @@ -60,6 +62,7 @@ func NewCompositeStateStore( cosmosStore: cosmosStore, config: ssConfig, } + snapshotSourceDirs := []string{dbHome} if ssConfig.EVMSplit { evmDir := ssConfig.EVMDBDirectory @@ -79,6 +82,16 @@ func NewCompositeStateStore( return nil, fmt.Errorf("failed to create EVM store: %w", err) } cs.evmStore = evmStore + if ssConfig.SeparateEVMSubDBs { + for _, storeType := range evm.AllEVMStoreTypes() { + snapshotSourceDirs = append( + snapshotSourceDirs, + filepath.Join(evmDir, evm.StoreTypeName(storeType)), + ) + } + } else { + snapshotSourceDirs = append(snapshotSourceDirs, evmDir) + } logger.Info("EVM state store enabled", "dir", evmDir, "separateDBs", ssConfig.SeparateEVMSubDBs, @@ -97,12 +110,22 @@ func NewCompositeStateStore( return nil, fmt.Errorf("failed to recover state store: %w", err) } - // Mismatched earliest versions = DBs from different snapshots; reads would diverge. - if err := cs.validateEVMSSPostRecovery(); err != nil { - _ = cs.Close() - return nil, err - } + cs.validateEVMSSPostRecovery() + if ssConfig.SnapshotInterval > 0 { + snapshotRoot := utils.GetStateStoreSnapshotsPath(homeDir) + if ssConfig.DBDirectory != "" { + cleanDBHome := filepath.Clean(dbHome) + snapshotRoot = filepath.Join( + filepath.Dir(cleanDBHome), + filepath.Base(cleanDBHome)+"-"+utils.StateStoreSnapshotsDirName, + ) + } + if err := cs.startSnapshotManager(snapshotRoot, snapshotSourceDirs); err != nil { + _ = cs.Close() + return nil, fmt.Errorf("start state store snapshot manager: %w", err) + } + } cs.StartPruning() return cs, nil @@ -150,20 +173,23 @@ func (s *CompositeStateStore) validateEVMSSPreRecovery() error { return nil } -// validateEVMSSPostRecovery rejects mismatched earliest versions between the two SS DBs. -func (s *CompositeStateStore) validateEVMSSPostRecovery() error { +// validateEVMSSPostRecovery reports mismatched earliest versions between SS DBs. +// Divergence is safe because GetEarliestVersion reports the highest member +// floor, which is the first version every routed store can serve. +func (s *CompositeStateStore) validateEVMSSPostRecovery() { if s.evmStore == nil { - return nil + return } cosmosEarliest := s.cosmosStore.GetEarliestVersion() evmEarliest := s.evmStore.GetEarliestVersion() if cosmosEarliest != evmEarliest && (cosmosEarliest > 0 || evmEarliest > 0) { - return fmt.Errorf( - "EVM SS earliest version %d does not match Cosmos SS earliest version %d: state sync the EVM SS DB, or set evm-ss-split=false", - evmEarliest, cosmosEarliest, + logger.Warn( + "EVM SS earliest version does not match Cosmos SS earliest version; serving the highest floor", + "evmEarliest", evmEarliest, + "cosmosEarliest", cosmosEarliest, + "reportedEarliest", max(cosmosEarliest, evmEarliest), ) } - return nil } func (s *CompositeStateStore) StartPruning() { @@ -216,11 +242,18 @@ func (s *CompositeStateStore) GetLatestVersion() int64 { } func (s *CompositeStateStore) GetEarliestVersion() int64 { - return s.cosmosStore.GetEarliestVersion() + earliest := s.cosmosStore.GetEarliestVersion() + if s.evmStore != nil { + earliest = max(earliest, s.evmStore.GetEarliestVersion()) + } + return earliest } func (s *CompositeStateStore) Close() error { s.closeOnce.Do(func() { + if s.snapshotMgr != nil { + s.snapshotMgr.stop() + } if s.pruningManager != nil { s.pruningManager.Stop() } @@ -306,6 +339,19 @@ func (s *CompositeStateStore) ApplyChangesetAsync(version int64, changesets []*p return nil } +// ScheduleSnapshot asks the snapshot manager to capture version once the caller +// has enqueued every state change for that version and nothing above it. +// +// This is deliberately not called from ApplyChangesetAsync. That method is part +// of the general StateStore interface and has callers outside the commit path, +// such as the benchmark wrappers, which would inherit a snapshot trigger they +// never asked for. The rootmulti commit path is the single choke point that +// sees both the populated and the empty block, so it owns the trigger. Direct +// writes such as import, recovery, and prune must not use this hook. +func (s *CompositeStateStore) ScheduleSnapshot(version int64) { + s.snapshotMgr.maybeSnapshot(version) +} + func filterEVMChangesets(changesets []*proto.NamedChangeSet) []*proto.NamedChangeSet { var evmCS []*proto.NamedChangeSet for _, cs := range changesets { diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 5b02d8ed15..c512335e49 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -4,6 +4,7 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" ) @@ -76,3 +77,24 @@ func (s *CosmosStateStore) Import(version int64, ch <-chan types.SnapshotNode) e func (s *CosmosStateStore) Close() error { return s.db.Close() } + +func (s *CosmosStateStore) SupportsCheckpoint() bool { + _, checkpointable := s.db.(types.Checkpointable) + _, barrier := s.db.(types.DrainBarrier) + _, versionSetter := s.db.(types.CheckpointVersionSetter) + return checkpointable && barrier && versionSetter +} + +func (s *CosmosStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { + management.ScheduleCheckpoint(s.db, destDir, shouldRun, done) +} + +func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) error { + return management.SetCheckpointVersion(s.db, destDir, version) +} + +func (s *CosmosStateStore) WaitForPendingWrites() { + if w, ok := s.db.(interface{ WaitForPendingWrites() }); ok { + w.WaitForPendingWrites() + } +} diff --git a/sei-db/state_db/ss/evm/db_test.go b/sei-db/state_db/ss/evm/db_test.go index 4b7682f302..69dc3ea2b6 100644 --- a/sei-db/state_db/ss/evm/db_test.go +++ b/sei-db/state_db/ss/evm/db_test.go @@ -31,6 +31,28 @@ func openTestStore(t *testing.T) types.StateStore { return store } +// GetEarliestVersion reports the highest sub-DB floor, which is the earliest +// version every routed sub-DB can serve. +func TestGetEarliestVersionReportsTheFurthestPrunedSubDB(t *testing.T) { + dir := t.TempDir() + cfg := testConfig() + cfg.SeparateEVMSubDBs = true + + store, err := NewEVMStateStore(dir, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + require.Greater(t, len(store.managedDBs), 1) + + require.Zero(t, store.GetEarliestVersion()) + + require.NoError(t, store.subDBs[StoreStorage].SetEarliestVersion(40, false)) + require.Equal(t, int64(40), store.GetEarliestVersion()) + + // Once the pass finishes, the reported floor stays the same. + require.NoError(t, store.SetEarliestVersion(40, false)) + require.Equal(t, int64(40), store.GetEarliestVersion()) +} + func TestEVMStateStoreDefaultUsesUnifiedDB(t *testing.T) { dir := t.TempDir() cfg := testConfig() diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 01337940f3..4c20aa6ceb 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -1,7 +1,9 @@ package evm import ( + "errors" "fmt" + "os" "path/filepath" "sync" @@ -10,6 +12,7 @@ import ( commonevm "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/backend" ) @@ -152,16 +155,13 @@ func (s *EVMStateStore) SetLatestVersion(version int64) error { } func (s *EVMStateStore) GetEarliestVersion() int64 { - var minVersion int64 = -1 + var maxVersion int64 for _, db := range s.managedDBs { - if v := db.GetEarliestVersion(); minVersion < 0 || v < minVersion { - minVersion = v + if v := db.GetEarliestVersion(); v > maxVersion { + maxVersion = v } } - if minVersion < 0 { - return 0 - } - return minVersion + return maxVersion } func (s *EVMStateStore) SetEarliestVersion(version int64, ignoreVersion bool) error { @@ -370,6 +370,100 @@ func (s *EVMStateStore) Close() error { return lastErr } +func (s *EVMStateStore) SupportsCheckpoint() bool { + for _, db := range s.managedDBs { + if _, checkpointable := db.(types.Checkpointable); !checkpointable { + return false + } + if _, barrier := db.(types.DrainBarrier); !barrier { + return false + } + if _, versionSetter := db.(types.CheckpointVersionSetter); !versionSetter { + return false + } + } + return len(s.managedDBs) > 0 +} + +// ScheduleCheckpoint places one barrier on each managed apply queue. +// +// Every sub-DB checkpoints at the same block without the sub-DBs having to agree +// on anything. The caller runs this after it has enqueued the target block on +// every sub-DB and before it enqueues any later block, so each barrier lands at +// the same point in its own queue: after that block and before the next one. A +// sub-DB then checkpoints its own state as of that block. Wall-clock times +// differ, and no lock is shared. A sub-DB that received no change at the target +// block stays at its last version, which is that sub-DB's correct state for the +// block. SetCheckpointVersion afterwards labels every sub-DB with the same +// block, so a reopened snapshot reports one version rather than five. +func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { + if !s.separateDBs { + db := s.primaryDB() + if db == nil { + // Unreachable: NewEVMStateStore either opens a managed DB or fails. + // Reporting success would publish a snapshot with no evm tree in it, + // which is only discovered by whoever tries to restore from it. + done(errors.New("EVM state store has no managed DB to checkpoint")) + return + } + management.ScheduleCheckpoint(db, destDir, shouldRun, done) + return + } + + if err := os.MkdirAll(destDir, 0o750); err != nil { + done(fmt.Errorf("create EVM checkpoint dir %q: %w", destDir, err)) + return + } + + storeTypes := AllEVMStoreTypes() + var ( + mu sync.Mutex + remaining = len(storeTypes) + firstErr error + ) + for _, storeType := range storeTypes { + name := StoreTypeName(storeType) + dest := filepath.Join(destDir, name) + management.ScheduleCheckpoint(s.subDBs[storeType], dest, shouldRun, func(err error) { + mu.Lock() + if err != nil && firstErr == nil { + firstErr = fmt.Errorf("checkpoint EVM sub-DB %s: %w", name, err) + } + remaining-- + last, outcome := remaining == 0, firstErr + mu.Unlock() + if last { + done(outcome) + } + }) + } +} + +func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) error { + if !s.separateDBs { + db := s.primaryDB() + if db == nil { + return errors.New("EVM state store has no managed DB to stamp") + } + return management.SetCheckpointVersion(db, destDir, version) + } + for _, storeType := range AllEVMStoreTypes() { + dest := filepath.Join(destDir, StoreTypeName(storeType)) + if err := management.SetCheckpointVersion(s.subDBs[storeType], dest, version); err != nil { + return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", StoreTypeName(storeType), err) + } + } + return nil +} + +func (s *EVMStateStore) WaitForPendingWrites() { + for _, db := range s.managedDBs { + if w, ok := db.(interface{ WaitForPendingWrites() }); ok { + w.WaitForPendingWrites() + } + } +} + func filterEVMChangesets(changesets []*proto.NamedChangeSet) []*proto.NamedChangeSet { filtered := make([]*proto.NamedChangeSet, 0, len(changesets)) for _, cs := range changesets { From e4be6a7710fa05666ce54dbaac2342837d9f24dd Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 14 Aug 2026 10:31:03 -0400 Subject: [PATCH 2/8] fix(seidb): enforce SS floors and preserve current snapshots Reject historical SS queries below the highest member floor before they can return partial routed state. Keep the snapshot referenced by current during retention so a failed link update cannot leave a dangling target. Also detach the snapshot design overview from exported const godoc and make recovery mismatch tests assert the tolerated highest-floor contract. Co-authored-by: Cursor --- sei-cosmos/storev2/rootmulti/store.go | 16 +++++++++++++ sei-cosmos/storev2/rootmulti/store_test.go | 15 ++++++++++++ sei-db/state_db/ss/composite/recovery_test.go | 21 ++++++++++++++++ sei-db/state_db/ss/composite/snapshot.go | 24 +++++++++++++++++++ sei-db/state_db/ss/composite/snapshot_test.go | 19 +++++++++++++++ sei-db/state_db/ss/composite/store.go | 24 +++++++++++++++++++ 6 files changed, 119 insertions(+) diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 2f04ff75f6..f7a23cd0a1 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -380,6 +380,9 @@ func (rs *Store) CacheMultiStoreWithVersion(version int64) (types.CacheMultiStor if version <= 0 { version = rs.ssStore.GetLatestVersion() } + if err := rs.validateSSReadVersion(version); err != nil { + return nil, err + } // add the transient/mem stores registered in current app. for k, store := range rs.ckvStores { if store.GetStoreType() != types.StoreTypeIAVL { @@ -398,6 +401,16 @@ func (rs *Store) CacheMultiStoreWithVersion(version int64) (types.CacheMultiStor return cachemulti.NewStore(nil, stores, rs.storeKeys, nil, nil, nil), nil } +// validateSSReadVersion rejects a historical query below the common SS floor +// before constructing stores that would otherwise return partial state. +func (rs *Store) validateSSReadVersion(version int64) error { + earliest := rs.ssStore.GetEarliestVersion() + if version < earliest { + return fmt.Errorf("state store version %d is below earliest available version %d", version, earliest) + } + return nil +} + func (rs *Store) CacheMultiStoreForExport(version int64) (types.CacheMultiStore, error) { if version <= 0 || (rs.lastCommitInfo != nil && version == rs.lastCommitInfo.Version) { return rs.CacheMultiStore(), nil @@ -882,6 +895,9 @@ func (rs *Store) Query(ctx context.Context, req abci.RequestQuery) abci.Response // Fast path: no proof + SS enabled if !needProof && rs.ssStore != nil { + if err := rs.validateSSReadVersion(version); err != nil { + return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidHeight, err.Error())) + } store := types.Queryable(state.NewStore(rs.ssStore, types.NewKVStoreKey(storeName), version)) return store.Query(ctx, req) } diff --git a/sei-cosmos/storev2/rootmulti/store_test.go b/sei-cosmos/storev2/rootmulti/store_test.go index 90c298eb97..e026add773 100644 --- a/sei-cosmos/storev2/rootmulti/store_test.go +++ b/sei-cosmos/storev2/rootmulti/store_test.go @@ -135,6 +135,21 @@ func TestSCSS_WriteAndHistoricalRead(t *testing.T) { }) require.EqualValues(t, 0, resp.Code) require.Equal(t, valV1, resp.Value) + + // Once SS reports a higher floor, historical SS queries below it must fail + // before a cache store can mix available Cosmos data with unavailable routed + // data. + require.NoError(t, store.ssStore.SetEarliestVersion(c2.Version, false)) + _, err = store.CacheMultiStoreWithVersion(c1.Version) + require.ErrorContains(t, err, "below earliest available version 2") + + resp = store.Query(context.Background(), abci.RequestQuery{ + Path: "/bank/key", + Data: keyBytes, + Height: c1.Version, + Prove: false, + }) + require.NotEqualValues(t, 0, resp.Code) } // flush owns the SS snapshot trigger for every block, so a boundary must be diff --git a/sei-db/state_db/ss/composite/recovery_test.go b/sei-db/state_db/ss/composite/recovery_test.go index 926e5d4ef7..7554e57599 100644 --- a/sei-db/state_db/ss/composite/recovery_test.go +++ b/sei-db/state_db/ss/composite/recovery_test.go @@ -85,15 +85,18 @@ func TestEVMSSPostRecoveryEarliestMismatch(t *testing.T) { evm := &fakeStateStore{latest: 100, earliest: 75} cs := newCompositeStateStoreWithStores(cosmos, evm, config.StateStoreConfig{EVMSplit: true}) cs.validateEVMSSPostRecovery() + require.Equal(t, int64(75), cs.GetEarliestVersion()) // Matching earliest → pass. evm.earliest = 50 cs.validateEVMSSPostRecovery() + require.Equal(t, int64(50), cs.GetEarliestVersion()) // Both zero → pass (fresh DBs). cosmos.earliest = 0 evm.earliest = 0 cs.validateEVMSSPostRecovery() + require.Zero(t, cs.GetEarliestVersion()) } func TestCompositeGetEarliestVersionReportsHighestMemberFloor(t *testing.T) { @@ -109,6 +112,24 @@ func TestCompositeGetEarliestVersionReportsHighestMemberFloor(t *testing.T) { require.Equal(t, int64(90), cs.GetEarliestVersion()) } +func TestCompositeReadsRejectVersionBelowHighestMemberFloor(t *testing.T) { + cosmos := &fakeStateStore{latest: 100, earliest: 50} + evmStore := &fakeStateStore{latest: 100, earliest: 75} + cs := newCompositeStateStoreWithStores(cosmos, evmStore, config.StateStoreConfig{EVMSplit: true}) + + _, err := cs.Get("bank", 74, []byte("key")) + require.ErrorContains(t, err, "below earliest available version 75") + + _, err = cs.Has(evm.EVMStoreKey, 74, []byte("key")) + require.ErrorContains(t, err, "below earliest available version 75") + + _, err = cs.Iterator("bank", 74, nil, nil) + require.ErrorContains(t, err, "below earliest available version 75") + + _, err = cs.ReverseIterator(evm.EVMStoreKey, 74, nil, nil) + require.ErrorContains(t, err, "below earliest available version 75") +} + // fakeStateStore stubs latest/earliest for validator tests. type fakeStateStore struct { types.StateStore diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index cd9be00f4a..f51e4712f5 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -96,6 +96,7 @@ import ( // link semantics should change with that work too: this implementation points to // the newest published snapshot, while FlatKV's current link points to the // active snapshot that open/rollback clones and replays from. + const ( // SnapshotsDirName is the directory under data/state_store that holds // online snapshots. @@ -633,11 +634,19 @@ func (m *snapshotManager) prune() { return } defer m.recordRetentionMetrics() + currentVersion, hasCurrent, err := m.currentSnapshotVersion() + if err != nil { + logger.Error("failed to resolve current state store snapshot before pruning", "error", err) + return + } keep := 1 + m.keepRecent if len(versions) <= keep { return } for _, v := range versions[:len(versions)-keep] { + if hasCurrent && v == currentVersion { + continue + } dir := filepath.Join(m.root, SnapshotDirName(v)) if err := os.RemoveAll(dir); err != nil { logger.Error("failed to prune state store snapshot", "dir", dir, "error", err) @@ -647,6 +656,21 @@ func (m *snapshotManager) prune() { } } +func (m *snapshotManager) currentSnapshotVersion() (version int64, exists bool, err error) { + target, err := os.Readlink(filepath.Join(m.root, snapshotCurrentLink)) + if err != nil { + if os.IsNotExist(err) { + return 0, false, nil + } + return 0, false, fmt.Errorf("read current snapshot link: %w", err) + } + version, ok := ParseSnapshotVersion(filepath.Base(target)) + if !ok { + return 0, false, fmt.Errorf("current snapshot link has invalid target %q", target) + } + return version, true, nil +} + func (m *snapshotManager) recordRetentionMetrics() { versions, err := ListSnapshotVersions(m.root) if err != nil { diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index 565a9fa45a..c0967ae875 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -774,6 +774,25 @@ func TestSnapshotPrune(t *testing.T) { require.Equal(t, SnapshotDirName(15), target) } +func TestSnapshotPruneKeepsCurrentTarget(t *testing.T) { + root := t.TempDir() + for _, version := range []int64{5, 10, 15, 20} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + require.NoError(t, os.Symlink(SnapshotDirName(5), filepath.Join(root, snapshotCurrentLink))) + + manager := &snapshotManager{root: root, keepRecent: 1} + manager.prune() + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5, 15, 20}, versions, + "retention may keep one extra snapshot but must not dangle current") + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(5), target) +} + func TestSnapshotManagerResumesFromNewestSnapshot(t *testing.T) { dir := t.TempDir() cfg := config.DefaultStateStoreConfig() diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index 70326b70da..ca5a09e4ef 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -205,7 +205,22 @@ func (s *CompositeStateStore) evmRouted(storeKey string) bool { return s.evmStore != nil && storeKey == evm.EVMStoreKey } +// validateReadVersion rejects a version that one or more routed stores can no +// longer serve. GetEarliestVersion reports the highest member floor, so applying +// it here prevents a query from combining valid data from one member with empty +// results from a further-pruned member. +func (s *CompositeStateStore) validateReadVersion(version int64) error { + earliest := s.GetEarliestVersion() + if version < earliest { + return fmt.Errorf("state store version %d is below earliest available version %d", version, earliest) + } + return nil +} + func (s *CompositeStateStore) Get(storeKey string, version int64, key []byte) ([]byte, error) { + if err := s.validateReadVersion(version); err != nil { + return nil, err + } if s.evmRouted(storeKey) { return s.evmStore.Get(storeKey, version, key) } @@ -213,6 +228,9 @@ func (s *CompositeStateStore) Get(storeKey string, version int64, key []byte) ([ } func (s *CompositeStateStore) Has(storeKey string, version int64, key []byte) (bool, error) { + if err := s.validateReadVersion(version); err != nil { + return false, err + } if s.evmRouted(storeKey) { return s.evmStore.Has(storeKey, version, key) } @@ -220,6 +238,9 @@ func (s *CompositeStateStore) Has(storeKey string, version int64, key []byte) (b } func (s *CompositeStateStore) Iterator(storeKey string, version int64, start, end []byte) (dbm.Iterator, error) { + if err := s.validateReadVersion(version); err != nil { + return nil, err + } if s.evmRouted(storeKey) { return s.evmStore.Iterator(storeKey, version, start, end) } @@ -227,6 +248,9 @@ func (s *CompositeStateStore) Iterator(storeKey string, version int64, start, en } func (s *CompositeStateStore) ReverseIterator(storeKey string, version int64, start, end []byte) (dbm.Iterator, error) { + if err := s.validateReadVersion(version); err != nil { + return nil, err + } if s.evmRouted(storeKey) { return s.evmStore.ReverseIterator(storeKey, version, start, end) } From 6f0bb142601ae6b84d4b979dc5cf787421b1328a Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 14 Aug 2026 11:30:18 -0400 Subject: [PATCH 3/8] fix(seidb): keep SS reads below the earliest floor non-fatal The composite read gate turned a prune that races an in-flight historical query into a process crash: the cosmos KVStore wrapper over a StateStore panics on any read error, and a query store keeps reading at its fixed version after the floor moved. Enforce the floor only where an error can be returned to the caller (query-store construction, Query, VersionExists) and let a pruned member report absence, which is what its engine already does. Co-authored-by: Cursor --- sei-db/state_db/ss/composite/recovery_test.go | 47 +++++++++++++++---- sei-db/state_db/ss/composite/store.go | 30 +++--------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/sei-db/state_db/ss/composite/recovery_test.go b/sei-db/state_db/ss/composite/recovery_test.go index 7554e57599..c6b0b14fe3 100644 --- a/sei-db/state_db/ss/composite/recovery_test.go +++ b/sei-db/state_db/ss/composite/recovery_test.go @@ -15,6 +15,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/wal" evmtypes "github.com/sei-protocol/sei-chain/x/evm/types" "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" ) func newCompositeStateStoreWithStores( @@ -112,33 +113,63 @@ func TestCompositeGetEarliestVersionReportsHighestMemberFloor(t *testing.T) { require.Equal(t, int64(90), cs.GetEarliestVersion()) } -func TestCompositeReadsRejectVersionBelowHighestMemberFloor(t *testing.T) { +// TestCompositeReadsBelowFloorDoNotError pins the read contract that keeps a +// prune racing an in-flight query from crashing the node: the cosmos KVStore +// wrapper panics on any read error, so a version below the reported floor must +// route through and report absence instead of returning an error. +func TestCompositeReadsBelowFloorDoNotError(t *testing.T) { cosmos := &fakeStateStore{latest: 100, earliest: 50} evmStore := &fakeStateStore{latest: 100, earliest: 75} cs := newCompositeStateStoreWithStores(cosmos, evmStore, config.StateStoreConfig{EVMSplit: true}) + require.Equal(t, int64(75), cs.GetEarliestVersion()) - _, err := cs.Get("bank", 74, []byte("key")) - require.ErrorContains(t, err, "below earliest available version 75") + value, err := cs.Get("bank", 74, []byte("key")) + require.NoError(t, err) + require.Nil(t, value) - _, err = cs.Has(evm.EVMStoreKey, 74, []byte("key")) - require.ErrorContains(t, err, "below earliest available version 75") + has, err := cs.Has(evm.EVMStoreKey, 74, []byte("key")) + require.NoError(t, err) + require.False(t, has) _, err = cs.Iterator("bank", 74, nil, nil) - require.ErrorContains(t, err, "below earliest available version 75") + require.NoError(t, err) _, err = cs.ReverseIterator(evm.EVMStoreKey, 74, nil, nil) - require.ErrorContains(t, err, "below earliest available version 75") + require.NoError(t, err) + + require.Equal(t, 4, cosmos.reads+evmStore.reads, "every read must reach its routed member") } -// fakeStateStore stubs latest/earliest for validator tests. +// fakeStateStore stubs latest/earliest and absent reads for validator tests. type fakeStateStore struct { types.StateStore latest, earliest int64 + reads int } func (f *fakeStateStore) GetLatestVersion() int64 { return f.latest } func (f *fakeStateStore) GetEarliestVersion() int64 { return f.earliest } +func (f *fakeStateStore) Get(string, int64, []byte) ([]byte, error) { + f.reads++ + return nil, nil +} + +func (f *fakeStateStore) Has(string, int64, []byte) (bool, error) { + f.reads++ + return false, nil +} + +func (f *fakeStateStore) Iterator(string, int64, []byte, []byte) (dbm.Iterator, error) { + f.reads++ + return nil, nil +} + +func (f *fakeStateStore) ReverseIterator(string, int64, []byte, []byte) (dbm.Iterator, error) { + f.reads++ + return nil, nil +} + func TestRecoverCompositeStateStore(t *testing.T) { dir, err := os.MkdirTemp("", "composite_recovery_test") require.NoError(t, err) diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index ca5a09e4ef..dbaf727921 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -205,22 +205,15 @@ func (s *CompositeStateStore) evmRouted(storeKey string) bool { return s.evmStore != nil && storeKey == evm.EVMStoreKey } -// validateReadVersion rejects a version that one or more routed stores can no -// longer serve. GetEarliestVersion reports the highest member floor, so applying -// it here prevents a query from combining valid data from one member with empty -// results from a further-pruned member. -func (s *CompositeStateStore) validateReadVersion(version int64) error { - earliest := s.GetEarliestVersion() - if version < earliest { - return fmt.Errorf("state store version %d is below earliest available version %d", version, earliest) - } - return nil -} +// The read methods below route by store key and do not re-check +// GetEarliestVersion. The cosmos KVStore wrapper over a StateStore panics on any +// read error, and pruning can raise the floor after a query store was built, so +// an error here would crash the process for a request that must merely fail. The +// floor is enforced where an error is representable: query-store construction +// and VersionExists. Below the floor a pruned member reports the key as absent, +// as its engine already does. func (s *CompositeStateStore) Get(storeKey string, version int64, key []byte) ([]byte, error) { - if err := s.validateReadVersion(version); err != nil { - return nil, err - } if s.evmRouted(storeKey) { return s.evmStore.Get(storeKey, version, key) } @@ -228,9 +221,6 @@ func (s *CompositeStateStore) Get(storeKey string, version int64, key []byte) ([ } func (s *CompositeStateStore) Has(storeKey string, version int64, key []byte) (bool, error) { - if err := s.validateReadVersion(version); err != nil { - return false, err - } if s.evmRouted(storeKey) { return s.evmStore.Has(storeKey, version, key) } @@ -238,9 +228,6 @@ func (s *CompositeStateStore) Has(storeKey string, version int64, key []byte) (b } func (s *CompositeStateStore) Iterator(storeKey string, version int64, start, end []byte) (dbm.Iterator, error) { - if err := s.validateReadVersion(version); err != nil { - return nil, err - } if s.evmRouted(storeKey) { return s.evmStore.Iterator(storeKey, version, start, end) } @@ -248,9 +235,6 @@ func (s *CompositeStateStore) Iterator(storeKey string, version int64, start, en } func (s *CompositeStateStore) ReverseIterator(storeKey string, version int64, start, end []byte) (dbm.Iterator, error) { - if err := s.validateReadVersion(version); err != nil { - return nil, err - } if s.evmRouted(storeKey) { return s.evmStore.ReverseIterator(storeKey, version, start, end) } From 317fd44fc714c59fb0ab975c3727d325df017a38 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 14 Aug 2026 14:46:20 -0400 Subject: [PATCH 4/8] refactor(seidb): move SS snapshots into each store Cosmos SS and EVM SS now own their snapshot roots, staging, current links, and retention, while the composite layer only coordinates cadence and best-effort joint publication. This prepares future GC wiring by adding ExternalPruning stand-down without implementing gc.PrunableStore before rollback can report an honest floor. Co-authored-by: Cursor --- app/config_fuzz_test.go | 3 + app/testdata/state-store.golden | 1 + .../config/testdata/server_config.golden | 1 + sei-db/config/ss_config.go | 17 +- sei-db/config/toml.go | 5 +- sei-db/state_db/ss/composite/snapshot.go | 780 ++++----------- sei-db/state_db/ss/composite/snapshot_test.go | 907 +++--------------- sei-db/state_db/ss/composite/store.go | 38 +- sei-db/state_db/ss/cosmos/store.go | 38 +- sei-db/state_db/ss/evm/store.go | 52 +- sei-db/state_db/ss/snapshot/manager.go | 561 +++++++++++ sei-db/state_db/ss/snapshot/manager_test.go | 141 +++ .../metrics.go} | 73 +- 13 files changed, 1194 insertions(+), 1423 deletions(-) create mode 100644 sei-db/state_db/ss/snapshot/manager.go create mode 100644 sei-db/state_db/ss/snapshot/manager_test.go rename sei-db/state_db/ss/{composite/snapshot_metrics.go => snapshot/metrics.go} (54%) diff --git a/app/config_fuzz_test.go b/app/config_fuzz_test.go index 1b8c7617ce..ec36038822 100644 --- a/app/config_fuzz_test.go +++ b/app/config_fuzz_test.go @@ -683,9 +683,12 @@ func TestManifestNamesEveryField(t *testing.T) { // point: they are tagged mapstructure:"-" so no key can bind them even in principle, // and AlignSSSnapshotWithSC derives all three at runtime from the state-commit cadence. // ss-snapshot-enable is the only SS-side knob, and it has a row of its own above. + // ExternalPruning is also runtime-only: future GC wiring sets it when it registers SS, + // not through app.toml. "SnapshotInterval", "SnapshotKeepRecent", "SnapshotMinTimeInterval", + "ExternalPruning", ) }) t.Run("light_invariance", func(t *testing.T) { diff --git a/app/testdata/state-store.golden b/app/testdata/state-store.golden index 57d01bd92f..953c105162 100644 --- a/app/testdata/state-store.golden +++ b/app/testdata/state-store.golden @@ -12,6 +12,7 @@ SnapshotEnable = bool(false) SnapshotInterval = int64(0) SnapshotKeepRecent = int(0) SnapshotMinTimeInterval = time.Duration(0s) +ExternalPruning = bool(false) EVMSplit = bool(false) EVMDBDirectory = string("") SeparateEVMSubDBs = bool(false) diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index 442b08b5d9..23f93cedba 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -144,6 +144,7 @@ StateStore.SnapshotEnable = bool(false) StateStore.SnapshotInterval = int64(0) StateStore.SnapshotKeepRecent = int(0) StateStore.SnapshotMinTimeInterval = time.Duration(0s) +StateStore.ExternalPruning = bool(false) StateStore.EVMSplit = bool(false) StateStore.EVMDBDirectory = string("") StateStore.SeparateEVMSubDBs = bool(false) diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index 567faf9138..f016780cfd 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -67,10 +67,9 @@ type StateStoreConfig struct { // SnapshotEnable controls whether the state store takes periodic online // snapshots. Snapshots are Pebble checkpoints (hardlink trees), so the - // backend must be pebbledb and every SS database must be able to hardlink - // into the snapshot root. Startup fails on either rather than running - // without snapshots. A custom Cosmos SS directory moves the snapshot root - // beside that directory, which keeps the link inside one filesystem. + // backend must be pebbledb and each SS database must be able to hardlink into + // its own snapshot root. Startup fails on either rather than running without + // snapshots. // // Taking a snapshot occupies each backend's SS apply goroutine for the WAL // flush, the filesystem sync, and the checkpoint. No data is copied up @@ -84,8 +83,8 @@ type StateStoreConfig struct { // rollback restore points, not an archive format. They have no lease in this // release, so node-external tools must not resolve a snapshot path and open it // later without first adding a hold mechanism. Attempts, skips, outcomes, - // duration, in-flight state, height, count, and apparent bytes are exported as - // ss_snapshot_* metrics. + // duration, in-flight state, per-store height, retained count, apparent bytes, + // and newest common height are exported as ss_snapshot_* metrics. // defaults to false SnapshotEnable bool `mapstructure:"snapshot-enable"` @@ -97,6 +96,12 @@ type StateStoreConfig struct { SnapshotKeepRecent int `mapstructure:"-"` SnapshotMinTimeInterval time.Duration `mapstructure:"-"` + // ExternalPruning hands SS history and snapshot retention to the + // StorageGarbageCollector. Internal pruning stands down when set. + // This field is set by the code wiring SS into a collector, not app.toml. + // defaults to false + ExternalPruning bool `mapstructure:"-"` + // --- EVM optimization fields --- // EVMSplit controls whether EVM data is routed to a dedicated SS backend. diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 6847fd3650..34e0d8b42b 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -143,9 +143,8 @@ ss-enable-read-write-metrics = {{ .StateStore.EnableReadWriteMetrics }} # SnapshotEnable turns on periodic online state-store snapshots. The cadence is # not configurable here: it mirrors the state-commit snapshot settings. # Two configurations fail startup rather than run without snapshots: an -# ss-backend other than "pebbledb", and SS databases that cannot hardlink into -# the snapshot root, which needs every SS database and that root on one -# filesystem. +# ss-backend other than "pebbledb", and any SS database that cannot hardlink into +# its own snapshot root. # Each retained snapshot pins the SST files it references against compaction, so # budget the write churn of one snapshot interval per retained snapshot. This is # substantial on a multi-TB state store. diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index f51e4712f5..1e65b670a0 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -1,62 +1,43 @@ package composite import ( - "context" "errors" "fmt" - "io/fs" - "os" - "path/filepath" - "slices" - "strconv" - "strings" "sync" + "sync/atomic" "time" - "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/management" + sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" ) -// Online state-store snapshots. Every SnapshotInterval blocks the store takes a -// Pebble checkpoint of each backend while the node keeps producing blocks. -// Checkpoints are hardlink trees, so they do not copy database contents, but -// each one occupies its backend's apply goroutine for the full checkpoint -// operation. Writes continue to enter the bounded queue, but a full queue -// applies backpressure until the checkpoint finishes. The result is an -// immutable, crash-consistent image of the query store. +// Online state-store snapshots. Every SnapshotInterval blocks the composite +// store asks each enabled SS member to stage a Pebble checkpoint while the node +// keeps producing blocks. Each member owns its own snapshot root, current link, +// retention, and metadata; composite only chooses the height and coordinates the +// best-effort joint publication. // -// These snapshots are an input to SS rollback, not an export format. The -// intended restore model matches SC FlatKV: restore from an SS snapshot, then -// replay the state WAL forward to the target height. State sync imports the SC -// snapshot stream and rebuilds SS from that stream; it does not consume these -// SS snapshot directories. +// These snapshots are an input to SS rollback, not an export format. The restore +// model matches SC FlatKV: restore from an SS snapshot, then replay the state WAL +// forward to the target height. State sync imports the SC snapshot stream and +// rebuilds SS from that stream; it does not consume these SS snapshot +// directories. // -// On-disk layout under the snapshot root. By default the root is -// /data/state_store/snapshots. A custom Cosmos SS directory moves it -// to the sibling -snapshots directory so Pebble can use hardlinks. +// For every accepted SS snapshot, the label is exact: it is the version the +// write path had just handed to the backends when the snapshot was requested. +// Placing a barrier in each backend's apply queue — rather than sampling what +// the backends had applied — makes that label exact without the request having +// to wait. The barrier orders only the async block-commit queues. Import, +// recovery, pruning, and direct version-marker writes bypass those queues and +// must not call ScheduleSnapshot. // -// snapshots/ -// current -> snapshot-NNNNN (symlink to newest snapshot) -// snapshot-NNNNN/ (immutable; NNNNN = label version) -// cosmos// (Pebble checkpoint of Cosmos SS) -// evm// (Pebble checkpoint of EVM SS, if split) -// / (when EVM sub-DBs are separate) -// -// Snapshots are eligible at the same interval boundaries and minimum time -// cadence as state commit. This composite implementation uses one trigger and -// one current link for all member stores, so its member snapshots share a label. -// That same label is a property of this layout, not a rollback requirement: -// rollback can replay the state WAL from each store's own nearest snapshot. For -// every accepted SS snapshot, the label is exact: it is the version the write -// path had just handed to the backends when the snapshot was requested. Placing -// a barrier in each backend's apply queue — rather than sampling what the -// backends had applied — makes that label exact without the request having to -// wait. See requestSnapshot. -// -// The barrier orders only the async block-commit queues. Import, recovery, -// pruning, and direct version-marker writes bypass those queues and must not -// call ScheduleSnapshot. The rootmulti commit path owns the trigger for every -// block, populated or empty, and is the only caller of ScheduleSnapshot. +// Cross-member pairing is best-effort, not an invariant. Composite stages every +// member and commits only after all members stage successfully, so the normal +// path publishes the same height everywhere. Once members have separate roots, +// however, publication is multiple renames and cannot be atomic across +// directories. Startup therefore does not delete an unpaired height; rollback +// must pick the newest snapshot height present in every required member. In an +// EVM-only process that rule degenerates to the newest EVM snapshot. // // Pruning is the one writer nothing orders a snapshot against. A checkpoint can // capture a partially applied prune — the same state a crash mid-prune leaves on @@ -65,112 +46,41 @@ import ( // range the DB has already dropped. Reopening a snapshot with different member // floors is allowed; the composite reports the highest floor any member carries. // -// SS rollback is not implemented in this feature. When it is added, it should -// use these snapshots the same way SC FlatKV does: restore from a snapshot -// boundary, then replay the state WAL forward. Until then, rolling back or -// state-syncing to a lower height in a reused home directory leaves two stale -// facts behind: lastRequested still carries the old high-water mark, so repeated -// boundaries can be skipped, and already published snapshot-NNNNN directories -// keep labels from the abandoned chain. Clear the snapshot root by hand in that -// case. -// // Managed snapshot directories have no lease because they are not a node-external // consumption API. Retention may remove any snapshot that rollback does not need. // If a future tool opens or copies these directories directly, it must first add // a lease or other hold mechanism. -// -// This file is the layer the planned per-SS restructure has to move. The -// lifecycle here — layout, retention, the current symlink, staging and -// publication, restart recovery — is reachable only as a method on -// *CompositeStateStore, and startSnapshotManager requires a checkpointable -// Cosmos store, so an EVM-only store cannot use it as written. The agreed -// direction is for each SS to own its own snapshot root, current link, creation, -// and retention behind gc.PrunableStore, mirroring SC FlatKV. Composite mode can -// then fan out to Cosmos SS and EVM SS, while Giga can use EVM SS directly. That -// also removes the second retention path this file adds: prune here is -// count-based and has no ExternalPruning stand-down, so pointing -// StorageGarbageCollector at SS before then would give a store two independent -// pruners. The shape waits on rollback not because GetRollbackFloor is unknown; -// SC FlatKV already defines that floor. It waits because gc.PrunableStore must -// not report a floor above what the store can actually restore to. The current -// link semantics should change with that work too: this implementation points to -// the newest published snapshot, while FlatKV's current link points to the -// active snapshot that open/rollback clones and replays from. const ( - // SnapshotsDirName is the directory under data/state_store that holds - // online snapshots. - SnapshotsDirName = utils.StateStoreSnapshotsDirName - - snapshotPrefix = "snapshot-" - // snapshotDirLen is "snapshot-" + 20-digit zero-padded version. - snapshotDirLen = len(snapshotPrefix) + 20 - - snapshotCurrentLink = "current" - snapshotCurrentTmpLink = "current-tmp" - snapshotTmpPrefix = "tmp-" - snapshotSizeFile = ".apparent-size" + SnapshotsDirName = sssnapshot.SnapshotsDirName + snapshotCurrentLink = "current" + snapshotTmpPrefix = "tmp-" ) -// SnapshotDirName returns the directory name for a snapshot labeled with the -// given version. func SnapshotDirName(version int64) string { - return fmt.Sprintf("%s%020d", snapshotPrefix, version) + return sssnapshot.SnapshotDirName(version) } -// ParseSnapshotVersion parses a snapshot directory name; ok is false for -// anything that is not a snapshot-<20 digits> name. -func ParseSnapshotVersion(name string) (version int64, ok bool) { - if !strings.HasPrefix(name, snapshotPrefix) || len(name) != snapshotDirLen { - return 0, false - } - v, err := strconv.ParseInt(name[len(snapshotPrefix):], 10, 64) - if err != nil || v < 0 { - return 0, false - } - return v, true -} - -// ListSnapshotVersions returns the labels of all snapshots under root in -// ascending order. A missing root is not an error (no snapshots yet). func ListSnapshotVersions(root string) ([]int64, error) { - entries, err := os.ReadDir(root) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, fmt.Errorf("read snapshots dir %q: %w", root, err) - } - var versions []int64 - for _, entry := range entries { - if !entry.IsDir() { - continue - } - if v, ok := ParseSnapshotVersion(entry.Name()); ok { - versions = append(versions, v) - } - } - slices.Sort(versions) - return versions, nil + return sssnapshot.ListSnapshotVersions(root) } -// snapshotManager owns the snapshots directory and the one-at-a-time discipline -// for filling it. It has no goroutine of its own: snapshots are requested from -// the write path and completed on the backends' apply goroutines. -type snapshotManager struct { - root string - backend string - interval int64 - keepRecent int - minTime time.Duration +type snapshotMember struct { + name string + manager *sssnapshot.Manager +} - cosmosScheduler management.CheckpointScheduler - evmScheduler management.CheckpointScheduler - snapshotSizes map[int64]int64 +// snapshotCoordinator owns the one-at-a-time cadence for composite mode. It has +// no snapshot root of its own. +type snapshotCoordinator struct { + interval int64 + minTime time.Duration + members []snapshotMember mu sync.Mutex - // lastRequested is the newest label already requested or on disk, so a - // boundary is not snapshotted twice across a restart or a re-sent version. + // lastRequested is the newest label already requested or present in any + // member, so an unpaired height does not get retried under an exact label the + // write path has already moved past. lastRequested int64 lastRequestAt time.Time inFlight bool @@ -180,239 +90,155 @@ type snapshotManager struct { scheduling sync.WaitGroup // publishing tracks the goroutine finishing the accepted snapshot off. publishing sync.WaitGroup - - // publishMu serializes the publish step, which reads and rewrites the - // shared directory (the current link, and pruning). - publishMu sync.Mutex - lastPublished int64 } -type checkpointTarget struct { - store management.CheckpointScheduler - dest string +func newSnapshotCoordinator(interval int64, minTime time.Duration, members []snapshotMember) *snapshotCoordinator { + c := &snapshotCoordinator{ + interval: interval, + minTime: minTime, + members: members, + } + c.lastRequested = newestMemberSnapshot(members) + c.lastRequestAt = newestMemberSnapshotModTime(members, c.lastRequested) + sssnapshot.RecordCommonHeight(newestCommonSnapshot(members)) + return c } -// startSnapshotManager wires the manager into the composite store. Snapshot -// enablement is fail-closed: every backend must support checkpoints, and every -// live DB must be able to hardlink into root. Pebble otherwise silently falls -// back to copying SSTs across filesystems while its apply worker is blocked. -func (s *CompositeStateStore) startSnapshotManager(root string, sourceDirs []string) error { +func (s *CompositeStateStore) startSnapshotManager(members []snapshotMember) error { if s.config.SnapshotInterval <= 0 { return nil } - cosmosScheduler, ok := s.cosmosStore.(management.CheckpointScheduler) - if !ok || !cosmosScheduler.SupportsCheckpoint() { - return fmt.Errorf("cosmos backend %q does not support checkpoints", s.config.Backend) - } - var evmScheduler management.CheckpointScheduler - if s.evmStore != nil { - evmScheduler, ok = s.evmStore.(management.CheckpointScheduler) - if !ok || !evmScheduler.SupportsCheckpoint() { - return fmt.Errorf("EVM backend %q does not support checkpoints", s.config.Backend) - } - } - if err := verifySnapshotHardlinks(root, sourceDirs); err != nil { - return err - } - m := &snapshotManager{ - root: root, - backend: s.config.Backend, - interval: s.config.SnapshotInterval, - keepRecent: s.config.SnapshotKeepRecent, - minTime: s.config.SnapshotMinTimeInterval, - cosmosScheduler: cosmosScheduler, - evmScheduler: evmScheduler, - snapshotSizes: map[int64]int64{}, + if len(members) == 0 { + return errors.New("no state store snapshot members") } - m.lastRequested = m.newestSnapshotVersion() - m.lastPublished = m.lastRequested - m.lastRequestAt = m.snapshotModTime(m.lastRequested) - m.removeStaleTmpDirs() - m.prune() - if m.lastPublished > 0 { - if err := m.updateCurrentLink(SnapshotDirName(m.lastPublished)); err != nil { - logger.Error("failed to restore state store snapshot current link", - "version", m.lastPublished, "error", err) + for _, member := range members { + if member.manager == nil { + return fmt.Errorf("state store snapshot member %q has no manager", member.name) } - snapshotMetrics.CurrentHeight.Record(context.Background(), m.lastPublished) } - s.snapshotMgr = m + s.snapshotMgr = newSnapshotCoordinator( + s.config.SnapshotInterval, + s.config.SnapshotMinTimeInterval, + members, + ) logger.Info("state store snapshotting enabled", - "root", root, - "interval", m.interval, - "minTimeInterval", m.minTime, - "keepRecent", m.keepRecent, + "interval", s.config.SnapshotInterval, + "minTimeInterval", s.config.SnapshotMinTimeInterval, + "keepRecent", s.config.SnapshotKeepRecent, + "members", len(members), ) return nil } -func verifySnapshotHardlinks(root string, sourceDirs []string) error { - if err := os.MkdirAll(root, 0o750); err != nil { - return fmt.Errorf("create snapshot root %q: %w", root, err) - } - for _, sourceDir := range sourceDirs { - probe, err := os.CreateTemp(sourceDir, ".ss-snapshot-link-probe-*") - if err != nil { - return fmt.Errorf("create hardlink probe in state store %q: %w", sourceDir, err) - } - source := probe.Name() - if err := probe.Close(); err != nil { - _ = os.Remove(source) - return fmt.Errorf("close hardlink probe in state store %q: %w", sourceDir, err) - } - target := filepath.Join(root, filepath.Base(source)) - if err := os.Link(source, target); err != nil { - _ = os.Remove(source) - return fmt.Errorf( - "state store %q cannot hardlink snapshots into %q; place all SS databases and the snapshot root on one filesystem: %w", - sourceDir, - root, - err, - ) - } - if err := os.Remove(source); err != nil { - _ = os.Remove(target) - return fmt.Errorf("remove hardlink probe %q: %w", source, err) - } - if err := os.Remove(target); err != nil { - return fmt.Errorf("remove hardlink probe %q: %w", target, err) - } - } - return nil -} - // stop prevents further snapshots, waits for accepted requests to enqueue their // barriers, and then waits for active publication. Queued barriers are canceled // before they start when backend close drains their queues. -func (m *snapshotManager) stop() { - m.mu.Lock() - m.stopped = true - m.mu.Unlock() - m.scheduling.Wait() - m.publishing.Wait() +func (c *snapshotCoordinator) stop() { + c.mu.Lock() + c.stopped = true + c.mu.Unlock() + c.scheduling.Wait() + c.publishing.Wait() } -func (m *snapshotManager) isRunning() bool { - m.mu.Lock() - defer m.mu.Unlock() - return !m.stopped +func (c *snapshotCoordinator) isRunning() bool { + c.mu.Lock() + defer c.mu.Unlock() + return !c.stopped } // maybeSnapshot takes a snapshot when version lands on an interval boundary. // It is called from the write path for every version, so the common case is the // modulo test and nothing else. -func (m *snapshotManager) maybeSnapshot(version int64) { - if m == nil || version <= 0 || m.interval <= 0 || version%m.interval != 0 { +func (c *snapshotCoordinator) maybeSnapshot(version int64) { + if c == nil || version <= 0 || c.interval <= 0 || version%c.interval != 0 { return } now := time.Now() - m.mu.Lock() - previous := m.lastRequested - previousRequestAt := m.lastRequestAt + c.mu.Lock() + previous := c.lastRequested + previousRequestAt := c.lastRequestAt var skipReason string accepted := false switch { - case m.stopped || version <= m.lastRequested: + case c.stopped || version <= c.lastRequested: // A repeated commit-path call is expected and is not a skipped attempt. - case m.inFlight: + case c.inFlight: skipReason = "in_flight" - case !m.lastRequestAt.IsZero() && now.Sub(m.lastRequestAt) < m.minTime: + case !c.lastRequestAt.IsZero() && now.Sub(c.lastRequestAt) < c.minTime: skipReason = "minimum_time_interval" default: - m.lastRequested = version - m.lastRequestAt = now - m.inFlight = true - m.scheduling.Add(1) - recordSnapshotInFlight(1) + c.lastRequested = version + c.lastRequestAt = now + c.inFlight = true + c.scheduling.Add(1) + sssnapshot.RecordInFlight(1) accepted = true } - m.mu.Unlock() + c.mu.Unlock() if !accepted { if skipReason != "" { - recordSnapshotSkipped(skipReason) + sssnapshot.RecordSkipped(skipReason) // A skipped boundary is the reason a snapshot an operator expected is // not on disk, so name the gate rather than leaving only a metric. logger.Info("skipping state store snapshot", "version", version, "reason", skipReason) } return } - defer m.scheduling.Done() + defer c.scheduling.Done() start := time.Now() - recordSnapshotAttempt() - if err := m.requestSnapshot(version, start); err != nil { - recordSnapshotCompletion(start, "failure") - m.mu.Lock() - if m.lastRequested == version { - m.lastRequested = previous - m.lastRequestAt = previousRequestAt - m.inFlight = false - recordSnapshotInFlight(0) + sssnapshot.RecordAttempt() + if err := c.requestSnapshot(version, start); err != nil { + sssnapshot.RecordCompletion(start, "failure") + c.mu.Lock() + if c.lastRequested == version { + c.lastRequested = previous + c.lastRequestAt = previousRequestAt + c.inFlight = false + sssnapshot.RecordInFlight(0) } - m.mu.Unlock() + c.mu.Unlock() logger.Error("state store snapshot failed", "version", version, "error", err) } } -func (m *snapshotManager) finishSnapshot() { - m.mu.Lock() - m.inFlight = false - recordSnapshotInFlight(0) - m.mu.Unlock() +func (c *snapshotCoordinator) finishSnapshot() { + c.mu.Lock() + c.inFlight = false + sssnapshot.RecordInFlight(0) + c.mu.Unlock() + sssnapshot.RecordCommonHeight(newestCommonSnapshot(c.members)) } -// requestSnapshot asks every backend to checkpoint itself into a staging +// requestSnapshot asks every member to checkpoint itself into a staging // directory and publishes the result once they all have. -// -// The label is exact because of when this runs: the caller has just enqueued -// version on the backends and has not enqueued anything above it, so a barrier -// placed in each apply queue now captures that backend with everything up to -// version applied and nothing after it. The backends reach their barriers -// independently and at different wall-clock times, and the caller waits for none -// of the checkpointing — enqueueing a barrier costs what enqueueing a changeset -// costs. The caller does wait for the staging directories below: one Stat, one -// RemoveAll and one MkdirAll per target, on the commit path and ahead of the SC -// apply. -func (m *snapshotManager) requestSnapshot(version int64, start time.Time) error { - name := SnapshotDirName(version) - finalDir := filepath.Join(m.root, name) - if _, err := os.Stat(finalDir); err == nil { - return fmt.Errorf("snapshot dir %q already exists", finalDir) - } else if !os.IsNotExist(err) { - return fmt.Errorf("inspect snapshot dir %q: %w", finalDir, err) +func (c *snapshotCoordinator) requestSnapshot(version int64, start time.Time) error { + if len(c.members) == 0 { + return errors.New("no state store snapshot members") } - tmpDir := filepath.Join(m.root, snapshotTmpPrefix+name) - if err := os.RemoveAll(tmpDir); err != nil { - return fmt.Errorf("clear stale snapshot tmp dir: %w", err) - } - - targets := []checkpointTarget{ - {m.cosmosScheduler, filepath.Join(tmpDir, "cosmos", m.backend)}, - } - if m.evmScheduler != nil { - targets = append(targets, checkpointTarget{ - m.evmScheduler, - filepath.Join(tmpDir, "evm", m.backend), - }) - } - for _, target := range targets { - if err := os.MkdirAll(filepath.Dir(target.dest), 0o750); err != nil { - _ = os.RemoveAll(tmpDir) - return fmt.Errorf("create snapshot dir: %w", err) - } + var canceled atomic.Bool + shouldRun := func() bool { + return c.isRunning() && !canceled.Load() } var ( mu sync.Mutex - remaining = len(targets) + remaining = len(c.members) + staged = make([]*sssnapshot.Staged, len(c.members)) firstErr error ) - // Set up before scheduling because callbacks can complete while the loop is - // still scheduling the remaining targets. - for _, target := range targets { - target.store.ScheduleCheckpoint(target.dest, m.isRunning, func(err error) { + for i, member := range c.members { + if member.manager == nil { + canceled.Store(true) + return fmt.Errorf("state store snapshot member %q has no manager", member.name) + } + err := member.manager.Stage(version, shouldRun, func(s *sssnapshot.Staged, err error) { mu.Lock() if err != nil && firstErr == nil { - firstErr = err + firstErr = fmt.Errorf("stage %s snapshot: %w", member.name, err) + } + if s != nil { + staged[i] = s } remaining-- last, outcome := remaining == 0, firstErr @@ -420,8 +246,12 @@ func (m *snapshotManager) requestSnapshot(version int64, start time.Time) error if !last { return } - m.startPublish(version, tmpDir, finalDir, targets, outcome, start) + c.startPublish(version, staged, outcome, start) }) + if err != nil { + canceled.Store(true) + return err + } } return nil } @@ -430,332 +260,108 @@ func (m *snapshotManager) requestSnapshot(version int64, start time.Time) error // on whichever backend's apply goroutine finished last, so it must not do the // work itself: publishing renames directories and prunes old snapshots, and a // writer stalled on that is a writer not applying blocks. -func (m *snapshotManager) startPublish( +func (c *snapshotCoordinator) startPublish( version int64, - tmpDir, finalDir string, - targets []checkpointTarget, + staged []*sssnapshot.Staged, checkpointErr error, start time.Time, ) { // Taken under the same lock stop uses, so no goroutine is registered after // stop has started waiting. - m.mu.Lock() - if m.stopped { - m.mu.Unlock() - _ = os.RemoveAll(tmpDir) - recordSnapshotCompletion(start, "canceled") - m.finishSnapshot() + c.mu.Lock() + if c.stopped { + c.mu.Unlock() + abortStaged(staged) + sssnapshot.RecordCompletion(start, "canceled") + c.finishSnapshot() return } - m.publishing.Add(1) - m.mu.Unlock() + c.publishing.Add(1) + c.mu.Unlock() go func() { - defer m.publishing.Done() - defer m.finishSnapshot() + defer c.publishing.Done() + defer c.finishSnapshot() if checkpointErr != nil { if errors.Is(checkpointErr, management.ErrCheckpointCanceled) { - recordSnapshotCompletion(start, "canceled") + sssnapshot.RecordCompletion(start, "canceled") } else { - recordSnapshotCompletion(start, "failure") + sssnapshot.RecordCompletion(start, "failure") logger.Error("state store snapshot failed", "version", version, "error", checkpointErr) } - _ = os.RemoveAll(tmpDir) + abortStaged(staged) return } - for _, target := range targets { - if err := target.store.SetCheckpointVersion(target.dest, version); err != nil { - recordSnapshotCompletion(start, "failure") - logger.Error("failed to set state store snapshot version", - "version", version, "dir", target.dest, "error", err) - _ = os.RemoveAll(tmpDir) + for i, member := range c.members { + if err := member.manager.Commit(staged[i]); err != nil { + sssnapshot.RecordCompletion(start, "failure") + logger.Error("failed to publish state store member snapshot", + "version", version, "member", member.name, "error", err) + abortStaged(staged[i+1:]) return } } - if m.publish(version, tmpDir, finalDir, start) { - recordSnapshotCompletion(start, "success") - } else { - recordSnapshotCompletion(start, "failure") - } + sssnapshot.RecordCompletion(start, "success") }() } -// publish moves a finished checkpoint into place and reports whether the whole -// publication succeeded. Retention runs either way. -// -// A boundary that fails anywhere past the barrier is given up on, and this is -// deliberate. maybeSnapshot restores lastRequested when requestSnapshot fails, -// because that failure happens before any barrier is enqueued and the boundary -// was never claimed. Once the barriers are out, the version they captured is -// the only image of that boundary there will ever be: the write path has moved -// on, so re-running the attempt would checkpoint a later state under the older -// label, which is the one thing the label is supposed to rule out. Recovery is -// therefore the next boundary rather than a retry of this one, at the cost of -// one snapshot interval of coverage. The error log and the outcome="failure" -// counter are the signal. -func (m *snapshotManager) publish(version int64, tmpDir, finalDir string, start time.Time) bool { - apparentBytes, sizeErr := snapshotDirApparentBytes(tmpDir) - if sizeErr != nil { - logger.Error("failed to measure state store snapshot", "dir", tmpDir, "error", sizeErr) - } else if err := writeSnapshotSize(tmpDir, apparentBytes); err != nil { - logger.Error("failed to persist state store snapshot size", "dir", tmpDir, "error", err) - sizeErr = err - } - - m.publishMu.Lock() - defer m.publishMu.Unlock() - defer m.prune() - - if err := os.Rename(tmpDir, finalDir); err != nil { - logger.Error("failed to finalize state store snapshot", "version", version, "error", err) - _ = os.RemoveAll(tmpDir) - return false - } - if err := syncDir(m.root); err != nil { - logger.Error("failed to persist state store snapshot publication", - "version", version, "dir", finalDir, "error", err) - return false - } - if sizeErr == nil { - if m.snapshotSizes == nil { - m.snapshotSizes = map[int64]int64{} - } - m.snapshotSizes[version] = apparentBytes - } - logger.Info("state store snapshot created", - "version", version, "dir", finalDir, "took", time.Since(start).String()) - - // Snapshots can finish out of order, so only move the link forward. - if version > m.lastPublished { - if err := m.updateCurrentLink(SnapshotDirName(version)); err != nil { - // The snapshot itself is intact and discoverable by name; only the - // convenience symlink is stale. The link is part of the publication - // contract, so record this attempt as a failure. - logger.Error("failed to update state store snapshot current link", - "version", version, "error", err) - return false +func abortStaged(staged []*sssnapshot.Staged) { + for _, s := range staged { + if s != nil { + s.Abort() } - m.lastPublished = version - } - snapshotMetrics.CurrentHeight.Record(context.Background(), m.lastPublished) - return true -} - -func (m *snapshotManager) newestSnapshotVersion() int64 { - versions, err := ListSnapshotVersions(m.root) - if err != nil { - logger.Error("failed to list state store snapshots", "error", err) - return 0 - } - if len(versions) == 0 { - return 0 - } - return versions[len(versions)-1] -} - -func (m *snapshotManager) snapshotModTime(version int64) time.Time { - if version <= 0 { - return time.Time{} } - info, err := os.Stat(filepath.Join(m.root, SnapshotDirName(version))) - if err != nil { - logger.Error("failed to read state store snapshot modification time", - "version", version, "error", err) - return time.Time{} - } - return info.ModTime() } -// removeStaleTmpDirs clears staging directories left behind by a crash or a -// shutdown that landed mid-snapshot. They are named after the snapshot they -// were staging, so they would otherwise sit there until that exact boundary -// came round again. -func (m *snapshotManager) removeStaleTmpDirs() { - tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) - if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) { - logger.Error("failed to remove stale state store snapshot link", "path", tmpLink, "error", err) - } - - entries, err := os.ReadDir(m.root) - if err != nil { - if !os.IsNotExist(err) { - logger.Error("failed to scan state store snapshots dir", "error", err) - } - return - } - for _, entry := range entries { - if !entry.IsDir() || !strings.HasPrefix(entry.Name(), snapshotTmpPrefix) { +func newestMemberSnapshot(members []snapshotMember) int64 { + var newest int64 + for _, member := range members { + if member.manager == nil { continue } - dir := filepath.Join(m.root, entry.Name()) - if err := os.RemoveAll(dir); err != nil { - logger.Error("failed to remove stale snapshot tmp dir", "dir", dir, "error", err) - continue - } - logger.Info("removed stale state store snapshot tmp dir", "dir", dir) - } -} - -// updateCurrentLink atomically points the current symlink at name. -func (m *snapshotManager) updateCurrentLink(name string) error { - tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) - _ = os.Remove(tmpLink) - if err := os.Symlink(name, tmpLink); err != nil { - return fmt.Errorf("create snapshot current symlink: %w", err) - } - if err := os.Rename(tmpLink, filepath.Join(m.root, snapshotCurrentLink)); err != nil { - return fmt.Errorf("swap snapshot current symlink: %w", err) - } - return syncDir(m.root) -} - -func syncDir(path string) error { - // #nosec G304 -- path is an internal database or snapshot directory, not request input. - dir, err := os.Open(path) - if err != nil { - return fmt.Errorf("open directory %q for sync: %w", path, err) - } - syncErr := dir.Sync() - closeErr := dir.Close() - if syncErr != nil { - syncErr = fmt.Errorf("sync directory %q: %w", path, syncErr) - } - if closeErr != nil { - closeErr = fmt.Errorf("close directory %q after sync: %w", path, closeErr) + newest = max(newest, member.manager.Newest()) } - return errors.Join(syncErr, closeErr) + return newest } -// prune removes all but the newest 1+keepRecent snapshots. -func (m *snapshotManager) prune() { - versions, err := ListSnapshotVersions(m.root) - if err != nil { - logger.Error("failed to list state store snapshots for pruning", "error", err) - return - } - defer m.recordRetentionMetrics() - currentVersion, hasCurrent, err := m.currentSnapshotVersion() - if err != nil { - logger.Error("failed to resolve current state store snapshot before pruning", "error", err) - return - } - keep := 1 + m.keepRecent - if len(versions) <= keep { - return - } - for _, v := range versions[:len(versions)-keep] { - if hasCurrent && v == currentVersion { - continue - } - dir := filepath.Join(m.root, SnapshotDirName(v)) - if err := os.RemoveAll(dir); err != nil { - logger.Error("failed to prune state store snapshot", "dir", dir, "error", err) +func newestMemberSnapshotModTime(members []snapshotMember, version int64) time.Time { + var newest time.Time + for _, member := range members { + if member.manager == nil { continue } - logger.Info("pruned state store snapshot", "dir", dir) - } -} - -func (m *snapshotManager) currentSnapshotVersion() (version int64, exists bool, err error) { - target, err := os.Readlink(filepath.Join(m.root, snapshotCurrentLink)) - if err != nil { - if os.IsNotExist(err) { - return 0, false, nil + modTime := member.manager.ModTime(version) + if modTime.After(newest) { + newest = modTime } - return 0, false, fmt.Errorf("read current snapshot link: %w", err) } - version, ok := ParseSnapshotVersion(filepath.Base(target)) - if !ok { - return 0, false, fmt.Errorf("current snapshot link has invalid target %q", target) - } - return version, true, nil + return newest } -func (m *snapshotManager) recordRetentionMetrics() { - versions, err := ListSnapshotVersions(m.root) - if err != nil { - logger.Error("failed to list state store snapshots for metrics", "error", err) - return - } - snapshotMetrics.RetainedCount.Record(context.Background(), int64(len(versions))) - - if m.snapshotSizes == nil { - m.snapshotSizes = map[int64]int64{} +func newestCommonSnapshot(members []snapshotMember) int64 { + if len(members) == 0 { + return 0 } - retained := make(map[int64]struct{}, len(versions)) - var apparentBytes int64 - for _, version := range versions { - retained[version] = struct{}{} - if size, ok := m.snapshotSizes[version]; ok { - apparentBytes += size - continue + counts := map[int64]int{} + for _, member := range members { + if member.manager == nil { + return 0 } - dir := filepath.Join(m.root, SnapshotDirName(version)) - size, err := readSnapshotSize(dir) + versions, err := member.manager.Versions() if err != nil { - if !os.IsNotExist(err) { - logger.Error("failed to read state store snapshot size", "dir", dir, "error", err) - } - continue + logger.Error("failed to list state store snapshots for common height", + "member", member.name, "error", err) + return 0 } - m.snapshotSizes[version] = size - apparentBytes += size - } - for version := range m.snapshotSizes { - if _, ok := retained[version]; !ok { - delete(m.snapshotSizes, version) + for _, version := range versions { + counts[version]++ } } - snapshotMetrics.ApparentBytes.Record(context.Background(), apparentBytes) -} - -func snapshotDirApparentBytes(dir string) (int64, error) { - var apparentBytes int64 - err := filepath.WalkDir(dir, func(_ string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - if !entry.Type().IsRegular() { - return nil - } - info, err := entry.Info() - if err != nil { - return err + var newest int64 + for version, count := range counts { + if count == len(members) && version > newest { + newest = version } - apparentBytes += info.Size() - return nil - }) - return apparentBytes, err -} - -func writeSnapshotSize(dir string, size int64) error { - path := filepath.Join(dir, snapshotSizeFile) - // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) - if err != nil { - return err - } - _, writeErr := fmt.Fprintf(file, "%d\n", size) - syncErr := file.Sync() - closeErr := file.Close() - if err := errors.Join(writeErr, syncErr, closeErr); err != nil { - return err - } - return syncDir(dir) -} - -func readSnapshotSize(dir string) (int64, error) { - // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. - data, err := os.ReadFile(filepath.Join(dir, snapshotSizeFile)) - if err != nil { - return 0, err - } - size, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) - if err != nil { - return 0, fmt.Errorf("parse snapshot size in %q: %w", dir, err) - } - if size < 0 { - return 0, fmt.Errorf("snapshot size in %q must be non-negative", dir) } - return size, nil + return newest } diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index c0967ae875..19746b2ad5 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -1,40 +1,23 @@ package composite import ( + "errors" "os" "path/filepath" "testing" "time" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/cosmos" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" + sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" "github.com/stretchr/testify/require" ) -type noCheckpointStateStore struct { - types.StateStore -} - -type noBarrierStateStore struct { - types.StateStore -} - -func (*noBarrierStateStore) Checkpoint(string) error { - return nil -} - -func (*noBarrierStateStore) SetCheckpointVersion(string, int64) error { - return nil -} - type controlledSnapshotScheduler struct { - pending chan func() - entered chan struct{} - checkpointCalls int + pending chan func() + fail bool } func (*controlledSnapshotScheduler) SupportsCheckpoint() bool { @@ -46,15 +29,15 @@ func (s *controlledSnapshotScheduler) ScheduleCheckpoint( shouldRun func() bool, done func(error), ) { - if s.entered != nil { - close(s.entered) - } s.pending <- func() { if !shouldRun() { done(management.ErrCheckpointCanceled) return } - s.checkpointCalls++ + if s.fail { + done(errors.New("checkpoint failed")) + return + } _ = os.MkdirAll(destDir, 0o750) done(nil) } @@ -64,6 +47,10 @@ func (*controlledSnapshotScheduler) SetCheckpointVersion(string, int64) error { return nil } +type pendingWaiter interface { + WaitForPendingWrites() +} + func bankChangeset(key, value string) []*proto.NamedChangeSet { return []*proto.NamedChangeSet{ { @@ -81,19 +68,17 @@ func evmStorageKey() []byte { return append([]byte{0x03}, make([]byte, 20+32)...) } -// setupSnapshotStore opens a store with snapshotting on at a small interval so -// tests can cross boundaries cheaply. It returns the store and its snapshots -// root. -func setupSnapshotStore(t *testing.T, interval int64, keepRecent int, separateEVMSubDBs bool) (*CompositeStateStore, string) { +func setupSnapshotStore(t *testing.T, interval int64, keepRecent int, separateEVMSubDBs bool) (*CompositeStateStore, string, string) { t.Helper() dir := t.TempDir() + evmDir := filepath.Join(dir, "evm_ss") store, err := NewCompositeStateStore(config.StateStoreConfig{ Backend: "pebbledb", AsyncWriteBuffer: 100, KeepRecent: 100000, EVMSplit: true, SeparateEVMSubDBs: separateEVMSubDBs, - EVMDBDirectory: filepath.Join(dir, "evm_ss"), + EVMDBDirectory: evmDir, SnapshotEnable: true, SnapshotInterval: interval, SnapshotKeepRecent: keepRecent, @@ -101,16 +86,11 @@ func setupSnapshotStore(t *testing.T, interval int64, keepRecent int, separateEV require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) require.NotNil(t, store.snapshotMgr) - return store, filepath.Join(dir, "data", "state_store", SnapshotsDirName) -} - -type pendingWaiter interface { - WaitForPendingWrites() + return store, + filepath.Join(dir, "data", "state_store", SnapshotsDirName), + evmDir + "-" + SnapshotsDirName } -// settle waits until every snapshot requested so far has been published and its -// pruning finished. Snapshot barriers sit in the backends' apply queues, so -// draining those queues is what guarantees the barriers ran. func settle(t *testing.T, store *CompositeStateStore) { t.Helper() if w, ok := store.cosmosStore.(pendingWaiter); ok { @@ -122,10 +102,6 @@ func settle(t *testing.T, store *CompositeStateStore) { store.snapshotMgr.publishing.Wait() } -// commitBlock is what rootmulti.flush does for a populated block: enqueue the -// changesets, then hand the version to the snapshot manager. ApplyChangesetAsync -// alone schedules nothing, so tests that expect a snapshot must come through -// here. func commitBlock(t *testing.T, store *CompositeStateStore, version int64, changesets []*proto.NamedChangeSet) { t.Helper() require.NoError(t, store.ApplyChangesetAsync(version, changesets)) @@ -150,8 +126,6 @@ func writeBlock(t *testing.T, store *CompositeStateStore, version int64) { }) } -// The snapshot manager keys off the mirrored cadence, so the ss-snapshot-enable -// switch has to reach it as a zero interval and leave no manager running. func TestSnapshotManagerRespectsSnapshotEnable(t *testing.T) { for _, tc := range []struct { name string @@ -182,787 +156,146 @@ func TestSnapshotManagerRespectsSnapshotEnable(t *testing.T) { } } -func TestCustomStateStoreDirectoryMovesSnapshotRootBesideDatabase(t *testing.T) { - home := t.TempDir() - customDB := filepath.Join(t.TempDir(), "cosmos-state") +func TestExternalPruningStandsDownWithoutSnapshots(t *testing.T) { cfg := config.DefaultStateStoreConfig() - cfg.Backend = config.PebbleDBBackend - cfg.DBDirectory = customDB - cfg.SnapshotEnable = true - cfg.SnapshotInterval = 5 - cfg.SnapshotKeepRecent = 1 + cfg.SnapshotEnable = false + cfg.ExternalPruning = true - store, err := NewCompositeStateStore(cfg, home) + store, err := NewCompositeStateStore(cfg, t.TempDir()) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) - require.Equal(t, customDB+"-"+SnapshotsDirName, store.snapshotMgr.root) -} - -func TestSnapshotHardlinkPreflightCleansProbeFiles(t *testing.T) { - source := t.TempDir() - root := t.TempDir() - require.NoError(t, verifySnapshotHardlinks(root, []string{source})) - - sourceEntries, err := os.ReadDir(source) - require.NoError(t, err) - require.Empty(t, sourceEntries) - rootEntries, err := os.ReadDir(root) - require.NoError(t, err) - require.Empty(t, rootEntries) -} - -func TestSnapshotHardlinkPreflightRejectsCrossFilesystem(t *testing.T) { - root, err := os.MkdirTemp("/dev/shm", "ss-snapshot-test-*") - if err != nil { - t.Skipf("no separate /dev/shm filesystem: %v", err) - } - t.Cleanup(func() { require.NoError(t, os.RemoveAll(root)) }) - - err = verifySnapshotHardlinks(root, []string{t.TempDir()}) - if err == nil { - t.Skip("temporary directory and /dev/shm use the same filesystem") - } - require.ErrorContains(t, err, "cannot hardlink snapshots") -} - -func TestSnapshotManagerRejectsUnsupportedBackend(t *testing.T) { - store := &CompositeStateStore{ - cosmosStore: cosmos.NewCosmosStateStore(&noCheckpointStateStore{}), - config: config.StateStoreConfig{ - Backend: config.RocksDBBackend, - SnapshotInterval: 10, - }, - } - - err := store.startSnapshotManager(t.TempDir(), nil) - require.ErrorContains(t, err, "does not support checkpoints") require.Nil(t, store.snapshotMgr) + require.Nil(t, store.pruningManager) + require.True(t, store.cosmosStore.(interface{ ExternalPruning() bool }).ExternalPruning()) } -func TestSnapshotManagerRejectsBackendWithoutBarrier(t *testing.T) { - store := &CompositeStateStore{ - cosmosStore: cosmos.NewCosmosStateStore(&noBarrierStateStore{}), - config: config.StateStoreConfig{ - Backend: config.PebbleDBBackend, - SnapshotInterval: 10, - }, - } - - err := store.startSnapshotManager(t.TempDir(), nil) - require.ErrorContains(t, err, "does not support checkpoints") - require.Nil(t, store.snapshotMgr) -} - -func TestSnapshotStopCancelsQueuedCheckpoint(t *testing.T) { - scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 1)} - manager := &snapshotManager{ - root: t.TempDir(), - backend: config.PebbleDBBackend, - interval: 5, - keepRecent: 1, - cosmosScheduler: scheduler, - } - - manager.maybeSnapshot(5) - manager.stop() - (<-scheduler.pending)() - - require.Zero(t, scheduler.checkpointCalls) - versions, err := ListSnapshotVersions(manager.root) - require.NoError(t, err) - require.Empty(t, versions) -} - -func TestSnapshotManagerAllowsOnlyOneInFlightSnapshot(t *testing.T) { - scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 2)} - manager := &snapshotManager{ - root: t.TempDir(), - backend: config.PebbleDBBackend, - interval: 5, - keepRecent: 1, - cosmosScheduler: scheduler, - } - - manager.maybeSnapshot(5) - manager.maybeSnapshot(10) - require.Len(t, scheduler.pending, 1, "a second boundary must not enqueue while one snapshot is active") - - (<-scheduler.pending)() - manager.publishing.Wait() - require.False(t, manager.inFlight) - require.Equal(t, int64(5), manager.lastRequested) -} - -func TestSnapshotManagerAppliesMinimumTimeInterval(t *testing.T) { - scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 2)} - manager := &snapshotManager{ - root: t.TempDir(), - backend: config.PebbleDBBackend, - interval: 5, - minTime: time.Hour, - keepRecent: 1, - cosmosScheduler: scheduler, - } - - manager.maybeSnapshot(5) - (<-scheduler.pending)() - manager.publishing.Wait() - - manager.maybeSnapshot(10) - require.Empty(t, scheduler.pending, "a rapid boundary must be skipped") - - manager.mu.Lock() - manager.lastRequestAt = time.Now().Add(-2 * time.Hour) - manager.mu.Unlock() - manager.maybeSnapshot(10) - require.Len(t, scheduler.pending, 1) - (<-scheduler.pending)() - manager.publishing.Wait() -} - -func TestSnapshotStopWaitsForBarrierScheduling(t *testing.T) { - scheduler := &controlledSnapshotScheduler{ - pending: make(chan func()), - entered: make(chan struct{}), - } - manager := &snapshotManager{ - root: t.TempDir(), - backend: config.PebbleDBBackend, - interval: 5, - keepRecent: 1, - cosmosScheduler: scheduler, - } - - requestDone := make(chan struct{}) - go func() { - manager.maybeSnapshot(5) - close(requestDone) - }() - <-scheduler.entered - - stopDone := make(chan struct{}) - go func() { - manager.stop() - close(stopDone) - }() - require.Never(t, func() bool { - select { - case <-stopDone: - return true - default: - return false - } - }, 50*time.Millisecond, 5*time.Millisecond) - - callback := <-scheduler.pending - <-requestDone - <-stopDone - callback() - require.False(t, manager.inFlight) -} - -// Snapshot labels are the interval boundaries themselves, not whatever version -// the store happened to be at when some background pass noticed. That is the -// property the in-queue barrier buys. It keeps each accepted SS snapshot's -// contents aligned with its own label even when SC independently skips that -// boundary. -func TestSnapshotTakenAtExactIntervalBoundaries(t *testing.T) { - store, root := setupSnapshotStore(t, 5, 5, false) - - for v := int64(1); v <= 12; v++ { - writeBlock(t, store, v) - if v%5 == 0 { - settle(t, store) - } - } - settle(t, store) - - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{5, 10}, versions, - "snapshots must land on interval boundaries and nowhere else") - - target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) - require.NoError(t, err) - require.Equal(t, SnapshotDirName(10), target) - - snapDir := filepath.Join(root, SnapshotDirName(10)) - apparentBytes, err := readSnapshotSize(snapDir) - require.NoError(t, err) - require.Positive(t, apparentBytes) - reopened, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: config.PebbleDBBackend, - AsyncWriteBuffer: 0, - KeepRecent: 100000, - EVMSplit: true, - DBDirectory: filepath.Join(snapDir, "cosmos", config.PebbleDBBackend), - EVMDBDirectory: filepath.Join(snapDir, "evm", config.PebbleDBBackend), - }, t.TempDir()) - require.NoError(t, err) - defer reopened.Close() - - require.Equal(t, int64(10), reopened.GetLatestVersion()) - cosmosValue, err := reopened.Get("bank", 12, []byte("balance")) - require.NoError(t, err) - require.Equal(t, []byte{10}, cosmosValue, "snapshot 10 must exclude Cosmos writes 11 and 12") - evmValue, err := reopened.Get(evm.EVMStoreKey, 12, evmStorageKey()) - require.NoError(t, err) - require.Equal(t, []byte{10}, evmValue, "snapshot 10 must exclude EVM writes 11 and 12") -} - -func TestSnapshotTakenAtExactIntervalBoundaryWithoutEVMSplit(t *testing.T) { - dir := t.TempDir() - ssConfig := config.DefaultStateStoreConfig() - ssConfig.Backend = config.PebbleDBBackend - ssConfig.AsyncWriteBuffer = 100 - ssConfig.KeepRecent = 100000 - ssConfig.EVMSplit = false - ssConfig.SnapshotEnable = true - ssConfig.SnapshotInterval = 5 - ssConfig.SnapshotKeepRecent = 1 - - store, err := NewCompositeStateStore(ssConfig, dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, store.Close()) }) - require.NotNil(t, store.snapshotMgr) - - for version := int64(1); version <= 5; version++ { - commitBlock(t, store, version, bankChangeset("balance", "value")) - } - settle(t, store) - - root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{5}, versions) -} - -// A snapshot must be a complete image of every version at or below its label, -// reopenable as a store in its own right. -func TestSnapshotReopensWithEveryVersionBelowLabel(t *testing.T) { - store, root := setupSnapshotStore(t, 10, 5, false) - - for v := int64(1); v <= 10; v++ { - writeBlock(t, store, v) - } - settle(t, store) - - const label = int64(10) - snapDir := filepath.Join(root, SnapshotDirName(label)) - require.DirExists(t, snapDir) - - reopened, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - EVMSplit: true, - DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), - EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), - }, t.TempDir()) - require.NoError(t, err) - defer reopened.Close() - - require.Equal(t, label, reopened.GetLatestVersion(), - "the label is the version the snapshot was requested at") - for v := int64(1); v <= label; v++ { - val, err := reopened.Get("bank", v, []byte("balance")) - require.NoError(t, err) - require.Equal(t, []byte{byte(v)}, val, "cosmos version %d missing from snapshot", v) - val, err = reopened.Get(evm.EVMStoreKey, v, evmStorageKey()) - require.NoError(t, err) - require.Equal(t, []byte{byte(v)}, val, "evm version %d missing from snapshot", v) - } -} - -// The property the barrier exists for: the label stays exact while the write -// path keeps going. Nothing is drained between block 10 and blocks 11 and 12, so -// the checkpoint runs with later versions already queued behind the barrier — the -// case a post-hoc "snapshot what has been applied" scheme would get wrong. -func TestSnapshotExcludesVersionsWrittenAfterTheBoundary(t *testing.T) { - store, root := setupSnapshotStore(t, 10, 5, false) - - const label = int64(10) - for v := int64(1); v <= label; v++ { - writeBlock(t, store, v) - } - for v := label + 1; v <= label+2; v++ { - writeBlock(t, store, v) - } - settle(t, store) - - snapDir := filepath.Join(root, SnapshotDirName(label)) - require.DirExists(t, snapDir) - - reopened, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - EVMSplit: true, - DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), - EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), - }, t.TempDir()) - require.NoError(t, err) - defer reopened.Close() - - require.Equal(t, label, reopened.GetLatestVersion()) - // Reading above the label returns the label's value rather than 11 or 12, - // which is what "excluded" means for an MVCC store: the later writes are not - // in this image at any version. - for _, above := range []int64{label + 1, label + 2} { - val, err := reopened.Get("bank", above, []byte("balance")) - require.NoError(t, err) - require.Equal(t, []byte{byte(label)}, val, - "cosmos read at %d saw a write from after the boundary", above) - val, err = reopened.Get(evm.EVMStoreKey, above, evmStorageKey()) - require.NoError(t, err) - require.Equal(t, []byte{byte(label)}, val, - "evm read at %d saw a write from after the boundary", above) - } - - // The live store keeps them, so the snapshot dropped them rather than the - // writes never landing. - val, err := store.Get("bank", label+2, []byte("balance")) - require.NoError(t, err) - require.Equal(t, []byte{byte(label + 2)}, val) -} - -// A snapshot inherits each database's earliest marker. The composite is allowed -// to reopen with different member floors because it reports the highest one. -func TestSnapshotInheritsPerStoreEarliestMarkers(t *testing.T) { - store, root := setupSnapshotStore(t, 10, 5, false) - - for v := int64(1); v <= 9; v++ { - writeBlock(t, store, v) - } - settle(t, store) - require.NoError(t, store.cosmosStore.SetEarliestVersion(2, false)) - require.NoError(t, store.evmStore.SetEarliestVersion(5, false)) - require.Equal(t, int64(2), store.cosmosStore.GetEarliestVersion()) - require.Equal(t, int64(5), store.evmStore.GetEarliestVersion()) - - writeBlock(t, store, 10) - settle(t, store) - - snapDir := filepath.Join(root, SnapshotDirName(10)) - require.DirExists(t, snapDir) - - reopened, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - EVMSplit: true, - DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), - EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), - }, t.TempDir()) - require.NoError(t, err, "a snapshot with different member floors must reopen") - defer reopened.Close() - - require.Equal(t, int64(2), reopened.cosmosStore.GetEarliestVersion()) - require.Equal(t, int64(5), reopened.GetEarliestVersion()) - require.Equal(t, int64(5), reopened.evmStore.GetEarliestVersion()) -} - -func TestSnapshotInheritsEarliestMarkerAfterPrune(t *testing.T) { - store, root := setupSnapshotStore(t, 10, 5, false) - - for v := int64(1); v <= 9; v++ { - writeBlock(t, store, v) - } - settle(t, store) - require.NoError(t, store.Prune(4)) - - writeBlock(t, store, 10) - settle(t, store) - - snapDir := filepath.Join(root, SnapshotDirName(10)) - reopened, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - EVMSplit: true, - DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), - EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), - }, t.TempDir()) - require.NoError(t, err) - defer reopened.Close() - - require.Equal(t, int64(5), reopened.GetEarliestVersion()) - require.Equal(t, int64(5), reopened.cosmosStore.GetEarliestVersion()) - require.Equal(t, int64(5), reopened.evmStore.GetEarliestVersion()) -} - -// With separate sub-DBs the latest label has to reach every one of them, not -// just the sub-DBs that took writes, or the snapshot is not self-describing at -// its exact boundary. -func TestSnapshotSetsLatestVersionEveryEVMSubDB(t *testing.T) { - store, root := setupSnapshotStore(t, 10, 5, true) - - for v := int64(1); v <= 9; v++ { - writeBlock(t, store, v) - } - - writeBlock(t, store, 10) - settle(t, store) - - snapDir := filepath.Join(root, SnapshotDirName(10)) - reopened, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - EVMSplit: true, - SeparateEVMSubDBs: true, - DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), - EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), - }, t.TempDir()) - require.NoError(t, err) - - require.Equal(t, int64(10), reopened.evmStore.GetLatestVersion()) - require.NoError(t, reopened.Close()) - - evmRoot := filepath.Join(snapDir, "evm", "pebbledb") - for _, storeType := range evm.AllEVMStoreTypes() { - subDir := filepath.Join(evmRoot, evm.StoreTypeName(storeType)) - subDB, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - UseDefaultComparer: true, - DBDirectory: subDir, - }, t.TempDir()) - require.NoError(t, err) - require.Equal(t, int64(10), subDB.GetLatestVersion(), "sub-DB %s latest marker", evm.StoreTypeName(storeType)) - require.NoError(t, subDB.Close()) - } -} - -// The reason the barrier has to be a message in every queue rather than a wait: -// a block that only touches storage keys is enqueued only on the storage sub-DB, -// so the idle sub-DBs never observe that version and no amount of waiting would -// tell them it passed. Every sub-DB must still be captured. -func TestSnapshotCapturesIdleEVMSubDBs(t *testing.T) { - store, root := setupSnapshotStore(t, 5, 5, true) - - // Storage keys only: codehash, code and misc sub-DBs stay idle throughout. - for v := int64(1); v <= 5; v++ { - commitBlock(t, store, v, []*proto.NamedChangeSet{ - { - Name: evm.EVMStoreKey, - Changeset: proto.ChangeSet{ - Pairs: []*proto.KVPair{{Key: evmStorageKey(), Value: []byte{byte(v)}}}, - }, - }, - }) - } - settle(t, store) - - evmRoot := filepath.Join(root, SnapshotDirName(5), "evm", "pebbledb") - for _, storeType := range evm.AllEVMStoreTypes() { - name := evm.StoreTypeName(storeType) - subDir := filepath.Join(evmRoot, name) - require.DirExists(t, subDir, "sub-DB %s missing from snapshot", name) - // A checkpoint always carries a manifest. An empty directory would mean - // the barrier never reached that sub-DB. - manifests, err := filepath.Glob(filepath.Join(subDir, "MANIFEST-*")) - require.NoError(t, err) - require.NotEmpty(t, manifests, "sub-DB %s was not checkpointed", name) - } - - // The storage sub-DB is the one that actually took writes, and it must be - // readable at every version up to the label. - storage, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - // EVM sub-DBs are opened with the plain byte comparer. - UseDefaultComparer: true, - DBDirectory: filepath.Join(evmRoot, evm.StoreTypeName(evm.StoreStorage)), - }, t.TempDir()) - require.NoError(t, err) - defer storage.Close() - - for v := int64(1); v <= 5; v++ { - val, err := storage.Get(evm.EVMStoreKey, v, evmStorageKey()) - require.NoError(t, err) - require.Equal(t, []byte{byte(v)}, val, "evm storage version %d missing from snapshot", v) - } -} - -// An interval boundary that happens to be an empty block arrives through -// SetLatestVersion rather than the changeset path, and must still snapshot — -// otherwise a quiet chain skips whole intervals. -func TestSnapshotTakenOnEmptyBoundaryBlock(t *testing.T) { - store, root := setupSnapshotStore(t, 5, 5, false) - - for v := int64(1); v <= 4; v++ { - writeBlock(t, store, v) - } - // Block 5 is empty: marker only, nothing enqueued. - require.NoError(t, store.SetLatestVersion(5)) - store.ScheduleSnapshot(5) - settle(t, store) +func TestPerStoreSnapshotRoots(t *testing.T) { + store, cosmosRoot, evmRoot := setupSnapshotStore(t, 5, 1, true) - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{5}, versions) - - // Every data version below the label is inside the snapshot, and the - // checkpoint marker advances to the empty block's version. - snapDir := filepath.Join(root, SnapshotDirName(5)) - reopened, err := NewCompositeStateStore(config.StateStoreConfig{ - Backend: "pebbledb", - AsyncWriteBuffer: 0, - KeepRecent: 100000, - DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), - }, t.TempDir()) - require.NoError(t, err) - defer reopened.Close() - - require.Equal(t, int64(5), reopened.GetLatestVersion()) - val, err := reopened.Get("bank", 4, []byte("balance")) - require.NoError(t, err) - require.Equal(t, []byte{4}, val) + require.Equal(t, cosmosRoot, store.cosmosStore.(cosmosStoreWithSnapshots).Snapshots().Root()) + require.Equal(t, evmRoot, store.evmStore.(*evm.EVMStateStore).Snapshots().Root()) } -func TestSetLatestVersionDoesNotSnapshotDuringImport(t *testing.T) { - store, root := setupSnapshotStore(t, 5, 5, false) - nodes := make(chan types.SnapshotNode) - importDone := make(chan error, 1) - go func() { - importDone <- store.Import(5, nodes) - }() - closed := false - t.Cleanup(func() { - if !closed { - close(nodes) - <-importDone - } - }) - - nodes <- types.SnapshotNode{StoreKey: "bank", Key: []byte("balance"), Value: []byte{5}} - require.NoError(t, store.SetLatestVersion(5)) - settle(t, store) - - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Empty(t, versions, "direct restore metadata writes must not trigger a snapshot") - - close(nodes) - closed = true - require.NoError(t, <-importDone) -} - -// TestSnapshotPrune verifies retention: with keepRecent=1, only the newest two -// snapshots survive and current tracks the newest. -func TestSnapshotPrune(t *testing.T) { - store, root := setupSnapshotStore(t, 5, 1, false) - - for v := int64(1); v <= 15; v++ { - writeBlock(t, store, v) - if v%5 == 0 { - settle(t, store) - } - } - settle(t, store) - - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{10, 15}, versions) - - target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) - require.NoError(t, err) - require.Equal(t, SnapshotDirName(15), target) -} - -func TestSnapshotPruneKeepsCurrentTarget(t *testing.T) { - root := t.TempDir() - for _, version := range []int64{5, 10, 15, 20} { - require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) - } - require.NoError(t, os.Symlink(SnapshotDirName(5), filepath.Join(root, snapshotCurrentLink))) - - manager := &snapshotManager{root: root, keepRecent: 1} - manager.prune() - - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{5, 15, 20}, versions, - "retention may keep one extra snapshot but must not dangle current") - target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) - require.NoError(t, err) - require.Equal(t, SnapshotDirName(5), target) -} - -func TestSnapshotManagerResumesFromNewestSnapshot(t *testing.T) { - dir := t.TempDir() +func TestCustomStateStoreDirectoryMovesCosmosSnapshotRootBesideDatabase(t *testing.T) { + home := t.TempDir() + customDB := filepath.Join(t.TempDir(), "cosmos-state") cfg := config.DefaultStateStoreConfig() cfg.Backend = config.PebbleDBBackend - cfg.AsyncWriteBuffer = 100 - cfg.KeepRecent = 100000 + cfg.DBDirectory = customDB cfg.SnapshotEnable = true cfg.SnapshotInterval = 5 cfg.SnapshotKeepRecent = 1 - cfg.SnapshotMinTimeInterval = time.Hour - - store, err := NewCompositeStateStore(cfg, dir) - require.NoError(t, err) - for version := int64(1); version <= 5; version++ { - commitBlock(t, store, version, bankChangeset("balance", "value")) - } - settle(t, store) - - root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) - snapshotDir := filepath.Join(root, SnapshotDirName(5)) - before, err := os.Stat(snapshotDir) - require.NoError(t, err) - require.NoError(t, store.Close()) - require.NoError(t, os.Remove(filepath.Join(root, snapshotCurrentLink))) - reopened, err := NewCompositeStateStore(cfg, dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, reopened.Close()) }) - require.Equal(t, int64(5), reopened.snapshotMgr.lastRequested) - require.WithinDuration(t, before.ModTime(), reopened.snapshotMgr.lastRequestAt, time.Second) - target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) - require.NoError(t, err) - require.Equal(t, SnapshotDirName(5), target) - - reopened.ScheduleSnapshot(5) - settle(t, reopened) - after, err := os.Stat(snapshotDir) + store, err := NewCompositeStateStore(cfg, home) require.NoError(t, err) - require.True(t, os.SameFile(before, after), "restart must not replace an existing boundary snapshot") + t.Cleanup(func() { require.NoError(t, store.Close()) }) - for version := int64(6); version <= 10; version++ { - commitBlock(t, reopened, version, bankChangeset("balance", "value")) - } - settle(t, reopened) - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{5}, versions, "restart must preserve the minimum-time gate") + require.Equal(t, customDB+"-"+SnapshotsDirName, store.cosmosStore.(cosmosStoreWithSnapshots).Snapshots().Root()) } -func TestOutOfOrderPublishDoesNotMoveCurrentBackward(t *testing.T) { - root := t.TempDir() - manager := &snapshotManager{root: root, keepRecent: 5} +func TestCompositeCoordinatorPublishesEveryMember(t *testing.T) { + rootA, rootB := t.TempDir(), t.TempDir() + schedulerA := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + schedulerB := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + managerA := openTestManager(t, "cosmos", rootA, schedulerA) + managerB := openTestManager(t, "evm", rootB, schedulerB) + coord := newSnapshotCoordinator(10, 0, []snapshotMember{ + {name: "cosmos", manager: managerA}, + {name: "evm", manager: managerB}, + }) - publish := func(version int64) { - tmpDir := filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(version)) - require.NoError(t, os.MkdirAll(tmpDir, 0o750)) - manager.publish(version, tmpDir, filepath.Join(root, SnapshotDirName(version)), time.Now()) - } - publish(10) - publish(5) + coord.maybeSnapshot(10) + (<-schedulerA.pending)() + (<-schedulerB.pending)() + coord.publishing.Wait() + + require.DirExists(t, filepath.Join(rootA, SnapshotDirName(10))) + require.DirExists(t, filepath.Join(rootB, SnapshotDirName(10))) + require.Equal(t, int64(10), newestCommonSnapshot(coord.members)) +} + +func TestCompositeCoordinatorAbortsEveryMemberOnStageFailure(t *testing.T) { + rootA, rootB := t.TempDir(), t.TempDir() + schedulerA := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + schedulerB := &controlledSnapshotScheduler{pending: make(chan func(), 1), fail: true} + managerA := openTestManager(t, "cosmos", rootA, schedulerA) + managerB := openTestManager(t, "evm", rootB, schedulerB) + coord := newSnapshotCoordinator(10, 0, []snapshotMember{ + {name: "cosmos", manager: managerA}, + {name: "evm", manager: managerB}, + }) - target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) - require.NoError(t, err) - require.Equal(t, SnapshotDirName(10), target) + coord.maybeSnapshot(10) + (<-schedulerA.pending)() + (<-schedulerB.pending)() + coord.publishing.Wait() + + require.NoDirExists(t, filepath.Join(rootA, SnapshotDirName(10))) + require.NoDirExists(t, filepath.Join(rootB, SnapshotDirName(10))) } -func TestSnapshotManagerPrunesExistingSnapshotsAtStartup(t *testing.T) { - dir := t.TempDir() - root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) - for _, version := range []int64{5, 10, 15} { - require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) - } +func TestUnpairedSnapshotHeightsSurviveStartupAndCommonHeightIsLower(t *testing.T) { + rootA, rootB := t.TempDir(), t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(rootA, SnapshotDirName(10)), 0o750)) + require.NoError(t, os.MkdirAll(filepath.Join(rootA, SnapshotDirName(20)), 0o750)) + require.NoError(t, os.MkdirAll(filepath.Join(rootB, SnapshotDirName(10)), 0o750)) - cfg := config.DefaultStateStoreConfig() - cfg.Backend = config.PebbleDBBackend - cfg.SnapshotEnable = true - cfg.SnapshotInterval = 5 - cfg.SnapshotKeepRecent = 1 - store, err := NewCompositeStateStore(cfg, dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, store.Close()) }) + managerA := openTestManager(t, "cosmos", rootA, &controlledSnapshotScheduler{pending: make(chan func(), 1)}) + managerB := openTestManager(t, "evm", rootB, &controlledSnapshotScheduler{pending: make(chan func(), 1)}) + coord := newSnapshotCoordinator(10, 0, []snapshotMember{ + {name: "cosmos", manager: managerA}, + {name: "evm", manager: managerB}, + }) - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{10, 15}, versions) + require.DirExists(t, filepath.Join(rootA, SnapshotDirName(20))) + require.Equal(t, int64(20), coord.lastRequested) + require.Equal(t, int64(10), newestCommonSnapshot(coord.members)) } -func TestFailedPublishStillEnforcesRetention(t *testing.T) { - root := t.TempDir() - for _, version := range []int64{5, 10, 15} { - require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) - } - manager := &snapshotManager{root: root, keepRecent: 1} +func TestSnapshotCoversEmptyBlock(t *testing.T) { + store, cosmosRoot, evmRoot := setupSnapshotStore(t, 5, 1, false) - published := manager.publish( - 20, - filepath.Join(root, "missing-staging-dir"), - filepath.Join(root, SnapshotDirName(20)), - time.Now(), - ) - require.False(t, published) + for i := int64(1); i <= 4; i++ { + writeBlock(t, store, i) + } + commitBlock(t, store, 5, nil) + settle(t, store) - versions, err := ListSnapshotVersions(root) - require.NoError(t, err) - require.Equal(t, []int64{10, 15}, versions) + require.DirExists(t, filepath.Join(cosmosRoot, SnapshotDirName(5))) + require.DirExists(t, filepath.Join(evmRoot, SnapshotDirName(5))) } -func TestRetentionMetricsCacheSnapshotSizes(t *testing.T) { - root := t.TempDir() - snapshotDir := filepath.Join(root, SnapshotDirName(5)) - require.NoError(t, os.MkdirAll(snapshotDir, 0o750)) - dataFile := filepath.Join(snapshotDir, "data.sst") - require.NoError(t, os.WriteFile(dataFile, []byte("one"), 0o600)) - require.NoError(t, writeSnapshotSize(snapshotDir, 3)) - - manager := &snapshotManager{root: root} - manager.recordRetentionMetrics() - require.Equal(t, int64(3), manager.snapshotSizes[5]) - - // Published snapshots are immutable, so later metric records reuse the - // cached total rather than walking every retained hardlink tree again. - require.NoError(t, os.WriteFile(dataFile, []byte("a longer value"), 0o600)) - manager.recordRetentionMetrics() - require.Equal(t, int64(3), manager.snapshotSizes[5]) - - require.NoError(t, os.RemoveAll(snapshotDir)) - manager.recordRetentionMetrics() - require.NotContains(t, manager.snapshotSizes, int64(5)) -} +func TestSnapshotMinTimeIntervalSkipsBoundary(t *testing.T) { + rootA := t.TempDir() + scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + manager := openTestManager(t, "cosmos", rootA, scheduler) + coord := newSnapshotCoordinator(10, time.Hour, []snapshotMember{{name: "cosmos", manager: manager}}) -func TestSnapshotRequestReturnsUnexpectedStatError(t *testing.T) { - root := t.TempDir() - name := SnapshotDirName(5) - require.NoError(t, os.Symlink(name, filepath.Join(root, name))) + coord.maybeSnapshot(10) + (<-scheduler.pending)() + coord.publishing.Wait() - manager := &snapshotManager{root: root, backend: config.PebbleDBBackend} - err := manager.requestSnapshot(5, time.Now()) - require.ErrorContains(t, err, "inspect snapshot dir") + coord.maybeSnapshot(20) + require.Empty(t, scheduler.pending) + require.NoDirExists(t, filepath.Join(rootA, SnapshotDirName(20))) } -// A crash mid-snapshot leaves a staging directory named after the boundary it -// was staging, which would otherwise sit there until that exact boundary came -// round again. -func TestStaleSnapshotTmpDirRemovedAtStartup(t *testing.T) { - dir := t.TempDir() - root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) - stale := filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(40)) - require.NoError(t, os.MkdirAll(filepath.Join(stale, "cosmos"), 0o750)) - tmpLink := filepath.Join(root, snapshotCurrentTmpLink) - require.NoError(t, os.Symlink(filepath.Base(stale), tmpLink)) - - ssConfig := config.DefaultStateStoreConfig() - ssConfig.SnapshotEnable = true - config.AlignSSSnapshotWithSC(config.DefaultStateCommitConfig(), &ssConfig) - store, err := NewCompositeStateStore(ssConfig, dir) +func openTestManager(t *testing.T, name, root string, scheduler *controlledSnapshotScheduler) *sssnapshot.Manager { + t.Helper() + source := t.TempDir() + manager, err := sssnapshot.Open(sssnapshot.Config{ + Name: name, + Root: root, + SourceDirs: []string{source}, + Backend: config.PebbleDBBackend, + KeepRecent: 1, + Scheduler: scheduler, + }) require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, store.Close()) }) - - require.NoDirExists(t, stale) - _, err = os.Lstat(tmpLink) - require.ErrorIs(t, err, os.ErrNotExist) + return manager } -func TestParseSnapshotVersion(t *testing.T) { - v, ok := ParseSnapshotVersion(SnapshotDirName(219140000)) - require.True(t, ok) - require.Equal(t, int64(219140000), v) - - for _, bad := range []string{"snapshot-", "snapshot-123", "current", "tmp-snapshot-00000000000000000010", "snapshot-0000000000000000001x"} { - _, ok := ParseSnapshotVersion(bad) - require.False(t, ok, "expected %q to be rejected", bad) - } +type cosmosStoreWithSnapshots interface { + Snapshots() *sssnapshot.Manager } diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index dbaf727921..9007f0b2dd 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -35,7 +35,7 @@ type CompositeStateStore struct { cosmosStore types.StateStore // CosmosStateStore wrapping MVCC DB evmStore types.StateStore // EVMStateStore wrapping sub MVCC DBs (nil if disabled) pruningManager *pruning.Manager - snapshotMgr *snapshotManager + snapshotMgr *snapshotCoordinator config config.StateStoreConfig closeOnce sync.Once closeErr error @@ -57,12 +57,12 @@ func NewCompositeStateStore( return nil, fmt.Errorf("failed to create cosmos MVCC DB: %w", err) } cosmosStore := cosmos.NewCosmosStateStore(mvccDB) + cosmosStore.SetExternalPruning(ssConfig.ExternalPruning) cs := &CompositeStateStore{ cosmosStore: cosmosStore, config: ssConfig, } - snapshotSourceDirs := []string{dbHome} if ssConfig.EVMSplit { evmDir := ssConfig.EVMDBDirectory @@ -82,16 +82,6 @@ func NewCompositeStateStore( return nil, fmt.Errorf("failed to create EVM store: %w", err) } cs.evmStore = evmStore - if ssConfig.SeparateEVMSubDBs { - for _, storeType := range evm.AllEVMStoreTypes() { - snapshotSourceDirs = append( - snapshotSourceDirs, - filepath.Join(evmDir, evm.StoreTypeName(storeType)), - ) - } - } else { - snapshotSourceDirs = append(snapshotSourceDirs, evmDir) - } logger.Info("EVM state store enabled", "dir", evmDir, "separateDBs", ssConfig.SeparateEVMSubDBs, @@ -113,15 +103,29 @@ func NewCompositeStateStore( cs.validateEVMSSPostRecovery() if ssConfig.SnapshotInterval > 0 { - snapshotRoot := utils.GetStateStoreSnapshotsPath(homeDir) + members := make([]snapshotMember, 0, 2) + cosmosSnapshotRoot := utils.GetStateStoreSnapshotsPath(homeDir) if ssConfig.DBDirectory != "" { cleanDBHome := filepath.Clean(dbHome) - snapshotRoot = filepath.Join( + cosmosSnapshotRoot = filepath.Join( filepath.Dir(cleanDBHome), filepath.Base(cleanDBHome)+"-"+utils.StateStoreSnapshotsDirName, ) } - if err := cs.startSnapshotManager(snapshotRoot, snapshotSourceDirs); err != nil { + if err := cosmosStore.StartSnapshots(cosmosSnapshotRoot, []string{dbHome}, ssConfig); err != nil { + _ = cs.Close() + return nil, fmt.Errorf("start Cosmos state store snapshot manager: %w", err) + } + members = append(members, snapshotMember{name: "cosmos", manager: cosmosStore.Snapshots()}) + if evmStore, ok := cs.evmStore.(*evm.EVMStateStore); ok { + evmSnapshotRoot := evmStore.Dir() + "-" + utils.StateStoreSnapshotsDirName + if err := evmStore.StartSnapshots(evmSnapshotRoot, ssConfig); err != nil { + _ = cs.Close() + return nil, fmt.Errorf("start EVM state store snapshot manager: %w", err) + } + members = append(members, snapshotMember{name: "evm", manager: evmStore.Snapshots()}) + } + if err := cs.startSnapshotManager(members); err != nil { _ = cs.Close() return nil, fmt.Errorf("start state store snapshot manager: %w", err) } @@ -193,6 +197,10 @@ func (s *CompositeStateStore) validateEVMSSPostRecovery() { } func (s *CompositeStateStore) StartPruning() { + if s.config.ExternalPruning { + logger.Info("state store internal pruning disabled by external pruning") + return + } pm := pruning.NewPruningManager(s, int64(s.config.KeepRecent), int64(s.config.PruneIntervalSeconds)) pm.Start() s.pruningManager = pm diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index c512335e49..291023957a 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -3,9 +3,11 @@ package cosmos import ( dbm "github.com/tendermint/tm-db" + "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" + sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" ) // Compile-time check: CosmosStateStore implements db_engine.StateStore. @@ -14,11 +16,13 @@ var _ types.StateStore = (*CosmosStateStore)(nil) // CosmosStateStore wraps a single StateStore (MVCC DB) and satisfies db_engine.StateStore. // It is the SS-layer adapter for the main Cosmos state (all non-EVM modules). type CosmosStateStore struct { - db types.StateStore + db types.StateStore + snapshotMgr *sssnapshot.Manager + externalPruning bool } // NewCosmosStateStore wraps an existing StateStore as a CosmosStateStore. -func NewCosmosStateStore(db types.StateStore) types.StateStore { +func NewCosmosStateStore(db types.StateStore) *CosmosStateStore { return &CosmosStateStore{db: db} } @@ -70,6 +74,14 @@ func (s *CosmosStateStore) Prune(version int64) error { return s.db.Prune(version) } +func (s *CosmosStateStore) ExternalPruning() bool { + return s.externalPruning +} + +func (s *CosmosStateStore) SetExternalPruning(enabled bool) { + s.externalPruning = enabled +} + func (s *CosmosStateStore) Import(version int64, ch <-chan types.SnapshotNode) error { return s.db.Import(version, ch) } @@ -93,6 +105,28 @@ func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) e return management.SetCheckpointVersion(s.db, destDir, version) } +func (s *CosmosStateStore) StartSnapshots(root string, sourceDirs []string, ssConfig config.StateStoreConfig) error { + manager, err := sssnapshot.Open(sssnapshot.Config{ + Name: "cosmos", + Root: root, + SourceDirs: sourceDirs, + Backend: ssConfig.Backend, + KeepRecent: ssConfig.SnapshotKeepRecent, + ExternalPruning: ssConfig.ExternalPruning, + Scheduler: s, + }) + if err != nil { + return err + } + s.snapshotMgr = manager + s.externalPruning = ssConfig.ExternalPruning + return nil +} + +func (s *CosmosStateStore) Snapshots() *sssnapshot.Manager { + return s.snapshotMgr +} + func (s *CosmosStateStore) WaitForPendingWrites() { if w, ok := s.db.(interface{ WaitForPendingWrites() }); ok { w.WaitForPendingWrites() diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 4c20aa6ceb..4a6bd7f295 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -15,6 +15,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/backend" + sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" ) var _ types.StateStore = (*EVMStateStore)(nil) @@ -27,6 +28,9 @@ type EVMStateStore struct { managedDBs []types.StateStore dir string separateDBs bool + snapshotMgr *sssnapshot.Manager + + externalPruning bool } // NewEVMStateStore opens either a single unified MVCC DB for all EVM state @@ -35,9 +39,10 @@ func NewEVMStateStore(dir string, ssConfig config.StateStoreConfig) (*EVMStateSt opener := backend.ResolveBackend(ssConfig.Backend) store := &EVMStateStore{ - subDBs: make(map[EVMStoreType]types.StateStore, NumEVMStoreTypes), - dir: dir, - separateDBs: ssConfig.SeparateEVMSubDBs, + subDBs: make(map[EVMStoreType]types.StateStore, NumEVMStoreTypes), + dir: dir, + separateDBs: ssConfig.SeparateEVMSubDBs, + externalPruning: ssConfig.ExternalPruning, } if ssConfig.SeparateEVMSubDBs { @@ -82,6 +87,10 @@ func (s *EVMStateStore) primaryDB() types.StateStore { return s.managedDBs[0] } +func (s *EVMStateStore) Dir() string { + return s.dir +} + func (s *EVMStateStore) routeKey(key []byte) types.StateStore { storeType, _ := commonevm.ParseEVMKey(key) if storeType == StoreEmpty { @@ -360,6 +369,21 @@ func (s *EVMStateStore) Prune(version int64) error { return nil } +func (s *EVMStateStore) ExternalPruning() bool { + return s.externalPruning +} + +func (s *EVMStateStore) snapshotSourceDirs() []string { + if !s.separateDBs { + return []string{s.dir} + } + dirs := make([]string, 0, len(AllEVMStoreTypes())) + for _, storeType := range AllEVMStoreTypes() { + dirs = append(dirs, filepath.Join(s.dir, StoreTypeName(storeType))) + } + return dirs +} + func (s *EVMStateStore) Close() error { var lastErr error for _, db := range s.managedDBs { @@ -456,6 +480,28 @@ func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) erro return nil } +func (s *EVMStateStore) StartSnapshots(root string, ssConfig config.StateStoreConfig) error { + manager, err := sssnapshot.Open(sssnapshot.Config{ + Name: "evm", + Root: root, + SourceDirs: s.snapshotSourceDirs(), + Backend: ssConfig.Backend, + KeepRecent: ssConfig.SnapshotKeepRecent, + ExternalPruning: ssConfig.ExternalPruning, + Scheduler: s, + }) + if err != nil { + return err + } + s.snapshotMgr = manager + s.externalPruning = ssConfig.ExternalPruning + return nil +} + +func (s *EVMStateStore) Snapshots() *sssnapshot.Manager { + return s.snapshotMgr +} + func (s *EVMStateStore) WaitForPendingWrites() { for _, db := range s.managedDBs { if w, ok := db.(interface{ WaitForPendingWrites() }); ok { diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go new file mode 100644 index 0000000000..bc73741b2f --- /dev/null +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -0,0 +1,561 @@ +package snapshot + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/management" + "github.com/sei-protocol/seilog" +) + +const ( + // SnapshotsDirName is the directory under data/state_store that holds + // online snapshots for the Cosmos SS member. + SnapshotsDirName = utils.StateStoreSnapshotsDirName + + snapshotPrefix = "snapshot-" + // snapshotDirLen is "snapshot-" + 20-digit zero-padded version. + snapshotDirLen = len(snapshotPrefix) + 20 + + snapshotCurrentLink = "current" + snapshotCurrentTmpLink = "current-tmp" + snapshotTmpPrefix = "tmp-" + snapshotSizeFile = ".apparent-size" +) + +// Config wires one SS member into a snapshot Manager. +type Config struct { + Name string + Root string + SourceDirs []string + Backend string + KeepRecent int + ExternalPruning bool + Scheduler management.CheckpointScheduler +} + +// Manager owns one SS member's snapshot root, staging directories, current +// symlink, metadata, and retention. It has no cadence of its own; a coordinator +// decides which height to snapshot and then calls Stage and Commit. +type Manager struct { + name string + root string + backend string + keepRecent int + externalPruning bool + scheduler management.CheckpointScheduler + snapshotSizes map[int64]int64 + + publishMu sync.Mutex + lastPublished int64 +} + +// Staged names a checkpoint that has been written to a staging directory but +// has not yet been published. +type Staged struct { + manager *Manager + version int64 + tmpDir string + finalDir string +} + +func (s *Staged) Abort() { + if s == nil || s.manager == nil { + return + } + s.manager.Abort(s) +} + +// SnapshotDirName returns the directory name for a snapshot labeled with the +// given version. +func SnapshotDirName(version int64) string { + return fmt.Sprintf("%s%020d", snapshotPrefix, version) +} + +// ParseSnapshotVersion parses a snapshot directory name; ok is false for +// anything that is not a snapshot-<20 digits> name. +func ParseSnapshotVersion(name string) (version int64, ok bool) { + if !strings.HasPrefix(name, snapshotPrefix) || len(name) != snapshotDirLen { + return 0, false + } + v, err := strconv.ParseInt(name[len(snapshotPrefix):], 10, 64) + if err != nil || v < 0 { + return 0, false + } + return v, true +} + +// ListSnapshotVersions returns the labels of all snapshots under root in +// ascending order. A missing root is not an error (no snapshots yet). +func ListSnapshotVersions(root string) ([]int64, error) { + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read snapshots dir %q: %w", root, err) + } + var versions []int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if v, ok := ParseSnapshotVersion(entry.Name()); ok { + versions = append(versions, v) + } + } + slices.Sort(versions) + return versions, nil +} + +// Open prepares a snapshot root and returns a Manager for one SS member. +func Open(cfg Config) (*Manager, error) { + if cfg.Scheduler == nil { + return nil, fmt.Errorf("%s snapshot scheduler is nil", cfg.Name) + } + if !cfg.Scheduler.SupportsCheckpoint() { + return nil, fmt.Errorf("%s backend %q does not support checkpoints", cfg.Name, cfg.Backend) + } + if err := verifyHardlinks(cfg.Root, cfg.SourceDirs); err != nil { + return nil, err + } + m := &Manager{ + name: cfg.Name, + root: cfg.Root, + backend: cfg.Backend, + keepRecent: cfg.KeepRecent, + externalPruning: cfg.ExternalPruning, + scheduler: cfg.Scheduler, + snapshotSizes: map[int64]int64{}, + } + m.lastPublished = m.Newest() + m.removeStaleTmpDirs() + m.prune() + if m.lastPublished > 0 { + if err := m.updateCurrentLink(SnapshotDirName(m.lastPublished)); err != nil { + logger.Error("failed to restore state store snapshot current link", + "store", m.name, "version", m.lastPublished, "error", err) + } + recordCurrentHeight(m.name, m.lastPublished) + } + logger.Info("state store member snapshotting enabled", + "store", m.name, + "root", m.root, + "keepRecent", m.keepRecent, + ) + return m, nil +} + +func (m *Manager) Name() string { + if m == nil { + return "" + } + return m.name +} + +func (m *Manager) Root() string { + if m == nil { + return "" + } + return m.root +} + +func (m *Manager) Stage(version int64, shouldRun func() bool, done func(*Staged, error)) error { + name := SnapshotDirName(version) + finalDir := filepath.Join(m.root, name) + if _, err := os.Stat(finalDir); err == nil { + return fmt.Errorf("%s snapshot dir %q already exists", m.name, finalDir) + } else if !os.IsNotExist(err) { + return fmt.Errorf("inspect %s snapshot dir %q: %w", m.name, finalDir, err) + } + tmpDir := filepath.Join(m.root, snapshotTmpPrefix+name) + if err := os.RemoveAll(tmpDir); err != nil { + return fmt.Errorf("clear stale %s snapshot tmp dir: %w", m.name, err) + } + if err := os.MkdirAll(filepath.Dir(tmpDir), 0o750); err != nil { + return fmt.Errorf("create %s snapshot root: %w", m.name, err) + } + staged := &Staged{ + manager: m, + version: version, + tmpDir: tmpDir, + finalDir: finalDir, + } + m.scheduler.ScheduleCheckpoint(tmpDir, shouldRun, func(err error) { + if err != nil { + done(nil, err) + return + } + done(staged, nil) + }) + return nil +} + +func (m *Manager) Commit(staged *Staged) error { + if staged == nil || staged.manager != m { + return fmt.Errorf("%s staged snapshot belongs to a different manager", m.name) + } + apparentBytes, sizeErr := snapshotDirApparentBytes(staged.tmpDir) + if sizeErr != nil { + logger.Error("failed to measure state store snapshot", + "store", m.name, "dir", staged.tmpDir, "error", sizeErr) + } else if err := writeSnapshotSize(staged.tmpDir, apparentBytes); err != nil { + logger.Error("failed to persist state store snapshot size", + "store", m.name, "dir", staged.tmpDir, "error", err) + sizeErr = err + } + + m.publishMu.Lock() + defer m.publishMu.Unlock() + defer m.prune() + + if err := m.scheduler.SetCheckpointVersion(staged.tmpDir, staged.version); err != nil { + _ = os.RemoveAll(staged.tmpDir) + return fmt.Errorf("set %s snapshot version: %w", m.name, err) + } + if err := os.Rename(staged.tmpDir, staged.finalDir); err != nil { + _ = os.RemoveAll(staged.tmpDir) + return fmt.Errorf("finalize %s snapshot: %w", m.name, err) + } + if err := syncDir(m.root); err != nil { + return fmt.Errorf("persist %s snapshot publication: %w", m.name, err) + } + if sizeErr == nil { + if m.snapshotSizes == nil { + m.snapshotSizes = map[int64]int64{} + } + m.snapshotSizes[staged.version] = apparentBytes + } + logger.Info("state store member snapshot created", + "store", m.name, "version", staged.version, "dir", staged.finalDir) + + // Snapshots can finish out of order, so only move the link forward. + if staged.version > m.lastPublished { + if err := m.updateCurrentLink(SnapshotDirName(staged.version)); err != nil { + return fmt.Errorf("update %s snapshot current link: %w", m.name, err) + } + m.lastPublished = staged.version + } + recordCurrentHeight(m.name, m.lastPublished) + return nil +} + +func (m *Manager) Abort(staged *Staged) { + if staged == nil || staged.manager != m { + return + } + if err := os.RemoveAll(staged.tmpDir); err != nil { + logger.Error("failed to remove aborted state store snapshot", + "store", m.name, "dir", staged.tmpDir, "error", err) + } +} + +func (m *Manager) Versions() ([]int64, error) { + if m == nil { + return nil, nil + } + return ListSnapshotVersions(m.root) +} + +func (m *Manager) Newest() int64 { + versions, err := m.Versions() + if err != nil { + logger.Error("failed to list state store snapshots", "store", m.name, "error", err) + return 0 + } + if len(versions) == 0 { + return 0 + } + return versions[len(versions)-1] +} + +func (m *Manager) ModTime(version int64) time.Time { + if version <= 0 { + return time.Time{} + } + info, err := os.Stat(filepath.Join(m.root, SnapshotDirName(version))) + if err != nil { + logger.Error("failed to read state store snapshot modification time", + "store", m.name, "version", version, "error", err) + return time.Time{} + } + return info.ModTime() +} + +func (m *Manager) PruneSnapshots(cutLine int64) error { + if m == nil || m.externalPruning { + return nil + } + versions, err := m.Versions() + if err != nil { + return err + } + currentVersion, hasCurrent, err := m.currentSnapshotVersion() + if err != nil { + return err + } + for _, v := range versions { + if v >= cutLine { + continue + } + if hasCurrent && v == currentVersion { + continue + } + dir := filepath.Join(m.root, SnapshotDirName(v)) + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("remove %s snapshot %q: %w", m.name, dir, err) + } + logger.Info("pruned state store snapshot", "store", m.name, "dir", dir) + } + m.recordRetentionMetrics() + return nil +} + +func (m *Manager) CurrentVersion() (int64, bool, error) { + return m.currentSnapshotVersion() +} + +func (m *Manager) removeStaleTmpDirs() { + tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) + if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) { + logger.Error("failed to remove stale state store snapshot link", + "store", m.name, "path", tmpLink, "error", err) + } + + entries, err := os.ReadDir(m.root) + if err != nil { + if !os.IsNotExist(err) { + logger.Error("failed to scan state store snapshots dir", + "store", m.name, "error", err) + } + return + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), snapshotTmpPrefix) { + continue + } + dir := filepath.Join(m.root, entry.Name()) + if err := os.RemoveAll(dir); err != nil { + logger.Error("failed to remove stale snapshot tmp dir", + "store", m.name, "dir", dir, "error", err) + continue + } + logger.Info("removed stale state store snapshot tmp dir", "store", m.name, "dir", dir) + } +} + +func (m *Manager) updateCurrentLink(name string) error { + tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) + _ = os.Remove(tmpLink) + if err := os.Symlink(name, tmpLink); err != nil { + return fmt.Errorf("create snapshot current symlink: %w", err) + } + if err := os.Rename(tmpLink, filepath.Join(m.root, snapshotCurrentLink)); err != nil { + return fmt.Errorf("swap snapshot current symlink: %w", err) + } + return syncDir(m.root) +} + +func (m *Manager) prune() { + if m.externalPruning { + return + } + versions, err := m.Versions() + if err != nil { + logger.Error("failed to list state store snapshots for pruning", + "store", m.name, "error", err) + return + } + defer m.recordRetentionMetrics() + currentVersion, hasCurrent, err := m.currentSnapshotVersion() + if err != nil { + logger.Error("failed to resolve current state store snapshot before pruning", + "store", m.name, "error", err) + return + } + keep := 1 + m.keepRecent + if len(versions) <= keep { + return + } + for _, v := range versions[:len(versions)-keep] { + if hasCurrent && v == currentVersion { + continue + } + dir := filepath.Join(m.root, SnapshotDirName(v)) + if err := os.RemoveAll(dir); err != nil { + logger.Error("failed to prune state store snapshot", + "store", m.name, "dir", dir, "error", err) + continue + } + logger.Info("pruned state store snapshot", "store", m.name, "dir", dir) + } +} + +func (m *Manager) currentSnapshotVersion() (version int64, exists bool, err error) { + target, err := os.Readlink(filepath.Join(m.root, snapshotCurrentLink)) + if err != nil { + if os.IsNotExist(err) { + return 0, false, nil + } + return 0, false, fmt.Errorf("read current snapshot link: %w", err) + } + version, ok := ParseSnapshotVersion(filepath.Base(target)) + if !ok { + return 0, false, fmt.Errorf("current snapshot link has invalid target %q", target) + } + return version, true, nil +} + +func (m *Manager) recordRetentionMetrics() { + versions, err := m.Versions() + if err != nil { + logger.Error("failed to list state store snapshots for metrics", + "store", m.name, "error", err) + return + } + recordRetainedCount(m.name, int64(len(versions))) + + if m.snapshotSizes == nil { + m.snapshotSizes = map[int64]int64{} + } + retained := make(map[int64]struct{}, len(versions)) + var apparentBytes int64 + for _, version := range versions { + retained[version] = struct{}{} + if size, ok := m.snapshotSizes[version]; ok { + apparentBytes += size + continue + } + dir := filepath.Join(m.root, SnapshotDirName(version)) + size, err := readSnapshotSize(dir) + if err != nil { + if !os.IsNotExist(err) { + logger.Error("failed to read state store snapshot size", + "store", m.name, "dir", dir, "error", err) + } + continue + } + m.snapshotSizes[version] = size + apparentBytes += size + } + for version := range m.snapshotSizes { + if _, ok := retained[version]; !ok { + delete(m.snapshotSizes, version) + } + } + recordApparentBytes(m.name, apparentBytes) +} + +func verifyHardlinks(root string, sourceDirs []string) error { + if err := os.MkdirAll(root, 0o750); err != nil { + return fmt.Errorf("create snapshot root %q: %w", root, err) + } + for _, sourceDir := range sourceDirs { + probe, err := os.CreateTemp(sourceDir, ".ss-snapshot-link-probe-*") + if err != nil { + return fmt.Errorf("create hardlink probe in state store %q: %w", sourceDir, err) + } + source := probe.Name() + if err := probe.Close(); err != nil { + _ = os.Remove(source) + return fmt.Errorf("close hardlink probe in state store %q: %w", sourceDir, err) + } + target := filepath.Join(root, filepath.Base(source)) + if err := os.Link(source, target); err != nil { + _ = os.Remove(source) + return fmt.Errorf( + "state store %q cannot hardlink snapshots into %q; place the SS database and its snapshot root on one filesystem: %w", + sourceDir, + root, + err, + ) + } + if err := os.Remove(source); err != nil { + _ = os.Remove(target) + return fmt.Errorf("remove hardlink probe %q: %w", source, err) + } + if err := os.Remove(target); err != nil { + return fmt.Errorf("remove hardlink probe %q: %w", target, err) + } + } + return nil +} + +func syncDir(path string) error { + // #nosec G304 -- path is an internal database or snapshot directory, not request input. + dir, err := os.Open(path) + if err != nil { + return fmt.Errorf("open directory %q for sync: %w", path, err) + } + syncErr := dir.Sync() + closeErr := dir.Close() + if syncErr != nil { + syncErr = fmt.Errorf("sync directory %q: %w", path, syncErr) + } + if closeErr != nil { + closeErr = fmt.Errorf("close directory %q after sync: %w", path, closeErr) + } + return errors.Join(syncErr, closeErr) +} + +func snapshotDirApparentBytes(dir string) (int64, error) { + var apparentBytes int64 + err := filepath.WalkDir(dir, func(_ string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.Type().IsRegular() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + apparentBytes += info.Size() + return nil + }) + return apparentBytes, err +} + +func writeSnapshotSize(dir string, size int64) error { + path := filepath.Join(dir, snapshotSizeFile) + // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, writeErr := fmt.Fprintf(file, "%d\n", size) + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(writeErr, syncErr, closeErr); err != nil { + return err + } + return syncDir(dir) +} + +func readSnapshotSize(dir string) (int64, error) { + // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. + data, err := os.ReadFile(filepath.Join(dir, snapshotSizeFile)) + if err != nil { + return 0, err + } + size, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse snapshot size in %q: %w", dir, err) + } + if size < 0 { + return 0, fmt.Errorf("snapshot size in %q must be non-negative", dir) + } + return size, nil +} + +var logger = seilog.NewLogger("db", "state-db", "ss", "snapshot") diff --git a/sei-db/state_db/ss/snapshot/manager_test.go b/sei-db/state_db/ss/snapshot/manager_test.go new file mode 100644 index 0000000000..4edd9e04f1 --- /dev/null +++ b/sei-db/state_db/ss/snapshot/manager_test.go @@ -0,0 +1,141 @@ +package snapshot + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/management" + "github.com/stretchr/testify/require" +) + +type controlledScheduler struct { + pending chan func() + fail bool +} + +func (*controlledScheduler) SupportsCheckpoint() bool { + return true +} + +func (s *controlledScheduler) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { + s.pending <- func() { + if !shouldRun() { + done(management.ErrCheckpointCanceled) + return + } + if s.fail { + done(errors.New("checkpoint failed")) + return + } + _ = os.MkdirAll(destDir, 0o750) + done(nil) + } +} + +func (*controlledScheduler) SetCheckpointVersion(string, int64) error { + return nil +} + +func TestManagerCommitPublishesCurrentAndPrunesByCount(t *testing.T) { + root := t.TempDir() + scheduler := &controlledScheduler{pending: make(chan func(), 1)} + manager := openManager(t, root, scheduler, 1, false) + + stageAndCommit(t, manager, scheduler, 10) + stageAndCommit(t, manager, scheduler, 20) + stageAndCommit(t, manager, scheduler, 30) + + require.NoDirExists(t, filepath.Join(root, SnapshotDirName(10))) + require.DirExists(t, filepath.Join(root, SnapshotDirName(20))) + require.DirExists(t, filepath.Join(root, SnapshotDirName(30))) + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(30), target) +} + +func TestManagerPruneKeepsCurrentTarget(t *testing.T) { + root := t.TempDir() + for _, version := range []int64{5, 10, 15} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + require.NoError(t, os.Symlink(SnapshotDirName(5), filepath.Join(root, snapshotCurrentLink))) + manager := openManager(t, root, &controlledScheduler{pending: make(chan func(), 1)}, 1, false) + + versions, err := manager.Versions() + require.NoError(t, err) + require.Equal(t, []int64{5, 10, 15}, versions) +} + +func TestManagerPruneSnapshotsByHeight(t *testing.T) { + root := t.TempDir() + for _, version := range []int64{5, 10, 15} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + manager := openManager(t, root, &controlledScheduler{pending: make(chan func(), 1)}, 10, false) + + require.NoError(t, manager.PruneSnapshots(12)) + + versions, err := manager.Versions() + require.NoError(t, err) + require.Equal(t, []int64{15}, versions) +} + +func TestManagerExternalPruningStandsDownInternalRetention(t *testing.T) { + root := t.TempDir() + scheduler := &controlledScheduler{pending: make(chan func(), 1)} + manager := openManager(t, root, scheduler, 0, true) + + stageAndCommit(t, manager, scheduler, 10) + stageAndCommit(t, manager, scheduler, 20) + require.NoError(t, manager.PruneSnapshots(20)) + + versions, err := manager.Versions() + require.NoError(t, err) + require.Equal(t, []int64{10, 20}, versions) +} + +func TestManagerAbortRemovesStagedSnapshot(t *testing.T) { + root := t.TempDir() + scheduler := &controlledScheduler{pending: make(chan func(), 1)} + manager := openManager(t, root, scheduler, 1, false) + var staged *Staged + + require.NoError(t, manager.Stage(10, func() bool { return true }, func(s *Staged, err error) { + require.NoError(t, err) + staged = s + })) + (<-scheduler.pending)() + staged.Abort() + + require.NoDirExists(t, filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(10))) + require.NoDirExists(t, filepath.Join(root, SnapshotDirName(10))) +} + +func openManager(t *testing.T, root string, scheduler *controlledScheduler, keepRecent int, external bool) *Manager { + t.Helper() + manager, err := Open(Config{ + Name: "test", + Root: root, + SourceDirs: []string{t.TempDir()}, + Backend: config.PebbleDBBackend, + KeepRecent: keepRecent, + ExternalPruning: external, + Scheduler: scheduler, + }) + require.NoError(t, err) + return manager +} + +func stageAndCommit(t *testing.T, manager *Manager, scheduler *controlledScheduler, version int64) { + t.Helper() + var staged *Staged + require.NoError(t, manager.Stage(version, func() bool { return true }, func(s *Staged, err error) { + require.NoError(t, err) + staged = s + })) + (<-scheduler.pending)() + require.NoError(t, manager.Commit(staged)) +} diff --git a/sei-db/state_db/ss/composite/snapshot_metrics.go b/sei-db/state_db/ss/snapshot/metrics.go similarity index 54% rename from sei-db/state_db/ss/composite/snapshot_metrics.go rename to sei-db/state_db/ss/snapshot/metrics.go index d908055f01..0d1beedaf1 100644 --- a/sei-db/state_db/ss/composite/snapshot_metrics.go +++ b/sei-db/state_db/ss/snapshot/metrics.go @@ -1,4 +1,4 @@ -package composite +package snapshot import ( "context" @@ -11,53 +11,58 @@ import ( commonmetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" ) -var snapshotMeter = otel.Meter("seidb_ss_snapshot") +var meter = otel.Meter("seidb_ss_snapshot") -var snapshotMetrics = struct { +var metrics = struct { Attempts metric.Int64Counter Skipped metric.Int64Counter Completions metric.Int64Counter Duration metric.Float64Histogram InFlight metric.Int64Gauge CurrentHeight metric.Int64Gauge + CommonHeight metric.Int64Gauge RetainedCount metric.Int64Gauge ApparentBytes metric.Int64Gauge }{ - Attempts: must(snapshotMeter.Int64Counter( + Attempts: must(meter.Int64Counter( "ss_snapshot_attempts", metric.WithDescription("Number of state-store snapshot attempts"), metric.WithUnit("{count}"), )), - Skipped: must(snapshotMeter.Int64Counter( + Skipped: must(meter.Int64Counter( "ss_snapshot_skipped", metric.WithDescription("Number of state-store snapshot boundaries skipped by a scheduling gate"), metric.WithUnit("{count}"), )), - Completions: must(snapshotMeter.Int64Counter( + Completions: must(meter.Int64Counter( "ss_snapshot_completions", metric.WithDescription("Number of completed state-store snapshot attempts"), metric.WithUnit("{count}"), )), - Duration: must(snapshotMeter.Float64Histogram( + Duration: must(meter.Float64Histogram( "ss_snapshot_duration", metric.WithDescription("Time from a state-store snapshot request to completion"), metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(commonmetrics.LongLatencyBuckets...), )), - InFlight: must(snapshotMeter.Int64Gauge( + InFlight: must(meter.Int64Gauge( "ss_snapshot_in_flight", metric.WithDescription("Whether one state-store snapshot is currently in flight"), )), - CurrentHeight: must(snapshotMeter.Int64Gauge( + CurrentHeight: must(meter.Int64Gauge( "ss_snapshot_current_height", metric.WithDescription("Height of the newest published state-store snapshot"), )), - RetainedCount: must(snapshotMeter.Int64Gauge( + CommonHeight: must(meter.Int64Gauge( + "ss_snapshot_common_height", + metric.WithDescription("Newest state-store snapshot height present in every enabled SS member"), + )), + RetainedCount: must(meter.Int64Gauge( "ss_snapshot_retained_count", metric.WithDescription("Number of retained state-store snapshots"), metric.WithUnit("{count}"), )), - ApparentBytes: must(snapshotMeter.Int64Gauge( + ApparentBytes: must(meter.Int64Gauge( "ss_snapshot_retained_apparent_bytes", metric.WithDescription("Apparent bytes referenced by retained state-store snapshots; hardlinks can share physical blocks"), metric.WithUnit("By"), @@ -71,24 +76,52 @@ func must[V any](instrument V, err error) V { return instrument } -func recordSnapshotAttempt() { - snapshotMetrics.Attempts.Add(context.Background(), 1) +func RecordAttempt() { + metrics.Attempts.Add(context.Background(), 1) } -func recordSnapshotSkipped(reason string) { - snapshotMetrics.Skipped.Add( +func RecordSkipped(reason string) { + metrics.Skipped.Add( context.Background(), 1, metric.WithAttributes(attribute.String("reason", reason)), ) } -func recordSnapshotInFlight(value int64) { - snapshotMetrics.InFlight.Record(context.Background(), value) +func RecordInFlight(value int64) { + metrics.InFlight.Record(context.Background(), value) } -func recordSnapshotCompletion(start time.Time, outcome string) { +func RecordCompletion(start time.Time, outcome string) { attrs := metric.WithAttributes(attribute.String("outcome", outcome)) - snapshotMetrics.Completions.Add(context.Background(), 1, attrs) - snapshotMetrics.Duration.Record(context.Background(), time.Since(start).Seconds(), attrs) + metrics.Completions.Add(context.Background(), 1, attrs) + metrics.Duration.Record(context.Background(), time.Since(start).Seconds(), attrs) +} + +func RecordCommonHeight(height int64) { + metrics.CommonHeight.Record(context.Background(), height) +} + +func recordCurrentHeight(store string, height int64) { + metrics.CurrentHeight.Record( + context.Background(), + height, + metric.WithAttributes(attribute.String("store", store)), + ) +} + +func recordRetainedCount(store string, count int64) { + metrics.RetainedCount.Record( + context.Background(), + count, + metric.WithAttributes(attribute.String("store", store)), + ) +} + +func recordApparentBytes(store string, bytes int64) { + metrics.ApparentBytes.Record( + context.Background(), + bytes, + metric.WithAttributes(attribute.String("store", store)), + ) } From 55cb87d1224bb228ce6eda49d0fb0719114919ac Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 14 Aug 2026 15:03:22 -0400 Subject: [PATCH 5/8] fix(seidb): let the SS snapshot cut line act under external pruning PruneSnapshots is the entry point an external collector prunes an SS through, and gc.PrunableStore only calls it when ExternalPruning reports true. Standing it down in that mode made it a silent no-op, so snapshots would grow without bound once the collector owned retention. Only internal count-based retention stands down now, which is what ExternalPruning is for. Co-authored-by: Cursor --- sei-db/state_db/ss/snapshot/manager.go | 5 ++++- sei-db/state_db/ss/snapshot/manager_test.go | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go index bc73741b2f..7fe13b3b7f 100644 --- a/sei-db/state_db/ss/snapshot/manager.go +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -291,8 +291,11 @@ func (m *Manager) ModTime(version int64) time.Time { return info.ModTime() } +// PruneSnapshots deletes every snapshot below cutLine, never the current one. It acts whether or not +// retention is external: an external collector prunes this store through here, and it is the internal +// count-based retention that stands down instead. func (m *Manager) PruneSnapshots(cutLine int64) error { - if m == nil || m.externalPruning { + if m == nil { return nil } versions, err := m.Versions() diff --git a/sei-db/state_db/ss/snapshot/manager_test.go b/sei-db/state_db/ss/snapshot/manager_test.go index 4edd9e04f1..96590d0338 100644 --- a/sei-db/state_db/ss/snapshot/manager_test.go +++ b/sei-db/state_db/ss/snapshot/manager_test.go @@ -90,11 +90,28 @@ func TestManagerExternalPruningStandsDownInternalRetention(t *testing.T) { stageAndCommit(t, manager, scheduler, 10) stageAndCommit(t, manager, scheduler, 20) + + versions, err := manager.Versions() + require.NoError(t, err) + require.Equal(t, []int64{10, 20}, versions, + "keepRecent would have dropped 10 if internal retention still ran") +} + +// PruneSnapshots is how an external collector prunes this store, and external pruning is the only mode +// it is called in, so the cut line must be honored there. The current snapshot survives it. +func TestManagerPruneSnapshotsActsUnderExternalPruning(t *testing.T) { + root := t.TempDir() + scheduler := &controlledScheduler{pending: make(chan func(), 1)} + manager := openManager(t, root, scheduler, 0, true) + + stageAndCommit(t, manager, scheduler, 10) + stageAndCommit(t, manager, scheduler, 20) + require.NoError(t, manager.PruneSnapshots(20)) versions, err := manager.Versions() require.NoError(t, err) - require.Equal(t, []int64{10, 20}, versions) + require.Equal(t, []int64{20}, versions) } func TestManagerAbortRemovesStagedSnapshot(t *testing.T) { From 6a4be669b9190c8231f4efa7291775e4f396c192 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 14 Aug 2026 15:13:37 -0400 Subject: [PATCH 6/8] fix(seidb): never half-queue an SS snapshot request A member that could not be staged left barriers queued for the members before it, while the coordinator released the version for another attempt. The retry then cleared the staging directory the queued barrier was writing into, and published whatever the interrupted checkpoint had reached under a label claiming to hold every version up to it. Staging is now split: every member reserves its directory before any barrier is queued, so a request either queues all of them or none. Preparing also refuses a staging directory that already exists rather than clearing it, since a checkpoint may still be writing there; Open clears the ones a crash left behind. A failed checkpoint now aborts the staging directories of its peers instead of leaving them until the next startup. Co-authored-by: Cursor --- sei-db/state_db/ss/composite/snapshot.go | 43 ++++++++++++------- sei-db/state_db/ss/composite/snapshot_test.go | 25 +++++++++++ sei-db/state_db/ss/snapshot/manager.go | 42 +++++++++++------- sei-db/state_db/ss/snapshot/manager_test.go | 39 ++++++++++++----- 4 files changed, 107 insertions(+), 42 deletions(-) diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 1e65b670a0..7de58f2c90 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -189,6 +189,9 @@ func (c *snapshotCoordinator) maybeSnapshot(version int64) { start := time.Now() sssnapshot.RecordAttempt() if err := c.requestSnapshot(version, start); err != nil { + // requestSnapshot only reports an error before it queues anything, so this version is free to + // be requested again: no barrier is writing under its label, and the write path has enqueued + // nothing above it yet, which is what keeps a retry within the same block exact. sssnapshot.RecordCompletion(start, "failure") c.mu.Lock() if c.lastRequested == version { @@ -216,6 +219,23 @@ func (c *snapshotCoordinator) requestSnapshot(version int64, start time.Time) er if len(c.members) == 0 { return errors.New("no state store snapshot members") } + // Every member is prepared before any of them is scheduled, so a member that cannot take this + // version fails the whole request with nothing queued. Half-queued requests are what let a retry + // of the same version reach a staging directory a barrier from the first attempt still writes to. + staged := make([]*sssnapshot.Staged, len(c.members)) + for i, member := range c.members { + if member.manager == nil { + abortStaged(staged) + return fmt.Errorf("state store snapshot member %q has no manager", member.name) + } + s, err := member.manager.Prepare(version) + if err != nil { + abortStaged(staged) + return fmt.Errorf("prepare %s snapshot: %w", member.name, err) + } + staged[i] = s + } + var canceled atomic.Bool shouldRun := func() bool { return c.isRunning() && !canceled.Load() @@ -224,21 +244,18 @@ func (c *snapshotCoordinator) requestSnapshot(version int64, start time.Time) er var ( mu sync.Mutex remaining = len(c.members) - staged = make([]*sssnapshot.Staged, len(c.members)) firstErr error ) for i, member := range c.members { - if member.manager == nil { - canceled.Store(true) - return fmt.Errorf("state store snapshot member %q has no manager", member.name) - } - err := member.manager.Stage(version, shouldRun, func(s *sssnapshot.Staged, err error) { + member.manager.Schedule(staged[i], shouldRun, func(err error) { mu.Lock() - if err != nil && firstErr == nil { - firstErr = fmt.Errorf("stage %s snapshot: %w", member.name, err) - } - if s != nil { - staged[i] = s + if err != nil { + if firstErr == nil { + firstErr = fmt.Errorf("stage %s snapshot: %w", member.name, err) + } + // A snapshot only publishes when every member has it, so a peer still queued has + // nothing left to produce. + canceled.Store(true) } remaining-- last, outcome := remaining == 0, firstErr @@ -248,10 +265,6 @@ func (c *snapshotCoordinator) requestSnapshot(version int64, start time.Time) er } c.startPublish(version, staged, outcome, start) }) - if err != nil { - canceled.Store(true) - return err - } } return nil } diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index 19746b2ad5..f5c0c75eb1 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -235,6 +235,31 @@ func TestCompositeCoordinatorAbortsEveryMemberOnStageFailure(t *testing.T) { require.NoDirExists(t, filepath.Join(rootB, SnapshotDirName(10))) } +// A request that fails must queue no barrier at all. The commit path can call maybeSnapshot twice for +// one block, so a failed request releases its version for another attempt; a barrier left behind by the +// first attempt would then be writing into the staging directory the retry reserves. +func TestCompositeCoordinatorQueuesNothingWhenAMemberCannotPrepare(t *testing.T) { + rootA, rootB := t.TempDir(), t.TempDir() + schedulerA := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + schedulerB := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + managerA := openTestManager(t, "cosmos", rootA, schedulerA) + managerB := openTestManager(t, "evm", rootB, schedulerB) + coord := newSnapshotCoordinator(10, 0, []snapshotMember{ + {name: "cosmos", manager: managerA}, + {name: "evm", manager: managerB}, + }) + // Occupy the name the evm member would publish version 10 under, so its prepare fails after the + // cosmos member has already been prepared. + require.NoError(t, os.WriteFile(filepath.Join(rootB, SnapshotDirName(10)), nil, 0o600)) + + coord.maybeSnapshot(10) + + require.Empty(t, schedulerA.pending, "a member must not be scheduled once another cannot prepare") + require.Empty(t, schedulerB.pending) + require.Equal(t, int64(0), coord.lastRequested, "the version stays available for another attempt") + require.False(t, coord.inFlight) +} + func TestUnpairedSnapshotHeightsSurviveStartupAndCommonHeightIsLower(t *testing.T) { rootA, rootB := t.TempDir(), t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(rootA, SnapshotDirName(10)), 0o750)) diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go index 7fe13b3b7f..8d6e8c8ba1 100644 --- a/sei-db/state_db/ss/snapshot/manager.go +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -169,35 +169,45 @@ func (m *Manager) Root() string { return m.root } -func (m *Manager) Stage(version int64, shouldRun func() bool, done func(*Staged, error)) error { +// Prepare reserves a staging directory for version without queueing any work. A caller that snapshots +// several members prepares all of them before it schedules any, so a member that cannot be prepared +// leaves no barrier queued for a version nothing will publish. +func (m *Manager) Prepare(version int64) (*Staged, error) { name := SnapshotDirName(version) finalDir := filepath.Join(m.root, name) if _, err := os.Stat(finalDir); err == nil { - return fmt.Errorf("%s snapshot dir %q already exists", m.name, finalDir) + return nil, fmt.Errorf("%s snapshot dir %q already exists", m.name, finalDir) } else if !os.IsNotExist(err) { - return fmt.Errorf("inspect %s snapshot dir %q: %w", m.name, finalDir, err) + return nil, fmt.Errorf("inspect %s snapshot dir %q: %w", m.name, finalDir, err) } tmpDir := filepath.Join(m.root, snapshotTmpPrefix+name) - if err := os.RemoveAll(tmpDir); err != nil { - return fmt.Errorf("clear stale %s snapshot tmp dir: %w", m.name, err) + // Refused rather than cleared: a checkpoint may still be writing into it, and deleting underneath + // one leaves a partial directory that is then published under an exact label. Open clears the + // staging directories a crash left behind, so a stale one does not block the next boundary. + if _, err := os.Stat(tmpDir); err == nil { + return nil, fmt.Errorf("%s snapshot staging dir %q already exists", m.name, tmpDir) + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("inspect %s snapshot staging dir %q: %w", m.name, tmpDir, err) } - if err := os.MkdirAll(filepath.Dir(tmpDir), 0o750); err != nil { - return fmt.Errorf("create %s snapshot root: %w", m.name, err) + if err := os.MkdirAll(m.root, 0o750); err != nil { + return nil, fmt.Errorf("create %s snapshot root: %w", m.name, err) } - staged := &Staged{ + return &Staged{ manager: m, version: version, tmpDir: tmpDir, finalDir: finalDir, + }, nil +} + +// Schedule queues staged's checkpoint behind the writes already enqueued on this member's backend. +// done is called exactly once, so a caller counting members can wait on it. +func (m *Manager) Schedule(staged *Staged, shouldRun func() bool, done func(error)) { + if staged == nil || staged.manager != m { + done(fmt.Errorf("%s staged snapshot belongs to a different manager", m.name)) + return } - m.scheduler.ScheduleCheckpoint(tmpDir, shouldRun, func(err error) { - if err != nil { - done(nil, err) - return - } - done(staged, nil) - }) - return nil + m.scheduler.ScheduleCheckpoint(staged.tmpDir, shouldRun, done) } func (m *Manager) Commit(staged *Staged) error { diff --git a/sei-db/state_db/ss/snapshot/manager_test.go b/sei-db/state_db/ss/snapshot/manager_test.go index 96590d0338..fdeb149c2f 100644 --- a/sei-db/state_db/ss/snapshot/manager_test.go +++ b/sei-db/state_db/ss/snapshot/manager_test.go @@ -118,19 +118,30 @@ func TestManagerAbortRemovesStagedSnapshot(t *testing.T) { root := t.TempDir() scheduler := &controlledScheduler{pending: make(chan func(), 1)} manager := openManager(t, root, scheduler, 1, false) - var staged *Staged - require.NoError(t, manager.Stage(10, func() bool { return true }, func(s *Staged, err error) { - require.NoError(t, err) - staged = s - })) - (<-scheduler.pending)() + staged := stage(t, manager, scheduler, 10) staged.Abort() require.NoDirExists(t, filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(10))) require.NoDirExists(t, filepath.Join(root, SnapshotDirName(10))) } +// A staging directory that already exists may hold a checkpoint still being written, so preparing the +// same version again must refuse rather than clear it: clearing it publishes whatever the interrupted +// checkpoint had reached under a label that claims to be exact. +func TestManagerPrepareRefusesExistingStagingDir(t *testing.T) { + root := t.TempDir() + scheduler := &controlledScheduler{pending: make(chan func(), 1)} + manager := openManager(t, root, scheduler, 1, false) + + staged := stage(t, manager, scheduler, 10) + require.DirExists(t, staged.tmpDir) + + _, err := manager.Prepare(10) + require.Error(t, err) + require.DirExists(t, staged.tmpDir, "the first attempt's checkpoint must survive the refusal") +} + func openManager(t *testing.T, root string, scheduler *controlledScheduler, keepRecent int, external bool) *Manager { t.Helper() manager, err := Open(Config{ @@ -148,11 +159,17 @@ func openManager(t *testing.T, root string, scheduler *controlledScheduler, keep func stageAndCommit(t *testing.T, manager *Manager, scheduler *controlledScheduler, version int64) { t.Helper() - var staged *Staged - require.NoError(t, manager.Stage(version, func() bool { return true }, func(s *Staged, err error) { + staged := stage(t, manager, scheduler, version) + require.NoError(t, manager.Commit(staged)) +} + +func stage(t *testing.T, manager *Manager, scheduler *controlledScheduler, version int64) *Staged { + t.Helper() + staged, err := manager.Prepare(version) + require.NoError(t, err) + manager.Schedule(staged, func() bool { return true }, func(err error) { require.NoError(t, err) - staged = s - })) + }) (<-scheduler.pending)() - require.NoError(t, manager.Commit(staged)) + return staged } From bc0b3861756fb08c5ae73109a112927a0c92a61d Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 14 Aug 2026 15:26:20 -0400 Subject: [PATCH 7/8] fix(seidb): keep the SS snapshot height every member shares MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each member counts only its own snapshot directories, so an unpaired newer height took the keep slot the newest shared height occupied and retention deleted it, in Open and after every publish. A restore reads the newest height every member holds, so that left nothing to restore from until the next fully paired publication. The coordinator now hands the members a shared floor — the newest height they all hold — which retention keeps regardless of the keep window, and republishes it after every publication. It is resolved before the members open, because opening one runs its retention. Also: the hardlink preflight probe takes a fixed name and is cleared on both sides before use, so a crash mid-probe leaves one known path per directory instead of accumulating; and pebbledb reports consecutive prune failures, since every pass raises the earliest-version marker before it deletes and repeated failures narrow the served range while nothing leaves disk. Co-authored-by: Cursor --- sei-db/db_engine/pebbledb/mvcc/db.go | 18 ++++ .../db_engine/pebbledb/mvcc/db_ascending.go | 1 + sei-db/db_engine/pebbledb/mvcc/metrics.go | 10 ++ sei-db/state_db/ss/composite/snapshot.go | 49 ++++----- sei-db/state_db/ss/composite/snapshot_test.go | 43 ++++++-- sei-db/state_db/ss/composite/store.go | 23 ++-- sei-db/state_db/ss/cosmos/store.go | 8 +- sei-db/state_db/ss/evm/store.go | 7 +- sei-db/state_db/ss/snapshot/manager.go | 102 ++++++++++++++++-- sei-db/state_db/ss/snapshot/manager_test.go | 57 ++++++++++ 10 files changed, 270 insertions(+), 48 deletions(-) diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index d4ea4c2f94..462692165c 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -92,6 +92,9 @@ type Database struct { // that has gone idle since would be skipped for as long as it stays idle. pruneIncomplete atomic.Bool + // pruneFailures counts the prune passes that have failed in a row, reported as a gauge. + pruneFailures atomic.Int64 + // Changelog used to support async write streamHandler wal.ChangelogWAL @@ -706,6 +709,20 @@ func (db *Database) getDescending(storeKey string, targetVersion int64, key []by // the same process rescan every store to reach them. A crash inside that window // loses the flag, and those rows stay on disk — unreachable by any read, since // the marker bounds reads too — until the store is written to again. +// recordPruneOutcome reports how many prune passes have failed in a row. A failed pass has already +// raised the earliest-version marker, so repeated failures narrow the range reads and checkpoints are +// allowed to serve once per prune interval while disk keeps growing. Nothing else makes that visible: +// the rows left behind are unreachable, so the store looks consistent from the outside. +func (db *Database) recordPruneOutcome(err error) { + failures := int64(0) + if err == nil { + db.pruneFailures.Store(0) + } else { + failures = db.pruneFailures.Add(1) + } + otelMetrics.pruneConsecutiveFailures.Record(context.Background(), failures) +} + func (db *Database) pruneDescending(version int64) (_err error) { // Defensive check: ensure database is not closed if db.storage == nil { @@ -714,6 +731,7 @@ func (db *Database) pruneDescending(version int64) (_err error) { startTime := time.Now() defer func() { + db.recordPruneOutcome(_err) otelMetrics.pruneLatency.Record( context.Background(), time.Since(startTime).Seconds(), diff --git a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go index 6932d06fd2..534f7c373f 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go @@ -98,6 +98,7 @@ func (db *Database) pruneAscending(version int64) (_err error) { startTime := time.Now() defer func() { + db.recordPruneOutcome(_err) otelMetrics.pruneLatency.Record( context.Background(), time.Since(startTime).Seconds(), diff --git a/sei-db/db_engine/pebbledb/mvcc/metrics.go b/sei-db/db_engine/pebbledb/mvcc/metrics.go index 6c1d6f2297..7a7a6dddad 100644 --- a/sei-db/db_engine/pebbledb/mvcc/metrics.go +++ b/sei-db/db_engine/pebbledb/mvcc/metrics.go @@ -13,6 +13,7 @@ var ( applyChangesetLatency metric.Float64Histogram applyChangesetAsyncLatency metric.Float64Histogram pruneLatency metric.Float64Histogram + pruneConsecutiveFailures metric.Int64Gauge importLatency metric.Float64Histogram batchWriteLatency metric.Float64Histogram @@ -58,6 +59,15 @@ var ( metric.WithDescription("Time taken to prune old versions from PebbleDB"), metric.WithUnit("s"), )), + pruneConsecutiveFailures: must(meter.Int64Gauge( + "pebble_prune_consecutive_failures", + metric.WithDescription( + "Prune passes that have failed in a row. Every pass raises the earliest-version marker "+ + "before it deletes, so a rising value means the served history window is narrowing "+ + "once per prune interval while nothing leaves disk", + ), + metric.WithUnit("{count}"), + )), importLatency: must(meter.Float64Histogram( "pebble_import_latency", metric.WithDescription("Time taken to import snapshot data to PebbleDB"), diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 7de58f2c90..8788e11525 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -76,6 +76,9 @@ type snapshotCoordinator struct { interval int64 minTime time.Duration members []snapshotMember + // floor carries the newest height every member holds to the members' own retention, which counts + // only its own directories and would otherwise let an unpaired newer height crowd it out. + floor *sssnapshot.Floor mu sync.Mutex // lastRequested is the newest label already requested or present in any @@ -92,19 +95,33 @@ type snapshotCoordinator struct { publishing sync.WaitGroup } -func newSnapshotCoordinator(interval int64, minTime time.Duration, members []snapshotMember) *snapshotCoordinator { +func newSnapshotCoordinator( + interval int64, + minTime time.Duration, + members []snapshotMember, + floor *sssnapshot.Floor, +) *snapshotCoordinator { c := &snapshotCoordinator{ interval: interval, minTime: minTime, members: members, + floor: floor, } c.lastRequested = newestMemberSnapshot(members) c.lastRequestAt = newestMemberSnapshotModTime(members, c.lastRequested) - sssnapshot.RecordCommonHeight(newestCommonSnapshot(members)) + c.recordCommonHeight() return c } -func (s *CompositeStateStore) startSnapshotManager(members []snapshotMember) error { +// recordCommonHeight republishes the height every member holds, which is both what retention must keep +// and what an operator watches to see the members drifting apart. +func (c *snapshotCoordinator) recordCommonHeight() { + common := newestCommonSnapshot(c.members) + c.floor.Set(common) + sssnapshot.RecordCommonHeight(common) +} + +func (s *CompositeStateStore) startSnapshotManager(members []snapshotMember, floor *sssnapshot.Floor) error { if s.config.SnapshotInterval <= 0 { return nil } @@ -120,6 +137,7 @@ func (s *CompositeStateStore) startSnapshotManager(members []snapshotMember) err s.config.SnapshotInterval, s.config.SnapshotMinTimeInterval, members, + floor, ) logger.Info("state store snapshotting enabled", "interval", s.config.SnapshotInterval, @@ -210,7 +228,7 @@ func (c *snapshotCoordinator) finishSnapshot() { c.inFlight = false sssnapshot.RecordInFlight(0) c.mu.Unlock() - sssnapshot.RecordCommonHeight(newestCommonSnapshot(c.members)) + c.recordCommonHeight() } // requestSnapshot asks every member to checkpoint itself into a staging @@ -352,29 +370,12 @@ func newestMemberSnapshotModTime(members []snapshotMember, version int64) time.T } func newestCommonSnapshot(members []snapshotMember) int64 { - if len(members) == 0 { - return 0 - } - counts := map[int64]int{} + roots := make([]string, 0, len(members)) for _, member := range members { if member.manager == nil { return 0 } - versions, err := member.manager.Versions() - if err != nil { - logger.Error("failed to list state store snapshots for common height", - "member", member.name, "error", err) - return 0 - } - for _, version := range versions { - counts[version]++ - } - } - var newest int64 - for version, count := range counts { - if count == len(members) && version > newest { - newest = version - } + roots = append(roots, member.manager.Root()) } - return newest + return sssnapshot.NewestCommonVersion(roots) } diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index f5c0c75eb1..92e855cb70 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -200,7 +200,7 @@ func TestCompositeCoordinatorPublishesEveryMember(t *testing.T) { schedulerB := &controlledSnapshotScheduler{pending: make(chan func(), 1)} managerA := openTestManager(t, "cosmos", rootA, schedulerA) managerB := openTestManager(t, "evm", rootB, schedulerB) - coord := newSnapshotCoordinator(10, 0, []snapshotMember{ + coord := newTestSnapshotCoordinator(10, 0, []snapshotMember{ {name: "cosmos", manager: managerA}, {name: "evm", manager: managerB}, }) @@ -221,7 +221,7 @@ func TestCompositeCoordinatorAbortsEveryMemberOnStageFailure(t *testing.T) { schedulerB := &controlledSnapshotScheduler{pending: make(chan func(), 1), fail: true} managerA := openTestManager(t, "cosmos", rootA, schedulerA) managerB := openTestManager(t, "evm", rootB, schedulerB) - coord := newSnapshotCoordinator(10, 0, []snapshotMember{ + coord := newTestSnapshotCoordinator(10, 0, []snapshotMember{ {name: "cosmos", manager: managerA}, {name: "evm", manager: managerB}, }) @@ -244,7 +244,7 @@ func TestCompositeCoordinatorQueuesNothingWhenAMemberCannotPrepare(t *testing.T) schedulerB := &controlledSnapshotScheduler{pending: make(chan func(), 1)} managerA := openTestManager(t, "cosmos", rootA, schedulerA) managerB := openTestManager(t, "evm", rootB, schedulerB) - coord := newSnapshotCoordinator(10, 0, []snapshotMember{ + coord := newTestSnapshotCoordinator(10, 0, []snapshotMember{ {name: "cosmos", manager: managerA}, {name: "evm", manager: managerB}, }) @@ -266,16 +266,23 @@ func TestUnpairedSnapshotHeightsSurviveStartupAndCommonHeightIsLower(t *testing. require.NoError(t, os.MkdirAll(filepath.Join(rootA, SnapshotDirName(20)), 0o750)) require.NoError(t, os.MkdirAll(filepath.Join(rootB, SnapshotDirName(10)), 0o750)) - managerA := openTestManager(t, "cosmos", rootA, &controlledSnapshotScheduler{pending: make(chan func(), 1)}) - managerB := openTestManager(t, "evm", rootB, &controlledSnapshotScheduler{pending: make(chan func(), 1)}) + // Retention that keeps one snapshot would drop 10 from the cosmos root, where the unpaired 20 holds + // the only keep slot, leaving the members with no height in common to restore from. + floor := sssnapshot.NewFloor(sssnapshot.NewestCommonVersion([]string{rootA, rootB})) + schedulerA := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + schedulerB := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + managerA := openTestManagerWithRetention(t, "cosmos", rootA, schedulerA, 0, floor) + managerB := openTestManagerWithRetention(t, "evm", rootB, schedulerB, 0, floor) coord := newSnapshotCoordinator(10, 0, []snapshotMember{ {name: "cosmos", manager: managerA}, {name: "evm", manager: managerB}, - }) + }, floor) + require.DirExists(t, filepath.Join(rootA, SnapshotDirName(10)), "the shared height must survive retention") require.DirExists(t, filepath.Join(rootA, SnapshotDirName(20))) require.Equal(t, int64(20), coord.lastRequested) require.Equal(t, int64(10), newestCommonSnapshot(coord.members)) + require.Equal(t, int64(10), floor.Height()) } func TestSnapshotCoversEmptyBlock(t *testing.T) { @@ -295,7 +302,7 @@ func TestSnapshotMinTimeIntervalSkipsBoundary(t *testing.T) { rootA := t.TempDir() scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 1)} manager := openTestManager(t, "cosmos", rootA, scheduler) - coord := newSnapshotCoordinator(10, time.Hour, []snapshotMember{{name: "cosmos", manager: manager}}) + coord := newTestSnapshotCoordinator(10, time.Hour, []snapshotMember{{name: "cosmos", manager: manager}}) coord.maybeSnapshot(10) (<-scheduler.pending)() @@ -307,6 +314,17 @@ func TestSnapshotMinTimeIntervalSkipsBoundary(t *testing.T) { } func openTestManager(t *testing.T, name, root string, scheduler *controlledSnapshotScheduler) *sssnapshot.Manager { + t.Helper() + return openTestManagerWithRetention(t, name, root, scheduler, 1, nil) +} + +func openTestManagerWithRetention( + t *testing.T, + name, root string, + scheduler *controlledSnapshotScheduler, + keepRecent int, + floor *sssnapshot.Floor, +) *sssnapshot.Manager { t.Helper() source := t.TempDir() manager, err := sssnapshot.Open(sssnapshot.Config{ @@ -314,13 +332,22 @@ func openTestManager(t *testing.T, name, root string, scheduler *controlledSnaps Root: root, SourceDirs: []string{source}, Backend: config.PebbleDBBackend, - KeepRecent: 1, + KeepRecent: keepRecent, Scheduler: scheduler, + Floor: floor, }) require.NoError(t, err) return manager } +func newTestSnapshotCoordinator( + interval int64, + minTime time.Duration, + members []snapshotMember, +) *snapshotCoordinator { + return newSnapshotCoordinator(interval, minTime, members, sssnapshot.NewFloor(0)) +} + type cosmosStoreWithSnapshots interface { Snapshots() *sssnapshot.Manager } diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index 9007f0b2dd..dba80ff811 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -20,6 +20,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/cosmos" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/pruning" + sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" "github.com/sei-protocol/sei-chain/sei-db/wal" "github.com/sei-protocol/seilog" ) @@ -103,7 +104,6 @@ func NewCompositeStateStore( cs.validateEVMSSPostRecovery() if ssConfig.SnapshotInterval > 0 { - members := make([]snapshotMember, 0, 2) cosmosSnapshotRoot := utils.GetStateStoreSnapshotsPath(homeDir) if ssConfig.DBDirectory != "" { cleanDBHome := filepath.Clean(dbHome) @@ -112,20 +112,31 @@ func NewCompositeStateStore( filepath.Base(cleanDBHome)+"-"+utils.StateStoreSnapshotsDirName, ) } - if err := cosmosStore.StartSnapshots(cosmosSnapshotRoot, []string{dbHome}, ssConfig); err != nil { + evmStore, hasEVM := cs.evmStore.(*evm.EVMStateStore) + var evmSnapshotRoot string + roots := []string{cosmosSnapshotRoot} + if hasEVM { + evmSnapshotRoot = evmStore.Dir() + "-" + utils.StateStoreSnapshotsDirName + roots = append(roots, evmSnapshotRoot) + } + // Resolved before any member opens, because opening one runs its retention, and the height a + // restore would start from is the newest the members share rather than the newest either holds. + floor := sssnapshot.NewFloor(sssnapshot.NewestCommonVersion(roots)) + + members := make([]snapshotMember, 0, len(roots)) + if err := cosmosStore.StartSnapshots(cosmosSnapshotRoot, []string{dbHome}, ssConfig, floor); err != nil { _ = cs.Close() return nil, fmt.Errorf("start Cosmos state store snapshot manager: %w", err) } members = append(members, snapshotMember{name: "cosmos", manager: cosmosStore.Snapshots()}) - if evmStore, ok := cs.evmStore.(*evm.EVMStateStore); ok { - evmSnapshotRoot := evmStore.Dir() + "-" + utils.StateStoreSnapshotsDirName - if err := evmStore.StartSnapshots(evmSnapshotRoot, ssConfig); err != nil { + if hasEVM { + if err := evmStore.StartSnapshots(evmSnapshotRoot, ssConfig, floor); err != nil { _ = cs.Close() return nil, fmt.Errorf("start EVM state store snapshot manager: %w", err) } members = append(members, snapshotMember{name: "evm", manager: evmStore.Snapshots()}) } - if err := cs.startSnapshotManager(members); err != nil { + if err := cs.startSnapshotManager(members, floor); err != nil { _ = cs.Close() return nil, fmt.Errorf("start state store snapshot manager: %w", err) } diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 291023957a..8254d8c3ce 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -105,7 +105,12 @@ func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) e return management.SetCheckpointVersion(s.db, destDir, version) } -func (s *CosmosStateStore) StartSnapshots(root string, sourceDirs []string, ssConfig config.StateStoreConfig) error { +func (s *CosmosStateStore) StartSnapshots( + root string, + sourceDirs []string, + ssConfig config.StateStoreConfig, + floor *sssnapshot.Floor, +) error { manager, err := sssnapshot.Open(sssnapshot.Config{ Name: "cosmos", Root: root, @@ -114,6 +119,7 @@ func (s *CosmosStateStore) StartSnapshots(root string, sourceDirs []string, ssCo KeepRecent: ssConfig.SnapshotKeepRecent, ExternalPruning: ssConfig.ExternalPruning, Scheduler: s, + Floor: floor, }) if err != nil { return err diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 4a6bd7f295..94fc67e174 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -480,7 +480,11 @@ func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) erro return nil } -func (s *EVMStateStore) StartSnapshots(root string, ssConfig config.StateStoreConfig) error { +func (s *EVMStateStore) StartSnapshots( + root string, + ssConfig config.StateStoreConfig, + floor *sssnapshot.Floor, +) error { manager, err := sssnapshot.Open(sssnapshot.Config{ Name: "evm", Root: root, @@ -489,6 +493,7 @@ func (s *EVMStateStore) StartSnapshots(root string, ssConfig config.StateStoreCo KeepRecent: ssConfig.SnapshotKeepRecent, ExternalPruning: ssConfig.ExternalPruning, Scheduler: s, + Floor: floor, }) if err != nil { return err diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go index 8d6e8c8ba1..e199dfac5f 100644 --- a/sei-db/state_db/ss/snapshot/manager.go +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/sei-protocol/sei-chain/sei-db/common/utils" @@ -30,6 +31,7 @@ const ( snapshotCurrentTmpLink = "current-tmp" snapshotTmpPrefix = "tmp-" snapshotSizeFile = ".apparent-size" + linkProbeName = ".ss-snapshot-link-probe" ) // Config wires one SS member into a snapshot Manager. @@ -41,6 +43,67 @@ type Config struct { KeepRecent int ExternalPruning bool Scheduler management.CheckpointScheduler + // Floor names a height this member's retention must keep. Leave it nil when the member is the only + // one that has to hold the height a restore starts from. + Floor *Floor +} + +// Floor is a height retention must keep, shared by the members of a coordinated snapshot set. A +// restore reads the newest height every member holds, and a member counting its own directories cannot +// see the other roots: an unpaired newer directory would otherwise consume the keep slot that height +// occupies. The coordinator publishes the shared height here after every publication. +// +// Keeping it costs one directory beyond KeepRecent while the members disagree. +type Floor struct { + height atomic.Int64 +} + +// NewFloor returns a Floor holding height, which may be 0 for "no height to keep". +func NewFloor(height int64) *Floor { + f := &Floor{} + f.Set(height) + return f +} + +func (f *Floor) Height() int64 { + if f == nil { + return 0 + } + return f.height.Load() +} + +func (f *Floor) Set(height int64) { + if f == nil { + return + } + f.height.Store(height) +} + +// NewestCommonVersion returns the newest snapshot height present under every root, 0 when the roots +// share none. A root that cannot be read is treated as holding nothing, so the answer never names a +// height that may be absent. +func NewestCommonVersion(roots []string) int64 { + if len(roots) == 0 { + return 0 + } + counts := map[int64]int{} + for _, root := range roots { + versions, err := ListSnapshotVersions(root) + if err != nil { + logger.Error("failed to list state store snapshots", "root", root, "error", err) + return 0 + } + for _, v := range versions { + counts[v]++ + } + } + var newest int64 + for version, count := range counts { + if count == len(roots) && version > newest { + newest = version + } + } + return newest } // Manager owns one SS member's snapshot root, staging directories, current @@ -53,6 +116,7 @@ type Manager struct { keepRecent int externalPruning bool scheduler management.CheckpointScheduler + floor *Floor snapshotSizes map[int64]int64 publishMu sync.Mutex @@ -135,6 +199,7 @@ func Open(cfg Config) (*Manager, error) { keepRecent: cfg.KeepRecent, externalPruning: cfg.ExternalPruning, scheduler: cfg.Scheduler, + floor: cfg.Floor, snapshotSizes: map[int64]int64{}, } m.lastPublished = m.Newest() @@ -316,6 +381,7 @@ func (m *Manager) PruneSnapshots(cutLine int64) error { if err != nil { return err } + floor := m.floor.Height() for _, v := range versions { if v >= cutLine { continue @@ -323,6 +389,9 @@ func (m *Manager) PruneSnapshots(cutLine int64) error { if hasCurrent && v == currentVersion { continue } + if v == floor { + continue + } dir := filepath.Join(m.root, SnapshotDirName(v)) if err := os.RemoveAll(dir); err != nil { return fmt.Errorf("remove %s snapshot %q: %w", m.name, dir, err) @@ -399,10 +468,14 @@ func (m *Manager) prune() { if len(versions) <= keep { return } + floor := m.floor.Height() for _, v := range versions[:len(versions)-keep] { if hasCurrent && v == currentVersion { continue } + if v == floor { + continue + } dir := filepath.Join(m.root, SnapshotDirName(v)) if err := os.RemoveAll(dir); err != nil { logger.Error("failed to prune state store snapshot", @@ -468,21 +541,27 @@ func (m *Manager) recordRetentionMetrics() { recordApparentBytes(m.name, apparentBytes) } +// verifyHardlinks proves the snapshot root can hold hardlinks to each source directory, which is how a +// checkpoint avoids copying the database. +// +// The probe has a fixed name in both places so a process that dies between the link and the removals +// leaves one known path per directory, which the next start reclaims rather than accumulating. func verifyHardlinks(root string, sourceDirs []string) error { if err := os.MkdirAll(root, 0o750); err != nil { return fmt.Errorf("create snapshot root %q: %w", root, err) } for _, sourceDir := range sourceDirs { - probe, err := os.CreateTemp(sourceDir, ".ss-snapshot-link-probe-*") - if err != nil { - return fmt.Errorf("create hardlink probe in state store %q: %w", sourceDir, err) + source := filepath.Join(sourceDir, linkProbeName) + target := filepath.Join(root, linkProbeName) + if err := removeIfExists(source); err != nil { + return fmt.Errorf("clear hardlink probe %q: %w", source, err) } - source := probe.Name() - if err := probe.Close(); err != nil { - _ = os.Remove(source) - return fmt.Errorf("close hardlink probe in state store %q: %w", sourceDir, err) + if err := removeIfExists(target); err != nil { + return fmt.Errorf("clear hardlink probe %q: %w", target, err) + } + if err := os.WriteFile(source, nil, 0o600); err != nil { + return fmt.Errorf("create hardlink probe in state store %q: %w", sourceDir, err) } - target := filepath.Join(root, filepath.Base(source)) if err := os.Link(source, target); err != nil { _ = os.Remove(source) return fmt.Errorf( @@ -503,6 +582,13 @@ func verifyHardlinks(root string, sourceDirs []string) error { return nil } +func removeIfExists(path string) error { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + func syncDir(path string) error { // #nosec G304 -- path is an internal database or snapshot directory, not request input. dir, err := os.Open(path) diff --git a/sei-db/state_db/ss/snapshot/manager_test.go b/sei-db/state_db/ss/snapshot/manager_test.go index fdeb149c2f..c29747653e 100644 --- a/sei-db/state_db/ss/snapshot/manager_test.go +++ b/sei-db/state_db/ss/snapshot/manager_test.go @@ -114,6 +114,50 @@ func TestManagerPruneSnapshotsActsUnderExternalPruning(t *testing.T) { require.Equal(t, []int64{20}, versions) } +// Retention counts only this member's directories, so a height an unpaired newer snapshot has pushed out +// of the keep window can still be the newest one every member holds — the height a restore starts from. +func TestManagerRetentionKeepsTheSharedFloor(t *testing.T) { + root := t.TempDir() + for _, version := range []int64{10, 20, 30} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + floor := NewFloor(10) + scheduler := &controlledScheduler{pending: make(chan func(), 1)} + manager := openManagerWithFloor(t, root, scheduler, 0, false, floor) + + versions, err := manager.Versions() + require.NoError(t, err) + require.Equal(t, []int64{10, 30}, versions, "only the unpaired 20 is beyond the keep window") + + // The members agree again, so the height they used to share is free to go. + floor.Set(30) + require.NoError(t, manager.PruneSnapshots(30)) + + versions, err = manager.Versions() + require.NoError(t, err) + require.Equal(t, []int64{30}, versions) +} + +// A hardlink probe left by a crash is reclaimed rather than accumulating, in the source directory and in +// the snapshot root alike. +func TestOpenClearsLeftoverHardlinkProbes(t *testing.T) { + root, source := t.TempDir(), t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(source, linkProbeName), nil, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, linkProbeName), nil, 0o600)) + + _, err := Open(Config{ + Name: "test", + Root: root, + SourceDirs: []string{source}, + Backend: config.PebbleDBBackend, + Scheduler: &controlledScheduler{pending: make(chan func(), 1)}, + }) + require.NoError(t, err) + + require.NoFileExists(t, filepath.Join(source, linkProbeName)) + require.NoFileExists(t, filepath.Join(root, linkProbeName)) +} + func TestManagerAbortRemovesStagedSnapshot(t *testing.T) { root := t.TempDir() scheduler := &controlledScheduler{pending: make(chan func(), 1)} @@ -143,6 +187,18 @@ func TestManagerPrepareRefusesExistingStagingDir(t *testing.T) { } func openManager(t *testing.T, root string, scheduler *controlledScheduler, keepRecent int, external bool) *Manager { + t.Helper() + return openManagerWithFloor(t, root, scheduler, keepRecent, external, nil) +} + +func openManagerWithFloor( + t *testing.T, + root string, + scheduler *controlledScheduler, + keepRecent int, + external bool, + floor *Floor, +) *Manager { t.Helper() manager, err := Open(Config{ Name: "test", @@ -152,6 +208,7 @@ func openManager(t *testing.T, root string, scheduler *controlledScheduler, keep KeepRecent: keepRecent, ExternalPruning: external, Scheduler: scheduler, + Floor: floor, }) require.NoError(t, err) return manager From d04377c52a2a7db18da186058792c53b90638cd5 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 14 Aug 2026 15:54:52 -0400 Subject: [PATCH 8/8] refactor(seidb): name the patterns the SS snapshot path repeats Each of these was written twice by the snapshot work, so a later change had two places to keep in agreement: - the fan-in over parallel checkpoints, now management.FanIn - the checkpoint capability test, now management.SupportsCheckpoint - the retention delete loop, now Manager.removeSnapshots - the prune marker prologue, now beginPrunePass/endPrunePass, which also holds the skip-baseline reasoning that four comments carried - the "-snapshots" sibling root, now one path helper - the pending-write capability, now types.PendingWriteWaiter PruneSnapshots takes publishMu, because publication renames a directory in and swaps current under it, so a cut line resolved a moment earlier could otherwise delete what current now names. Retention attempts every candidate and joins the errors, which is what the count-based path already did. Co-authored-by: Cursor --- sei-db/common/utils/path.go | 7 ++ sei-db/db_engine/pebbledb/mvcc/db.go | 79 +++++++++++-------- .../db_engine/pebbledb/mvcc/db_ascending.go | 24 +----- sei-db/db_engine/types/types.go | 10 ++- sei-db/management/checkpoint_scheduler.go | 35 ++++++++ sei-db/state_db/ss/composite/snapshot.go | 40 ++-------- sei-db/state_db/ss/composite/store.go | 9 +-- sei-db/state_db/ss/cosmos/store.go | 7 +- sei-db/state_db/ss/evm/store.go | 50 +++++------- sei-db/state_db/ss/snapshot/manager.go | 69 +++++++--------- 10 files changed, 157 insertions(+), 173 deletions(-) diff --git a/sei-db/common/utils/path.go b/sei-db/common/utils/path.go index d073091bad..6b8ffb021d 100644 --- a/sei-db/common/utils/path.go +++ b/sei-db/common/utils/path.go @@ -70,6 +70,13 @@ func GetStateStoreSnapshotsPath(homePath string) string { return filepath.Join(homePath, "data", "state_store", StateStoreSnapshotsDirName) } +// GetStateStoreSnapshotsSiblingPath returns the snapshot root beside an SS database directory, which is +// where that member's snapshots live when its database sits outside the default home layout. Beside +// rather than inside, because a checkpoint hardlinks into it and the two must share a filesystem. +func GetStateStoreSnapshotsSiblingPath(dbDir string) string { + return filepath.Clean(dbDir) + "-" + StateStoreSnapshotsDirName +} + // GetReceiptStorePath returns the path for the receipt store. // New nodes use data/ledger/receipt/{backend}; existing nodes with // data/receipt.db continue using the legacy path for backward compatibility. diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index 462692165c..6e80b16f2b 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -434,6 +434,47 @@ func (db *Database) advanceEarliestVersion(target int64) error { return db.SetEarliestVersion(target, false) } +// beginPrunePass raises the earliest-version marker to earliestVersion and returns the version below +// which an unmodified store's keys may be skipped. +// +// The baseline is the marker as it stood before this pass: the raised value would skip every store whose +// latest update is at or below the prune height. It is 0 when an earlier pass failed after raising the +// marker, since the rows it left behind are below that marker and an idle store would never be revisited. +func (db *Database) beginPrunePass(earliestVersion int64) (skipBelow int64, err error) { + skipBelow = db.GetEarliestVersion() + if err := db.advanceEarliestVersion(earliestVersion); err != nil { + return 0, err + } + if db.pruneIncomplete.Load() { + skipBelow = 0 + } + db.pruneIncomplete.Store(true) + return skipBelow, nil +} + +// endPrunePass clears the incomplete flag for a pass that finished its deletes, and reports the pass +// outcome. A failed pass leaves the flag set, so the next one rescans every store. +func (db *Database) endPrunePass(err error) { + if err == nil { + db.pruneIncomplete.Store(false) + } + db.recordPruneOutcome(err) +} + +// recordPruneOutcome reports how many prune passes have failed in a row. A failed pass has already +// raised the earliest-version marker, so repeated failures narrow the range reads and checkpoints may +// serve, once per prune interval, while disk keeps growing. Nothing else makes that visible: the rows +// left behind are unreachable, so the store looks consistent from the outside. +func (db *Database) recordPruneOutcome(err error) { + failures := int64(0) + if err == nil { + db.pruneFailures.Store(0) + } else { + failures = db.pruneFailures.Add(1) + } + otelMetrics.pruneConsecutiveFailures.Record(context.Background(), failures) +} + // Retrieves earliest version from db, if not found, return 0 func retrieveEarliestVersion(db *pebble.DB) (int64, error) { return retrieveVersionKey(db, earliestVersionKey) @@ -709,20 +750,6 @@ func (db *Database) getDescending(storeKey string, targetVersion int64, key []by // the same process rescan every store to reach them. A crash inside that window // loses the flag, and those rows stay on disk — unreachable by any read, since // the marker bounds reads too — until the store is written to again. -// recordPruneOutcome reports how many prune passes have failed in a row. A failed pass has already -// raised the earliest-version marker, so repeated failures narrow the range reads and checkpoints are -// allowed to serve once per prune interval while disk keeps growing. Nothing else makes that visible: -// the rows left behind are unreachable, so the store looks consistent from the outside. -func (db *Database) recordPruneOutcome(err error) { - failures := int64(0) - if err == nil { - db.pruneFailures.Store(0) - } else { - failures = db.pruneFailures.Add(1) - } - otelMetrics.pruneConsecutiveFailures.Record(context.Background(), failures) -} - func (db *Database) pruneDescending(version int64) (_err error) { // Defensive check: ensure database is not closed if db.storage == nil { @@ -731,7 +758,7 @@ func (db *Database) pruneDescending(version int64) (_err error) { startTime := time.Now() defer func() { - db.recordPruneOutcome(_err) + db.endPrunePass(_err) otelMetrics.pruneLatency.Record( context.Background(), time.Since(startTime).Seconds(), @@ -742,22 +769,10 @@ func (db *Database) pruneDescending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version - skipBelow := db.GetEarliestVersion() - if err := db.advanceEarliestVersion(earliestVersion); err != nil { + skipBelow, err := db.beginPrunePass(earliestVersion) + if err != nil { return err } - if db.pruneIncomplete.Load() { - // A previous pass raised the marker and then stopped short of its - // deletes, so the marker no longer bounds what is on disk. Scan every - // store to reach the rows it left behind. - skipBelow = 0 - } - db.pruneIncomplete.Store(true) - defer func() { - if _err == nil { - db.pruneIncomplete.Store(false) - } - }() itr, err := db.storage.NewIter(nil) if err != nil { @@ -804,11 +819,7 @@ func (db *Database) pruneDescending(version int64) (_err error) { prevStore = storeKey updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) - // The marker is advanced before deletes so checkpoints never claim - // history that the prune has already dropped. skipBelow is the marker - // as it stood before this pass raised it; comparing against the raised - // value would skip every store whose latest update is at or below the - // prune height. + // skipBelow is the marker as it stood before this pass raised it; see beginPrunePass. if !ok || (typeOk && versionUpdated < skipBelow) { itr.SeekGE(storePrefix(storeKey + "0")) continue diff --git a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go index 534f7c373f..6237466ab4 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go @@ -98,7 +98,7 @@ func (db *Database) pruneAscending(version int64) (_err error) { startTime := time.Now() defer func() { - db.recordPruneOutcome(_err) + db.endPrunePass(_err) otelMetrics.pruneLatency.Record( context.Background(), time.Since(startTime).Seconds(), @@ -109,22 +109,10 @@ func (db *Database) pruneAscending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version - skipBelow := db.GetEarliestVersion() - if err := db.advanceEarliestVersion(earliestVersion); err != nil { + skipBelow, err := db.beginPrunePass(earliestVersion) + if err != nil { return err } - if db.pruneIncomplete.Load() { - // A previous pass raised the marker and then stopped short of its - // deletes, so the marker no longer bounds what is on disk. Scan every - // store to reach the rows it left behind. - skipBelow = 0 - } - db.pruneIncomplete.Store(true) - defer func() { - if _err == nil { - db.pruneIncomplete.Store(false) - } - }() itr, err := db.storage.NewIter(nil) if err != nil { @@ -171,11 +159,7 @@ func (db *Database) pruneAscending(version int64) (_err error) { prevStore = storeKey updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) - // The marker is advanced before deletes so checkpoints never claim - // history that the prune has already dropped. skipBelow is the marker - // as it stood before this pass raised it; comparing against the raised - // value would skip every store whose latest update is at or below the - // prune height. + // skipBelow is the marker as it stood before this pass raised it; see beginPrunePass. if !ok || (typeOk && versionUpdated < skipBelow) { itr.SeekGE(storePrefix(storeKey + "0")) continue diff --git a/sei-db/db_engine/types/types.go b/sei-db/db_engine/types/types.go index 24856dc539..66b3ad2f19 100644 --- a/sei-db/db_engine/types/types.go +++ b/sei-db/db_engine/types/types.go @@ -139,8 +139,14 @@ type DrainBarrier interface { ScheduleAtDrain(fn func()) } -// The three interfaces above are engine capabilities. Deciding when a checkpoint -// runs, and what version it is labeled with, is coordination rather than engine +// PendingWriteWaiter is an optional capability for engines that apply changesets +// from an async queue. It blocks until the queue is empty. +type PendingWriteWaiter interface { + WaitForPendingWrites() +} + +// The interfaces above are engine capabilities. Deciding when a checkpoint runs, +// and what version it is labeled with, is coordination rather than engine // behavior and lives in sei-db/management: CheckpointScheduler, // ScheduleCheckpoint, SetCheckpointVersion and ErrCheckpointCanceled. diff --git a/sei-db/management/checkpoint_scheduler.go b/sei-db/management/checkpoint_scheduler.go index 8e68bb92b9..0471ebcd8d 100644 --- a/sei-db/management/checkpoint_scheduler.go +++ b/sei-db/management/checkpoint_scheduler.go @@ -6,6 +6,7 @@ package management import ( "errors" "fmt" + "sync" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" ) @@ -26,6 +27,40 @@ type CheckpointScheduler interface { // it started. var ErrCheckpointCanceled = errors.New("state store checkpoint canceled") +// SupportsCheckpoint reports whether db carries every engine capability a scheduled checkpoint needs. +// A store built from several engines answers for the set: one engine short of the capabilities makes +// the whole snapshot unpublishable. +func SupportsCheckpoint(db types.StateStore) bool { + _, checkpointable := db.(types.Checkpointable) + _, barrier := db.(types.DrainBarrier) + _, versionSetter := db.(types.CheckpointVersionSetter) + return checkpointable && barrier && versionSetter +} + +// FanIn returns a report callback for n parallel branches. Each branch calls it once, and the last +// call passes done the first error any branch reported, or nil. +func FanIn(n int, done func(error)) func(error) { + var ( + mu sync.Mutex + remaining = n + firstErr error + ) + return func(err error) { + mu.Lock() + if err != nil && firstErr == nil { + firstErr = err + } + remaining-- + isLast := remaining == 0 + // Read under the lock: a peer branch may report between the unlock and the call to done. + outcome := firstErr + mu.Unlock() + if isLast { + done(outcome) + } + } +} + // ScheduleCheckpoint checkpoints an engine after all writes already enqueued // on it have been applied. func ScheduleCheckpoint(db types.StateStore, destDir string, shouldRun func() bool, done func(error)) { diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 8788e11525..71c77e270e 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -51,11 +51,7 @@ import ( // If a future tool opens or copies these directories directly, it must first add // a lease or other hold mechanism. -const ( - SnapshotsDirName = sssnapshot.SnapshotsDirName - snapshotCurrentLink = "current" - snapshotTmpPrefix = "tmp-" -) +const SnapshotsDirName = sssnapshot.SnapshotsDirName func SnapshotDirName(version int64) string { return sssnapshot.SnapshotDirName(version) @@ -242,10 +238,6 @@ func (c *snapshotCoordinator) requestSnapshot(version int64, start time.Time) er // of the same version reach a staging directory a barrier from the first attempt still writes to. staged := make([]*sssnapshot.Staged, len(c.members)) for i, member := range c.members { - if member.manager == nil { - abortStaged(staged) - return fmt.Errorf("state store snapshot member %q has no manager", member.name) - } s, err := member.manager.Prepare(version) if err != nil { abortStaged(staged) @@ -259,29 +251,18 @@ func (c *snapshotCoordinator) requestSnapshot(version int64, start time.Time) er return c.isRunning() && !canceled.Load() } - var ( - mu sync.Mutex - remaining = len(c.members) - firstErr error - ) + report := management.FanIn(len(c.members), func(err error) { + c.startPublish(version, staged, err, start) + }) for i, member := range c.members { member.manager.Schedule(staged[i], shouldRun, func(err error) { - mu.Lock() if err != nil { - if firstErr == nil { - firstErr = fmt.Errorf("stage %s snapshot: %w", member.name, err) - } // A snapshot only publishes when every member has it, so a peer still queued has // nothing left to produce. canceled.Store(true) + err = fmt.Errorf("stage %s snapshot: %w", member.name, err) } - remaining-- - last, outcome := remaining == 0, firstErr - mu.Unlock() - if !last { - return - } - c.startPublish(version, staged, outcome, start) + report(err) }) } return nil @@ -347,9 +328,6 @@ func abortStaged(staged []*sssnapshot.Staged) { func newestMemberSnapshot(members []snapshotMember) int64 { var newest int64 for _, member := range members { - if member.manager == nil { - continue - } newest = max(newest, member.manager.Newest()) } return newest @@ -358,9 +336,6 @@ func newestMemberSnapshot(members []snapshotMember) int64 { func newestMemberSnapshotModTime(members []snapshotMember, version int64) time.Time { var newest time.Time for _, member := range members { - if member.manager == nil { - continue - } modTime := member.manager.ModTime(version) if modTime.After(newest) { newest = modTime @@ -372,9 +347,6 @@ func newestMemberSnapshotModTime(members []snapshotMember, version int64) time.T func newestCommonSnapshot(members []snapshotMember) int64 { roots := make([]string, 0, len(members)) for _, member := range members { - if member.manager == nil { - return 0 - } roots = append(roots, member.manager.Root()) } return sssnapshot.NewestCommonVersion(roots) diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index dba80ff811..212c801c23 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -4,7 +4,6 @@ import ( "encoding/binary" "fmt" "os" - "path/filepath" "sync" dbm "github.com/tendermint/tm-db" @@ -106,17 +105,13 @@ func NewCompositeStateStore( if ssConfig.SnapshotInterval > 0 { cosmosSnapshotRoot := utils.GetStateStoreSnapshotsPath(homeDir) if ssConfig.DBDirectory != "" { - cleanDBHome := filepath.Clean(dbHome) - cosmosSnapshotRoot = filepath.Join( - filepath.Dir(cleanDBHome), - filepath.Base(cleanDBHome)+"-"+utils.StateStoreSnapshotsDirName, - ) + cosmosSnapshotRoot = utils.GetStateStoreSnapshotsSiblingPath(dbHome) } evmStore, hasEVM := cs.evmStore.(*evm.EVMStateStore) var evmSnapshotRoot string roots := []string{cosmosSnapshotRoot} if hasEVM { - evmSnapshotRoot = evmStore.Dir() + "-" + utils.StateStoreSnapshotsDirName + evmSnapshotRoot = utils.GetStateStoreSnapshotsSiblingPath(evmStore.Dir()) roots = append(roots, evmSnapshotRoot) } // Resolved before any member opens, because opening one runs its retention, and the height a diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 8254d8c3ce..75fc232490 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -91,10 +91,7 @@ func (s *CosmosStateStore) Close() error { } func (s *CosmosStateStore) SupportsCheckpoint() bool { - _, checkpointable := s.db.(types.Checkpointable) - _, barrier := s.db.(types.DrainBarrier) - _, versionSetter := s.db.(types.CheckpointVersionSetter) - return checkpointable && barrier && versionSetter + return management.SupportsCheckpoint(s.db) } func (s *CosmosStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { @@ -134,7 +131,7 @@ func (s *CosmosStateStore) Snapshots() *sssnapshot.Manager { } func (s *CosmosStateStore) WaitForPendingWrites() { - if w, ok := s.db.(interface{ WaitForPendingWrites() }); ok { + if w, ok := s.db.(types.PendingWriteWaiter); ok { w.WaitForPendingWrites() } } diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 94fc67e174..c8d70d04d6 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -377,13 +377,20 @@ func (s *EVMStateStore) snapshotSourceDirs() []string { if !s.separateDBs { return []string{s.dir} } - dirs := make([]string, 0, len(AllEVMStoreTypes())) - for _, storeType := range AllEVMStoreTypes() { - dirs = append(dirs, filepath.Join(s.dir, StoreTypeName(storeType))) + storeTypes := AllEVMStoreTypes() + dirs := make([]string, 0, len(storeTypes)) + for _, storeType := range storeTypes { + dirs = append(dirs, subDBPath(s.dir, storeType)) } return dirs } +// subDBPath returns the directory a sub-DB occupies under base. The live database, a checkpoint, and a +// snapshot source all use this layout. +func subDBPath(base string, storeType EVMStoreType) string { + return filepath.Join(base, StoreTypeName(storeType)) +} + func (s *EVMStateStore) Close() error { var lastErr error for _, db := range s.managedDBs { @@ -396,13 +403,7 @@ func (s *EVMStateStore) Close() error { func (s *EVMStateStore) SupportsCheckpoint() bool { for _, db := range s.managedDBs { - if _, checkpointable := db.(types.Checkpointable); !checkpointable { - return false - } - if _, barrier := db.(types.DrainBarrier); !barrier { - return false - } - if _, versionSetter := db.(types.CheckpointVersionSetter); !versionSetter { + if !management.SupportsCheckpoint(db) { return false } } @@ -440,25 +441,14 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool } storeTypes := AllEVMStoreTypes() - var ( - mu sync.Mutex - remaining = len(storeTypes) - firstErr error - ) + report := management.FanIn(len(storeTypes), done) for _, storeType := range storeTypes { name := StoreTypeName(storeType) - dest := filepath.Join(destDir, name) - management.ScheduleCheckpoint(s.subDBs[storeType], dest, shouldRun, func(err error) { - mu.Lock() - if err != nil && firstErr == nil { - firstErr = fmt.Errorf("checkpoint EVM sub-DB %s: %w", name, err) - } - remaining-- - last, outcome := remaining == 0, firstErr - mu.Unlock() - if last { - done(outcome) + management.ScheduleCheckpoint(s.subDBs[storeType], subDBPath(destDir, storeType), shouldRun, func(err error) { + if err != nil { + err = fmt.Errorf("checkpoint EVM sub-DB %s: %w", name, err) } + report(err) }) } } @@ -472,9 +462,9 @@ func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) erro return management.SetCheckpointVersion(db, destDir, version) } for _, storeType := range AllEVMStoreTypes() { - dest := filepath.Join(destDir, StoreTypeName(storeType)) - if err := management.SetCheckpointVersion(s.subDBs[storeType], dest, version); err != nil { - return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", StoreTypeName(storeType), err) + name := StoreTypeName(storeType) + if err := management.SetCheckpointVersion(s.subDBs[storeType], subDBPath(destDir, storeType), version); err != nil { + return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", name, err) } } return nil @@ -509,7 +499,7 @@ func (s *EVMStateStore) Snapshots() *sssnapshot.Manager { func (s *EVMStateStore) WaitForPendingWrites() { for _, db := range s.managedDBs { - if w, ok := db.(interface{ WaitForPendingWrites() }); ok { + if w, ok := db.(types.PendingWriteWaiter); ok { w.WaitForPendingWrites() } } diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go index e199dfac5f..1b6c085254 100644 --- a/sei-db/state_db/ss/snapshot/manager.go +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -136,7 +136,7 @@ func (s *Staged) Abort() { if s == nil || s.manager == nil { return } - s.manager.Abort(s) + s.manager.abort(s) } // SnapshotDirName returns the directory name for a snapshot labeled with the @@ -305,9 +305,6 @@ func (m *Manager) Commit(staged *Staged) error { return fmt.Errorf("persist %s snapshot publication: %w", m.name, err) } if sizeErr == nil { - if m.snapshotSizes == nil { - m.snapshotSizes = map[int64]int64{} - } m.snapshotSizes[staged.version] = apparentBytes } logger.Info("state store member snapshot created", @@ -324,7 +321,7 @@ func (m *Manager) Commit(staged *Staged) error { return nil } -func (m *Manager) Abort(staged *Staged) { +func (m *Manager) abort(staged *Staged) { if staged == nil || staged.manager != m { return } @@ -373,37 +370,46 @@ func (m *Manager) PruneSnapshots(cutLine int64) error { if m == nil { return nil } + // Publication renames a directory in and swaps current under this lock, so retention has to hold it + // too: otherwise a cut line resolved a moment earlier can delete what current now names. + m.publishMu.Lock() + defer m.publishMu.Unlock() + defer m.recordRetentionMetrics() + versions, err := m.Versions() if err != nil { return err } + candidates := make([]int64, 0, len(versions)) + for _, v := range versions { + if v < cutLine { + candidates = append(candidates, v) + } + } + return m.removeSnapshots(candidates) +} + +// removeSnapshots deletes each candidate except the current snapshot and the shared floor. Every +// candidate is attempted, the removals being independent of each other. +func (m *Manager) removeSnapshots(candidates []int64) error { currentVersion, hasCurrent, err := m.currentSnapshotVersion() if err != nil { return err } floor := m.floor.Height() - for _, v := range versions { - if v >= cutLine { - continue - } - if hasCurrent && v == currentVersion { - continue - } - if v == floor { + var errs []error + for _, v := range candidates { + if (hasCurrent && v == currentVersion) || v == floor { continue } dir := filepath.Join(m.root, SnapshotDirName(v)) if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("remove %s snapshot %q: %w", m.name, dir, err) + errs = append(errs, fmt.Errorf("remove %s snapshot %q: %w", m.name, dir, err)) + continue } logger.Info("pruned state store snapshot", "store", m.name, "dir", dir) } - m.recordRetentionMetrics() - return nil -} - -func (m *Manager) CurrentVersion() (int64, bool, error) { - return m.currentSnapshotVersion() + return errors.Join(errs...) } func (m *Manager) removeStaleTmpDirs() { @@ -458,31 +464,12 @@ func (m *Manager) prune() { return } defer m.recordRetentionMetrics() - currentVersion, hasCurrent, err := m.currentSnapshotVersion() - if err != nil { - logger.Error("failed to resolve current state store snapshot before pruning", - "store", m.name, "error", err) - return - } keep := 1 + m.keepRecent if len(versions) <= keep { return } - floor := m.floor.Height() - for _, v := range versions[:len(versions)-keep] { - if hasCurrent && v == currentVersion { - continue - } - if v == floor { - continue - } - dir := filepath.Join(m.root, SnapshotDirName(v)) - if err := os.RemoveAll(dir); err != nil { - logger.Error("failed to prune state store snapshot", - "store", m.name, "dir", dir, "error", err) - continue - } - logger.Info("pruned state store snapshot", "store", m.name, "dir", dir) + if err := m.removeSnapshots(versions[:len(versions)-keep]); err != nil { + logger.Error("failed to prune state store snapshots", "store", m.name, "error", err) } }