From 7649c185f6a56cfac497dd7841419b0a4a46a644 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 14 Aug 2026 15:37:45 -0500 Subject: [PATCH] Remove metadata DB --- sei-db/state_db/sc/composite/flatkv_needed.go | 60 ----- .../sc/composite/flatkv_needed_test.go | 223 ------------------ sei-db/state_db/sc/composite/store.go | 140 ++++------- .../state_db/sc/composite/store_auto_test.go | 41 ++-- .../state_db/sc/composite/store_load_test.go | 93 ++++++++ sei-db/state_db/sc/composite/store_test.go | 80 +------ sei-db/state_db/sc/flatkv/api.go | 8 - sei-db/state_db/sc/flatkv/config/config.go | 16 +- .../state_db/sc/flatkv/config/config_test.go | 2 - .../sc/flatkv/config/flatkv_test_config.go | 2 - sei-db/state_db/sc/flatkv/hashlog.go | 3 +- sei-db/state_db/sc/flatkv/ktype/meta.go | 12 +- .../state_db/sc/flatkv/perdb_lthash_test.go | 35 ++- sei-db/state_db/sc/flatkv/snapshot.go | 113 ++------- sei-db/state_db/sc/flatkv/snapshot_test.go | 120 +--------- sei-db/state_db/sc/flatkv/store.go | 124 +++++----- sei-db/state_db/sc/flatkv/store_iteration.go | 2 +- sei-db/state_db/sc/flatkv/store_lifecycle.go | 11 +- sei-db/state_db/sc/flatkv/store_meta.go | 159 ++----------- sei-db/state_db/sc/flatkv/store_meta_test.go | 187 +++++---------- sei-db/state_db/sc/flatkv/store_replay.go | 39 ++- .../state_db/sc/flatkv/store_replay_test.go | 67 +++++- sei-db/state_db/sc/flatkv/store_test.go | 47 +--- sei-db/state_db/sc/flatkv/store_write.go | 31 +-- sei-db/state_db/sc/flatkv/testutil_test.go | 19 ++ sei-db/state_db/sc/flatkv/verify.go | 36 +++ .../tools/cmd/seidb/operations/dump_flatkv.go | 133 +---------- .../cmd/seidb/operations/dump_flatkv_test.go | 79 +------ 28 files changed, 531 insertions(+), 1351 deletions(-) delete mode 100644 sei-db/state_db/sc/composite/flatkv_needed.go delete mode 100644 sei-db/state_db/sc/composite/flatkv_needed_test.go create mode 100644 sei-db/state_db/sc/composite/store_load_test.go diff --git a/sei-db/state_db/sc/composite/flatkv_needed.go b/sei-db/state_db/sc/composite/flatkv_needed.go deleted file mode 100644 index b72d7750e1..0000000000 --- a/sei-db/state_db/sc/composite/flatkv_needed.go +++ /dev/null @@ -1,60 +0,0 @@ -package composite - -import ( - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" -) - -// FlatKVNeededAtHeight reports whether flatkv holds any consensus state at height, i.e. whether a store -// serving reads at that height must open flatkv or can be served completely by memiavl alone. -// -// A chain's backend layout is not fixed for the life of the chain. Under types.Auto a chain runs memiavl-only -// until a MigrateEVM transition materializes flatkv and seeds it at some height; below that height every -// consensus value — including evm — lives in memiavl, and flatkv has no data at all. Block history therefore -// splits into a pre-flatkv era and an in-flatkv era, and a reader at an old height must know which side it is -// on. Answering "yes" for a pre-era height sends the caller into a flatkv open that cannot succeed ("no -// snapshot found ..."), failing historical queries that worked when the node was configured memiavl_only. -// -// Answering "no" when flatkv does hold state at that height is the far more dangerous direction: the migration -// deletes migrated keys out of memiavl, so a memiavl-only reader would report a key as absent rather than -// erroring — fabricating a nonexistence answer. Every uncertain case therefore resolves toward "yes", never -// toward a silent "no". -// -// The answer keys on flatkv's earliest-history record rather than on a flatkv open failing, because "no -// snapshot at target" is also what a pruned or corrupt in-history height produces. Serving those from -// post-migration memiavl is the fabricated-nonexistence case above, so they must keep failing loudly. -// -// This performs no I/O and cannot fail, which is what lets historical queries call it per read. -func FlatKVNeededAtHeight( - // Whether a flatkv backend was materialized at all. Under types.Auto the constructor only builds - // flatkv when its directory already exists on disk, so absence is itself the answer: no flatkv - // instance means no height was ever served by one. - flatKVPresent bool, - // The height flatkv's history begins at, or 0 when that is unknown — history begins at genesis, or - // seeding never ran. Both zero cases resolve toward "yes" per the safety direction above. Callers - // pass CompositeCommitStore.flatKVEarliestVersion, which is read from disk once at construction; - // reading flatkv's own copy instead is wrong, because it is populated as a side effect of a load and - // so is zero on a handle that has never been loaded. - earliestVersion int64, - // The configured write mode, not the effective one. A fixed mode short circuits to "yes", preserving - // the pinned fail-loud behavior of modes that cannot re-derive an effective memiavl-only layout. - configuredMode types.WriteMode, - // The height being read, where 0 means latest. - height int64, -) bool { - if !flatKVPresent { - return false - } - if configuredMode != types.Auto { - return true - } - if height <= 0 { - // The latest height is always in-era when flatkv exists at all. - return true - } - if earliestVersion > 0 && height < earliestVersion { - logger.Info("height predates flatkv history; memiavl serves it alone", - "height", height, "flatkvEarliestVersion", earliestVersion) - return false - } - return true -} diff --git a/sei-db/state_db/sc/composite/flatkv_needed_test.go b/sei-db/state_db/sc/composite/flatkv_needed_test.go deleted file mode 100644 index 86fc611812..0000000000 --- a/sei-db/state_db/sc/composite/flatkv_needed_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package composite - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/sei-protocol/sei-chain/sei-db/common/keys" - "github.com/sei-protocol/sei-chain/sei-db/common/utils" - "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" -) - -// TestFlatKVNeededAtHeight covers the classification table. Every row is a pure function of the cached -// earliest-history record, the configured mode and the height — the function performs no I/O, which is what -// lets historical queries call it per read. -func TestFlatKVNeededAtHeight(t *testing.T) { - for _, tc := range []struct { - name string - // present is whether the constructor materialized a flatkv backend at all. - present bool - // earliest is flatkv's earliest-history record, 0 when unseeded. - earliest int64 - mode types.WriteMode - height int64 - want bool - }{ - // Under types.Auto the constructor only builds flatkv when its directory already exists, so - // absence is itself the answer and must not be treated as "unknown". - {name: "not materialized", present: false, earliest: 0, mode: types.Auto, height: 5, want: false}, - - // A fixed configured mode cannot re-derive an effective memiavl-only layout, so it answers from - // config alone — even at a height the record would call pre-era. - {name: "fixed mode ignores era", present: true, earliest: 10, mode: types.EVMMigrated, - height: 5, want: true}, - - // The latest height is in-era by definition whenever flatkv exists at all. - {name: "latest", present: true, earliest: 10, mode: types.Auto, height: 0, want: true}, - - // The pre-era case this whole mechanism exists for, and its boundary: the record is the first - // in-era height, so height == earliest still needs flatkv. - {name: "below earliest", present: true, earliest: 10, mode: types.Auto, height: 9, want: false}, - {name: "at earliest", present: true, earliest: 10, mode: types.Auto, height: 10, want: true}, - {name: "above earliest", present: true, earliest: 10, mode: types.Auto, height: 11, want: true}, - - // An unseeded record means history begins at genesis or seeding never ran. Both resolve toward - // "yes": answering "no" would serve the height from a memiavl whose migrated keys are deleted, - // fabricating a nonexistence answer, whereas "yes" fails loudly on the flatkv open. - {name: "unseeded record", present: true, earliest: 0, mode: types.Auto, height: 5, want: true}, - } { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, FlatKVNeededAtHeight(tc.present, tc.earliest, tc.mode, tc.height)) - }) - } -} - -// TestNewCompositeCommitStore_UnreadableFlatKVMetadataFails pins the fail-loud direction at the moment the -// era record is read. A flatkv directory that exists but whose metadata cannot be read is corrupt or -// mid-materialization from a crashed transition, and those are not distinguishable here. Defaulting to an -// unseeded record would classify every height as in-era, which is safe, but defaulting the other way — or -// treating the directory as absent — would serve heights from a memiavl whose migrated keys were deleted. So -// the constructor refuses to produce a store at all. -func TestNewCompositeCommitStore_UnreadableFlatKVMetadataFails(t *testing.T) { - dir := t.TempDir() - - // A regular file where the working metadata DB belongs: the directory exists, so the constructor - // builds flatkv, but the point read cannot open it. - metaDir := filepath.Join(utils.GetFlatKVPath(dir), "working", "metadata") - require.NoError(t, os.MkdirAll(filepath.Dir(metaDir), 0o750)) - require.NoError(t, os.WriteFile(metaDir, []byte("not a pebble db"), 0o600)) - - _, err := NewCompositeCommitStore(t.Context(), dir, autoExportConfig()) - require.Error(t, err) - require.ErrorContains(t, err, "failed to read FlatKV earliest version") -} - -// TestComposite_Auto_EraRecordTracksLiveTransition guards the one moment the constructor's on-disk read goes -// stale: a MigrateEVM transition that materializes and seeds flatkv mid-run. A store that kept the value it -// read at construction (0, because the directory did not exist yet) would classify every pre-transition -// height as in-era and send historical reads into a flatkv that has no data there. -func TestComposite_Auto_EraRecordTracksLiveTransition(t *testing.T) { - dir := t.TempDir() - cfg := autoExportConfig() - - cs := openAutoStoreWithConfig(t, dir, cfg, 100) - defer func() { _ = cs.Close() }() - - require.Nil(t, cs.flatKV, "fixture precondition: flatkv must not be materialized yet") - require.Zero(t, cs.flatKVEarliestVersion) - - for i := 1; i <= 5; i++ { - require.NoError(t, cs.ApplyChangeSets([]*proto.NamedChangeSet{ - {Name: keys.BankStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - {Key: []byte("k"), Value: []byte{0x10 + byte(i)}}, - }}}, - })) - _, err := cs.Commit() - require.NoError(t, err) - } - - require.NoError(t, cs.SetWriteMode(types.MigrateEVM)) - require.NotNil(t, cs.flatKV, "the transition must have materialized flatkv") - require.Equal(t, cs.flatKV.EarliestVersion(), cs.flatKVEarliestVersion, - "the cached record must follow the seeding the transition performed") - require.Equal(t, int64(5), cs.flatKVEarliestVersion) - - // The classification must now split history at the transition height. - require.False(t, FlatKVNeededAtHeight(true, cs.flatKVEarliestVersion, cfg.WriteMode, 3)) - require.True(t, FlatKVNeededAtHeight(true, cs.flatKVEarliestVersion, cfg.WriteMode, 5)) -} - -// TestComposite_Auto_ReadOnlyPreEraHeightOnNeverLoadedStore is the regression guard for the gap this function -// closes. It drives the same pre-era scenario as TestComposite_Auto_ReadOnlyPreFlatKVEraHeight, but serves the -// historical read from a freshly constructed store that has never been loaded — the shape rootmulti uses for -// `seid export --height N`, where the era decision cannot come from flatkv's in-memory bookkeeping because -// nothing has populated it. -func TestComposite_Auto_ReadOnlyPreEraHeightOnNeverLoadedStore(t *testing.T) { - dir := t.TempDir() - cfg := autoExportConfig() - - valAt := func(i int) []byte { return []byte{0x10 + byte(i)} } - - cs := openAutoStoreWithConfig(t, dir, cfg, 100) - - // Heights 1..5 in the memiavl-only era, each with a distinct value so the as-of-height read is checkable. - for i := 1; i <= 5; i++ { - require.NoError(t, cs.ApplyChangeSets([]*proto.NamedChangeSet{ - {Name: keys.BankStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - {Key: []byte("k"), Value: valAt(i)}, - }}}, - })) - _, err := cs.Commit() - require.NoError(t, err) - } - - // Transition at height 5, then drive blocks so flatkv accumulates committed history. - require.NoError(t, cs.SetWriteMode(types.MigrateEVM)) - for i := 0; i < 3; i++ { - require.NoError(t, cs.ApplyChangeSets(nil)) - _, err := cs.Commit() - require.NoError(t, err) - } - require.Equal(t, int64(5), cs.flatKV.EarliestVersion(), - "fixture precondition: flatkv history must begin at the transition height") - require.NoError(t, cs.Close()) - - // Reopen without loading. Initialize mirrors what rootmulti does before vending a historical view; the - // deliberate omission is LoadLatest, which is what would otherwise populate flatkv's bookkeeping. - fresh, err := NewCompositeCommitStore(t.Context(), dir, cfg) - require.NoError(t, err) - defer func() { _ = fresh.Close() }() - require.NoError(t, fresh.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey})) - - // Pre-era height: must be served memiavl-only, with the value as of that height. - preEra, err := fresh.LoadVersionReadOnly(3) - require.NoError(t, err, "pre-flatkv-era heights must be queryable from a never-loaded store") - preEraStore, ok := preEra.(*CompositeCommitStore) - require.True(t, ok) - require.Nil(t, preEraStore.flatKV, "a pre-era height must not open flatkv") - require.Equal(t, types.MemiavlOnly, preEraStore.currentWriteMode) - val, found, err := preEraStore.Get(keys.BankStoreKey, []byte("k")) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, valAt(3), val, "value as-of height 3") - require.NoError(t, preEraStore.Close()) - - // In-era height: flatkv must still be opened, from the same never-loaded store. - inEra, err := fresh.LoadVersionReadOnly(7) - require.NoError(t, err) - inEraStore, ok := inEra.(*CompositeCommitStore) - require.True(t, ok) - defer func() { _ = inEraStore.Close() }() - require.NotNil(t, inEraStore.flatKV, "in-era heights must keep loading flatkv") -} - -// TestDerivedStoreRefusesLoads pins that a store which does not own the data directory rejects every load. -// Adopting a view and continuing to load through it used to depend on the view's backend mix to fail: a -// flatkv-carrying view errored ("store is read-only") but a memiavl-only view silently succeeded, leaving reads -// served from the version the view was built at. This fixture is deliberately memiavl-only, which is the -// configuration that was silent. -func TestDerivedStoreRefusesLoads(t *testing.T) { - dir := t.TempDir() - cfg := autoExportConfig() - - cs := openAutoStoreWithConfig(t, dir, cfg, 100) - for i := 1; i <= 3; i++ { - require.NoError(t, cs.ApplyChangeSets([]*proto.NamedChangeSet{ - {Name: keys.BankStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - {Key: []byte("k"), Value: []byte{byte(0x10 + i)}}, - }}}, - })) - _, err := cs.Commit() - require.NoError(t, err) - } - require.Nil(t, cs.flatKV, "fixture precondition: flatkv must not be materialized") - require.NoError(t, cs.Close()) - - fresh, err := NewCompositeCommitStore(t.Context(), dir, cfg) - require.NoError(t, err) - defer func() { _ = fresh.Close() }() - require.NoError(t, fresh.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey})) - - view, err := fresh.LoadVersion(2, false) - require.NoError(t, err) - viewStore, ok := view.(*CompositeCommitStore) - require.True(t, ok) - defer func() { _ = viewStore.Close() }() - require.True(t, viewStore.derived, "a read-only view must be marked derived") - - // Every load path must refuse, so a caller that adopted the view cannot keep loading through it. - require.ErrorIs(t, viewStore.LoadLatest(), errDerivedStore) - _, err = viewStore.LoadVersion(0, false) - require.ErrorIs(t, err, errDerivedStore) - _, err = viewStore.LoadVersion(2, false) - require.ErrorIs(t, err, errDerivedStore) - _, err = viewStore.LoadVersionReadOnly(0) - require.ErrorIs(t, err, errDerivedStore) - - // The view still serves the height it was built at. - require.Equal(t, int64(2), viewStore.Version()) -} diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 0bca128e2d..bf0d72d67f 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -43,15 +43,6 @@ type CompositeCommitStore struct { // The flatKV backend. Will be nil if migration to flatKV has not yet started. flatKV flatkv.Store - // flatKVEarliestVersion is the height flatkv's history begins at, or 0 when flatkv holds no history - // (never materialized, or seeded from genesis). Heights below it belong to the pre-flatkv era and are - // served by memiavl alone; see FlatKVNeededAtHeight. - // - // Read from disk by the constructor, before any flatkv instance exists to hold a PebbleDB lock, and - // refreshed by materializeFlatKV when a live transition seeds flatkv mid-run. Those are the only two - // moments the underlying record can change, so every read of this field is free. - flatKVEarliestVersion int64 - // Manages routing of traffic between the memiavl and flatkv backends. // Built (and rebuilt) inside LoadVersion against the just-opened // backends so that lazily-eager constructors like @@ -179,22 +170,8 @@ func NewCompositeCommitStore( openFlatKV = false } - // Resolve the flatkv era boundary here, while the directory is still unopened. Both era-classifying - // reads (this one and FlatKVNeededAtHeight) are properties of the directory rather than of any store, - // and PebbleDB takes an exclusive directory lock — so the only moment a plain point read of flatkv's - // metadata is possible is before the flatkv instance below exists to hold that lock. A directory that - // exists but whose metadata cannot be read is corrupt or mid-materialization, which must not be - // mistaken for "no flatkv here": those keys were deleted out of memiavl, so continuing would answer - // "absent" for keys that exist. - var earliestVersion int64 var flatKV flatkv.Store if openFlatKV { - v, err := flatkv.GetEarliestVersion(cfg.FlatKVConfig.DataDir) - if err != nil { - return nil, fmt.Errorf("failed to read FlatKV earliest version: %w", err) - } - earliestVersion = v - stateWAL, err := flatkv.OpenStateWAL(&cfg.FlatKVConfig) if err != nil { return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err) @@ -208,13 +185,12 @@ func NewCompositeCommitStore( } return &CompositeCommitStore{ - memIAVL: memIAVL, - flatKV: flatKV, - flatKVEarliestVersion: earliestVersion, - homeDir: homeDir, - config: cfg, - currentWriteMode: cfg.WriteMode, - ctx: ctx, + memIAVL: memIAVL, + flatKV: flatKV, + homeDir: homeDir, + config: cfg, + currentWriteMode: cfg.WriteMode, + ctx: ctx, }, nil } @@ -334,7 +310,6 @@ func (cs *CompositeCommitStore) SetInitialVersion(initialVersion int64) error { if err := cs.flatKV.SetInitialVersion(initialVersion); err != nil { return fmt.Errorf("flatkv SetInitialVersion: %w", err) } - cs.flatKVEarliestVersion = cs.flatKV.EarliestVersion() } return nil } @@ -399,7 +374,6 @@ func (cs *CompositeCommitStore) LoadLatest() error { return fmt.Errorf("failed to seed flatkv to memiavl version %d: %w", cs.memIAVL.Version(), err) } - cs.flatKVEarliestVersion = cs.flatKV.EarliestVersion() } // A crash between the sequential cosmos and EVM commits can leave the backends at different @@ -455,7 +429,7 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type } } - if FlatKVNeededAtHeight(cs.flatKV != nil, cs.flatKVEarliestVersion, cs.config.WriteMode, targetVersion) { + if cs.flatKV != nil { fkv, err := cs.flatKV.LoadVersionReadOnly(targetVersion) if err != nil { return nil, fmt.Errorf("failed to load FlatKV version: %w", err) @@ -468,13 +442,12 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type // inherits cs.ctx so cancellation of the parent context cascades, but buildRouter installs its own // child cancel so closing this handle does not affect the parent. ro := &CompositeCommitStore{ - memIAVL: memIAVLCommitter, - flatKV: flatKVStore, - flatKVEarliestVersion: cs.flatKVEarliestVersion, - homeDir: cs.homeDir, - config: cs.config, - ctx: cs.ctx, - derived: true, + memIAVL: memIAVLCommitter, + flatKV: flatKVStore, + homeDir: cs.homeDir, + config: cs.config, + ctx: cs.ctx, + derived: true, } if err := ro.resolveCurrentWriteMode(false); err != nil { return nil, fmt.Errorf("failed to resolve effective write mode for read-only handle: %w", err) @@ -750,9 +723,6 @@ func (cs *CompositeCommitStore) materializeFlatKV() error { } } cs.flatKV = loaded - // The seeding above is what writes flatkv's earliest-history record, so this is the one moment the - // constructor's on-disk read goes out of date. Take the value the load already has in memory. - cs.flatKVEarliestVersion = loaded.EarliestVersion() return nil } @@ -1307,61 +1277,37 @@ func (cs *CompositeCommitStore) Exporter(version int64) (types.Exporter, error) includeFlatKV := cs.flatKV != nil if includeFlatKV && exportNeedsMetadataGating(cs.config.WriteMode) { - // Distinguish a genuinely pre-flatkv-era version from an in-history - // flatkv load failure using flatkv's persisted earliest-history - // record, NOT the load failing — mirroring FlatKVNeededAtHeight. - // "no snapshot at target" / version-mismatch is also what a pruned or - // corrupt in-history version produces (flatkv prunes old snapshots and - // truncates the WAL beneath them while the earliest-history record stays - // fixed at the seeded value), so keying on the load failing would silently - // emit a memiavl-only snapshot that drops consensus-visible flatkv state - // and is byte-indistinguishable from a legitimate pre-era stream. - // - // Read the composite's copy rather than cs.flatKV.EarliestVersion(): the - // latter is populated as a side effect of a load, so it is zero on a - // never-loaded handle, which would classify every version as in-era. - earliest := cs.flatKVEarliestVersion - if earliest > 0 && version < earliest { - // Genuinely pre-flatkv era: every consensus value at this height - // lived in memiavl, so omitting flatkv is correct. - logger.Info("export version predates flatkv history; exporting memiavl only", - "version", version, "flatkvEarliestVersion", earliest) - includeFlatKV = false - } else { - // Evaluate the hash predicates against metadata as-of the - // exported version: flatkv read-only clones replay the WAL to the - // target version, so the boundary/version keys reflect historical - // state, not the live store's. - ro, err := cs.flatKV.LoadVersionReadOnly(version) - if err != nil { - // In-history load failure (pruned snapshot/WAL, corruption, or - // a transient fault) at a version where flatkv participates in - // the AppHash. Silently omitting flatkv here would produce a - // consensus-incomplete snapshot, so fail loud instead. - return nil, fmt.Errorf("failed to load flatkv at export version %d (>= earliest %d): %w", - version, earliest, err) - } - started, gateErr := migrationStarted(ro) - var bankDone bool - if gateErr == nil { - bankDone, gateErr = migration.IsModeComplete(ro, types.MigrateBank) - } - closeErr := ro.Close() - if gateErr != nil { - return nil, fmt.Errorf("failed to read migration metadata for export gating: %w", gateErr) - } - if closeErr != nil { - return nil, fmt.Errorf("failed to close export gating handle: %w", closeErr) - } - if cs.config.WriteMode == types.MigrateBank { - // Fixed MigrateBank descends from a flatkv-bearing - // predecessor; flatkv is in its hash at every version it - // can serve. Only the memiavl side is version-dependent. - started = true - } - includeFlatKV = started - includeMemiavl = includeMemiavl && !bankDone + // Evaluate the hash predicates against metadata as-of the exported + // version: flatkv read-only clones replay the WAL to the target + // version, so the boundary/version keys reflect historical state, not + // the live store's. + ro, err := cs.flatKV.LoadVersionReadOnly(version) + if err != nil { + // Silently omitting flatkv here would produce a consensus-incomplete + // snapshot, byte-indistinguishable from a legitimate memiavl-only + // stream, so fail loud instead. + return nil, fmt.Errorf("failed to load flatkv at export version %d: %w", version, err) + } + started, gateErr := migrationStarted(ro) + var bankDone bool + if gateErr == nil { + bankDone, gateErr = migration.IsModeComplete(ro, types.MigrateBank) + } + closeErr := ro.Close() + if gateErr != nil { + return nil, fmt.Errorf("failed to read migration metadata for export gating: %w", gateErr) + } + if closeErr != nil { + return nil, fmt.Errorf("failed to close export gating handle: %w", closeErr) + } + if cs.config.WriteMode == types.MigrateBank { + // Fixed MigrateBank descends from a flatkv-bearing predecessor; + // flatkv is in its hash at every version it can serve. Only the + // memiavl side is version-dependent. + started = true } + includeFlatKV = started + includeMemiavl = includeMemiavl && !bankDone } var memIAVLExporter types.Exporter diff --git a/sei-db/state_db/sc/composite/store_auto_test.go b/sei-db/state_db/sc/composite/store_auto_test.go index 7138223560..560b77519c 100644 --- a/sei-db/state_db/sc/composite/store_auto_test.go +++ b/sei-db/state_db/sc/composite/store_auto_test.go @@ -568,13 +568,17 @@ func TestComposite_Auto_ReadOnlyHandle(t *testing.T) { requireOracleMatches(t, ro, workload.snapshotOracle()) } -// TestComposite_Auto_ReadOnlyPreFlatKVEraHeight pins the era-aware -// read-only path: heights that predate flatkv's history (the chain ran -// effectively memiavl-only) must remain queryable after the migration has -// begun. The handle skips flatkv entirely — at such heights all consensus -// data lives in memiavl — instead of failing the flatkv load. In-era -// heights keep loading flatkv. -func TestComposite_Auto_ReadOnlyPreFlatKVEraHeight(t *testing.T) { +// TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails records a guarantee that was deliberately +// given up: heights predating flatkv's history used to be served memiavl-only, because flatkv kept a +// persisted record of the height its history began at and the read path consulted it. +// +// That record is gone, so nothing distinguishes "this height predates flatkv" from "flatkv failed to +// load at this height", and the read path can only take the safe branch — attempt the load and +// surface the failure. Answering the other way would serve the height from a memiavl whose migrated +// keys were deleted, fabricating a nonexistence answer, so failing here is the correct direction. +// +// In-era heights are unaffected. Delete this test only alongside a decision to restore pre-era reads. +func TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails(t *testing.T) { dir := t.TempDir() cs := openAutoStoreWithConfig(t, dir, autoExportConfig(), 100) defer func() { _ = cs.Close() }() @@ -599,26 +603,15 @@ func TestComposite_Auto_ReadOnlyPreFlatKVEraHeight(t *testing.T) { _, err := cs.Commit() require.NoError(t, err) } - require.Equal(t, int64(5), cs.flatKV.EarliestVersion(), - "flatkv history must begin at the seeded (transition) height") - // Pre-era height: served memiavl-only, with as-of-height values. - roCommitter, err := cs.LoadVersionReadOnly(3) - require.NoError(t, err, "pre-flatkv-era heights must remain queryable") - ro, ok := roCommitter.(*CompositeCommitStore) - require.True(t, ok) - require.Equal(t, types.MemiavlOnly, ro.currentWriteMode) - require.Nil(t, ro.flatKV) - val, found, err := ro.Get(keys.BankStoreKey, []byte("k")) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, valAt(3), val, "value as-of height 3") - require.NoError(t, ro.Close()) + // Pre-era height: the load is attempted against a flatkv that has no such height, and fails. + _, err := cs.LoadVersionReadOnly(3) + require.Error(t, err, "a pre-flatkv-era height is no longer distinguishable from a load failure") - // In-era height: flatkv loads as before. - roCommitter, err = cs.LoadVersionReadOnly(7) + // In-era height: unchanged. + roCommitter, err := cs.LoadVersionReadOnly(7) require.NoError(t, err) - ro, ok = roCommitter.(*CompositeCommitStore) + ro, ok := roCommitter.(*CompositeCommitStore) require.True(t, ok) defer func() { _ = ro.Close() }() require.NotNil(t, ro.flatKV, "in-era heights must keep loading flatkv") diff --git a/sei-db/state_db/sc/composite/store_load_test.go b/sei-db/state_db/sc/composite/store_load_test.go new file mode 100644 index 0000000000..81b0f786b5 --- /dev/null +++ b/sei-db/state_db/sc/composite/store_load_test.go @@ -0,0 +1,93 @@ +package composite + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" +) + +// TestCorruptFlatKVDirFailsOnLoad pins where the corrupt-flatkv-directory tripwire lives. Under +// types.Auto the constructor treats directory presence as the signal that flatkv participates, so a +// directory that exists but cannot be opened must not be mistaken for "no flatkv here" — those keys +// were deleted out of memiavl, and continuing would answer "absent" for keys that exist. +// +// The constructor no longer reads the directory, so the failure surfaces when LoadLatest opens the +// DBs. That only holds for a working directory the open path will not silently rebuild: a working dir +// whose SNAPSHOT_BASE does not match the current snapshot is wiped and re-cloned by createWorkingDir, +// which repairs the damage rather than reporting it. This fixture commits first so SNAPSHOT_BASE is +// present and the re-clone is skipped. +func TestCorruptFlatKVDirFailsOnLoad(t *testing.T) { + dir := t.TempDir() + cs := openAutoStoreWithConfig(t, dir, autoExportConfig(), 100) + require.NoError(t, cs.SetWriteMode(types.MigrateEVM)) + require.NoError(t, cs.ApplyChangeSets(nil)) + _, err := cs.Commit() + require.NoError(t, err) + require.NoError(t, cs.Close()) + + // Replace the working misc DB with a regular file, leaving SNAPSHOT_BASE intact so the open path + // reuses this working dir instead of re-cloning it. + miscDir := filepath.Join(utils.GetFlatKVPath(dir), "working", "misc") + require.NoError(t, os.RemoveAll(miscDir)) + require.NoError(t, os.WriteFile(miscDir, []byte("not a pebble db"), 0o600)) + + reopened, err := NewCompositeCommitStore(t.Context(), dir, autoExportConfig()) + require.NoError(t, err, "construction does not open the DBs, so it cannot detect this") + defer func() { _ = reopened.Close() }() + + require.Error(t, reopened.LoadLatest(), "a flatkv directory that cannot be opened must fail the load") +} + +// TestDerivedStoreRefusesLoads pins that a store which does not own the data directory rejects every load. +// Adopting a view and continuing to load through it used to depend on the view's backend mix to fail: a +// flatkv-carrying view errored ("store is read-only") but a memiavl-only view silently succeeded, leaving reads +// served from the version the view was built at. This fixture is deliberately memiavl-only, which is the +// configuration that was silent. +func TestDerivedStoreRefusesLoads(t *testing.T) { + dir := t.TempDir() + cfg := autoExportConfig() + + cs := openAutoStoreWithConfig(t, dir, cfg, 100) + for i := 1; i <= 3; i++ { + require.NoError(t, cs.ApplyChangeSets([]*proto.NamedChangeSet{ + {Name: keys.BankStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: []byte("k"), Value: []byte{byte(0x10 + i)}}, + }}}, + })) + _, err := cs.Commit() + require.NoError(t, err) + } + require.Nil(t, cs.flatKV, "fixture precondition: flatkv must not be materialized") + require.NoError(t, cs.Close()) + + fresh, err := NewCompositeCommitStore(t.Context(), dir, cfg) + require.NoError(t, err) + defer func() { _ = fresh.Close() }() + require.NoError(t, fresh.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey})) + + view, err := fresh.LoadVersion(2, false) + require.NoError(t, err) + viewStore, ok := view.(*CompositeCommitStore) + require.True(t, ok) + defer func() { _ = viewStore.Close() }() + require.True(t, viewStore.derived, "a read-only view must be marked derived") + + // Every load path must refuse, so a caller that adopted the view cannot keep loading through it. + require.ErrorIs(t, viewStore.LoadLatest(), errDerivedStore) + _, err = viewStore.LoadVersion(0, false) + require.ErrorIs(t, err, errDerivedStore) + _, err = viewStore.LoadVersion(2, false) + require.ErrorIs(t, err, errDerivedStore) + _, err = viewStore.LoadVersionReadOnly(0) + require.ErrorIs(t, err, errDerivedStore) + + // The view still serves the height it was built at. + require.Equal(t, int64(2), viewStore.Version()) +} diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index be28d1b6e5..00e616d148 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -53,7 +53,6 @@ func (f *failingEVMStore) Iterator(string, []byte, []byte, bool) (dbm.Iterator, func (f *failingEVMStore) RootHash() []byte { return nil } func (f *failingEVMStore) Version() int64 { return 0 } func (f *failingEVMStore) PendingVersion() int64 { return 0 } -func (f *failingEVMStore) EarliestVersion() int64 { return 0 } func (f *failingEVMStore) GetLatestVersion() (int64, error) { return 0, nil } func (f *failingEVMStore) WriteSnapshot(string) error { return nil } func (f *failingEVMStore) Rollback(int64) error { return nil } @@ -66,18 +65,6 @@ func (f *failingEVMStore) RecordHashes(hashlog.HashLogger, uint64) error { retur func (f *failingEVMStore) CleanupOrphanedReadOnlyDirs() error { return nil } func (f *failingEVMStore) Close() error { return nil } -// eraFailingEVMStore is a failingEVMStore with a configurable -// EarliestVersion, used to exercise Exporter's pre-era vs in-history -// classification of a flatkv load failure. -type eraFailingEVMStore struct { - failingEVMStore - earliest int64 -} - -var _ flatkv.Store = (*eraFailingEVMStore)(nil) - -func (f *eraFailingEVMStore) EarliestVersion() int64 { return f.earliest } - func padLeft32(val ...byte) []byte { var b [32]byte copy(b[32-len(val):], val) @@ -1182,13 +1169,12 @@ func TestExportMemiavlOnlyHasNoFlatKVModule(t *testing.T) { } } -// TestExporterFailsLoudOnInHistoryFlatKVLoadFailure verifies that when -// flatkv fails to load at an export version within flatkv's history -// (version >= EarliestVersion), Exporter returns an error rather than -// silently emitting a memiavl-only snapshot that would drop -// consensus-visible flatkv state. Mirrors FlatKVNeededAtHeight's fail-loud -// contract for pruned/corrupt in-history versions. -func TestExporterFailsLoudOnInHistoryFlatKVLoadFailure(t *testing.T) { +// TestExporterFailsLoudOnFlatKVLoadFailure verifies that a flatkv load failure at an export +// version surfaces as an error rather than silently emitting a memiavl-only snapshot that would +// drop consensus-visible flatkv state — such a snapshot is byte-indistinguishable from a +// legitimate memiavl-only stream, so a restored node would be missing consensus state with no +// signal that anything went wrong. +func TestExporterFailsLoudOnFlatKVLoadFailure(t *testing.T) { dir := t.TempDir() cfg := config.DefaultStateCommitConfig() cfg.MemIAVLConfig.AsyncCommitBuffer = 0 @@ -1209,64 +1195,14 @@ func TestExporterFailsLoudOnInHistoryFlatKVLoadFailure(t *testing.T) { _, err = cs.Commit() require.NoError(t, err) - // Inject a flatkv whose load fails at an in-history version: export - // version 1 is >= the earliest-history record of 1, so the pre-era - // short-circuit does not apply and the load failure must surface as an - // error. - cs.flatKV = &eraFailingEVMStore{earliest: 1} - cs.flatKVEarliestVersion = 1 + // Inject a flatkv whose load always fails; the failure must surface. + cs.flatKV = &failingEVMStore{} _, err = cs.Exporter(1) require.Error(t, err, "Exporter must fail loud on an in-history flatkv load failure") require.Contains(t, err.Error(), "failed to load flatkv at export version") } -// TestExporterOmitsFlatKVForPreEraVersion verifies that when the export -// version predates flatkv's history (version < EarliestVersion), Exporter -// omits flatkv and returns a memiavl-only snapshot without error — the -// flatkv load is never attempted. This is the legitimate pre-era case that -// must remain non-fatal even though a load at that version would fail. -func TestExporterOmitsFlatKVForPreEraVersion(t *testing.T) { - dir := t.TempDir() - cfg := config.DefaultStateCommitConfig() - cfg.MemIAVLConfig.SnapshotInterval = 1 - cfg.MemIAVLConfig.SnapshotMinTimeInterval = 0 - cfg.MemIAVLConfig.AsyncCommitBuffer = 0 - cfg.WriteMode = types.MigrateEVM - cs, err := NewCompositeCommitStore(t.Context(), dir, cfg) - require.NoError(t, err) - require.NoError(t, cs.SetMigrationBatchSize(100)) - require.NoError(t, cs.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey})) - err = cs.LoadLatest() - require.NoError(t, err) - - err = cs.ApplyChangeSets([]*proto.NamedChangeSet{ - {Name: keys.BankStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - {Key: []byte("key1"), Value: []byte("val1")}, - }}}, - }) - require.NoError(t, err) - _, err = cs.Commit() - require.NoError(t, err) - - // Inject a flatkv whose history starts above the export height, so version - // 1 is pre-era. LoadVersion would fail, but the pre-era check - // short-circuits before it is called: flatkv is omitted, no error. - cs.flatKV = &eraFailingEVMStore{earliest: 10} - cs.flatKVEarliestVersion = 10 - - exporter, err := cs.Exporter(1) - require.NoError(t, err, "pre-era export must omit flatkv without error") - items := drainCompositeExporter(t, exporter) - require.NoError(t, exporter.Close()) - require.NoError(t, cs.Close()) - - for _, it := range items { - require.NotEqual(t, keys.FlatKVStoreKey, it.moduleName, - "flatkv module must not appear in a pre-era export") - } -} - func TestCompositeImporterRouting(t *testing.T) { // Verify that the composite importer routes evm_flatkv exclusively // to the evm importer and other modules only to cosmos. diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index 9f7a6a36c5..5735b3b419 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -66,14 +66,6 @@ type Store interface { // initialVersion <= 0. SetInitialVersion(initialVersion int64) error - // EarliestVersion returns the version this store's history begins at - // (the seeded version recorded by SetInitialVersion), or 0 when - // unknown (genesis stores, and stores created before the record - // existed). A non-zero result means versions below it predate the - // store entirely — the chain ran without flatkv — as opposed to - // pruned or corrupt in-history versions, which still fail to load. - EarliestVersion() int64 - // Get returns the value for a key within the given module. // For EVM keys (moduleName == "evm"), the key is a memiavl EVM key // routed to account/storage/code/misc DBs internally. diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index 2a866a3f84..5ee296fcd0 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -19,7 +19,7 @@ type Config struct { // Must be set before calling Validate(). DataDir string - // Fsync controls whether PebbleDB writes (data DBs + metadataDB) use fsync. + // Fsync controls whether PebbleDB writes to the data DBs use fsync. // WAL always uses NoSync (matching memiavl); crash recovery relies on // WAL catchup, which is idempotent. // Default: false @@ -87,12 +87,6 @@ type Config struct { // MiscCacheConfig defines the cache configuration for the misc database. MiscCacheConfig dbcache.CacheConfig - // MetadataDBConfig defines the PebbleDB configuration for the metadata database. - MetadataDBConfig pebbledb.PebbleDBConfig - - // MetadataCacheConfig defines the cache configuration for the metadata database. - MetadataCacheConfig dbcache.CacheConfig - // Controls the number of goroutines in the DB read pool. The number of threads in this pool is equal to // ReaderThreadsPerCore * runtime.NumCPU() + ReaderConstantThreadCount. ReaderThreadsPerCore float64 @@ -135,8 +129,6 @@ func DefaultConfig() *Config { StorageCacheConfig: dbcache.DefaultCacheConfig(), MiscDBConfig: pebbledb.DefaultConfig(), MiscCacheConfig: dbcache.DefaultCacheConfig(), - MetadataDBConfig: pebbledb.DefaultConfig(), - MetadataCacheConfig: dbcache.DefaultCacheConfig(), ReaderThreadsPerCore: 2.0, ReaderConstantThreadCount: 0, ReaderPoolQueueSize: 1024, @@ -172,9 +164,6 @@ func (c *Config) Validate() error { if err := c.MiscCacheConfig.Validate(); err != nil { return fmt.Errorf("misc cache config is invalid: %w", err) } - if err := c.MetadataCacheConfig.Validate(); err != nil { - return fmt.Errorf("metadata cache config is invalid: %w", err) - } if c.DataDir == "" { return fmt.Errorf("data dir is required") } @@ -190,9 +179,6 @@ func (c *Config) Validate() error { if err := c.MiscDBConfig.Validate(); err != nil { return fmt.Errorf("misc db config is invalid: %w", err) } - if err := c.MetadataDBConfig.Validate(); err != nil { - return fmt.Errorf("metadata db config is invalid: %w", err) - } if c.ReaderThreadsPerCore <= 0 { return fmt.Errorf("reader threads per core must be greater than 0") diff --git a/sei-db/state_db/sc/flatkv/config/config_test.go b/sei-db/state_db/sc/flatkv/config/config_test.go index 4c5cf3ac9d..18586a7b6f 100644 --- a/sei-db/state_db/sc/flatkv/config/config_test.go +++ b/sei-db/state_db/sc/flatkv/config/config_test.go @@ -15,7 +15,6 @@ func validBaseConfig() *Config { cfg.CodeDBConfig.DataDir = "/tmp/test/code" cfg.StorageDBConfig.DataDir = "/tmp/test/storage" cfg.MiscDBConfig.DataDir = "/tmp/test/misc" - cfg.MetadataDBConfig.DataDir = "/tmp/test/metadata" return cfg } @@ -92,7 +91,6 @@ func TestDefaultConfigValidExceptDataDir(t *testing.T) { cfg.CodeDBConfig.DataDir = "/tmp/test/code" cfg.StorageDBConfig.DataDir = "/tmp/test/storage" cfg.MiscDBConfig.DataDir = "/tmp/test/misc" - cfg.MetadataDBConfig.DataDir = "/tmp/test/metadata" require.NoError(t, cfg.Validate()) } diff --git a/sei-db/state_db/sc/flatkv/config/flatkv_test_config.go b/sei-db/state_db/sc/flatkv/config/flatkv_test_config.go index 0b04fbe604..fa5a3f5727 100644 --- a/sei-db/state_db/sc/flatkv/config/flatkv_test_config.go +++ b/sei-db/state_db/sc/flatkv/config/flatkv_test_config.go @@ -38,8 +38,6 @@ func DefaultTestConfig(t *testing.T) *Config { StorageCacheConfig: smallTestCacheConfig(), MiscDBConfig: smallTestPebbleConfig(), MiscCacheConfig: smallTestCacheConfig(), - MetadataDBConfig: smallTestPebbleConfig(), - MetadataCacheConfig: smallTestCacheConfig(), ReaderThreadsPerCore: 2.0, ReaderPoolQueueSize: 1024, MiscPoolThreadsPerCore: 4.0, diff --git a/sei-db/state_db/sc/flatkv/hashlog.go b/sei-db/state_db/sc/flatkv/hashlog.go index 63a7ed084a..effe0ff024 100644 --- a/sei-db/state_db/sc/flatkv/hashlog.go +++ b/sei-db/state_db/sc/flatkv/hashlog.go @@ -7,8 +7,7 @@ import ( ) // Hash logger category names owned by the flatKV backend. flatKVDBHashPrefix is joined with a data DB -// directory name (e.g. "flatKV/db/account"). The metadata DB is intentionally excluded — it holds only -// watermarks, not state. +// directory name (e.g. "flatKV/db/account"). const ( FlatKVRootHashType = "flatKV/root" flatKVDBHashPrefix = "flatKV/db/" diff --git a/sei-db/state_db/sc/flatkv/ktype/meta.go b/sei-db/state_db/sc/flatkv/ktype/meta.go index 2490e144d7..ce59086c46 100644 --- a/sei-db/state_db/sc/flatkv/ktype/meta.go +++ b/sei-db/state_db/sc/flatkv/ktype/meta.go @@ -9,14 +9,13 @@ import ( const metaKeyPrefix = "_meta/" const ( - metaVersion = metaKeyPrefix + "version" - metaLtHash = metaKeyPrefix + "hash" - metaEarliest = metaKeyPrefix + "earliest" + metaVersion = metaKeyPrefix + "version" + metaLtHash = metaKeyPrefix + "hash" // moduleLtHashPrefix brackets the per-module metadata keys stored in each // data DB, e.g. "_meta/x:evm/hash", "_meta/x:gov/stats". The "x:" segment // namespaces module names so they never collide with the fixed per-DB keys - // (version / hash / earliest). Each module has a "/hash" key (its per-module + // (version / hash). Each module has a "/hash" key (its per-module // LtHash) and a "/stats" key (its per-module key-count / byte totals). moduleLtHashPrefix = metaKeyPrefix + "x:" moduleLtHashSuffix = "/hash" @@ -27,11 +26,6 @@ var ( MetaKeyPrefixBytes = []byte(metaKeyPrefix) MetaVersionKey = []byte(metaVersion) MetaLtHashKey = []byte(metaLtHash) - // MetaEarliestVersionKey records the version a seeded store's history - // begins at (written once by SetInitialVersion, global metadata DB - // only). Absent on genesis stores and stores predating the record. - MetaEarliestVersionKey = []byte(metaEarliest) - // ModuleLtHashPrefixBytes is the inclusive lower bound for iterating the // per-module LtHash keys ("_meta/x:") within a data DB. ModuleLtHashPrefixBytes = []byte(moduleLtHashPrefix) diff --git a/sei-db/state_db/sc/flatkv/perdb_lthash_test.go b/sei-db/state_db/sc/flatkv/perdb_lthash_test.go index 73b03326ee..2e06c8d9ef 100644 --- a/sei-db/state_db/sc/flatkv/perdb_lthash_test.go +++ b/sei-db/state_db/sc/flatkv/perdb_lthash_test.go @@ -86,10 +86,13 @@ func commitMixedState(t *testing.T, s *CommitStore, round byte) { require.NoError(t, err) } -// Test: Crash recovery where metadataDB is behind data DBs. -// Simulates a crash after commitBatches (step 2) but before -// commitGlobalMetadata (step 4) by rolling back metadataDB's -// global version. Data DBs and their LocalMeta remain at v2. +// Test: crash recovery where one data DB's version record is behind the others. +// +// A data DB commits its version in the same batch as its data, so a torn commit +// leaves the DBs at different versions. The store then opens at the lowest and +// replays from there — into DBs that already hold those blocks. This pins that +// the replay is idempotent: re-applying a block to a DB that already has it is a +// no-op for the LtHash, because the old value read back is the new value. func TestPerDBLtHashSkewRecovery(t *testing.T) { dir := t.TempDir() dbDir := filepath.Join(dir, flatkvRootDir) @@ -105,18 +108,22 @@ func TestPerDBLtHashSkewRecovery(t *testing.T) { commitMixedState(t, s1, 1) commitMixedState(t, s1, 2) verifyPerDBLtHash(t, s1) + wantRoot := bytes.Clone(s1.CommittedRootHash()) + wantPerDB := make(map[string][32]byte, len(dataDBDirs)) + for _, dbDir := range dataDBDirs { + wantPerDB[dbDir] = s1.perDBWorkingLtHash[dbDir].Checksum() + } require.NoError(t, s1.Close()) - // Roll back metadataDB global version to 1 to simulate crash - // after commitBatches completed but before commitGlobalMetadata. + // Rewind accountDB's version record to 1, leaving its data — and every + // other DB — at 2. The store must open at 1 and replay block 2. snapDir, _, err := currentSnapshotDir(dbDir) require.NoError(t, err) - metaDBPath := filepath.Join(snapDir, metadataDir) - metaCfg := pebbledb.DefaultConfig() - metaCfg.DataDir = metaDBPath - metaCfg.EnableMetrics = false - db, err := pebbledb.Open(t.Context(), &metaCfg) + acctCfg := pebbledb.DefaultConfig() + acctCfg.DataDir = filepath.Join(snapDir, accountDBDir) + acctCfg.EnableMetrics = false + db, err := pebbledb.Open(t.Context(), &acctCfg) require.NoError(t, err) require.NoError(t, db.Set(ktype.MetaVersionKey, versionToBytes(1), types.WriteOptions{Sync: true})) require.NoError(t, db.Close()) @@ -131,6 +138,12 @@ func TestPerDBLtHashSkewRecovery(t *testing.T) { require.NoError(t, err) defer s2.Close() + require.Equal(t, wantRoot, s2.CommittedRootHash(), + "replaying an already-applied block must reproduce the same global root") + for _, dbDir := range dataDBDirs { + require.Equal(t, wantPerDB[dbDir], s2.perDBWorkingLtHash[dbDir].Checksum(), + "%s per-DB root must be bit-identical after replay", dbDir) + } require.Equal(t, int64(2), s2.Version()) verifyPerDBLtHash(t, s2) verifyLtHashAtHeight(t, s2, 2) diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index ce8052ce17..cb297e2297 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -1,7 +1,6 @@ package flatkv import ( - "encoding/binary" "errors" "fmt" "io" @@ -12,9 +11,7 @@ import ( "strings" "time" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/pebbledb" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" "go.opentelemetry.io/otel/metric" ) @@ -27,10 +24,9 @@ import ( // account/ (PebbleDB: addr → AccountValue) // code/ (PebbleDB: addr → bytecode) // storage/ (PebbleDB: addr||slot → value) -// misc/ (PebbleDB: full key → value) -// metadata/ (PebbleDB: version + LtHash) +// misc/ (PebbleDB: full key → value) // working/ (mutable clone of active snapshot) -// account/, code/, storage/, misc/, metadata/ +// account/, code/, storage/, misc/ // SNAPSHOT_BASE (records source snapshot name) // changelog/ (WAL, shared across snapshots) const ( @@ -178,9 +174,6 @@ func updateCurrentSymlink(root, snapshotDir string) error { return nil } -// snapshotDBDirs lists the DB subdirectory names included in a snapshot. -var snapshotDBDirs = []string{accountDBDir, codeDBDir, storageDBDir, miscDBDir, metadataDir} - // removeTmpDirs removes any directories ending in "-tmp" or "-removing" // left over from interrupted snapshot writes or deletes. func removeTmpDirs(dir string) error { @@ -219,7 +212,7 @@ func createWorkingDir(snapDir, workDir string) error { return err } - for _, sub := range snapshotDBDirs { + for _, sub := range dataDBDirs { srcPath := filepath.Join(snapDir, sub) dstPath := filepath.Join(workDir, sub) @@ -321,9 +314,9 @@ func atomicRemoveDir(path string) error { } // resolveSnapshotDir returns the full path to the active snapshot directory. -// It handles four cases: (1) current symlink exists, (2) migration from -// pre-snapshot flat layout, (3) recovery from a partial migration crash, -// or (4) initialization of a fresh empty snapshot. +// It handles three cases: (1) current symlink exists, (2) recovery of an +// orphaned snapshot whose symlink was never created, or (3) initialization of a +// fresh empty snapshot. func (s *CommitStore) resolveSnapshotDir(flatkvDir string) (string, error) { snapDir, _, err := currentSnapshotDir(flatkvDir) if err == nil { @@ -333,20 +326,8 @@ func (s *CommitStore) resolveSnapshotDir(flatkvDir string) (string, error) { return "", fmt.Errorf("read current symlink: %w", err) } - hasFlatDirs := false - for _, sub := range snapshotDBDirs { - if _, err := os.Stat(filepath.Join(flatkvDir, sub)); err == nil { - hasFlatDirs = true - break - } - } - if hasFlatDirs { - return s.migrateFlatLayout(flatkvDir) - } - - // No flat dirs. Check for an orphaned snapshot directory — this happens - // when a previous migration moved all dirs but crashed before creating - // the current symlink. + // Check for an orphaned snapshot directory — this happens when a previous + // write moved everything into place but crashed before creating the symlink. var latestSnap int64 = -1 _ = traverseSnapshots(flatkvDir, false, func(v int64) (bool, error) { latestSnap = v @@ -363,7 +344,7 @@ func (s *CommitStore) resolveSnapshotDir(flatkvDir string) (string, error) { initSnap := snapshotName(0) initDir := filepath.Join(flatkvDir, initSnap) - for _, sub := range snapshotDBDirs { + for _, sub := range dataDBDirs { if err := os.MkdirAll(filepath.Join(initDir, sub), 0750); err != nil { return "", fmt.Errorf("create initial snapshot subdir %s: %w", sub, err) } @@ -374,61 +355,6 @@ func (s *CommitStore) resolveSnapshotDir(flatkvDir string) (string, error) { return initDir, nil } -// migrateFlatLayout moves the existing flat DB directories -// (account/, code/, storage/, metadata/) into a snapshot directory and -// creates the current symlink. -// -// The function is idempotent: directories that were already moved by a -// previous partial attempt are skipped, so recovery from a mid-migration -// crash completes the remaining moves. -func (s *CommitStore) migrateFlatLayout(flatkvDir string) (string, error) { - logger.Info("FlatKV: migrating from flat layout to snapshot layout") - - // Determine version for the snapshot name. The metadata DB might still - // be at the flat location or might have been moved in a prior attempt. - var version int64 - metaCfg := s.config.MetadataDBConfig - metaCfg.DataDir = filepath.Join(flatkvDir, metadataDir) - tmpMeta, err := pebbledb.Open(s.ctx, &metaCfg) - if err == nil { - verData, verErr := tmpMeta.Get(ktype.MetaVersionKey) - _ = tmpMeta.Close() - if verErr == nil && len(verData) == 8 { - version = int64(binary.BigEndian.Uint64(verData)) //nolint:gosec // block height, always < MaxInt64 - } - } else { - // Metadata already moved — look for the snapshot dir from a prior attempt. - _ = traverseSnapshots(flatkvDir, false, func(v int64) (bool, error) { - version = v - return true, nil - }) - } - - snapName := snapshotName(version) - snapDir := filepath.Join(flatkvDir, snapName) - if err := os.MkdirAll(snapDir, 0750); err != nil { - return "", fmt.Errorf("migration: create snapshot dir: %w", err) - } - - for _, sub := range snapshotDBDirs { - src := filepath.Join(flatkvDir, sub) - dst := filepath.Join(snapDir, sub) - if _, err := os.Stat(src); os.IsNotExist(err) { - continue - } - if err := os.Rename(src, dst); err != nil { - return "", fmt.Errorf("migration: move %s -> %s: %w", src, dst, err) - } - } - - if err := updateCurrentSymlink(flatkvDir, snapName); err != nil { - return "", fmt.Errorf("migration: update current symlink: %w", err) - } - - logger.Info("FlatKV: migration complete", "snapshot", snapName) - return snapDir, nil -} - // WriteSnapshot creates a PebbleDB checkpoint of the committed state. // The snapshot is written into a versioned subdirectory under the flatkv root // (e.g. flatkv/snapshot-00000000000000000100) and the current symlink is updated. @@ -472,26 +398,15 @@ func (s *CommitStore) WriteSnapshot(_ string) (err error) { } }() - // Deterministic order (slice, not map) for reproducibility. - type namedDB struct { - name string - db types.KeyValueDB - } - dbs := []namedDB{ - {accountDBDir, s.accountDB}, - {codeDBDir, s.codeDB}, - {storageDBDir, s.storageDB}, - {miscDBDir, s.miscDB}, - {metadataDir, s.metadataDB}, - } - for _, ndb := range dbs { + // namedDataDBs has a fixed iteration order, so the checkpoint is reproducible. + for _, ndb := range s.namedDataDBs() { cp, ok := ndb.db.(types.Checkpointable) if !ok { - return fmt.Errorf("db %s does not support Checkpoint", ndb.name) + return fmt.Errorf("db %s does not support Checkpoint", ndb.dir) } - dest := filepath.Join(tmpPath, ndb.name) + dest := filepath.Join(tmpPath, ndb.dir) if err := cp.Checkpoint(dest); err != nil { - return fmt.Errorf("checkpoint %s: %w", ndb.name, err) + return fmt.Errorf("checkpoint %s: %w", ndb.dir, err) } } diff --git a/sei-db/state_db/sc/flatkv/snapshot_test.go b/sei-db/state_db/sc/flatkv/snapshot_test.go index 12d08582a7..b827d4fded 100644 --- a/sei-db/state_db/sc/flatkv/snapshot_test.go +++ b/sei-db/state_db/sc/flatkv/snapshot_test.go @@ -53,7 +53,7 @@ func TestSnapshotCreatesDir(t *testing.T) { // Verify snapshot directory exists with all 4 DB subdirs snapDir := filepath.Join(flatkvDir, snapshotName(1)) - for _, sub := range snapshotDBDirs { + for _, sub := range dataDBDirs { info, err := os.Stat(filepath.Join(snapDir, sub)) require.NoError(t, err, "subdir %s should exist", sub) require.True(t, info.IsDir()) @@ -272,57 +272,6 @@ func TestPartialSnapshotCleanup(t *testing.T) { _ = s.Close() } -func TestMigrationFromFlatLayout(t *testing.T) { - dir := t.TempDir() - flatkvDir := filepath.Join(dir, flatkvRootDir) - - // Simulate the old flat layout by creating DB dirs directly - for _, sub := range []string{accountDBDir, codeDBDir, storageDBDir, metadataDir, miscDBDir} { - dbPath := filepath.Join(flatkvDir, sub) - require.NoError(t, os.MkdirAll(dbPath, 0750)) - // Create an actual PebbleDB so Open works - cfg := pebbledb.DefaultTestConfig(t) - cfg.DataDir = dbPath - db, err := pebbledb.Open(t.Context(), &cfg) - require.NoError(t, err) - require.NoError(t, db.Close()) - } - - // Ensure no current symlink exists - _, err := os.Lstat(currentPath(flatkvDir)) - require.True(t, os.IsNotExist(err)) - - // Open the store - should trigger migration - cfg := config.DefaultTestConfig(t) - cfg.DataDir = filepath.Join(dir, flatkvRootDir) - s, err := newCommitStoreWithWAL(t.Context(), cfg) - require.NoError(t, err) - err = s.LoadLatest() - require.NoError(t, err) - defer s.Close() - - // current symlink should now exist - target, err := os.Readlink(currentPath(flatkvDir)) - require.NoError(t, err) - require.Equal(t, snapshotName(0), target) - - // The old flat dirs should be gone (moved into the snapshot) - for _, sub := range snapshotDBDirs { - _, err := os.Stat(filepath.Join(flatkvDir, sub)) - require.True(t, os.IsNotExist(err), "flat dir %s should have been moved", sub) - } - - // The snapshot dir should have the DB subdirs - snapDir := filepath.Join(flatkvDir, snapshotName(0)) - for _, sub := range snapshotDBDirs { - info, err := os.Stat(filepath.Join(snapDir, sub)) - require.NoError(t, err) - require.True(t, info.IsDir()) - } - - require.Equal(t, int64(0), s.Version()) -} - func TestOpenVersionValidation(t *testing.T) { dir := t.TempDir() @@ -966,7 +915,7 @@ func TestCreateWorkingDirReusesExisting(t *testing.T) { dir := t.TempDir() snapDir := filepath.Join(dir, snapshotName(5)) - for _, sub := range snapshotDBDirs { + for _, sub := range dataDBDirs { require.NoError(t, os.MkdirAll(filepath.Join(snapDir, sub), 0750)) } @@ -988,7 +937,7 @@ func TestCreateWorkingDirReclones(t *testing.T) { snap5 := filepath.Join(dir, snapshotName(5)) snap10 := filepath.Join(dir, snapshotName(10)) - for _, sub := range snapshotDBDirs { + for _, sub := range dataDBDirs { require.NoError(t, os.MkdirAll(filepath.Join(snap5, sub), 0750)) require.NoError(t, os.MkdirAll(filepath.Join(snap10, sub), 0750)) } @@ -1097,7 +1046,7 @@ func TestOrphanSnapshotRecovery(t *testing.T) { flatkvDir := filepath.Join(dir, flatkvRootDir) snapDir := filepath.Join(flatkvDir, snapshotName(5)) - for _, sub := range snapshotDBDirs { + for _, sub := range dataDBDirs { require.NoError(t, os.MkdirAll(filepath.Join(snapDir, sub), 0750)) } @@ -1720,51 +1669,6 @@ func TestSingleDBOpenFailure(t *testing.T) { require.Error(t, err, "open should fail when storageDB is corrupted in both working and snapshot") } -// ============================================================================= -// Global Metadata Corruption (W-P3-2) -// ============================================================================= - -func TestGlobalMetadataCorruption(t *testing.T) { - dir := t.TempDir() - dbDir := filepath.Join(dir, flatkvRootDir) - - cfg := config.DefaultConfig() - cfg.DataDir = dbDir - s, err := newCommitStoreWithWAL(t.Context(), cfg) - require.NoError(t, err) - err = s.LoadLatest() - require.NoError(t, err) - commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0xAA}) - require.NoError(t, s.WriteSnapshot("")) - require.NoError(t, s.Close()) - - workingMeta := filepath.Join(dbDir, "working", metadataDir) - metaCfg := pebbledb.DefaultConfig() - metaCfg.DataDir = workingMeta - metaCfg.EnableMetrics = false - db, err := pebbledb.Open(context.Background(), &metaCfg) - require.NoError(t, err) - require.NoError(t, db.Set(ktype.MetaVersionKey, []byte{0xFF, 0xFF, 0xFF}, types.WriteOptions{Sync: true})) - require.NoError(t, db.Close()) - - snapMeta := filepath.Join(dbDir, snapshotName(1), metadataDir) - metaCfg2 := pebbledb.DefaultConfig() - metaCfg2.DataDir = snapMeta - metaCfg2.EnableMetrics = false - db2, err := pebbledb.Open(context.Background(), &metaCfg2) - require.NoError(t, err) - require.NoError(t, db2.Set(ktype.MetaVersionKey, []byte{0xFF, 0xFF, 0xFF}, types.WriteOptions{Sync: true})) - require.NoError(t, db2.Close()) - _ = os.Remove(filepath.Join(dbDir, "working", snapshotBaseFile)) - - cfg2 := config.DefaultConfig() - cfg2.DataDir = dbDir - s2, err := newCommitStoreWithWAL(context.Background(), cfg2) - require.NoError(t, err) - err = s2.LoadLatest() - require.Error(t, err, "open should fail when global metadata is corrupted") -} - // ============================================================================= // WAL Directory Deleted (W-P3-5) // ============================================================================= @@ -1850,7 +1754,7 @@ func TestLocalMetaCorruption(t *testing.T) { require.NoError(t, err) err = s2.LoadLatest() require.Error(t, err, "open should fail when meta version is corrupted") - require.Contains(t, err.Error(), "invalid meta version length") + require.Contains(t, err.Error(), "invalid _meta/version length") } // TestWALSegmentCorruption simulates WAL data loss caused by segment corruption. @@ -1872,11 +1776,11 @@ func TestWALSegmentCorruption(t *testing.T) { commitStorageEntry(t, s, ktype.Address{0x02}, ktype.Slot{0x02}, []byte{0xBB}) // v2 require.NoError(t, s.Close()) - // Simulate crash between commitBatches (v2 written) and commitGlobalMetadata: - // rewind global version to v1 so catchup needs to replay v2 from WAL. - workingMeta := filepath.Join(dbDir, "working", metadataDir) + // Simulate a torn commit: rewind accountDB's version record to v1 so the + // store's watermark drops to v1 and catchup needs to replay v2 from the WAL. + workingAccount := filepath.Join(dbDir, "working", accountDBDir) metaCfg := pebbledb.DefaultConfig() - metaCfg.DataDir = workingMeta + metaCfg.DataDir = workingAccount metaCfg.EnableMetrics = false mdb, err := pebbledb.Open(context.Background(), &metaCfg) require.NoError(t, err) @@ -1902,7 +1806,7 @@ func TestWALSegmentCorruption(t *testing.T) { } require.Greater(t, corrupted, 0, "should have found at least one WAL segment to corrupt") - // Request version 2: global says v1, WAL auto-truncated (empty), can't catchup to v2. + // Request version 2: the watermark says v1, the WAL auto-truncated (empty), so v2 is unreachable. cfg2 := config.DefaultConfig() cfg2.DataDir = dbDir s2, err := newCommitStoreWithWAL(context.Background(), cfg2) @@ -2007,9 +1911,9 @@ func TestAccountRowDeleteSurvivesWALReplay(t *testing.T) { hashAtV2 := s.RootHash() require.NoError(t, s.Close()) - // Simulate crash: rewind global version to v1 so catchup must replay v2 + // Simulate a torn commit: rewind accountDB's version record to v1 so catchup must replay v2 metaCfg := pebbledb.DefaultTestConfig(t) - metaCfg.DataDir = filepath.Join(dbDir, "working", metadataDir) + metaCfg.DataDir = filepath.Join(dbDir, "working", accountDBDir) mdb, err := pebbledb.Open(context.Background(), &metaCfg) require.NoError(t, err) versionBuf := make([]byte, 8) diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 5f7c07c0c9..9b59502e9a 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" "runtime" @@ -44,7 +45,6 @@ const ( codeDBDir = "code" storageDBDir = "storage" miscDBDir = "misc" - metadataDir = "metadata" // Suffixes for atomic directory operations tmpSuffix = "-tmp" @@ -86,13 +86,12 @@ type CommitStore struct { config config.Config dbDir string - // Five separate PebbleDB instances. + // Four separate PebbleDB instances. // Physical key format: "module/" + type_prefix + stripped_key. - metadataDB seidbtypes.KeyValueDB // Global version + LtHash watermark - accountDB seidbtypes.KeyValueDB // "evm/"+0x0a+addr(20) → vtype.AccountData - codeDB seidbtypes.KeyValueDB // "evm/"+0x07+addr(20) → vtype.CodeData - storageDB seidbtypes.KeyValueDB // "evm/"+0x03+addr(20)||slot(32) → vtype.StorageData - miscDB seidbtypes.KeyValueDB // "module/"+key → vtype.MiscData + accountDB seidbtypes.KeyValueDB // "evm/"+0x0a+addr(20) → vtype.AccountData + codeDB seidbtypes.KeyValueDB // "evm/"+0x07+addr(20) → vtype.CodeData + storageDB seidbtypes.KeyValueDB // "evm/"+0x03+addr(20)||slot(32) → vtype.StorageData + miscDB seidbtypes.KeyValueDB // "module/"+key → vtype.MiscData // Per-DB committed version, keyed by DB dir name (e.g. accountDBDir). localMeta map[string]*ktype.LocalMeta @@ -102,12 +101,6 @@ type CommitStore struct { committedLtHash *lthash.LtHash workingLtHash *lthash.LtHash - // earliestVersion is the version this store's history begins at, as - // recorded by SetInitialVersion (the seeded version). 0 when unknown: - // genesis stores and stores created before the record existed. See - // EarliestVersion. - earliestVersion int64 - // Per-DB working LTHash tracking. Authoritative copies live in each // DB's LocalMeta (atomically committed with data). On startup the // working hashes are loaded from LocalMeta. @@ -185,7 +178,7 @@ type CommitStore struct { } // dataDBs returns the four data PebbleDB instances in fixed iteration order: -// accountDB, codeDB, storageDB, miscDB. metadataDB is excluded. +// accountDB, codeDB, storageDB, miscDB. func (s *CommitStore) dataDBs() []seidbtypes.KeyValueDB { return []seidbtypes.KeyValueDB{s.accountDB, s.codeDB, s.storageDB, s.miscDB} } @@ -410,7 +403,6 @@ func (s *CommitStore) LoadVersionReadOnly(targetVersion int64) (opened Store, re ro.config.CodeDBConfig.DataDir = filepath.Join(workDir, codeDBDir) ro.config.StorageDBConfig.DataDir = filepath.Join(workDir, storageDBDir) ro.config.MiscDBConfig.DataDir = filepath.Join(workDir, miscDBDir) - ro.config.MetadataDBConfig.DataDir = filepath.Join(workDir, metadataDir) // Transfer the lazily-acquired lock to the view so that ro.Close() // releases it, preventing a leak when this store is never closed. @@ -617,7 +609,6 @@ func (s *CommitStore) openDBs(dbDir string) (retErr error) { for _, c := range toClose { _ = c.Close() } - s.metadataDB = nil s.accountDB = nil s.codeDB = nil s.storageDB = nil @@ -651,12 +642,6 @@ func (s *CommitStore) openDBs(dbDir string) (retErr error) { } toClose = append(toClose, s.miscDB) - s.metadataDB, err = s.openPebbleDB(&s.config.MetadataDBConfig, &s.config.MetadataCacheConfig) - if err != nil { - return fmt.Errorf("failed to open metadata DB: %w", err) - } - toClose = append(toClose, s.metadataDB) - for _, ndb := range s.namedDataDBs() { meta, err := loadLocalMeta(ndb.db) if err != nil { @@ -668,34 +653,20 @@ func (s *CommitStore) openDBs(dbDir string) (retErr error) { return nil } +// loadGlobalMetadata rebuilds the store's in-memory global state from the data +// DBs' metadata. func (s *CommitStore) loadGlobalMetadata() error { - globalVersion, err := s.loadGlobalVersion() - if err != nil { - return fmt.Errorf("failed to load global version: %w", err) - } - s.committedVersion = globalVersion - - earliestVersion, err := s.loadGlobalEarliestVersion() - if err != nil { - return fmt.Errorf("failed to load global earliest version: %w", err) - } - s.earliestVersion = earliestVersion - - globalLtHash, err := s.loadGlobalLtHash() - if err != nil { - return fmt.Errorf("failed to load global LtHash: %w", err) - } - if globalLtHash != nil { - s.committedLtHash = globalLtHash - s.workingLtHash = globalLtHash.Clone() - } else { - s.committedLtHash = lthash.New() - s.workingLtHash = lthash.New() + if err := s.hydratePerDBState(); err != nil { + return err } + s.deriveGlobalState() + return nil +} - // Load per-DB LtHashes from each DB's LocalMeta (already loaded in openDBs). - // If any DB's version is behind the global version (partial commit or - // corruption), lower committedVersion so catchup replays from there. +// hydratePerDBState populates the working per-DB and per-module hash state from +// each data DB's LocalMeta. It rejects a DB whose per-module hashes do not sum +// to its recorded root. +func (s *CommitStore) hydratePerDBState() error { for _, dbDir := range dataDBDirs { meta := s.localMeta[dbDir] if err := validatePerModuleMetadata(dbDir, meta); err != nil { @@ -713,15 +684,56 @@ func (s *CommitStore) loadGlobalMetadata() error { s.perDBModuleWorkingLtHash[dbDir] = make(map[string]*lthash.LtHash) s.perDBModuleWorkingStats[dbDir] = make(map[string]lthash.ModuleStats) } - if meta != nil && meta.CommittedVersion < s.committedVersion { - logger.Warn("DB LocalMeta version behind global version, will catchup", + } + return nil +} + +// deriveGlobalState sets the committed version to the lowest version any data DB +// reached and the committed LtHash to the homomorphic sum of their roots. +func (s *CommitStore) deriveGlobalState() { + version := int64(math.MaxInt64) + global := lthash.New() + for _, dbDir := range dataDBDirs { + global.MixIn(s.perDBWorkingLtHash[dbDir]) + if meta := s.localMeta[dbDir]; meta != nil && meta.CommittedVersion < version { + version = meta.CommittedVersion + } + } + if version == math.MaxInt64 { + version = 0 + } + + for _, dbDir := range dataDBDirs { + if meta := s.localMeta[dbDir]; meta != nil && meta.CommittedVersion > version { + logger.Warn("data DB versions disagree, catchup will replay from the lowest", "db", dbDir, "localVersion", meta.CommittedVersion, - "globalVersion", s.committedVersion) - s.committedVersion = meta.CommittedVersion + "storeVersion", version) } } + s.committedVersion = version + s.committedLtHash = global + s.workingLtHash = global.Clone() +} + +// requireAlignedDataDBs returns an error unless every data DB sits at the +// store's committed version. The condition holds once catchup has run; before +// then the DBs may legally disagree. +func (s *CommitStore) requireAlignedDataDBs() error { + for _, dbDir := range dataDBDirs { + meta := s.localMeta[dbDir] + if meta == nil { + return fmt.Errorf("flatkv: %s has no local metadata after load", dbDir) + } + if meta.CommittedVersion != s.committedVersion { + return fmt.Errorf( + "flatkv: %s is at version %d but the store is at %d; this store holds a block "+ + "its write-ahead log lost, which no replay can reconcile (restore from a snapshot)", + dbDir, meta.CommittedVersion, s.committedVersion, + ) + } + } return nil } @@ -747,11 +759,6 @@ func (s *CommitStore) CommittedRootHash() []byte { return checksum[:] } -// EarliestVersion implements Store. -func (s *CommitStore) EarliestVersion() int64 { - return s.earliestVersion -} - func (s *CommitStore) Importer(version int64) (types.Importer, error) { if s.readOnly { return nil, errReadOnly @@ -888,9 +895,6 @@ func InitializeDataDirectories(c *config.Config) { if c.MiscDBConfig.DataDir == "" { c.MiscDBConfig.DataDir = filepath.Join(workDir, miscDBDir) } - if c.MetadataDBConfig.DataDir == "" { - c.MetadataDBConfig.DataDir = filepath.Join(workDir, metadataDir) - } applyPebbleMetricsConfig(c) } @@ -901,11 +905,9 @@ func applyPebbleMetricsConfig(c *config.Config) { c.CodeDBConfig.EnableMetrics = c.EnablePebbleMetrics c.StorageDBConfig.EnableMetrics = c.EnablePebbleMetrics c.MiscDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.MetadataDBConfig.EnableMetrics = c.EnablePebbleMetrics c.AccountDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics c.CodeDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics c.StorageDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics c.MiscDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.MetadataDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics } diff --git a/sei-db/state_db/sc/flatkv/store_iteration.go b/sei-db/state_db/sc/flatkv/store_iteration.go index 937c3e91ed..01caaceb68 100644 --- a/sei-db/state_db/sc/flatkv/store_iteration.go +++ b/sei-db/state_db/sc/flatkv/store_iteration.go @@ -16,7 +16,7 @@ import ( // RawGlobalIterator returns an iterator over all committed keys across the // data DBs (account, code, storage, misc), merged in global lexicographic // order. Within each DB, keys are in Pebble order. Per-DB _meta/* keys are -// skipped. Pending writes are not visible. metadataDB is not included. +// skipped. Pending writes are not visible. func (s *CommitStore) RawGlobalIterator() (dbm.Iterator, error) { // Read lock for the construction span: the returned iterator pins a Pebble // view and may then outlive a concurrent ApplyChangeSets/Commit. diff --git a/sei-db/state_db/sc/flatkv/store_lifecycle.go b/sei-db/state_db/sc/flatkv/store_lifecycle.go index f32fe58b13..51db39f477 100644 --- a/sei-db/state_db/sc/flatkv/store_lifecycle.go +++ b/sei-db/state_db/sc/flatkv/store_lifecycle.go @@ -13,8 +13,8 @@ import ( // isClosed reports whether the store's DB handles have been released. func (s *CommitStore) isClosed() bool { - return s.metadataDB == nil && s.accountDB == nil && - s.codeDB == nil && s.storageDB == nil && s.miscDB == nil + return s.accountDB == nil && s.codeDB == nil && + s.storageDB == nil && s.miscDB == nil } // closeDBsOnly closes all database handles but retains the file lock, preventing a race window during @@ -24,13 +24,6 @@ func (s *CommitStore) isClosed() bool { func (s *CommitStore) closeDBsOnly() error { var errs []error - if s.metadataDB != nil { - if err := s.metadataDB.Close(); err != nil { - errs = append(errs, fmt.Errorf("metadataDB close: %w", err)) - } - s.metadataDB = nil - } - if s.storageDB != nil { if err := s.storageDB.Close(); err != nil { errs = append(errs, fmt.Errorf("storageDB close: %w", err)) diff --git a/sei-db/state_db/sc/flatkv/store_meta.go b/sei-db/state_db/sc/flatkv/store_meta.go index 08a8fd8f5c..636d7f7d95 100644 --- a/sei-db/state_db/sc/flatkv/store_meta.go +++ b/sei-db/state_db/sc/flatkv/store_meta.go @@ -38,10 +38,10 @@ func loadLocalMeta(db types.KeyValueDB) (*ktype.LocalMeta, error) { } return nil, fmt.Errorf("could not read meta version: %w", err) } - if len(versionData) != 8 { - return nil, fmt.Errorf("invalid meta version length: got %d, want 8", len(versionData)) + meta.CommittedVersion, err = decodeVersion(ktype.MetaVersionKey, versionData) + if err != nil { + return nil, err } - meta.CommittedVersion = int64(binary.BigEndian.Uint64(versionData)) //nolint:gosec // version won't exceed int64 max hashData, err := db.Get(ktype.MetaLtHashKey) if err != nil && !errorutils.IsNotFound(err) { @@ -232,77 +232,19 @@ func cloneModuleStats(src map[string]lthash.ModuleStats) map[string]lthash.Modul return dst } -// loadGlobalVersion reads the global committed version from metadata DB. -// Returns 0 if not found (fresh start). -func (s *CommitStore) loadGlobalVersion() (int64, error) { - data, err := s.metadataDB.Get(ktype.MetaVersionKey) - if errorutils.IsNotFound(err) { - return 0, nil - } - if err != nil { - return 0, fmt.Errorf("failed to read global version: %w", err) - } +// decodeVersion parses an 8-byte big-endian version record; key names it in +// error messages. +func decodeVersion(key []byte, data []byte) (int64, error) { if len(data) != 8 { - return 0, fmt.Errorf("invalid global version length: got %d, want 8", len(data)) + return 0, fmt.Errorf("invalid %s length: got %d, want 8", key, len(data)) } v := binary.BigEndian.Uint64(data) if v > math.MaxInt64 { - return 0, fmt.Errorf("global version overflow: %d exceeds max int64", v) + return 0, fmt.Errorf("%s overflow: %d exceeds max int64", key, v) } return int64(v), nil //nolint:gosec // overflow checked above } -// loadGlobalEarliestVersion reads the earliest-history version recorded by -// SetInitialVersion. Returns 0 if not found (genesis stores, or stores -// created before this record existed). -func (s *CommitStore) loadGlobalEarliestVersion() (int64, error) { - data, err := s.metadataDB.Get(ktype.MetaEarliestVersionKey) - if errorutils.IsNotFound(err) { - return 0, nil - } - if err != nil { - return 0, fmt.Errorf("failed to read global earliest version: %w", err) - } - if len(data) != 8 { - return 0, fmt.Errorf("invalid global earliest version length: got %d, want 8", len(data)) - } - v := binary.BigEndian.Uint64(data) - if v > math.MaxInt64 { - return 0, fmt.Errorf("global earliest version overflow: %d exceeds max int64", v) - } - return int64(v), nil //nolint:gosec // overflow checked above -} - -// loadGlobalLtHash reads the global committed LtHash from metadata DB. -// Returns nil if not found (fresh start). -func (s *CommitStore) loadGlobalLtHash() (*lthash.LtHash, error) { - data, err := s.metadataDB.Get(ktype.MetaLtHashKey) - if errorutils.IsNotFound(err) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("failed to read global lthash: %w", err) - } - return lthash.Unmarshal(data) -} - -// commitGlobalMetadata atomically commits global version and global LtHash -// to metadata DB. Per-DB LtHashes are stored in each DB's LocalMeta -// (committed atomically with data in commitBatches). -func (s *CommitStore) commitGlobalMetadata(version int64, hash *lthash.LtHash) error { - batch := s.metadataDB.NewBatch() - defer func() { _ = batch.Close() }() - - if err := batch.Set(ktype.MetaVersionKey, versionToBytes(version)); err != nil { - return fmt.Errorf("failed to set global version: %w", err) - } - if err := batch.Set(ktype.MetaLtHashKey, hash.Marshal()); err != nil { - return fmt.Errorf("failed to set global lthash: %w", err) - } - - return batch.Commit(types.WriteOptions{Sync: s.config.Fsync}) -} - // newPerDBLtHashMap returns a map with a fresh zero LtHash for each data DB. func newPerDBLtHashMap() map[string]*lthash.LtHash { m := make(map[string]*lthash.LtHash, len(dataDBDirs)) @@ -338,13 +280,11 @@ func newPerDBModuleStatsMap() map[string]map[string]lthash.ModuleStats { // rejected on read-only stores, and persists durably across restart. // // Implementation notes: -// - We persist version = initialVersion - 1 to both the global metadata DB -// and every per-DB LocalMeta, so Commit(initialVersion) is ahead of the -// current watermark. -// - Write order is "global first, per-DB second" so that any partial-write -// crash recovers as "fresh store" (loadGlobalMetadata lowers the global -// watermark to the minimum per-DB watermark; per-DB at 0 forces global -// back to 0). A retry with the same initialVersion is idempotent. +// - We persist version = initialVersion - 1 to every per-DB LocalMeta, so +// Commit(initialVersion) is ahead of the current watermark. +// - Any partial-write crash recovers as "fresh store": the store's watermark +// is the minimum per-DB watermark, so an unseeded DB holds it at 0. A retry +// with the same initialVersion is idempotent. // - LtHashes stay at their zero values (lthash.New()) — a freshly seeded // store has no data, so committed/working LtHashes remain the identity. func (s *CommitStore) SetInitialVersion(initialVersion int64) error { @@ -358,34 +298,12 @@ func (s *CommitStore) SetInitialVersion(initialVersion int64) error { return fmt.Errorf("flatkv: SetInitialVersion can only be called on a fresh store; committedVersion=%d", s.committedVersion) } - if s.metadataDB == nil { + if s.miscDB == nil { return fmt.Errorf("flatkv: SetInitialVersion called before LoadLatest") } seededVersion := initialVersion - 1 - if err := s.commitGlobalMetadata(seededVersion, s.committedLtHash); err != nil { - return fmt.Errorf("flatkv: SetInitialVersion: persist global metadata: %w", err) - } - - // Record where this store's history begins. Versions below this mark - // predate the store entirely (the chain ran without flatkv), which is - // distinct from pruned or corrupt in-history versions; the composite - // store's era-aware read-only path keys on it. - { - batch := s.metadataDB.NewBatch() - if err := batch.Set(ktype.MetaEarliestVersionKey, versionToBytes(seededVersion)); err != nil { - _ = batch.Close() - return fmt.Errorf("flatkv: SetInitialVersion: set earliest version: %w", err) - } - if err := batch.Commit(types.WriteOptions{Sync: s.config.Fsync}); err != nil { - _ = batch.Close() - return fmt.Errorf("flatkv: SetInitialVersion: persist earliest version: %w", err) - } - _ = batch.Close() - s.earliestVersion = seededVersion - } - syncOpt := types.WriteOptions{Sync: s.config.Fsync} for _, ndb := range s.namedDataDBs() { ltHash := s.perDBWorkingLtHash[ndb.dir] @@ -430,66 +348,39 @@ func (s *CommitStore) SetInitialVersion(initialVersion int64) error { // process startup). Returns 0 when the store has never been opened or // has no commits yet. // -// The truth source is MetaVersionKey in working/metadata. The working +// The truth source is MetaVersionKey in working/misc. The working // dir survives across restarts and is updated on every Commit, so this // matches the precision of memiavl.GetLatestVersion (which reads the // WAL tail). It must not be called concurrently with a running // CommitStore on dir, because the underlying PebbleDB takes an // exclusive file lock. +// An absent directory or key reads as 0. func GetLatestVersion(dir string) (int64, error) { - return readVersionRecord(dir, ktype.MetaVersionKey) -} - -// GetEarliestVersion returns the version the history of the store under dir -// begins at, without holding an open *CommitStore. It is the on-disk twin of -// CommitStore.EarliestVersion, and carries the same meaning: a non-zero result -// means versions below it predate the store entirely, as opposed to pruned or -// corrupt in-history versions. Returns 0 when the store was never seeded. -// -// The truth source is MetaEarliestVersionKey in working/metadata, written once -// by SetInitialVersion. It never travels through the state WAL, so no replay -// can change it and this answer does not depend on one. Like GetLatestVersion, -// it must not be called concurrently with a running CommitStore on dir. -func GetEarliestVersion(dir string) (int64, error) { - return readVersionRecord(dir, ktype.MetaEarliestVersionKey) -} - -// readVersionRecord reads one 8-byte big-endian version record out of the -// working metadata DB under dir, opening and closing that single PebbleDB -// around the read. An absent directory or key reads as 0. -func readVersionRecord(dir string, key []byte) (int64, error) { - metaDir := filepath.Join(dir, workingDirName, metadataDir) - if _, err := os.Stat(metaDir); err != nil { + miscDir := filepath.Join(dir, workingDirName, miscDBDir) + if _, err := os.Stat(miscDir); err != nil { if os.IsNotExist(err) { return 0, nil } - return 0, fmt.Errorf("flatkv: stat working metadata dir %q: %w", metaDir, err) + return 0, fmt.Errorf("flatkv: stat working misc dir %q: %w", miscDir, err) } cfg := pebbledb.DefaultConfig() - cfg.DataDir = metaDir + cfg.DataDir = miscDir cfg.EnableMetrics = false db, err := pebbledb.Open(context.Background(), &cfg) if err != nil { - return 0, fmt.Errorf("flatkv: open working metadata at %q: %w", cfg.DataDir, err) + return 0, fmt.Errorf("flatkv: open working misc at %q: %w", cfg.DataDir, err) } defer func() { _ = db.Close() }() - data, err := db.Get(key) + data, err := db.Get(ktype.MetaVersionKey) if errorutils.IsNotFound(err) { return 0, nil } if err != nil { - return 0, fmt.Errorf("flatkv: read %s: %w", key, err) + return 0, fmt.Errorf("flatkv: read %s: %w", ktype.MetaVersionKey, err) } - if len(data) != 8 { - return 0, fmt.Errorf("flatkv: invalid %s length: got %d, want 8", key, len(data)) - } - v := binary.BigEndian.Uint64(data) - if v > math.MaxInt64 { - return 0, fmt.Errorf("flatkv: %s overflow: %d exceeds max int64", key, v) - } - return int64(v), nil //nolint:gosec // overflow checked above + return decodeVersion(ktype.MetaVersionKey, data) } // GetLatestVersion returns the latest committed version. When the store @@ -497,7 +388,7 @@ func readVersionRecord(dir string, key []byte) (int64, error) { // LoadLatest has run, it falls back to the free-standing on-disk // helper. Either path returns 0 on a fresh store. func (s *CommitStore) GetLatestVersion() (int64, error) { - if s.metadataDB != nil { + if !s.isClosed() { return s.committedVersion, nil } return GetLatestVersion(s.flatkvDir()) diff --git a/sei-db/state_db/sc/flatkv/store_meta_test.go b/sei-db/state_db/sc/flatkv/store_meta_test.go index 36a8222e39..59db35ae29 100644 --- a/sei-db/state_db/sc/flatkv/store_meta_test.go +++ b/sei-db/state_db/sc/flatkv/store_meta_test.go @@ -52,7 +52,7 @@ func TestLoadLocalMeta(t *testing.T) { _, err := loadLocalMeta(db) require.Error(t, err) - require.Contains(t, err.Error(), "invalid meta version length") + require.Contains(t, err.Error(), "invalid _meta/version length") }) } @@ -197,79 +197,6 @@ func TestStoreCommitBatchesUpdatesLocalMeta(t *testing.T) { require.Equal(t, int64(1), int64(binary.BigEndian.Uint64(data))) } -func TestStoreMetadataOperations(t *testing.T) { - t.Run("LoadGlobalVersion_NewDB", func(t *testing.T) { - s := setupTestStore(t) - defer s.Close() - - version, err := s.loadGlobalVersion() - require.NoError(t, err) - require.Equal(t, int64(0), version) - }) - - t.Run("LoadGlobalLtHash_NewDB", func(t *testing.T) { - s := setupTestStore(t) - defer s.Close() - - hash, err := s.loadGlobalLtHash() - require.NoError(t, err) - require.Nil(t, hash) - }) - - t.Run("CommitGlobalMetadata_RoundTrip", func(t *testing.T) { - s := setupTestStore(t) - defer s.Close() - - // Commit metadata - expectedVersion := int64(100) - expectedHash := lthash.New() - - err := s.commitGlobalMetadata(expectedVersion, expectedHash) - require.NoError(t, err) - - // Load it back - version, err := s.loadGlobalVersion() - require.NoError(t, err) - require.Equal(t, expectedVersion, version) - - hash, err := s.loadGlobalLtHash() - require.NoError(t, err) - require.NotNil(t, hash) - require.Equal(t, expectedHash.Marshal(), hash.Marshal()) - }) - - t.Run("CommitGlobalMetadata_Atomicity", func(t *testing.T) { - s := setupTestStore(t) - defer s.Close() - - // Commit multiple times - for v := int64(1); v <= 10; v++ { - hash := lthash.New() - err := s.commitGlobalMetadata(v, hash) - require.NoError(t, err) - - // Verify immediately - version, err := s.loadGlobalVersion() - require.NoError(t, err) - require.Equal(t, v, version) - } - }) - - t.Run("LoadGlobalVersion_InvalidData", func(t *testing.T) { - s := setupTestStore(t) - defer s.Close() - - // Write invalid data (wrong size) - err := s.metadataDB.Set(ktype.MetaVersionKey, []byte{0x01}, types.WriteOptions{}) - require.NoError(t, err) - - // Should return error - _, err = s.loadGlobalVersion() - require.Error(t, err) - require.Contains(t, err.Error(), "invalid global version length") - }) -} - // ============================================================================= // SetInitialVersion // ============================================================================= @@ -315,28 +242,6 @@ func TestSetInitialVersion_GenesisSkipsSeededSnapshot(t *testing.T) { require.Equal(t, int64(1), v, "first Commit after SetInitialVersion(1) must produce version 1") } -func TestSetInitialVersion_PersistsEarliestVersion(t *testing.T) { - cfg := config.DefaultTestConfig(t) - s, err := newCommitStoreWithWAL(t.Context(), cfg) - require.NoError(t, err) - err = s.LoadLatest() - require.NoError(t, err) - require.Equal(t, int64(0), s.EarliestVersion(), - "a fresh store has no earliest-version record") - - require.NoError(t, s.SetInitialVersion(100)) - require.Equal(t, int64(99), s.EarliestVersion()) - require.NoError(t, s.Close()) - - reopened, err := newCommitStoreWithWAL(t.Context(), cfg) - require.NoError(t, err) - err = reopened.LoadLatest() - require.NoError(t, err) - defer reopened.Close() - require.Equal(t, int64(99), reopened.EarliestVersion(), - "the earliest-version record must survive reopen") -} - func TestSetInitialVersion_RejectsAfterCommit(t *testing.T) { s := setupTestStore(t) defer s.Close() @@ -437,10 +342,10 @@ func TestSetInitialVersion_RollbackBelowSeededVersionFails(t *testing.T) { } // ============================================================================= -// Global Metadata Persistence After Commit + Reopen +// Derived Global State After Commit + Reopen // ============================================================================= -func TestGlobalMetadataPersistence(t *testing.T) { +func TestDerivedGlobalStatePersistence(t *testing.T) { dir := t.TempDir() dbDir := filepath.Join(dir, flatkvRootDir) @@ -454,13 +359,17 @@ func TestGlobalMetadataPersistence(t *testing.T) { commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0xAA}) commitStorageEntry(t, s, ktype.Address{0x02}, ktype.Slot{0x02}, []byte{0xBB}) - globalVer, err := s.loadGlobalVersion() - require.NoError(t, err) - require.Equal(t, int64(2), globalVer) - - globalHash, err := s.loadGlobalLtHash() - require.NoError(t, err) - require.Equal(t, s.committedLtHash.Checksum(), globalHash.Checksum()) + // The store keeps no global record: its version is the minimum of the data + // DBs' own version records and its root is the sum of their roots. Read both + // off disk and check the derivation rather than a stored copy. + derived := lthash.New() + for _, ndb := range s.namedDataDBs() { + meta, err := loadLocalMeta(ndb.db) + require.NoError(t, err) + require.Equal(t, int64(2), meta.CommittedVersion, "%s version record", ndb.dir) + derived.MixIn(meta.LtHash) + } + require.Equal(t, s.committedLtHash.Checksum(), derived.Checksum()) expectedHash := s.committedLtHash.Checksum() require.NoError(t, s.Close()) @@ -490,7 +399,7 @@ func TestGetLatestVersionFreshDirReturnsZero(t *testing.T) { "never-opened flatkv dir must report version 0, not an error") } -func TestGetLatestVersionAfterCommitsReadsWorkingMeta(t *testing.T) { +func TestGetLatestVersionAfterCommitsReadsWorkingMisc(t *testing.T) { dir := t.TempDir() dbDir := filepath.Join(dir, flatkvRootDir) @@ -510,7 +419,7 @@ func TestGetLatestVersionAfterCommitsReadsWorkingMeta(t *testing.T) { v, err := GetLatestVersion(dbDir) require.NoError(t, err) require.Equal(t, int64(3), v, - "helper must read MetaVersionKey from working/metadata after a clean close") + "helper must read MetaVersionKey from working/misc after a clean close") } func TestGetLatestVersionMissingKeyReturnsZero(t *testing.T) { @@ -575,53 +484,69 @@ func TestCommitStoreGetLatestVersionFallsBackToDiskWhenUnloaded(t *testing.T) { } // ============================================================================= -// GetEarliestVersion (free-standing helper) +// Data DB alignment // ============================================================================= -func TestGetEarliestVersionFreshDirReturnsZero(t *testing.T) { - dir := t.TempDir() - v, err := GetEarliestVersion(filepath.Join(dir, flatkvRootDir)) - require.NoError(t, err) - require.Equal(t, int64(0), v, - "never-opened flatkv dir must report an unseeded history, not an error") -} - -func TestGetEarliestVersionUnseededStoreReturnsZero(t *testing.T) { +// TestOpenRejectsDataDBAheadOfWAL pins the guard that makes deriving the store's +// root from the per-DB roots safe. A DB left above the WAL tail holds a block no +// replay can reconcile, and summing its root with the others would produce a root +// for a state that never existed — which is what feeds the AppHash. +func TestOpenRejectsDataDBAheadOfWAL(t *testing.T) { dir := t.TempDir() dbDir := filepath.Join(dir, flatkvRootDir) - cfg := config.DefaultConfig() + cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) require.NoError(t, s.LoadLatest()) commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0xAA}) + commitStorageEntry(t, s, ktype.Address{0x02}, ktype.Slot{0x02}, []byte{0xBB}) + + // Push accountDB one block past the WAL tail. Nothing can replay it away. + rewindVersionRecords(t, s, 3, accountDBDir) require.NoError(t, s.Close()) - v, err := GetEarliestVersion(dbDir) + cfg2 := config.DefaultTestConfig(t) + cfg2.DataDir = dbDir + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) require.NoError(t, err) - require.Equal(t, int64(0), v, - "a store that was never seeded has no earliest-history record") + defer s2.Close() + + err = s2.LoadLatest() + require.Error(t, err, "a data DB above the WAL tail must refuse to open") + require.ErrorContains(t, err, accountDBDir) + require.ErrorContains(t, err, "write-ahead log lost") } -// TestGetEarliestVersionMatchesLoadedStore is the property the composite store depends on: the free-standing -// read of working/metadata answers exactly what a loaded store reports in memory. The record is written by -// SetInitialVersion and never travels through the state WAL, so no replay is needed to see it. -func TestGetEarliestVersionMatchesLoadedStore(t *testing.T) { +// TestEmptyBlockAdvancesWatermarkAcrossReopen pins that a block touching no data +// DB still moves every DB's version record. The store's watermark is the minimum +// of those records, so a DB that skipped the block would hold the whole store +// back and force the next open to replay from there. +func TestEmptyBlockAdvancesWatermarkAcrossReopen(t *testing.T) { dir := t.TempDir() dbDir := filepath.Join(dir, flatkvRootDir) - cfg := config.DefaultConfig() + cfg := config.DefaultTestConfig(t) cfg.DataDir = dbDir s, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) require.NoError(t, s.LoadLatest()) - require.NoError(t, s.SetInitialVersion(43)) - inMemory := s.EarliestVersion() - require.Equal(t, int64(42), inMemory, "the record stores initialVersion-1") + + commitStorageEntry(t, s, ktype.Address{0x01}, ktype.Slot{0x01}, []byte{0xAA}) + _, err = s.Commit(s.Version() + 1) // block 2: no ApplyChangeSets at all + require.NoError(t, err) + require.Equal(t, int64(2), s.Version()) + + for _, ndb := range s.namedDataDBs() { + meta, err := loadLocalMeta(ndb.db) + require.NoError(t, err) + require.Equal(t, int64(2), meta.CommittedVersion, + "%s must record the empty block, not stay behind at 1", ndb.dir) + } require.NoError(t, s.Close()) - v, err := GetEarliestVersion(dbDir) + v, err := GetLatestVersion(dbDir) require.NoError(t, err) - require.Equal(t, inMemory, v) + require.Equal(t, int64(2), v, "the empty block must be durable, not replayed again") } diff --git a/sei-db/state_db/sc/flatkv/store_replay.go b/sei-db/state_db/sc/flatkv/store_replay.go index db1cddd20b..7088c00b90 100644 --- a/sei-db/state_db/sc/flatkv/store_replay.go +++ b/sei-db/state_db/sc/flatkv/store_replay.go @@ -13,12 +13,22 @@ import ( // persists nothing. Everything under them is shared, ordered callers first. // replayIntoMutableStore brings this store up to targetVersion from its own WAL, or to the end of the WAL when -// targetVersion <= 0, and then persists the result so a later open does not replay it again. +// targetVersion <= 0, and rejects a store whose data DBs did not all reach that version. // -// It runs at startup (open/openTo) and during Rollback, never concurrently with live commits, so it reads the -// WAL unlocked. A nil WAL is legal only if this store already sits at targetVersion — the outer context owns -// the WAL pipeline in that case. -func (s *CommitStore) replayIntoMutableStore(targetVersion int64) (err error) { +// It runs at startup (open/openTo) and during Rollback, never concurrently with live commits. +func (s *CommitStore) replayIntoMutableStore(targetVersion int64) error { + if err := s.catchUpFromWAL(targetVersion); err != nil { + return err + } + return s.requireAlignedDataDBs() +} + +// catchUpFromWAL replays this store's own WAL up to targetVersion, or to the end of the WAL when +// targetVersion <= 0. +// +// It reads the WAL unlocked; callers must not run it concurrently with live commits. A nil WAL is legal only +// if this store already sits at targetVersion — the outer context owns the WAL pipeline in that case. +func (s *CommitStore) catchUpFromWAL(targetVersion int64) (err error) { var replayed int obs := s.observeOp("catchup", otelMetrics.CatchupLatency, "targetVersion", targetVersion) // Replayed blocks are reported regardless of outcome. CurrentVersion is intentionally NOT recorded here — @@ -59,28 +69,35 @@ func (s *CommitStore) replayIntoMutableStore(targetVersion int64) (err error) { } if !s.config.Fsync { - // With Fsync=false, per-block batch commits may leave data only in OS/page cache. Flush once before - // advancing global metadata so the global watermark never gets ahead of data durability. + // With Fsync=false, per-block batch commits may leave data only in OS/page cache. Flushing here bounds + // how much of a long catchup a crash forces us to redo. if err = s.flushAllDBs(); err != nil { return fmt.Errorf("catchup flush: %w", err) } } - if err = s.commitGlobalMetadata(s.committedVersion, s.committedLtHash); err != nil { - return fmt.Errorf("catchup global meta: %w", err) - } logger.Info("FlatKV catchup complete", "replayed", replayed, "version", s.committedVersion, "elapsed", obs.elapsed()) return nil } // replayIntoReadOnlyCopy advances a read-only clone from the snapshot boundary it opened at up to targetVersion, -// or to this store's latest WAL block when targetVersion <= 0. +// or to this store's latest WAL block when targetVersion <= 0, and rejects a clone whose data DBs did not all +// reach that version. // // The clone has a nil WAL of its own — this store owns the WAL — so the blocks it needs have to be fed to it // from here. Nothing is persisted afterwards: the clone's databases live in a directory discarded on Close. A // gap between the clone's snapshot boundary and the start of the WAL fails only the export; this store's own // state is untouched. func (s *CommitStore) replayIntoReadOnlyCopy(clone *CommitStore, targetVersion int64) error { + if err := s.feedWALToReadOnlyCopy(clone, targetVersion); err != nil { + return err + } + return clone.requireAlignedDataDBs() +} + +// feedWALToReadOnlyCopy replays this store's WAL into clone up to targetVersion, or to the latest WAL block +// when targetVersion <= 0. +func (s *CommitStore) feedWALToReadOnlyCopy(clone *CommitStore, targetVersion int64) error { if s.wal == nil { if targetVersion > 0 && clone.committedVersion != targetVersion { return fmt.Errorf("readonly: nil WAL cannot replay to version %d (opened at %d)", diff --git a/sei-db/state_db/sc/flatkv/store_replay_test.go b/sei-db/state_db/sc/flatkv/store_replay_test.go index 6cda933714..f1f959f862 100644 --- a/sei-db/state_db/sc/flatkv/store_replay_test.go +++ b/sei-db/state_db/sc/flatkv/store_replay_test.go @@ -1,6 +1,7 @@ package flatkv import ( + "bytes" "path/filepath" "testing" @@ -8,7 +9,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/stretchr/testify/require" ) @@ -136,7 +136,7 @@ func TestLoadVersionSurfacesCatchupGap(t *testing.T) { require.NoError(t, s.CommitBlock(10, []*proto.NamedChangeSet{cs})) // Rewind the persisted watermark so the reopened store needs blocks 6-9, which this WAL never held. - require.NoError(t, s.commitGlobalMetadata(5, lthash.New())) + rewindVersionRecords(t, s, 5) require.NoError(t, s.Close()) reopened, err := newCommitStoreWithWAL(t.Context(), cfg) @@ -328,3 +328,66 @@ func TestReplayIntoReadOnlyCopyDoesNotDisturbPrimary(t *testing.T) { require.Equal(t, primaryVersion, s.committedVersion, "feeding a clone must not move the primary") require.Equal(t, primaryHash, s.RootHash()) } + +// TestReplayConvergesOnPartialAccountFieldWrites pins the one case where replaying +// a block into a DB that already holds it is not obviously a no-op. An account row +// is a merge, not an overwrite: deriveNewAccountValues folds a nonce-only or +// codehash-only update onto whatever is currently on disk. Replaying a range where +// different blocks touch different fields therefore rebuilds the row field by field +// through intermediate values that were never on-chain. It converges because the +// last block to write each field writes it last — and the LtHash must land on the +// same value either way, since that value feeds the AppHash. +func TestReplayConvergesOnPartialAccountFieldWrites(t *testing.T) { + dir := t.TempDir() + dbDir := filepath.Join(dir, flatkvRootDir) + + cfg := config.DefaultTestConfig(t) + cfg.DataDir = dbDir + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + + addr := ktype.Address{0xAB} + // Block 1 sets the nonce, block 2 is unrelated, block 3 sets only the codehash. + require.NoError(t, s.CommitBlock(1, []*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addr, 7)}}, + }})) + require.NoError(t, s.CommitBlock(2, []*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ + {Key: keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addr, ktype.Slot{0x01})), + Value: padLeft32(0x22)}, + }}, + }})) + require.NoError(t, s.CommitBlock(3, []*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{codePair(addr, []byte{0x60, 0x0A})}}, + }})) + + wantRoot := bytes.Clone(s.CommittedRootHash()) + wantAccount, found := s.Get(keys.EVMStoreKey, keys.BuildEVMKey(keys.EVMKeyNonce, addr[:])) + require.True(t, found) + require.NoError(t, s.Close()) + + // Rewind accountDB alone to block 1, leaving its rows at block 3. Replay of + // blocks 2 and 3 now runs against an account row that already holds both fields. + s2, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s2.LoadLatest()) + rewindVersionRecords(t, s2, 1, accountDBDir) + require.NoError(t, s2.Close()) + + s3, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s3.LoadLatest()) + defer s3.Close() + + require.Equal(t, int64(3), s3.Version()) + require.Equal(t, wantRoot, s3.CommittedRootHash(), + "rebuilding an account row through partial-field replays must land on the same root") + gotAccount, found := s3.Get(keys.EVMStoreKey, keys.BuildEVMKey(keys.EVMKeyNonce, addr[:])) + require.True(t, found) + require.Equal(t, wantAccount, gotAccount) + require.NoError(t, VerifyLtHash(s3)) +} diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index 12e5953087..ad733d2f22 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -55,7 +55,6 @@ func TestInitializeDataDirectoriesPropagatesPebbleMetrics(t *testing.T) { cfg.CodeDBConfig.EnableMetrics = true cfg.StorageDBConfig.EnableMetrics = true cfg.MiscDBConfig.EnableMetrics = true - cfg.MetadataDBConfig.EnableMetrics = true InitializeDataDirectories(cfg) @@ -63,7 +62,6 @@ func TestInitializeDataDirectoriesPropagatesPebbleMetrics(t *testing.T) { require.False(t, cfg.CodeDBConfig.EnableMetrics) require.False(t, cfg.StorageDBConfig.EnableMetrics) require.False(t, cfg.MiscDBConfig.EnableMetrics) - require.False(t, cfg.MetadataDBConfig.EnableMetrics) } func TestStoreClose(t *testing.T) { @@ -1638,41 +1636,6 @@ func TestCrashRecoveryLtHashConsistencyAfterAllPaths(t *testing.T) { verifyLtHashConsistency(t, s3) } -func TestCrashRecoveryCorruptLtHashBlobInMetadata(t *testing.T) { - dir := t.TempDir() - cfg := config.DefaultTestConfig(t) - cfg.DataDir = filepath.Join(dir, flatkvRootDir) - - s, err := newCommitStoreWithWAL(t.Context(), cfg) - require.NoError(t, err) - err = s.LoadLatest() - require.NoError(t, err) - - cs := makeChangeSet( - keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addrN(0x01), slotN(0x01))), - padLeft32(0x11), false, - ) - require.NoError(t, s.ApplyChangeSets(s.Version()+1, []*proto.NamedChangeSet{cs})) - _, err = s.Commit(s.Version() + 1) - require.NoError(t, err) - - // Write garbage to the global _meta/hash key in metadataDB. - batch := s.metadataDB.NewBatch() - require.NoError(t, batch.Set(ktype.MetaLtHashKey, []byte{0xDE, 0xAD, 0xBE, 0xEF})) - require.NoError(t, batch.Commit(types.WriteOptions{Sync: true})) - _ = batch.Close() - - require.NoError(t, s.Close()) - - // Reopen should fail with an LtHash unmarshal error. - s2, err := newCommitStoreWithWAL(t.Context(), cfg) - require.NoError(t, err) - defer s2.Close() - err = s2.LoadLatest() - require.Error(t, err) - require.Contains(t, err.Error(), "invalid LtHash size") -} - func TestCrashRecoveryCorruptLtHashBlobInPerDBMeta(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) @@ -1708,7 +1671,7 @@ func TestCrashRecoveryCorruptLtHashBlobInPerDBMeta(t *testing.T) { require.Contains(t, err.Error(), "invalid LtHash size") } -func TestCrashRecoveryGlobalVersionOverflow(t *testing.T) { +func TestCrashRecoveryVersionRecordOverflow(t *testing.T) { dir := t.TempDir() cfg := config.DefaultTestConfig(t) cfg.DataDir = filepath.Join(dir, flatkvRootDir) @@ -1726,10 +1689,10 @@ func TestCrashRecoveryGlobalVersionOverflow(t *testing.T) { _, err = s.Commit(s.Version() + 1) require.NoError(t, err) - // Write a version value that exceeds math.MaxInt64 to the global metadata. + // Write a version value that exceeds math.MaxInt64 to accountDB's metadata. overflowBytes := make([]byte, 8) overflowBytes[0] = 0xFF // 0xFF00000000000000 > MaxInt64 - batch := s.metadataDB.NewBatch() + batch := s.accountDB.NewBatch() require.NoError(t, batch.Set(ktype.MetaVersionKey, overflowBytes)) require.NoError(t, batch.Commit(types.WriteOptions{Sync: true})) _ = batch.Close() @@ -1742,7 +1705,7 @@ func TestCrashRecoveryGlobalVersionOverflow(t *testing.T) { defer s2.Close() err = s2.LoadLatest() require.Error(t, err) - require.Contains(t, err.Error(), "global version overflow") + require.Contains(t, err.Error(), "overflow") } func TestInitializeDataDirectories(t *testing.T) { @@ -1752,7 +1715,6 @@ func TestInitializeDataDirectories(t *testing.T) { cfg.CodeDBConfig.DataDir = "" cfg.StorageDBConfig.DataDir = "" cfg.MiscDBConfig.DataDir = "" - cfg.MetadataDBConfig.DataDir = "" InitializeDataDirectories(cfg) @@ -1760,7 +1722,6 @@ func TestInitializeDataDirectories(t *testing.T) { require.Equal(t, "/base/flatkv/working/code", cfg.CodeDBConfig.DataDir) require.Equal(t, "/base/flatkv/working/storage", cfg.StorageDBConfig.DataDir) require.Equal(t, "/base/flatkv/working/misc", cfg.MiscDBConfig.DataDir) - require.Equal(t, "/base/flatkv/working/metadata", cfg.MetadataDBConfig.DataDir) } func TestInitializeDataDirectoriesPreservesExisting(t *testing.T) { diff --git a/sei-db/state_db/sc/flatkv/store_write.go b/sei-db/state_db/sc/flatkv/store_write.go index 6c80fe3f63..60bd49a596 100644 --- a/sei-db/state_db/sc/flatkv/store_write.go +++ b/sei-db/state_db/sc/flatkv/store_write.go @@ -32,7 +32,7 @@ func (s *CommitStore) CommitBlock(version int64, changesets []*proto.NamedChange // block; version must equal the height the pending writes were stamped with. Consecutive commits must also // be contiguous: the state WAL rejects a version that skips a height, though the first block written to an // empty WAL may be any height. -// Protocol: WAL → per-DB batch (with LocalMeta) → flush → update metaDB. +// Protocol: WAL → per-DB batch (with LocalMeta) → flush. // On crash, catchup replays WAL to recover incomplete commits. func (s *CommitStore) Commit(version int64) (committed int64, err error) { start := time.Now() @@ -98,25 +98,12 @@ func (s *CommitStore) Commit(version int64) (committed int64, err error) { return version, fmt.Errorf("db commit: %w", err) } - // Step 3: Persist global metadata to metadata DB. - // This must succeed before we update in-memory state; otherwise a - // metadataDB write failure would leave committedVersion advanced while - // the caller sees an error, making the store's internal state - // inconsistent. Per-DB data is already committed (Step 2) and the WAL - // (Step 1) is the source of truth, so a restart will self-heal via - // catchup even if we fail here. - s.phaseTimer.SetPhase("commit_write_metadata") - committedLtHash := s.workingLtHash.Clone() - if err := s.commitGlobalMetadata(version, committedLtHash); err != nil { - return version, fmt.Errorf("metadata DB commit: %w", err) - } - - // Step 4: Update in-memory committed state (only after metadata persisted) + // Step 3: Update in-memory committed state. Step 2 already made this commit durable. s.phaseTimer.SetPhase("commit_update_lt_hash") s.committedVersion = version - s.committedLtHash = committedLtHash + s.committedLtHash = s.workingLtHash.Clone() - // Step 5: Clear pending buffers + // Step 4: Clear pending buffers s.phaseTimer.SetPhase("commit_clear_pending_writes") s.clearPendingWrites() recordPendingWrites(s.ctx, accountDBDir, 0) @@ -408,9 +395,10 @@ type rawKVPair struct { Value []byte } -// FinalizeImport persists per-DB metadata (version + LtHash) and global -// metadata after all import data has been written. This must be called -// exactly once at the end of an import to make the data durable across restarts. +// FinalizeImport persists each data DB's metadata (version + LtHash) after all +// import data has been written, and recomputes the store's global state from it. +// This must be called exactly once at the end of an import to make the data +// durable across restarts. func (s *CommitStore) FinalizeImport(version int64) error { syncOpt := types.WriteOptions{Sync: true} for _, ndb := range s.namedDataDBs() { @@ -441,8 +429,5 @@ func (s *CommitStore) FinalizeImport(version int64) error { s.workingLtHash = globalHash s.committedVersion = version s.committedLtHash = s.workingLtHash.Clone() - if err := s.commitGlobalMetadata(version, s.committedLtHash); err != nil { - return fmt.Errorf("import global metadata: %w", err) - } return nil } diff --git a/sei-db/state_db/sc/flatkv/testutil_test.go b/sei-db/state_db/sc/flatkv/testutil_test.go index b49f3142e7..df750b016f 100644 --- a/sei-db/state_db/sc/flatkv/testutil_test.go +++ b/sei-db/state_db/sc/flatkv/testutil_test.go @@ -23,6 +23,25 @@ import ( // ============================================================================= // evmStorageKey builds a prefix-encoded storage key for the external Get/Has API. +// rewindVersionRecords rewrites every data DB's version record, lowering the +// store's watermark the way a torn commit does. Pass one dir to skew a single DB. +func rewindVersionRecords(t *testing.T, s *CommitStore, version int64, dirs ...string) { + t.Helper() + want := make(map[string]struct{}, len(dirs)) + for _, d := range dirs { + want[d] = struct{}{} + } + for _, ndb := range s.namedDataDBs() { + if len(want) > 0 { + if _, ok := want[ndb.dir]; !ok { + continue + } + } + require.NoError(t, ndb.db.Set(ktype.MetaVersionKey, versionToBytes(version), + types.WriteOptions{Sync: true})) + } +} + func evmStorageKey(addr ktype.Address, slot ktype.Slot) []byte { internal := ktype.StorageKey(addr, slot) return keys.BuildEVMKey(keys.EVMKeyStorage, internal) diff --git a/sei-db/state_db/sc/flatkv/verify.go b/sei-db/state_db/sc/flatkv/verify.go index 49dbc98af2..447785f231 100644 --- a/sei-db/state_db/sc/flatkv/verify.go +++ b/sei-db/state_db/sc/flatkv/verify.go @@ -58,6 +58,9 @@ func verifyLtHashInternal(cs *CommitStore) error { if err != nil { return err } + if err := cs.verifyPersistedDBMetadata(ndb.dir, ndb.db, dbRoot); err != nil { + return err + } global.MixIn(dbRoot) } @@ -125,6 +128,39 @@ func scanDBByModule(db seidbtypes.KeyValueDB) (map[string]*lthash.LtHash, map[st return hashes, stats, nil } +// verifyPersistedDBMetadata reads one DB's LocalMeta off disk and checks its +// version against the store's committed version and its root against scanRoot. +func (cs *CommitStore) verifyPersistedDBMetadata( + dir string, + db seidbtypes.KeyValueDB, + scanRoot *lthash.LtHash, +) error { + meta, err := loadLocalMeta(db) + if err != nil { + return fmt.Errorf("VerifyLtHash: read %s persisted metadata: %w", dir, err) + } + if meta.CommittedVersion != cs.committedVersion { + return fmt.Errorf( + "VerifyLtHash: %s is persisted at version %d but the store is at %d", + dir, meta.CommittedVersion, cs.committedVersion, + ) + } + // A DB that has never been written records no root at all, which reads as + // the identity — the same value a scan of its empty keyspace produces. + persistedRoot := meta.LtHash + if persistedRoot == nil { + persistedRoot = lthash.New() + } + if !persistedRoot.Equal(scanRoot) { + return fmt.Errorf( + "VerifyLtHash: persisted per-DB root mismatch for %s at version %d"+ + "\n persisted: %x\n full-scan: %x", + dir, cs.committedVersion, persistedRoot.Checksum(), scanRoot.Checksum(), + ) + } + return nil +} + // verifyDBModuleMetadata checks the maintained per-module hashes and stats for // one DB against a fresh scan, verifies they homomorphically sum to the // maintained per-DB root, and returns that (scan-derived) per-DB root for the diff --git a/sei-db/tools/cmd/seidb/operations/dump_flatkv.go b/sei-db/tools/cmd/seidb/operations/dump_flatkv.go index 6e7488439b..17dbf06781 100644 --- a/sei-db/tools/cmd/seidb/operations/dump_flatkv.go +++ b/sei-db/tools/cmd/seidb/operations/dump_flatkv.go @@ -7,13 +7,8 @@ import ( "fmt" "io/fs" "os" - "path/filepath" - "strconv" - errorutils "github.com/sei-protocol/sei-chain/sei-db/common/errors" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/pebbledb" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/sei-protocol/sei-chain/sei-db/tools/utils" "github.com/spf13/cobra" @@ -37,8 +32,6 @@ const ( minReadBurstBytes = 4 << 20 bytesPerMiB = 1 << 20 - - flatkvMetadataDir = "metadata" ) const ( @@ -189,24 +182,9 @@ func isFlatKVBucket(name string) bool { // Physical keys are emitted verbatim, including their "/" + type // prefix header, because they are not byte-for-byte comparable with // memIAVL logical keys anyway (different type prefixes per domain). The -// FlatKV metadataDB and the per-DB _meta/* rows are intentionally excluded: -// they are internal bookkeeping and RawGlobalIterator already filters the -// per-DB ones for us. +// The per-DB _meta/* rows are intentionally excluded: they are internal +// bookkeeping and RawGlobalIterator already filters them for us. func DumpFlatKVData(dbDir, outputDir string, height int64, bucket string, withLtHash bool, lthashOnly bool, readLimitMiBps float64) error { - // Determine, before the main scan, whether the snapshot selected for this - // height carries an LtHash watermark. CommittedRootHash() on the opened - // store cannot tell a full-state hash apart from a partial WAL-deltas-only - // hash, so we check the selected snapshot's metadata DB directly. See - // snapshotCommittedLtHashIsFullState. - committedIsFullState := true - if withLtHash { - var probeErr error - committedIsFullState, probeErr = snapshotCommittedLtHashIsFullState(dbDir, height) - if probeErr != nil { - return fmt.Errorf("probe snapshot lthash watermark: %w", probeErr) - } - } - store, err := openFlatKVReadOnly(dbDir, height) if err != nil { return fmt.Errorf("open flatkv read-only: %w", err) @@ -216,106 +194,14 @@ func DumpFlatKVData(dbDir, outputDir string, height int64, bucket string, withLt version := store.Version() fmt.Printf("Opened FlatKV at version %d\n", version) - return dumpFlatKVFromStore(store, outputDir, version, bucket, withLtHash, lthashOnly, - committedIsFullState, readLimitMiBps) -} - -// snapshotMetadataMakesCommittedHashFullState decides, from the snapshot's own -// version and whether its metadata DB contains a global LtHash watermark, -// whether a store opened on top of that snapshot will carry a full-state -// committed LtHash (verifiable against a full re-scan). -// -// - snapshotVersion == 0: a genesis/empty baseline contributes nothing, so -// the committed hash is built entirely from replayed history and is -// full-state. -// - snapshotVersion > 0 without MetaLtHashKey: the snapshot had committed -// data but no LtHash watermark, so once any WAL replays on top the -// committed hash becomes a partial (deltas-only) hash. Not full-state. -// - snapshotVersion > 0 with MetaLtHashKey present (even if the stored -// checksum is all-zero): the snapshot's watermark exists and seeds the -// committed hash. Full-state. -func snapshotMetadataMakesCommittedHashFullState(snapshotVersion int64, hasLtHashMetadata bool) bool { - if snapshotVersion == 0 { - return true - } - return hasLtHashMetadata -} - -// snapshotCommittedLtHashIsFullState probes the FlatKV snapshot selected for -// height and reports whether a store opened on top of it will have a -// full-state committed LtHash. It checks the selected snapshot's metadata DB -// for ktype.MetaLtHashKey directly instead of using CommittedRootHash(): a -// legitimate LtHash watermark may be all-zero, so hash value alone cannot -// distinguish "metadata present" from "metadata absent". -func snapshotCommittedLtHashIsFullState(dbDir string, height int64) (bool, error) { - snapshotName, err := selectFlatKVSnapshot(dbDir, height) - if err != nil { - return false, fmt.Errorf("select snapshot: %w", err) - } - snapshotVersion, err := strconv.ParseInt(snapshotName[len(flatkvSnapshotPrefix):], 10, 64) - if err != nil { - return false, fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) - } - if snapshotVersion == 0 { - return true, nil - } - - hasMetadata, err := selectedSnapshotHasLtHashMetadata(dbDir, snapshotName) - if err != nil { - return false, err - } - return snapshotMetadataMakesCommittedHashFullState(snapshotVersion, hasMetadata), nil -} - -// selectedSnapshotHasLtHashMetadata checks whether the selected immutable -// snapshot's metadata DB contains the global LtHash watermark key. The source -// snapshot is not opened directly: Pebble would create lock/log files. Instead -// we hardlink-clone just the metadata DB into a temp dir under dbDir, open that -// clone, and read ktype.MetaLtHashKey. -func selectedSnapshotHasLtHashMetadata(dbDir, snapshotName string) (bool, error) { - srcMetadataDir := filepath.Join(dbDir, snapshotName, flatkvMetadataDir) - if _, err := os.Stat(srcMetadataDir); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, fmt.Errorf("stat snapshot metadata dir %s: %w", srcMetadataDir, err) - } - - tempDir, err := os.MkdirTemp(dbDir, ".seidb-flatkv-meta-probe-*") - if err != nil { - return false, fmt.Errorf("create metadata probe dir under %s: %w", dbDir, err) - } - defer func() { _ = os.RemoveAll(tempDir) }() - - probeMetadataDir := filepath.Join(tempDir, flatkvMetadataDir) - if err := cloneDirRecursive(srcMetadataDir, probeMetadataDir); err != nil { - return false, fmt.Errorf("clone snapshot metadata %s: %w", srcMetadataDir, err) - } - - cfg := pebbledb.DefaultConfig() - cfg.DataDir = probeMetadataDir - cfg.EnableMetrics = false - db, err := pebbledb.Open(context.Background(), &cfg) - if err != nil { - return false, fmt.Errorf("open cloned snapshot metadata %s: %w", probeMetadataDir, err) - } - defer func() { _ = db.Close() }() - - _, err = db.Get(ktype.MetaLtHashKey) - if errorutils.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, fmt.Errorf("read snapshot LtHash metadata key: %w", err) - } - return true, nil + return dumpFlatKVFromStore(store, outputDir, version, bucket, withLtHash, lthashOnly, readLimitMiBps) } // dumpFlatKVFromStore is the core scan+write path, split out so tests can // exercise it against an in-memory store without going through the // snapshot clone machinery used by the CLI. func dumpFlatKVFromStore(store flatkv.Store, outputDir string, version int64, bucket string, - withLtHash bool, lthashOnly bool, committedIsFullState bool, readLimitMiBps float64, + withLtHash bool, lthashOnly bool, readLimitMiBps float64, ) error { limiter := newReadLimiter(readLimitMiBps) ctx := context.Background() @@ -410,16 +296,7 @@ func dumpFlatKVFromStore(store flatkv.Store, outputDir string, version int64, bu if withLtHash { printFlatKVLtHash(hashers, version) - // committedIsFullState is false when the selected snapshot predates - // LtHash metadata: the store opened with a zero baseline LtHash and - // catchup only mixed in the deltas of the WAL blocks replayed on top, - // so CommittedRootHash() is a partial hash (WAL deltas only, not the - // snapshot's pre-existing rows). Cross-checking a full re-scan against - // it would falsely fail, so skip verification; it becomes verifiable - // again once a new snapshot with LtHash metadata exists. - if !committedIsFullState { - fmt.Println("\nLtHash verification: skipped (snapshot predates LtHash metadata; committed hash covers only replayed WAL deltas, not full state)") - } else if err := verifyFlatKVLtHash(store, hashers); err != nil { + if err := verifyFlatKVLtHash(store, hashers); err != nil { return err } } diff --git a/sei-db/tools/cmd/seidb/operations/dump_flatkv_test.go b/sei-db/tools/cmd/seidb/operations/dump_flatkv_test.go index dd26abd95c..6ef734aa40 100644 --- a/sei-db/tools/cmd/seidb/operations/dump_flatkv_test.go +++ b/sei-db/tools/cmd/seidb/operations/dump_flatkv_test.go @@ -2,17 +2,13 @@ package operations import ( "bufio" - "context" "os" "path/filepath" "strings" "testing" "github.com/sei-protocol/sei-chain/sei-db/common/keys" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/pebbledb" - dbtypes "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/stretchr/testify/require" ) @@ -52,7 +48,7 @@ func TestDumpFlatKVFromStoreAllBuckets(t *testing.T) { require.NoError(t, err) outDir := t.TempDir() - require.NoError(t, dumpFlatKVFromStore(store, outDir, store.Version(), "", true, false, true, 0)) + require.NoError(t, dumpFlatKVFromStore(store, outDir, store.Version(), "", true, false, 0)) type expect struct { lines int @@ -116,7 +112,7 @@ func TestDumpFlatKVFromStoreSingleBucket(t *testing.T) { require.NoError(t, err) outDir := t.TempDir() - require.NoError(t, dumpFlatKVFromStore(store, outDir, store.Version(), "storage", true, false, true, 0)) + require.NoError(t, dumpFlatKVFromStore(store, outDir, store.Version(), "storage", true, false, 0)) // Only storage file should exist; the others must not be created. for _, name := range flatkvBucketOrder { @@ -173,74 +169,6 @@ func TestBucketLtHasherMatchesSingleShot(t *testing.T) { "MixIn of per-bucket hashes must equal the LtHash over the union of all pairs") } -// TestSnapshotMetadataMakesCommittedHashFullState pins the decision that -// drives whether dump-flatkv --lthash verifies or skips: a snapshot at version -// 0 is always full-state; a snapshot at version > 0 is full-state iff it -// carried the LtHash metadata key. Presence matters, not the hash value, -// because a legitimate LtHash watermark can be all-zero. -func TestSnapshotMetadataMakesCommittedHashFullState(t *testing.T) { - require.True(t, snapshotMetadataMakesCommittedHashFullState(0, false), - "version 0 baseline is always full-state") - require.True(t, snapshotMetadataMakesCommittedHashFullState(0, true), - "version 0 baseline is full-state regardless of metadata presence") - require.False(t, snapshotMetadataMakesCommittedHashFullState(100, false), - "version>0 without the LtHash metadata key predates LtHash metadata: not full-state") - require.True(t, snapshotMetadataMakesCommittedHashFullState(100, true), - "version>0 with the LtHash metadata key present is full-state") -} - -func TestSelectedSnapshotHasLtHashMetadata(t *testing.T) { - dbDir := t.TempDir() - snapshotName := flatkvSnapshotPrefix + "00000000000000000100" - - hasMetadata, err := selectedSnapshotHasLtHashMetadata(dbDir, snapshotName) - require.NoError(t, err) - require.False(t, hasMetadata, "missing metadata dir means the snapshot has no LtHash metadata") - - metaDir := filepath.Join(dbDir, snapshotName, flatkvMetadataDir) - require.NoError(t, os.MkdirAll(metaDir, 0o750)) - cfg := pebbledb.DefaultConfig() - cfg.DataDir = metaDir - cfg.EnableMetrics = false - db, err := pebbledb.Open(context.Background(), &cfg) - require.NoError(t, err) - - hasMetadata, err = selectedSnapshotHasLtHashMetadata(dbDir, snapshotName) - require.NoError(t, err) - require.False(t, hasMetadata, "metadata dir without MetaLtHashKey is still pre-LtHash") - - zeroHash := lthash.New() - require.NoError(t, db.Set(ktype.MetaLtHashKey, zeroHash.Marshal(), dbtypes.WriteOptions{Sync: true})) - require.NoError(t, db.Close()) - - hasMetadata, err = selectedSnapshotHasLtHashMetadata(dbDir, snapshotName) - require.NoError(t, err) - require.True(t, hasMetadata, "MetaLtHashKey presence matters even when the stored watermark is all-zero") -} - -// TestDumpFlatKVFromStoreSkipsVerifyWhenNotFullState confirms that passing -// committedIsFullState=false skips LtHash verification (returns nil) rather -// than comparing a full re-scan against a partial committed hash. -func TestDumpFlatKVFromStoreSkipsVerifyWhenNotFullState(t *testing.T) { - store := newTestFlatKVStore(t) - defer func() { require.NoError(t, store.Close()) }() - - addrA := addrN(0x11) - require.NoError(t, store.ApplyChangeSets(store.Version()+1, []*proto.NamedChangeSet{{ - Name: keys.EVMStoreKey, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{ - storagePair(addrA, slotN(0x01), 0xAA), - }}, - }})) - _, err := store.Commit(store.Version() + 1) - require.NoError(t, err) - - outDir := t.TempDir() - // committedIsFullState=false -> verification is skipped, so the dump - // succeeds even though we are not cross-checking the committed hash. - require.NoError(t, dumpFlatKVFromStore(store, outDir, store.Version(), "", true, false, false, 0)) -} - func TestDumpFlatKVFromStoreLtHashOnlyWritesNoBucketFiles(t *testing.T) { store := newTestFlatKVStore(t) defer func() { require.NoError(t, store.Close()) }() @@ -258,7 +186,7 @@ func TestDumpFlatKVFromStoreLtHashOnlyWritesNoBucketFiles(t *testing.T) { require.NoError(t, err) outDir := filepath.Join(t.TempDir(), "must-not-be-created") - require.NoError(t, dumpFlatKVFromStore(store, outDir, store.Version(), "", true, true, true, 0)) + require.NoError(t, dumpFlatKVFromStore(store, outDir, store.Version(), "", true, true, 0)) _, statErr := os.Stat(outDir) require.True(t, os.IsNotExist(statErr), "lthash-only mode must not create output dir or bucket files") } @@ -268,6 +196,5 @@ func TestIsFlatKVBucket(t *testing.T) { require.True(t, isFlatKVBucket(b), "%s should be accepted", b) } require.False(t, isFlatKVBucket(""), "empty should not validate") - require.False(t, isFlatKVBucket("metadata"), "metadata is intentionally excluded from dump-flatkv") require.False(t, isFlatKVBucket("evm"), "evm is a module, not a bucket") }