Skip to content

Commit d09acd8

Browse files
seidroid[bot]alrevueltaalexander-sei
authored
Backport release/v6.7: Fix pruning issue in SS causing huge disk spike (#4508)
Backport of #4321 to `release/v6.7`. Co-authored-by: Alvaro Revuelta <alvrevuelta@gmail.com> Co-authored-by: alexander-sei <alexanderh@seinetwork.io>
1 parent 210b31b commit d09acd8

1 file changed

Lines changed: 33 additions & 2 deletions

File tree

  • sei-db/db_engine/pebbledb/mvcc

‎sei-db/db_engine/pebbledb/mvcc/db.go‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import (
2222

2323
dbm "github.com/tendermint/tm-db"
2424

25+
"github.com/sei-protocol/seilog"
26+
2527
errorutils "github.com/sei-protocol/sei-chain/sei-db/common/errors"
2628
"github.com/sei-protocol/sei-chain/sei-db/common/utils"
2729
"github.com/sei-protocol/sei-chain/sei-db/config"
@@ -33,6 +35,8 @@ import (
3335

3436
var _ types.ContextIteratorStore = (*Database)(nil)
3537

38+
var logger = seilog.NewLogger("db", "db-engine", "pebbledb", "mvcc")
39+
3640
const (
3741
VersionSize = 8
3842

@@ -906,7 +910,22 @@ func (db *Database) pruneDescending(version int64) (_err error) {
906910
if err != nil {
907911
return err
908912
}
909-
defer func() { _ = itr.Close() }()
913+
// itr pins Pebble's readState for as long as it's open, which blocks Pebble
914+
// from deleting any sstable that readState might still need to serve reads
915+
// from — including ones a concurrent or subsequent compaction (see
916+
// compactPrunedRange below) has already superseded. itrOpen guards against
917+
// closing it twice: the scan closes it explicitly as soon as it's done
918+
// reading, right before compaction, and this defer only still applies if an
919+
// error returned from inside the loop below.
920+
itrOpen := true
921+
closeItr := func() error {
922+
if !itrOpen {
923+
return nil
924+
}
925+
itrOpen = false
926+
return itr.Close()
927+
}
928+
defer func() { _ = closeItr() }()
910929

911930
batch := db.storage.NewBatch()
912931
defer func() { _ = batch.Close() }()
@@ -1008,6 +1027,15 @@ func (db *Database) pruneDescending(version int64) (_err error) {
10081027
itr.Next()
10091028
}
10101029

1030+
// Close the scan iterator now rather than leaving it to the deferred close
1031+
// at function return: compactPrunedRange below runs a compaction that can
1032+
// take a long time on a large deleted span, and every obsolete sstable it
1033+
// produces stays undeletable for as long as this iterator's readState is
1034+
// still pinning them.
1035+
if err := closeItr(); err != nil {
1036+
return err
1037+
}
1038+
10111039
// Commit any leftover delete ops in batch
10121040
if counter > 0 {
10131041
writeCount := int64(batch.Count())
@@ -1019,7 +1047,10 @@ func (db *Database) pruneDescending(version int64) (_err error) {
10191047
}
10201048
db.operationMetrics.AddRead(scanReads)
10211049

1022-
return db.compactPrunedRange(firstDeletedKey, lastDeletedKey)
1050+
compactStart := time.Now()
1051+
err = db.compactPrunedRange(firstDeletedKey, lastDeletedKey)
1052+
logger.Info("pruneDescending: compacted pruned range", "version", version, "elapsed", time.Since(compactStart), "err", err)
1053+
return err
10231054
}
10241055

10251056
func (db *Database) iteratorDescending(ctx context.Context, storeKey string, version int64, start, end []byte) (dbm.Iterator, error) {

0 commit comments

Comments
 (0)