move pebble checkpoints off of the execution goroutine

app/config_fuzz_test.go

@@ -167,14 +167,14 @@ var genesisKeys = []configtest.KeySpec{
 // else, so the compiler keeps this list out of CheckRow and out of the discriminating-seed check,
 // where a row that predicted a resolved value would be wrong for all three.
 //
-// The four flatkv names are here for a different reason than the three above them. Nothing in this
+// The five flatkv names are here for a different reason than the three above them. Nothing in this
 // package reads them: sei-cosmos/server/config.GetConfig is their only reader, and its guardedKeys
 // target drives them. They are recorded on this section's record rather than on a second one in that
 // package so that [state-commit] has one list of operator-facing names, which is where someone
 // checking a spelling will look.
 //
 // What that record does and does not do is worth being exact about, in both directions. Renaming one
-// of these four in GetConfig fails that package's own targets, not this record, because nothing
+// of these five in GetConfig fails that package's own targets, not this record, because nothing
 // compares this list against the read site. Verified by renaming
 // state-commit.flatkv.snapshot-interval in GetConfig, which reddens three tests in sei-cosmos and
 // none here. And deleting one of them from GetConfig leaves this record green while it names a key no
@@ -192,6 +192,7 @@ var scKeysWithTargetsOfTheirOwn = []configtest.KeyName{
 	"state-commit.flatkv.async-write-buffer",
 	"state-commit.flatkv.snapshot-interval",
 	"state-commit.flatkv.snapshot-keep-recent",
+	"state-commit.flatkv.max-snapshot-lag-blocks",
 }
 
 // genesisKeysWithTargetsOfTheirOwn are the [genesis] names no row claims.

app/testdata/state-commit.golden

@@ -15,6 +15,7 @@ FlatKVConfig.Fsync = bool(false)
 FlatKVConfig.AsyncWriteBuffer = int(0)
 FlatKVConfig.SnapshotInterval = uint32(10000)
 FlatKVConfig.SnapshotKeepRecent = uint32(1)
+FlatKVConfig.MaxSnapshotLagBlocks = uint32(64)
 FlatKVConfig.ExternalPruning = bool(false)
 FlatKVConfig.EnablePebbleMetrics = bool(true)
 FlatKVConfig.EnableReadWriteMetrics = bool(false)
@@ -28,7 +29,7 @@ FlatKVConfig.AccountStoreConfig.EstimatedOverheadPerEntry = uint64(256)
 FlatKVConfig.AccountStoreConfig.Name = string("account")
 FlatKVConfig.AccountStoreConfig.MetricsEnabled = bool(true)
 FlatKVConfig.AccountStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(4)
+FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(128)
 FlatKVConfig.AccountStoreConfig.TargetBytesPerFlush = uint64(4194304)
 FlatKVConfig.AccountStoreConfig.ReservedPrefix = string("_meta/")
 FlatKVConfig.AccountStoreConfig.FlushSync = bool(false)
@@ -42,7 +43,7 @@ FlatKVConfig.CodeStoreConfig.EstimatedOverheadPerEntry = uint64(256)
 FlatKVConfig.CodeStoreConfig.Name = string("code")
 FlatKVConfig.CodeStoreConfig.MetricsEnabled = bool(true)
 FlatKVConfig.CodeStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(4)
+FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(128)
 FlatKVConfig.CodeStoreConfig.TargetBytesPerFlush = uint64(4194304)
 FlatKVConfig.CodeStoreConfig.ReservedPrefix = string("_meta/")
 FlatKVConfig.CodeStoreConfig.FlushSync = bool(false)
@@ -56,7 +57,7 @@ FlatKVConfig.StorageStoreConfig.EstimatedOverheadPerEntry = uint64(256)
 FlatKVConfig.StorageStoreConfig.Name = string("storage")
 FlatKVConfig.StorageStoreConfig.MetricsEnabled = bool(true)
 FlatKVConfig.StorageStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(4)
+FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(128)
 FlatKVConfig.StorageStoreConfig.TargetBytesPerFlush = uint64(4194304)
 FlatKVConfig.StorageStoreConfig.ReservedPrefix = string("_meta/")
 FlatKVConfig.StorageStoreConfig.FlushSync = bool(false)
@@ -70,7 +71,7 @@ FlatKVConfig.MiscStoreConfig.EstimatedOverheadPerEntry = uint64(256)
 FlatKVConfig.MiscStoreConfig.Name = string("misc")
 FlatKVConfig.MiscStoreConfig.MetricsEnabled = bool(true)
 FlatKVConfig.MiscStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(4)
+FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(128)
 FlatKVConfig.MiscStoreConfig.TargetBytesPerFlush = uint64(4194304)
 FlatKVConfig.MiscStoreConfig.ReservedPrefix = string("_meta/")
 FlatKVConfig.MiscStoreConfig.FlushSync = bool(false)

app/testdata/state-commit.keys.golden

@@ -23,3 +23,4 @@
 "state-commit.flatkv.async-write-buffer"
 "state-commit.flatkv.snapshot-interval"
 "state-commit.flatkv.snapshot-keep-recent"
+"state-commit.flatkv.max-snapshot-lag-blocks"

sei-cosmos/server/config/config.go

@@ -489,6 +489,9 @@ func GetConfig(v *viper.Viper) (Config, error) {
 	if v.IsSet("state-commit.flatkv.snapshot-keep-recent") {
 		flatKVConfig.SnapshotKeepRecent = v.GetUint32("state-commit.flatkv.snapshot-keep-recent")
 	}
+	if v.IsSet("state-commit.flatkv.max-snapshot-lag-blocks") {
+		flatKVConfig.MaxSnapshotLagBlocks = v.GetUint32("state-commit.flatkv.max-snapshot-lag-blocks")
+	}
 	if v.IsSet("state-commit.flatkv.enable-read-write-metrics") {
 		flatKVConfig.EnableReadWriteMetrics = v.GetBool("state-commit.flatkv.enable-read-write-metrics")
 	}

sei-cosmos/server/config/config_fuzz_test.go

@@ -315,6 +315,10 @@ var guardedKeys = []guardedKey{
 	},
 	{Key: "state-commit.flatkv.snapshot-interval", Path: "StateCommit.FlatKVConfig.SnapshotInterval", Set: 777},
 	{Key: "state-commit.flatkv.snapshot-keep-recent", Path: "StateCommit.FlatKVConfig.SnapshotKeepRecent", Set: 6},
+	{
+		Key: "state-commit.flatkv.max-snapshot-lag-blocks", Path: "StateCommit.FlatKVConfig.MaxSnapshotLagBlocks",
+		Set: 32,
+	},
 	{
 		Key: "state-commit.flatkv.enable-read-write-metrics", Path: "StateCommit.FlatKVConfig.EnableReadWriteMetrics",
 		Set: true, DefaultIsZero: true,

sei-cosmos/server/config/testdata/server_config.golden

@@ -73,6 +73,7 @@ StateCommit.FlatKVConfig.Fsync = bool(false)
 StateCommit.FlatKVConfig.AsyncWriteBuffer = int(0)
 StateCommit.FlatKVConfig.SnapshotInterval = uint32(10000)
 StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(1)
+StateCommit.FlatKVConfig.MaxSnapshotLagBlocks = uint32(64)
 StateCommit.FlatKVConfig.ExternalPruning = bool(false)
 StateCommit.FlatKVConfig.EnablePebbleMetrics = bool(true)
 StateCommit.FlatKVConfig.EnableReadWriteMetrics = bool(false)
@@ -86,7 +87,7 @@ StateCommit.FlatKVConfig.AccountStoreConfig.EstimatedOverheadPerEntry = uint64(2
 StateCommit.FlatKVConfig.AccountStoreConfig.Name = string("account")
 StateCommit.FlatKVConfig.AccountStoreConfig.MetricsEnabled = bool(true)
 StateCommit.FlatKVConfig.AccountStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-StateCommit.FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(4)
+StateCommit.FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(128)
 StateCommit.FlatKVConfig.AccountStoreConfig.TargetBytesPerFlush = uint64(4194304)
 StateCommit.FlatKVConfig.AccountStoreConfig.ReservedPrefix = string("_meta/")
 StateCommit.FlatKVConfig.AccountStoreConfig.FlushSync = bool(false)
@@ -100,7 +101,7 @@ StateCommit.FlatKVConfig.CodeStoreConfig.EstimatedOverheadPerEntry = uint64(256)
 StateCommit.FlatKVConfig.CodeStoreConfig.Name = string("code")
 StateCommit.FlatKVConfig.CodeStoreConfig.MetricsEnabled = bool(true)
 StateCommit.FlatKVConfig.CodeStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-StateCommit.FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(4)
+StateCommit.FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(128)
 StateCommit.FlatKVConfig.CodeStoreConfig.TargetBytesPerFlush = uint64(4194304)
 StateCommit.FlatKVConfig.CodeStoreConfig.ReservedPrefix = string("_meta/")
 StateCommit.FlatKVConfig.CodeStoreConfig.FlushSync = bool(false)
@@ -114,7 +115,7 @@ StateCommit.FlatKVConfig.StorageStoreConfig.EstimatedOverheadPerEntry = uint64(2
 StateCommit.FlatKVConfig.StorageStoreConfig.Name = string("storage")
 StateCommit.FlatKVConfig.StorageStoreConfig.MetricsEnabled = bool(true)
 StateCommit.FlatKVConfig.StorageStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-StateCommit.FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(4)
+StateCommit.FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(128)
 StateCommit.FlatKVConfig.StorageStoreConfig.TargetBytesPerFlush = uint64(4194304)
 StateCommit.FlatKVConfig.StorageStoreConfig.ReservedPrefix = string("_meta/")
 StateCommit.FlatKVConfig.StorageStoreConfig.FlushSync = bool(false)
@@ -128,7 +129,7 @@ StateCommit.FlatKVConfig.MiscStoreConfig.EstimatedOverheadPerEntry = uint64(256)
 StateCommit.FlatKVConfig.MiscStoreConfig.Name = string("misc")
 StateCommit.FlatKVConfig.MiscStoreConfig.MetricsEnabled = bool(true)
 StateCommit.FlatKVConfig.MiscStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
-StateCommit.FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(4)
+StateCommit.FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(128)
 StateCommit.FlatKVConfig.MiscStoreConfig.TargetBytesPerFlush = uint64(4194304)
 StateCommit.FlatKVConfig.MiscStoreConfig.ReservedPrefix = string("_meta/")
 StateCommit.FlatKVConfig.MiscStoreConfig.FlushSync = bool(false)

sei-cosmos/storev2/rootmulti/flatkv_helpers_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/sei-protocol/sei-chain/sei-cosmos/store/types"
 	errorutils "github.com/sei-protocol/sei-chain/sei-db/common/errors"
@@ -19,9 +20,49 @@ import (
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
 	scmemiavl "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl"
 	sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types"
+	sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite"
 	"github.com/stretchr/testify/require"
 )
 
+// requireStateStoreCaughtUp waits for the state store to reach version.
+//
+// The state store applies a block asynchronously: ApplyChangesetAsync hands it to a background writer
+// whose queue is AsyncWriteBuffer deep, so a commit returning says nothing about the state store having
+// applied that block. Any test that asserts a state store version, or that rolls back — rollback
+// refuses a state store sitting below the target — has to establish this first.
+func requireStateStoreCaughtUp(t *testing.T, store *Store, version int64) {
+	t.Helper()
+	require.NotNil(t, store.ssStore)
+	require.Eventually(t, func() bool {
+		return store.ssStore.GetLatestVersion() >= version
+	}, 10*time.Second, 5*time.Millisecond,
+		"state store never caught up to version %d", version)
+}
+
+// requireStateStoreSnapshotAtOrBelow waits for a state store snapshot at or below target to be on
+// disk, which is what rolling the state store back to target requires as its base.
+//
+// State store snapshots are published asynchronously and their coordinator declines a boundary while
+// another snapshot is still in flight, so committing past a boundary does not mean its snapshot exists.
+// With a per-block interval only the first boundary is usually accepted, the rest reporting "in_flight".
+func requireStateStoreSnapshotAtOrBelow(t *testing.T, homeDir string, target int64) {
+	t.Helper()
+	root := utils.GetStateStoreSnapshotsPath(homeDir)
+	require.Eventually(t, func() bool {
+		versions, err := sscomposite.ListSnapshotVersions(root)
+		if err != nil {
+			return false
+		}
+		for _, version := range versions {
+			if version > 0 && version <= target {
+				return true
+			}
+		}
+		return false
+	}, 10*time.Second, 5*time.Millisecond,
+		"no state store snapshot at or below version %d was ever published under %s", target, root)
+}
+
 // ---------------------------------------------------------------------------
 // Config helpers
 // ---------------------------------------------------------------------------

sei-cosmos/storev2/rootmulti/flatkv_recovery_test.go

@@ -81,6 +81,8 @@ func TestRollbackToVersionRollsBackStateStore(t *testing.T) {
 	for block := 1; block <= 5; block++ {
 		simulateBlock(t, store, storeKeys, block, evmData)
 	}
+	requireStateStoreCaughtUp(t, store, 5)
+	requireStateStoreSnapshotAtOrBelow(t, dir, 3)
 
 	require.NoError(t, store.RollbackToVersion(3))
 	require.Equal(t, int64(3), store.LastCommitID().Version)
@@ -110,6 +112,7 @@ func TestRollbackToVersionWithoutStateStoreSnapshots(t *testing.T) {
 	for block := 1; block <= 5; block++ {
 		simulateBlock(t, store, storeKeys, block, evmData)
 	}
+	requireStateStoreCaughtUp(t, store, 5)
 
 	require.NoError(t, store.RollbackToVersion(3))
 	require.Equal(t, int64(3), store.LastCommitID().Version)
@@ -140,6 +143,8 @@ func TestRollbackToVersionProceedsWhenStateStoreCannotFollow(t *testing.T) {
 	for block := 1; block <= 5; block++ {
 		simulateBlock(t, store, storeKeys, block, evmData)
 	}
+	requireStateStoreCaughtUp(t, store, 5)
+
 	rollbackable := store.ssStore
 	store.ssStore = nonRollbackableStateStore{rollbackable}
 

sei-db/controller/prunable_store.go

@@ -20,8 +20,13 @@ type PrunableStore interface {
 	// Only called when ExternalPruning reports true.
 	PruneHistory(blockNumber uint64) error
 
-	// PruneSnapshots may drop every snapshot strictly below blockNumber. A store that keeps no
-	// snapshots returns nil.
+	// PruneSnapshots may drop every snapshot strictly below blockNumber, and may do so
+	// asynchronously. A store that keeps no snapshots returns nil.
+	//
+	// A store that defers the work may let a later call supersede a pending one, so a given
+	// blockNumber is not guaranteed to be acted on at all — only that retention converges on the
+	// most recent one. A nil return therefore means the request was accepted, not that the
+	// snapshots are gone.
 	//
 	// blockNumber is never 0, and never above the block this store last returned from
 	// GetRollbackFloor.

sei-db/db_engine/view/view_manager_config.go

@@ -74,7 +74,7 @@ func DefaultViewManagerConfig(name string, reservedPrefix string) *ViewManagerCo
 		Name:                         name,
 		MetricsEnabled:               true,
 		MetricsScrapeIntervalSeconds: 10,
-		MaxUnflushedVersions:         4,
+		MaxUnflushedVersions:         128,
 		TargetBytesPerFlush:          unit.MB * 4,
 		ReservedPrefix:               reservedPrefix,
 		FlushSync:                    false,

sei-db/state_db/sc/composite/store_auto_test.go

@@ -383,6 +383,18 @@ func TestComposite_Auto_ExportImportRoundTrip(t *testing.T) {
 	runBlocks(t, src, workload, 1)
 	h := src.Version()
 
+	// FlatKV writes snapshots off the execution thread, so a commit returning no longer means its
+	// snapshot churn has finished. Exporting reads a snapshot by copying its directory, and pruning the
+	// oldest snapshot is the last step of publishing a new one, so without this wait the writer can
+	// delete the directory the export is part way through copying.
+	//
+	// Reached through the concrete store because quiescing the writer is not part of the flatkv.Store
+	// abstraction: no production caller needs it, and this test only does because it drives commits and
+	// reads from one goroutine and so has a quiet period to establish.
+	flatKVStore, ok := src.flatKV.(*flatkv.CommitStore)
+	require.True(t, ok)
+	require.NoError(t, flatKVStore.FlushSnapshots())
+
 	exp, err := src.Exporter(h)
 	require.NoError(t, err)
 	items := drainCompositeExporter(t, exp)

sei-db/state_db/sc/composite/store_test.go

@@ -54,7 +54,6 @@ func (f *failingEVMStore) RootHash() ([]byte, int64)                     { retur
 func (f *failingEVMStore) Version() int64                                { return 0 }
 func (f *failingEVMStore) PendingVersion() int64                         { return 0 }
 func (f *failingEVMStore) GetLatestVersion() (int64, error)              { return 0, nil }
-func (f *failingEVMStore) WriteSnapshot(string) error                    { return nil }
 func (f *failingEVMStore) Rollback(int64) error                          { return nil }
 func (f *failingEVMStore) Exporter(int64) (types.Exporter, error)        { return nil, nil }
 func (f *failingEVMStore) Importer(int64) (types.Importer, error)        { return nil, nil }

sei-db/state_db/sc/flatkv/api.go

@@ -152,9 +152,6 @@ type Store interface {
 	// inspect the store's height without taking ownership of it.
 	GetLatestVersion() (int64, error)
 
-	// WriteSnapshot writes a complete snapshot to dir.
-	WriteSnapshot(dir string) error
-
 	// Rollback rewinds a store opened with LoadLatest to targetVersion and prunes everything above it:
 	// snapshots, WAL blocks and committed state. It is the only way to move a committable store backwards,
 	// and the result keeps committing from targetVersion+1. An unreachable target is rejected before

sei-db/state_db/sc/flatkv/config/config.go

@@ -8,11 +8,6 @@ import (
 	"github.com/sei-protocol/sei-chain/sei-db/db_engine/view"
 )
 
-const (
-	DefaultSnapshotInterval   uint32 = 10000
-	DefaultSnapshotKeepRecent uint32 = 1
-)
-
 // Config defines configuration for the FlatKV (EVM) commit store.
 type Config struct {
 	// DataDir is the root directory for the FlatKV data files.
@@ -22,27 +17,32 @@ type Config struct {
 	// Fsync controls whether every view manager's flush is fsync'd. It overwrites each store
 	// config's FlushSync, so the four databases are always synced alike. The state WAL is
 	// unaffected and always writes NoSync.
-	// Default: false
 	Fsync bool `mapstructure:"fsync"`
 
 	// AsyncWriteBuffer defines the size of the async write buffer for data DBs.
 	// Set <= 0 for synchronous writes.
-	// Default: 0 (synchronous)
 	AsyncWriteBuffer int `mapstructure:"async-write-buffer"`
 
 	// SnapshotInterval defines how often (in blocks) a PebbleDB checkpoint
 	// snapshot is taken. 0 disables auto-snapshots.
 	// Without periodic snapshots the WAL grows unbounded and every restart
 	// replays the entire history from snapshot-0.
-	// Default: 10000
 	SnapshotInterval uint32 `mapstructure:"snapshot-interval"`
 
 	// SnapshotKeepRecent defines how many old snapshots to keep besides the
 	// latest one. 0 means keep only the current snapshot (no old snapshots).
 	// Ignored entirely when ExternalPruning is set.
-	// Default: 1
 	SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"`
 
+	// MaxSnapshotLagBlocks is how many committed blocks may queue up behind a snapshot that is still
+	// being written before Commit blocks. A value below 1 is treated as 1.
+	//
+	// A snapshot being written holds every database pinned at its own height, so no later block can
+	// reach disk until it completes, and each one is retained in memory meanwhile. This bounds how far
+	// that can run, trading a pause in block production for the memory the backlog would otherwise
+	// consume. It bounds blocks rather than bytes, so it mitigates exhaustion rather than preventing it.
+	MaxSnapshotLagBlocks uint32 `mapstructure:"max-snapshot-lag-blocks"`
+
 	// ExternalPruning hands retention to the StorageGarbageCollector: the store stops pruning its
 	// own snapshots (SnapshotKeepRecent) and stops truncating the state WAL.
 	//
@@ -51,16 +51,12 @@ type Config struct {
 	//
 	// With it on, snapshots are retained by height rather than by count, so the number kept becomes
 	// RollbackWindow / SnapshotInterval instead of SnapshotKeepRecent + 1.
-	//
-	// Default: false
 	ExternalPruning bool `mapstructure:"-"`
 
 	// EnablePebbleMetrics defines if the Pebble metrics should be enabled.
-	// Default: true
 	EnablePebbleMetrics bool `mapstructure:"enable-pebble-metrics"`
 
 	// EnableReadWriteMetrics emits simple estimated read/write counters for FlatKV's Pebble DBs.
-	// Default: false
 	EnableReadWriteMetrics bool `mapstructure:"enable-read-write-metrics"`
 
 	// AccountDBConfig defines the PebbleDB configuration for the account database.
@@ -133,8 +129,9 @@ func DefaultConfig() *Config {
 	cfg := &Config{
 		Fsync:                     false,
 		AsyncWriteBuffer:          0,
-		SnapshotInterval:          DefaultSnapshotInterval,
-		SnapshotKeepRecent:        DefaultSnapshotKeepRecent,
+		SnapshotInterval:          10000,
+		SnapshotKeepRecent:        1,
+		MaxSnapshotLagBlocks:      64,
 		EnablePebbleMetrics:       true,
 		AccountDBConfig:           pebbledb.DefaultConfig(),
 		AccountStoreConfig:        defaultStoreConfig("account"),

sei-db/state_db/sc/flatkv/config/flatkv_test_config.go

@@ -28,8 +28,8 @@ func DefaultTestConfig(t *testing.T) *Config {
 	t.Helper()
 	return &Config{
 		DataDir:                filepath.Join(t.TempDir(), "flatkv"),
-		SnapshotInterval:       DefaultSnapshotInterval,
-		SnapshotKeepRecent:     DefaultSnapshotKeepRecent,
+		SnapshotInterval:       10000,
+		SnapshotKeepRecent:     1,
 		AccountDBConfig:        smallTestPebbleConfig(),
 		AccountStoreConfig:     smallTestViewManagerConfig("account"),
 		CodeDBConfig:           smallTestPebbleConfig(),