Archive Node Online Migration

app/app.go

@@ -368,6 +368,7 @@
 
 	genesisImportConfig genesistypes.GenesisImportConfig
 
+	stateStore   seidb.StateStore
 	receiptStore seidb.StateStore
 }
 
@@ -396,7 +397,7 @@
 	cdc := encodingConfig.Amino
 	interfaceRegistry := encodingConfig.InterfaceRegistry
 
-	bAppOptions := SetupSeiDB(logger, homePath, appOpts, baseAppOptions)
+	bAppOptions, stateStore := SetupSeiDB(logger, homePath, appOpts, baseAppOptions)
 
 	bApp := baseapp.NewBaseApp(AppName, logger, db, encodingConfig.TxConfig.TxDecoder(), tmConfig, appOpts, bAppOptions...)
 	bApp.SetCommitMultiStoreTracer(traceStore)
@@ -429,6 +430,7 @@
 		versionInfo:       version.NewInfo(),
 		metricCounter:     &map[string]float32{},
 		encodingConfig:    encodingConfig,
+		stateStore:        stateStore,
 	}
 
 	for _, option := range appOptions {
@@ -1064,6 +1066,9 @@
 // GetBaseApp returns the base app of the application
 func (app App) GetBaseApp() *baseapp.BaseApp { return app.BaseApp }
 
+// GetStateStore returns the state store of the application
+func (app App) GetStateStore() seidb.StateStore { return app.stateStore }
+
 // BeginBlocker application updates every begin block
 func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock {
 	metrics.GaugeSeidVersionAndCommit(app.versionInfo.Version, app.versionInfo.GitCommit)

app/seidb.go

@@ -7,6 +7,7 @@
 	servertypes "github.com/cosmos/cosmos-sdk/server/types"
 	"github.com/cosmos/cosmos-sdk/storev2/rootmulti"
 	"github.com/sei-protocol/sei-db/config"
+	seidb "github.com/sei-protocol/sei-db/ss/types"
 	"github.com/spf13/cast"
 	"github.com/tendermint/tendermint/libs/log"
 )
@@ -33,18 +34,20 @@
 
 	// Other configs
 	FlagSnapshotInterval = "state-sync.snapshot-interval"
+	FlagMigrateIAVL      = "migrate-iavl"
+	FlagMigrateHeight    = "migrate-height"
 )
 
 func SetupSeiDB(
 	logger log.Logger,
 	homePath string,
 	appOpts servertypes.AppOptions,
 	baseAppOptions []func(*baseapp.BaseApp),
-) []func(*baseapp.BaseApp) {
+) ([]func(*baseapp.BaseApp), seidb.StateStore) {
 	scEnabled := cast.ToBool(appOpts.Get(FlagSCEnable))
 	if !scEnabled {
 		logger.Info("SeiDB is disabled, falling back to IAVL")
-		return baseAppOptions
+		return baseAppOptions, nil
 	}
 	logger.Info("SeiDB SC is enabled, running node with StoreV2 commit store")
 	scConfig := parseSCConfigs(appOpts)
@@ -56,14 +59,21 @@
 
 	// cms must be overridden before the other options, because they may use the cms,
 	// make sure the cms aren't be overridden by the other options later on.
-	cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig)
+	cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig, cast.ToBool(appOpts.Get("migrate-iavl")))
+	migrationEnabled := cast.ToBool(appOpts.Get(FlagMigrateIAVL))
+	migrationHeight := cast.ToInt64(appOpts.Get(FlagMigrateHeight))
 	baseAppOptions = append([]func(*baseapp.BaseApp){
 		func(baseApp *baseapp.BaseApp) {
+			if migrationEnabled {
+				originalCMS := baseApp.CommitMultiStore()
+				baseApp.SetQueryMultiStore(originalCMS)
+				baseApp.SetMigrationHeight(migrationHeight)
+			}
 			baseApp.SetCMS(cms)
 		},
 	}, baseAppOptions...)
 
-	return baseAppOptions
+	return baseAppOptions, cms.GetStateStore()
 }
 
 func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {

app/test_state_store.go

@@ -244,6 +244,26 @@
 	return nil
 }
 
+func (s *InMemoryStateStore) SetLatestMigratedModule(module string) error {
+	// TODO: Add set call here
+	return nil
+}
+
+func (s *InMemoryStateStore) GetLatestMigratedModule() (string, error) {
+	// TODO: Add get call here
+	return "", nil
+}
+
+func (s *InMemoryStateStore) SetLatestMigratedKey(key []byte) error {
+	// TODO: Add set call here
+	return nil
+}
+
+func (s *InMemoryStateStore) GetLatestMigratedKey() ([]byte, error) {
+	// TODO: Add get call here
+	return nil, nil
+}
+
 func (s *InMemoryStateStore) Prune(version int64) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()

cmd/seid/cmd/root.go

@@ -38,6 +38,7 @@
 	"github.com/sei-protocol/sei-chain/app/params"
 	"github.com/sei-protocol/sei-chain/evmrpc"
 	"github.com/sei-protocol/sei-chain/tools"
+	"github.com/sei-protocol/sei-chain/tools/migration/ss"
 	"github.com/sei-protocol/sei-chain/x/evm/blocktest"
 	"github.com/sei-protocol/sei-chain/x/evm/querier"
 	"github.com/sei-protocol/sei-chain/x/evm/replay"
@@ -220,6 +221,8 @@
 
 func addModuleInitFlags(startCmd *cobra.Command) {
 	crisis.AddModuleInitFlags(startCmd)
+	startCmd.Flags().Bool("migrate-iavl", false, "Run migration of IAVL data store to SeiDB State Store")
+	startCmd.Flags().Int64("migrate-height", 0, "Height at which to start the migration")
 }
 
 // newApp creates a new Cosmos SDK app
@@ -266,7 +269,7 @@
 	// This makes it such that the wasm VM gas converts to sdk gas at a 6.66x rate vs that of the previous multiplier
 	wasmGasRegisterConfig.GasMultiplier = 21_000_000
 
-	return app.New(
+	app := app.New(
 		logger,
 		db,
 		traceStore,
@@ -302,6 +305,21 @@
 		baseapp.SetSnapshotDirectory(cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))),
 		baseapp.SetOccEnabled(cast.ToBool(appOpts.Get(baseapp.FlagOccEnabled))),
 	)
+
+	// Start migration if --migrate flag is set
+	if cast.ToBool(appOpts.Get("migrate-iavl")) {
+		go func() {
+			homeDir := cast.ToString(appOpts.Get(flags.FlagHome))
+			stateStore := app.GetStateStore()
+			migrationHeight := cast.ToInt64(appOpts.Get("migrate-height"))
+			migrator := ss.NewMigrator(homeDir, db, stateStore)
+			if err := migrator.Migrate(migrationHeight, homeDir); err != nil {
+				panic(err)
+			}
+		}()
+	}
+
+	return app
 }
 
 // appExport creates a new simapp (optionally at a given height)

docs/migration/seidb_archive_migration.md

@@ -0,0 +1,150 @@
+# SeiDB Archive Migration Guide
+
+## Overview
+SeiDB is the next generation of chain storage in SeiV2.
+One issue for running SeiDB on archive nodes is that we need to keep the full state of the chain, so we can't
+state sync it and clear out previous iavl data.
+
+In order to run an archive node with SeiDB, we need to run a migration from iavl to sei-db.
+
+The overall process will work as follows:
+
+1. Stop archive node and note down its height, call it MIGRATION_HEIGHT
+2. Update config to enable SeiDB (state committment + state store)
+3. Run sc migration at the MIGRATION_HEIGHT
+4. Re start seid with `--migrate-iavl` enabled (migrating state store in background)
+5. Verify migration at various sampled heights once state store is complete
+6. Stop seid, clear out iavl and restart seid normally, now only using SeiDB fully
+
+You may need to ensure you have sufficient disk space available, as during the migration process, both IAVL and SeiDB state stores will need to be maintained simultaneously. This could potentially double your storage requirements temporarily.
+
+
+## Migration Steps
+
+### Step 1: Stop Node and note down latest height
+Stop the seid process and note down the latest height. Save it as an env var $MIGRATION_HEIGHT.
+```bash
+systemctl stop seid
+MIGRATION_HEIGHT=<>
+```
+
+### Step 2: Add SeiDB Configurations
+We can enable SeiDB by adding the following configs to app.toml file.
+Usually you can find this file under ~/.sei/config/app.toml.
+```bash
+#############################################################################
+###                             SeiDB Configuration                       ###
+#############################################################################
+
+[state-commit]
+# Enable defines if the SeiDB should be enabled to override existing IAVL db backend.
+sc-enable = true
+
+# AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up
+# performance, <=0 means synchronous commit.
+sc-async-commit-buffer = 100
+
+# SnapshotKeepRecent defines how many memiavl snapshots (beyond the latest one) to keep
+# Recommend to set to 1 to make sure IBC relayers work.
+sc-keep-recent = 1
+
+# SnapshotInterval defines the number of blocks interval the memiavl snapshot is taken, default to 10000 blocks.
+# Adjust based on your needs:
+# Setting this too low could lead to lot of extra heavy disk IO
+# Setting this too high could lead to slow restart
+sc-snapshot-interval = 10000
+
+# SnapshotWriterLimit defines the max concurrency for taking memiavl snapshot
+sc-snapshot-writer-limit = 1
+
+# CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000.
+sc-cache-size = 100000
+
+[state-store]
+# Enable defines if the state-store should be enabled for historical queries.
+# In order to use state-store, you need to make sure to enable state-commit at the same time.
+# Validator nodes should turn this off.
+# State sync node or full nodes should turn this on.
+ss-enable = true
+
+# DBBackend defines the backend database used for state-store.
+# Supported backends: pebbledb, rocksdb
+# defaults to pebbledb (recommended)
+ss-backend = "pebbledb"
+
+# AsyncWriteBuffer defines the async queue length for commits to be applied to State Store
+# Set <= 0 for synchronous writes, which means commits also need to wait for data to be persisted in State Store.
+# defaults to 100
+ss-async-write-buffer = 100
+
+# KeepRecent defines the number of versions to keep in state store
+# Setting it to 0 means keep everything, default to 100000
+ss-keep-recent = 0
+
+# PruneIntervalSeconds defines the minimum interval in seconds + some random delay to trigger pruning.
+# It is more efficient to trigger pruning less frequently with large interval.
+# default to 600 seconds
+ss-prune-interval = 600
+
+# ImportNumWorkers defines the concurrency for state sync import
+# defaults to 1
+ss-import-num-workers = 1
+```
+
+
+### Step 3: Run SC Migration
+
+```bash
+seid tools migrate-iavl --target-db SC --home-dir /root/.sei
+```
+
+This may take a couple hours to run. You will see logs of form
+`Start restoring SC store for height`
+
+
+### Step 4: Restart seid with background SS migration
+```bash
+seid start --migrate-iavl --migrate-height $MIGRATION_HEIGHT --chain-id pacific-1
+```
+
+Seid will run normally and the migration will run in the background. Data from iavl
+will be written to SS and new writes will be directed at SS not iavl.
+
+You will see logs of form 
+`SeiDB Archive Migration: Iterating through %s module...` and 
+`SeiDB Archive Migration: Last 1,000,000 iterations took:...`
+
+
+NOTE: While this is running, any historical queries will be routed to iavl if
+they are for a height BEFORE the migrate-height. Any queries on heights
+AFTER the migrate-height will be routed to state store (pebbbledb).
+
+
+### Step 5: Verify State Store Migration after completion
+Once State Store Migration is complete, you will see logs of form
+`SeiDB Archive Migration: DB scanning completed. Total time taken:...`
+
+You DO NOT immediately need to do anything. Your node will continue to run
+and will operate normally. However we added a verification tool that will iterate through
+all keys in iavl at a specific height and verify they exist in State Store.
+
+You should run the following command for a selection of different heights
+```bash
+seid tools verify-migration --version $VERIFICATION_HEIGHT
+```
+
+This will output `Verification Succeeded` if the verification was successful.
+
+
+### Step 6: Clear out Iavl and restart seid
+Once the verification is complete, we can proceed to clear out the iavl and
+restart seid normally.
+
+```bash
+systemctl stop seid
+rm -rf ~/.sei/data/application.db
+seid start --chain-id pacific-1
+```
+
+
+## FAQ

go.mod

@@ -346,12 +346,12 @@ require (
 replace (
 	github.com/CosmWasm/wasmd => github.com/sei-protocol/sei-wasmd v0.2.4
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
-	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.3.37-0.20240923023912-aa7a702d42cc
+	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.3.37-0.20241007142359-ca29513f17aa
 	github.com/cosmos/iavl => github.com/sei-protocol/sei-iavl v0.2.0
 	github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.2
 	github.com/ethereum/go-ethereum => github.com/sei-protocol/go-ethereum v1.13.5-sei-9.0.20240923025222-815b87dde97b
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
-	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.44
+	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.45-0.20241009171123-c9b6c253819b
 	// Latest goleveldb is broken, we have to stick to this version
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.3.8

go.sum

@@ -1347,10 +1347,10 @@ github.com/sei-protocol/go-ethereum v1.13.5-sei-9.0.20240923025222-815b87dde97b
 github.com/sei-protocol/go-ethereum v1.13.5-sei-9.0.20240923025222-815b87dde97b/go.mod h1:kcRZmuzRn1lVejiFNTz4l4W7imnpq1bDAnuKS/RyhbQ=
 github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQpIbXDA=
 github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
-github.com/sei-protocol/sei-cosmos v0.3.37-0.20240923023912-aa7a702d42cc h1:srWLbsoS0NYBIl8OjZOFuPmIeqf+fJTkfsK39MmG3+k=
-github.com/sei-protocol/sei-cosmos v0.3.37-0.20240923023912-aa7a702d42cc/go.mod h1:ZwWxF/69WlcLEn4BzVjPPToTFkE2sjPanU8PNNyKoOk=
-github.com/sei-protocol/sei-db v0.0.44 h1:HMgcyDTQlmXdJysHJxmIo66EKeXn1CSQT9qXDnxjJgI=
-github.com/sei-protocol/sei-db v0.0.44/go.mod h1:F/ZKZA8HJPcUzSZPA8yt6pfwlGriJ4RDR4eHKSGLStI=
+github.com/sei-protocol/sei-cosmos v0.3.37-0.20241007142359-ca29513f17aa h1:F27bny97cLUl1PAUX67qnFEAwH2OtLXH/b3r5S3mfVA=
+github.com/sei-protocol/sei-cosmos v0.3.37-0.20241007142359-ca29513f17aa/go.mod h1:ZwWxF/69WlcLEn4BzVjPPToTFkE2sjPanU8PNNyKoOk=
+github.com/sei-protocol/sei-db v0.0.45-0.20241009171123-c9b6c253819b h1:q25KpfxNwTrS8UZgkjdTvQkNI1Wo84KkavDpPA3aHwI=
+github.com/sei-protocol/sei-db v0.0.45-0.20241009171123-c9b6c253819b/go.mod h1:F/ZKZA8HJPcUzSZPA8yt6pfwlGriJ4RDR4eHKSGLStI=
 github.com/sei-protocol/sei-iavl v0.2.0 h1:OisPjXiDT+oe+aeckzDEFgkZCYuUjHgs/PP8DPicN+I=
 github.com/sei-protocol/sei-iavl v0.2.0/go.mod h1:qRf8QYUPfrAO7K6VDB2B2l/N7K5L76OorioGBcJBIbw=
 github.com/sei-protocol/sei-ibc-go/v3 v3.3.2 h1:BaMZ6gjwqe3R/5dLmcJ1TkSZ3omcWy2TjaAZAeOJH44=

tools/migration/cmd/cmd.go

@@ -7,7 +7,10 @@ import (
 	"github.com/cosmos/cosmos-sdk/store/rootmulti"
 	"github.com/sei-protocol/sei-chain/tools/migration/sc"
 	"github.com/sei-protocol/sei-chain/tools/migration/ss"
+	"github.com/sei-protocol/sei-db/config"
+	sstypes "github.com/sei-protocol/sei-db/ss"
 	"github.com/spf13/cobra"
+	"github.com/tendermint/tendermint/libs/log"
 	dbm "github.com/tendermint/tm-db"
 )
 
@@ -51,13 +54,16 @@ func migrateSC(version int64, homeDir string, db dbm.DB) error {
 }
 
 func migrateSS(version int64, homeDir string, db dbm.DB) error {
-	migrator := ss.NewMigrator(homeDir, db)
-	return migrator.Migrate(version, homeDir)
-}
+	ssConfig := config.DefaultStateStoreConfig()
+	ssConfig.Enable = true
 
-func verifySS(version int64, homeDir string, db dbm.DB) error {
-	migrator := ss.NewMigrator(homeDir, db)
-	return migrator.Verify(version)
+	stateStore, err := sstypes.NewStateStore(log.NewNopLogger(), homeDir, ssConfig)
+	if err != nil {
+		return err
+	}
+
+	migrator := ss.NewMigrator(homeDir, db, stateStore)
+	return migrator.Migrate(version, homeDir)
 }
 
 func VerifyMigrationCmd() *cobra.Command {
@@ -97,3 +103,16 @@ func verify(cmd *cobra.Command, _ []string) {
 
 	fmt.Println("Verification Succeeded")
 }
+
+func verifySS(version int64, homeDir string, db dbm.DB) error {
+	ssConfig := config.DefaultStateStoreConfig()
+	ssConfig.Enable = true
+
+	stateStore, err := sstypes.NewStateStore(log.NewNopLogger(), homeDir, ssConfig)
+	if err != nil {
+		return err
+	}
+
+	migrator := ss.NewMigrator(homeDir, db, stateStore)
+	return migrator.Verify(version)
+}