Persist Pebble Hashes during scanner (#2065)

* Add tools to scan and compute hash for IAVL db * Fix lint * Fix lint * Fix * Fix * Fix * Pebble scanner (#2061) * Pebble scanner * Update name * Update command * Persist hashes * Update method * add backfill flag * Bump seidb --------- Co-authored-by: yzang2019 <[email protected]>
sei-protocol · Feb 3, 2025 · 83d6980 · 83d6980
1 parent 51f6572
commit 83d6980
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 6 deletions.
diff --git a/app/test_state_store.go b/app/test_state_store.go
@@ -294,6 +294,10 @@ func (s *InMemoryStateStore) Close() error {
 	return nil
 }
 
+func (s *InMemoryStateStore) WriteBlockRangeHash(storeKey string, beginBlockRange, endBlockRange int64, hash []byte) error {
+	panic("implement me")
+}
+
 type InMemoryIterator struct {
 	data    map[string][]byte
 	keys    []string

diff --git a/go.mod b/go.mod
@@ -356,7 +356,7 @@ replace (
 	github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.5
 	github.com/ethereum/go-ethereum => github.com/sei-protocol/go-ethereum v1.13.5-sei-9.0.20241224143343-21ee50facc96
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
-	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.46
+	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.47
 	// Latest goleveldb is broken, we have to stick to this version
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.4.7

diff --git a/go.sum b/go.sum
@@ -1350,8 +1350,8 @@ github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQp
 github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
 github.com/sei-protocol/sei-cosmos v0.3.52 h1:oEd4t9BqPxipiTqAYwgKcN1qVHZUUnOuQOzbLNrVA5Y=
 github.com/sei-protocol/sei-cosmos v0.3.52/go.mod h1:X/Fhh/QNNBKIVvJJUT3IKz4nyJ8PhB5/iFD0LSlyB74=
-github.com/sei-protocol/sei-db v0.0.46 h1:naXfSp1I3UgJJm/iSvXpdFzr9nofEOxp/EekcAVj7wY=
-github.com/sei-protocol/sei-db v0.0.46/go.mod h1:m5g7p0QeAS3dNJHIl28zQpzOgxQmvYqPb7t4hwgIOCA=
+github.com/sei-protocol/sei-db v0.0.47 h1:/1/btOKcNd+JPbCNTb87DENlKix7lnpubxAPWFgvrig=
+github.com/sei-protocol/sei-db v0.0.47/go.mod h1:m5g7p0QeAS3dNJHIl28zQpzOgxQmvYqPb7t4hwgIOCA=
 github.com/sei-protocol/sei-iavl v0.2.0 h1:OisPjXiDT+oe+aeckzDEFgkZCYuUjHgs/PP8DPicN+I=
 github.com/sei-protocol/sei-iavl v0.2.0/go.mod h1:qRf8QYUPfrAO7K6VDB2B2l/N7K5L76OorioGBcJBIbw=
 github.com/sei-protocol/sei-ibc-go/v3 v3.3.5 h1:SQRzWi9KSMuGNGd3f5RWAmsGGk7yeY1zhnEnrr/nqug=

diff --git a/tools/hash_verification/cmd/cmd.go b/tools/hash_verification/cmd/cmd.go
@@ -44,12 +44,14 @@ func GeneratePebbleHashCmd() *cobra.Command {
 	}
 	cmd.PersistentFlags().String("home-dir", "/root/.sei", "Sei home directory")
 	cmd.PersistentFlags().Int64("blocks-interval", 1_000_000, "Generate a hash every N blocks")
+	cmd.PersistentFlags().Bool("backfill", false, "Whether to write block range hashes back to the database")
 	return cmd
 }
 
 func generatePebbleHash(cmd *cobra.Command, _ []string) {
 	homeDir, _ := cmd.Flags().GetString("home-dir")
 	blocksInterval, _ := cmd.Flags().GetInt64("blocks-interval")
+	backfill, _ := cmd.Flags().GetBool("backfill")
 
 	ssConfig := config.DefaultStateStoreConfig()
 	ssConfig.Enable = true
@@ -60,6 +62,6 @@ func generatePebbleHash(cmd *cobra.Command, _ []string) {
 		panic(err)
 	}
 
-	scanner := pebbledb.NewHashScanner(stateStore, blocksInterval)
+	scanner := pebbledb.NewHashScanner(stateStore, blocksInterval, backfill)
 	scanner.ScanAllModules()
 }
diff --git a/tools/hash_verification/pebbledb/scanner.go b/tools/hash_verification/pebbledb/scanner.go
@@ -13,9 +13,10 @@ type HashScanner struct {
 	latestVersion  int64
 	blocksInterval int64
 	hashResult     map[string][][]byte
+	backfill       bool
 }
 
-func NewHashScanner(db types.StateStore, blocksInterval int64) *HashScanner {
+func NewHashScanner(db types.StateStore, blocksInterval int64, backfill bool) *HashScanner {
 	latestVersion, err := db.GetLatestVersion()
 	if err != nil {
 		panic(err)
@@ -26,14 +27,26 @@ func NewHashScanner(db types.StateStore, blocksInterval int64) *HashScanner {
 		latestVersion:  latestVersion,
 		blocksInterval: blocksInterval,
 		hashResult:     make(map[string][][]byte),
+		backfill:       backfill,
 	}
 }
 
 func (s *HashScanner) ScanAllModules() {
 	for _, moduleName := range utils.Modules {
 		result := s.scanAllHeights(moduleName)
 		for i, hashResult := range result {
-			fmt.Printf("Module %s height %d hash is: %X\n", moduleName, s.blocksInterval*(int64(i)+1), hashResult)
+			// Calculate the block range for this hash.
+			beginBlockRange := s.blocksInterval * int64(i)
+			endBlockRange := s.blocksInterval * (int64(i) + 1)
+
+			fmt.Printf("Module %s block range %d-%d hash is: %X\n", moduleName, beginBlockRange, endBlockRange, hashResult)
+
+			// Write the block range hash to the database only if backfill is enabled
+			if s.backfill {
+				if err := s.db.WriteBlockRangeHash(moduleName, beginBlockRange, endBlockRange, hashResult); err != nil {
+					panic(fmt.Errorf("failed to write block range hash: %w", err))
+				}
+			}
 		}
 	}
 }