Pebble scanner (#2061)

* Pebble scanner

* Update name

* Update command

tools/cmd.go

@@ -18,5 +18,6 @@ func ToolCmd() *cobra.Command {
 	toolsCmd.AddCommand(migration.VerifyMigrationCmd())
 	toolsCmd.AddCommand(migration.GenerateStats())
 	toolsCmd.AddCommand(hasher.GenerateIavlHashCmd())
+	toolsCmd.AddCommand(hasher.GeneratePebbleHashCmd())
 	return toolsCmd
 }

tools/hash_verification/cmd/cmd.go

@@ -6,6 +6,10 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/sei-protocol/sei-chain/tools/hash_verification/iavl"
+	"github.com/sei-protocol/sei-chain/tools/hash_verification/pebbledb"
+	"github.com/sei-protocol/sei-db/config"
+	sstypes "github.com/sei-protocol/sei-db/ss"
+	"github.com/tendermint/tendermint/libs/log"
 	dbm "github.com/tendermint/tm-db"
 )
 
@@ -31,3 +35,31 @@ func generateIavlHash(cmd *cobra.Command, _ []string) {
 	scanner := iavl.NewHashScanner(db, blocksInterval)
 	scanner.ScanAllModules()
 }
+
+func GeneratePebbleHashCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "generate-pebble-hash",
+		Short: "A tool to scan full Pebble archive database and generate a hash for every N blocks per module",
+		Run:   generatePebbleHash,
+	}
+	cmd.PersistentFlags().String("home-dir", "/root/.sei", "Sei home directory")
+	cmd.PersistentFlags().Int64("blocks-interval", 1_000_000, "Generate a hash every N blocks")
+	return cmd
+}
+
+func generatePebbleHash(cmd *cobra.Command, _ []string) {
+	homeDir, _ := cmd.Flags().GetString("home-dir")
+	blocksInterval, _ := cmd.Flags().GetInt64("blocks-interval")
+
+	ssConfig := config.DefaultStateStoreConfig()
+	ssConfig.Enable = true
+	ssConfig.KeepRecent = 0
+	stateStore, err := sstypes.NewStateStore(log.NewNopLogger(), homeDir, ssConfig)
+
+	if err != nil {
+		panic(err)
+	}
+
+	scanner := pebbledb.NewHashScanner(stateStore, blocksInterval)
+	scanner.ScanAllModules()
+}

tools/hash_verification/pebbledb/scanner.go

@@ -0,0 +1,70 @@
+package pebbledb
+
+import (
+	"fmt"
+
+	"github.com/sei-protocol/sei-chain/tools/hash_verification/hasher"
+	"github.com/sei-protocol/sei-chain/tools/utils"
+	"github.com/sei-protocol/sei-db/ss/types"
+)
+
+type HashScanner struct {
+	db             types.StateStore
+	latestVersion  int64
+	blocksInterval int64
+	hashResult     map[string][][]byte
+}
+
+func NewHashScanner(db types.StateStore, blocksInterval int64) *HashScanner {
+	latestVersion, err := db.GetLatestVersion()
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("Detected Pebbledb latest version: %d\n", latestVersion)
+	return &HashScanner{
+		db:             db,
+		latestVersion:  latestVersion,
+		blocksInterval: blocksInterval,
+		hashResult:     make(map[string][][]byte),
+	}
+}
+
+func (s *HashScanner) ScanAllModules() {
+	for _, moduleName := range utils.Modules {
+		result := s.scanAllHeights(moduleName)
+		for i, hashResult := range result {
+			fmt.Printf("Module %s height %d hash is: %X\n", moduleName, s.blocksInterval*(int64(i)+1), hashResult)
+		}
+	}
+}
+
+func (s *HashScanner) scanAllHeights(module string) [][]byte {
+	dataCh := make(chan types.RawSnapshotNode, 10000)
+	hashCalculator := hasher.NewXorHashCalculator(s.blocksInterval, int(s.latestVersion/s.blocksInterval+1), dataCh)
+	fmt.Printf("Starting to scan module: %s\n", module)
+	go func() {
+		count := 0
+		_, err := s.db.RawIterate(module, func(key, value []byte, version int64) bool {
+			dataCh <- types.RawSnapshotNode{
+				StoreKey: module,
+				Key:      key,
+				Value:    value,
+				Version:  version,
+			}
+
+			count++
+			if count%1000000 == 0 {
+				fmt.Printf("Scanned %d items for module %s\n", count, module)
+			}
+
+			return false
+		})
+		if err != nil {
+			panic(fmt.Errorf("RawIterate error: %w", err))
+		}
+		close(dataCh)
+	}()
+	allHashes := hashCalculator.ComputeHashes()
+	s.hashResult[module] = allHashes
+	return allHashes
+}