all: enable pbss (#600)

* trie: enable pathdb: add path config and enable tests

* core/rawdb: now also inspect the state freezer in pathdb; rename

* cmd: working on cmd ronin

* core: refactor; add pathbase config; fix tests

- all: fix and enable tests for pathbase
- blockchain: open triedb explicitly in blockchain functions and close right after use, since diskLayer inside pathdb is a skeleton
- blockchain: when writeBlockWithState, pathbase will skip the explicit garbage collector, which is only needed for hashbase
- genesis.go: nit: change check genesis state, ref ethereum/go-ethereum@08bf8a6

* tests: enable path tests

* eth: enable path scheme

- all: fix tests, enable path scheme tests
- state_accessor: split function to retrieve statedb from block to hash scheme and path scheme

* light, miner, les, ethclient: clean up tests

* trie: refactor triereader, return err when state reader won't be created in hash and path

* trie: fix failed test in iterator and sync test tie

* trie,core: improve trie reader and add checking config nil when initing
database

* trie: statedb instance is committed, then it's not usable, a new instance must be created based on new root updated database, reference by commit 6d2aeb4

* cmd,les,eth: fixed unittest and adding flag Parrallel correctly

* core, eth: fix tests

* core: refactor and fix sync_test logic

* tmp: disable pathbase for TestIsPeriodBlock, TestIsTrippEffective

---------

Co-authored-by: Huy Ngo <[email protected]>

Author: Francesco4203

accounts/abi/bind/backends/simulated.go

@@ -43,6 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // This nil assignment ensures at compile time that SimulatedBackend implements bind.ContractBackend.
@@ -77,7 +78,7 @@ type SimulatedBackend struct {
 // A simulated backend always uses chainID 1337.
 func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
-	genesis.MustCommit(database)
+	genesis.MustCommit(database, trie.NewDatabase(database, trie.HashDefaults))
 	blockchain, _ := core.NewBlockChain(database, nil, &genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
 
 	backend := &SimulatedBackend{

cmd/evm/internal/t8ntool/execution.go

@@ -258,6 +258,12 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		Difficulty:  (*math.HexOrDecimal256)(vmContext.Difficulty),
 		GasUsed:     (math.HexOrDecimal64)(gasUsed),
 	}
+	// Re-create statedb instance with new root upon the updated database
+	// for accessing latest states.
+	statedb, err = state.New(root, statedb.Database(), nil)
+	if err != nil {
+		return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err))
+	}
 	return statedb, execRs, nil
 }
 

cmd/evm/runner.go

@@ -30,13 +30,16 @@ import (
 
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/internal/flags"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
 
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/log"
@@ -139,11 +142,22 @@ func runCmd(ctx *cli.Context) error {
 		gen := readGenesis(ctx.String(GenesisFlag.Name))
 		genesisConfig = gen
 		db := rawdb.NewMemoryDatabase()
-		genesis := gen.MustCommit(db)
-		statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil)
+		triedb := trie.NewDatabase(db, &trie.Config{
+			HashDB: hashdb.Defaults,
+		})
+		defer triedb.Close()
+		genesis := gen.MustCommit(db, triedb)
+		sdb := state.NewDatabaseWithNodeDB(db, triedb)
+		statedb, _ = state.New(genesis.Root(), sdb, nil)
 		chainConfig = gen.Config
 	} else {
-		statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
+		db := rawdb.NewMemoryDatabase()
+		triedb := trie.NewDatabase(db, &trie.Config{
+			HashDB: hashdb.Defaults,
+		})
+		defer triedb.Close()
+		sdb := state.NewDatabaseWithNodeDB(db, triedb)
+		statedb, _ = state.New(types.EmptyRootHash, sdb, nil)
 		genesisConfig = new(core.Genesis)
 	}
 	if ctx.String(SenderFlag.Name) != "" {

cmd/evm/staterunner.go

@@ -20,11 +20,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"io/ioutil"
 	"os"
 
+	"github.com/ethereum/go-ethereum/eth/tracers/logger"
+
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
@@ -96,21 +99,22 @@ func stateTestCmd(ctx *cli.Context) error {
 	results := make([]StatetestResult, 0, len(tests))
 	for key, test := range tests {
 		for _, st := range test.Subtests() {
+			dump := ctx.Bool(DumpFlag.Name)
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
-			_, s, err := test.Run(st, cfg, false)
-			// print state root for evmlab tracing
-			if ctx.Bool(MachineFlag.Name) && s != nil {
-				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false))
-			}
-			if err != nil {
-				// Test failed, mark as so and dump any state to aid debugging
-				result.Pass, result.Error = false, err.Error()
-				if ctx.Bool(DumpFlag.Name) && s != nil {
-					dump := s.RawDump(nil)
-					result.State = &dump
+			test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
+				if err != nil {
+					// Test failed, mark as so and dump any state to aid debugging
+					result.Pass, result.Error = false, err.Error()
+					if dump {
+						dump := state.RawDump(nil)
+						result.State = &dump
+					}
 				}
-			}
+				if ctx.Bool(MachineFlag.Name) && state != nil {
+					fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+				}
+			})
 
 			results = append(results, *result)
 

cmd/ronin/chaincmd.go

@@ -38,7 +38,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/trie"
 	"github.com/urfave/cli/v2"
 )
 
@@ -52,6 +51,8 @@ var (
 			utils.DataDirFlag,
 			utils.DBEngineFlag,
 			utils.ForceOverrideChainConfigFlag,
+			utils.CachePreimagesFlag,
+			utils.StateSchemeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -106,6 +107,9 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to
 			utils.MetricsInfluxDBBucketFlag,
 			utils.MetricsInfluxDBOrganizationFlag,
 			utils.TxLookupLimitFlag,
+			utils.TransactionHistoryFlag,
+			utils.StateSchemeFlag,
+			utils.StateHistoryFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -125,6 +129,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 			utils.DBEngineFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
+			utils.StateSchemeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -226,14 +231,13 @@ func initGenesis(ctx *cli.Context) error {
 			utils.Fatalf("Failed to open database: %v", err)
 		}
 		// Create triedb firstly
-		triedb := trie.NewDatabaseWithConfig(chaindb, &trie.Config{
-			Preimages: ctx.Bool(utils.CachePreimagesFlag.Name),
-		})
+
+		triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false)
+		defer chaindb.Close()
 		_, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis, overrideChainConfig)
 		if err != nil {
 			utils.Fatalf("Failed to write genesis block: %v", err)
 		}
-		chaindb.Close()
 		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
 	return nil
@@ -471,10 +475,10 @@ func dump(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	config := &trie.Config{
-		Preimages: true, // always enable preimage lookup
-	}
-	state, err := state.New(root, state.NewDatabaseWithConfig(db, config), nil)
+	triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup
+	defer triedb.Close()
+	state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
+
 	if err != nil {
 		return err
 	}

cmd/ronin/dbcmd.go

@@ -190,6 +190,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 			utils.SepoliaFlag,
 			utils.RinkebyFlag,
 			utils.GoerliFlag,
+			utils.StateSchemeFlag,
 		},
 		Description: "This command looks up the specified database key from the database.",
 	}
@@ -476,6 +477,8 @@ func dbDumpTrie(ctx *cli.Context) error {
 
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	defer db.Close()
+	triedb := utils.MakeTrieDatabase(ctx, db, false, true)
+	defer triedb.Close()
 	var (
 		state   []byte
 		storage []byte
@@ -509,12 +512,16 @@ func dbDumpTrie(ctx *cli.Context) error {
 		}
 	}
 	id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage))
-	theTrie, err := trie.New(id, trie.NewDatabase(db))
+	theTrie, err := trie.New(id, triedb)
+	if err != nil {
+		return err
+	}
+	trieIt, err := theTrie.NodeIterator(start)
 	if err != nil {
 		return err
 	}
 	var count int64
-	it := trie.NewIterator(theTrie.NodeIterator(start))
+	it := trie.NewIterator(trieIt)
 	for it.Next() {
 		if max > 0 && count == max {
 			fmt.Printf("Exiting after %d values\n", count)

cmd/ronin/genesis_test.go

@@ -173,12 +173,12 @@ func TestCustomBackend(t *testing.T) {
 		{ // Can't start pebble on top of leveldb
 			initArgs:   []string{"--db.engine", "leveldb"},
 			execArgs:   []string{"--db.engine", "pebble"},
-			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
+			execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
 		},
 		{ // Can't start leveldb on top of pebble
 			initArgs:   []string{"--db.engine", "pebble"},
 			execArgs:   []string{"--db.engine", "leveldb"},
-			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
+			execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
 		},
 		{ // Reject invalid backend choice
 			initArgs:   []string{"--db.engine", "mssql"},

cmd/ronin/main.go

@@ -101,7 +101,10 @@ var (
 		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
 		utils.SnapshotFlag,
-		utils.TxLookupLimitFlag,
+		utils.TxLookupLimitFlag, // deprecated
+		utils.TransactionHistoryFlag,
+		utils.StateSchemeFlag,
+		utils.StateHistoryFlag,
 		utils.TriesInMemoryFlag,
 		utils.LightServeFlag,
 		utils.LightIngressFlag,