From ef814c0f6627a3f4035cedbae7dd6f54779ead46 Mon Sep 17 00:00:00 2001 From: tony Date: Fri, 25 Oct 2024 11:34:02 +0800 Subject: [PATCH] add back pruner tests --- api/debug/debug_test.go | 2 +- api/metrics_test.go | 148 +++--- api/subscriptions/subscriptions_test.go | 2 +- bft/engine.go | 2 +- cmd/thor/pruner/pruner_test.go | 609 +++++++++++------------- go.mod | 2 +- trie/trie.go | 3 - txpool/tx_object_map_test.go | 2 +- txpool/tx_pool_test.go | 6 +- 9 files changed, 357 insertions(+), 419 deletions(-) diff --git a/api/debug/debug_test.go b/api/debug/debug_test.go index e3febb49e..03cbff91b 100644 --- a/api/debug/debug_test.go +++ b/api/debug/debug_test.go @@ -126,7 +126,7 @@ func TestStorageRangeFunc(t *testing.T) { func TestStorageRangeMaxResult(t *testing.T) { db := muxdb.NewMem() - state := state.New(db, thor.Bytes32{}, 0, 0, 0) + state := state.New(db, trie.Root{}) addr := thor.BytesToAddress([]byte("account1")) for i := 0; i < 1001; i++ { diff --git a/api/metrics_test.go b/api/metrics_test.go index ef06d6f59..bbbdbb473 100644 --- a/api/metrics_test.go +++ b/api/metrics_test.go @@ -7,7 +7,9 @@ package api import ( "bytes" + "crypto/rand" "io" + "math" "net/http" "net/http/httptest" "net/url" @@ -18,12 +20,15 @@ import ( "github.com/gorilla/websocket" "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/api/accounts" "github.com/vechain/thor/v2/api/subscriptions" "github.com/vechain/thor/v2/chain" + "github.com/vechain/thor/v2/cmd/thor/solo" "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/metrics" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/txpool" ) @@ -31,78 +36,77 @@ func init() { metrics.InitializePrometheusMetrics() } -// TODO: add back the test -// func TestMetricsMiddleware(t *testing.T) { -// db := muxdb.NewMem() -// stater := state.NewStater(db) -// gene := genesis.NewDevnet() - -// b, _, _, err := gene.Build(stater) -// if err != nil { -// t.Fatal(err) -// } -// repo, _ := chain.NewRepository(db, b) - -// // inject some invalid data to db -// data := db.NewStore("chain.data") -// var blkID thor.Bytes32 -// rand.Read(blkID[:]) -// data.Put(blkID[:], []byte("invalid data")) - -// // get summary should fail since the block data is not rlp encoded -// _, err = repo.GetBlockSummary(blkID) -// assert.NotNil(t, err) - -// router := mux.NewRouter() -// acc := accounts.New(repo, stater, math.MaxUint64, thor.NoFork, solo.NewBFTEngine(repo)) -// acc.Mount(router, "/accounts") -// router.PathPrefix("/metrics").Handler(metrics.HTTPHandler()) -// router.Use(metricsMiddleware) -// ts := httptest.NewServer(router) - -// httpGet(t, ts.URL+"/accounts/0x") -// httpGet(t, ts.URL+"/accounts/"+thor.Address{}.String()) - -// _, code := httpGet(t, ts.URL+"/accounts/"+thor.Address{}.String()+"?revision="+blkID.String()) -// assert.Equal(t, 500, code) - -// body, _ := httpGet(t, ts.URL+"/metrics") -// parser := expfmt.TextParser{} -// metrics, err := parser.TextToMetricFamilies(bytes.NewReader(body)) -// assert.Nil(t, err) - -// m := metrics["thor_metrics_api_request_count"].GetMetric() -// assert.Equal(t, 3, len(m), "should be 3 metric entries") -// assert.Equal(t, float64(1), m[0].GetCounter().GetValue()) -// assert.Equal(t, float64(1), m[1].GetCounter().GetValue()) - -// labels := m[0].GetLabel() -// assert.Equal(t, 3, len(labels)) -// assert.Equal(t, "code", labels[0].GetName()) -// assert.Equal(t, "200", labels[0].GetValue()) -// assert.Equal(t, "method", labels[1].GetName()) -// assert.Equal(t, "GET", labels[1].GetValue()) -// assert.Equal(t, "name", labels[2].GetName()) -// assert.Equal(t, "accounts_get_account", labels[2].GetValue()) - -// labels = m[1].GetLabel() -// assert.Equal(t, 3, len(labels)) -// assert.Equal(t, "code", labels[0].GetName()) -// assert.Equal(t, "400", labels[0].GetValue()) -// assert.Equal(t, "method", labels[1].GetName()) -// assert.Equal(t, "GET", labels[1].GetValue()) -// assert.Equal(t, "name", labels[2].GetName()) -// assert.Equal(t, "accounts_get_account", labels[2].GetValue()) - -// labels = m[2].GetLabel() -// assert.Equal(t, 3, len(labels)) -// assert.Equal(t, "code", labels[0].GetName()) -// assert.Equal(t, "500", labels[0].GetValue()) -// assert.Equal(t, "method", labels[1].GetName()) -// assert.Equal(t, "GET", labels[1].GetValue()) -// assert.Equal(t, "name", labels[2].GetName()) -// assert.Equal(t, "accounts_get_account", labels[2].GetValue()) -// } +func TestMetricsMiddleware(t *testing.T) { + db := muxdb.NewMem() + stater := state.NewStater(db) + gene := genesis.NewDevnet() + + b, _, _, err := gene.Build(stater) + if err != nil { + t.Fatal(err) + } + repo, _ := chain.NewRepository(db, b) + + // inject some invalid data to db + data := db.NewStore("chain.hdr") + var blkID thor.Bytes32 + rand.Read(blkID[:]) + data.Put(blkID[:], []byte("invalid data")) + + // get summary should fail since the block data is not rlp encoded + _, err = repo.GetBlockSummary(blkID) + assert.NotNil(t, err) + + router := mux.NewRouter() + acc := accounts.New(repo, stater, math.MaxUint64, thor.NoFork, solo.NewBFTEngine(repo)) + acc.Mount(router, "/accounts") + router.PathPrefix("/metrics").Handler(metrics.HTTPHandler()) + router.Use(metricsMiddleware) + ts := httptest.NewServer(router) + + httpGet(t, ts.URL+"/accounts/0x") + httpGet(t, ts.URL+"/accounts/"+thor.Address{}.String()) + + _, code := httpGet(t, ts.URL+"/accounts/"+thor.Address{}.String()+"?revision="+blkID.String()) + assert.Equal(t, 500, code) + + body, _ := httpGet(t, ts.URL+"/metrics") + parser := expfmt.TextParser{} + metrics, err := parser.TextToMetricFamilies(bytes.NewReader(body)) + assert.Nil(t, err) + + m := metrics["thor_metrics_api_request_count"].GetMetric() + assert.Equal(t, 3, len(m), "should be 3 metric entries") + assert.Equal(t, float64(1), m[0].GetCounter().GetValue()) + assert.Equal(t, float64(1), m[1].GetCounter().GetValue()) + + labels := m[0].GetLabel() + assert.Equal(t, 3, len(labels)) + assert.Equal(t, "code", labels[0].GetName()) + assert.Equal(t, "200", labels[0].GetValue()) + assert.Equal(t, "method", labels[1].GetName()) + assert.Equal(t, "GET", labels[1].GetValue()) + assert.Equal(t, "name", labels[2].GetName()) + assert.Equal(t, "accounts_get_account", labels[2].GetValue()) + + labels = m[1].GetLabel() + assert.Equal(t, 3, len(labels)) + assert.Equal(t, "code", labels[0].GetName()) + assert.Equal(t, "400", labels[0].GetValue()) + assert.Equal(t, "method", labels[1].GetName()) + assert.Equal(t, "GET", labels[1].GetValue()) + assert.Equal(t, "name", labels[2].GetName()) + assert.Equal(t, "accounts_get_account", labels[2].GetValue()) + + labels = m[2].GetLabel() + assert.Equal(t, 3, len(labels)) + assert.Equal(t, "code", labels[0].GetName()) + assert.Equal(t, "500", labels[0].GetValue()) + assert.Equal(t, "method", labels[1].GetName()) + assert.Equal(t, "GET", labels[1].GetValue()) + assert.Equal(t, "name", labels[2].GetName()) + assert.Equal(t, "accounts_get_account", labels[2].GetValue()) +} func TestWebsocketMetrics(t *testing.T) { db := muxdb.NewMem() diff --git a/api/subscriptions/subscriptions_test.go b/api/subscriptions/subscriptions_test.go index cce2fd8f2..c89dafae2 100644 --- a/api/subscriptions/subscriptions_test.go +++ b/api/subscriptions/subscriptions_test.go @@ -293,7 +293,7 @@ func initChainMultipleBlocks(t *testing.T, blockCount int) (*chain.Repository, [ if _, err := stage.Commit(); err != nil { t.Fatal(err) } - if err := repo.AddBlock(blk, receipts, 0); err != nil { + if err := repo.AddBlock(blk, receipts, 0, false); err != nil { t.Fatal(err) } if err := repo.SetBestBlockID(blk.Header().ID()); err != nil { diff --git a/bft/engine.go b/bft/engine.go index a952f8fab..3e0c88059 100644 --- a/bft/engine.go +++ b/bft/engine.go @@ -391,7 +391,7 @@ func (engine *Engine) findCheckpointByQuality(target uint32, finalized, headID t return c.GetBlockID(searchStart + uint32(num)*thor.CheckpointInterval) } -func (engine *BFTEngine) getMaxBlockProposers(sum *chain.BlockSummary) (uint64, error) { +func (engine *Engine) getMaxBlockProposers(sum *chain.BlockSummary) (uint64, error) { state := engine.stater.NewState(sum.Root()) params, err := builtin.Params.Native(state).Get(thor.KeyMaxBlockProposers) if err != nil { diff --git a/cmd/thor/pruner/pruner_test.go b/cmd/thor/pruner/pruner_test.go index 1de852697..cfcb027d8 100644 --- a/cmd/thor/pruner/pruner_test.go +++ b/cmd/thor/pruner/pruner_test.go @@ -5,339 +5,276 @@ package pruner -// TODO: add test back -// import ( -// "context" -// "crypto/ecdsa" -// "encoding/binary" -// "math" -// "math/big" -// "os" -// "path/filepath" -// "testing" - -// "github.com/ethereum/go-ethereum/crypto" -// "github.com/ethereum/go-ethereum/rlp" -// "github.com/stretchr/testify/assert" -// "github.com/vechain/thor/v2/block" -// "github.com/vechain/thor/v2/chain" -// "github.com/vechain/thor/v2/genesis" -// "github.com/vechain/thor/v2/muxdb" -// "github.com/vechain/thor/v2/state" -// "github.com/vechain/thor/v2/thor" -// "github.com/vechain/thor/v2/trie" -// "github.com/vechain/thor/v2/tx" -// ) - -// func fastForwardTo(from uint32, to uint32, db *muxdb.MuxDB, steadyID thor.Bytes32) (thor.Bytes32, error) { -// id := thor.Bytes32{} -// binary.BigEndian.PutUint32(id[:], to) - -// var summary = &chain.BlockSummary{ -// Header: &block.Header{}, -// Conflicts: 0, -// SteadyNum: block.Number(steadyID), -// } - -// data, err := rlp.EncodeToBytes(summary) -// if err != nil { -// return thor.Bytes32{}, err -// } - -// store := db.NewStore("chain.data") -// err = store.Put(id.Bytes(), data) -// if err != nil { -// return thor.Bytes32{}, err -// } - -// trie := db.NewNonCryptoTrie("i", trie.NonCryptoNodeHash, from, 0) -// if err := trie.Update(id[:4], id[:], nil); err != nil { -// return thor.Bytes32{}, err -// } - -// if steadyID == (thor.Bytes32{}) { -// if err := trie.Update(steadyID[:4], steadyID[:], nil); err != nil { -// return thor.Bytes32{}, err -// } -// } - -// _, commit := trie.Stage(to, 0) -// err = commit() -// if err != nil { -// return thor.Bytes32{}, err -// } -// return id, nil -// } - -// func newBlock(parentID thor.Bytes32, score uint64, stateRoot thor.Bytes32, priv *ecdsa.PrivateKey) *block.Block { -// blk := new(block.Builder).ParentID(parentID).TotalScore(score).StateRoot(stateRoot).Build() - -// if priv != nil { -// sig, _ := crypto.Sign(blk.Header().SigningHash().Bytes(), priv) -// return blk.WithSignature(sig) -// } -// return blk -// } - -// func TestStatus(t *testing.T) { -// db := muxdb.NewMem() - -// store := db.NewStore("test") - -// s := &status{} -// err := s.Load(store) -// assert.Nil(t, err, "load should not error") -// assert.Equal(t, uint32(0), s.Base) -// assert.Equal(t, uint32(0), s.PruneBase) - -// s.Base = 1 -// s.PruneBase = 2 - -// err = s.Save(store) -// assert.Nil(t, err, "save should not error") - -// s2 := &status{} -// err = s2.Load(store) -// assert.Nil(t, err, "load should not error") -// assert.Equal(t, uint32(1), s.Base) -// assert.Equal(t, uint32(2), s.PruneBase) -// } - -// func TestNewOptimizer(t *testing.T) { -// db := muxdb.NewMem() -// stater := state.NewStater(db) -// gene := genesis.NewDevnet() -// b0, _, _, _ := gene.Build(stater) -// repo, _ := chain.NewRepository(db, b0) - -// op := New(db, repo, false) -// op.Stop() -// } - -// func newTempFileDB() (*muxdb.MuxDB, func() error, error) { -// dir := os.TempDir() - -// opts := muxdb.Options{ -// TrieNodeCacheSizeMB: 128, -// TrieRootCacheCapacity: 256, -// TrieCachedNodeTTL: 30, // 5min -// TrieLeafBankSlotCapacity: 256, -// TrieDedupedPartitionFactor: math.MaxUint32, -// TrieWillCleanHistory: true, -// OpenFilesCacheCapacity: 512, -// ReadCacheMB: 256, // rely on os page cache other than huge db read cache. -// WriteBufferMB: 128, -// TrieHistPartitionFactor: 1000, -// } -// path := filepath.Join(dir, "main.db") -// db, err := muxdb.Open(path, &opts) -// if err != nil { -// return nil, nil, err -// } - -// close := func() error { -// err = db.Close() -// if err != nil { -// return err -// } -// err = os.RemoveAll(path) -// if err != nil { -// return err -// } -// return nil -// } - -// return db, close, nil -// } - -// func TestProcessDump(t *testing.T) { -// db, closeDB, err := newTempFileDB() -// assert.Nil(t, err) -// stater := state.NewStater(db) -// gene := genesis.NewDevnet() -// b0, _, _, _ := gene.Build(stater) -// repo, _ := chain.NewRepository(db, b0) - -// devAccounts := genesis.DevAccounts() - -// // fast forward to 1999 -// parentID, err := fastForwardTo(0, 1999, db, repo.SteadyBlockID()) -// assert.Nil(t, err) - -// var parentScore uint64 = 1999 * 2 -// // add new blocks with signature -// for i := 0; i < 3; i++ { -// blk := newBlock(parentID, parentScore+2, b0.Header().StateRoot(), devAccounts[i%2].PrivateKey) -// err := repo.AddBlock(blk, tx.Receipts{}, 0) -// assert.Nil(t, err) - -// parentID = blk.Header().ID() -// parentScore = blk.Header().TotalScore() -// } - -// repo.SetBestBlockID(parentID) - -// op := New(db, repo, false) -// op.Stop() - -// var s status -// assert.Nil(t, s.Load(op.db.NewStore(propsStoreName))) -// assert.Equal(t, uint32(2000), s.Base) - -// // fast forward to 3999 -// parentID, err = fastForwardTo(block.Number(parentID), 3999, db, repo.SteadyBlockID()) -// assert.Nil(t, err) - -// // add new blocks with signature -// for i := 0; i < 3; i++ { -// blk := newBlock(parentID, parentScore+2, b0.Header().StateRoot(), devAccounts[i%2].PrivateKey) -// err := repo.AddBlock(blk, tx.Receipts{}, 0) -// assert.Nil(t, err) - -// parentID = blk.Header().ID() -// parentScore = blk.Header().TotalScore() -// } -// repo.SetBestBlockID(parentID) - -// op = New(db, repo, true) -// op.Stop() - -// assert.Nil(t, s.Load(op.db.NewStore(propsStoreName))) -// assert.Equal(t, uint32(4000), s.Base) - -// closeDB() -// } - -// func TestWaitUntil(t *testing.T) { -// db := muxdb.NewMem() -// stater := state.NewStater(db) -// gene := genesis.NewDevnet() -// b0, _, _, _ := gene.Build(stater) -// repo, _ := chain.NewRepository(db, b0) -// devAccounts := genesis.DevAccounts() - -// ctx, cancel := context.WithCancel(context.Background()) -// op := &Optimizer{ -// repo: repo, -// db: db, -// ctx: ctx, -// cancel: cancel, -// } - -// parentID := b0.Header().ID() -// var parentScore uint64 = 0 -// for i := 0; i < 6; i++ { -// blk := newBlock(parentID, parentScore+2, b0.Header().StateRoot(), devAccounts[0].PrivateKey) -// err := repo.AddBlock(blk, tx.Receipts{}, 0) -// assert.Nil(t, err) - -// parentID = blk.Header().ID() -// parentScore = blk.Header().TotalScore() -// } -// repo.SetBestBlockID(parentID) - -// parentID, err := fastForwardTo(block.Number(parentID), 100000-1, db, repo.SteadyBlockID()) -// assert.Nil(t, err) - -// parentScore = (100000 - 1) * 2 -// for i := 0; i < 3; i++ { -// signer := devAccounts[0].PrivateKey -// score := parentScore + 1 -// blk := newBlock(parentID, score, b0.Header().StateRoot(), signer) -// err := repo.AddBlock(blk, tx.Receipts{}, 0) -// assert.Nil(t, err) - -// parentID = blk.Header().ID() -// parentScore = blk.Header().TotalScore() -// } -// repo.SetBestBlockID(parentID) - -// go func() { -// cancel() -// }() - -// // not enough signer, will wait for 1 sec -// // backoff will increase for more waiting -// // cancel here and restart a new test case -// _, err = op.awaitUntilSteady(100000) -// assert.NotNil(t, err) - -// for i := 0; i < 3; i++ { -// signer := devAccounts[i%2].PrivateKey -// score := parentScore + 2 -// blk := newBlock(parentID, score, b0.Header().StateRoot(), signer) - -// err := repo.AddBlock(blk, tx.Receipts{}, 0) -// assert.Nil(t, err) -// parentID = blk.Header().ID() -// parentScore = blk.Header().TotalScore() -// } -// repo.SetBestBlockID(parentID) - -// ctx, cancel = context.WithCancel(context.Background()) -// op.ctx = ctx -// op.cancel = cancel - -// chain, err := op.awaitUntilSteady(100000) -// assert.Nil(t, err) - -// assert.True(t, block.Number(chain.HeadID()) >= 10000) -// } - -// func TestDumpAndPrune(t *testing.T) { -// db, closeDB, err := newTempFileDB() -// assert.Nil(t, err) - -// stater := state.NewStater(db) -// gene := genesis.NewDevnet() -// b0, _, _, _ := gene.Build(stater) -// repo, _ := chain.NewRepository(db, b0) -// devAccounts := genesis.DevAccounts() - -// ctx, cancel := context.WithCancel(context.Background()) -// op := &Optimizer{ -// repo: repo, -// db: db, -// ctx: ctx, -// cancel: cancel, -// } - -// acc1 := thor.BytesToAddress([]byte("account1")) -// acc2 := thor.BytesToAddress([]byte("account2")) -// key := thor.BytesToBytes32([]byte("key")) -// value := thor.BytesToBytes32([]byte("value")) -// code := []byte("code") - -// parentID := b0.Header().ID() -// for i := 0; i < 9; i++ { -// blk := newBlock(parentID, 10, b0.Header().StateRoot(), nil) - -// err := repo.AddBlock(blk, tx.Receipts{}, 0) -// assert.Nil(t, err) -// parentID = blk.Header().ID() -// } - -// st := stater.NewState(b0.Header().StateRoot(), b0.Header().Number(), 0, 0) -// st.SetBalance(acc1, big.NewInt(1e18)) -// st.SetCode(acc2, code) -// st.SetStorage(acc2, key, value) -// stage, err := st.Stage(10, 0) -// assert.Nil(t, err) -// root, err := stage.Commit() -// assert.Nil(t, err) - -// blk := newBlock(parentID, 10, root, devAccounts[0].PrivateKey) -// err = repo.AddBlock(blk, tx.Receipts{}, 0) -// assert.Nil(t, err) -// parentID = blk.Header().ID() - -// repo.SetBestBlockID(parentID) - -// err = op.dumpStateLeaves(repo.NewBestChain(), 0, block.Number(parentID)+1) -// assert.Nil(t, err) - -// err = op.pruneTries(repo.NewBestChain(), 0, block.Number(parentID)+1) -// assert.Nil(t, err) - -// closeDB() -// } +import ( + "context" + "crypto/ecdsa" + "encoding/binary" + "math" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/block" + "github.com/vechain/thor/v2/chain" + "github.com/vechain/thor/v2/genesis" + "github.com/vechain/thor/v2/muxdb" + "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/trie" + "github.com/vechain/thor/v2/tx" +) + +func fastForwardTo(from uint32, to uint32, db *muxdb.MuxDB) (thor.Bytes32, error) { + var ( + parentID thor.Bytes32 + id thor.Bytes32 + ) + binary.BigEndian.PutUint32(parentID[:], to-1) + binary.BigEndian.PutUint32(id[:], to) + + blk := new(block.Builder).ParentID(parentID).Build() + var summary = &chain.BlockSummary{ + Header: blk.Header(), + Conflicts: 0, + } + + data, err := rlp.EncodeToBytes(summary) + if err != nil { + return thor.Bytes32{}, err + } + + store := db.NewStore("chain.hdr") + err = store.Put(id.Bytes(), data) + if err != nil { + return thor.Bytes32{}, err + } + + indexTrie := db.NewTrie("i", trie.Root{ + Hash: thor.BytesToBytes32([]byte{1}), + Ver: trie.Version{ + Major: from, + Minor: 0, + }, + }) + if err := indexTrie.Update(id[:4], id[:], nil); err != nil { + return thor.Bytes32{}, err + } + + if err := indexTrie.Commit(trie.Version{Major: to, Minor: 0}, true); err != nil { + return thor.Bytes32{}, err + } + return id, nil +} + +func newBlock(parentID thor.Bytes32, score uint64, stateRoot thor.Bytes32, priv *ecdsa.PrivateKey) *block.Block { + now := uint64(time.Now().Unix()) + blk := new(block.Builder).ParentID(parentID).TotalScore(score).StateRoot(stateRoot).Timestamp(now - now%10 - 10).Build() + + if priv != nil { + sig, _ := crypto.Sign(blk.Header().SigningHash().Bytes(), priv) + return blk.WithSignature(sig) + } + return blk +} + +func TestStatus(t *testing.T) { + db := muxdb.NewMem() + + store := db.NewStore("test") + + s := &status{} + err := s.Load(store) + assert.Nil(t, err, "load should not error") + assert.Equal(t, uint32(0), s.Base) + + s.Base = 1 + + err = s.Save(store) + assert.Nil(t, err, "save should not error") + + s2 := &status{} + err = s2.Load(store) + assert.Nil(t, err, "load should not error") + assert.Equal(t, uint32(1), s.Base) +} + +func TestNewPruner(t *testing.T) { + db := muxdb.NewMem() + stater := state.NewStater(db) + gene := genesis.NewDevnet() + b0, _, _, _ := gene.Build(stater) + repo, _ := chain.NewRepository(db, b0) + + pr := New(db, repo) + pr.Stop() +} + +func newTempFileDB() (*muxdb.MuxDB, func() error, error) { + dir := os.TempDir() + + opts := muxdb.Options{ + TrieNodeCacheSizeMB: 128, + TrieCachedNodeTTL: 30, // 5min + TrieDedupedPartitionFactor: math.MaxUint32, + TrieWillCleanHistory: true, + OpenFilesCacheCapacity: 512, + ReadCacheMB: 256, // rely on os page cache other than huge db read cache. + WriteBufferMB: 128, + TrieHistPartitionFactor: 1000, + } + path := filepath.Join(dir, "main.db") + db, err := muxdb.Open(path, &opts) + if err != nil { + return nil, nil, err + } + + close := func() error { + err = db.Close() + if err != nil { + return err + } + err = os.RemoveAll(path) + if err != nil { + return err + } + return nil + } + + return db, close, nil +} + +func TestWaitUntil(t *testing.T) { + db := muxdb.NewMem() + stater := state.NewStater(db) + gene := genesis.NewDevnet() + b0, _, _, _ := gene.Build(stater) + repo, _ := chain.NewRepository(db, b0) + devAccounts := genesis.DevAccounts() + + ctx, cancel := context.WithCancel(context.Background()) + pruner := &Pruner{ + repo: repo, + db: db, + ctx: ctx, + cancel: cancel, + } + + parentID := b0.Header().ID() + var parentScore uint64 = 0 + for i := 0; i < 6; i++ { + blk := newBlock(parentID, parentScore+2, b0.Header().StateRoot(), devAccounts[0].PrivateKey) + err := repo.AddBlock(blk, tx.Receipts{}, 0, false) + assert.Nil(t, err) + + parentID = blk.Header().ID() + parentScore = blk.Header().TotalScore() + } + repo.SetBestBlockID(parentID) + + parentID, err := fastForwardTo(block.Number(parentID), 100000-1, db) + assert.Nil(t, err) + + parentScore = (100000 - 1) * 2 + for i := 0; i < 3; i++ { + signer := devAccounts[0].PrivateKey + score := parentScore + 1 + blk := newBlock(parentID, score, b0.Header().StateRoot(), signer) + err := repo.AddBlock(blk, tx.Receipts{}, 0, false) + assert.Nil(t, err) + + parentID = blk.Header().ID() + parentScore = blk.Header().TotalScore() + } + repo.SetBestBlockID(parentID) + + go func() { + cancel() + }() + + // not enough signer, will wait for 1 sec + // backoff will increase for more waiting + // cancel here and restart a new test case + _, err = pruner.awaitUntilSteady(100000) + assert.NotNil(t, err) + + for i := 0; i < 3; i++ { + signer := devAccounts[i%2].PrivateKey + score := parentScore + 2 + blk := newBlock(parentID, score, b0.Header().StateRoot(), signer) + + err := repo.AddBlock(blk, tx.Receipts{}, 0, false) + assert.Nil(t, err) + parentID = blk.Header().ID() + parentScore = blk.Header().TotalScore() + } + repo.SetBestBlockID(parentID) + + ctx, cancel = context.WithCancel(context.Background()) + pruner.ctx = ctx + pruner.cancel = cancel + + chain, err := pruner.awaitUntilSteady(100000) + assert.Nil(t, err) + + assert.True(t, block.Number(chain.HeadID()) >= 10000) +} + +func TestPrune(t *testing.T) { + db, closeDB, err := newTempFileDB() + assert.Nil(t, err) + + stater := state.NewStater(db) + gene := genesis.NewDevnet() + b0, _, _, _ := gene.Build(stater) + repo, _ := chain.NewRepository(db, b0) + devAccounts := genesis.DevAccounts() + + ctx, cancel := context.WithCancel(context.Background()) + pruner := &Pruner{ + repo: repo, + db: db, + ctx: ctx, + cancel: cancel, + } + + acc1 := thor.BytesToAddress([]byte("account1")) + acc2 := thor.BytesToAddress([]byte("account2")) + key := thor.BytesToBytes32([]byte("key")) + value := thor.BytesToBytes32([]byte("value")) + code := []byte("code") + + parentID := b0.Header().ID() + for i := 0; i < 9; i++ { + blk := newBlock(parentID, 10, b0.Header().StateRoot(), nil) + + err := repo.AddBlock(blk, tx.Receipts{}, 0, false) + assert.Nil(t, err) + parentID = blk.Header().ID() + } + + st := stater.NewState(trie.Root{Hash: b0.Header().StateRoot(), Ver: trie.Version{Major: 0, Minor: 0}}) + st.SetBalance(acc1, big.NewInt(1e18)) + st.SetCode(acc2, code) + st.SetStorage(acc2, key, value) + stage, err := st.Stage(trie.Version{Major: 10, Minor: 0}) + assert.Nil(t, err) + root, err := stage.Commit() + assert.Nil(t, err) + + blk := newBlock(parentID, 10, root, devAccounts[0].PrivateKey) + err = repo.AddBlock(blk, tx.Receipts{}, 0, false) + assert.Nil(t, err) + parentID = blk.Header().ID() + + repo.SetBestBlockID(parentID) + + err = pruner.pruneTries(repo.NewBestChain(), 0, block.Number(parentID)+1) + assert.Nil(t, err) + + closeDB() +} diff --git a/go.mod b/go.mod index 700a6599e..2ae96305e 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,8 @@ require ( github.com/prometheus/client_model v0.5.0 github.com/prometheus/common v0.45.0 github.com/qianbin/directcache v0.9.7 - github.com/stretchr/testify v1.8.4 github.com/qianbin/drlp v0.0.0-20240102101024-e0e02518b5f9 + github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765 golang.org/x/crypto v0.21.0 diff --git a/trie/trie.go b/trie/trie.go index d405b1e5a..bb75f1ee0 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -22,15 +22,12 @@ import ( "fmt" "github.com/ethereum/go-ethereum/rlp" - "github.com/vechain/thor/v2/log" "github.com/vechain/thor/v2/thor" ) var ( // This is the known root hash of an empty trie. emptyRoot = thor.Blake2b(rlp.EmptyString) - - logger = log.WithContext("pkg", "trie") ) // Version is the version number of a standalone trie node. diff --git a/txpool/tx_object_map_test.go b/txpool/tx_object_map_test.go index 9a0b38629..d4e3613f9 100644 --- a/txpool/tx_object_map_test.go +++ b/txpool/tx_object_map_test.go @@ -158,7 +158,7 @@ func TestPendingCost(t *testing.T) { chain := repo.NewBestChain() best := repo.BestBlockSummary() - state := stater.NewState(best.Header.StateRoot(), best.Header.Number(), best.Conflicts, best.SteadyNum) + state := stater.NewState(best.Root()) var err error txObj1.executable, err = txObj1.Executable(chain, state, best.Header) diff --git a/txpool/tx_pool_test.go b/txpool/tx_pool_test.go index d79e2a3e9..68ee33997 100644 --- a/txpool/tx_pool_test.go +++ b/txpool/tx_pool_test.go @@ -615,8 +615,8 @@ func TestAddOverPendingCost(t *testing.T) { b0, _, _, err := builder.Build(state.NewStater(db)) assert.Nil(t, err) - st := state.New(db, b0.Header().StateRoot(), 0, 0, 0) - stage, err := st.Stage(1, 0) + st := state.New(db, trie.Root{Hash: b0.Header().StateRoot()}) + stage, err := st.Stage(trie.Version{Major: 1}) assert.Nil(t, err) root, err := stage.Commit() assert.Nil(t, err) @@ -632,7 +632,7 @@ func TestAddOverPendingCost(t *testing.T) { TransactionFeatures(feat).Build() repo, _ := chain.NewRepository(db, b0) - repo.AddBlock(b1, tx.Receipts{}, 0) + repo.AddBlock(b1, tx.Receipts{}, 0, false) repo.SetBestBlockID(b1.Header().ID()) pool := New(repo, state.NewStater(db), Options{ Limit: LIMIT,