From accef36b9eef3605c5d89cfdcbb74afa61cbb82f Mon Sep 17 00:00:00 2001 From: Jeongseo Park Date: Thu, 21 Sep 2023 17:46:34 +0900 Subject: [PATCH 1/2] Fix race-test failure in network --- network/network_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/network/network_test.go b/network/network_test.go index 6a6f8de87..6ce9a9489 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -337,8 +337,6 @@ func generateNetwork(name string, n int, t *testing.T, roles ...module.Role) []* r := newTestReactor(fmt.Sprintf("%s_%d", name, i), nm, ProtoTestNetwork, t) r.nt = nt r.c = c - failIfError(t, r.nt.Listen(), "fail to listen", r.name) - failIfError(t, nm.Start(), "fail to start", r.name) arr[i] = r } return arr @@ -562,6 +560,12 @@ func baseNetwork(t *testing.T) (m map[string][]*testReactor, ch chan context.Con } } } + for _, v := range m { + for _, r := range v { + failIfError(t, r.nt.Listen(), "fail to listen", r.name) + failIfError(t, r.nm.Start(), "fail to start", r.name) + } + } connMap, maxD, err := waitConnection(ch, defaultConnectionLimit, n, 10*DefaultSeedPeriod) t.Log(time.Now(), "max:", maxD, connMap) From 4538ce7c04521e37b8023f21fa5bff8438c5fde9 Mon Sep 17 00:00:00 2001 From: MoonKyu Song Date: Fri, 22 Sep 2023 13:18:54 +0900 Subject: [PATCH 2/2] Feature replay chain task --- chain/taskreplay.go | 229 ++++++++++++++++++++++++++++++++++++++ service/showresultdiff.go | 217 ++++++++++++++++++++++++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 chain/taskreplay.go create mode 100644 service/showresultdiff.go diff --git a/chain/taskreplay.go b/chain/taskreplay.go new file mode 100644 index 000000000..52a9a4ffb --- /dev/null +++ b/chain/taskreplay.go @@ -0,0 +1,229 @@ +/* + * Copyright 2023 ICON Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package chain + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/icon-project/goloop/common/db" + "github.com/icon-project/goloop/common/errors" + "github.com/icon-project/goloop/module" + "github.com/icon-project/goloop/service" +) + +type verifyParams struct { + Start int64 `json:"start"` + End int64 `json:"end"` + Detail bool `json:"detail"` +} + +type taskReplay struct { + chain *singleChain + tmpDB db.LayerDB + result resultStore + height int64 + start int64 + end int64 + detail bool + stop chan error +} + +func (t *taskReplay) Stop() { + t.stop <- errors.ErrInterrupted +} + +func (t *taskReplay) Wait() error { + return t.result.Wait() +} + +func (t *taskReplay) String() string { + return fmt.Sprintf("Replay(start=%d,end=%d,detail=%v)", + t.start, t.end, t.detail) +} + +func (t *taskReplay) DetailOf(s State) string { + switch s { + case Started: + return fmt.Sprintf("replay started height=%d", t.height) + default: + return "replay " + s.String() + } +} + +func (t *taskReplay) initTransition() (module.Block, module.Transition, error) { + sm := t.chain.ServiceManager() + bm := t.chain.BlockManager() + blk, err := bm.GetBlockByHeight(t.height) + if err != nil { + return nil, nil, err + } + tr, err := sm.CreateInitialTransition(blk.Result(), blk.NextValidators()) + return blk, tr, err +} + +type transitionCallback chan error + +func (t transitionCallback) OnValidate(transition module.Transition, err error) { + t <- err +} + +func (t transitionCallback) OnExecute(transition module.Transition, err error) { + t <- err +} + +func (t *taskReplay) doReplay() error { + defer func() { + t.chain.releaseManagers() + t.chain.database = t.tmpDB.Unwrap() + }() + var err error + + t.height = t.start + + bm := t.chain.BlockManager() + sm := t.chain.ServiceManager() + logger := t.chain.Logger() + + end := t.end + if last, err := bm.GetLastBlock(); err != nil { + return err + } else { + lastHeight := last.Height() + if end == 0 || end > lastHeight-1 { + end = lastHeight - 1 + } + } + + blk, ptr, err := t.initTransition() + if err != nil { + return err + } + var nblk module.Block + var tr module.Transition + for t.height <= end { + // next block for votes and consensus information + nblk, err = bm.GetBlockByHeight(t.height + 1) + if err != nil { + return err + } + csi, err := bm.NewConsensusInfo(blk) + if err != nil { + return err + } + tr, err = sm.CreateTransition(ptr, blk.NormalTransactions(), blk, csi, true) + if err != nil { + return err + } + ptxs := nblk.PatchTransactions() + if len(ptxs.Hash()) > 0 { + tr = sm.PatchTransition(tr, ptxs, nblk) + } + cb := make(chan error, 2) + cancel, err := tr.Execute(transitionCallback(cb)) + if err != nil { + return err + } + + // wait for OnValidate + select { + case err := <-t.stop: + cancel() + return err + case err := <-cb: + if err != nil { + return err + } + } + + // wait for OnExecute + select { + case err := <-t.stop: + cancel() + return err + case err := <-cb: + if err != nil { + return err + } + } + + // check the result + if !bytes.Equal(tr.Result(), nblk.Result()) { + logger.Errorf("INVALID RESULT res=%#x exp=%#x", + tr.Result(), nblk.Result()) + if t.detail { + _ = sm.Finalize(tr, module.FinalizeResult) + if err := service.ShowResultDiff(t.tmpDB, t.chain.plt, logger, nblk.Result(), tr.Result()); err != nil { + logger.Errorf("FAIL to show diff err=%+v", err) + } + } + return errors.InvalidStateError.New("InvalidResult") + } else { + if err := service.FinalizeTransition(tr, + module.FinalizeNormalTransaction|module.FinalizePatchTransaction|module.FinalizeResult, + false); err != nil { + return err + } + _ = t.tmpDB.Flush(false) + } + t.height += 1 + ptr, tr = tr, nil + blk, nblk = nblk, nil + } + return nil +} + +func (t *taskReplay) Start() error { + t.tmpDB = db.NewLayerDB(t.chain.database) + t.chain.database = t.tmpDB + + if err := t.chain.prepareManagers(); err != nil { + t.chain.database = t.tmpDB.Unwrap() + t.result.SetValue(err) + return err + } + t.stop = make(chan error, 1) + go func() { + err := t.doReplay() + defer t.result.SetValue(err) + }() + return nil +} + +func taskReplayFactory(chain *singleChain, param json.RawMessage) (chainTask, error) { + var p verifyParams + if err := json.Unmarshal(param, &p); err != nil { + return nil, err + } + if (p.End != 0 && p.End < p.Start) || p.Start < 0 { + return nil, errors.IllegalArgumentError.Errorf( + "InvalidParameter(start=%d,end=%d)", p.Start, p.End) + } + task := &taskReplay{ + chain: chain, + start: p.Start, + end: p.End, + detail: p.Detail, + } + return task, nil +} + +func init() { + registerTaskFactory("replay", taskReplayFactory) +} diff --git a/service/showresultdiff.go b/service/showresultdiff.go new file mode 100644 index 000000000..8f0987aae --- /dev/null +++ b/service/showresultdiff.go @@ -0,0 +1,217 @@ +/* + * Copyright 2023 ICON Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package service + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + + "github.com/icon-project/goloop/chain/base" + "github.com/icon-project/goloop/common/db" + "github.com/icon-project/goloop/common/log" + "github.com/icon-project/goloop/common/trie" + "github.com/icon-project/goloop/common/trie/trie_manager" + "github.com/icon-project/goloop/module" + "github.com/icon-project/goloop/service/state" + "github.com/icon-project/goloop/service/txresult" +) + +const ( + DNExtension = "extension" +) +type ObjectDetailHandler func(name string, key []byte, exp, real trie.Object) + +type DiffContext interface { + Database() db.Database + Logger() log.Logger + + ShowObjectMPTDiff(name string, dbase db.Database, t reflect.Type, e, r []byte, handler ObjectDetailHandler) error +} + +type PlatformWithShowDiff interface { + ShowDiff(ctx DiffContext, name string, e, r []byte) error +} + +type diffContext struct { + plt base.Platform + dbase db.Database + log log.Logger +} + +func (c *diffContext) Database() db.Database { + return c.dbase +} + +func (c *diffContext) Logger() log.Logger { + return c.log +} + +func (c *diffContext) GetObjectDiffHandlerFor(name string, handler ObjectDetailHandler) trie_manager.ObjectDifferenceHandler { + return func(op int, key []byte, exp, real trie.Object) { + switch op { + case -1: + c.log.Errorf("%s [-] key=%#x value=%+v\n", name, key, exp) + case 0: + if exp.Equal(real) { + c.log.Errorf("%s [=] key=%#x exp=<%#x> real=<%#x>\n", name, key, exp.Bytes(), real.Bytes()) + } else { + c.log.Errorf("%s [=] key=%#x exp=%+v real=%+v\n", name, key, exp, real) + } + if handler != nil { + handler(name, key, exp, real) + } + case 1: + c.log.Errorf("%s [+] key=%#x value=%+v\n", name, key, real) + } + } +} + +func (c *diffContext) GetBytesDiffHandlerFor(name string) trie_manager.BytesDifferenceHandler { + return func(op int, key []byte, exp, real []byte) { + switch op { + case -1: + c.log.Errorf("%s [-] key=%#x value=<%#x>\n", name, key, exp) + case 0: + c.log.Errorf("%s [=] key=%#x exp=<%#x> real=<%#x>\n", name, key, exp, real) + case 1: + c.log.Errorf("%s [+] key=%#x value=<%#x>\n", name, key, real) + } + } +} + +func (c *diffContext) ShowObjectMPTDiff(name string, dbase db.Database, t reflect.Type, e, r []byte, handler ObjectDetailHandler) error { + et := trie_manager.NewImmutableForObject(dbase, e, t) + rt := trie_manager.NewImmutableForObject(dbase, r, t) + return trie_manager.CompareImmutableForObject(et, rt, c.GetObjectDiffHandlerFor(name, handler)) +} + +func (c *diffContext) AccountDetailHandler() ObjectDetailHandler { + type Storer interface { + Store() trie.Immutable + } + return func (name string, key []byte, exp, real trie.Object) { + var eStore, rStore trie.Immutable + if eASS, ok := exp.(Storer); ok { + eStore = eASS.Store() + } + if rASS, ok := real.(Storer); ok { + rStore = rASS.Store() + } + if eStore == rStore { + return + } + if eStore == nil { + c.log.Errorf("%s [+] key=%#x real=%+v", name+".store", key, rStore) + return + } else if rStore == nil { + c.log.Errorf("%s [-] key=%#x exp=%+v", name+".store", key, eStore) + return + } + accountHash := fmt.Sprintf("%#x", key) + err := trie_manager.CompareImmutable(eStore, rStore, + c.GetBytesDiffHandlerFor(accountHash)) + if err != nil { + c.log.Errorf("%s fail to compare store", name) + } + } +} + +func JSONMarshalIndent(obj interface{}) ([]byte, error) { + type ToJSONer interface { + ToJSON(version module.JSONVersion) (interface{}, error) + } + if jsoner, ok := obj.(ToJSONer); ok { + if jso, err := jsoner.ToJSON(module.JSONVersionLast); err == nil { + obj = jso + } else { + log.Warnf("Failure in ToJSON err=%+v", err) + } + } + return json.MarshalIndent(obj, "", " ") +} + + +func (c *diffContext) showReceiptDiff(name string, e, r []byte) error { + el := txresult.NewReceiptListFromHash(c.dbase, e) + rl := txresult.NewReceiptListFromHash(c.dbase, r) + idx := 0 + for expect, result := el.Iterator(), rl.Iterator(); expect.Has() && result.Has(); _, _, idx = expect.Next(), result.Next(), idx+1 { + rct1, _ := expect.Get() + rct2, _ := result.Get() + if err := rct1.Check(rct2); err != nil { + rct1js, _ := JSONMarshalIndent(rct1) + rct2js, _ := JSONMarshalIndent(rct2) + c.log.Errorf("Expected %s Receipt[%d]:%s", name, idx, rct1js) + c.log.Errorf("Returned %s Receipt[%d]:%s", name, idx, rct2js) + } + } + return nil +} + +func (c *diffContext) showResultDiff(e, r *transitionResult) error { + if !bytes.Equal(e.StateHash, r.StateHash) { + if err := c.ShowObjectMPTDiff("world", c.dbase, state.AccountType, + e.StateHash, r.StateHash, c.AccountDetailHandler()) ; err != nil { + return err + } + } + if !bytes.Equal(e.PatchReceiptHash, r.PatchReceiptHash) { + if err := c.showReceiptDiff("Patch", e.PatchReceiptHash, r.PatchReceiptHash) ; err != nil { + return err + } + } + if !bytes.Equal(e.NormalReceiptHash, r.NormalReceiptHash) { + if err := c.showReceiptDiff("Normal", e.NormalReceiptHash, r.NormalReceiptHash) ; err != nil { + return err + } + } + if !bytes.Equal(e.ExtensionData, r.ExtensionData) { + c.log.Errorf("ExtensionData [=] e=<%#x> r=<%#x>", e.ExtensionData, r.ExtensionData) + if plt, ok := c.plt.(PlatformWithShowDiff); ok { + if err := plt.ShowDiff(c, DNExtension, e.ExtensionData, r.ExtensionData); err != nil { + return err + } + } + } + if !bytes.Equal(e.BTPData, r.BTPData) { + c.log.Errorf("BTPData [=] e=<%#x> r=<%#x>", e.BTPData, r.BTPData) + } + return nil +} + +func ShowResultDiff(dbase db.Database, plt base.Platform, logger log.Logger, exp, real []byte) error { + if bytes.Equal(exp, real) { + return nil + } + eResult, err := newTransitionResultFromBytes(exp) + if err != nil { + return err + } + rResult, err := newTransitionResultFromBytes(real) + if err != nil { + return err + } + c := &diffContext{ + plt: plt, + dbase: dbase, + log: logger, + } + return c.showResultDiff(eResult, rResult) +}