fix delete nodes

Author: Hitenjain14

core/util/wmpt/trie.go

@@ -27,7 +27,7 @@ type WeightedMerkleTrie struct {
 	root        Node
 	db          storage.StorageAdapter
 	oldRoot     hashNode
-	deleted     [][]byte
+	deleted     map[[32]byte]bool
 	tempDeleted [][]byte
 	created     [][]byte
 	sync.Mutex
@@ -38,7 +38,7 @@ func New(root Node, db storage.StorageAdapter) *WeightedMerkleTrie {
 	if root == nil {
 		root = emptyNode
 	}
-	return &WeightedMerkleTrie{db: db, root: root}
+	return &WeightedMerkleTrie{db: db, root: root, deleted: make(map[[32]byte]bool)}
 }
 
 func (t *WeightedMerkleTrie) CopyRoot(collapseLevel int) Node {
@@ -322,8 +322,8 @@ func (t *WeightedMerkleTrie) Rollback() {
 func (t *WeightedMerkleTrie) DeleteNodes() error {
 	if len(t.deleted) > 0 {
 		batcher := t.db.NewBatch()
-		for _, key := range t.deleted {
-			err := batcher.Delete(key)
+		for key := range t.deleted {
+			err := batcher.Delete(key[:])
 			if err != nil {
 				return err
 			}
@@ -333,7 +333,12 @@ func (t *WeightedMerkleTrie) DeleteNodes() error {
 			return err
 		}
 	}
-	t.deleted = t.tempDeleted
+	clear(t.deleted)
+	for _, key := range t.tempDeleted {
+		var k [32]byte
+		copy(k[:], key)
+		t.deleted[k] = true
+	}
 	t.tempDeleted = nil
 	return nil
 }
@@ -541,6 +546,10 @@ func (t *WeightedMerkleTrie) collectDeleteAndCreated(deleteChan, createdChan cha
 	}()
 	go func() {
 		for hash := range createdChan {
+			//check if hash is in deleted, if so, remove it
+			var k [32]byte
+			copy(k[:], hash)
+			delete(t.deleted, k)
 			t.created = append(t.created, hash)
 		}
 		wg.Done()

core/util/wmpt/trie_test.go

@@ -207,3 +207,98 @@ func TestUpdateTrie(t *testing.T) {
 	assert.Equal(t, trie.root.Weight(), uint64(15))
 	assert.NotEqual(t, h1, h2)
 }
+
+func TestCommitAndRollback(t *testing.T) {
+	wd, err := os.Getwd()
+	assert.NoError(t, err)
+	pebDir := filepath.Join(wd, "pebble_storage")
+	assert.NoError(t, os.RemoveAll(pebDir))
+	assert.NoError(t, os.MkdirAll(pebDir, 0777))
+	db, err := kv.NewPebbleAdapter(pebDir, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		db.Close()
+		os.RemoveAll(pebDir)
+	}()
+	t1 := New(nil, db)
+	keys := make([][]byte, 0, 5)
+	for i := 0; i < 5; i++ {
+		hash := sha256.Sum256([]byte(strconv.Itoa(i)))
+		keys = append(keys, hash[:])
+	}
+	t1.SaveRoot()
+	t1.Update(keys[0], []byte("a"), 10)
+	b, err := t1.Commit(3)
+	assert.NoError(t, err)
+	err = b.Commit(true)
+	assert.NoError(t, err)
+	assert.NoError(t, t1.DeleteNodes())
+	t1.SaveRoot()
+	assert.NoError(t, t1.Update(keys[0], nil, 0))
+	b, err = t1.Commit(3)
+	assert.NoError(t, err)
+	err = b.Commit(true)
+	assert.NoError(t, err)
+	assert.NoError(t, t1.DeleteNodes())
+	h := t1.Root()
+	assert.Equal(t, h, emptyState)
+	data, err := t1.GetPath(nil)
+	assert.NoError(t, err)
+	t2 := New(nil, nil)
+	assert.NoError(t, t2.Deserialize(data))
+	h2 := t2.GetRoot().CalcHash()
+	assert.Equal(t, h2, h)
+	t1.SaveRoot()
+	t1.Rollback()
+	assert.Equal(t, t1.Root(), h)
+}
+
+func TestUploadDelete(t *testing.T) {
+	wd, err := os.Getwd()
+	assert.NoError(t, err)
+	pebDir := filepath.Join(wd, "pebble_storage")
+	assert.NoError(t, os.RemoveAll(pebDir))
+	assert.NoError(t, os.MkdirAll(pebDir, 0777))
+	db, err := kv.NewPebbleAdapter(pebDir, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		db.Close()
+		os.RemoveAll(pebDir)
+	}()
+	t1 := New(nil, db)
+	keys := make([][]byte, 0, 5)
+	for i := 0; i < 5; i++ {
+		hash := sha256.Sum256([]byte(strconv.Itoa(i)))
+		keys = append(keys, hash[:])
+	}
+	t1.SaveRoot()
+	t1.Update(keys[0], []byte("a"), 10)
+	b, err := t1.Commit(3)
+	assert.NoError(t, err)
+	err = b.Commit(true)
+	assert.NoError(t, err)
+	assert.NoError(t, t1.DeleteNodes())
+	t1.SaveRoot()
+	assert.NoError(t, t1.Update(keys[0], nil, 0))
+	b, err = t1.Commit(3)
+	assert.NoError(t, err)
+	err = b.Commit(true)
+	assert.NoError(t, err)
+	assert.NoError(t, t1.DeleteNodes())
+	t1.Update(keys[0], []byte("a"), 10)
+	b, err = t1.Commit(3)
+	assert.NoError(t, err)
+	err = b.Commit(true)
+	assert.NoError(t, err)
+	assert.NoError(t, t1.DeleteNodes())
+	t2 := New(&hashNode{
+		weight: 10,
+		hash:   t1.GetRoot().CalcHash(),
+	}, db)
+	_, _, err = t2.GetBlockProof(6)
+	assert.NoError(t, err)
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/0chain/common
 
-go 1.18
+go 1.21
 
 require (
 	github.com/cockroachdb/pebble v1.1.2

go.sum

@@ -40,6 +40,7 @@ github.com/0chain/msgp v1.1.62 h1:D/s/TYygUwQnMlvfHUjh56FEaI0wP2VfPVS0TTGGu9I=
 github.com/0chain/msgp v1.1.62/go.mod h1:g7jEyb18KPe65d9RRhGw+ThaJr5duyBH8eaFgBUor7Y=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -67,6 +68,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
 github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
 github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
 github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
@@ -90,13 +92,15 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
+github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
 github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -154,6 +158,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -228,6 +233,7 @@ github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaF
 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 h1:6ob53CVz+ja2i7easAStApZJlh7sxyq3Cm7g1Di6iqA=
 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -632,6 +638,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=