From 5a7e1b74b112e24aef0b065fc6a0f1a8fac90008 Mon Sep 17 00:00:00 2001 From: cwbhhjl Date: Mon, 30 Jan 2023 10:24:09 +0800 Subject: [PATCH 1/6] pruning without batch (cherry picked from commit a79c1da1f9ad5112f4daeeb7c0aac3845a580457) --- libs/iavl/mutable_tree.go | 2 +- libs/iavl/mutable_tree_oec.go | 4 ++- libs/iavl/nodedb.go | 57 ++++++++++++++++++++++++++++++++-- libs/iavl/nodedb_oec.go | 58 ++++++++++++++++++++++++++++++++--- 4 files changed, 112 insertions(+), 9 deletions(-) diff --git a/libs/iavl/mutable_tree.go b/libs/iavl/mutable_tree.go index 7f9a9c47d9..6a4961cb2e 100644 --- a/libs/iavl/mutable_tree.go +++ b/libs/iavl/mutable_tree.go @@ -906,7 +906,7 @@ func (tree *MutableTree) deleteVersion(batch dbm.Batch, version int64, versions return err } - tree.ndb.deleteVersion(batch, version, true) + tree.ndb.deleteVersion(batch, version, true, false) return nil } diff --git a/libs/iavl/mutable_tree_oec.go b/libs/iavl/mutable_tree_oec.go index e47bb42167..ec9e05ec9c 100644 --- a/libs/iavl/mutable_tree_oec.go +++ b/libs/iavl/mutable_tree_oec.go @@ -225,6 +225,7 @@ func (tree *MutableTree) persist(version int64) { } func (tree *MutableTree) commitSchedule() { + tree.ndb.cleanPruningInDB() tree.loadVersionToCommittedHeightMap() for event := range tree.commitCh { if event.version < 0 { @@ -275,9 +276,10 @@ func (tree *MutableTree) pruningSchedule() { for event := range tree.pruneCh { if event.version >= 0 { trc := trace.NewTracer("pruningSchedule") + noBatch := IavlCommitAsyncNoBatch batch := tree.ndb.NewBatch() trc.Pin("deleteVersion") - tree.ndb.deleteVersion(batch, event.version, true) + tree.ndb.deleteVersion(batch, event.version, true, noBatch) trc.Pin("Commit") if err := tree.ndb.Commit(batch); err != nil { panic(err) diff --git a/libs/iavl/nodedb.go b/libs/iavl/nodedb.go index 0b6148e540..16b18f3b56 100644 --- a/libs/iavl/nodedb.go +++ b/libs/iavl/nodedb.go @@ -31,6 +31,8 @@ const ( // Using semantic versioning: https://semver.org/ defaultStorageVersionValue = "1.0.0" fastStorageVersionValue = "1.1.0" + + pruningVersionKey = "pruning_version" ) var ( @@ -629,7 +631,7 @@ func (ndb *nodeDB) deleteOrphans(batch dbm.Batch, version int64) { if predecessor < fromVersion || fromVersion == toVersion { ndb.log(IavlDebug, "DELETE", "predecessor", predecessor, "fromVersion", fromVersion, "toVersion", toVersion, "hash", hash) batch.Delete(ndb.nodeKey(hash)) - ndb.syncUnCacheNode(hash) + ndb.uncacheNode(hash) ndb.state.increaseDeletedCount() } else { ndb.log(IavlDebug, "MOVE", "predecessor", predecessor, "fromVersion", fromVersion, "toVersion", toVersion, "hash", hash) @@ -638,6 +640,48 @@ func (ndb *nodeDB) deleteOrphans(batch dbm.Batch, version int64) { }) } +func (ndb *nodeDB) deleteOrphansFromDB(version int64) { + // Will be zero if there is no previous version. + predecessor := ndb.getPreviousVersion(version) + + // Traverse orphans with a lifetime ending at the version specified. + // TODO optimize. + ndb.traverseOrphansVersion(version, func(key, hash []byte) { + var fromVersion, toVersion int64 + + // See comment on `orphanKeyFmt`. Note that here, `version` and + // `toVersion` are always equal. + orphanKeyFormat.Scan(key, &toVersion, &fromVersion) + + // Delete orphan key and reverse-lookup key. + err := ndb.db.Delete(key) + if err != nil { + panic(err) + } + + // If there is no predecessor, or the predecessor is earlier than the + // beginning of the lifetime (ie: negative lifetime), or the lifetime + // spans a single version and that version is the one being deleted, we + // can delete the orphan. Otherwise, we shorten its lifetime, by + // moving its endpoint to the previous version. + if predecessor < fromVersion || fromVersion == toVersion { + ndb.log(IavlDebug, "DELETE", "predecessor", predecessor, "fromVersion", fromVersion, "toVersion", toVersion, "hash", hash) + err = ndb.db.Delete(ndb.nodeKey(hash)) + if err != nil { + panic(err) + } + ndb.uncacheNode(hash) + ndb.state.increaseDeletedCount() + } else { + ndb.log(IavlDebug, "MOVE", "predecessor", predecessor, "fromVersion", fromVersion, "toVersion", toVersion, "hash", hash) + err = ndb.saveOrphanToDB(hash, fromVersion, predecessor) + if err != nil { + panic(err) + } + } + }) +} + func (ndb *nodeDB) nodeKey(hash []byte) []byte { return nodeKeyFormat.KeyBytes(hash) } @@ -710,11 +754,18 @@ func (ndb *nodeDB) getPreviousVersion(version int64) int64 { } // deleteRoot deletes the root entry from disk, but not the node it points to. -func (ndb *nodeDB) deleteRoot(batch dbm.Batch, version int64, checkLatestVersion bool) { +func (ndb *nodeDB) deleteRoot(batch dbm.Batch, version int64, checkLatestVersion bool, writeToDB bool) { if checkLatestVersion && version == ndb.getLatestVersion() { panic("Tried to delete latest version") } - batch.Delete(ndb.rootKey(version)) + if !writeToDB { + batch.Delete(ndb.rootKey(version)) + } else { + err := ndb.db.Delete(ndb.rootKey(version)) + if err != nil { + panic(err) + } + } } func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { diff --git a/libs/iavl/nodedb_oec.go b/libs/iavl/nodedb_oec.go index 4da8e52724..04c287e286 100644 --- a/libs/iavl/nodedb_oec.go +++ b/libs/iavl/nodedb_oec.go @@ -5,6 +5,7 @@ import ( "container/list" "encoding/binary" "fmt" + "strconv" cmap "github.com/orcaman/concurrent-map" @@ -269,13 +270,62 @@ func (ndb *nodeDB) DeleteVersion(batch dbm.Batch, version int64, checkLatestVers return err } - ndb.deleteVersion(batch, version, checkLatestVersion) + ndb.deleteVersion(batch, version, checkLatestVersion, false) return nil } -func (ndb *nodeDB) deleteVersion(batch dbm.Batch, version int64, checkLatestVersion bool) { - ndb.deleteOrphans(batch, version) - ndb.deleteRoot(batch, version, checkLatestVersion) +func (ndb *nodeDB) deleteVersion(batch dbm.Batch, version int64, checkLatestVersion bool, writeToDB bool) { + if !writeToDB { + ndb.deleteOrphans(batch, version) + ndb.deleteRoot(batch, version, checkLatestVersion, writeToDB) + } else { + ndb.setPruningRoot(version, checkLatestVersion) + ndb.deleteRoot(batch, version, checkLatestVersion, writeToDB) + ndb.deleteOrphansFromDB(version) + ndb.deletePruningRoot() + } +} + +func (ndb *nodeDB) cleanPruningInDB() { + version, exist := ndb.getPruningRoot() + if !exist { + return + } + ndb.deleteRoot(nil, version, false, true) + ndb.deleteOrphansFromDB(version) + ndb.deletePruningRoot() +} + +func (ndb *nodeDB) setPruningRoot(version int64, checkLatestVersion bool) { + if checkLatestVersion && version == ndb.getLatestVersion() { + panic("Tried to delete latest version") + } + err := ndb.db.Set(metadataKeyFormat.Key([]byte(pruningVersionKey)), []byte(strconv.FormatInt(version, 10))) + if err != nil { + panic(err) + } +} + +func (ndb *nodeDB) deletePruningRoot() { + err := ndb.db.Delete(metadataKeyFormat.Key([]byte(pruningVersionKey))) + if err != nil { + panic(err) + } +} + +func (ndb *nodeDB) getPruningRoot() (int64, bool) { + bz, err := ndb.db.Get(metadataKeyFormat.Key([]byte(pruningVersionKey))) + if err != nil { + panic(err) + } + if len(bz) == 0 { + return 0, false + } + v, err := strconv.ParseInt(string(bz), 10, 64) + if err != nil { + panic(err) + } + return v, true } func (ndb *nodeDB) checkoutVersionReaders(version int64) error { From 704f72d31e1581b671ea40644812cf93d989844f Mon Sep 17 00:00:00 2001 From: cwbhhjl Date: Wed, 8 Feb 2023 16:36:47 +0800 Subject: [PATCH 2/6] update cleanPruningInDB --- libs/iavl/mutable_tree_map.go | 1 + libs/iavl/mutable_tree_oec.go | 1 - libs/iavl/nodedb_oec.go | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/iavl/mutable_tree_map.go b/libs/iavl/mutable_tree_map.go index 9953125794..44926203ee 100644 --- a/libs/iavl/mutable_tree_map.go +++ b/libs/iavl/mutable_tree_map.go @@ -26,6 +26,7 @@ func (tm *TreeMap) addNewTree(tree *MutableTree) { defer tm.mtx.Unlock() if _, ok := tm.mutableTreeSavedMap[tree.GetModuleName()]; !ok { tm.mutableTreeSavedMap[tree.GetModuleName()] = tree + tree.ndb.cleanPruningInDB() go tree.commitSchedule() if EnablePruningHistoryState { go tree.pruningSchedule() diff --git a/libs/iavl/mutable_tree_oec.go b/libs/iavl/mutable_tree_oec.go index ec9e05ec9c..a4d669f8eb 100644 --- a/libs/iavl/mutable_tree_oec.go +++ b/libs/iavl/mutable_tree_oec.go @@ -225,7 +225,6 @@ func (tree *MutableTree) persist(version int64) { } func (tree *MutableTree) commitSchedule() { - tree.ndb.cleanPruningInDB() tree.loadVersionToCommittedHeightMap() for event := range tree.commitCh { if event.version < 0 { diff --git a/libs/iavl/nodedb_oec.go b/libs/iavl/nodedb_oec.go index 04c287e286..be6f15131f 100644 --- a/libs/iavl/nodedb_oec.go +++ b/libs/iavl/nodedb_oec.go @@ -291,9 +291,11 @@ func (ndb *nodeDB) cleanPruningInDB() { if !exist { return } + ndb.log(IavlErr, "start cleanPruningInDB", "name", ndb.name, "version", version) ndb.deleteRoot(nil, version, false, true) ndb.deleteOrphansFromDB(version) ndb.deletePruningRoot() + ndb.log(IavlErr, "cleanPruningInDB is done", "name", ndb.name, "version", version) } func (ndb *nodeDB) setPruningRoot(version int64, checkLatestVersion bool) { From 0af257338172f1872441f5ffba42b696dbfa6688 Mon Sep 17 00:00:00 2001 From: cwbhhjl Date: Wed, 8 Feb 2023 17:15:24 +0800 Subject: [PATCH 3/6] delete orphans with batch in cleanPruning --- libs/iavl/nodedb_oec.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/iavl/nodedb_oec.go b/libs/iavl/nodedb_oec.go index be6f15131f..286bd35303 100644 --- a/libs/iavl/nodedb_oec.go +++ b/libs/iavl/nodedb_oec.go @@ -293,7 +293,12 @@ func (ndb *nodeDB) cleanPruningInDB() { } ndb.log(IavlErr, "start cleanPruningInDB", "name", ndb.name, "version", version) ndb.deleteRoot(nil, version, false, true) - ndb.deleteOrphansFromDB(version) + batch := ndb.db.NewBatch() + defer batch.Close() + ndb.deleteOrphans(batch, version) + if err := batch.Write(); err != nil { + panic(err) + } ndb.deletePruningRoot() ndb.log(IavlErr, "cleanPruningInDB is done", "name", ndb.name, "version", version) } From 703134d4cbf75ed1edf7cad67656328ffeb34ea4 Mon Sep 17 00:00:00 2001 From: cwbhhjl Date: Wed, 15 Feb 2023 11:07:10 +0800 Subject: [PATCH 4/6] can;t delete origin orphans key before update orphans key or delete node --- libs/iavl/nodedb.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libs/iavl/nodedb.go b/libs/iavl/nodedb.go index 16b18f3b56..a9c4d8a350 100644 --- a/libs/iavl/nodedb.go +++ b/libs/iavl/nodedb.go @@ -648,17 +648,12 @@ func (ndb *nodeDB) deleteOrphansFromDB(version int64) { // TODO optimize. ndb.traverseOrphansVersion(version, func(key, hash []byte) { var fromVersion, toVersion int64 + var err error // See comment on `orphanKeyFmt`. Note that here, `version` and // `toVersion` are always equal. orphanKeyFormat.Scan(key, &toVersion, &fromVersion) - // Delete orphan key and reverse-lookup key. - err := ndb.db.Delete(key) - if err != nil { - panic(err) - } - // If there is no predecessor, or the predecessor is earlier than the // beginning of the lifetime (ie: negative lifetime), or the lifetime // spans a single version and that version is the one being deleted, we @@ -679,6 +674,12 @@ func (ndb *nodeDB) deleteOrphansFromDB(version int64) { panic(err) } } + + // Delete orphan key and reverse-lookup key. + err = ndb.db.Delete(key) + if err != nil { + panic(err) + } }) } From 5669e979931ae08143c7f5244a0ccbb649ba9a9b Mon Sep 17 00:00:00 2001 From: cwbhhjl Date: Thu, 16 Feb 2023 15:45:45 +0800 Subject: [PATCH 5/6] Update logic for deleting root and orphans in nodeDB --- libs/iavl/nodedb_oec.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/iavl/nodedb_oec.go b/libs/iavl/nodedb_oec.go index 286bd35303..b322c20f2d 100644 --- a/libs/iavl/nodedb_oec.go +++ b/libs/iavl/nodedb_oec.go @@ -278,12 +278,12 @@ func (ndb *nodeDB) deleteVersion(batch dbm.Batch, version int64, checkLatestVers if !writeToDB { ndb.deleteOrphans(batch, version) ndb.deleteRoot(batch, version, checkLatestVersion, writeToDB) - } else { - ndb.setPruningRoot(version, checkLatestVersion) - ndb.deleteRoot(batch, version, checkLatestVersion, writeToDB) - ndb.deleteOrphansFromDB(version) - ndb.deletePruningRoot() + return } + ndb.setPruningRoot(version, checkLatestVersion) + ndb.deleteRoot(batch, version, checkLatestVersion, writeToDB) + ndb.deleteOrphansFromDB(version) + ndb.deletePruningRoot() } func (ndb *nodeDB) cleanPruningInDB() { From d3aa6346182097d43cb1b346a9eebc9c852987f7 Mon Sep 17 00:00:00 2001 From: cwbhhjl Date: Thu, 16 Feb 2023 15:53:51 +0800 Subject: [PATCH 6/6] update comment --- libs/iavl/nodedb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/iavl/nodedb.go b/libs/iavl/nodedb.go index a9c4d8a350..04f8df6121 100644 --- a/libs/iavl/nodedb.go +++ b/libs/iavl/nodedb.go @@ -676,6 +676,7 @@ func (ndb *nodeDB) deleteOrphansFromDB(version int64) { } // Delete orphan key and reverse-lookup key. + // we can't delete origin orphanKey before delete node or update orphanKey. err = ndb.db.Delete(key) if err != nil { panic(err)