Skip to content

Commit

Permalink
feat(lite): optimize DbLite tool (#5647)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lurais
lurais authored Jan 5, 2024
1 parent 1e2986a commit db9f3be
Showing 1 changed file with 15 additions and 73 deletions.
88 changes: 15 additions & 73 deletions plugins/src/main/java/org/tron/plugins/DbLite.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Bytes;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import java.io.File;
import java.io.FileNotFoundException;
Expand Down Expand Up @@ -155,10 +153,10 @@ public void generateSnapshot(String sourceDir, String snapshotDir) {
long start = System.currentTimeMillis();
snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString();
try {
mergeCheckpoint(sourceDir);
hasEnoughBlock(sourceDir);
List<String> snapshotDbs = getSnapshotDbs(sourceDir);
split(sourceDir, snapshotDir, snapshotDbs);
mergeCheckpoint2Snapshot(sourceDir, snapshotDir);
// write genesisBlock , latest recent blocks and trans
fillSnapshotBlockAndTransDb(sourceDir, snapshotDir);
// save min block to info
Expand Down Expand Up @@ -192,9 +190,9 @@ public void generateHistory(String sourceDir, String historyDir) {
throw new IllegalStateException(
String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir));
}
mergeCheckpoint(sourceDir);
hasEnoughBlock(sourceDir);
split(sourceDir, historyDir, archiveDbs);
mergeCheckpoint2History(sourceDir, historyDir);
// save max block to info
generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(),
getLatestBlockHeaderNum(sourceDir));
Expand Down Expand Up @@ -263,15 +261,6 @@ private List<String> getSnapshotDbs(String sourceDir) {
return snapshotDbs;
}

private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) {
List<String> snapshotDbs = getSnapshotDbs(sourceDir);
mergeCheckpoint(sourceDir, historyDir, snapshotDbs);
}

private void mergeCheckpoint2History(String sourceDir, String destDir) {
mergeCheckpoint(sourceDir, destDir, archiveDbs);
}

private void split(String sourceDir, String destDir, List<String> dbs) throws IOException {
logger.info("Begin to split the dbs.");
spec.commandLine().getOut().println("Begin to split the dbs.");
Expand All @@ -289,7 +278,7 @@ private void split(String sourceDir, String destDir, List<String> dbs) throws IO
FileUtils.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs);
}

private void mergeCheckpoint(String sourceDir, String destDir, List<String> destDbs) {
private void mergeCheckpoint(String sourceDir) {
logger.info("Begin to merge checkpoint to dataset.");
spec.commandLine().getOut().println("Begin to merge checkpoint to dataset.");
try {
Expand All @@ -298,18 +287,18 @@ private void mergeCheckpoint(String sourceDir, String destDir, List<String> dest
for (String cp : cpList) {
DBInterface checkpointDb = DbTool.getDB(
sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, cp);
recover(checkpointDb, destDir, destDbs);
recover(checkpointDb, sourceDir);
}
} else if (Paths.get(sourceDir, CHECKPOINT_DB).toFile().exists()) {
DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB);
recover(tmpDb, destDir, destDbs);
recover(tmpDb, sourceDir);
}
} catch (IOException | RocksDBException e) {
throw new RuntimeException(e);
}
}

private void recover(DBInterface db, String destDir, List<String> destDbs)
private void recover(DBInterface db, String destDir)
throws IOException, RocksDBException {
try (DBIterator iterator = db.iterator()) {
for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) {
Expand All @@ -323,17 +312,15 @@ private void recover(DBInterface db, String destDir, List<String> destDbs)
byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length);
byte[] realValue =
value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length);
if (destDbs != null && destDbs.contains(dbName)) {
DBInterface destDb = DbTool.getDB(destDir, dbName);
if (realValue != null) {
destDb.put(realKey, realValue);
DBInterface destDb = DbTool.getDB(destDir, dbName);
if (realValue != null) {
destDb.put(realKey, realValue);
} else {
byte op = value[0];
if (DBUtils.Operator.DELETE.getValue() == op) {
destDb.delete(realKey);
} else {
byte op = value[0];
if (DBUtils.Operator.DELETE.getValue() == op) {
destDb.delete(realKey);
} else {
destDb.put(realKey, new byte[0]);
}
destDb.put(realKey, new byte[0]);
}
}
}
Expand All @@ -353,38 +340,14 @@ private void generateInfoProperties(String propertyfile, long num)
}

private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException {
// query latest_block_header_number from checkpoint first
final String latestBlockHeaderNumber = "latest_block_header_number";
List<String> cpList = getCheckpointV2List(databaseDir);
DBInterface checkpointDb;
if (cpList.size() > 0) {
String lastestCp = cpList.get(cpList.size() - 1);
checkpointDb = DbTool.getDB(
databaseDir + "/" + DBUtils.CHECKPOINT_DB_V2, lastestCp);
} else {
checkpointDb = DbTool.getDB(databaseDir, CHECKPOINT_DB);
}
Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb,
latestBlockHeaderNumber.getBytes());
if (blockNumber != null) {
return blockNumber;
}
// query from propertiesDb if checkpoint not contains latest_block_header_number
DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME);
return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber)))
.map(ByteArray::toLong)
.orElseThrow(
() -> new IllegalArgumentException("not found latest block header number"));
}

private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) {
byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key));
if (value != null && value.length > 1) {
return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length));
}
return null;
}

/**
* recent blocks, trans and genesis block.
*/
Expand Down Expand Up @@ -451,15 +414,6 @@ private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDB
return result;
}

private static byte[] simpleEncode(String s) {
byte[] bytes = s.getBytes();
byte[] length = Ints.toByteArray(bytes.length);
byte[] r = new byte[4 + bytes.length];
System.arraycopy(length, 0, r, 0, 4);
System.arraycopy(bytes, 0, r, 4, bytes.length);
return r;
}

private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir)
throws IOException, RocksDBException {
logger.info("Check the compatibility of this history.");
Expand Down Expand Up @@ -531,7 +485,6 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo)
DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME);
DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME);


ProgressBar.wrap(LongStream.rangeClosed(start, end)
.boxed()
.sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> {
Expand Down Expand Up @@ -566,7 +519,6 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws
return;
}


Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME);
logger.info("Begin to merge {} to database, start {} end {}.", bakDir, start, end);
spec.commandLine().getOut()
Expand All @@ -593,17 +545,7 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws

private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key)
throws IOException, RocksDBException {
DBInterface sourceDb = DbTool.getDB(sourceDir, dbName);
DBInterface checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB);
// get data from tmp first.
byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key));
byte[] value;
if (isEmptyBytes(valueFromTmp)) {
value = sourceDb.get(key);
} else {
value = valueFromTmp.length == 1
? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length);
}
byte[] value = DbTool.getDB(sourceDir, dbName).get(key);
if (isEmptyBytes(value)) {
throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s",
dbName, Arrays.toString(key)));
Expand Down

0 comments on commit db9f3be

Please sign in to comment.