Skip to content

Commit

Permalink
Merge master into develop branch (#5668)
Browse files Browse the repository at this point in the history 
* feat(api):fix a concurrency issue for toString in BlockCapsule (#5657)

* feat(db):optimize cache settings (#5659)

* feat(lite):optimize DbLite tool (#5658)
  • Loading branch information
@lurais
lurais authored Jan 12, 2024
1 parent 52b3327 commit 0f5a8de
Show file tree
Hide file tree
Showing 6 changed files with 98 additions and 48 deletions.
22 changes: 1 addition & 21 deletions chainbase/src/main/java/org/tron/core/store/WitnessStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,17 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.tron.common.cache.CacheManager;
import org.tron.common.cache.CacheStrategies;
import org.tron.common.cache.CacheType;
import org.tron.common.cache.TronCache;
import org.tron.core.capsule.WitnessCapsule;
import org.tron.core.config.Parameter;
import org.tron.core.db.TronStoreWithRevoking;

@Slf4j(topic = "DB")
@Component
public class WitnessStore extends TronStoreWithRevoking<WitnessCapsule> {
// cache for 127 SR
private final TronCache<Integer, List<WitnessCapsule>> witnessStandbyCache;

@Autowired
protected WitnessStore(@Value("witness") String dbName) {
super(dbName);
String strategy = String.format(CacheStrategies.PATTERNS, 1, 1, "30s", 1);
witnessStandbyCache = CacheManager.allocate(CacheType.witnessStandby, strategy);
}

/**
Expand All @@ -48,19 +40,8 @@ public WitnessCapsule get(byte[] key) {
}

public List<WitnessCapsule> getWitnessStandby() {
List<WitnessCapsule> list =
witnessStandbyCache.getIfPresent(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH);
if (list != null) {
return list;
}
return updateWitnessStandby(null);
}

public List<WitnessCapsule> updateWitnessStandby(List<WitnessCapsule> all) {
List<WitnessCapsule> ret;
if (all == null) {
all = getAllWitnesses();
}
List<WitnessCapsule> all = getAllWitnesses();
all.sort(Comparator.comparingLong(WitnessCapsule::getVoteCount)
.reversed().thenComparing(Comparator.comparingInt(
(WitnessCapsule w) -> w.getAddress().hashCode()).reversed()));
Expand All @@ -71,7 +52,6 @@ public List<WitnessCapsule> updateWitnessStandby(List<WitnessCapsule> all) {
}
// trim voteCount = 0
ret.removeIf(w -> w.getVoteCount() < 1);
witnessStandbyCache.put(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH, ret);
return ret;
}

Expand Down
4 changes: 0 additions & 4 deletions consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,6 @@ public List<WitnessCapsule> getAllWitnesses() {
return witnessStore.getAllWitnesses();
}

public List<WitnessCapsule> updateWitnessStandby(List<WitnessCapsule> all) {
return witnessStore.updateWitnessStandby(all);
}

public void saveStateFlag(int flag) {
dynamicPropertiesStore.saveStateFlag(flag);
}
Expand Down
4 changes: 1 addition & 3 deletions consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,11 @@ public void doMaintenance() {
if (dynamicPropertiesStore.allowChangeDelegation()) {
long nextCycle = dynamicPropertiesStore.getCurrentCycleNumber() + 1;
dynamicPropertiesStore.saveCurrentCycleNumber(nextCycle);
List<WitnessCapsule> all = consensusDelegate.getAllWitnesses();
all.forEach(witness -> {
consensusDelegate.getAllWitnesses().forEach(witness -> {
delegationStore.setBrokerage(nextCycle, witness.createDbKey(),
delegationStore.getBrokerage(witness.createDbKey()));
delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount());
});
consensusDelegate.updateWitnessStandby(all);
}
}

Expand Down
6 changes: 3 additions & 3 deletions framework/src/main/java/org/tron/program/Version.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

public class Version {

public static final String VERSION_NAME = "GreatVoyage-v4.7.2-140-g9d13f9cb69";
public static final String VERSION_CODE = "18173";
private static final String VERSION = "4.7.3";
public static final String VERSION_NAME = "GreatVoyage-v4.7.3-5-g788136ebe";
public static final String VERSION_CODE = "18180";
private static final String VERSION = "4.7.3.1";

public static String getVersion() {
return VERSION;
Expand Down
17 changes: 15 additions & 2 deletions framework/src/test/java/org/tron/core/db/ManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,14 +581,24 @@ public void pushSwitchFork()
AccountResourceInsufficientException, EventBloomException {

String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62";
String key2 = "c85ef7d79691fe79573b1a7064c19c1a9819ebdbd1faaab1a8ec92344438aaf4";
byte[] privateKey = ByteArray.fromHexString(key);
final ECKey ecKey = ECKey.fromPrivate(privateKey);
byte[] address = ecKey.getAddress();

WitnessCapsule sr1 = new WitnessCapsule(
ByteString.copyFrom(address), "www.tron.net/first");
sr1.setVoteCount(1000000000L);
byte[] privateKey2 = ByteArray.fromHexString(key2);
final ECKey ecKey2 = ECKey.fromPrivate(privateKey2);
byte[] address2 = ecKey2.getAddress();
WitnessCapsule sr2 = new WitnessCapsule(
ByteString.copyFrom(address2), "www.tron.net/second");
sr2.setVoteCount(100000L);
chainManager.getWitnessStore().put(address, sr1);
WitnessCapsule witnessCapsule = new WitnessCapsule(ByteString.copyFrom(address));
chainManager.getWitnessScheduleStore().saveActiveWitnesses(new ArrayList<>());
chainManager.addWitness(ByteString.copyFrom(address));

List<WitnessCapsule> witnessStandby1 = chainManager.getWitnessStore().getWitnessStandby();
Block block = getSignedBlock(witnessCapsule.getAddress(), 1533529947843L, privateKey);
dbManager.pushBlock(new BlockCapsule(block));

Expand Down Expand Up @@ -625,6 +635,9 @@ public void pushSwitchFork()
} catch (Exception e) {
Assert.assertTrue(e instanceof Exception);
}
chainManager.getWitnessStore().put(address, sr2);
List<WitnessCapsule> witnessStandby2 = chainManager.getWitnessStore().getWitnessStandby();
Assert.assertNotEquals(witnessStandby1, witnessStandby2);
}


Expand Down
93 changes: 78 additions & 15 deletions plugins/src/main/java/org/tron/plugins/DbLite.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Bytes;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import java.io.File;
import java.io.FileNotFoundException;
Expand Down Expand Up @@ -153,10 +155,10 @@ public void generateSnapshot(String sourceDir, String snapshotDir) {
long start = System.currentTimeMillis();
snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString();
try {
mergeCheckpoint(sourceDir);
hasEnoughBlock(sourceDir);
List<String> snapshotDbs = getSnapshotDbs(sourceDir);
split(sourceDir, snapshotDir, snapshotDbs);
mergeCheckpoint2Snapshot(sourceDir, snapshotDir);
// write genesisBlock , latest recent blocks and trans
fillSnapshotBlockAndTransDb(sourceDir, snapshotDir);
// save min block to info
Expand Down Expand Up @@ -190,9 +192,9 @@ public void generateHistory(String sourceDir, String historyDir) {
throw new IllegalStateException(
String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir));
}
mergeCheckpoint(sourceDir);
hasEnoughBlock(sourceDir);
split(sourceDir, historyDir, archiveDbs);
mergeCheckpoint2History(sourceDir, historyDir);
// save max block to info
generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(),
getLatestBlockHeaderNum(sourceDir));
Expand Down Expand Up @@ -261,6 +263,15 @@ private List<String> getSnapshotDbs(String sourceDir) {
return snapshotDbs;
}

private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) {
List<String> snapshotDbs = getSnapshotDbs(sourceDir);
mergeCheckpoint(sourceDir, historyDir, snapshotDbs);
}

private void mergeCheckpoint2History(String sourceDir, String destDir) {
mergeCheckpoint(sourceDir, destDir, archiveDbs);
}

private void split(String sourceDir, String destDir, List<String> dbs) throws IOException {
logger.info("Begin to split the dbs.");
spec.commandLine().getOut().println("Begin to split the dbs.");
Expand All @@ -278,7 +289,7 @@ private void split(String sourceDir, String destDir, List<String> dbs) throws IO
FileUtils.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs);
}

private void mergeCheckpoint(String sourceDir) {
private void mergeCheckpoint(String sourceDir, String destDir, List<String> destDbs) {
logger.info("Begin to merge checkpoint to dataset.");
spec.commandLine().getOut().println("Begin to merge checkpoint to dataset.");
try {
Expand All @@ -287,18 +298,18 @@ private void mergeCheckpoint(String sourceDir) {
for (String cp : cpList) {
DBInterface checkpointDb = DbTool.getDB(
sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, cp);
recover(checkpointDb, sourceDir);
recover(checkpointDb, destDir, destDbs);
}
} else if (Paths.get(sourceDir, CHECKPOINT_DB).toFile().exists()) {
DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB);
recover(tmpDb, sourceDir);
recover(tmpDb, destDir, destDbs);
}
} catch (IOException | RocksDBException e) {
throw new RuntimeException(e);
}
}

private void recover(DBInterface db, String destDir)
private void recover(DBInterface db, String destDir, List<String> destDbs)
throws IOException, RocksDBException {
try (DBIterator iterator = db.iterator()) {
for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) {
Expand All @@ -312,15 +323,17 @@ private void recover(DBInterface db, String destDir)
byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length);
byte[] realValue =
value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length);
DBInterface destDb = DbTool.getDB(destDir, dbName);
if (realValue != null) {
destDb.put(realKey, realValue);
} else {
byte op = value[0];
if (DBUtils.Operator.DELETE.getValue() == op) {
destDb.delete(realKey);
if (destDbs != null && destDbs.contains(dbName)) {
DBInterface destDb = DbTool.getDB(destDir, dbName);
if (realValue != null) {
destDb.put(realKey, realValue);
} else {
destDb.put(realKey, new byte[0]);
byte op = value[0];
if (DBUtils.Operator.DELETE.getValue() == op) {
destDb.delete(realKey);
} else {
destDb.put(realKey, new byte[0]);
}
}
}
}
Expand All @@ -340,14 +353,30 @@ private void generateInfoProperties(String propertyfile, long num)
}

private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException {
// query latest_block_header_number from checkpoint first
final String latestBlockHeaderNumber = "latest_block_header_number";
DBInterface checkpointDb = getCheckpointDb(databaseDir);
Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb,
latestBlockHeaderNumber.getBytes());
if (blockNumber != null) {
return blockNumber;
}
// query from propertiesDb if checkpoint not contains latest_block_header_number
DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME);
return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber)))
.map(ByteArray::toLong)
.orElseThrow(
() -> new IllegalArgumentException("not found latest block header number"));
}

private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) {
byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key));
if (value != null && value.length > 1) {
return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length));
}
return null;
}

/**
* recent blocks, trans and genesis block.
*/
Expand Down Expand Up @@ -414,6 +443,15 @@ private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDB
return result;
}

private static byte[] simpleEncode(String s) {
byte[] bytes = s.getBytes();
byte[] length = Ints.toByteArray(bytes.length);
byte[] r = new byte[4 + bytes.length];
System.arraycopy(length, 0, r, 0, 4);
System.arraycopy(bytes, 0, r, 4, bytes.length);
return r;
}

private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir)
throws IOException, RocksDBException {
logger.info("Check the compatibility of this history.");
Expand Down Expand Up @@ -485,6 +523,7 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo)
DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME);
DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME);


ProgressBar.wrap(LongStream.rangeClosed(start, end)
.boxed()
.sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> {
Expand Down Expand Up @@ -519,6 +558,7 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws
return;
}


Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME);
logger.info("Begin to merge {} to database, start {} end {}.", bakDir, start, end);
spec.commandLine().getOut()
Expand All @@ -545,7 +585,17 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws

private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key)
throws IOException, RocksDBException {
byte[] value = DbTool.getDB(sourceDir, dbName).get(key);
DBInterface sourceDb = DbTool.getDB(sourceDir, dbName);
DBInterface checkpointDb = getCheckpointDb(sourceDir);
// get data from tmp first.
byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key));
byte[] value;
if (isEmptyBytes(valueFromTmp)) {
value = sourceDb.get(key);
} else {
value = valueFromTmp.length == 1
? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length);
}
if (isEmptyBytes(value)) {
throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s",
dbName, Arrays.toString(key)));
Expand Down Expand Up @@ -614,6 +664,19 @@ private long getSecondBlock(String databaseDir) throws RocksDBException, IOExcep
return num;
}

private DBInterface getCheckpointDb(String sourceDir) throws IOException, RocksDBException {
List<String> cpList = getCheckpointV2List(sourceDir);
DBInterface checkpointDb;
if (cpList.size() > 0) {
String latestCp = cpList.get(cpList.size() - 1);
checkpointDb = DbTool.getDB(
sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, latestCp);
} else {
checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB);
}
return checkpointDb;
}

@VisibleForTesting
public static void setRecentBlks(long recentBlks) {
RECENT_BLKS = recentBlks;
Expand Down

0 comments on commit 0f5a8de

Please sign in to comment.