From 4b989e6113262ed29b5d874b474e9bd1ada68516 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sat, 27 Jul 2024 19:50:51 +0800 Subject: [PATCH 01/10] fix: Resolve the inconsistency between member configuration and log index in snapshot meta --- cmake/braft.cmake | 4 ++-- pikiwidb.conf | 2 +- save_load.sh | 2 +- src/praft/praft.cc | 13 ++++++++++++- src/praft/praft.h | 3 +++ src/praft/psnapshot.cc | 21 ++++++++++++++++++++- 6 files changed, 39 insertions(+), 6 deletions(-) mode change 100755 => 100644 save_load.sh diff --git a/cmake/braft.cmake b/cmake/braft.cmake index bb8e688..2cf294c 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -19,8 +19,8 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being - GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" - GIT_TAG v1.1.2-alpha2 + GIT_REPOSITORY https://github.com/panlei-coder/braft.git # "https://github.com/pikiwidb/braft.git" + GIT_TAG get_configuration # v1.1.2-alpha2 GIT_SHALLOW true CMAKE_ARGS -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} diff --git a/pikiwidb.conf b/pikiwidb.conf index 38d7371..775bb85 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft no +use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset raft-port-offset 10 diff --git a/save_load.sh b/save_load.sh old mode 100755 new mode 100644 index 09f36c2..435dc48 --- a/save_load.sh +++ b/save_load.sh @@ -15,4 +15,4 @@ redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset redis-cli -p 7777 raft.node dosnapshot redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 +redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 \ No newline at end of file diff --git a/src/praft/praft.cc b/src/praft/praft.cc index c171ff4..92486d1 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -259,6 +259,16 @@ storage::LogIndex PRaft::GetLastLogIndex(bool is_flush) { return node_->get_last_log_index(is_flush); } +void PRaft::GetConfigurationByIndex(const int64_t index, braft::ConfigurationEntry* conf, + braft::ConfigurationEntry* learner_conf) { + if (!node_) { + ERROR("Node is not initialized"); + return; + } + + node_->get_configuration(index, conf, learner_conf); +} + void PRaft::SendNodeRequest(PClient* client) { assert(client); @@ -663,8 +673,9 @@ int PRaft::on_snapshot_load(braft::SnapshotReader* reader) { 2. When a node is improperly shut down and restarted, the minimum flush-index should be obtained as the starting point for fault recovery. */ + // replay from uint64_t replay_point = PSTORE.GetBackend(db_id_)->GetStorage()->GetSmallestFlushedLogIndex(); - node_->set_self_playback_point(replay_point); + node_->set_last_applied_index_and_term(replay_point); is_node_first_start_up_ = false; INFO("set replay_point: {}", replay_point); diff --git a/src/praft/praft.h b/src/praft/praft.h index 60c9e47..c09e1dc 100644 --- a/src/praft/praft.h +++ b/src/praft/praft.h @@ -14,6 +14,7 @@ #include #include +#include "braft/configuration_manager.h" #include "braft/file_system_adaptor.h" #include "braft/raft.h" #include "brpc/server.h" @@ -140,6 +141,8 @@ class PRaft : public braft::StateMachine { butil::Status GetListPeers(std::vector* peers); storage::LogIndex GetTerm(uint64_t log_index); storage::LogIndex GetLastLogIndex(bool is_flush = false); + void GetConfigurationByIndex(const int64_t index, braft::ConfigurationEntry* conf, + braft::ConfigurationEntry* learner_conf); bool IsInitialized() const { return node_ != nullptr && server_ != nullptr; } diff --git a/src/praft/psnapshot.cc b/src/praft/psnapshot.cc index 28fc99c..1dc4e1f 100644 --- a/src/praft/psnapshot.cc +++ b/src/praft/psnapshot.cc @@ -10,6 +10,8 @@ #include "psnapshot.h" +#include "braft/configuration.h" +#include "braft/configuration_manager.h" #include "braft/local_file_meta.pb.h" #include "braft/snapshot.h" #include "butil/files/file_path.h" @@ -79,12 +81,29 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o PSTORE.HandleTaskSpecificDB(tasks); AddAllFiles(snapshot_path, &snapshot_meta_memtable, snapshot_path); - // update snapshot last log index and last_log_term + // update snapshot last log index, last_log_term, conf of last log index and learners auto& new_meta = const_cast(snapshot_meta_memtable.meta()); auto last_log_index = PSTORE.GetBackend(db_id)->GetStorage()->GetSmallestFlushedLogIndex(); new_meta.set_last_included_index(last_log_index); auto last_log_term = PRAFT.GetTerm(last_log_index); new_meta.set_last_included_term(last_log_term); + braft::ConfigurationEntry conf_entry; + braft::ConfigurationEntry learner_conf_entry; + PRAFT.GetConfigurationByIndex(last_log_index, &conf_entry, &learner_conf_entry); + new_meta.clear_peers(); + for (braft::Configuration::const_iterator iter = conf_entry.conf.begin(); iter != conf_entry.conf.end(); ++iter) { + *new_meta.add_peers() = iter->to_string(); + } + new_meta.clear_old_peers(); + for (braft::Configuration::const_iterator iter = conf_entry.old_conf.begin(); iter != conf_entry.old_conf.end(); + ++iter) { + *new_meta.add_old_peers() = iter->to_string(); + } + new_meta.clear_learners(); + for (braft::Configuration::const_iterator iter = learner_conf_entry.conf.begin(); + iter != learner_conf_entry.conf.end(); ++iter) { + *new_meta.add_learners() = iter->to_string(); + } INFO("Succeed to fix db_{} snapshot meta: {}, {}", db_id, last_log_index, last_log_term); auto rc = snapshot_meta_memtable.save_to_file(fs, meta_path); From 6af2c77494f208b64462c563b78d83e15f35bca1 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 01:49:06 +0800 Subject: [PATCH 02/10] fix: fix a bug --- build.sh | 2 +- src/pikiwidb.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 644a5df..53532ea 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ do --) shift break - ;; + ;; esac shift done diff --git a/src/pikiwidb.cc b/src/pikiwidb.cc index 5108e8d..3bf60c5 100644 --- a/src/pikiwidb.cc +++ b/src/pikiwidb.cc @@ -227,7 +227,7 @@ static int InitLimit() { limit.rlim_cur = maxfiles; limit.rlim_max = maxfiles; if (setrlimit(RLIMIT_NOFILE, &limit) != -1) { - WARN("your 'limit -n ' of {} is not enough for PikiwiDB to start. PikiwiDB have successfully reconfig it to ", + WARN("your 'limit -n ' of {} is not enough for PikiwiDB to start. PikiwiDB have successfully reconfig it to {}", old_limit, limit.rlim_cur); } else { ERROR( @@ -282,7 +282,7 @@ int main(int ac, char* av[]) { daemonize(); } - InitLimit(); + // InitLimit(); // Code problem, program startup direct segment error pstd::InitRandom(); SignalSetup(); InitLogs(); From 08f323e4a0d35b8df0d9465796a5be41acc80637 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 02:18:08 +0800 Subject: [PATCH 03/10] fix: fix the bug of conf file --- pikiwidb.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pikiwidb.conf b/pikiwidb.conf index 775bb85..e72a5e7 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft no +use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset -raft-port-offset 10 +raft-port-offset 10 \ No newline at end of file From bae5da681bc56f7d8b976ee5c43ed74cd42fc3ca Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 08:43:20 +0800 Subject: [PATCH 04/10] fix: fix a comment --- cmake/braft.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/braft.cmake b/cmake/braft.cmake index 2cf294c..79bf4e8 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -19,8 +19,8 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being - GIT_REPOSITORY https://github.com/panlei-coder/braft.git # "https://github.com/pikiwidb/braft.git" - GIT_TAG get_configuration # v1.1.2-alpha2 + GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" + GIT_TAG v1.1.2.1 GIT_SHALLOW true CMAKE_ARGS -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} From c074a99dbc2fa50c9df8b2c6633774696add1d66 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 08:51:58 +0800 Subject: [PATCH 05/10] fix: Add a blank line at the end of the pikiwidb.conf file --- pikiwidb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pikiwidb.conf b/pikiwidb.conf index e72a5e7..38d7371 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -345,4 +345,4 @@ rocksdb-periodic-second 259200; ############################### RAFT ############################### use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset -raft-port-offset 10 \ No newline at end of file +raft-port-offset 10 From 46eb97809bb97be98a318d089b12534961a0f5ab Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 10:45:21 +0800 Subject: [PATCH 06/10] fix: fix comment --- src/praft/psnapshot.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/praft/psnapshot.cc b/src/praft/psnapshot.cc index 1dc4e1f..cc2b9de 100644 --- a/src/praft/psnapshot.cc +++ b/src/praft/psnapshot.cc @@ -91,17 +91,15 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o braft::ConfigurationEntry learner_conf_entry; PRAFT.GetConfigurationByIndex(last_log_index, &conf_entry, &learner_conf_entry); new_meta.clear_peers(); - for (braft::Configuration::const_iterator iter = conf_entry.conf.begin(); iter != conf_entry.conf.end(); ++iter) { + for (auto iter = conf_entry.conf.begin(); iter != conf_entry.conf.end(); ++iter) { *new_meta.add_peers() = iter->to_string(); } new_meta.clear_old_peers(); - for (braft::Configuration::const_iterator iter = conf_entry.old_conf.begin(); iter != conf_entry.old_conf.end(); - ++iter) { + for (auto iter = conf_entry.old_conf.begin(); iter != conf_entry.old_conf.end(); ++iter) { *new_meta.add_old_peers() = iter->to_string(); } new_meta.clear_learners(); - for (braft::Configuration::const_iterator iter = learner_conf_entry.conf.begin(); - iter != learner_conf_entry.conf.end(); ++iter) { + for (auto iter = learner_conf_entry.conf.begin(); iter != learner_conf_entry.conf.end(); ++iter) { *new_meta.add_learners() = iter->to_string(); } INFO("Succeed to fix db_{} snapshot meta: {}, {}", db_id, last_log_index, last_log_term); From df2344758804da58b4d27281c561c314a2947a8c Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:16:34 +0800 Subject: [PATCH 07/10] fix: resolve conflict --- build.sh | 2 +- save_load.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 53532ea..644a5df 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ do --) shift break - ;; + ;; esac shift done diff --git a/save_load.sh b/save_load.sh index 435dc48..09f36c2 100644 --- a/save_load.sh +++ b/save_load.sh @@ -15,4 +15,4 @@ redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset redis-cli -p 7777 raft.node dosnapshot redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 \ No newline at end of file +redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 From f409652f9f4e9569e9b9579a0753db04a894a02c Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:22:24 +0800 Subject: [PATCH 08/10] fix: resolve conflict --- save_load.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/save_load.sh b/save_load.sh index 09f36c2..a45170d 100644 --- a/save_load.sh +++ b/save_load.sh @@ -3,8 +3,13 @@ killall -9 pikiwidb mkdir leader follower1 -cd leader && ulimit -n 99999 && rm -fr * && ../bin/pikiwidb ../pikiwidb.conf --port 7777 & -cd follower1 && ulimit -n 99999 && rm -fr * && ../bin/pikiwidb ../pikiwidb.conf --port 8888 & + +PWD=`pwd` +PROJECT_HOME="${PWD}/../" +BIN="${PROJECT_HOME}/bin/pikiwidb" +CONF="${PROJECT_HOME}/etc/conf/pikiwidb.conf" +cd leader && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 7777 & +cd follower1 && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 8888 & sleep 5 redis-cli -p 7777 raft.cluster init From 9ec19206af0a2947b33011db96fc646a4021b4e9 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:27:17 +0800 Subject: [PATCH 09/10] fix: resolve conflict --- save_load.sh => etc/script/save_load.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename save_load.sh => etc/script/save_load.sh (100%) diff --git a/save_load.sh b/etc/script/save_load.sh similarity index 100% rename from save_load.sh rename to etc/script/save_load.sh From 70c63421c1f2a7cd38a83cf3670e7bd4a5f8423f Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:28:26 +0800 Subject: [PATCH 10/10] fix: resolve conflict --- etc/script/save_load.sh | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 etc/script/save_load.sh diff --git a/etc/script/save_load.sh b/etc/script/save_load.sh deleted file mode 100644 index a45170d..0000000 --- a/etc/script/save_load.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -killall -9 pikiwidb -mkdir leader follower1 - - -PWD=`pwd` -PROJECT_HOME="${PWD}/../" -BIN="${PROJECT_HOME}/bin/pikiwidb" -CONF="${PROJECT_HOME}/etc/conf/pikiwidb.conf" -cd leader && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 7777 & -cd follower1 && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 8888 & -sleep 5 - -redis-cli -p 7777 raft.cluster init - -redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 7777 raft.node dosnapshot -redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 7777 raft.node dosnapshot -redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset - -redis-cli -p 8888 raft.cluster join 127.0.0.1:7777