forked from erigontech/silkworm
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge stage-senders into master (erigontech#40)
* Wrong WIN env var * LOCALAPPDATA returns %USERPROFILE%\AppData\Local thus the final path return would be %USERPROFILE%\AppData\Local\AppData\Roaming\TurboGeth\tg\chaindata * use APPDATA instead which returns %USERPROFILE%\AppData\Roaming * [WIP] Add check_senders.cpp * taken check_changes as template * Add check_senders executable * Add Boost components and CLI11 * Extend ecdsa methods * [WIP] check_senders * Method naming and constants for ECDSA * Adapt tests to new method naming for ECDSA * [WIP] - complete loop for senders recovery *no persistence yet * [WIP] Idempotent * Intercept SIGTERM too * Amend test Ecrecover test error * Libgmp on Wndows * snake_case naming * Add editorconfig * CMakeSettings.json formatting * Add end line * Style formatting *line length 120 *indent width 4 * Add Boost::thread dependency * Worker object * a thread worker abstraction * [WIP] stage-senders * distinct recovery threads * main thread only for db reading * output data dumped to temp files * TODO persist to real db * [WIP] Small changes * amend help * timing per interval * add timestamp to all log lines * [WIP] Wait for completion * explain why CTR+C does not stop immediately * Overload block_key * Implement Cursor::next * Implement Cursor::Next * [WIP] Rough implementation of cursor fetching * Indent namespaces * Introduce a Singleton for ChainDb * [WIP] Lmdb wrapper * [WIP] Wrong ptr for releaser * [WIP] Small test * [WIP] lmdb abstraction * propedutic to removal of lmdb.h++ * [WIP] lmdb abstraction * simplified classes for db handling : bucket class (bkt) mimics behavior of both a cursor and a dbi * rewrite of check_db.cpp to simulate the loop of scanning blocks in search of transactions * [WIP] * rewrite check_senders2 using new lmdb abstraction layer * look for blocks bound to canonical headers only * constrained worker threads to return in the right order to preserve key sorting * differnt output logging * Wrong variable naming * Refactoring of check_senders2 * now fully idempotent * persistance is effective (no more fake) * Small code clean-up * default batch set to 100k txs * Use std lib when available instead of boost * Moved check_senders2 to check_senders * Rebase on master * Wrong variables * mdb_size_t is no longer present : use size_t * chain config erroneously adopting etc instead of eth * Notify cursors to close on commit or abort instead of moving around a register of opened cursors * Clean up removed abseil-cpp submodule * Remove abseil-cpp submodule * Git adds this file * Try another tack with the abseil-cpp submodule * Remove abseil-cpp dir * Delete abseil-cpp subdir * Evmone submodule * Make chain_id of transaction uint64_t * Unify Andrew's work * Amend wrong logic in signing * only v r s if not for signing * else apply EIP-155 only if valued * ecdsa::recover must use recovery_id * narrow cast the result of substraction and division not v and then do math * Amend wrong logic * Submodule updates * Fix compilation under macOS * Replace filesystem with boost::filesystem * at the moment of writing only GCC 8.x+ supports it * Mark Txn dirty when some Bkt writes * Refactor signature recovery ID * No need to check recovery_id in is_valid_signature since get_signature_recovery_id always returns 0 or 1 * Apply eip155_chain_id only if not for_signing * previously got applied wether it was valued or not regardless the for_signing value * Dont' commit if user requested abort * Comments for get_signature_recovery_id * CLI argument to data * use of --datadir as in TG to avoid confusion * Wrong flag handling on chaindb * readonly option is inverted * [LMDB] Imrpoved error handling * cascading events amongst related objects * Revised check_db * Revised check_senders * Change naming of bucket to table * Remove unnecessary double open of bodies * Keep map of opened MDB_dbi(s) * lmdb classes renaming * Env -> Environment * Txn -> Transaction * Tbl -> Table * Clearing a table actually invalidates the cursor * If you clear the table you need to reopen it * Data type mismatch * Rewritten body scan loop * less convoluted * amended counting error Co-authored-by: yperbasis <[email protected]>
- Loading branch information
1 parent
2049559
commit adb743b
Showing
29 changed files
with
2,553 additions
and
501 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,5 @@ | ||
| Language: Cpp | ||
| BasedOnStyle: Google | ||
| ColumnLimit: 100 | ||
| ColumnLimit: 120 | ||
| IndentWidth: 4 | ||
| NamespaceIndentation: All |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # EditorConfig helps developers define and maintain consistent | ||
| # coding styles between different editors and IDEs | ||
| # http://editorconfig.org | ||
|
|
||
| root = true | ||
|
|
||
| [*] | ||
| end_of_line = lf | ||
| charset = utf-8 | ||
| trim_trailing_whitespace = true | ||
| insert_final_newline = true | ||
|
|
||
| [*.{cpp,c,hpp,h}] | ||
| indent_style = space | ||
| indent_size = 4 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| /* | ||
| Copyright 2020 The Silkworm Authors | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| #include <CLI/CLI.hpp> | ||
| #include <boost/endian/conversion.hpp> | ||
| #include <boost/filesystem.hpp> | ||
| #include <csignal> | ||
| #include <iostream> | ||
| #include <silkworm/chain/config.hpp> | ||
| #include <silkworm/db/bucket.hpp> | ||
| #include <silkworm/db/chaindb.hpp> | ||
| #include <silkworm/db/util.hpp> | ||
| #include <silkworm/types/block.hpp> | ||
| #include <regex> | ||
| #include <string> | ||
|
|
||
| namespace bfs = boost::filesystem; | ||
| using namespace silkworm; | ||
|
|
||
| bool shouldStop{false}; | ||
| int errorCode{0}; | ||
|
|
||
| void sig_handler(int signum) { | ||
| (void)signum; | ||
| std::cout << std::endl << "Request for termination intercepted. Stopping ..." << std::endl << std::endl; | ||
| shouldStop = true; | ||
| } | ||
|
|
||
| std::optional<uint64_t> parse_size(const std::string& strsize) { | ||
| std::regex pattern{ "^([0-9]{1,})([\\ ]{0,})?(B|KB|MB|GB|TB|EB)?$" }; | ||
| std::smatch matches; | ||
| if (!std::regex_search(strsize, matches, pattern, std::regex_constants::match_default)) { | ||
| return {}; | ||
| }; | ||
|
|
||
| uint64_t number{ std::strtoull(matches[1].str().c_str(), nullptr, 10) }; | ||
|
|
||
| if (matches[3].length() == 0) { | ||
| return { number }; | ||
| } | ||
| std::string suffix = matches[3].str(); | ||
| if (suffix == "B") { | ||
| return { number }; | ||
| } | ||
| else if (suffix == "KB") { | ||
| return { number * (1ull << 10) }; | ||
| } | ||
| else if (suffix == "MB") { | ||
| return { number * (1ull << 20) }; | ||
| } | ||
| else if (suffix == "GB") { | ||
| return { number * (1ull << 30) }; | ||
| } | ||
| else if (suffix == "TB") { | ||
| return { number * (1ull << 40) }; | ||
| } | ||
| else if (suffix == "EB") { | ||
| return { number * (1ull << 50) }; | ||
| } | ||
| else { | ||
| return {}; | ||
| } | ||
| } | ||
|
|
||
| int main(int argc, char* argv[]) { | ||
| CLI::App app("Tests db interfaces."); | ||
|
|
||
| std::string po_data_dir{silkworm::db::default_path()}; // Default database path | ||
| std::string po_mapsize_str{"0"}; // Default lmdb map size | ||
| bool po_debug{false}; // Might be ignored | ||
| CLI::Range range32(1u, UINT32_MAX); | ||
|
|
||
| // Check whether or not default db_path exists and | ||
| // has some files in it | ||
| bfs::path db_path(po_data_dir); | ||
| CLI::Option* db_path_set = | ||
| app.add_option("--datadir", po_data_dir, "Path to chain db", true)->check(CLI::ExistingDirectory); | ||
| if (!bfs::exists(db_path) || !bfs::is_directory(db_path) || db_path.empty()) { | ||
| db_path_set->required(); | ||
| } | ||
|
|
||
| app.add_flag("-d,--debug", po_debug, "May be ignored."); | ||
| app.add_option("--lmdb.mapSize", po_mapsize_str, "Lmdb map size", true); | ||
|
|
||
| CLI11_PARSE(app, argc, argv); | ||
| std::optional<uint64_t>lmdb_mapSize = parse_size(po_mapsize_str); | ||
| if (!lmdb_mapSize) { | ||
| std::cout << "Invalid map size" << std::endl; | ||
| return -1; | ||
| } | ||
|
|
||
| signal(SIGINT, sig_handler); | ||
| signal(SIGTERM, sig_handler); | ||
|
|
||
| // If database path is provided (and has passed CLI::ExistingDirectory validator | ||
| // check whether it is empty | ||
| db_path = bfs::path(po_data_dir); | ||
| if (db_path.empty()) { | ||
| std::cerr << "Provided --datadir [" << po_data_dir << "] is an empty directory" << std::endl | ||
| << "Try --help for help" << std::endl; | ||
| return -1; | ||
| } | ||
|
|
||
| std::shared_ptr<db::lmdb::Environment> lmdb_env{nullptr}; // Main lmdb environment | ||
| std::unique_ptr<db::lmdb::Transaction> lmdb_txn{nullptr}; // Main lmdb transaction | ||
|
|
||
| try { | ||
|
|
||
| // Open db and start transaction | ||
| db::lmdb::options opts{}; | ||
| if (*lmdb_mapSize) opts.map_size = *lmdb_mapSize; | ||
| lmdb_env = db::get_env(po_data_dir.c_str(), opts, /* forwriting=*/true); | ||
| std::cout << "Database is " << (lmdb_env->is_opened() ? "" : "NOT ") << "opened" << std::endl; | ||
| lmdb_txn = lmdb_env->begin_rw_transaction(); | ||
|
|
||
| MDB_envinfo i; | ||
| MDB_stat s; | ||
| MDB_val key, data; | ||
|
|
||
| lmdb_env->get_info(&i); | ||
|
|
||
| std::cout << "Database page size : " << i.me_mapsize << std::endl; | ||
|
|
||
| // A list of tables stored into the database | ||
| auto unnamed = lmdb_txn->open(0); | ||
|
|
||
| unnamed->get_stat(&s); | ||
| std::cout << "Database contains " << s.ms_entries << " named tables" << std::endl; | ||
| int rc{unnamed->get_first(&key, &data)}; | ||
| while (!shouldStop && rc == MDB_SUCCESS) { | ||
| std::string_view v{static_cast<char*>(key.mv_data), key.mv_size}; | ||
| std::cout << "Bucket " << v << " with "; | ||
| { | ||
| size_t rcount{0}; | ||
| auto b = lmdb_txn->open(v.data()); | ||
| b->get_rcount(&rcount); | ||
| std::cout << rcount << " record(s)\n"; | ||
| b->close(); | ||
| } | ||
| rc = unnamed->get_next(&key, &data); | ||
| } | ||
| std::cout << "\n" << std::endl; | ||
| std::cout << "Independent cursor navigation\n"; | ||
|
|
||
| // Independent cursor navigation sample | ||
| rc = unnamed->get_first(&key, &data); | ||
| MDB_val key_rev, data_rev; | ||
| auto unnamed_rev = lmdb_txn->open(0); | ||
| rc = unnamed_rev->get_last(&key_rev, &data_rev); | ||
| while (!shouldStop && rc == MDB_SUCCESS) { | ||
| std::string_view v{ static_cast<char*>(key.mv_data), key.mv_size }; | ||
| std::string_view v_rev{ static_cast<char*>(key_rev.mv_data), key_rev.mv_size }; | ||
| std::cout << "Cursor 1 Key " << v << " Cursor 2 Key " << v_rev << "\n"; | ||
| rc = unnamed->get_next(&key, &data); | ||
| rc = unnamed_rev->get_prev(&key_rev, &data_rev); | ||
| } | ||
|
|
||
| std::cout << "\n" << std::endl; | ||
| unnamed->close(); | ||
| unnamed_rev->close(); | ||
|
|
||
| } catch (db::lmdb::exception& ex) { | ||
| // This handles specific lmdb errors | ||
| std::cout << ex.err() << " " << ex.what() << std::endl; | ||
| } catch (std::runtime_error& ex) { | ||
| // This handles runtime ligic errors | ||
| // eg. trying to open two rw txns | ||
| std::cout << ex.what() << std::endl; | ||
| } | ||
|
|
||
| if (lmdb_txn) lmdb_txn->abort(); | ||
| if (lmdb_env) { | ||
| lmdb_env->close(); | ||
| std::cout << "Database is " << (lmdb_env->is_opened() ? "" : "NOT ") << "opened" << std::endl; | ||
| } | ||
|
|
||
| return 0; | ||
| } |
Oops, something went wrong.