From d268f3e29251b99511516a5c283d00f7c71f13c8 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:35:01 +0100 Subject: [PATCH 01/28] Store TraceAssembler state locally in working dir (#92) * Store TraceAssembler state locally in working dir * clean up * Index DNS entries * testnet flag * fix * fix small bugs in ton-smc-scanner, add storage bag id for dns entry * support t.me domains * Fixed unsigned problems * Gc fix, but still broken * fix saving state * save TA state to files * fix log level * fix case expected_seqno_ == 0 --------- Co-authored-by: K-Dimentional Tree --- .../src/BlockInterfacesDetector.h | 3 + ton-index-postgres-v2/src/IndexScheduler.cpp | 68 ++-- ton-index-postgres-v2/src/IndexScheduler.h | 7 +- .../src/InsertManagerPostgres.cpp | 163 ++++++++-- ton-index-postgres-v2/src/main.cpp | 19 +- ton-smc-scanner/src/PostgreSQLInserter.cpp | 198 ++++++++++-- ton-smc-scanner/src/PostgreSQLInserter.h | 6 +- ton-smc-scanner/src/SmcScanner.cpp | 105 +++++- ton-smc-scanner/src/SmcScanner.h | 4 +- ton-smc-scanner/src/main.cpp | 6 + ton-trace-emulator/src/Serializer.hpp | 66 +--- tondb-scanner/CMakeLists.txt | 5 +- tondb-scanner/src/IndexData.h | 11 +- tondb-scanner/src/TraceAssembler.cpp | 302 ++++++++++-------- tondb-scanner/src/TraceAssembler.h | 74 ++--- tondb-scanner/src/msgpack-utils.h | 67 ++++ tondb-scanner/src/smc-interfaces/NftSale.cpp | 2 +- tondb-scanner/src/smc-interfaces/Tokens.cpp | 166 +++++++++- tondb-scanner/src/smc-interfaces/Tokens.h | 17 +- .../{common-utils.h => execute-smc.cpp} | 18 +- .../src/smc-interfaces/execute-smc.h | 24 ++ tondb-scanner/src/tlb/tokens.tlb | 21 +- 22 files changed, 984 insertions(+), 368 deletions(-) create mode 100644 tondb-scanner/src/msgpack-utils.h rename tondb-scanner/src/smc-interfaces/{common-utils.h => execute-smc.cpp} (64%) create mode 100644 tondb-scanner/src/smc-interfaces/execute-smc.h diff --git a/ton-index-postgres-v2/src/BlockInterfacesDetector.h b/ton-index-postgres-v2/src/BlockInterfacesDetector.h index 02cfe572..a7ffff86 100644 --- a/ton-index-postgres-v2/src/BlockInterfacesDetector.h +++ b/ton-index-postgres-v2/src/BlockInterfacesDetector.h @@ -112,6 +112,9 @@ class BlockInterfaceProcessor: public td::actor::Actor { nft_item_data.last_transaction_now = last_trans_now; nft_item_data.code_hash = code_hash; nft_item_data.data_hash = data_hash; + if (arg.dns_entry) { + nft_item_data.dns_entry = NFTItemDataV2::DNSEntry{arg.dns_entry->domain, arg.dns_entry->wallet, arg.dns_entry->next_resolver, arg.dns_entry->site_adnl}; + } interfaces_[address].push_back(nft_item_data); } else if constexpr (std::is_same_v) { GetGemsNftAuctionData auction_data; diff --git a/ton-index-postgres-v2/src/IndexScheduler.cpp b/ton-index-postgres-v2/src/IndexScheduler.cpp index 8e5f33b0..fed204b3 100644 --- a/ton-index-postgres-v2/src/IndexScheduler.cpp +++ b/ton-index-postgres-v2/src/IndexScheduler.cpp @@ -6,7 +6,7 @@ void IndexScheduler::start_up() { - trace_assembler_ = td::actor::create_actor("trace_assembler", from_seqno_); + trace_assembler_ = td::actor::create_actor("trace_assembler", working_dir_ + "/trace_assembler", max_queue_.mc_blocks_); } std::string get_time_string(double seconds) { @@ -67,16 +67,6 @@ void IndexScheduler::run() { td::actor::send_closure(SelfId, &IndexScheduler::got_existing_seqnos, std::move(R)); }); td::actor::send_closure(insert_manager_, &InsertManagerInterface::get_existing_seqnos, std::move(P), from_seqno_, to_seqno_); - - // restore TraceAssembler state - auto Q = td::PromiseCreator::lambda([trace_assembler = trace_assembler_.get()](td::Result R) { - if (R.is_error()) { - LOG(ERROR) << "Failed to read TraceAssemblerState from database: " << R.move_as_error(); - std::_Exit(2); - } - td::actor::send_closure(trace_assembler, &TraceAssembler::restore_trace_assembler_state, R.move_as_ok()); - }); - td::actor::send_closure(insert_manager_, &InsertManagerInterface::get_trace_assembler_state, std::move(Q)); } void IndexScheduler::got_existing_seqnos(td::Result> R) { @@ -85,39 +75,61 @@ void IndexScheduler::got_existing_seqnos(td::Result> return; } - std::vector seqnos_{R.move_as_ok()}; - if (seqnos_.size()) { - std::sort(seqnos_.begin(), seqnos_.end()); - std::int32_t next_seqno = seqnos_[0]; - for (auto value : seqnos_) { + ton::BlockSeqno next_seqno = from_seqno_; + std::vector seqnos{R.move_as_ok()}; + if (seqnos.size()) { + std::sort(seqnos.begin(), seqnos.end()); + next_seqno = seqnos[0]; + for (auto value : seqnos) { if (value == next_seqno) { ++next_seqno; - existing_seqnos_.insert(value); } else { break; } } - LOG(INFO) << "Accepted " << existing_seqnos_.size() << " of " << seqnos_.size() - << " existing seqnos. Next seqno: " << next_seqno; - td::actor::send_closure(trace_assembler_, &TraceAssembler::update_expected_seqno, next_seqno); + size_t accepted_cnt = next_seqno - from_seqno_; + LOG(INFO) << "Accepted " << accepted_cnt << " of " << seqnos.size() << " existing seqnos. Next seqno: " << next_seqno; } - alarm_timestamp() = td::Timestamp::in(1.0); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), next_seqno](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "TraceAssembler state not found for seqno " << next_seqno - 1; + LOG(WARNING) << "Traces that started before block " << next_seqno << " will be marked as broken and not inserted."; + td::actor::send_closure(SelfId, &IndexScheduler::got_trace_assembler_last_state_seqno, next_seqno - 1); + } else { + LOG(INFO) << "Restored TraceAssembler state for seqno " << R.ok(); + td::actor::send_closure(SelfId, &IndexScheduler::got_trace_assembler_last_state_seqno, R.move_as_ok()); + } + }); + + td::actor::send_closure(trace_assembler_, &TraceAssembler::restore_state, next_seqno - 1, std::move(P)); +} + +void IndexScheduler::got_trace_assembler_last_state_seqno(ton::BlockSeqno last_state_seqno) { + for (auto seqno = from_seqno_; seqno <= last_state_seqno; ++seqno) { + existing_seqnos_.insert(seqno); + } + + LOG(INFO) << "Starting indexing from seqno: " << last_state_seqno + 1; + + td::actor::send_closure(trace_assembler_, &TraceAssembler::set_expected_seqno, last_state_seqno + 1); + alarm_timestamp() = td::Timestamp::now(); } void IndexScheduler::got_last_known_seqno(std::uint32_t last_known_seqno) { if (to_seqno_ > 0 && last_known_seqno_ > to_seqno_) return; - int skipped_count_ = 0; - for(auto seqno = last_known_seqno_ + 1; seqno <= last_known_seqno; ++seqno) { + int skipped_count = 0; + for (auto seqno = last_known_seqno_ + 1; seqno <= last_known_seqno; ++seqno) { if (!force_index_ && (existing_seqnos_.find(seqno) != existing_seqnos_.end())) { - ++skipped_count_; + ++skipped_count; } else if ((from_seqno_ <= 0 || seqno >= from_seqno_) && (to_seqno_ <= 0 || seqno <= to_seqno_)) { queued_seqnos_.push(seqno); } } - if (skipped_count_ > 0) { - LOG(INFO) << "Skipped " << skipped_count_ << " existing seqnos"; + if (skipped_count > 0) { + LOG(INFO) << "Skipped " << skipped_count << " existing seqnos"; } last_known_seqno_ = last_known_seqno; } @@ -173,7 +185,7 @@ void IndexScheduler::seqno_parsed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_ } void IndexScheduler::seqno_traces_assembled(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block) { - LOG(DEBUG) << "Assebled traces for seqno " << mc_seqno; + LOG(DEBUG) << "Assembled traces for seqno " << mc_seqno; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), mc_seqno](td::Result R) { if (R.is_error()) { @@ -187,6 +199,8 @@ void IndexScheduler::seqno_traces_assembled(std::uint32_t mc_seqno, ParsedBlockP } void IndexScheduler::seqno_interfaces_processed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block) { + LOG(DEBUG) << "Interfaces processed for seqno " << mc_seqno; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), mc_seqno](td::Result R) { if (R.is_error()) { LOG(ERROR) << "Failed to detect actions for seqno " << mc_seqno << ": " << R.move_as_error(); diff --git a/ton-index-postgres-v2/src/IndexScheduler.h b/ton-index-postgres-v2/src/IndexScheduler.h index 47f56f08..f948c643 100644 --- a/ton-index-postgres-v2/src/IndexScheduler.h +++ b/ton-index-postgres-v2/src/IndexScheduler.h @@ -26,6 +26,8 @@ class IndexScheduler: public td::actor::Actor { td::actor::ActorOwn trace_assembler_; std::shared_ptr watcher_; + std::string working_dir_; + std::uint32_t max_active_tasks_{32}; std::int32_t last_known_seqno_{0}; std::int32_t last_indexed_seqno_{0}; @@ -43,10 +45,10 @@ class IndexScheduler: public td::actor::Actor { td::Timestamp next_print_stats_; public: IndexScheduler(td::actor::ActorId db_scanner, td::actor::ActorId insert_manager, - td::actor::ActorId parse_manager, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0, bool force_index = false, + td::actor::ActorId parse_manager, std::string working_dir, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0, bool force_index = false, std::uint32_t max_active_tasks = 32, QueueState max_queue = QueueState{30000, 30000, 500000, 500000}, std::int32_t stats_timeout = 10, std::shared_ptr watcher = nullptr) - : db_scanner_(db_scanner), insert_manager_(insert_manager), parse_manager_(parse_manager), + : db_scanner_(db_scanner), insert_manager_(insert_manager), parse_manager_(parse_manager), working_dir_(std::move(working_dir)), from_seqno_(from_seqno), to_seqno_(to_seqno), force_index_(force_index), max_active_tasks_(max_active_tasks), max_queue_(std::move(max_queue)), stats_timeout_(stats_timeout), watcher_(watcher) {}; @@ -67,6 +69,7 @@ class IndexScheduler: public td::actor::Actor { void seqno_inserted(std::uint32_t mc_seqno, td::Unit result); void got_existing_seqnos(td::Result> R); + void got_trace_assembler_last_state_seqno(ton::BlockSeqno last_state_seqno); void got_last_known_seqno(std::uint32_t last_known_seqno); void got_insert_queue_state(QueueState status); diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index ae0b0679..f26e636f 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -1040,14 +1040,14 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { } std::ostringstream query; - query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, " - "account_status, timestamp, last_trans_hash, last_trans_lt, " - "frozen_hash, data_hash, code_hash, " - "data_boc, code_boc) VALUES "; bool is_first = true; for (auto i = latest_account_states.begin(); i != latest_account_states.end(); ++i) { auto& account_state = i->second; if (is_first) { + query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, " + "account_status, timestamp, last_trans_hash, last_trans_lt, " + "frozen_hash, data_hash, code_hash, " + "data_boc, code_boc) VALUES "; is_first = false; } else { query << ", "; @@ -1102,24 +1102,46 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { << data_str << "," << code_str << ")"; } - if (is_first) { - return ""; + if (!is_first) { + query << " ON CONFLICT (account) DO UPDATE SET " + << "account_friendly = EXCLUDED.account_friendly, " + << "hash = EXCLUDED.hash, " + << "balance = EXCLUDED.balance, " + << "account_status = EXCLUDED.account_status, " + << "timestamp = EXCLUDED.timestamp, " + << "last_trans_hash = EXCLUDED.last_trans_hash, " + << "last_trans_lt = EXCLUDED.last_trans_lt, " + << "frozen_hash = EXCLUDED.frozen_hash, " + << "data_hash = EXCLUDED.data_hash, " + << "code_hash = EXCLUDED.code_hash, " + << "data_boc = EXCLUDED.data_boc, " + << "code_boc = EXCLUDED.code_boc " + << "WHERE latest_account_states.last_trans_lt < EXCLUDED.last_trans_lt;\n"; } - query << " ON CONFLICT (account) DO UPDATE SET " - << "account_friendly = EXCLUDED.account_friendly, " - << "hash = EXCLUDED.hash, " - << "balance = EXCLUDED.balance, " - << "account_status = EXCLUDED.account_status, " - << "timestamp = EXCLUDED.timestamp, " - << "last_trans_hash = EXCLUDED.last_trans_hash, " - << "last_trans_lt = EXCLUDED.last_trans_lt, " - << "frozen_hash = EXCLUDED.frozen_hash, " - << "data_hash = EXCLUDED.data_hash, " - << "code_hash = EXCLUDED.code_hash, " - << "data_boc = EXCLUDED.data_boc, " - << "code_boc = EXCLUDED.code_boc " - << "WHERE latest_account_states.last_trans_lt < EXCLUDED.last_trans_lt;\n"; - // LOG(INFO) << "Latest account states query size: " << double(query.str().length()) / 1024 / 1024; + + is_first = true; + for (auto i = latest_account_states.begin(); i != latest_account_states.end(); ++i) { + auto& account_state = i->second; + if (is_first) { + query << "INSERT INTO address_book (address, code_hash) VALUES "; + is_first = false; + } else { + query << ", "; + } + + std::optional code_hash_str; + if (account_state.code_hash) { + code_hash_str = td::base64_encode(account_state.code_hash.value().as_slice()); + } + query << "(" + << txn.quote(convert::to_raw_address(account_state.account)) << "," + << TO_SQL_OPTIONAL_STRING(code_hash_str, txn) << ")"; + } + if (!is_first) { + query << " ON CONFLICT (address) DO UPDATE SET " + << "code_hash = EXCLUDED.code_hash;\n"; + } + return query.str(); } @@ -1240,7 +1262,6 @@ std::string InsertBatchPostgres::insert_jetton_wallets(pqxx::work &txn) { } else { query << ", "; } - LOG(INFO) << "Indexed mintless jetton: " << convert::to_raw_address(addr); query << "(" << txn.quote(convert::to_raw_address(addr)) << ", FALSE)"; } query << " ON CONFLICT (address) DO NOTHING;\n"; @@ -1315,10 +1336,10 @@ std::string InsertBatchPostgres::insert_nft_items(pqxx::work &txn) { } std::ostringstream query; - query << "INSERT INTO nft_items (address, init, index, collection_address, owner_address, content, last_transaction_lt, code_hash, data_hash) VALUES "; bool is_first = true; for (const auto& [addr, nft_item] : nft_items) { if (is_first) { + query << "INSERT INTO nft_items (address, init, index, collection_address, owner_address, content, last_transaction_lt, code_hash, data_hash) VALUES "; is_first = false; } else { query << ", "; @@ -1343,18 +1364,73 @@ std::string InsertBatchPostgres::insert_nft_items(pqxx::work &txn) { << txn.quote(td::base64_encode(nft_item.data_hash.as_slice())) << ")"; } - if (is_first) { - return ""; + if (!is_first) { + query << " ON CONFLICT (address) DO UPDATE SET " + << "init = EXCLUDED.init, " + << "index = EXCLUDED.index, " + << "collection_address = EXCLUDED.collection_address, " + << "owner_address = EXCLUDED.owner_address, " + << "content = EXCLUDED.content, " + << "last_transaction_lt = EXCLUDED.last_transaction_lt, " + << "code_hash = EXCLUDED.code_hash, " + << "data_hash = EXCLUDED.data_hash WHERE nft_items.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; } - query << " ON CONFLICT (address) DO UPDATE SET " - << "init = EXCLUDED.init, " - << "index = EXCLUDED.index, " - << "collection_address = EXCLUDED.collection_address, " - << "owner_address = EXCLUDED.owner_address, " - << "content = EXCLUDED.content, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_hash = EXCLUDED.code_hash, " - << "data_hash = EXCLUDED.data_hash WHERE nft_items.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; + + is_first = true; + for (const auto& [addr, nft_item] : nft_items) { + if (!nft_item.dns_entry) { + continue; + } + if (is_first) { + query << "INSERT INTO dns_entries (nft_item_address, nft_item_owner, domain, dns_next_resolver, dns_wallet, dns_site_adnl, dns_storage_bag_id, last_transaction_lt) VALUES "; + is_first = false; + } else { + query << ", "; + } + + std::optional raw_owner_address; + if (nft_item.owner_address) { + raw_owner_address = convert::to_raw_address(nft_item.owner_address.value()); + } + std::optional raw_dns_next_resolver; + if (nft_item.dns_entry->next_resolver) { + raw_dns_next_resolver = convert::to_raw_address(nft_item.dns_entry->next_resolver.value()); + } + std::optional raw_dns_wallet; + if (nft_item.dns_entry->wallet) { + raw_dns_wallet = convert::to_raw_address(nft_item.dns_entry->wallet.value()); + } + std::optional raw_dns_site; + if (nft_item.dns_entry->site_adnl) { + raw_dns_site = nft_item.dns_entry->site_adnl->to_hex(); + } + std::optional raw_dns_storage_bag_id; + if (nft_item.dns_entry->storage_bag_id) { + raw_dns_storage_bag_id = nft_item.dns_entry->storage_bag_id->to_hex(); + } + query << "(" + << txn.quote(convert::to_raw_address(nft_item.address)) << "," + << TO_SQL_OPTIONAL_STRING(raw_owner_address, txn) << "," + << txn.quote(nft_item.dns_entry->domain) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_next_resolver, txn) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_wallet, txn) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_site, txn) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_storage_bag_id, txn) << "," + << nft_item.last_transaction_lt + << ")"; + } + if (!is_first) { + query << " ON CONFLICT (nft_item_address) DO UPDATE SET " + << "nft_item_owner = EXCLUDED.nft_item_owner, " + << "domain = EXCLUDED.domain, " + << "dns_next_resolver = EXCLUDED.dns_next_resolver, " + << "dns_wallet = EXCLUDED.dns_wallet, " + << "dns_site_adnl = EXCLUDED.dns_site_adnl, " + << "dns_storage_bag_id = EXCLUDED.dns_storage_bag_id, " + << "last_transaction_lt = EXCLUDED.last_transaction_lt " + << "WHERE dns_entries.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; + } + return query.str(); } @@ -2251,6 +2327,25 @@ void InsertManagerPostgres::start_up() { "custom_payload_api_uri varchar[]);\n" ); + query += ( + "create table if not exists dns_entries (" + "nft_item_address tonaddr not null primary key, " + "nft_item_owner tonaddr, " + "domain varchar, " + "dns_next_resolver tonaddr, " + "dns_wallet tonaddr, " + "dns_site_adnl varchar(64), " + "dns_storage_bag_id varchar(64), " + "last_transaction_lt bigint);\n" + ); + + query += ( + "create table if not exists address_book (" + "address tonaddr not null primary key, " + "code_hash tonhash, " + "domain varchar);\n" + ); + LOG(DEBUG) << query; txn.exec0(query); txn.commit(); diff --git a/ton-index-postgres-v2/src/main.cpp b/ton-index-postgres-v2/src/main.cpp index d853da4e..b231da85 100644 --- a/ton-index-postgres-v2/src/main.cpp +++ b/ton-index-postgres-v2/src/main.cpp @@ -3,6 +3,7 @@ #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/check.h" +#include "td/utils/port/path.h" #include "crypto/vm/cp0.h" @@ -38,6 +39,7 @@ int main(int argc, char *argv[]) { bool custom_types = false; bool create_indexes = false; InsertManagerPostgres::Credential credential; + bool testnet = false; std::uint32_t max_active_tasks = 7; std::uint32_t max_insert_actors = 12; @@ -45,7 +47,7 @@ int main(int argc, char *argv[]) { std::int32_t max_queue_size{-1}; std::int32_t max_batch_size{-1}; - QueueState max_queue{100000, 100000, 100000, 100000}; + QueueState max_queue{10000, 100000, 100000, 100000}; QueueState batch_size{2000, 2000, 10000, 10000}; td::OptionParser p; @@ -97,6 +99,10 @@ int main(int argc, char *argv[]) { LOG(WARNING) << "Indexes will be created on launch. It may take several hours!"; }); + p.add_option('\0', "testnet", "Use for testnet. It is used for correct indexing of .ton DNS entries (in testnet .ton collection has a different address)", [&]() { + testnet = true; + }); + p.add_checked_option('f', "from", "Masterchain seqno to start indexing from", [&](td::Slice value) { int v; try { @@ -278,9 +284,10 @@ int main(int argc, char *argv[]) { std::_Exit(2); } if (working_dir.size() == 0) { - working_dir = PSTRING() << "/tmp/index_worker_" << getpid(); - LOG(WARNING) << "Working dir not specified, using " << working_dir; + LOG(ERROR) << "Please specify working directory with -W or --working-dir"; + std::_Exit(2); } + td::mkdir(working_dir).ensure(); if (max_queue_size > 0) { max_queue.mc_blocks_ = max_queue_size; @@ -298,6 +305,8 @@ int main(int argc, char *argv[]) { batch_size.traces_ = max_batch_size; } + NftItemDetectorR::is_testnet = testnet; + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); @@ -305,10 +314,10 @@ int main(int argc, char *argv[]) { td::actor::Scheduler scheduler({td::actor::Scheduler::NodeInfo{threads, io_workers}}); scheduler.run_in_context([&] { insert_manager_ = td::actor::create_actor("insertmanager", credential, custom_types, create_indexes); }); scheduler.run_in_context([&] { parse_manager_ = td::actor::create_actor("parsemanager"); }); - scheduler.run_in_context([&] { db_scanner_ = td::actor::create_actor("scanner", db_root, dbs_secondary, working_dir); }); + scheduler.run_in_context([&] { db_scanner_ = td::actor::create_actor("scanner", db_root, dbs_secondary, working_dir + "/secondary_logs"); }); scheduler.run_in_context([&, watcher = std::move(watcher)] { index_scheduler_ = td::actor::create_actor("indexscheduler", db_scanner_.get(), - insert_manager_.get(), parse_manager_.get(), from_seqno, to_seqno, force_index, max_active_tasks, max_queue, stats_timeout, watcher); + insert_manager_.get(), parse_manager_.get(), working_dir, from_seqno, to_seqno, force_index, max_active_tasks, max_queue, stats_timeout, watcher); }); scheduler.run_in_context([&] { td::actor::send_closure(insert_manager_, &InsertManagerPostgres::set_parallel_inserts_actors, max_insert_actors); diff --git a/ton-smc-scanner/src/PostgreSQLInserter.cpp b/ton-smc-scanner/src/PostgreSQLInserter.cpp index f013a639..fe90fe60 100644 --- a/ton-smc-scanner/src/PostgreSQLInserter.cpp +++ b/ton-smc-scanner/src/PostgreSQLInserter.cpp @@ -36,6 +36,8 @@ void PostgreSQLInserter::start_up() { insert_latest_account_states(txn); insert_jetton_masters(txn); insert_jetton_wallets(txn); + insert_nft_items(txn); + insert_nft_collections(txn); txn.commit(); } @@ -132,15 +134,15 @@ void PostgreSQLInserter::insert_latest_account_states(pqxx::work &transaction) { } void PostgreSQLInserter::insert_jetton_masters(pqxx::work &transaction) { - std::vector jetton_masters; + std::vector jetton_masters; for (auto &data : data_) { - if (std::holds_alternative(data)) { - jetton_masters.push_back(std::get(data)); + if (std::holds_alternative(data)) { + jetton_masters.push_back(std::get(data)); } } std::ostringstream query; - query << "INSERT INTO jetton_masters (address, total_supply, mintable, admin_address, jetton_content, jetton_wallet_code_hash, data_hash, code_hash, last_transaction_lt, code_boc, data_boc) VALUES "; + query << "INSERT INTO jetton_masters (address, total_supply, mintable, admin_address, jetton_content, jetton_wallet_code_hash, data_hash, code_hash, last_transaction_lt) VALUES "; bool is_first = true; for (const auto &jetton_master : jetton_masters) { if (is_first) { @@ -148,18 +150,20 @@ void PostgreSQLInserter::insert_jetton_masters(pqxx::work &transaction) { } else { query << ", "; } + std::optional raw_admin_address; + if (jetton_master.admin_address) { + raw_admin_address = convert::to_raw_address(jetton_master.admin_address.value()); + } query << "(" - << TO_SQL_STRING(jetton_master.address, transaction) << "," + << TO_SQL_STRING(convert::to_raw_address(jetton_master.address), transaction) << "," << jetton_master.total_supply << "," << TO_SQL_BOOL(jetton_master.mintable) << "," - << TO_SQL_OPTIONAL_STRING(jetton_master.admin_address, transaction) << "," + << TO_SQL_OPTIONAL_STRING(raw_admin_address, transaction) << "," << (jetton_master.jetton_content ? TO_SQL_STRING(content_to_json_string(jetton_master.jetton_content.value()), transaction) : "NULL") << "," << TO_SQL_STRING(td::base64_encode(jetton_master.jetton_wallet_code_hash.as_slice()), transaction) << "," << TO_SQL_STRING(td::base64_encode(jetton_master.data_hash.as_slice()), transaction) << "," << TO_SQL_STRING(td::base64_encode(jetton_master.code_hash.as_slice()), transaction) << "," - << jetton_master.last_transaction_lt << "," - << TO_SQL_STRING(jetton_master.code_boc, transaction) << "," - << TO_SQL_STRING(jetton_master.data_boc, transaction) + << jetton_master.last_transaction_lt << ")"; } if (is_first) { @@ -173,19 +177,17 @@ void PostgreSQLInserter::insert_jetton_masters(pqxx::work &transaction) { << "jetton_wallet_code_hash = EXCLUDED.jetton_wallet_code_hash, " << "data_hash = EXCLUDED.data_hash, " << "code_hash = EXCLUDED.code_hash, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_boc = EXCLUDED.code_boc, " - << "data_boc = EXCLUDED.data_boc WHERE jetton_masters.last_transaction_lt <= EXCLUDED.last_transaction_lt"; + << "last_transaction_lt = EXCLUDED.last_transaction_lt WHERE jetton_masters.last_transaction_lt <= EXCLUDED.last_transaction_lt"; // LOG(DEBUG) << "Running SQL query: " << query.str(); transaction.exec0(query.str()); } void PostgreSQLInserter::insert_jetton_wallets(pqxx::work &transaction) { - std::vector jetton_wallets; + std::vector jetton_wallets; for (auto& data : data_) { - if (std::holds_alternative(data)) { - jetton_wallets.push_back(std::get(data)); + if (std::holds_alternative(data)) { + jetton_wallets.push_back(std::get(data)); } } @@ -198,11 +200,12 @@ void PostgreSQLInserter::insert_jetton_wallets(pqxx::work &transaction) { } else { query << ", "; } + query << "(" << jetton_wallet.balance << "," - << TO_SQL_STRING(jetton_wallet.address, transaction) << "," - << TO_SQL_STRING(jetton_wallet.owner, transaction) << "," - << TO_SQL_STRING(jetton_wallet.jetton, transaction) << "," + << TO_SQL_STRING(convert::to_raw_address(jetton_wallet.address), transaction) << "," + << TO_SQL_STRING(convert::to_raw_address(jetton_wallet.owner), transaction) << "," + << TO_SQL_STRING(convert::to_raw_address(jetton_wallet.jetton), transaction) << "," << jetton_wallet.last_transaction_lt << "," << TO_SQL_STRING(TO_B64_HASH(jetton_wallet.code_hash), transaction) << "," << TO_SQL_STRING(TO_B64_HASH(jetton_wallet.data_hash), transaction) @@ -223,6 +226,157 @@ void PostgreSQLInserter::insert_jetton_wallets(pqxx::work &transaction) { transaction.exec0(query.str()); } +void PostgreSQLInserter::insert_nft_collections(pqxx::work &txn) { + std::vector nft_collections; + for (auto& data : data_) { + if (std::holds_alternative(data)) { + nft_collections.push_back(std::get(data)); + } + } + + std::ostringstream query; + query << "INSERT INTO nft_collections (address, next_item_index, owner_address, collection_content, last_transaction_lt, code_hash, data_hash) VALUES "; + bool is_first = true; + for (const auto& nft_collection : nft_collections) { + if (is_first) { + is_first = false; + } else { + query << ", "; + } + std::optional raw_owner_address; + if (nft_collection.owner_address) { + raw_owner_address = convert::to_raw_address(nft_collection.owner_address.value()); + } + query << "(" + << txn.quote(convert::to_raw_address(nft_collection.address)) << "," + << nft_collection.next_item_index << "," + << TO_SQL_OPTIONAL_STRING(raw_owner_address, txn) << "," + << (nft_collection.collection_content ? txn.quote(content_to_json_string(nft_collection.collection_content.value())) : "NULL") << "," + << nft_collection.last_transaction_lt << "," + << txn.quote(td::base64_encode(nft_collection.code_hash.as_slice())) << "," + << txn.quote(td::base64_encode(nft_collection.data_hash.as_slice())) + << ")"; + } + if (is_first) { + return; + } + query << " ON CONFLICT (address) DO UPDATE SET " + << "next_item_index = EXCLUDED.next_item_index, " + << "owner_address = EXCLUDED.owner_address, " + << "collection_content = EXCLUDED.collection_content, " + << "last_transaction_lt = EXCLUDED.last_transaction_lt, " + << "code_hash = EXCLUDED.code_hash, " + << "data_hash = EXCLUDED.data_hash WHERE nft_collections.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; + + txn.exec0(query.str()); +} + +void PostgreSQLInserter::insert_nft_items(pqxx::work &txn) { + std::vector nft_items; + for (auto& data : data_) { + if (std::holds_alternative(data)) { + nft_items.push_back(std::get(data)); + } + } + std::ostringstream query; + bool is_first = true; + for (const auto& nft_item : nft_items) { + if (is_first) { + query << "INSERT INTO nft_items (address, init, index, collection_address, owner_address, content, last_transaction_lt, code_hash, data_hash) VALUES "; + is_first = false; + } else { + query << ", "; + } + std::optional raw_collection_address; + if (nft_item.collection_address) { + raw_collection_address = convert::to_raw_address(nft_item.collection_address.value()); + } + std::optional raw_owner_address; + if (nft_item.owner_address) { + raw_owner_address = convert::to_raw_address(nft_item.owner_address.value()); + } + query << "(" + << txn.quote(convert::to_raw_address(nft_item.address)) << "," + << TO_SQL_BOOL(nft_item.init) << "," + << nft_item.index << "," + << TO_SQL_OPTIONAL_STRING(raw_collection_address, txn) << "," + << TO_SQL_OPTIONAL_STRING(raw_owner_address, txn) << "," + << (nft_item.content ? txn.quote(content_to_json_string(nft_item.content.value())) : "NULL") << "," + << nft_item.last_transaction_lt << "," + << txn.quote(td::base64_encode(nft_item.code_hash.as_slice())) << "," + << txn.quote(td::base64_encode(nft_item.data_hash.as_slice())) + << ")"; + } + if (!is_first) { + query << " ON CONFLICT (address) DO UPDATE SET " + << "init = EXCLUDED.init, " + << "index = EXCLUDED.index, " + << "collection_address = EXCLUDED.collection_address, " + << "owner_address = EXCLUDED.owner_address, " + << "content = EXCLUDED.content, " + << "last_transaction_lt = EXCLUDED.last_transaction_lt, " + << "code_hash = EXCLUDED.code_hash, " + << "data_hash = EXCLUDED.data_hash WHERE nft_items.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; + } + + is_first = true; + for (const auto& nft_item : nft_items) { + if (!nft_item.dns_entry) { + continue; + } + if (is_first) { + query << "INSERT INTO dns_entries (nft_item_address, nft_item_owner, domain, dns_next_resolver, dns_wallet, dns_site_adnl, dns_storage_bag_id, last_transaction_lt) VALUES "; + is_first = false; + } else { + query << ", "; + } + + std::optional raw_owner_address; + if (nft_item.owner_address) { + raw_owner_address = convert::to_raw_address(nft_item.owner_address.value()); + } + std::optional raw_dns_next_resolver; + if (nft_item.dns_entry->next_resolver) { + raw_dns_next_resolver = convert::to_raw_address(nft_item.dns_entry->next_resolver.value()); + } + std::optional raw_dns_wallet; + if (nft_item.dns_entry->wallet) { + raw_dns_wallet = convert::to_raw_address(nft_item.dns_entry->wallet.value()); + } + std::optional raw_dns_site; + if (nft_item.dns_entry->site_adnl) { + raw_dns_site = nft_item.dns_entry->site_adnl->to_hex(); + } + std::optional raw_dns_storage_bag_id; + if (nft_item.dns_entry->storage_bag_id) { + raw_dns_storage_bag_id = nft_item.dns_entry->storage_bag_id->to_hex(); + } + query << "(" + << txn.quote(convert::to_raw_address(nft_item.address)) << "," + << TO_SQL_OPTIONAL_STRING(raw_owner_address, txn) << "," + << txn.quote(nft_item.dns_entry->domain) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_next_resolver, txn) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_wallet, txn) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_site, txn) << "," + << TO_SQL_OPTIONAL_STRING(raw_dns_storage_bag_id, txn) << "," + << nft_item.last_transaction_lt + << ")"; + } + if (!is_first) { + query << " ON CONFLICT (nft_item_address) DO UPDATE SET " + << "nft_item_owner = EXCLUDED.nft_item_owner, " + << "domain = EXCLUDED.domain, " + << "dns_next_resolver = EXCLUDED.dns_next_resolver, " + << "dns_wallet = EXCLUDED.dns_wallet, " + << "dns_site_adnl = EXCLUDED.dns_site_adnl, " + << "dns_storage_bag_id = EXCLUDED.dns_storage_bag_id, " + << "last_transaction_lt = EXCLUDED.last_transaction_lt " + << "WHERE dns_entries.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; + } + + txn.exec0(query.str()); +} + void PostgreSQLInsertManager::start_up() { queue_.reserve(batch_size_ * 2); alarm_timestamp() = td::Timestamp::in(10.0); @@ -230,7 +384,7 @@ void PostgreSQLInsertManager::start_up() { void PostgreSQLInsertManager::alarm() { alarm_timestamp() = td::Timestamp::in(10.0); - LOG(INFO) << "Inserted: " << inserted_count_ << " In progress: " << in_progress_; + LOG(INFO) << "Inserted: " << inserted_count_ << ". Batches in progress: " << in_progress_; check_queue(true); } @@ -246,17 +400,17 @@ void PostgreSQLInsertManager::check_queue(bool force) { } else { // LOG(INFO) << "Inserted " << len << " states"; } - td::actor::send_closure(SelfId, &PostgreSQLInsertManager::insert_done); + td::actor::send_closure(SelfId, &PostgreSQLInsertManager::insert_done, len); }); ++in_progress_; - inserted_count_ += to_insert.size(); td::actor::create_actor("PostgresInserter", connection_string_, to_insert, std::move(P)).release(); } } -void PostgreSQLInsertManager::insert_done() { +void PostgreSQLInsertManager::insert_done(size_t cnt) { --in_progress_; + inserted_count_ += cnt; } void PostgreSQLInsertManager::checkpoint(ton::ShardIdFull shard, td::Bits256 cur_addr_) { diff --git a/ton-smc-scanner/src/PostgreSQLInserter.h b/ton-smc-scanner/src/PostgreSQLInserter.h index fde00248..1e9f675a 100644 --- a/ton-smc-scanner/src/PostgreSQLInserter.h +++ b/ton-smc-scanner/src/PostgreSQLInserter.h @@ -9,7 +9,7 @@ -using InsertData = std::variant; +using InsertData = std::variant; class PostgreSQLInserter : public td::actor::Actor { public: @@ -21,6 +21,8 @@ class PostgreSQLInserter : public td::actor::Actor { void insert_latest_account_states(pqxx::work &transaction); void insert_jetton_masters(pqxx::work &transaction); void insert_jetton_wallets(pqxx::work &transaction); + void insert_nft_items(pqxx::work &transaction); + void insert_nft_collections(pqxx::work &transaction); std::string connection_string_; std::vector data_; @@ -34,7 +36,7 @@ class PostgreSQLInsertManager : public td::actor::Actor { void start_up() override; void alarm() override; void insert_data(std::vector data); - void insert_done(); + void insert_done(size_t cnt); void checkpoint(ton::ShardIdFull shard, td::Bits256 cur_addr_); void checkpoint_read(ton::ShardIdFull shard, td::Promise promise); void checkpoint_reset(ton::ShardIdFull shard); diff --git a/ton-smc-scanner/src/SmcScanner.cpp b/ton-smc-scanner/src/SmcScanner.cpp index be7e8cd2..a1c871fc 100644 --- a/ton-smc-scanner/src/SmcScanner.cpp +++ b/ton-smc-scanner/src/SmcScanner.cpp @@ -64,7 +64,7 @@ void ShardStateScanner::schedule_next() { if(!finished) { alarm_timestamp() = td::Timestamp::in(0.1); } else { - LOG(ERROR) << "Finished!"; + LOG(INFO) << "Shard " << shard_.to_str() << " is finished!"; stop(); } } @@ -96,7 +96,7 @@ void ShardStateScanner::start_up() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), shard = shard_](td::Result R) { td::Bits256 cur_addr{td::Bits256::zero()}; if (R.is_error()) { - LOG(ERROR) << "Failed to restore state for shard (" << shard.workchain << "," << shard.shard << ")"; + LOG(ERROR) << "Failed to restore state for shard " << shard.to_str() << ": " << R.move_as_error(); } else { cur_addr = R.move_as_ok(); } @@ -110,7 +110,7 @@ void ShardStateScanner::start_up() { } void ShardStateScanner::alarm() { - LOG(INFO) << "workchain: " << shard_.workchain << " shard: " << static_cast(shard_.shard) << " cur_addr: " << cur_addr_.to_hex(); + LOG(INFO) << "Shard " << shard_.to_str() << " cur addr: " << cur_addr_.to_hex(); schedule_next(); } @@ -123,20 +123,91 @@ void ShardStateScanner::got_checkpoint(td::Bits256 cur_addr) { alarm_timestamp() = td::Timestamp::in(0.1); } -void StateBatchParser::interfaces_detected(std::vector ifaces) { - LOG(ERROR) << "Detected, but not queued! Interfaces will not be inserted!!!"; - // TODO: here should be added logic of addition to result_ +void StateBatchParser::interfaces_detected(block::StdAddress address, std::vector interfaces, + td::Bits256 code_hash, td::Bits256 data_hash, uint64_t last_trans_lt, uint32_t last_trans_now, td::Promise promise) { + for (auto& interface : interfaces) { + std::visit([&](auto&& arg) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + JettonMasterDataV2 jetton_master_data; + jetton_master_data.address = address; + jetton_master_data.total_supply = arg.total_supply; + jetton_master_data.mintable = arg.mintable; + jetton_master_data.admin_address = arg.admin_address; + jetton_master_data.jetton_content = arg.jetton_content; + jetton_master_data.jetton_wallet_code_hash = arg.jetton_wallet_code_hash.bits(); + jetton_master_data.data_hash = data_hash; + jetton_master_data.code_hash = code_hash; + jetton_master_data.last_transaction_lt = last_trans_lt; + jetton_master_data.last_transaction_now = last_trans_now; + interfaces_[address].push_back(jetton_master_data); + } else if constexpr (std::is_same_v) { + JettonWalletDataV2 jetton_wallet_data; + jetton_wallet_data.balance = arg.balance; + jetton_wallet_data.address = address; + jetton_wallet_data.owner = arg.owner; + jetton_wallet_data.jetton = arg.jetton; + jetton_wallet_data.mintless_is_claimed = arg.mintless_is_claimed; + jetton_wallet_data.last_transaction_lt = last_trans_lt; + jetton_wallet_data.last_transaction_now = last_trans_now; + jetton_wallet_data.code_hash = code_hash; + jetton_wallet_data.data_hash = data_hash; + interfaces_[address].push_back(jetton_wallet_data); + } else if constexpr (std::is_same_v) { + NFTCollectionDataV2 nft_collection_data; + nft_collection_data.address = address; + nft_collection_data.next_item_index = arg.next_item_index; + nft_collection_data.owner_address = arg.owner_address; + nft_collection_data.collection_content = arg.collection_content; + nft_collection_data.last_transaction_lt = last_trans_lt; + nft_collection_data.last_transaction_now = last_trans_now; + nft_collection_data.code_hash = code_hash; + nft_collection_data.data_hash = data_hash; + interfaces_[address].push_back(nft_collection_data); + } else if constexpr (std::is_same_v) { + NFTItemDataV2 nft_item_data; + nft_item_data.address = address; + nft_item_data.init = arg.init; + nft_item_data.index = arg.index; + nft_item_data.collection_address = arg.collection_address; + nft_item_data.owner_address = arg.owner_address; + nft_item_data.content = arg.content; + nft_item_data.last_transaction_lt = last_trans_lt; + nft_item_data.last_transaction_now = last_trans_now; + nft_item_data.code_hash = code_hash; + nft_item_data.data_hash = data_hash; + if (arg.dns_entry) { + nft_item_data.dns_entry = NFTItemDataV2::DNSEntry{arg.dns_entry->domain, arg.dns_entry->wallet, arg.dns_entry->next_resolver, arg.dns_entry->site_adnl}; + } + interfaces_[address].push_back(nft_item_data); + } + }, interface); + } + promise.set_value(td::Unit()); } void StateBatchParser::process_account_states(std::vector account_states) { // LOG(INFO) << "Processing account state " << account.account; + + auto P = td::PromiseCreator::lambda([&, SelfId=actor_id(this)](td::Result res) mutable { + td::actor::send_closure(SelfId, &StateBatchParser::processing_finished); + }); + + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(P)); + for (auto &account : account_states) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), addr = account.account](td::Result> R) { + if (account.code.is_null() || account.data.is_null()) { + continue; + } + interfaces_[account.account] = {}; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), account, promise = ig.get_promise()](td::Result> R) mutable { if (R.is_error()) { - LOG(ERROR) << "Failed to detect interfaces of account '" << addr << "'"; + LOG(ERROR) << "Failed to detect interfaces of account '" << account.account << "'"; return; } - td::actor::send_closure(SelfId, &StateBatchParser::interfaces_detected, R.move_as_ok()); + td::actor::send_closure(SelfId, &StateBatchParser::interfaces_detected, account.account, R.move_as_ok(), account.code_hash.value(), account.data_hash.value(), account.last_trans_lt, account.timestamp, std::move(promise)); }); td::actor::create_actor("InterfacesDetector", account.account, account.code, account.data, shard_state_data_->shard_states_, shard_state_data_->config_, std::move(P)).release(); } @@ -146,7 +217,7 @@ void StateBatchParser::start_up() { // if (cur_addr_.to_hex() != "E753CF93EAEDD2EC01B5DE8F49A334622BD630A8728806ABA65F1443EB7C8FD7") { // continue; // } - std::vector state_list_; + std::vector state_list; for (auto &[addr_, acc_info] : data_) { int account_tag = block::gen::t_Account.get_tag(vm::load_cell_slice(acc_info.account)); switch (account_tag) { @@ -160,23 +231,27 @@ void StateBatchParser::start_up() { LOG(ERROR) << "Failed to parse account " << addr_.to_hex() << ": " << account_r.move_as_error(); break; } - auto account_state_ = account_r.move_as_ok(); - result_.push_back(account_state_); - state_list_.push_back(account_state_); + auto account_state = account_r.move_as_ok(); + state_list.push_back(account_state); break; } default: LOG(ERROR) << "Unknown account tag"; break; } } + + std::copy(state_list.begin(), state_list.end(), std::back_inserter(result_)); + if (options_.index_interfaces_) { - process_account_states(state_list_); + process_account_states(state_list); } else { - std::copy(state_list_.begin(), state_list_.end(), std::back_inserter(result_)); processing_finished(); } } void StateBatchParser::processing_finished() { + for (auto& [addr, ifaces] : interfaces_ ) { + std::copy(ifaces.begin(), ifaces.end(), std::back_inserter(result_)); + } td::actor::send_closure(options_.insert_manager_, &PostgreSQLInsertManager::insert_data, std::move(result_)); td::actor::send_closure(shard_state_scanner_, &ShardStateScanner::batch_inserted); stop(); diff --git a/ton-smc-scanner/src/SmcScanner.h b/ton-smc-scanner/src/SmcScanner.h index de32cd56..bd319b2e 100644 --- a/ton-smc-scanner/src/SmcScanner.h +++ b/ton-smc-scanner/src/SmcScanner.h @@ -33,6 +33,7 @@ class StateBatchParser: public td::actor::Actor { td::actor::ActorId shard_state_scanner_; Options options_; + std::unordered_map, AddressHasher> interfaces_; std::vector result_; public: StateBatchParser(std::vector> data, ShardStateDataPtr shard_state_data, td::actor::ActorId shard_state_scanner, Options options) @@ -40,7 +41,8 @@ class StateBatchParser: public td::actor::Actor { void start_up() override; void processing_finished(); private: - void interfaces_detected(std::vector ifaces); + void interfaces_detected(block::StdAddress address, std::vector interfaces, + td::Bits256 code_hash, td::Bits256 data_hash, uint64_t last_trans_lt, uint32_t last_trans_now, td::Promise promise); void process_account_states(std::vector account_states); }; diff --git a/ton-smc-scanner/src/main.cpp b/ton-smc-scanner/src/main.cpp index 16381b6c..eeebd6ff 100644 --- a/ton-smc-scanner/src/main.cpp +++ b/ton-smc-scanner/src/main.cpp @@ -20,6 +20,7 @@ int main(int argc, char *argv[]) { std::string db_root = "/var/lib/ton-work/db"; std::string pg_dsn = "postgresql://localhost:5432/ton_index"; Options options_; + bool is_testnet = false; td::OptionParser p; p.set_description("Scan all accounts at some seqno, detect interfaces and save them to postgres"); @@ -72,6 +73,9 @@ int main(int argc, char *argv[]) { p.add_option('f', "force", "Reset checkpoints", [&]() { options_.from_checkpoint = false; }); + p.add_option('\0', "testnet", "Use for testnet. It is used for correct indexing of .ton DNS entries (in testnet .ton collection has a different address)", [&]() { + is_testnet = true; + }); auto S = p.run(argc, argv); if (S.is_error()) { @@ -79,6 +83,8 @@ int main(int argc, char *argv[]) { std::_Exit(2); } + NftItemDetectorR::is_testnet = is_testnet; + td::actor::Scheduler scheduler({threads}); td::actor::ActorOwn db_scanner; td::actor::ActorOwn insert_manager; diff --git a/ton-trace-emulator/src/Serializer.hpp b/ton-trace-emulator/src/Serializer.hpp index 2f117374..cc302534 100644 --- a/ton-trace-emulator/src/Serializer.hpp +++ b/ton-trace-emulator/src/Serializer.hpp @@ -1,73 +1,9 @@ #pragma once -#include #include "crypto/block/block-auto.h" #include "crypto/block/block-parse.h" +#include "msgpack-utils.h" #include "TraceEmulator.h" -namespace msgpack { - MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) { - namespace adaptor { - - template<> - struct pack { - template - msgpack::packer& operator()(msgpack::packer& o, const block::StdAddress& v) const { - std::string addr = std::to_string(v.workchain) + ":" + v.addr.to_hex(); - o.pack(addr); - return o; - } - }; - - template <> - struct convert { - msgpack::object const& operator()(msgpack::object const& o, block::StdAddress& v) const { - if (o.type != msgpack::type::STR) throw msgpack::type_error(); - std::string addr = o.as(); - if (!v.parse_addr(addr)) throw std::runtime_error("Failed to deserialize block::StdAddress"); - return o; - } - }; - - template<> - struct pack { - template - msgpack::packer& operator()(msgpack::packer& o, const td::Bits256& v) const { - o.pack(v.to_hex()); - return o; - } - }; - - template <> - struct convert { - msgpack::object const& operator()(msgpack::object const& o, td::Bits256& v) const { - if (o.type != msgpack::type::STR) throw msgpack::type_error(); - std::string hex = o.as(); - if (v.from_hex(hex) < 0) throw std::runtime_error("Failed to deserialize td::Bits256"); - return o; - } - }; - - template<> - struct pack { - template - msgpack::packer& operator()(msgpack::packer& o, const td::RefInt256& v) const { - o.pack(v->to_dec_string()); - return o; - } - }; - - template <> - struct convert { - msgpack::object const& operator()(msgpack::object const& o, td::RefInt256& v) const { - throw std::runtime_error("Deserializion of td::RefInt256 is not implemented"); - return o; - } - }; - - } // namespace adaptor - } // MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) -} // namespace msgpack - enum AccountStatus { uninit = block::gen::AccountStatus::acc_state_uninit, diff --git a/tondb-scanner/CMakeLists.txt b/tondb-scanner/CMakeLists.txt index 969b70da..db81bc45 100644 --- a/tondb-scanner/CMakeLists.txt +++ b/tondb-scanner/CMakeLists.txt @@ -14,22 +14,25 @@ add_library(tondb-scanner STATIC src/tokens.cpp src/smc-interfaces/Tokens.cpp src/smc-interfaces/NftSale.cpp + src/smc-interfaces/execute-smc.cpp ) target_include_directories(tondb-scanner PUBLIC external/ton PUBLIC external/libpqxx + PUBLIC external/msgpack-c PUBLIC src/ ) target_link_directories(tondb-scanner PUBLIC external/ton PUBLIC external/libpqxx + PUBLIC external/msgpack-c ) target_compile_features(tondb-scanner PRIVATE cxx_std_17) target_link_libraries(tondb-scanner overlay tdutils tdactor adnl tl_api dht catchain validatorsession validator-disk ton_validator validator-disk smc-envelope - pqxx) + pqxx msgpack-cxx) set(TLB_TOKENS ${CMAKE_CURRENT_SOURCE_DIR}/src/tokens.cpp diff --git a/tondb-scanner/src/IndexData.h b/tondb-scanner/src/IndexData.h index 7eae3c81..39f18fbe 100644 --- a/tondb-scanner/src/IndexData.h +++ b/tondb-scanner/src/IndexData.h @@ -11,8 +11,6 @@ namespace schema { -struct Message; - enum AccountStatus { uninit = block::gen::AccountStatus::acc_state_uninit, frozen = block::gen::AccountStatus::acc_state_frozen, @@ -474,6 +472,14 @@ struct NFTItemData { }; struct NFTItemDataV2 { + struct DNSEntry { + std::string domain; + std::optional wallet; + std::optional next_resolver; + std::optional site_adnl; + std::optional storage_bag_id; + }; + block::StdAddress address; bool init; td::RefInt256 index; @@ -484,6 +490,7 @@ struct NFTItemDataV2 { uint32_t last_transaction_now; td::Bits256 code_hash; td::Bits256 data_hash; + std::optional dns_entry; }; struct NFTTransfer { diff --git a/tondb-scanner/src/TraceAssembler.cpp b/tondb-scanner/src/TraceAssembler.cpp index b83a5f45..210c1d6a 100644 --- a/tondb-scanner/src/TraceAssembler.cpp +++ b/tondb-scanner/src/TraceAssembler.cpp @@ -1,40 +1,13 @@ #include -#include - +#include #include "TraceAssembler.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/path.h" +#include "common/delay.h" #include "convert-utils.h" -// -// Utils -// -#define B64HASH(x) (td::base64_encode((x).as_slice())) - -template -std::optional to_std_optional(td::optional item) { - return (item ? std::optional(item.value()) : std::optional()); -} - -// -// Utils -// -std::string TraceEdgeImpl::str() const { - td::StringBuilder sb; - sb << "TraceEdge(" - << trace_id << ", " - << msg_hash << ", " - << (left_tx.has_value() ? B64HASH(left_tx.value()) : "null") << ", " - << (right_tx.has_value() ? B64HASH(right_tx.value()) : "null") << ")"; - return sb.as_cslice().str(); -} - -std::string TraceImpl::str() const { - td::StringBuilder sb; - sb << "Trace(" << trace_id - << ", nodes=" << nodes_ - << ", edges=" << edges_ - << ", pending_edges=" << pending_edges_ << ")"; - return sb.as_cslice().str(); -} +namespace fs = std::filesystem; schema::TraceEdge TraceEdgeImpl::to_schema() const { schema::TraceEdge result; @@ -49,19 +22,6 @@ schema::TraceEdge TraceEdgeImpl::to_schema() const { return result; } -TraceEdgeImpl TraceEdgeImpl::from_schema(const schema::TraceEdge& edge) { - TraceEdgeImpl edge_impl; - edge_impl.trace_id = edge.trace_id; - edge_impl.msg_hash = edge.msg_hash; - edge_impl.msg_lt = edge.msg_lt; - edge_impl.left_tx = edge.left_tx; - edge_impl.right_tx = edge.right_tx; - edge_impl.type = edge.type; - edge_impl.incomplete = edge.incomplete; - edge_impl.broken = edge.broken; - return std::move(edge_impl); -} - schema::Trace TraceImpl::to_schema() const { schema::Trace result; result.trace_id = trace_id; @@ -73,74 +33,167 @@ schema::Trace TraceImpl::to_schema() const { result.end_lt = end_lt; result.end_utime = end_utime; result.state = state; - result.pending_edges_ = pending_edges_; - result.edges_ = edges_; - result.nodes_ = nodes_; + result.pending_edges_ = pending_edges; + result.edges_ = edges; + result.nodes_ = nodes; return result; } -TraceImplPtr TraceImpl::from_schema(const schema::Trace& trace) { - TraceImplPtr trace_impl = std::make_shared(); - trace_impl->trace_id = trace.trace_id; - trace_impl->external_hash = trace.external_hash; - trace_impl->mc_seqno_start = trace.mc_seqno_start; - trace_impl->mc_seqno_end = trace.mc_seqno_end; - trace_impl->start_lt = trace.start_lt; - trace_impl->start_utime = trace.start_utime; - trace_impl->end_lt = trace.end_lt; - trace_impl->end_utime = trace.end_utime; - trace_impl->state = trace.state; - trace_impl->pending_edges_ = trace.pending_edges_; - trace_impl->edges_ = trace.edges_; - trace_impl->nodes_ = trace.nodes_; - return std::move(trace_impl); +TraceAssembler::TraceAssembler(std::string db_path, size_t gc_distance) : + db_path_(db_path), gc_distance_(gc_distance) { + td::mkdir(db_path).ensure(); } -// -// TraceAssembler -// void TraceAssembler::start_up() { alarm_timestamp() = td::Timestamp::in(10.0); } +td::Status gc_states(std::string db_path, ton::BlockSeqno current_seqno, size_t keep_last) { + std::map> fileMap; + + for (const auto& entry : fs::directory_iterator(db_path)) { + if (fs::is_regular_file(entry.status())) { + auto filename = entry.path().stem().string(); + auto extension = entry.path().extension().string(); + + if (extension == ".tastate") { + try { + int seqno = std::stoi(filename); + fileMap[seqno] = entry.path(); + } catch (const std::exception& e) { + LOG(ERROR) << "Error reading seqno of trace assembler state " << entry.path().string(); + } + } + } + } + + int count = 0; + for (const auto& [seqno, filepath] : fileMap) { + if (seqno > current_seqno) { + LOG(WARNING) << "Deleting state " << filepath.string() << " that is higher than currently processing seqno " << current_seqno; + fs::remove(filepath); + } else if (count >= keep_last) { + fs::remove(filepath); + LOG(DEBUG) << "Deleting old state: " << filepath.string(); + } else { + LOG(DEBUG) << "Keeping state: " << filepath.string(); + count++; + } + } + return td::Status::OK(); +} + +td::Status save_state(std::string db_path, ton::BlockSeqno seqno, + std::unordered_map pending_traces, + std::unordered_map pending_edges) { + std::stringstream buffer; + msgpack::pack(buffer, pending_traces); + msgpack::pack(buffer, pending_edges); + + auto path = db_path + "/" + std::to_string(seqno) + ".tastate"; + return td::atomic_write_file(path, buffer.str()); +} + void TraceAssembler::alarm() { alarm_timestamp() = td::Timestamp::in(10.0); + if (expected_seqno_ == 0) { + return; + } + + ton::delay_action([this, seqno = expected_seqno_ - 1, pending_traces = pending_traces_, pending_edges = pending_edges_]() { + auto S = save_state(this->db_path_, expected_seqno_ - 1, pending_traces_, pending_edges_); + if (S.is_error()) { + LOG(ERROR) << "Error while saving Trace Assembler state: " << S.move_as_error(); + } + }, td::Timestamp::now()); + + ton::delay_action([this]() { + auto S = gc_states(this->db_path_, this->expected_seqno_, 10); + if (S.is_error()) { + LOG(ERROR) << "Error while garbage collecting Trace Assembler states: " << S.move_as_error(); + } + }, td::Timestamp::now()); + LOG(INFO) << " Pending traces: " << pending_traces_.size() << " Pending edges: " << pending_edges_.size() << " Broken traces: " << broken_count_; } -void TraceAssembler::assemble(std::int32_t seqno, ParsedBlockPtr block, td::Promise promise) { +void TraceAssembler::assemble(ton::BlockSeqno seqno, ParsedBlockPtr block, td::Promise promise) { queue_.emplace(seqno, Task{seqno, std::move(block), std::move(promise)}); - if (is_ready_) { - process_queue(); - } + process_queue(); } -void TraceAssembler::update_expected_seqno(std::int32_t new_expected_seqno) { +void TraceAssembler::set_expected_seqno(ton::BlockSeqno new_expected_seqno) { expected_seqno_ = new_expected_seqno; - LOG(INFO) << "Updating the expected senqo. New expected seqno is " << expected_seqno_; - if (is_ready_) { - process_queue(); - } } -void TraceAssembler::restore_trace_assembler_state(schema::TraceAssemblerState state) { - LOG(INFO) << "Got TraceAssemblerState with " << state.pending_traces_.size() - << " pending traces and " << state.pending_edges_.size() << " pending edges."; +td::Result TraceAssembler::restore_state(ton::BlockSeqno seqno) { + std::map> fileMap; + try { + for (const auto& entry : fs::directory_iterator(db_path_)) { + if (fs::is_regular_file(entry.status())) { + auto filename = entry.path().stem().string(); // Get filename without extension + auto extension = entry.path().extension().string(); // Get file extension - for (auto &trace : state.pending_traces_) { - pending_traces_.emplace(trace.trace_id, TraceImpl::from_schema(trace)); + if (extension == ".tastate") { + try { + int seqno = std::stoi(filename); + fileMap[seqno] = entry.path(); + } catch (const std::exception& e) { + LOG(ERROR) << "Error reading seqno of trace assembler state " << entry.path().string(); + } + } + } + } + } catch (const std::exception& e) { + return td::Status::Error(PSLICE() << "Error while searching for TraceAssembler states: " << e.what()); } - for (auto &edge : state.pending_edges_) { - pending_edges_.emplace(edge.msg_hash, TraceEdgeImpl::from_schema(edge)); + + for (const auto& [state_seqno, path] : fileMap) { + LOG(INFO) << "Found TA state seqno:" << seqno << " - path: " << path.string() << '\n'; + if (state_seqno > seqno) { + LOG(WARNING) << "Found trace assembler state " << state_seqno << " newer than requested " << seqno; + continue; + } + auto buffer_r = td::read_file(path.string()); + if (buffer_r.is_error()) { + LOG(ERROR) << "Failed to read trace assembler state file " << path.string() << ": " << buffer_r.move_as_error(); + continue; + } + auto buffer = buffer_r.move_as_ok(); + + std::unordered_map pending_traces; + std::unordered_map pending_edges; + try { + size_t offset = 0; + msgpack::unpacked pending_traces_res; + msgpack::unpack(pending_traces_res, buffer.data(), buffer.size(), offset); + msgpack::object pending_traces_obj = pending_traces_res.get(); + + msgpack::unpacked pending_edges_res; + msgpack::unpack(pending_edges_res, buffer.data(), buffer.size(), offset); + msgpack::object pending_edges_obj = pending_edges_res.get(); + + pending_traces_obj.convert(pending_traces); + pending_edges_obj.convert(pending_edges); + } catch (const std::exception &e) { + LOG(ERROR) << "Failed to unpack state for seqno " << state_seqno << ": " << e.what(); + continue; + } + + pending_traces_ = std::move(pending_traces); + pending_edges_ = std::move(pending_edges); + + return state_seqno; } - is_ready_ = true; - process_queue(); + + return td::Status::Error(ton::ErrorCode::warning, "TraceAssembler state not found"); } + void TraceAssembler::process_queue() { auto it = queue_.find(expected_seqno_); while(it != queue_.end()) { @@ -149,12 +202,13 @@ void TraceAssembler::process_queue() { // block processed queue_.erase(it); + expected_seqno_ += 1; it = queue_.find(expected_seqno_); } } -void TraceAssembler::process_block(std::int32_t seqno, ParsedBlockPtr block) { +void TraceAssembler::process_block(ton::BlockSeqno seqno, ParsedBlockPtr block) { // sort transactions by lt std::vector> sorted_txs; for(auto& blk: block->blocks_) { @@ -173,38 +227,32 @@ void TraceAssembler::process_block(std::int32_t seqno, ParsedBlockPtr block) { }); // process transactions - std::vector edges_found_; - std::unordered_set updated_traces_; - std::unordered_set updated_edges_; + std::vector completed_edges; + std::unordered_set updated_traces; + std::unordered_set pending_edges_added; for(auto &tx : sorted_txs) { - process_transaction(seqno, tx.get(), edges_found_, updated_traces_, updated_edges_); + process_transaction(seqno, tx.get(), completed_edges, updated_traces, pending_edges_added); } std::unordered_map trace_map; - for (auto & trace_id : updated_traces_) { + for (auto &trace_id : updated_traces) { auto trace = pending_traces_[trace_id]; - if(trace->pending_edges_ == 0) { + if (trace->pending_edges == 0) { if (trace->state != TraceImpl::State::broken) { trace->state = TraceImpl::State::complete; } pending_traces_.erase(trace->trace_id); - } else if (trace->pending_edges_ < 0) { + } else if (trace->pending_edges < 0) { trace->state = TraceImpl::State::broken; } trace_map[trace_id] = trace->to_schema(); } - for (auto & edge_hash : updated_edges_) { - auto edge = pending_edges_.find(edge_hash); - if (edge == pending_edges_.end()) { - LOG(ERROR) << "No edge found!"; - std::_Exit(42); - } - edges_found_.push_back(edge->second); - if (!edge->second.incomplete) { - LOG(WARNING) << "Complete edge in pending_edges_!"; - pending_edges_.erase(edge); - } + std::vector block_all_edges = completed_edges; + for (const auto &edge_hash : pending_edges_added) { + const auto &edge = pending_edges_[edge_hash]; + assert(edge.incomplete); + block_all_edges.push_back(edge); } - for (auto &edge : edges_found_) { + for (const auto &edge : block_all_edges) { // update trace auto &trace_schema = trace_map[edge.trace_id]; trace_schema.edges.push_back(edge.to_schema()); @@ -214,8 +262,8 @@ void TraceAssembler::process_block(std::int32_t seqno, ParsedBlockPtr block) { } } -void TraceAssembler::process_transaction(std::int32_t seqno, schema::Transaction& tx, std::vector& edges_found_, - std::unordered_set& updated_traces_, std::unordered_set& updated_edges_) { +void TraceAssembler::process_transaction(ton::BlockSeqno seqno, schema::Transaction& tx, std::vector& completed_edges, + std::unordered_set& updated_traces, std::unordered_set& pending_edges_added) { TraceImplPtr trace = nullptr; if (tx.in_msg.has_value()) { auto &msg = tx.in_msg.value(); @@ -234,7 +282,7 @@ void TraceAssembler::process_transaction(std::int32_t seqno, schema::Transaction edge.type = TraceEdgeImpl::Type::ext; edge.incomplete = false; edge.broken = false; - } else if (msg.source && msg.source.value() == "-1:0000000000000000000000000000000000000000000000000000000000000000") { + } else if (msg.source.value() == "-1:0000000000000000000000000000000000000000000000000000000000000000") { // system edge.trace_id = tx.hash; edge.msg_hash = msg.hash; @@ -242,7 +290,7 @@ void TraceAssembler::process_transaction(std::int32_t seqno, schema::Transaction edge.left_tx = std::nullopt; edge.right_tx = tx.hash; edge.type = TraceEdgeImpl::Type::sys; - edge.incomplete = tx.out_msgs.size() > 0; + edge.incomplete = false; edge.broken = false; } else { // broken edge @@ -258,17 +306,17 @@ void TraceAssembler::process_transaction(std::int32_t seqno, schema::Transaction // trace trace = std::make_shared(seqno, tx); - trace->edges_ += !edge.incomplete; - trace->pending_edges_ += edge.incomplete; + trace->edges += !edge.incomplete; + trace->pending_edges += edge.incomplete; if(edge.broken) { trace->state = TraceImpl::State::broken; ++broken_count_; } if (edge.incomplete) { pending_edges_.insert_or_assign(edge.msg_hash, edge); - updated_edges_.insert(edge.msg_hash); + pending_edges_added.insert(edge.msg_hash); } else { - edges_found_.push_back(edge); + completed_edges.push_back(edge); } pending_traces_.insert_or_assign(trace->trace_id, trace); } else { @@ -285,8 +333,8 @@ void TraceAssembler::process_transaction(std::int32_t seqno, schema::Transaction // LOG(ERROR) << "Broken trace for in_msg of tx: " << tx.hash; // create a broken trace trace = std::make_shared(seqno, tx); - trace->edges_ += !edge.incomplete; - trace->pending_edges_ += edge.incomplete; + trace->edges += !edge.incomplete; + trace->pending_edges += edge.incomplete; trace->state = TraceImpl::State::broken; ++broken_count_; @@ -297,25 +345,23 @@ void TraceAssembler::process_transaction(std::int32_t seqno, schema::Transaction } } - edges_found_.push_back(edge); + completed_edges.push_back(edge); pending_edges_.erase(edge_it); - if (updated_edges_.find(edge.msg_hash) != updated_edges_.end()) { - updated_edges_.erase(edge.msg_hash); - } + pending_edges_added.erase(edge.msg_hash); - trace->pending_edges_ -= 1; - trace->edges_ += 1; - trace->nodes_ += 1; + trace->pending_edges -= 1; + trace->edges += 1; + trace->nodes += 1; } } - updated_traces_.insert(trace->trace_id); + updated_traces.insert(trace->trace_id); tx.trace_id = trace->trace_id; msg.trace_id = trace->trace_id; } else { trace = std::make_shared(seqno, tx); pending_traces_.insert_or_assign(trace->trace_id, trace); - updated_traces_.insert(trace->trace_id); + updated_traces.insert(trace->trace_id); tx.trace_id = trace->trace_id; } @@ -333,13 +379,13 @@ void TraceAssembler::process_transaction(std::int32_t seqno, schema::Transaction edge.broken = false; } - trace->pending_edges_ += edge.incomplete; - trace->edges_ += !edge.incomplete; + trace->pending_edges += edge.incomplete; + trace->edges += !edge.incomplete; if (edge.incomplete) { pending_edges_.insert_or_assign(edge.msg_hash, edge); - updated_edges_.insert(edge.msg_hash); + pending_edges_added.insert(edge.msg_hash); } else { - edges_found_.push_back(edge); + completed_edges.push_back(edge); } msg.trace_id = trace->trace_id; } diff --git a/tondb-scanner/src/TraceAssembler.h b/tondb-scanner/src/TraceAssembler.h index 87d5715d..c1ab03b1 100644 --- a/tondb-scanner/src/TraceAssembler.h +++ b/tondb-scanner/src/TraceAssembler.h @@ -1,11 +1,8 @@ #pragma once +#include "msgpack-utils.h" #include "IndexData.h" -#include -// -// Utils -// struct Bits256Hasher { std::size_t operator()(const td::Bits256& k) const { std::size_t seed = 0; @@ -16,15 +13,12 @@ struct Bits256Hasher { } }; -// -// Trace datatypes -// struct TraceEdgeImpl { using Type = schema::TraceEdge::Type; td::Bits256 trace_id; td::Bits256 msg_hash; - std::uint64_t msg_lt; + uint64_t msg_lt; std::optional left_tx; std::optional right_tx; @@ -33,10 +27,12 @@ struct TraceEdgeImpl { bool broken{false}; // methods - std::string str() const; schema::TraceEdge to_schema() const; static TraceEdgeImpl from_schema(const schema::TraceEdge& edge); + + MSGPACK_DEFINE(trace_id, msg_hash, msg_lt, left_tx, right_tx, type, incomplete, broken); }; +MSGPACK_ADD_ENUM(TraceEdgeImpl::Type); struct TraceImpl; using TraceImplPtr = std::shared_ptr; @@ -47,65 +43,63 @@ struct TraceImpl { // info std::optional external_hash; - std::int32_t mc_seqno_start; - std::int32_t mc_seqno_end; + ton::BlockSeqno mc_seqno_start; + ton::BlockSeqno mc_seqno_end; - std::uint64_t start_lt; - std::uint32_t start_utime; + uint64_t start_lt; + uint32_t start_utime; - std::uint64_t end_lt{0}; - std::uint32_t end_utime{0}; + uint64_t end_lt{0}; + uint32_t end_utime{0}; State state{State::pending}; - std::int64_t pending_edges_{0}; - std::int64_t edges_{0}; - std::int64_t nodes_{0}; + size_t pending_edges{0}; + size_t edges{0}; + size_t nodes{0}; // methods TraceImpl() {} - TraceImpl(std::int32_t seqno, const schema::Transaction &tx) : + TraceImpl(ton::BlockSeqno seqno, const schema::Transaction &tx) : trace_id(tx.hash), external_hash((tx.in_msg.has_value() ? std::optional(tx.in_msg.value().hash) : std::nullopt)), mc_seqno_start(seqno), mc_seqno_end(seqno), start_lt(tx.lt), start_utime(tx.now), end_lt(tx.lt), end_utime(tx.now), - state(State::pending), nodes_(1) {} + state(State::pending), nodes(1) {} - std::string str() const; schema::Trace to_schema() const; static TraceImplPtr from_schema(const schema::Trace& trace); + + MSGPACK_DEFINE(trace_id, external_hash, mc_seqno_start, mc_seqno_end, start_lt, start_utime, end_lt, end_utime, state, pending_edges, edges, nodes); }; +MSGPACK_ADD_ENUM(TraceImpl::State); + -// -// TraceAssembler -// class TraceAssembler: public td::actor::Actor { struct Task { - std::int32_t seqno_; + ton::BlockSeqno seqno_; ParsedBlockPtr block_; td::Promise promise_; }; - // assembler state and queue - std::int32_t expected_seqno_; - std::map queue_; - bool is_ready_{false}; - std::int64_t broken_count_{0}; + std::string db_path_; + size_t gc_distance_; + ton::BlockSeqno expected_seqno_{0}; + std::map queue_; + size_t broken_count_{0}; - // trace assembly std::unordered_map pending_traces_; std::unordered_map pending_edges_; public: - TraceAssembler(std::int32_t expected_seqno, bool is_ready = false) : - expected_seqno_(expected_seqno), is_ready_(is_ready) {} + TraceAssembler(std::string db_path, size_t gc_distance); - void assemble(int mc_seqno, ParsedBlockPtr mc_block_, td::Promise promise); - void update_expected_seqno(std::int32_t new_expected_seqno); - void restore_trace_assembler_state(schema::TraceAssemblerState state); - void process_queue(); - + void assemble(ton::BlockSeqno mc_seqno, ParsedBlockPtr mc_block_, td::Promise promise); + + td::Result restore_state(ton::BlockSeqno expected_seqno); + void set_expected_seqno(ton::BlockSeqno expected_seqno); void start_up() override; void alarm() override; private: - void process_block(std::int32_t seqno, ParsedBlockPtr block); - void process_transaction(std::int32_t seqno, schema::Transaction& tx, std::vector& edges_found_, + void process_queue(); + void process_block(ton::BlockSeqno seqno, ParsedBlockPtr block); + void process_transaction(ton::BlockSeqno seqno, schema::Transaction& tx, std::vector& edges_found_, std::unordered_set& updated_traces_, std::unordered_set& updated_edges_); }; diff --git a/tondb-scanner/src/msgpack-utils.h b/tondb-scanner/src/msgpack-utils.h new file mode 100644 index 00000000..46f00add --- /dev/null +++ b/tondb-scanner/src/msgpack-utils.h @@ -0,0 +1,67 @@ +#include +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" + +namespace msgpack { + MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) { + namespace adaptor { + + template<> + struct pack { + template + msgpack::packer& operator()(msgpack::packer& o, const block::StdAddress& v) const { + std::string addr = std::to_string(v.workchain) + ":" + v.addr.to_hex(); + o.pack(addr); + return o; + } + }; + + template <> + struct convert { + msgpack::object const& operator()(msgpack::object const& o, block::StdAddress& v) const { + if (o.type != msgpack::type::STR) throw msgpack::type_error(); + std::string addr = o.as(); + if (!v.parse_addr(addr)) throw std::runtime_error("Failed to deserialize block::StdAddress"); + return o; + } + }; + + template<> + struct pack { + template + msgpack::packer& operator()(msgpack::packer& o, const td::Bits256& v) const { + o.pack(v.to_hex()); + return o; + } + }; + + template <> + struct convert { + msgpack::object const& operator()(msgpack::object const& o, td::Bits256& v) const { + if (o.type != msgpack::type::STR) throw msgpack::type_error(); + std::string hex = o.as(); + if (v.from_hex(hex) < 0) throw std::runtime_error("Failed to deserialize td::Bits256"); + return o; + } + }; + + template<> + struct pack { + template + msgpack::packer& operator()(msgpack::packer& o, const td::RefInt256& v) const { + o.pack(v->to_dec_string()); + return o; + } + }; + + template <> + struct convert { + msgpack::object const& operator()(msgpack::object const& o, td::RefInt256& v) const { + throw std::runtime_error("Deserializion of td::RefInt256 is not implemented"); + return o; + } + }; + + } // namespace adaptor + } // MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) +} // namespace msgpack diff --git a/tondb-scanner/src/smc-interfaces/NftSale.cpp b/tondb-scanner/src/smc-interfaces/NftSale.cpp index 700862f6..ab325d94 100644 --- a/tondb-scanner/src/smc-interfaces/NftSale.cpp +++ b/tondb-scanner/src/smc-interfaces/NftSale.cpp @@ -1,6 +1,6 @@ #include "NftSale.h" #include "convert-utils.h" -#include "common-utils.h" +#include "execute-smc.h" GetGemsNftFixPriceSale::GetGemsNftFixPriceSale(block::StdAddress address, diff --git a/tondb-scanner/src/smc-interfaces/Tokens.cpp b/tondb-scanner/src/smc-interfaces/Tokens.cpp index 669eb24a..159a050c 100644 --- a/tondb-scanner/src/smc-interfaces/Tokens.cpp +++ b/tondb-scanner/src/smc-interfaces/Tokens.cpp @@ -5,7 +5,9 @@ #include "DataParser.h" #include "Tokens.h" #include "parse_token_data.h" -#include "smc-interfaces/common-utils.h" +#include "smc-interfaces/execute-smc.h" +#include "tokens.h" +#include "common/checksum.h" class FetchAccountFromShardV2: public td::actor::Actor { @@ -313,6 +315,26 @@ void NftItemDetectorR::start_up() { } } +bool NftItemDetectorR::is_testnet = false; + +const auto dot_ton_dns_root_addr_mainnet = block::StdAddress::parse("0:B774D95EB20543F186C06B371AB88AD704F7E256130CAF96189368A7D0CB6CCF").move_as_ok(); +const auto dot_ton_dns_root_addr_testnet = block::StdAddress::parse("0:E33ED33A42EB2032059F97D90C706F8400BB256D32139CA707F1564AD699C7DD").move_as_ok(); +const auto dot_t_dot_me_dns_root_addr_mainnet = block::StdAddress::parse("0:80D78A35F955A14B679FAA887FF4CD5BFC0F43B4A4EEA2A7E6927F3701B273C2").move_as_ok(); + +block::StdAddress NftItemDetectorR::get_dot_ton_dns_root_addr() { + if (is_testnet) { + return dot_ton_dns_root_addr_testnet; + } + return dot_ton_dns_root_addr_mainnet; +} + +std::optional NftItemDetectorR::dot_t_dot_me_dns_root_addr() { + if (is_testnet) { + return std::nullopt; + } + return dot_t_dot_me_dns_root_addr_mainnet; +} + void NftItemDetectorR::got_collection(Result item_data, td::Ref ind_content, td::Ref collection_code, td::Ref collection_data) { auto verify = verify_with_collection(item_data.collection_address.value(), collection_code, collection_data, item_data.index); if (verify.is_error()) { @@ -329,6 +351,12 @@ void NftItemDetectorR::got_collection(Result item_data, td::Ref ind_co } item_data.content = content.move_as_ok(); + process_domain_and_dns_data(get_dot_ton_dns_root_addr(), [this](){ return this->get_ton_domain(); }, item_data); + auto t_me_root = dot_t_dot_me_dns_root_addr(); + if (t_me_root) { + process_domain_and_dns_data(t_me_root.value(), [this](){ return this->get_t_me_domain(); }, item_data); + } + promise_.set_value(std::move(item_data)); stop(); } @@ -355,18 +383,104 @@ td::Result> NftItemDetectorR::get_content(td: TRY_RESULT(stack, execute_smc_method<1>(collection_address, collection_code, collection_data, config_, "get_nft_content", {vm::StackEntry(index), vm::StackEntry(ind_content)}, {vm::StackEntry::Type::t_cell})); - const std::string ton_dns_root_addr = "0:B774D95EB20543F186C06B371AB88AD704F7E256130CAF96189368A7D0CB6CCF"; + return parse_token_data(stack[0].as_cell()); +} - if (convert::to_raw_address(collection_address) == ton_dns_root_addr) { - std::map result; - TRY_RESULT_ASSIGN(result["domain"], get_domain()); - return result; +void NftItemDetectorR::process_domain_and_dns_data(const block::StdAddress& root_address, const std::function()>& get_domain_function, Result& item_data) { + if (!item_data.collection_address.has_value() || !(item_data.collection_address.value() == root_address)) { + return; } + auto domain = get_domain_function(); + if (domain.is_error()) { + LOG(ERROR) << "Failed to get domain for " << address_ << ": " << domain.move_as_error(); + } else { + item_data.content.value()["domain"] = domain.ok(); - return parse_token_data(stack[0].as_cell()); + auto dns_data = get_dns_entry_data(); + if (dns_data.is_ok()) { + dns_data.ok_ref().domain = domain.move_as_ok(); + item_data.dns_entry = dns_data.move_as_ok(); + } + } } -td::Result NftItemDetectorR::get_domain() { +td::Result NftItemDetectorR::get_dns_entry_data() { + auto zero_byte_cell = vm::CellBuilder().store_bytes("\0").finalize(); + auto zero_byte_slice = vm::load_cell_slice_ref(zero_byte_cell); + td::RefInt256 categories{true, 0}; + + TRY_RESULT(stack, execute_smc_method<2>(address_, code_cell_, data_cell_, config_, "dnsresolve", + {vm::StackEntry(zero_byte_slice), vm::StackEntry(categories)}, {vm::StackEntry::Type::t_int, vm::StackEntry::Type::t_cell})); + + auto resolved_bits_cnt = stack[0].as_int()->to_long(); + if (resolved_bits_cnt != 8) { + return td::Status::Error("dnsresolve returned unexpected bits cnt resolved"); + } + + auto recordset_cell = stack[1].as_cell(); + if (recordset_cell.is_null()) { + return td::Status::Error("recordset is null"); + } + + vm::Dictionary records{recordset_cell, 256}; + Result::DNSEntry result; + + // TODO: support ProtoList + auto site_cell = records.lookup_ref(td::sha256_bits256("site")); + if (site_cell.not_null()) { + tokens::gen::DNSRecord::Record_dns_adnl_address site_record; + if (!tlb::unpack_cell(site_cell, site_record)) { + LOG(ERROR) << "Failed to unpack DNSRecord site for " << address_; + } else { + result.site_adnl = site_record.adnl_addr; + } + } + + // TODO: support SmcCapList + auto wallet_cell = records.lookup_ref(td::sha256_bits256("wallet")); + if (wallet_cell.not_null()) { + tokens::gen::DNSRecord::Record_dns_smc_address wallet_record; + if (!tlb::unpack_cell(wallet_cell, wallet_record)) { + LOG(ERROR) << "Failed to unpack DNSRecord wallet"; + } else { + auto wallet = convert::to_std_address(wallet_record.smc_addr); + if (wallet.is_error()) { + LOG(ERROR) << "Failed to parse DNSRecord wallet address"; + } else { + result.wallet = wallet.move_as_ok(); + } + } + } + + auto next_resolver_cell = records.lookup_ref(td::sha256_bits256("dns_next_resolver")); + if (next_resolver_cell.not_null()) { + tokens::gen::DNSRecord::Record_dns_next_resolver next_resolver_record; + if (!tlb::unpack_cell(next_resolver_cell, next_resolver_record)) { + LOG(ERROR) << "Failed to unpack DNSRecord next_resolver"; + } else { + auto next_resolver = convert::to_std_address(next_resolver_record.resolver); + if (next_resolver.is_error()) { + LOG(ERROR) << "Failed to parse DNSRecord next_resolver address"; + } else { + result.next_resolver = next_resolver.move_as_ok(); + } + } + } + + auto storage_bag_id_cell = records.lookup_ref(td::sha256_bits256("storage")); + if (storage_bag_id_cell.not_null()) { + tokens::gen::DNSRecord::Record_dns_storage_address dns_storage_record; + if (!tlb::unpack_cell(storage_bag_id_cell, dns_storage_record)) { + LOG(ERROR) << "Failed to unpack DNSRecord storage"; + } else { + result.storage_bag_id = dns_storage_record.bag_id; + } + } + + return result; +} + +td::Result NftItemDetectorR::get_ton_domain() { TRY_RESULT(stack, execute_smc_method<1>(address_, code_cell_, data_cell_, config_, "get_domain", {}, {vm::StackEntry::Type::t_slice})); auto cs = stack[0].as_slice(); @@ -384,6 +498,42 @@ td::Result NftItemDetectorR::get_domain() { return td::Status::Error("get_domain returned unexpected result"); } +td::Result NftItemDetectorR::get_t_me_domain() { + TRY_RESULT(stack, execute_smc_method<1>(address_, code_cell_, data_cell_, config_, "get_full_domain", {}, {vm::StackEntry::Type::t_slice})); + auto cs = stack[0].as_slice(); + + if (cs.not_null()) { + auto size = cs->size(); + if (size % 8 == 0) { + auto cnt = size / 8; + unsigned char tmp[1024]; + cs.write().fetch_bytes(tmp, cnt); + std::string s{tmp, tmp + cnt}; + + // convert "me\0t\0username\0" to "username.t.me" + std::string c; + std::vector parts; + for (size_t i = 0; i < s.size(); i++) { + if (s[i] == '\0') { + parts.push_back(c); + c = ""; + } else { + c += s[i]; + } + } + std::string result; + for (int16_t i = parts.size() - 1; i >= 0; i--) { + result += parts[i]; + if (i != 0) { + result += "."; + } + } + return result; + } + } + return td::Status::Error("get_full_domain returned unexpected result"); +} + NftCollectionDetectorR::NftCollectionDetectorR(block::StdAddress address, td::Ref code_cell, td::Ref data_cell, diff --git a/tondb-scanner/src/smc-interfaces/Tokens.h b/tondb-scanner/src/smc-interfaces/Tokens.h index 394d71bf..bc6bd1b8 100644 --- a/tondb-scanner/src/smc-interfaces/Tokens.h +++ b/tondb-scanner/src/smc-interfaces/Tokens.h @@ -67,14 +67,24 @@ class JettonMasterDetectorR: public td::actor::Actor { class NftItemDetectorR: public td::actor::Actor { public: struct Result { + struct DNSEntry { + std::string domain; + std::optional wallet; + std::optional next_resolver; + std::optional site_adnl; + std::optional storage_bag_id; + }; block::StdAddress address; bool init; td::RefInt256 index; std::optional collection_address; std::optional owner_address; std::optional> content; + std::optional dns_entry; }; + static bool is_testnet; + NftItemDetectorR(block::StdAddress address, td::Ref code_cell, td::Ref data_cell, @@ -89,7 +99,12 @@ class NftItemDetectorR: public td::actor::Actor { td::Status verify_with_collection(block::StdAddress collection_address, td::Ref collection_code, td::Ref collection_data, td::RefInt256 index); td::Result> get_content(td::RefInt256 index, td::Ref ind_content, block::StdAddress collection_address, td::Ref collection_code, td::Ref collection_data); - td::Result get_domain(); + void process_domain_and_dns_data(const block::StdAddress& root_address, const std::function()>& get_domain_function, Result& item_data); + td::Result get_dns_entry_data(); + td::Result get_ton_domain(); + td::Result get_t_me_domain(); + static block::StdAddress get_dot_ton_dns_root_addr(); + static std::optional dot_t_dot_me_dns_root_addr(); block::StdAddress address_; td::Ref code_cell_; diff --git a/tondb-scanner/src/smc-interfaces/common-utils.h b/tondb-scanner/src/smc-interfaces/execute-smc.cpp similarity index 64% rename from tondb-scanner/src/smc-interfaces/common-utils.h rename to tondb-scanner/src/smc-interfaces/execute-smc.cpp index 677af768..8ac339fa 100644 --- a/tondb-scanner/src/smc-interfaces/common-utils.h +++ b/tondb-scanner/src/smc-interfaces/execute-smc.cpp @@ -1,10 +1,10 @@ -#include "smc-envelope/SmartContract.h" +#include +#include "execute-smc.h" -template td::Result> execute_smc_method(const block::StdAddress& address, td::Ref code, td::Ref data, std::shared_ptr config, const std::string& method_id, - std::vector input, const std::array& expected_types) { + std::vector input) { ton::SmartContract smc({code, data}); ton::SmartContract::Args args; args.set_libraries(vm::Dictionary(config->get_libraries_root(), 256)); @@ -19,16 +19,8 @@ td::Result> execute_smc_method(const block::StdAddre if (!res.success) { return td::Status::Error(method_id + " failed"); } - if (res.stack->depth() != expected_types.size()) { - return td::Status::Error(method_id + " unexpected result stack depth"); - } - auto stack = res.stack->extract_contents(); - for (size_t i = 0; i < expected_types.size(); i++) { - if (stack[i].type() != expected_types[i]) { - return td::Status::Error(method_id + " unexpected stack entry type"); - } - } + auto stack = res.stack->extract_contents(); return stack; -} +} \ No newline at end of file diff --git a/tondb-scanner/src/smc-interfaces/execute-smc.h b/tondb-scanner/src/smc-interfaces/execute-smc.h new file mode 100644 index 00000000..e2d98798 --- /dev/null +++ b/tondb-scanner/src/smc-interfaces/execute-smc.h @@ -0,0 +1,24 @@ +#include "smc-envelope/SmartContract.h" + +td::Result> execute_smc_method(const block::StdAddress& address, + td::Ref code, td::Ref data, + std::shared_ptr config, const std::string& method_id, + std::vector input); + +template +td::Result> execute_smc_method(const block::StdAddress& address, td::Ref code, td::Ref data, + std::shared_ptr config, const std::string& method_id, + std::vector input, const std::array& expected_types) { + TRY_RESULT(stack, execute_smc_method(address, code, data, config, method_id, std::move(input))); + + if (stack.size() != expected_types.size()) { + return td::Status::Error(method_id + " unexpected result stack depth"); + } + for (size_t i = 0; i < expected_types.size(); i++) { + if (stack[i].type() != expected_types[i]) { + return td::Status::Error(method_id + " unexpected stack entry type"); + } + } + + return stack; +} diff --git a/tondb-scanner/src/tlb/tokens.tlb b/tondb-scanner/src/tlb/tokens.tlb index adf64a88..d48f647c 100644 --- a/tondb-scanner/src/tlb/tokens.tlb +++ b/tondb-scanner/src/tlb/tokens.tlb @@ -79,4 +79,23 @@ text#_ data:Cell = Text; // text#_ { snake#00 data:Cell = ContentData; // snake#00 data:(SnakeData ~n) = ContentData; chunks#01 data:ChunkedData = ContentData; onchain#00 data:(HashmapE 256 ^ContentData) = FullContent; -offchain#01 uri:Text = FullContent; \ No newline at end of file +offchain#01 uri:Text = FullContent; + +proto_http#4854 = Protocol; +proto_list_nil$0 = ProtoList; + +proto_list_next$1 head:Protocol tail:ProtoList = ProtoList; + +cap_is_wallet#2177 = SmcCapability; + +cap_list_nil$0 = SmcCapList; +cap_list_next$1 head:SmcCapability tail:SmcCapList = SmcCapList; + +dns_smc_address#9fd3 smc_addr:MsgAddressInt flags:(## 8) { flags <= 1 } + cap_list:flags . 0?SmcCapList = DNSRecord; +dns_next_resolver#ba93 resolver:MsgAddressInt = DNSRecord; +dns_adnl_address#ad01 adnl_addr:bits256 flags:(## 8) { flags <= 1 } + proto_list:flags . 0?ProtoList = DNSRecord; +dns_storage_address#7473 bag_id:bits256 = DNSRecord; + +_ (HashmapE 256 ^DNSRecord) = DNS_RecordSet; From d5f7c784a4cc4760fd115c6ffdc77ea2155f42ce Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:35:27 +0100 Subject: [PATCH 02/28] Support token data dict value as direct slice (#93) --- tondb-scanner/src/parse_token_data.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tondb-scanner/src/parse_token_data.cpp b/tondb-scanner/src/parse_token_data.cpp index 5dadd9bb..c122258b 100644 --- a/tondb-scanner/src/parse_token_data.cpp +++ b/tondb-scanner/src/parse_token_data.cpp @@ -60,8 +60,7 @@ td::Result parse_chunks_data(td::Ref data) { } } -td::Result parse_content_data(td::Ref data) { - auto cs = vm::load_cell_slice_ref(data); +td::Result parse_content_data(td::Ref cs) { switch (tokens::gen::t_ContentData.check_tag(*cs)) { case tokens::gen::ContentData::snake: { tokens::gen::ContentData::Record_snake snake_record; @@ -113,9 +112,18 @@ td::Result> parse_token_data(td::Ref res; for (auto attr : attributes) { - auto value = dict.lookup_ref(td::sha256_bits256(attr)); - if (value.not_null()) { - auto attr_data_r = parse_content_data(value); + auto value_cs = dict.lookup(td::sha256_bits256(attr)); + if (value_cs.not_null()) { + // TEP-64 standard requires that all attributes are stored in a dictionary with a single reference as a value: + // onchain#00 data:(HashmapE 256 ^ContentData) = FullContent; + // however, some contracts store attributes as a direct value: + // onchain#00 data:(HashmapE 256 ContentData) = FullContent; + // so we need to handle both cases. + if (value_cs->size() == 0 && value_cs->size_refs() == 1) { + value_cs = vm::load_cell_slice_ref(value_cs->prefetch_ref()); + } + + auto attr_data_r = parse_content_data(value_cs); if (attr_data_r.is_error()) { LOG(ERROR) << "Failed to parse attribute " << attr << ": " << attr_data_r.move_as_error().message(); continue; From d648022997348fbf1f30f5b650dddc73cbc1be66 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 21 Dec 2024 14:42:02 +0100 Subject: [PATCH 03/28] Add support of extra currencies (#95) * extra currencies, small refactoring of models * fixes * new columns when creating tables --- CMakeLists.txt | 1 - sandbox-cpp/CMakeLists.txt | 20 - sandbox-cpp/src/DbReader.cpp | 9 - sandbox-cpp/src/DbReader.h | 11 - sandbox-cpp/src/Scheduler.cpp | 35 -- sandbox-cpp/src/Scheduler.h | 27 -- sandbox-cpp/src/main.cpp | 86 ---- .../src/InsertManagerClickhouse.cpp | 85 ++-- .../src/InsertManagerPostgres.cpp | 388 ++++-------------- .../src/InsertManagerPostgres.cpp | 55 ++- ton-smc-scanner/src/PostgreSQLInserter.cpp | 3 +- ton-trace-emulator/src/Serializer.hpp | 47 ++- tondb-scanner/src/DataParser.cpp | 59 ++- tondb-scanner/src/DataParser.h | 1 + tondb-scanner/src/IndexData.h | 73 ++-- tondb-scanner/src/convert-utils.cpp | 38 +- tondb-scanner/src/convert-utils.h | 7 +- 17 files changed, 274 insertions(+), 671 deletions(-) delete mode 100644 sandbox-cpp/CMakeLists.txt delete mode 100644 sandbox-cpp/src/DbReader.cpp delete mode 100644 sandbox-cpp/src/DbReader.h delete mode 100644 sandbox-cpp/src/Scheduler.cpp delete mode 100644 sandbox-cpp/src/Scheduler.h delete mode 100644 sandbox-cpp/src/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c79c6189..d7e7445c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,6 @@ add_subdirectory(ton-index-clickhouse) add_subdirectory(ton-integrity-checker) add_subdirectory(ton-smc-scanner) add_subdirectory(ton-trace-emulator) -add_subdirectory(sandbox-cpp) if (PGTON) message("Building pgton") diff --git a/sandbox-cpp/CMakeLists.txt b/sandbox-cpp/CMakeLists.txt deleted file mode 100644 index 2dbbe01d..00000000 --- a/sandbox-cpp/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -cmake_minimum_required(VERSION 3.16) - -add_executable(sandbox-cpp - src/main.cpp - src/Scheduler.cpp - src/DbReader.cpp -) - -target_include_directories(sandbox-cpp - PUBLIC external/ton - PUBLIC tondb-scanner/src - PUBLIC src/ -) - -target_link_directories(sandbox-cpp - PUBLIC external/ton -) -target_compile_features(sandbox-cpp PRIVATE cxx_std_17) -target_link_libraries(sandbox-cpp tondb-scanner overlay tdutils tdactor adnl tl_api dht ton_crypto - catchain validatorsession validator-disk ton_validator validator-disk smc-envelope) diff --git a/sandbox-cpp/src/DbReader.cpp b/sandbox-cpp/src/DbReader.cpp deleted file mode 100644 index 735b6afe..00000000 --- a/sandbox-cpp/src/DbReader.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "DbReader.h" - -void DbReader::start_up() { - LOG(INFO) << "DbReader start_up"; -} - -void DbReader::alarm() { - LOG(INFO) << "DbReader alarm"; -} diff --git a/sandbox-cpp/src/DbReader.h b/sandbox-cpp/src/DbReader.h deleted file mode 100644 index 939c2942..00000000 --- a/sandbox-cpp/src/DbReader.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once -#include "td/actor/actor.h" - -class DbReader: public td::actor::Actor { - std::string db_root_; -public: - DbReader(std::string db_root): db_root_(db_root) {} - - void start_up() override; - void alarm() override; -}; diff --git a/sandbox-cpp/src/Scheduler.cpp b/sandbox-cpp/src/Scheduler.cpp deleted file mode 100644 index ad46aa95..00000000 --- a/sandbox-cpp/src/Scheduler.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "Scheduler.h" - -#include "td/utils/StringBuilder.h" - - -void Scheduler::start_up() { - LOG(INFO) << "Scheduler start_up"; -} - -void Scheduler::alarm() { - LOG(INFO) << "Scheduler: alarm"; -} - -void Scheduler::run() { - LOG(INFO) << "Scheduler: run"; - ton::BlockSeqno mc_seqno{1024}; -} - -void Scheduler::got_block_state_data(ton::BlockSeqno mc_seqno, MasterchainBlockDataState R) { - LOG(INFO) << "Scheduler: Got first block " << mc_seqno; - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), mc_seqno, R_ = std::move(R)](td::Result res) { - if(res.is_error()) { - LOG(ERROR) << "DbScanner: Failed to fetch seqno " << mc_seqno; - return; - } - td::actor::send_closure(SelfId, &Scheduler::got_block_state_data_2, mc_seqno, std::move(R_), res.move_as_ok()); - }); - td::actor::send_closure(db_scanner_, &DbScanner::fetch_seqno, mc_seqno, std::move(P)); -} - -void Scheduler::got_block_state_data_2(ton::BlockSeqno mc_seqno, MasterchainBlockDataState R1, MasterchainBlockDataState R2) { - LOG(INFO) << "Scheduler: Got both blocks " << mc_seqno; - std::_Exit(0); -} diff --git a/sandbox-cpp/src/Scheduler.h b/sandbox-cpp/src/Scheduler.h deleted file mode 100644 index 4c42a35f..00000000 --- a/sandbox-cpp/src/Scheduler.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once -#include "td/actor/actor.h" - -#include "DbReader.h" -#include "DataParser.h" -#include "DbScanner.h" - - -class Scheduler: public td::actor::Actor { -private: - td::actor::ActorId db_scanner_; - td::actor::ActorId db_reader_; - td::actor::ActorId parse_manager_; - - ton::BlockSeqno last_known_seqno_{2}; -public: - Scheduler(td::actor::ActorId db_scanner, - td::actor::ActorId db_reader, - td::actor::ActorId parse_manager, td::int32 last_known_seqno) - : db_scanner_(db_scanner), db_reader_(db_reader), parse_manager_(parse_manager), last_known_seqno_(last_known_seqno) {} - void start_up() override; - void alarm() override; - void run(); - - void got_block_state_data(ton::BlockSeqno mc_seqno, MasterchainBlockDataState R); - void got_block_state_data_2(ton::BlockSeqno mc_seqno, MasterchainBlockDataState R1, MasterchainBlockDataState R2); -}; diff --git a/sandbox-cpp/src/main.cpp b/sandbox-cpp/src/main.cpp deleted file mode 100644 index dbf743b8..00000000 --- a/sandbox-cpp/src/main.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "td/utils/port/signals.h" -#include "td/utils/OptionParser.h" -#include "td/utils/format.h" -#include "td/utils/logging.h" -#include "td/utils/check.h" - -#include "crypto/vm/cp0.h" - -#include "DataParser.h" -#include "DbScanner.h" -#include "DbReader.h" -#include "Scheduler.h" - - -int main(int argc, char *argv[]) { - SET_VERBOSITY_LEVEL(verbosity_INFO); - td::set_default_failure_signal_handler().ensure(); - - CHECK(vm::init_op_cp0()); - - td::actor::ActorOwn db_scanner_; - td::actor::ActorOwn db_reader_; - td::actor::ActorOwn parse_manager_; - td::actor::ActorOwn scheduler_; - - // options - td::uint32 threads = 7; - td::int32 last_known_seqno = 2; - std::string db_root; - - td::OptionParser p; - p.set_description("Sandbox"); - p.add_option('\0', "help", "prints_help", [&]() { - char b[10240]; - td::StringBuilder sb(td::MutableSlice{b, 10000}); - sb << p; - std::cout << sb.as_cslice().c_str(); - std::exit(2); - }); - p.add_option('D', "db", "Path to TON DB folder", [&](td::Slice fname) { - db_root = fname.str(); - }); - p.add_checked_option('f', "from", "Masterchain seqno to start indexing from", [&](td::Slice fname) { - int v; - try { - v = std::stoi(fname.str()); - } catch (...) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --from: not a number"); - } - last_known_seqno = v; - return td::Status::OK(); - }); - // scheduler settings - p.add_checked_option('t', "threads", "Scheduler threads (default: 7)", [&](td::Slice fname) { - int v; - try { - v = std::stoi(fname.str()); - } catch (...) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); - } - threads = v; - return td::Status::OK(); - }); - auto S = p.run(argc, argv); - if (S.is_error()) { - LOG(ERROR) << "failed to parse options: " << S.move_as_error(); - std::_Exit(2); - } - - td::actor::Scheduler scheduler({threads}); - scheduler.run_in_context([&] { - parse_manager_ = td::actor::create_actor("parsemanager"); - db_scanner_ = td::actor::create_actor("scanner", db_root, dbs_secondary); - db_reader_ = td::actor::create_actor("reader", db_root); - - scheduler_ = td::actor::create_actor("scheduler", db_scanner_.get(), db_reader_.get(), - parse_manager_.get(), last_known_seqno); - }); - scheduler.run_in_context([&] { td::actor::send_closure(scheduler_, &Scheduler::run); }); - - while(scheduler.run(1)) { - // do something - } - LOG(INFO) << "Done!"; - return 0; -} diff --git a/ton-index-clickhouse/src/InsertManagerClickhouse.cpp b/ton-index-clickhouse/src/InsertManagerClickhouse.cpp index cb013980..3b69cab3 100644 --- a/ton-index-clickhouse/src/InsertManagerClickhouse.cpp +++ b/ton-index-clickhouse/src/InsertManagerClickhouse.cpp @@ -9,8 +9,6 @@ #include "convert-utils.h" -#define TO_STD_OPTIONAL(x) ((x) ? std::optional(x.value()) : std::nullopt) - void InsertManagerClickhouse::start_up() { LOG(INFO) << "Clickhouse start_up"; try { @@ -691,8 +689,8 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { auto split_info__sibling_addr_col = std::make_shared>(); auto store_storage_ph = [&](const schema::TrStoragePhase& storage_ph) { - storage_ph__storage_fees_collected_col->Append(storage_ph.storage_fees_collected); - storage_ph__storage_fees_due_col->Append(storage_ph.storage_fees_due); + storage_ph__storage_fees_collected_col->Append(storage_ph.storage_fees_collected->to_long()); + storage_ph__storage_fees_due_col->Append(storage_ph.storage_fees_due ? std::optional(storage_ph.storage_fees_due.value()->to_long()) : std::nullopt); storage_ph__status_change_col->Append(storage_ph.status_change); }; auto store_empty_storage_ph = [&]() { @@ -701,8 +699,8 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { storage_ph__status_change_col->Append(std::nullopt); }; auto store_credit_ph = [&](const schema::TrCreditPhase& credit_ph) { - credit_ph__due_fees_collected_col->Append(credit_ph.due_fees_collected); - credit_ph__credit_col->Append(credit_ph.credit); + credit_ph__due_fees_collected_col->Append(credit_ph.due_fees_collected ? std::optional(credit_ph.due_fees_collected.value()->to_long()) : std::nullopt); + credit_ph__credit_col->Append(credit_ph.credit.grams->to_long()); }; auto store_empty_credit_ph = [&]() { credit_ph__due_fees_collected_col->Append(std::nullopt); @@ -732,7 +730,7 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { compute_ph__vm__success_col->Append(v->success); compute_ph__vm__msg_state_used_col->Append(v->msg_state_used); compute_ph__vm__account_activated_col->Append(v->account_activated); - compute_ph__vm__gas_fees_col->Append(v->gas_fees); + compute_ph__vm__gas_fees_col->Append(v->gas_fees->to_long()); compute_ph__vm__gas_used_col->Append(v->gas_used); compute_ph__vm__gas_limit_col->Append(v->gas_limit); compute_ph__vm__gas_credit_col->Append(v->gas_credit); @@ -766,8 +764,8 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { action__valid_col->Append(action.valid); action__no_funds_col->Append(action.no_funds); action__status_change_col->Append(action.status_change); - action__total_fwd_fees_col->Append(action.total_fwd_fees); - action__total_action_fees_col->Append(action.total_action_fees); + action__total_fwd_fees_col->Append(action.total_fwd_fees ? std::optional(action.total_fwd_fees.value()->to_long()) : std::nullopt); + action__total_action_fees_col->Append(action.total_action_fees ? std::optional(action.total_action_fees.value()->to_long()) : std::nullopt); action__result_code_col->Append(action.result_code); action__result_arg_col->Append(action.result_arg); action__tot_actions_col->Append(action.tot_actions); @@ -807,7 +805,7 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { bounce_col->Append(std::optional(1)); bounce__msg_size__cells_col->Append(v->msg_size.cells); bounce__msg_size__bits_col->Append(v->msg_size.bits); - bounce__no_funds__req_fwd_fees_col->Append(v->req_fwd_fees); + bounce__no_funds__req_fwd_fees_col->Append(v->req_fwd_fees->to_long()); bounce__ok__msg_fees_col->Append(std::nullopt); bounce__ok__fwd_fees_col->Append(std::nullopt); } else if (auto* v = std::get_if(&bounce)) { @@ -815,8 +813,8 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { bounce__msg_size__cells_col->Append(v->msg_size.cells); bounce__msg_size__bits_col->Append(v->msg_size.bits); bounce__no_funds__req_fwd_fees_col->Append(std::nullopt); - bounce__ok__msg_fees_col->Append(v->msg_fees); - bounce__ok__fwd_fees_col->Append(v->fwd_fees); + bounce__ok__msg_fees_col->Append(v->msg_fees->to_long()); + bounce__ok__fwd_fees_col->Append(v->fwd_fees->to_long()); } }; auto store_empty_bounce_ph = [&]() { @@ -851,7 +849,7 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { now_col->Append(tx_.now); orig_status_col->Append(tx_.orig_status); end_status_col->Append(tx_.end_status); - total_fees_col->Append(tx_.total_fees); + total_fees_col->Append(tx_.total_fees.grams->to_long()); account_state_hash_before_col->Append(td::base64_encode(tx_.account_state_hash_after.as_slice())); account_state_hash_after_col->Append(td::base64_encode(tx_.account_state_hash_after.as_slice())); block_workchain_col->Append(blk_.workchain); @@ -1115,18 +1113,29 @@ void InsertBatchClickhouse::insert_messages(clickhouse::Client &client) { mc_block_seqno_col->Append(blk.mc_block_seqno.value()); direction_col->Append(direction); hash_col->Append(td::base64_encode(msg.hash.as_slice())); - source_col->Append(TO_STD_OPTIONAL(msg.source)); - destination_col->Append(TO_STD_OPTIONAL(msg.destination)); - value_col->Append(TO_STD_OPTIONAL(msg.value)); - fwd_fee_col->Append(TO_STD_OPTIONAL(msg.fwd_fee)); - ihr_fee_col->Append(TO_STD_OPTIONAL(msg.ihr_fee)); - created_lt_col->Append(TO_STD_OPTIONAL(msg.created_lt)); - created_at_col->Append(TO_STD_OPTIONAL(msg.created_at)); - opcode_col->Append(TO_STD_OPTIONAL(msg.opcode)); - ihr_disabled_col->Append(TO_STD_OPTIONAL(msg.ihr_disabled)); - bounce_col->Append(TO_STD_OPTIONAL(msg.bounce)); - bounced_col->Append(TO_STD_OPTIONAL(msg.bounced)); - import_fee_col->Append(TO_STD_OPTIONAL(msg.import_fee)); + source_col->Append(msg.source); + destination_col->Append(msg.destination); + value_col->Append(msg.value ? std::optional(msg.value->grams->to_long()) : std::nullopt); + fwd_fee_col->Append(msg.fwd_fee ? std::optional(msg.fwd_fee.value()->to_long()) : std::nullopt); + ihr_fee_col->Append(msg.ihr_fee ? std::optional(msg.ihr_fee.value()->to_long()) : std::nullopt); + created_lt_col->Append(msg.created_lt); + created_at_col->Append(msg.created_at); + opcode_col->Append(msg.opcode); + ihr_disabled_col->Append(msg.ihr_disabled); + bounce_col->Append(msg.bounce); + bounced_col->Append(msg.bounced); + // msg.import_fee is defined by user and can be too large for uint64, so we need to check it + // and if it is too large, we will insert NULL. + // TODO: change uint64 to uint256 + std::optional import_fee_val; + if (msg.import_fee) { + import_fee_val = msg.import_fee.value()->to_long(); + if (import_fee_val.value() == (~0ULL << 63)) { + LOG(WARNING) << "Import fee of msg " << msg.hash.to_hex() << " is too large for bigint: " << msg.import_fee.value(); + import_fee_val = std::nullopt; + } + } + import_fee_col->Append(import_fee_val); td::Bits256 body_hash = msg.body->get_hash().bits(); if(body_hashes.find(body_hash) == body_hashes.end()) { @@ -1238,7 +1247,7 @@ void InsertBatchClickhouse::insert_account_states(clickhouse::Client &client) { hash_col->Append(td::base64_encode(state_.hash.as_slice())); account_col->Append(convert::to_raw_address(state_.account)); timestamp_col->Append(state_.timestamp); - balance_col->Append(state_.balance); + balance_col->Append(state_.balance.grams->to_long()); account_status_col->Append(state_.account_status); frozen_hash_col->Append(frozen_hash); code_hash_col->Append(code_hash); @@ -1362,7 +1371,7 @@ void InsertBatchClickhouse::insert_nfts(clickhouse::Client &client) { LOG(ERROR) << "Collection " << collection.address << " has null in next_item_index!"; next_item_index->Append(Int128()); } - owner_address->Append(TO_STD_OPTIONAL(collection.owner_address)); + owner_address->Append(collection.owner_address); if (collection.collection_content) { collection_content->Append(collection.collection_content.value()); } else { @@ -1456,10 +1465,10 @@ void InsertBatchClickhouse::insert_nfts(clickhouse::Client &client) { for (const auto& task : insert_tasks_) { for (const auto& transfer : task.parsed_block_->get_events()) { auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); - auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : td::optional{}; + auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; auto nft_item_address = convert::to_raw_address(transfer.nft_item); Int128 nft_item_index_value{-2}; @@ -1491,13 +1500,13 @@ void InsertBatchClickhouse::insert_nfts(clickhouse::Client &client) { old_owner->Append(transfer.old_owner); new_owner->Append(transfer.new_owner); response_destination->Append(transfer.response_destination); - custom_payload->Append(TO_STD_OPTIONAL(custom_payload_boc)); + custom_payload->Append(custom_payload_boc); if (transfer.forward_amount.not_null()) { forward_amount->Append(str2int128(transfer.forward_amount->to_dec_string())); } else { forward_amount->Append(Int128(-1)); } - forward_payload->Append(TO_STD_OPTIONAL(forward_payload_boc)); + forward_payload->Append(forward_payload_boc); } } block.AppendColumn("transaction_hash", transaction_hash); @@ -1573,7 +1582,7 @@ void InsertBatchClickhouse::insert_jettons(clickhouse::Client &client) { total_supply->Append(Int128()); } mintable->Append(master.mintable); - admin_address->Append(TO_STD_OPTIONAL(master.admin_address)); + admin_address->Append(master.admin_address); if (master.jetton_content) { jetton_content->Append(master.jetton_content.value()); } else { @@ -1656,10 +1665,10 @@ void InsertBatchClickhouse::insert_jettons(clickhouse::Client &client) { for (const auto& task : insert_tasks_) { for (const auto& transfer : task.parsed_block_->get_events()) { auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); - auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : td::optional{}; + auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; std::string jetton_master_address; { @@ -1684,13 +1693,13 @@ void InsertBatchClickhouse::insert_jettons(clickhouse::Client &client) { jetton_wallet->Append(transfer.jetton_wallet); jetton_master->Append(jetton_master_address); response_destination->Append(transfer.response_destination); - custom_payload->Append(TO_STD_OPTIONAL(custom_payload_boc)); + custom_payload->Append(custom_payload_boc); if (transfer.forward_ton_amount.not_null()) { forward_ton_amount->Append(str2int128(transfer.forward_ton_amount->to_dec_string())); } else { forward_ton_amount->Append(Int128(-1)); } - forward_payload->Append(TO_STD_OPTIONAL(forward_payload_boc)); + forward_payload->Append(forward_payload_boc); } } block.AppendColumn("transaction_hash", transaction_hash); @@ -1727,7 +1736,7 @@ void InsertBatchClickhouse::insert_jettons(clickhouse::Client &client) { for (const auto& task : insert_tasks_) { for (const auto& burn : task.parsed_block_->get_events()) { auto custom_payload_boc_r = convert::to_bytes(burn.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; std::string jetton_master_address; { @@ -1751,7 +1760,7 @@ void InsertBatchClickhouse::insert_jettons(clickhouse::Client &client) { amount->Append(Int128(-1)); } response_destination->Append(burn.response_destination); - custom_payload->Append(TO_STD_OPTIONAL(custom_payload_boc)); + custom_payload->Append(custom_payload_boc); } } block.AppendColumn("transaction_hash", transaction_hash); diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index f26e636f..c720c39f 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -7,6 +7,7 @@ #define TO_SQL_OPTIONAL(x) ((x) ? std::to_string(x.value()) : "NULL") #define TO_SQL_OPTIONAL_BOOL(x) ((x) ? ((x.value()) ? "TRUE" : "FALSE") : "NULL") #define TO_SQL_OPTIONAL_STRING(x, txn) ((x) ? (txn.quote(x.value())) : "NULL") +#define TO_SQL_OPTIONAL_REFINT(x) ((x) ? (x.value()->to_dec_string()) : "NULL") std::string content_to_json_string(const std::map &content) { td::JsonBuilder jetton_content_json; @@ -22,6 +23,17 @@ std::string content_to_json_string(const std::map &con return jetton_content_json.string_builder().as_cslice().str(); } +std::string extra_currencies_to_json_string(const std::map &extra_currencies) { + td::JsonBuilder extra_currencies_json; + auto obj = extra_currencies_json.enter_object(); + for (auto ¤cy : extra_currencies) { + obj(std::to_string(currency.first), currency.second->to_dec_string()); + } + obj.leave(); + + return extra_currencies_json.string_builder().as_cslice().str(); +} + std::string InsertManagerPostgres::Credential::get_connection_string(std::string dbname) const { if ((dbname.length() == 0) && this->dbname.length()) { @@ -174,257 +186,6 @@ std::string InsertBatchPostgres::stringify(schema::Trace::State state) { UNREACHABLE(); } - -// std::string InsertBatchPostgres::jsonify(const schema::SplitMergeInfo& info) { -// auto jb = td::JsonBuilder(); -// auto c = jb.enter_object(); -// c("cur_shard_pfx_len", static_cast(info.cur_shard_pfx_len)); -// c("acc_split_depth", static_cast(info.acc_split_depth)); -// c("this_addr", info.this_addr.to_hex()); -// c("sibling_addr", info.sibling_addr.to_hex()); -// c.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(const schema::StorageUsedShort& s) { -// auto jb = td::JsonBuilder(); -// auto c = jb.enter_object(); -// c("cells", std::to_string(s.cells)); -// c("bits", std::to_string(s.bits)); -// c.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(const schema::TrStoragePhase& s) { -// auto jb = td::JsonBuilder(); -// auto c = jb.enter_object(); -// c("storage_fees_collected", std::to_string(s.storage_fees_collected)); -// if (s.storage_fees_due) { -// c("storage_fees_due", std::to_string(*(s.storage_fees_due))); -// } -// c("status_change", stringify(s.status_change)); -// c.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(const schema::TrCreditPhase& c) { -// auto jb = td::JsonBuilder(); -// auto cc = jb.enter_object(); -// if (c.due_fees_collected) { -// cc("due_fees_collected", std::to_string(*(c.due_fees_collected))); -// } -// cc("credit", std::to_string(c.credit)); -// cc.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(const schema::TrActionPhase& action) { -// auto jb = td::JsonBuilder(); -// auto c = jb.enter_object(); -// c("success", td::JsonBool(action.success)); -// c("valid", td::JsonBool(action.valid)); -// c("no_funds", td::JsonBool(action.no_funds)); -// c("status_change", stringify(action.status_change)); -// if (action.total_fwd_fees) { -// c("total_fwd_fees", std::to_string(*(action.total_fwd_fees))); -// } -// if (action.total_action_fees) { -// c("total_action_fees", std::to_string(*(action.total_action_fees))); -// } -// c("result_code", action.result_code); -// if (action.result_arg) { -// c("result_arg", *(action.result_arg)); -// } -// c("tot_actions", action.tot_actions); -// c("spec_actions", action.spec_actions); -// c("skipped_actions", action.skipped_actions); -// c("msgs_created", action.msgs_created); -// c("action_list_hash", td::base64_encode(action.action_list_hash.as_slice())); -// c("tot_msg_size", td::JsonRaw(jsonify(action.tot_msg_size))); -// c.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(const schema::TrBouncePhase& bounce) { -// auto jb = td::JsonBuilder(); -// auto c = jb.enter_object(); -// if (std::holds_alternative(bounce)) { -// c("type", "negfunds"); -// } else if (std::holds_alternative(bounce)) { -// const auto& nofunds = std::get(bounce); -// c("type", "nofunds"); -// c("msg_size", td::JsonRaw(jsonify(nofunds.msg_size))); -// c("req_fwd_fees", std::to_string(nofunds.req_fwd_fees)); -// } else if (std::holds_alternative(bounce)) { -// const auto& ok = std::get(bounce); -// c("type", "ok"); -// c("msg_size", td::JsonRaw(jsonify(ok.msg_size))); -// c("msg_fees", std::to_string(ok.msg_fees)); -// c("fwd_fees", std::to_string(ok.fwd_fees)); -// } -// c.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(const schema::TrComputePhase& compute) { -// auto jb = td::JsonBuilder(); -// auto c = jb.enter_object(); -// if (std::holds_alternative(compute)) { -// c("type", "skipped"); -// c("skip_reason", stringify(std::get(compute).reason)); -// } else if (std::holds_alternative(compute)) { -// c("type", "vm"); -// auto& computed = std::get(compute); -// c("success", td::JsonBool(computed.success)); -// c("msg_state_used", td::JsonBool(computed.msg_state_used)); -// c("account_activated", td::JsonBool(computed.account_activated)); -// c("gas_fees", std::to_string(computed.gas_fees)); -// c("gas_used",std::to_string(computed.gas_used)); -// c("gas_limit", std::to_string(computed.gas_limit)); -// if (computed.gas_credit) { -// c("gas_credit", std::to_string(*(computed.gas_credit))); -// } -// c("mode", computed.mode); -// c("exit_code", computed.exit_code); -// if (computed.exit_arg) { -// c("exit_arg", *(computed.exit_arg)); -// } -// c("vm_steps", static_cast(computed.vm_steps)); -// c("vm_init_state_hash", td::base64_encode(computed.vm_init_state_hash.as_slice())); -// c("vm_final_state_hash", td::base64_encode(computed.vm_final_state_hash.as_slice())); -// } -// c.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(schema::TransactionDescr descr) { -// char tmp[10000]; // Adjust the size if needed -// td::StringBuilder sb(td::MutableSlice{tmp, sizeof(tmp)}); -// td::JsonBuilder jb(std::move(sb)); - -// auto obj = jb.enter_object(); -// if (std::holds_alternative(descr)) { -// const auto& ord = std::get(descr); -// obj("type", "ord"); -// obj("credit_first", td::JsonBool(ord.credit_first)); -// obj("storage_ph", td::JsonRaw(jsonify(ord.storage_ph))); -// obj("credit_ph", td::JsonRaw(jsonify(ord.credit_ph))); -// obj("compute_ph", td::JsonRaw(jsonify(ord.compute_ph))); -// if (ord.action.has_value()) { -// obj("action", td::JsonRaw(jsonify(ord.action.value()))); -// } -// obj("aborted", td::JsonBool(ord.aborted)); -// if (ord.bounce.has_value()) { -// obj("bounce", td::JsonRaw(jsonify(ord.bounce.value()))); -// } -// obj("destroyed", td::JsonBool(ord.destroyed)); -// obj.leave(); -// } -// else if (std::holds_alternative(descr)) { -// const auto& storage = std::get(descr); -// obj("type", "storage"); -// obj("storage_ph", td::JsonRaw(jsonify(storage.storage_ph))); -// obj.leave(); -// } -// else if (std::holds_alternative(descr)) { -// const auto& tt = std::get(descr); -// obj("type", "tick_tock"); -// obj("is_tock", td::JsonBool(tt.is_tock)); -// obj("storage_ph", td::JsonRaw(jsonify(tt.storage_ph))); -// obj("compute_ph", td::JsonRaw(jsonify(tt.compute_ph))); -// if (tt.action.has_value()) { -// obj("action", td::JsonRaw(jsonify(tt.action.value()))); -// } -// obj("aborted", td::JsonBool(tt.aborted)); -// obj("destroyed", td::JsonBool(tt.destroyed)); -// obj.leave(); -// } -// else if (std::holds_alternative(descr)) { -// const auto& split = std::get(descr); -// obj("type", "split_prepare"); -// obj("split_info", td::JsonRaw(jsonify(split.split_info))); -// if (split.storage_ph.has_value()) { -// obj("storage_ph", td::JsonRaw(jsonify(split.storage_ph.value()))); -// } -// obj("compute_ph", td::JsonRaw(jsonify(split.compute_ph))); -// if (split.action.has_value()) { -// obj("action", td::JsonRaw(jsonify(split.action.value()))); -// } -// obj("aborted", td::JsonBool(split.aborted)); -// obj("destroyed", td::JsonBool(split.destroyed)); -// obj.leave(); -// } -// else if (std::holds_alternative(descr)) { -// const auto& split = std::get(descr); -// obj("type", "split_install"); -// obj("split_info", td::JsonRaw(jsonify(split.split_info))); -// obj("installed", td::JsonBool(split.installed)); -// obj.leave(); -// } -// else if (std::holds_alternative(descr)) { -// const auto& merge = std::get(descr); -// obj("type", "merge_prepare"); -// obj("split_info", td::JsonRaw(jsonify(merge.split_info))); -// obj("storage_ph", td::JsonRaw(jsonify(merge.storage_ph))); -// obj("aborted", td::JsonBool(merge.aborted)); -// obj.leave(); -// } -// else if (std::holds_alternative(descr)) { -// const auto& merge = std::get(descr); -// obj("type", "merge_install"); -// obj("split_info", td::JsonRaw(jsonify(merge.split_info))); -// if (merge.storage_ph.has_value()) { -// obj("storage_ph", td::JsonRaw(jsonify(merge.storage_ph.value()))); -// } -// if (merge.credit_ph.has_value()) { -// obj("credit_ph", td::JsonRaw(jsonify(merge.credit_ph.value()))); -// } -// obj("compute_ph", td::JsonRaw(jsonify(merge.compute_ph))); -// if (merge.action.has_value()) { -// obj("action", td::JsonRaw(jsonify(merge.action.value()))); -// } -// obj("aborted", td::JsonBool(merge.aborted)); -// obj("destroyed", td::JsonBool(merge.destroyed)); -// obj.leave(); -// } - -// return jb.string_builder().as_cslice().str(); -// } - - -// std::string InsertBatchPostgres::jsonify(const schema::BlockReference& block_ref) { -// td::JsonBuilder jb; -// auto obj = jb.enter_object(); - -// obj("workchain", td::JsonInt(block_ref.workchain)); -// obj("shard", td::JsonLong(block_ref.shard)); -// obj("seqno", td::JsonInt(block_ref.seqno)); -// obj.leave(); - -// return jb.string_builder().as_cslice().str(); -// } - - - -// std::string InsertBatchPostgres::jsonify(const std::vector& prev_blocks) { -// td::JsonBuilder jb; -// auto obj = jb.enter_array(); - -// for (auto & p : prev_blocks) { -// obj.enter_value() << td::JsonRaw(jsonify(p)); -// } -// obj.leave(); -// return jb.string_builder().as_cslice().str(); -// } - - std::string InsertBatchPostgres::insert_blocks(pqxx::work &txn) { std::ostringstream query; query << "INSERT INTO blocks (workchain, shard, seqno, root_hash, file_hash, mc_block_workchain, " @@ -534,31 +295,15 @@ std::string InsertBatchPostgres::insert_shard_state(pqxx::work &txn) { return ""; } -template -std::string to_int64(std::optional value) { - return ((value) ? std::to_string(static_cast(value.value())) : "NULL"); -} - -template -std::string to_int64(td::optional value) { - return ((value) ? std::to_string(static_cast(value.value())) : "NULL"); -} - -template -std::string to_int64(T value) { - return std::to_string(static_cast(value)); -} - - std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { std::ostringstream query; query << "INSERT INTO transactions (account, hash, lt, block_workchain, block_shard, block_seqno, " "mc_block_seqno, trace_id, prev_trans_hash, prev_trans_lt, now, " - "orig_status, end_status, total_fees, account_state_hash_before, " - "account_state_hash_after, descr, aborted, destroyed, " + "orig_status, end_status, total_fees, total_fees_extra_currencies, " + "account_state_hash_before, account_state_hash_after, descr, aborted, destroyed, " "credit_first, is_tock, installed, storage_fees_collected, " "storage_fees_due, storage_status_change, credit_due_fees_collected, " - "credit, compute_skipped, skipped_reason, compute_success, " + "credit, credit_extra_currencies, compute_skipped, skipped_reason, compute_success, " "compute_msg_state_used, compute_account_activated, compute_gas_fees, " "compute_gas_used, compute_gas_limit, compute_gas_credit, compute_mode, " "compute_exit_code, compute_exit_arg, compute_vm_steps, " @@ -572,19 +317,20 @@ std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { "split_info_acc_split_depth, split_info_this_addr, split_info_sibling_addr) VALUES "; auto store_storage_ph = [&](const schema::TrStoragePhase& storage_ph) { - query << to_int64(storage_ph.storage_fees_collected) << "," - << to_int64(storage_ph.storage_fees_due) << "," + query << storage_ph.storage_fees_collected << "," + << TO_SQL_OPTIONAL_REFINT(storage_ph.storage_fees_due) << "," << txn.quote(stringify(storage_ph.status_change)) << ","; }; auto store_empty_storage_ph = [&]() { query << "NULL, NULL, NULL,"; }; auto store_credit_ph = [&](const schema::TrCreditPhase& credit_ph) { - query << to_int64(credit_ph.due_fees_collected) << "," - << to_int64(credit_ph.credit) << ","; + query << TO_SQL_OPTIONAL_REFINT(credit_ph.due_fees_collected) << "," + << credit_ph.credit.grams << "," + << txn.quote(extra_currencies_to_json_string(credit_ph.credit.extra_currencies)) << ","; }; auto store_empty_credit_ph = [&]() { - query << "NULL,NULL,"; + query << "NULL,NULL,NULL,"; }; auto store_compute_ph = [&](const schema::TrComputePhase& compute_ph) { if (auto* v = std::get_if(&compute_ph)) { @@ -599,10 +345,10 @@ std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { << TO_SQL_BOOL(v->success) << "," << TO_SQL_BOOL(v->msg_state_used) << "," << TO_SQL_BOOL(v->account_activated) << "," - << to_int64(v->gas_fees) << "," - << to_int64(v->gas_used) << "," - << to_int64(v->gas_limit) << "," - << to_int64(v->gas_credit) << "," + << v->gas_used << "," + << v->gas_fees << "," + << v->gas_limit << "," + << TO_SQL_OPTIONAL(v->gas_credit) << "," << std::to_string(v->mode) << "," << v->exit_code << "," << TO_SQL_OPTIONAL(v->exit_arg) << "," @@ -633,8 +379,8 @@ std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { << TO_SQL_BOOL(action.valid) << "," << TO_SQL_BOOL(action.no_funds) << "," << txn.quote(stringify(action.status_change)) << "," - << to_int64(action.total_fwd_fees) << "," - << to_int64(action.total_action_fees) << "," + << TO_SQL_OPTIONAL_REFINT(action.total_fwd_fees) << "," + << TO_SQL_OPTIONAL_REFINT(action.total_action_fees) << "," << action.result_code << "," << TO_SQL_OPTIONAL(action.result_arg) << "," << action.tot_actions << "," @@ -642,8 +388,8 @@ std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { << action.skipped_actions << "," << action.msgs_created << "," << txn.quote(td::base64_encode(action.action_list_hash.as_slice())) << "," - << to_int64(action.tot_msg_size.cells) << "," - << to_int64(action.tot_msg_size.bits) << ","; + << action.tot_msg_size.cells << "," + << action.tot_msg_size.bits << ","; }; auto store_empty_action_ph = [&]() { query << "NULL," @@ -668,17 +414,17 @@ std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { << "NULL,NULL,NULL,NULL,NULL,"; } else if (auto* v = std::get_if(&bounce)) { query << "'nofunds'," - << to_int64(v->msg_size.cells) << "," - << to_int64(v->msg_size.bits) << "," - << to_int64(v->req_fwd_fees) << "," + << v->msg_size.cells << "," + << v->msg_size.bits << "," + << v->req_fwd_fees << "," << "NULL,NULL,"; } else if (auto* v = std::get_if(&bounce)) { query << "'ok'," - << to_int64(v->msg_size.cells) << "," - << to_int64(v->msg_size.bits) << "," + << v->msg_size.cells << "," + << v->msg_size.bits << "," << "NULL," - << to_int64(v->msg_fees) << "," - << to_int64(v->fwd_fees) << ","; + << v->msg_fees << "," + << v->fwd_fees << ","; } }; auto store_empty_bounce_ph = [&]() { @@ -717,7 +463,8 @@ std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { << transaction.now << "," << txn.quote(stringify(transaction.orig_status)) << "," << txn.quote(stringify(transaction.end_status)) << "," - << transaction.total_fees << "," + << transaction.total_fees.grams << "," + << txn.quote(extra_currencies_to_json_string(transaction.total_fees.extra_currencies)) << "," << txn.quote(td::base64_encode(transaction.account_state_hash_before.as_slice())) << "," << txn.quote(td::base64_encode(transaction.account_state_hash_after.as_slice())) << ","; // insert description @@ -870,7 +617,7 @@ std::string InsertBatchPostgres::insert_messages(pqxx::work &txn) { { std::ostringstream query; query << "INSERT INTO messages (tx_hash, tx_lt, msg_hash, direction, trace_id, source, " - "destination, value, fwd_fee, ihr_fee, created_lt, " + "destination, value, value_extra_currencies, fwd_fee, ihr_fee, created_lt, " "created_at, opcode, ihr_disabled, bounce, bounced, " "import_fee, body_hash, init_state_hash) VALUES "; bool is_first = true; @@ -880,6 +627,17 @@ std::string InsertBatchPostgres::insert_messages(pqxx::work &txn) { } else { query << ", "; } + // msg.import_fee is defined by user and can be too large for bigint, so we need to check it + // and if it is too large, we will insert NULL. + // TODO: change bigint to numeric + std::optional import_fee_val; + if (msg.import_fee) { + import_fee_val = msg.import_fee.value()->to_long(); + if (import_fee_val.value() == (~0ULL << 63)) { + LOG(WARNING) << "Import fee of msg " << msg.hash.to_hex() << " is too large for bigint: " << msg.import_fee.value(); + import_fee_val = std::nullopt; + } + } query << "(" << txn.quote(td::base64_encode(tx.hash.as_slice())) << "," << tx.lt << "," @@ -888,16 +646,17 @@ std::string InsertBatchPostgres::insert_messages(pqxx::work &txn) { << txn.quote(td::base64_encode(msg.trace_id.as_slice())) << "," << TO_SQL_OPTIONAL_STRING(msg.source, txn) << "," << TO_SQL_OPTIONAL_STRING(msg.destination, txn) << "," - << to_int64(msg.value) << "," - << to_int64(msg.fwd_fee) << "," - << to_int64(msg.ihr_fee) << "," - << to_int64(msg.created_lt) << "," + << (msg.value ? msg.value->grams->to_dec_string() : "NULL") << "," + << (msg.value ? txn.quote(extra_currencies_to_json_string(msg.value->extra_currencies)) : "NULL") << "," + << TO_SQL_OPTIONAL_REFINT(msg.fwd_fee) << "," + << TO_SQL_OPTIONAL_REFINT(msg.ihr_fee) << "," + << TO_SQL_OPTIONAL(msg.created_lt) << "," << TO_SQL_OPTIONAL(msg.created_at) << "," << TO_SQL_OPTIONAL(msg.opcode) << "," << TO_SQL_OPTIONAL_BOOL(msg.ihr_disabled) << "," << TO_SQL_OPTIONAL_BOOL(msg.bounce) << "," << TO_SQL_OPTIONAL_BOOL(msg.bounced) << "," - << to_int64(msg.import_fee) << "," + << TO_SQL_OPTIONAL(import_fee_val) << "," << txn.quote(td::base64_encode(msg.body->get_hash().as_slice())) << "," << (msg.init_state.not_null() ? txn.quote(td::base64_encode(msg.init_state->get_hash().as_slice())) : "NULL") << ")"; @@ -977,7 +736,7 @@ std::string InsertBatchPostgres::insert_messages(pqxx::work &txn) { std::string InsertBatchPostgres::insert_account_states(pqxx::work &txn) { std::ostringstream query; - query << "INSERT INTO account_states (hash, account, balance, account_status, frozen_hash, code_hash, data_hash) VALUES "; + query << "INSERT INTO account_states (hash, account, balance, balance_extra_currencies, account_status, frozen_hash, code_hash, data_hash) VALUES "; bool is_first = true; for (const auto& task : insert_tasks_) { for (const auto& account_state : task.parsed_block_->account_states_) { @@ -1005,7 +764,8 @@ std::string InsertBatchPostgres::insert_account_states(pqxx::work &txn) { query << "(" << txn.quote(td::base64_encode(account_state.hash.as_slice())) << "," << txn.quote(convert::to_raw_address(account_state.account)) << "," - << account_state.balance << "," + << account_state.balance.grams << "," + << txn.quote(extra_currencies_to_json_string(account_state.balance.extra_currencies)) << "," << txn.quote(account_state.account_status) << "," << TO_SQL_OPTIONAL_STRING(frozen_hash, txn) << "," << TO_SQL_OPTIONAL_STRING(code_hash, txn) << "," @@ -1044,7 +804,7 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { for (auto i = latest_account_states.begin(); i != latest_account_states.end(); ++i) { auto& account_state = i->second; if (is_first) { - query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, " + query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, balance_extra_currencies, " "account_status, timestamp, last_trans_hash, last_trans_lt, " "frozen_hash, data_hash, code_hash, " "data_boc, code_boc) VALUES "; @@ -1091,11 +851,12 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { << txn.quote(convert::to_raw_address(account_state.account)) << "," << "NULL," << txn.quote(td::base64_encode(account_state.hash.as_slice())) << "," - << account_state.balance << "," + << account_state.balance.grams << "," + << txn.quote(extra_currencies_to_json_string(account_state.balance.extra_currencies)) << "," << txn.quote(account_state.account_status) << "," << account_state.timestamp << "," << txn.quote(td::base64_encode(account_state.last_trans_hash.as_slice())) << "," - << to_int64(account_state.last_trans_lt) << "," + << account_state.last_trans_lt << "," << TO_SQL_OPTIONAL_STRING(frozen_hash, txn) << "," << TO_SQL_OPTIONAL_STRING(data_hash, txn) << "," << TO_SQL_OPTIONAL_STRING(code_hash, txn) << "," @@ -1107,6 +868,7 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { << "account_friendly = EXCLUDED.account_friendly, " << "hash = EXCLUDED.hash, " << "balance = EXCLUDED.balance, " + << "balance_extra_currencies = EXCLUDED.balance_extra_currencies, " << "account_status = EXCLUDED.account_status, " << "timestamp = EXCLUDED.timestamp, " << "last_trans_hash = EXCLUDED.last_trans_hash, " @@ -1600,10 +1362,10 @@ std::string InsertBatchPostgres::insert_jetton_transfers(pqxx::work &txn) { query << ", "; } auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); - auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : td::optional{}; + auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; query << "(" << txn.quote(td::base64_encode(transfer.transaction_hash.as_slice())) << "," @@ -1648,7 +1410,7 @@ std::string InsertBatchPostgres::insert_jetton_burns(pqxx::work &txn) { } auto custom_payload_boc_r = convert::to_bytes(burn.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; query << "(" << txn.quote(td::base64_encode(burn.transaction_hash.as_slice())) << "," @@ -1689,10 +1451,10 @@ std::string InsertBatchPostgres::insert_nft_transfers(pqxx::work &txn) { query << ", "; } auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); - auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : td::optional{}; + auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; query << "(" << txn.quote(td::base64_encode(transfer.transaction_hash.as_slice())) << "," @@ -1972,6 +1734,7 @@ void InsertManagerPostgres::start_up() { "orig_status account_status_type, " "end_status account_status_type, " "total_fees bigint, " + "total_fees_extra_currencies jsonb, " "account_state_hash_before tonhash, " "account_state_hash_after tonhash, " "descr descr_type, " @@ -1985,6 +1748,7 @@ void InsertManagerPostgres::start_up() { "storage_status_change status_change_type, " "credit_due_fees_collected bigint, " "credit bigint, " + "credit_extra_currencies jsonb, " "compute_skipped boolean, " "skipped_reason skipped_reason_type, " "compute_success boolean, " @@ -2039,6 +1803,7 @@ void InsertManagerPostgres::start_up() { "source tonaddr, " "destination tonaddr, " "value bigint, " + "value_extra_currencies jsonb, " "fwd_fee bigint, " "ihr_fee bigint, " "created_lt bigint, " @@ -2065,6 +1830,7 @@ void InsertManagerPostgres::start_up() { "hash tonhash not null primary key, " "account tonaddr, " "balance bigint, " + "balance_extra_currencies jsonb, " "account_status account_status_type, " "frozen_hash tonhash, " "data_hash tonhash, " @@ -2079,6 +1845,7 @@ void InsertManagerPostgres::start_up() { "account_friendly tonaddr, " "hash tonhash not null, " "balance bigint, " + "balance_extra_currencies jsonb, " "account_status account_status_type, " "timestamp integer, " "last_trans_hash tonhash, " @@ -2477,13 +2244,18 @@ void InsertManagerPostgres::start_up() { std::string query = ""; - // some necessary indexes query += ( "alter table jetton_wallets add column if not exists mintless_is_claimed boolean;\n" "alter table jetton_wallets add column if not exists mintless_amount numeric;\n" "alter table jetton_wallets add column if not exists mintless_start_from bigint;\n" "alter table jetton_wallets add column if not exists mintless_expire_at bigint;\n" "alter table mintless_jetton_masters add column if not exists custom_payload_api_uri varchar[];\n" + + "alter table transactions add column if not exists total_fees_extra_currencies jsonb;\n" + "alter table transactions add column if not exists credit_extra_currencies jsonb;\n" + "alter table messages add column if not exists value_extra_currencies jsonb;\n" + "alter table account_states add column if not exists balance_extra_currencies jsonb;\n" + "alter table latest_account_states add column if not exists balance_extra_currencies jsonb;\n" ); LOG(DEBUG) << query; diff --git a/ton-index-postgres/src/InsertManagerPostgres.cpp b/ton-index-postgres/src/InsertManagerPostgres.cpp index 52d5e898..bd66e152 100644 --- a/ton-index-postgres/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres/src/InsertManagerPostgres.cpp @@ -213,9 +213,9 @@ std::string InsertBatchPostgres::jsonify(const schema::StorageUsedShort& s) { std::string InsertBatchPostgres::jsonify(const schema::TrStoragePhase& s) { auto jb = td::JsonBuilder(); auto c = jb.enter_object(); - c("storage_fees_collected", std::to_string(s.storage_fees_collected)); + c("storage_fees_collected", s.storage_fees_collected->to_dec_string()); if (s.storage_fees_due) { - c("storage_fees_due", std::to_string(*(s.storage_fees_due))); + c("storage_fees_due", s.storage_fees_due.value()->to_dec_string()); } c("status_change", stringify(s.status_change)); c.leave(); @@ -227,9 +227,9 @@ std::string InsertBatchPostgres::jsonify(const schema::TrCreditPhase& c) { auto jb = td::JsonBuilder(); auto cc = jb.enter_object(); if (c.due_fees_collected) { - cc("due_fees_collected", std::to_string(*(c.due_fees_collected))); + cc("due_fees_collected", c.due_fees_collected.value()->to_dec_string()); } - cc("credit", std::to_string(c.credit)); + cc("credit", c.credit.grams->to_dec_string()); cc.leave(); return jb.string_builder().as_cslice().str(); } @@ -243,10 +243,10 @@ std::string InsertBatchPostgres::jsonify(const schema::TrActionPhase& action) { c("no_funds", td::JsonBool(action.no_funds)); c("status_change", stringify(action.status_change)); if (action.total_fwd_fees) { - c("total_fwd_fees", std::to_string(*(action.total_fwd_fees))); + c("total_fwd_fees", action.total_fwd_fees.value()->to_dec_string()); } if (action.total_action_fees) { - c("total_action_fees", std::to_string(*(action.total_action_fees))); + c("total_action_fees", action.total_action_fees.value()->to_dec_string()); } c("result_code", action.result_code); if (action.result_arg) { @@ -272,13 +272,13 @@ std::string InsertBatchPostgres::jsonify(const schema::TrBouncePhase& bounce) { const auto& nofunds = std::get(bounce); c("type", "nofunds"); c("msg_size", td::JsonRaw(jsonify(nofunds.msg_size))); - c("req_fwd_fees", std::to_string(nofunds.req_fwd_fees)); + c("req_fwd_fees", nofunds.req_fwd_fees->to_dec_string()); } else if (std::holds_alternative(bounce)) { const auto& ok = std::get(bounce); c("type", "ok"); c("msg_size", td::JsonRaw(jsonify(ok.msg_size))); - c("msg_fees", std::to_string(ok.msg_fees)); - c("fwd_fees", std::to_string(ok.fwd_fees)); + c("msg_fees", ok.msg_fees->to_dec_string()); + c("fwd_fees", ok.fwd_fees->to_dec_string()); } c.leave(); return jb.string_builder().as_cslice().str(); @@ -297,7 +297,7 @@ std::string InsertBatchPostgres::jsonify(const schema::TrComputePhase& compute) c("success", td::JsonBool(computed.success)); c("msg_state_used", td::JsonBool(computed.msg_state_used)); c("account_activated", td::JsonBool(computed.account_activated)); - c("gas_fees", std::to_string(computed.gas_fees)); + c("gas_fees", computed.gas_fees->to_dec_string()); c("gas_used",std::to_string(computed.gas_used)); c("gas_limit", std::to_string(computed.gas_limit)); if (computed.gas_credit) { @@ -555,7 +555,7 @@ void InsertBatchPostgres::insert_transactions(pqxx::work &transaction, const std << tx.now << "," << TO_SQL_STRING(stringify(tx.orig_status)) << "," << TO_SQL_STRING(stringify(tx.end_status)) << "," - << tx.total_fees << "," + << tx.total_fees.grams << "," << TO_SQL_STRING(td::base64_encode(tx.account_state_hash_before.as_slice())) << "," << TO_SQL_STRING(td::base64_encode(tx.account_state_hash_after.as_slice())) << "," << "'" << jsonify(tx.description) << "'" // FIXME: remove for production @@ -641,20 +641,31 @@ void InsertBatchPostgres::insert_messages_impl(const std::vector import_fee_val; + if (message.import_fee) { + import_fee_val = message.import_fee.value()->to_long(); + if (import_fee_val.value() == int64_t(~0ULL << 63)) { + LOG(WARNING) << "Import fee of msg " << message.hash.to_hex() << " is too large for bigint: " << message.import_fee.value(); + import_fee_val = std::nullopt; + } + } query << "(" << "'" << td::base64_encode(message.hash.as_slice()) << "'," << (message.source ? "'" + message.source.value() + "'" : "NULL") << "," << (message.destination ? "'" + message.destination.value() + "'" : "NULL") << "," - << (message.value ? std::to_string(message.value.value()) : "NULL") << "," - << (message.fwd_fee ? std::to_string(message.fwd_fee.value()) : "NULL") << "," - << (message.ihr_fee ? std::to_string(message.ihr_fee.value()) : "NULL") << "," + << (message.value ? message.value->grams->to_dec_string() : "NULL") << "," + << (message.fwd_fee ? message.fwd_fee.value()->to_dec_string() : "NULL") << "," + << (message.ihr_fee ? message.ihr_fee.value()->to_dec_string() : "NULL") << "," << (message.created_lt ? std::to_string(message.created_lt.value()) : "NULL") << "," << (message.created_at ? std::to_string(message.created_at.value()) : "NULL") << "," << (message.opcode ? std::to_string(message.opcode.value()) : "NULL") << "," << (message.ihr_disabled ? TO_SQL_BOOL(message.ihr_disabled.value()) : "NULL") << "," << (message.bounce ? TO_SQL_BOOL(message.bounce.value()) : "NULL") << "," << (message.bounced ? TO_SQL_BOOL(message.bounced.value()) : "NULL") << "," - << (message.import_fee ? std::to_string(message.import_fee.value()) : "NULL") << "," + << TO_SQL_OPTIONAL(import_fee_val) << "," << "'" << td::base64_encode(message.body->get_hash().as_slice()) << "'," << (message.init_state.not_null() ? TO_SQL_STRING(td::base64_encode(message.init_state->get_hash().as_slice())) : "NULL") << ")"; @@ -735,7 +746,7 @@ void InsertBatchPostgres::insert_account_states(pqxx::work &transaction, const s query << "(" << TO_SQL_STRING(td::base64_encode(account_state.hash.as_slice())) << "," << TO_SQL_STRING(convert::to_raw_address(account_state.account)) << "," - << account_state.balance << "," + << account_state.balance.grams << "," << TO_SQL_STRING(account_state.account_status) << "," << TO_SQL_OPTIONAL_STRING(frozen_hash) << "," << TO_SQL_OPTIONAL_STRING(code_hash) << "," @@ -791,7 +802,7 @@ void InsertBatchPostgres::insert_latest_account_states(pqxx::work &transaction, query << "(" << TO_SQL_STRING(convert::to_raw_address(account_state.account)) << "," << TO_SQL_STRING(td::base64_encode(account_state.hash.as_slice())) << "," - << account_state.balance << "," + << account_state.balance.grams << "," << account_state.last_trans_lt << "," << account_state.timestamp << "," << TO_SQL_STRING(account_state.account_status) << "," @@ -1038,10 +1049,10 @@ void InsertBatchPostgres::insert_jetton_transfers(pqxx::work &transaction, const query << ", "; } auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); - auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : td::optional{}; + auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; query << "(" << TO_SQL_STRING(td::base64_encode(transfer.transaction_hash.as_slice())) << "," @@ -1079,7 +1090,7 @@ void InsertBatchPostgres::insert_jetton_burns(pqxx::work &transaction, const std } auto custom_payload_boc_r = convert::to_bytes(burn.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; query << "(" << TO_SQL_STRING(td::base64_encode(burn.transaction_hash.as_slice())) << "," @@ -1113,10 +1124,10 @@ void InsertBatchPostgres::insert_nft_transfers(pqxx::work &transaction, const st query << ", "; } auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); - auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : td::optional{}; + auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); - auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : td::optional{}; + auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; query << "(" << TO_SQL_STRING(td::base64_encode(transfer.transaction_hash.as_slice())) << "," diff --git a/ton-smc-scanner/src/PostgreSQLInserter.cpp b/ton-smc-scanner/src/PostgreSQLInserter.cpp index fe90fe60..6c89df0a 100644 --- a/ton-smc-scanner/src/PostgreSQLInserter.cpp +++ b/ton-smc-scanner/src/PostgreSQLInserter.cpp @@ -97,11 +97,12 @@ void PostgreSQLInserter::insert_latest_account_states(pqxx::work &transaction) { if (account_state.data_hash) { data_hash = td::base64_encode(account_state.data_hash.value().as_slice()); } + // TODO: extracurrencies query << "(" << transaction.quote(convert::to_raw_address(account_state.account)) << "," << "NULL," << transaction.quote(td::base64_encode(account_state.hash.as_slice())) << "," - << account_state.balance << "," + << account_state.balance.grams << "," << transaction.quote(account_state.account_status) << "," << account_state.timestamp << "," << transaction.quote(td::base64_encode(account_state.last_trans_hash.as_slice())) << "," diff --git a/ton-trace-emulator/src/Serializer.hpp b/ton-trace-emulator/src/Serializer.hpp index cc302534..07d66251 100644 --- a/ton-trace-emulator/src/Serializer.hpp +++ b/ton-trace-emulator/src/Serializer.hpp @@ -199,6 +199,23 @@ struct TraceNode { MSGPACK_DEFINE(transaction, emulated); }; +td::Result to_balance(vm::CellSlice& balance_slice) { + auto balance = block::tlb::t_Grams.as_integer_skip(balance_slice); + if (balance.is_null()) { + return td::Status::Error("Failed to unpack balance"); + } + auto res = balance->to_long(); + if (res == td::int64(~0ULL << 63)) { + return td::Status::Error("Failed to unpack balance (2)"); + } + return res; +} + +td::Result to_balance(td::Ref balance_ref) { + vm::CellSlice balance_slice = *balance_ref; + return to_balance(balance_slice); +} + td::Result parse_message(td::Ref msg_cell) { Message msg; msg.hash = msg_cell->get_hash().bits(); @@ -253,11 +270,11 @@ td::Result parse_message(td::Ref msg_cell) { return td::Status::Error("Failed to unpack CommonMsgInfo::int_msg_info"); } - TRY_RESULT_ASSIGN(msg.value, convert::to_balance(msg_info.value)); + TRY_RESULT_ASSIGN(msg.value, to_balance(msg_info.value)); TRY_RESULT_ASSIGN(msg.source, convert::to_raw_address(msg_info.src)); TRY_RESULT_ASSIGN(msg.destination, convert::to_raw_address(msg_info.dest)); - TRY_RESULT_ASSIGN(msg.fwd_fee, convert::to_balance(msg_info.fwd_fee)); - TRY_RESULT_ASSIGN(msg.ihr_fee, convert::to_balance(msg_info.ihr_fee)); + TRY_RESULT_ASSIGN(msg.fwd_fee, to_balance(msg_info.fwd_fee)); + TRY_RESULT_ASSIGN(msg.ihr_fee, to_balance(msg_info.ihr_fee)); msg.created_lt = msg_info.created_lt; msg.created_at = msg_info.created_at; msg.bounce = msg_info.bounce; @@ -273,7 +290,7 @@ td::Result parse_message(td::Ref msg_cell) { // msg.source = null, because it is external TRY_RESULT_ASSIGN(msg.destination, convert::to_raw_address(msg_info.dest)) - TRY_RESULT_ASSIGN(msg.import_fee, convert::to_balance(msg_info.import_fee)); + TRY_RESULT_ASSIGN(msg.import_fee, to_balance(msg_info.import_fee)); return msg; } case block::gen::CommonMsgInfo::ext_out_msg_info: { @@ -299,10 +316,10 @@ td::Result parse_tr_storage_phase(vm::CellSlice& cs) { return td::Status::Error("Failed to unpack TrStoragePhase"); } TrStoragePhase phase; - TRY_RESULT_ASSIGN(phase.storage_fees_collected, convert::to_balance(phase_data.storage_fees_collected)); + TRY_RESULT_ASSIGN(phase.storage_fees_collected, to_balance(phase_data.storage_fees_collected)); auto& storage_fees_due = phase_data.storage_fees_due.write(); if (storage_fees_due.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(phase.storage_fees_due, convert::to_balance(storage_fees_due)); + TRY_RESULT_ASSIGN(phase.storage_fees_due, to_balance(storage_fees_due)); } phase.status_change = static_cast(phase_data.status_change); return phase; @@ -316,9 +333,9 @@ td::Result parse_tr_credit_phase(vm::CellSlice& cs) { TrCreditPhase phase; auto& due_fees_collected = phase_data.due_fees_collected.write(); if (due_fees_collected.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(phase.due_fees_collected, convert::to_balance(due_fees_collected)); + TRY_RESULT_ASSIGN(phase.due_fees_collected, to_balance(due_fees_collected)); } - TRY_RESULT_ASSIGN(phase.credit, convert::to_balance(phase_data.credit)); + TRY_RESULT_ASSIGN(phase.credit, to_balance(phase_data.credit)); return phase; } @@ -333,7 +350,7 @@ td::Result parse_tr_compute_phase(vm::CellSlice& cs) { res.success = compute_vm.success; res.msg_state_used = compute_vm.msg_state_used; res.account_activated = compute_vm.account_activated; - TRY_RESULT_ASSIGN(res.gas_fees, convert::to_balance(compute_vm.gas_fees)); + TRY_RESULT_ASSIGN(res.gas_fees, to_balance(compute_vm.gas_fees)); res.gas_used = block::tlb::t_VarUInteger_7.as_uint(*compute_vm.r1.gas_used); res.gas_limit = block::tlb::t_VarUInteger_7.as_uint(*compute_vm.r1.gas_limit); auto& gas_credit = compute_vm.r1.gas_credit.write(); @@ -383,11 +400,11 @@ td::Result parse_tr_action_phase(vm::CellSlice& cs) { res.status_change = static_cast(info.status_change); auto& total_fwd_fees = info.total_fwd_fees.write(); if (total_fwd_fees.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(res.total_fwd_fees, convert::to_balance(info.total_fwd_fees)); + TRY_RESULT_ASSIGN(res.total_fwd_fees, to_balance(info.total_fwd_fees)); } auto& total_action_fees = info.total_action_fees.write(); if (total_action_fees.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(res.total_action_fees, convert::to_balance(info.total_action_fees)); + TRY_RESULT_ASSIGN(res.total_action_fees, to_balance(info.total_action_fees)); } res.result_code = info.result_code; auto& result_arg = info.result_arg.write(); @@ -420,7 +437,7 @@ td::Result parse_tr_bounce_phase(vm::CellSlice& cs) { } TrBouncePhase_nofunds res; TRY_RESULT_ASSIGN(res.msg_size, parse_storage_used_short(nofunds.msg_size.write())); - TRY_RESULT_ASSIGN(res.req_fwd_fees, convert::to_balance(nofunds.req_fwd_fees)); + TRY_RESULT_ASSIGN(res.req_fwd_fees, to_balance(nofunds.req_fwd_fees)); return res; } case block::gen::TrBouncePhase::tr_phase_bounce_ok: { @@ -430,8 +447,8 @@ td::Result parse_tr_bounce_phase(vm::CellSlice& cs) { } TrBouncePhase_ok res; TRY_RESULT_ASSIGN(res.msg_size, parse_storage_used_short(ok.msg_size.write())); - TRY_RESULT_ASSIGN(res.msg_fees, convert::to_balance(ok.msg_fees)); - TRY_RESULT_ASSIGN(res.fwd_fees, convert::to_balance(ok.fwd_fees)); + TRY_RESULT_ASSIGN(res.msg_fees, to_balance(ok.msg_fees)); + TRY_RESULT_ASSIGN(res.fwd_fees, to_balance(ok.fwd_fees)); return res; } default: @@ -507,7 +524,7 @@ td::Result parse_tx(td::Ref root, ton::WorkchainId workch schema_tx.orig_status = static_cast(trans.orig_status); schema_tx.end_status = static_cast(trans.end_status); - TRY_RESULT_ASSIGN(schema_tx.total_fees, convert::to_balance(trans.total_fees)); + TRY_RESULT_ASSIGN(schema_tx.total_fees, to_balance(trans.total_fees)); if (trans.r1.in_msg->prefetch_long(1)) { auto msg = trans.r1.in_msg->prefetch_ref(); diff --git a/tondb-scanner/src/DataParser.cpp b/tondb-scanner/src/DataParser.cpp index 23e850dd..3692d4a6 100644 --- a/tondb-scanner/src/DataParser.cpp +++ b/tondb-scanner/src/DataParser.cpp @@ -124,6 +124,25 @@ schema::Block ParseQuery::parse_block(const td::Ref& root_cell, const return block; } +td::Result ParseQuery::parse_currency_collection(td::Ref csr) { + td::RefInt256 grams; + std::map extra_currencies; + td::Ref extra; + if (!block::unpack_CurrencyCollection(csr, grams, extra)) { + return td::Status::Error(PSLICE() << "Failed to unpack currency collection"); + } + vm::Dictionary extra_currencies_dict{extra, 32}; + auto it = extra_currencies_dict.begin(); + while (!it.eof()) { + auto id = td::BitArray<32>(it.cur_pos()).to_ulong(); + auto value_cs = it.cur_value(); + auto value = block::tlb::t_VarUInteger_32.as_integer(value_cs); + extra_currencies[id] = value; + ++it; + } + return schema::CurrencyCollection{std::move(grams), std::move(extra_currencies)}; +} + td::Result ParseQuery::parse_message(td::Ref msg_cell) { schema::Message msg; msg.hash = msg_cell->get_hash().bits(); @@ -180,11 +199,11 @@ td::Result ParseQuery::parse_message(td::Ref msg_cell return td::Status::Error("Failed to unpack CommonMsgInfo::int_msg_info"); } - TRY_RESULT_ASSIGN(msg.value, convert::to_balance(msg_info.value)); + TRY_RESULT_ASSIGN(msg.value, parse_currency_collection(msg_info.value)); TRY_RESULT_ASSIGN(msg.source, convert::to_raw_address(msg_info.src)); TRY_RESULT_ASSIGN(msg.destination, convert::to_raw_address(msg_info.dest)); - TRY_RESULT_ASSIGN(msg.fwd_fee, convert::to_balance(msg_info.fwd_fee)); - TRY_RESULT_ASSIGN(msg.ihr_fee, convert::to_balance(msg_info.ihr_fee)); + msg.fwd_fee = block::tlb::t_Grams.as_integer_skip(msg_info.fwd_fee.write()); + msg.ihr_fee = block::tlb::t_Grams.as_integer_skip(msg_info.ihr_fee.write()); msg.created_lt = msg_info.created_lt; msg.created_at = msg_info.created_at; msg.bounce = msg_info.bounce; @@ -200,12 +219,7 @@ td::Result ParseQuery::parse_message(td::Ref msg_cell // msg.source = null, because it is external TRY_RESULT_ASSIGN(msg.destination, convert::to_raw_address(msg_info.dest)) - auto import_fee = convert::to_balance(msg_info.import_fee); - if (import_fee.is_error()) { - LOG(ERROR) << "Failed to convert import fee to int64"; - } else { - msg.import_fee = import_fee.move_as_ok(); - } + msg.import_fee = block::tlb::t_Grams.as_integer_skip(msg_info.import_fee.write()); return msg; } @@ -231,10 +245,10 @@ td::Result ParseQuery::parse_tr_storage_phase(vm::CellSl return td::Status::Error("Failed to unpack TrStoragePhase"); } schema::TrStoragePhase phase; - TRY_RESULT_ASSIGN(phase.storage_fees_collected, convert::to_balance(phase_data.storage_fees_collected)); + phase.storage_fees_collected = block::tlb::t_Grams.as_integer_skip(phase_data.storage_fees_collected.write()); auto& storage_fees_due = phase_data.storage_fees_due.write(); if (storage_fees_due.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(phase.storage_fees_due, convert::to_balance(storage_fees_due)); + phase.storage_fees_due = block::tlb::t_Grams.as_integer_skip(storage_fees_due); } phase.status_change = static_cast(phase_data.status_change); return phase; @@ -248,9 +262,10 @@ td::Result ParseQuery::parse_tr_credit_phase(vm::CellSlic schema::TrCreditPhase phase; auto& due_fees_collected = phase_data.due_fees_collected.write(); if (due_fees_collected.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(phase.due_fees_collected, convert::to_balance(due_fees_collected)); + phase.due_fees_collected = block::tlb::t_Grams.as_integer_skip(due_fees_collected); } - TRY_RESULT_ASSIGN(phase.credit, convert::to_balance(phase_data.credit)); + // TRY_RESULT_ASSIGN(phase.credit, convert::to_balance(phase_data.credit)); + TRY_RESULT_ASSIGN(phase.credit, parse_currency_collection(phase_data.credit)); return phase; } @@ -265,7 +280,7 @@ td::Result ParseQuery::parse_tr_compute_phase(vm::CellSl res.success = compute_vm.success; res.msg_state_used = compute_vm.msg_state_used; res.account_activated = compute_vm.account_activated; - TRY_RESULT_ASSIGN(res.gas_fees, convert::to_balance(compute_vm.gas_fees)); + res.gas_fees = block::tlb::t_Grams.as_integer_skip(compute_vm.gas_fees.write()); res.gas_used = block::tlb::t_VarUInteger_7.as_uint(*compute_vm.r1.gas_used); res.gas_limit = block::tlb::t_VarUInteger_7.as_uint(*compute_vm.r1.gas_limit); auto& gas_credit = compute_vm.r1.gas_credit.write(); @@ -315,11 +330,11 @@ td::Result ParseQuery::parse_tr_action_phase(vm::CellSlic res.status_change = static_cast(info.status_change); auto& total_fwd_fees = info.total_fwd_fees.write(); if (total_fwd_fees.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(res.total_fwd_fees, convert::to_balance(info.total_fwd_fees)); + res.total_fwd_fees = block::tlb::t_Grams.as_integer_skip(info.total_fwd_fees.write()); } auto& total_action_fees = info.total_action_fees.write(); if (total_action_fees.fetch_ulong(1) == 1) { - TRY_RESULT_ASSIGN(res.total_action_fees, convert::to_balance(info.total_action_fees)); + res.total_action_fees = block::tlb::t_Grams.as_integer_skip(info.total_action_fees.write()); } res.result_code = info.result_code; auto& result_arg = info.result_arg.write(); @@ -352,7 +367,7 @@ td::Result ParseQuery::parse_tr_bounce_phase(vm::CellSlic } schema::TrBouncePhase_nofunds res; TRY_RESULT_ASSIGN(res.msg_size, parse_storage_used_short(nofunds.msg_size.write())); - TRY_RESULT_ASSIGN(res.req_fwd_fees, convert::to_balance(nofunds.req_fwd_fees)); + res.req_fwd_fees = block::tlb::t_Grams.as_integer_skip(nofunds.req_fwd_fees.write()); return res; } case block::gen::TrBouncePhase::tr_phase_bounce_ok: { @@ -362,8 +377,8 @@ td::Result ParseQuery::parse_tr_bounce_phase(vm::CellSlic } schema::TrBouncePhase_ok res; TRY_RESULT_ASSIGN(res.msg_size, parse_storage_used_short(ok.msg_size.write())); - TRY_RESULT_ASSIGN(res.msg_fees, convert::to_balance(ok.msg_fees)); - TRY_RESULT_ASSIGN(res.fwd_fees, convert::to_balance(ok.fwd_fees)); + res.msg_fees = block::tlb::t_Grams.as_integer_skip(ok.msg_fees.write()); + res.fwd_fees = block::tlb::t_Grams.as_integer_skip(ok.fwd_fees.write()); return res; } default: @@ -573,7 +588,7 @@ td::Result> ParseQuery::parse_transactions(cons schema_tx.orig_status = static_cast(trans.orig_status); schema_tx.end_status = static_cast(trans.end_status); - TRY_RESULT_ASSIGN(schema_tx.total_fees, convert::to_balance(trans.total_fees)); + TRY_RESULT_ASSIGN(schema_tx.total_fees, parse_currency_collection(trans.total_fees)); if (trans.r1.in_msg->prefetch_long(1)) { auto msg = trans.r1.in_msg->prefetch_ref(); @@ -665,7 +680,7 @@ td::Result ParseQuery::parse_none_account(td::Refget_hash().bits(); schema_account.timestamp = gen_utime; schema_account.account_status = "nonexist"; - schema_account.balance = 0; + schema_account.balance = schema::CurrencyCollection{td::RefInt256{true, 0}, {}}; schema_account.last_trans_hash = last_trans_hash; schema_account.last_trans_lt = last_trans_lt; return schema_account; @@ -685,7 +700,7 @@ td::Result ParseQuery::parse_account(td::Ref acc schema_account.hash = account_root->get_hash().bits(); TRY_RESULT(account_addr, convert::to_raw_address(account.addr)); TRY_RESULT_ASSIGN(schema_account.account, block::StdAddress::parse(account_addr)); - TRY_RESULT_ASSIGN(schema_account.balance, convert::to_balance(storage.balance)); + TRY_RESULT_ASSIGN(schema_account.balance, parse_currency_collection(storage.balance)); schema_account.timestamp = gen_utime; schema_account.last_trans_hash = last_trans_hash; schema_account.last_trans_lt = last_trans_lt; diff --git a/tondb-scanner/src/DataParser.h b/tondb-scanner/src/DataParser.h index 30b9e935..2bcfbcb8 100644 --- a/tondb-scanner/src/DataParser.h +++ b/tondb-scanner/src/DataParser.h @@ -20,6 +20,7 @@ class ParseQuery: public td::actor::Actor { schema::Block parse_block(const td::Ref& root_cell, const ton::BlockIdExt& blk_id, block::gen::Block::Record& blk, const block::gen::BlockInfo::Record& info, const block::gen::BlockExtra::Record& extra, td::optional &mc_block); schema::MasterchainBlockShard parse_shard_state(schema::Block mc_block, const ton::BlockIdExt& shard_blk_id); + static td::Result parse_currency_collection(td::Ref csr); td::Result parse_message(td::Ref msg_cell); td::Result parse_tr_storage_phase(vm::CellSlice& cs); td::Result parse_tr_credit_phase(vm::CellSlice& cs); diff --git a/tondb-scanner/src/IndexData.h b/tondb-scanner/src/IndexData.h index 39f18fbe..08412bea 100644 --- a/tondb-scanner/src/IndexData.h +++ b/tondb-scanner/src/IndexData.h @@ -11,6 +11,11 @@ namespace schema { +struct CurrencyCollection { + td::RefInt256 grams; + std::map extra_currencies; +}; + enum AccountStatus { uninit = block::gen::AccountStatus::acc_state_uninit, frozen = block::gen::AccountStatus::acc_state_frozen, @@ -25,14 +30,14 @@ enum AccStatusChange { }; struct TrStoragePhase { - uint64_t storage_fees_collected; - std::optional storage_fees_due; + td::RefInt256 storage_fees_collected; + std::optional storage_fees_due; AccStatusChange status_change; }; struct TrCreditPhase { - std::optional due_fees_collected; - uint64_t credit; + std::optional due_fees_collected; + CurrencyCollection credit; }; enum ComputeSkipReason { @@ -50,7 +55,7 @@ struct TrComputePhase_vm { bool success; bool msg_state_used; bool account_activated; - uint64_t gas_fees; + td::RefInt256 gas_fees; uint64_t gas_used; uint64_t gas_limit; std::optional gas_credit; @@ -75,8 +80,8 @@ struct TrActionPhase { bool valid; bool no_funds; AccStatusChange status_change; - std::optional total_fwd_fees; - std::optional total_action_fees; + std::optional total_fwd_fees; + std::optional total_action_fees; int32_t result_code; std::optional result_arg; uint16_t tot_actions; @@ -92,13 +97,13 @@ struct TrBouncePhase_negfunds { struct TrBouncePhase_nofunds { StorageUsedShort msg_size; - uint64_t req_fwd_fees; + td::RefInt256 req_fwd_fees; }; struct TrBouncePhase_ok { StorageUsedShort msg_size; - uint64_t msg_fees; - uint64_t fwd_fees; + td::RefInt256 msg_fees; + td::RefInt256 fwd_fees; }; using TrBouncePhase = std::variant source; - td::optional destination; - td::optional value; - td::optional fwd_fee; - td::optional ihr_fee; - td::optional created_lt; - td::optional created_at; - td::optional opcode; - td::optional ihr_disabled; - td::optional bounce; - td::optional bounced; - td::optional import_fee; + std::optional source; + std::optional destination; + std::optional value; + std::optional fwd_fee; + std::optional ihr_fee; + std::optional created_lt; + std::optional created_at; + std::optional opcode; + std::optional ihr_disabled; + std::optional bounce; + std::optional bounced; + std::optional import_fee; td::Ref body; std::string body_boc; td::Ref init_state; - td::optional init_state_boc; + std::optional init_state_boc; td::Bits256 trace_id; }; @@ -214,7 +219,7 @@ struct Transaction { std::optional in_msg; std::vector out_msgs; - uint64_t total_fees; + CurrencyCollection total_fees; td::Bits256 account_state_hash_before; td::Bits256 account_state_hash_after; @@ -236,9 +241,9 @@ struct Block { std::string root_hash; std::string file_hash; - td::optional mc_block_workchain; - td::optional mc_block_shard; - td::optional mc_block_seqno; + std::optional mc_block_workchain; + std::optional mc_block_shard; + std::optional mc_block_seqno; int32_t global_id; int32_t version; @@ -258,7 +263,7 @@ struct Block { int32_t min_ref_mc_seqno; int32_t prev_key_block_seqno; int32_t vert_seqno; - td::optional master_ref_seqno; + std::optional master_ref_seqno; std::string rand_seed; std::string created_by; @@ -281,7 +286,7 @@ struct AccountState { block::StdAddress account; std::string account_friendly; // TODO: add account friendly uint32_t timestamp; - uint64_t balance; + CurrencyCollection balance; std::string account_status; // "uninit", "frozen", "active", "nonexist" std::optional frozen_hash; td::Ref code; @@ -354,8 +359,8 @@ struct JettonMasterData { std::string address; td::RefInt256 total_supply; bool mintable; - td::optional admin_address; - td::optional> jetton_content; + std::optional admin_address; + std::optional> jetton_content; vm::CellHash jetton_wallet_code_hash; vm::CellHash data_hash; vm::CellHash code_hash; @@ -437,8 +442,8 @@ struct JettonBurn { struct NFTCollectionData { std::string address; td::RefInt256 next_item_index; - td::optional owner_address; - td::optional> collection_content; + std::optional owner_address; + std::optional> collection_content; vm::CellHash data_hash; vm::CellHash code_hash; uint64_t last_transaction_lt; @@ -464,7 +469,7 @@ struct NFTItemData { td::RefInt256 index; std::string collection_address; std::string owner_address; - td::optional> content; + std::optional> content; uint64_t last_transaction_lt; uint32_t last_transaction_now; vm::CellHash code_hash; diff --git a/tondb-scanner/src/convert-utils.cpp b/tondb-scanner/src/convert-utils.cpp index ed16db3e..fbf91637 100644 --- a/tondb-scanner/src/convert-utils.cpp +++ b/tondb-scanner/src/convert-utils.cpp @@ -68,43 +68,9 @@ std::string convert::to_raw_address(block::StdAddress address) { return std::to_string(address.workchain) + ":" + address.addr.to_hex(); } -td::Result convert::to_balance(vm::CellSlice& balance_slice) { - auto balance = block::tlb::t_Grams.as_integer_skip(balance_slice); - if (balance.is_null()) { - return td::Status::Error("Failed to unpack balance"); - } - auto res = balance->to_long(); - if (res == td::int64(~0ULL << 63)) { - return td::Status::Error("Failed to unpack balance (2)"); - } - return res; -} - -td::Result convert::to_balance(td::Ref balance_ref) { - vm::CellSlice balance_slice = *balance_ref; - return to_balance(balance_slice); -} - -// td::Result convert::to_balance256(vm::CellSlice& balance_slice) { -// auto balance = block::tlb::t_Grams.as_integer_skip(balance_slice); -// if (balance.is_null()) { -// return td::Status::Error("Failed to unpack balance"); -// } -// auto res = balance->to_long(); -// if (res == td::int64(~0ULL << 63)) { -// return td::Status::Error("Failed to unpack balance (2)"); -// } -// return balance; -// } - -// td::Result convert::to_balance256(td::Ref balance_ref) { -// vm::CellSlice balance_slice = *balance_ref; -// return to_balance(balance_slice); -// } - -td::Result> convert::to_bytes(td::Ref cell) { +td::Result> convert::to_bytes(td::Ref cell) { if (cell.is_null()) { - return td::optional(); + return std::nullopt; } TRY_RESULT(boc, vm::std_boc_serialize(cell, vm::BagOfCells::Mode::WithCRC32C)); return td::base64_encode(boc.as_slice().str()); diff --git a/tondb-scanner/src/convert-utils.h b/tondb-scanner/src/convert-utils.h index 489e1d64..0e7daddf 100644 --- a/tondb-scanner/src/convert-utils.h +++ b/tondb-scanner/src/convert-utils.h @@ -8,10 +8,5 @@ namespace convert { td::Result to_std_address(td::Ref cs); - td::Result to_balance(vm::CellSlice& balance_slice); - td::Result to_balance(td::Ref balance_ref); - // td::Result to_balance256(vm::CellSlice& balance_slice); - // td::Result to_balance256(td::Ref balance_ref); - - td::Result> to_bytes(td::Ref cell); + td::Result> to_bytes(td::Ref cell); } From b46f3f111a4e944d54b7b8432286748d030cca6c Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 21 Dec 2024 14:42:52 +0100 Subject: [PATCH 04/28] Rebased ton submodule on upstream testnet (#97) --- external/ton | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/ton b/external/ton index 74d973c5..e1c31967 160000 --- a/external/ton +++ b/external/ton @@ -1 +1 @@ -Subproject commit 74d973c5bd86cac491d30bb7fe1cd09f44f7d6aa +Subproject commit e1c31967f1b566c87cddb83a6e679cc0df9b719a From 3ba62212a7d819168c733c5a5df9ec7bbef6be95 Mon Sep 17 00:00:00 2001 From: Marat S Date: Sat, 21 Dec 2024 14:50:06 +0000 Subject: [PATCH 05/28] Clean up address_book and trace_edges --- .../src/InsertManagerClickhouse.cpp | 4 - .../src/InsertManagerClickhouse.h | 1 - .../src/InsertManagerPostgres.cpp | 184 +----------------- .../src/InsertManagerPostgres.h | 1 - .../src/InsertManagerPostgres.cpp | 6 +- .../src/InsertManagerPostgres.h | 1 - tondb-scanner/src/InsertManager.h | 1 - 7 files changed, 4 insertions(+), 194 deletions(-) diff --git a/ton-index-clickhouse/src/InsertManagerClickhouse.cpp b/ton-index-clickhouse/src/InsertManagerClickhouse.cpp index 3b69cab3..7c972ac5 100644 --- a/ton-index-clickhouse/src/InsertManagerClickhouse.cpp +++ b/ton-index-clickhouse/src/InsertManagerClickhouse.cpp @@ -399,10 +399,6 @@ void InsertManagerClickhouse::get_existing_seqnos(td::Promise promise) { - UNREACHABLE(); -} - clickhouse::ClientOptions InsertManagerClickhouse::Credential::get_clickhouse_options() { clickhouse::ClientOptions options; diff --git a/ton-index-clickhouse/src/InsertManagerClickhouse.h b/ton-index-clickhouse/src/InsertManagerClickhouse.h index 133df690..7766c714 100644 --- a/ton-index-clickhouse/src/InsertManagerClickhouse.h +++ b/ton-index-clickhouse/src/InsertManagerClickhouse.h @@ -28,7 +28,6 @@ class InsertManagerClickhouse: public InsertManagerBase { void create_insert_actor(std::vector insert_tasks, td::Promise promise) override; void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) override; - void get_trace_assembler_state(td::Promise promise) override; }; diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index c720c39f..9ee931ff 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -880,29 +880,6 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { << "code_boc = EXCLUDED.code_boc " << "WHERE latest_account_states.last_trans_lt < EXCLUDED.last_trans_lt;\n"; } - - is_first = true; - for (auto i = latest_account_states.begin(); i != latest_account_states.end(); ++i) { - auto& account_state = i->second; - if (is_first) { - query << "INSERT INTO address_book (address, code_hash) VALUES "; - is_first = false; - } else { - query << ", "; - } - - std::optional code_hash_str; - if (account_state.code_hash) { - code_hash_str = td::base64_encode(account_state.code_hash.value().as_slice()); - } - query << "(" - << txn.quote(convert::to_raw_address(account_state.account)) << "," - << TO_SQL_OPTIONAL_STRING(code_hash_str, txn) << ")"; - } - if (!is_first) { - query << " ON CONFLICT (address) DO UPDATE SET " - << "code_hash = EXCLUDED.code_hash;\n"; - } return query.str(); } @@ -1490,22 +1467,17 @@ std::string InsertBatchPostgres::insert_nft_transfers(pqxx::work &txn) { #define B64HASH(x) (td::base64_encode((x).as_slice())) std::string InsertBatchPostgres::insert_traces(pqxx::work &txn) { - std::string full_query; std::ostringstream traces_query; - std::ostringstream edges_query; traces_query << "INSERT INTO traces (trace_id, external_hash, mc_seqno_start, mc_seqno_end, " "start_lt, start_utime, end_lt, end_utime, state, pending_edges_, edges_, nodes_) VALUES "; - edges_query << "INSERT INTO trace_edges (trace_id, msg_hash, left_tx, right_tx, incomplete, broken) VALUES "; bool is_first_trace = true; - bool is_first_edge = true; std::unordered_map traces_map; - std::unordered_map, schema::TraceEdge> edges_map; for (const auto& task : insert_tasks_) { for(auto &trace : task.parsed_block_->traces_) { - { + if (trace.state == schema::Trace::State::complete) { auto it = traces_map.find(trace.trace_id); if (it != traces_map.end() && it->second.end_lt < trace.end_lt) { it->second = trace; @@ -1513,15 +1485,6 @@ std::string InsertBatchPostgres::insert_traces(pqxx::work &txn) { traces_map.insert({trace.trace_id, trace}); } } - for(auto &edge : trace.edges) { - auto key = std::make_pair(edge.trace_id, edge.msg_hash); - auto it = edges_map.find(key); - if (it != edges_map.end() && it->second.incomplete && !edge.incomplete) { - it->second = edge; - } else { - edges_map.insert({key, edge}); - } - } } } for(auto &[_, trace] : traces_map) { @@ -1546,22 +1509,6 @@ std::string InsertBatchPostgres::insert_traces(pqxx::work &txn) { << trace.nodes_ << ")"; } - // edges - for(auto &[_, edge] : edges_map) { - if(is_first_edge) { - is_first_edge = false; - } else { - edges_query << ", "; - } - edges_query << "(" - << txn.quote(B64HASH(edge.trace_id)) << "," - << txn.quote(B64HASH(edge.msg_hash)) << "," - << (edge.left_tx.has_value() ? txn.quote(B64HASH(edge.left_tx.value())) : "NULL" ) << "," - << (edge.right_tx.has_value() ? txn.quote(B64HASH(edge.right_tx.value())) : "NULL" ) << "," - << TO_SQL_BOOL(edge.incomplete) << "," - << TO_SQL_BOOL(edge.broken) - << ")"; - } if (!is_first_trace) { traces_query << " ON CONFLICT (trace_id) DO UPDATE SET " << "mc_seqno_end = EXCLUDED.mc_seqno_end, " @@ -1572,20 +1519,9 @@ std::string InsertBatchPostgres::insert_traces(pqxx::work &txn) { << "edges_ = EXCLUDED.edges_, " << "nodes_ = EXCLUDED.nodes_ " << "WHERE traces.end_lt < EXCLUDED.end_lt;\n"; - full_query = traces_query.str(); - } - if (!is_first_edge) { - edges_query << " ON CONFLICT (trace_id, msg_hash) DO UPDATE SET " - << "trace_id = EXCLUDED.trace_id, " - << "msg_hash = EXCLUDED.msg_hash, " - << "left_tx = EXCLUDED.left_tx, " - << "right_tx = EXCLUDED.right_tx, " - << "incomplete = EXCLUDED.incomplete, " - << "broken = EXCLUDED.broken " - << "WHERE trace_edges.incomplete is true and EXCLUDED.incomplete is false and EXCLUDED.broken is false;\n"; - full_query += edges_query.str(); } - return full_query; + + return traces_query.str(); } // @@ -2039,19 +1975,6 @@ void InsertManagerPostgres::start_up() { "primary key (trace_id)" ");\n" ); - - query += ( - "create table if not exists trace_edges (" - "trace_id tonhash, " - "msg_hash tonhash, " - "left_tx tonhash, " - "right_tx tonhash, " - "incomplete boolean, " - "broken boolean, " - "primary key (trace_id, msg_hash), " - "foreign key (trace_id) references traces" - ");\n" - ); query += ( "create table if not exists actions (" @@ -2106,13 +2029,6 @@ void InsertManagerPostgres::start_up() { "last_transaction_lt bigint);\n" ); - query += ( - "create table if not exists address_book (" - "address tonaddr not null primary key, " - "code_hash tonhash, " - "domain varchar);\n" - ); - LOG(DEBUG) << query; txn.exec0(query); txn.commit(); @@ -2130,8 +2046,6 @@ void InsertManagerPostgres::start_up() { // some necessary indexes query += ( - "create index if not exists traces_index_1 on traces (state);\n" - "create index if not exists trace_edges_index_1 on trace_edges (incomplete);\n" "create index if not exists trace_unclassified_index on traces (state, start_lt) include (trace_id, nodes_) where (classification_state = 'unclassified');\n" ); @@ -2221,8 +2135,6 @@ void InsertManagerPostgres::start_up() { "-- create index if not exists traces_index_5 on traces (external_hash, end_lt asc);\n" "-- create index if not exists traces_index_6 on traces (external_hash, end_utime asc);\n" "create index if not exists traces_index_7 on traces (classification_state);\n" - "create index if not exists trace_edges_index_1 on trace_edges (incomplete);\n" - "-- create index if not exists trace_edges_index_2 on trace_edges (msg_hash);\n" "-- create index if not exists actions_index_1 on actions (trace_id, start_lt, end_lt);\n" "create index if not exists actions_index_2 on actions (action_id);" ); @@ -2275,96 +2187,6 @@ void InsertManagerPostgres::set_max_data_depth(std::int32_t value) { max_data_depth_ = value; } -void InsertManagerPostgres::get_trace_assembler_state(td::Promise promise) { - pqxx::connection c(credential_.get_connection_string()); - - auto to_bits256 = [](std::string value) { - auto R = td::base64_decode(value); - if (R.is_error()) { - LOG(ERROR) << "Failed to decode b64 string: " << value; - std::_Exit(2); - } - return td::Bits256(td::ConstBitPtr(td::Slice(R.move_as_ok()).ubegin())); - }; - try { - schema::TraceAssemblerState state; - { - pqxx::work txn(c); - std::string query = "select trace_id, external_hash, mc_seqno_start, mc_seqno_end, start_lt, start_utime, end_lt, end_utime, state, pending_edges_, edges_, nodes_ from traces where state = 'pending';"; - pqxx::result result = txn.exec(query); - txn.commit(); - for (auto row : result) { - schema::Trace trace; - trace.trace_id = to_bits256(row[0].as()); - if (!row[1].is_null()) { - trace.external_hash = to_bits256(row[1].as()); - } - if (!row[2].is_null()) { - trace.mc_seqno_start = row[2].as(); - } - if (!row[3].is_null()) { - trace.mc_seqno_end = row[3].as(); - } - if (!row[4].is_null()) { - trace.start_lt = row[4].as(); - } - if (!row[5].is_null()) { - trace.start_utime = row[5].as(); - } - if (!row[6].is_null()) { - trace.end_lt = row[6].as(); - } - if (!row[7].is_null()) { - trace.end_utime = row[7].as(); - } - if (row[8].as() != "pending") { - LOG(ERROR) << "Error in request. Got non-pending trace!"; - } - trace.state = schema::Trace::State::pending; - if (!row[9].is_null()) { - trace.pending_edges_ = row[9].as(); - } - if (!row[10].is_null()) { - trace.edges_ = row[10].as(); - } - if (!row[11].is_null()) { - trace.nodes_ = row[11].as(); - } - - state.pending_traces_.push_back(std::move(trace)); - } - } - { - pqxx::work txn(c); - std::string query = "select trace_id, msg_hash, left_tx, right_tx, incomplete, broken from trace_edges where incomplete;"; - pqxx::result result = txn.exec(query); - txn.commit(); - for (auto row : result) { - schema::TraceEdge edge; - edge.trace_id = to_bits256(row[0].as()); - edge.msg_hash = to_bits256(row[1].as()); - if (!row[2].is_null()) { - edge.left_tx = to_bits256(row[2].as()); - } - if (!row[3].is_null()) { - edge.right_tx = to_bits256(row[3].as()); - } - edge.type = schema::TraceEdge::Type::ord; - edge.incomplete = row[4].as(); - if (edge.incomplete != true) { - LOG(ERROR) << "Error in request. Got non-pending edge!"; - } - edge.broken = row[5].as(); - - state.pending_edges_.push_back(std::move(edge)); - } - } - promise.set_value(std::move(state)); - } catch (const std::exception &e) { - promise.set_error(td::Status::Error(ErrorCode::DB_ERROR, PSLICE() << "Error selecting from PG: " << e.what())); - } -} - void InsertManagerPostgres::create_insert_actor(std::vector insert_tasks, td::Promise promise) { td::actor::create_actor("insert_batch_postgres", credential_, std::move(insert_tasks), std::move(promise), max_data_depth_).release(); } diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.h b/ton-index-postgres-v2/src/InsertManagerPostgres.h index 419bb1a3..a6d04ab3 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.h +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.h @@ -33,7 +33,6 @@ class InsertManagerPostgres: public InsertManagerBase { void create_insert_actor(std::vector insert_tasks, td::Promise promise) override; void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) override; - void get_trace_assembler_state(td::Promise promise) override; }; diff --git a/ton-index-postgres/src/InsertManagerPostgres.cpp b/ton-index-postgres/src/InsertManagerPostgres.cpp index bd66e152..65a3575c 100644 --- a/ton-index-postgres/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres/src/InsertManagerPostgres.cpp @@ -1177,8 +1177,4 @@ void InsertManagerPostgres::get_existing_seqnos(td::Promise promise) { - UNREACHABLE(); -} +} \ No newline at end of file diff --git a/ton-index-postgres/src/InsertManagerPostgres.h b/ton-index-postgres/src/InsertManagerPostgres.h index a4615b83..1c8bdc4f 100644 --- a/ton-index-postgres/src/InsertManagerPostgres.h +++ b/ton-index-postgres/src/InsertManagerPostgres.h @@ -24,7 +24,6 @@ class InsertManagerPostgres: public InsertManagerBase { void create_insert_actor(std::vector insert_tasks, td::Promise promise) override; void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) override; - void get_trace_assembler_state(td::Promise promise) override; }; diff --git a/tondb-scanner/src/InsertManager.h b/tondb-scanner/src/InsertManager.h index 6bb39237..39e9e2f2 100644 --- a/tondb-scanner/src/InsertManager.h +++ b/tondb-scanner/src/InsertManager.h @@ -22,7 +22,6 @@ class InsertManagerInterface: public td::actor::Actor { virtual void insert(std::uint32_t mc_seqno, ParsedBlockPtr block_ds, td::Promise queued_promise, td::Promise inserted_promise) = 0; virtual void get_insert_queue_state(td::Promise promise) = 0; virtual void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) = 0; - virtual void get_trace_assembler_state(td::Promise promise) = 0; // // helper template functions // template From 05b85722daf91b12b9b7959431c5070eace825fb Mon Sep 17 00:00:00 2001 From: Marat S Date: Tue, 24 Dec 2024 12:54:54 +0000 Subject: [PATCH 06/28] fix segfault --- tondb-scanner/src/TraceAssembler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tondb-scanner/src/TraceAssembler.cpp b/tondb-scanner/src/TraceAssembler.cpp index 210c1d6a..5523cf9f 100644 --- a/tondb-scanner/src/TraceAssembler.cpp +++ b/tondb-scanner/src/TraceAssembler.cpp @@ -101,8 +101,8 @@ void TraceAssembler::alarm() { return; } - ton::delay_action([this, seqno = expected_seqno_ - 1, pending_traces = pending_traces_, pending_edges = pending_edges_]() { - auto S = save_state(this->db_path_, expected_seqno_ - 1, pending_traces_, pending_edges_); + ton::delay_action([db_path = this->db_path_, seqno = expected_seqno_ - 1, pending_traces = pending_traces_, pending_edges = pending_edges_]() { + auto S = save_state(db_path, seqno, pending_traces, pending_edges); if (S.is_error()) { LOG(ERROR) << "Error while saving Trace Assembler state: " << S.move_as_error(); } From 67b6d66d8760fdbb906ecfcc547ec6dc74cbe252 Mon Sep 17 00:00:00 2001 From: Marat S Date: Thu, 2 Jan 2025 02:03:55 +0000 Subject: [PATCH 07/28] improve trace assembler states handling --- tondb-scanner/src/TraceAssembler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tondb-scanner/src/TraceAssembler.cpp b/tondb-scanner/src/TraceAssembler.cpp index 5523cf9f..a3ac541a 100644 --- a/tondb-scanner/src/TraceAssembler.cpp +++ b/tondb-scanner/src/TraceAssembler.cpp @@ -109,7 +109,7 @@ void TraceAssembler::alarm() { }, td::Timestamp::now()); ton::delay_action([this]() { - auto S = gc_states(this->db_path_, this->expected_seqno_, 10); + auto S = gc_states(this->db_path_, this->expected_seqno_, 100); if (S.is_error()) { LOG(ERROR) << "Error while garbage collecting Trace Assembler states: " << S.move_as_error(); } @@ -153,7 +153,7 @@ td::Result TraceAssembler::restore_state(ton::BlockSeqno seqno) } for (const auto& [state_seqno, path] : fileMap) { - LOG(INFO) << "Found TA state seqno:" << seqno << " - path: " << path.string() << '\n'; + LOG(INFO) << "Found TA state seqno: " << state_seqno << " - path: " << path.string() << '\n'; if (state_seqno > seqno) { LOG(WARNING) << "Found trace assembler state " << state_seqno << " newer than requested " << seqno; continue; From 061412274004c29399af28e3d8a15390cc859f38 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 18 Jan 2025 00:37:19 +0100 Subject: [PATCH 08/28] Optimize reading of account states (#99) --- CMakeLists.txt | 1 + celldb-migrate/CMakeLists.txt | 12 + celldb-migrate/src/main.cpp | 231 +++++++++++++++++++ ton-index-postgres-v2/src/IndexScheduler.cpp | 7 +- tondb-scanner/src/DataParser.cpp | 61 ++--- tondb-scanner/src/DataParser.h | 19 +- tondb-scanner/src/DbScanner.cpp | 4 + tondb-scanner/src/DbScanner.h | 1 + 8 files changed, 293 insertions(+), 43 deletions(-) create mode 100644 celldb-migrate/CMakeLists.txt create mode 100644 celldb-migrate/src/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d7e7445c..b65b0aac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ add_subdirectory(ton-index-clickhouse) add_subdirectory(ton-integrity-checker) add_subdirectory(ton-smc-scanner) add_subdirectory(ton-trace-emulator) +add_subdirectory(celldb-migrate) if (PGTON) message("Building pgton") diff --git a/celldb-migrate/CMakeLists.txt b/celldb-migrate/CMakeLists.txt new file mode 100644 index 00000000..24bbe017 --- /dev/null +++ b/celldb-migrate/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.16) + +add_executable(celldb-migrate + src/main.cpp +) +target_include_directories(celldb-migrate + PUBLIC src +) +target_compile_features(celldb-migrate PRIVATE cxx_std_17) +target_link_libraries(celldb-migrate tondb-scanner) + +install(TARGETS celldb-migrate RUNTIME DESTINATION bin) diff --git a/celldb-migrate/src/main.cpp b/celldb-migrate/src/main.cpp new file mode 100644 index 00000000..90db6a07 --- /dev/null +++ b/celldb-migrate/src/main.cpp @@ -0,0 +1,231 @@ +#include "td/utils/port/signals.h" +#include "td/utils/OptionParser.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/check.h" +#include "td/utils/port/path.h" +#include "td/actor/actor.h" +#include "crypto/vm/cp0.h" +#include "tddb/td/db/RocksDb.h" +#include +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "crypto/vm/db/DynamicBagOfCellsDb.h" +#include "crypto/vm/db/CellStorage.h" + + +static td::Status from_rocksdb(rocksdb::Status status) { + if (status.ok()) { + return td::Status::OK(); + } + return td::Status::Error(status.ToString()); +} +static td::Slice from_rocksdb(rocksdb::Slice slice) { + return td::Slice(slice.data(), slice.size()); +} +static rocksdb::Slice to_rocksdb(td::Slice slice) { + return rocksdb::Slice(slice.data(), slice.size()); +} + +class MigrateBatchActor: public td::actor::Actor { + td::Bits256 from_; + td::Bits256 to_; + std::shared_ptr db_; + int new_compress_depth_; + td::Promise promise_; + + std::unique_ptr loader_; + std::shared_ptr boc_; + + uint32_t migrated_{0}; +public: + MigrateBatchActor(td::Bits256 from, td::Bits256 to, std::shared_ptr db, int new_compress_depth, td::Promise promise) + : from_(from), to_(to), db_(db), new_compress_depth_(new_compress_depth), promise_(std::move(promise)) { + + loader_ = std::make_unique(db_); + boc_ = vm::DynamicBagOfCellsDb::create(); + boc_->set_celldb_compress_depth(new_compress_depth_); // probably not necessary in this context + boc_->set_loader(std::make_unique(db_)); + } + + void start_up() override { + vm::CellStorer storer{*db_}; + + std::unique_ptr it; + it.reset(db_->raw_db()->NewIterator({})); + db_->begin_write_batch().ensure(); + for (it->Seek(to_rocksdb(from_.as_slice())); it->Valid(); it->Next()) { + auto key = from_rocksdb(it->key()); + if (key.size() != 32) { + LOG(WARNING) << "CellDb: skipping key with size " << key.size(); + continue; + } + td::Bits256 hash = td::Bits256(td::ConstBitPtr{key.ubegin()}); + if (!(hash < to_)) { + break; + } + + auto value = from_rocksdb(it->value()); + migrate_cell(hash, value, storer); + } + db_->commit_write_batch().ensure(); + + LOG(INFO) << "Migrating batch from " << from_.to_hex() << " to " << to_.to_hex() << " done. Migrated " << migrated_ << " cells"; + promise_.set_value(td::Unit()); + stop(); + } + + td::Status migrate_cell(const td::Bits256& hash, const td::Slice& value, vm::CellStorer& storer) { + auto R = loader_->load(hash.as_slice(), value, true, boc_->as_ext_cell_creator()); + if (R.is_error()) { + LOG(WARNING) << "CellDb: failed to load cell: " << R.move_as_error(); + return td::Status::OK(); + } + if (R.ok().status == vm::CellLoader::LoadResult::NotFound) { + LOG(WARNING) << "CellDb: cell not found"; + return td::Status::OK(); + } + bool expected_stored_boc = R.ok().cell_->get_depth() == new_compress_depth_; + if (expected_stored_boc != R.ok().stored_boc_) { + ++migrated_; + storer.set(R.ok().refcnt(), R.ok().cell_, expected_stored_boc).ensure(); + LOG(DEBUG) << "Migrating cell " << hash.to_hex(); + } + return td::Status::OK(); + } +}; + +class MigrateCellDBActor: public td::actor::Actor { + std::string db_root_; + int new_compress_depth_; + int max_parallel_batches_; + + std::shared_ptr db_; + + uint32_t migrated_{0}; + + td::Bits256 current_; + + int cur_parallel_batches_{0}; +public: + MigrateCellDBActor(std::string db_root, int new_compress_depth, int max_parallel_batches) + : db_root_(db_root), new_compress_depth_(new_compress_depth), max_parallel_batches_(max_parallel_batches) { + td::RocksDbOptions read_db_options; + read_db_options.use_direct_reads = true; + auto db_r = td::RocksDb::open(db_root_ + "/celldb", std::move(read_db_options)); + if (db_r.is_error()) { + LOG(FATAL) << "failed to open db: " << db_r.error(); + stop(); + return; + } + db_ = std::make_shared(db_r.move_as_ok()); + + current_ = td::Bits256::zero(); + } + + void start_up() override { + uint64_t count; + db_->raw_db()->GetIntProperty("rocksdb.estimate-num-keys", &count); + LOG(INFO) << "Estimated total number of keys: " << count; + + deploy_batches(); + } + + void deploy_batches() { + using namespace td::literals; + const auto interval_bi = "0000100000000000000000000000000000000000000000000000000000000000"_rx256; + CHECK(interval_bi.not_null()); + + while (cur_parallel_batches_ < max_parallel_batches_) { + auto current_bi = td::bits_to_refint(current_.bits(), 256, false); + auto to_bi = current_bi + interval_bi; + if (!to_bi->is_valid()) { + LOG(INFO) << "New to_bi is invalid. Stopping."; + return; + } + td::Bits256 to; + std::string to_hex = to_bi->to_hex_string(); + if (to_hex.size() < 64) { + to_hex = std::string(64 - to_hex.size(), '0') + to_hex; + } + if (to.from_hex(to_hex) >= 256) { + LOG(INFO) << "New to_bi is too large. Stopping."; + return; + } + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), current = current_, to = to](td::Result R) { + if (R.is_error()) { + LOG(ERROR) << "failed to migrate batch from " << current.to_hex() << " to " << to.to_hex() << ": " << R.error(); + } else { + LOG(INFO) << "migrated batch from " << current.to_hex() << " to " << to.to_hex(); + } + td::actor::send_closure(SelfId, &MigrateCellDBActor::on_batch_migrated); + }); + auto db_clone = std::make_shared(db_->clone()); + td::actor::create_actor("migrate", current_, to, db_clone, new_compress_depth_, std::move(P)).release(); + current_ = to; + + cur_parallel_batches_++; + } + } + + void on_batch_migrated() { + cur_parallel_batches_--; + + deploy_batches(); + + if (cur_parallel_batches_ == 0) { + LOG(INFO) << "Migrated all batches"; + stop(); + } + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_default_failure_signal_handler().ensure(); + + td::OptionParser p; + std::string db_root; + int new_compress_depth = 0; + p.set_description("Migrate CellDB to another compress db value"); + p.add_option('\0', "help", "prints_help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_option('D', "db", "Path to TON DB folder", [&](td::Slice fname) { + db_root = fname.str(); + }); + p.add_checked_option('\0', "new-celldb-compress-depth", "New value of celldb compress depth", [&](td::Slice fname) { + int v; + try { + v = std::stoi(fname.str()); + } catch (...) { + return td::Status::Error("bad value for --new-celldb-compress-depth: not a number"); + } + new_compress_depth = v; + return td::Status::OK(); + }); + auto S = p.run(argc, argv); + if (S.is_error()) { + LOG(ERROR) << "failed to parse options: " << S.move_as_error(); + std::_Exit(2); + } + if (db_root.empty()) { + LOG(ERROR) << "db path is empty"; + std::_Exit(2); + } + if (new_compress_depth <= 0) { + LOG(ERROR) << "new compress depth is invalid"; + std::_Exit(2); + } + + td::actor::Scheduler scheduler({32}); + scheduler.run_in_context( + [&] { td::actor::create_actor("migrate", db_root, new_compress_depth, 32).release(); }); + while (scheduler.run(1)) { + } + return 0; +} \ No newline at end of file diff --git a/ton-index-postgres-v2/src/IndexScheduler.cpp b/ton-index-postgres-v2/src/IndexScheduler.cpp index fed204b3..0cc9432d 100644 --- a/ton-index-postgres-v2/src/IndexScheduler.cpp +++ b/ton-index-postgres-v2/src/IndexScheduler.cpp @@ -167,7 +167,12 @@ void IndexScheduler::seqno_fetched(std::uint32_t mc_seqno, MasterchainBlockDataS } td::actor::send_closure(SelfId, &IndexScheduler::seqno_parsed, mc_seqno, R.move_as_ok()); }); - td::actor::send_closure(parse_manager_, &ParseManager::parse, mc_seqno, std::move(block_data_state), std::move(P)); + + td::actor::send_closure(db_scanner_, &DbScanner::get_cell_db_reader, + [SelfId = actor_id(this), parse_manager = parse_manager_, mc_seqno, block_data_state, P = std::move(P)](td::Result> cell_db_reader) mutable { + CHECK(cell_db_reader.is_ok()); + td::actor::send_closure(parse_manager, &ParseManager::parse, mc_seqno, std::move(block_data_state), cell_db_reader.move_as_ok(), std::move(P)); + }); } void IndexScheduler::seqno_parsed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block) { diff --git a/tondb-scanner/src/DataParser.cpp b/tondb-scanner/src/DataParser.cpp index 3692d4a6..2745c4b0 100644 --- a/tondb-scanner/src/DataParser.cpp +++ b/tondb-scanner/src/DataParser.cpp @@ -40,11 +40,14 @@ td::Status ParseQuery::parse_impl() { } // transactions and messages - std::set addresses; - TRY_RESULT_ASSIGN(schema_block.transactions, parse_transactions(block_ds.block_data->block_id(), blk, info, extra, addresses)); + std::map account_states_to_get; + TRY_RESULT_ASSIGN(schema_block.transactions, parse_transactions(block_ds.block_data->block_id(), blk, info, extra, account_states_to_get)); - // account states - TRY_STATUS(parse_account_states(block_ds.block_state, addresses)); + TRY_RESULT(account_states_fast, parse_account_states_new(schema_block.workchain, schema_block.gen_utime, account_states_to_get)); + + for (auto &acc : account_states_fast) { + result->account_states_.push_back(acc); + } // config if (block_ds.block_data->block_id().is_masterchain()) { @@ -532,7 +535,7 @@ td::Result ParseQuery::process_transaction_descr(vm::C td::Result> ParseQuery::parse_transactions(const ton::BlockIdExt& blk_id, const block::gen::Block::Record &block, const block::gen::BlockInfo::Record &info, const block::gen::BlockExtra::Record &extra, - std::set &addresses) { + std::map &account_states) { std::vector res; try { vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, block::tlb::aug_ShardAccountBlocks}; @@ -615,8 +618,8 @@ td::Result> ParseQuery::parse_transactions(cons TRY_RESULT_ASSIGN(schema_tx.description, process_transaction_descr(descr_cs)); res.push_back(schema_tx); - - addresses.insert(cur_addr); + + account_states[cur_addr] = {schema_tx.account_state_hash_after, schema_tx.lt, schema_tx.hash}; } } } catch (vm::VmError err) { @@ -625,50 +628,36 @@ td::Result> ParseQuery::parse_transactions(cons return res; } -td::Status ParseQuery::parse_account_states(const td::Ref& block_state_root, std::set &addresses) { - auto root = block_state_root; - block::gen::ShardStateUnsplit::Record sstate; - if (!tlb::unpack_cell(block_state_root, sstate)) { - return td::Status::Error("Failed to unpack ShardStateUnsplit"); - } - block::gen::ShardIdent::Record shard_id; - if (!tlb::csr_unpack(sstate.shard_id, shard_id)) { - return td::Status::Error("Failed to unpack ShardIdent"); - } - vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; - for (auto &addr : addresses) { - auto shard_account_csr = accounts_dict.lookup(addr); - if (shard_account_csr.is_null()) { - // account is uninitialized after this block - continue; - } - block::gen::ShardAccount::Record acc_info; - if(!tlb::csr_unpack(std::move(shard_account_csr), acc_info)) { - LOG(ERROR) << "Failed to unpack ShardAccount " << addr; +td::Result> ParseQuery::parse_account_states_new(ton::WorkchainId workchain_id, uint32_t gen_utime, std::map &account_states) { + std::vector res; + res.reserve(account_states.size()); + for (auto &[addr, state_short] : account_states) { + auto account_cell_r = cell_db_reader_->load_cell(state_short.account_cell_hash.as_slice()); + if (account_cell_r.is_error()) { + LOG(ERROR) << "Failed to load account state cell " << state_short.account_cell_hash.to_hex(); continue; } - int account_tag = block::gen::t_Account.get_tag(vm::load_cell_slice(acc_info.account)); + auto account_cell = account_cell_r.move_as_ok(); + int account_tag = block::gen::t_Account.get_tag(vm::load_cell_slice(account_cell)); switch (account_tag) { case block::gen::Account::account_none: { - auto address = block::StdAddress(shard_id.workchain_id, addr); - TRY_RESULT(account, parse_none_account(std::move(acc_info.account), address, sstate.gen_utime, acc_info.last_trans_hash, acc_info.last_trans_lt)); - result->account_states_.push_back(account); + auto address = block::StdAddress(workchain_id, addr); + TRY_RESULT(account, parse_none_account(std::move(account_cell), address, gen_utime, state_short.last_transaction_hash, state_short.last_transaction_lt)); + res.push_back(account); break; } case block::gen::Account::account: { - TRY_RESULT(account, parse_account(std::move(acc_info.account), sstate.gen_utime, acc_info.last_trans_hash, acc_info.last_trans_lt)); - result->account_states_.push_back(account); + TRY_RESULT(account, parse_account(std::move(account_cell), gen_utime, state_short.last_transaction_hash, state_short.last_transaction_lt)); + res.push_back(account); break; } default: return td::Status::Error("Unknown account tag"); } } - // LOG(DEBUG) << "Parsed " << result->account_states_.size() << " account states"; - return td::Status::OK(); + return res; } -// td::Result ParseQuery::parse_shard_account(td::Ref account_csr) td::Result ParseQuery::parse_none_account(td::Ref account_root, block::StdAddress address, uint32_t gen_utime, td::Bits256 last_trans_hash, uint64_t last_trans_lt) { block::gen::Account::Record_account_none account_none; diff --git a/tondb-scanner/src/DataParser.h b/tondb-scanner/src/DataParser.h index 2bcfbcb8..6b8a360e 100644 --- a/tondb-scanner/src/DataParser.h +++ b/tondb-scanner/src/DataParser.h @@ -6,11 +6,12 @@ class ParseQuery: public td::actor::Actor { private: const int mc_seqno_; MasterchainBlockDataState mc_block_; + std::shared_ptr cell_db_reader_; ParsedBlockPtr result; td::Promise promise_; public: - ParseQuery(int mc_seqno, MasterchainBlockDataState mc_block, td::Promise promise) - : mc_seqno_(mc_seqno), mc_block_(std::move(mc_block)), result(std::make_shared()), promise_(std::move(promise)) {} + ParseQuery(int mc_seqno, MasterchainBlockDataState mc_block, std::shared_ptr cell_db_reader, td::Promise promise) + : mc_seqno_(mc_seqno), mc_block_(std::move(mc_block)), cell_db_reader_(std::move(cell_db_reader)), result(std::make_shared()), promise_(std::move(promise)) {} void start_up() override; @@ -30,10 +31,16 @@ class ParseQuery: public td::actor::Actor { td::Result parse_tr_bounce_phase(vm::CellSlice& cs); td::Result parse_split_merge_info(td::Ref& cs); td::Result process_transaction_descr(vm::CellSlice& td_cs); + + struct AccountStateShort { + td::Bits256 account_cell_hash; + uint64_t last_transaction_lt; + td::Bits256 last_transaction_hash; + }; td::Result> parse_transactions(const ton::BlockIdExt& blk_id, const block::gen::Block::Record &block, const block::gen::BlockInfo::Record &info, const block::gen::BlockExtra::Record &extra, - std::set &addresses); - td::Status parse_account_states(const td::Ref& block_state_root, std::set &addresses); + std::map &account_states); + td::Result> parse_account_states_new(ton::WorkchainId workchain_id, uint32_t gen_utime, std::map &account_states); td::Result parse_none_account(td::Ref account_root, block::StdAddress address, uint32_t gen_utime, td::Bits256 last_trans_hash, uint64_t last_trans_lt); public: //TODO: refactor @@ -45,7 +52,7 @@ class ParseManager: public td::actor::Actor { public: ParseManager() {} - void parse(int mc_seqno, MasterchainBlockDataState mc_block, td::Promise promise) { - td::actor::create_actor("parsequery", mc_seqno, std::move(mc_block), std::move(promise)).release(); + void parse(int mc_seqno, MasterchainBlockDataState mc_block, std::shared_ptr cell_db_reader, td::Promise promise) { + td::actor::create_actor("parsequery", mc_seqno, std::move(mc_block), cell_db_reader, std::move(promise)).release(); } }; \ No newline at end of file diff --git a/tondb-scanner/src/DbScanner.cpp b/tondb-scanner/src/DbScanner.cpp index 190d6d27..120255ba 100644 --- a/tondb-scanner/src/DbScanner.cpp +++ b/tondb-scanner/src/DbScanner.cpp @@ -306,6 +306,10 @@ void DbScanner::get_mc_block_handle(ton::BlockSeqno seqno, td::Promise> promise) { + td::actor::send_closure(db_, &RootDb::get_cell_db_reader, std::move(promise)); +} + void DbScanner::catch_up_with_primary() { auto R = td::PromiseCreator::lambda([SelfId = actor_id(this), this](td::Result R) { R.ensure(); diff --git a/tondb-scanner/src/DbScanner.h b/tondb-scanner/src/DbScanner.h index cb1edc6f..47492e53 100644 --- a/tondb-scanner/src/DbScanner.h +++ b/tondb-scanner/src/DbScanner.h @@ -30,6 +30,7 @@ class DbScanner: public td::actor::Actor { void get_last_mc_seqno(td::Promise promise); void get_oldest_mc_seqno(td::Promise promise); void get_mc_block_handle(ton::BlockSeqno seqno, td::Promise promise); + void get_cell_db_reader(td::Promise> promise); private: void catch_up_with_primary(); }; From bc9aa657ca9f225c3cd177a5bd683b3a09f0c7bc Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 18 Jan 2025 00:37:50 +0100 Subject: [PATCH 09/28] Refactor PG insert manager, try insert with COPY (#100) --- .../src/BlockInterfacesDetector.h | 19 +- .../src/InsertManagerPostgres.cpp | 2106 ++++++++--------- .../src/InsertManagerPostgres.h | 20 +- tondb-scanner/src/IndexData.h | 4 +- 4 files changed, 1043 insertions(+), 1106 deletions(-) diff --git a/ton-index-postgres-v2/src/BlockInterfacesDetector.h b/ton-index-postgres-v2/src/BlockInterfacesDetector.h index a7ffff86..abe6f201 100644 --- a/ton-index-postgres-v2/src/BlockInterfacesDetector.h +++ b/ton-index-postgres-v2/src/BlockInterfacesDetector.h @@ -3,12 +3,6 @@ #include #include "IndexData.h" -struct AccountStateHasher { - std::size_t operator()(const schema::AccountState& account_state) const { - return BitArrayHasher()(account_state.hash); - } -}; - class BlockInterfaceProcessor: public td::actor::Actor { private: ParsedBlockPtr block_; @@ -19,17 +13,14 @@ class BlockInterfaceProcessor: public td::actor::Actor { block_(std::move(block)), promise_(std::move(promise)) {} void start_up() override { - std::unordered_set account_states_to_detect; + std::unordered_map account_states_to_detect; for (const auto& account_state : block_->account_states_) { if (!account_state.code_hash || !account_state.data_hash) { continue; } - auto existing = account_states_to_detect.find(account_state); - if (existing != account_states_to_detect.end() && account_state.last_trans_lt > existing->last_trans_lt) { - account_states_to_detect.erase(existing); - account_states_to_detect.insert(account_state); - } else { - account_states_to_detect.insert(account_state); + auto existing = account_states_to_detect.find(account_state.account); + if (existing == account_states_to_detect.end() || account_state.last_trans_lt > existing->second.last_trans_lt) { + account_states_to_detect[account_state.account] = account_state; interfaces_[account_state.account] = {}; } } @@ -47,7 +38,7 @@ class BlockInterfaceProcessor: public td::actor::Actor { auto ig = mp.init_guard(); ig.add_promise(std::move(P)); - for (const auto& account_state : account_states_to_detect) { + for (const auto& [_, account_state] : account_states_to_detect) { if (account_state.code.is_null()) { continue; } diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index 9ee931ff..164078de 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -3,11 +3,301 @@ #include "InsertManagerPostgres.h" #include "convert-utils.h" -#define TO_SQL_BOOL(x) ((x) ? "TRUE" : "FALSE") -#define TO_SQL_OPTIONAL(x) ((x) ? std::to_string(x.value()) : "NULL") -#define TO_SQL_OPTIONAL_BOOL(x) ((x) ? ((x.value()) ? "TRUE" : "FALSE") : "NULL") -#define TO_SQL_OPTIONAL_STRING(x, txn) ((x) ? (txn.quote(x.value())) : "NULL") -#define TO_SQL_OPTIONAL_REFINT(x) ((x) ? (x.value()->to_dec_string()) : "NULL") + +namespace pqxx +{ + +template<> struct nullness : pqxx::no_null {}; + +template<> struct string_traits +{ + static constexpr bool converts_to_string{true}; + static constexpr bool converts_from_string{false}; + + static zview to_buf(char *begin, char *end, schema::BlockReference const &value) { + return zview{ + begin, + static_cast(into_buf(begin, end, value) - begin - 1)}; + } + + static char *into_buf(char *begin, char *end, schema::BlockReference const &value) { + std::ostringstream stream; + stream << "(" << value.workchain << ", " << value.shard << ", " << value.seqno << ")"; + auto text = stream.str(); + if (pqxx::internal::cmp_greater_equal(std::size(text), end - begin)) + throw conversion_overrun{"Not enough buffer for schema::BlockReference."}; + std::memcpy(begin, text.c_str(), std::size(text) + 1); + return begin + std::size(text) + 1; + } + static std::size_t size_buffer(schema::BlockReference const &value) noexcept { + return 64; + } +}; + +template<> struct nullness : pqxx::no_null {}; + +template<> struct string_traits +{ + static constexpr bool converts_to_string{true}; + static constexpr bool converts_from_string{false}; + + static zview to_buf(char *begin, char *end, td::RefInt256 const &value) { + return zview{begin, static_cast(into_buf(begin, end, value) - begin - 1)}; + } + + static char *into_buf(char *begin, char *end, td::RefInt256 const &value) { + auto text = value->to_dec_string(); + if (pqxx::internal::cmp_greater_equal(std::size(text), end - begin)) + throw conversion_overrun{"Not enough buffer for td::RefInt256."}; + std::memcpy(begin, text.c_str(), std::size(text) + 1); + return begin + std::size(text) + 1; + } + static std::size_t size_buffer(td::RefInt256 const &value) noexcept { + return 128; + } +}; + +template<> struct nullness : pqxx::no_null {}; + +template<> struct string_traits +{ + static constexpr bool converts_to_string{true}; + static constexpr bool converts_from_string{false}; + + static zview to_buf(char *begin, char *end, block::StdAddress const &value) { + return zview{begin, static_cast(into_buf(begin, end, value) - begin - 1)}; + } + + static char *into_buf(char *begin, char *end, block::StdAddress const &value) { + std::ostringstream stream; + stream << value.workchain << ":" << value.addr; + auto text = stream.str(); + if (pqxx::internal::cmp_greater_equal(std::size(text), end - begin)) + throw conversion_overrun{"Not enough buffer for block::StdAddress."}; + std::memcpy(begin, text.c_str(), std::size(text) + 1); + return begin + std::size(text) + 1; + } + static std::size_t size_buffer(block::StdAddress const &value) noexcept { + return 80; + } +}; + +template<> struct nullness : pqxx::no_null {}; + +template<> struct string_traits +{ + static constexpr bool converts_to_string{true}; + static constexpr bool converts_from_string{false}; + + static zview to_buf(char *begin, char *end, td::Bits256 const &value) { + return zview{begin, static_cast(into_buf(begin, end, value) - begin - 1)}; + } + + static char *into_buf(char *begin, char *end, td::Bits256 const &value) { + auto text = td::base64_encode(value.as_slice()); + if (pqxx::internal::cmp_greater_equal(std::size(text), end - begin)) + throw conversion_overrun{"Not enough buffer for td::Bits256."}; + std::memcpy(begin, text.c_str(), std::size(text) + 1); + return begin + std::size(text) + 1; + } + static std::size_t size_buffer(td::Bits256 const &value) noexcept { + return 64; + } +}; + +template<> struct nullness : pqxx::no_null {}; + +template<> struct string_traits +{ + static constexpr bool converts_to_string{true}; + static constexpr bool converts_from_string{false}; + + static zview to_buf(char *begin, char *end, vm::CellHash const &value) { + return zview{begin, static_cast(into_buf(begin, end, value) - begin - 1)}; + } + + static char *into_buf(char *begin, char *end, vm::CellHash const &value) { + auto text = td::base64_encode(value.as_slice()); + if (pqxx::internal::cmp_greater_equal(std::size(text), end - begin)) + throw conversion_overrun{"Not enough buffer for vm::CellHash."}; + std::memcpy(begin, text.c_str(), std::size(text) + 1); + return begin + std::size(text) + 1; + } + static std::size_t size_buffer(vm::CellHash const &value) noexcept { + return 64; + } +}; +} + +class PopulateTableStream { +private: + pqxx::work& txn_; + std::string table_name_; + std::initializer_list column_names_; + int batch_size_; + bool with_copy_; + bool is_first_row_{true}; + std::optional copy_stream_; + std::ostringstream insert_stream_; + std::string conflict_clause_; + + bool conflict_clause_added_{false}; + + enum class ConflictAction { None, DoNothing, DoUpdate }; + ConflictAction conflict_action_{ConflictAction::None}; + std::initializer_list conflict_columns_; + std::string update_condition_; + +public: + struct UpsertConfig { + std::initializer_list conflict_columns; + std::string_view update_condition; + }; + + PopulateTableStream( + pqxx::work& txn, + std::string_view table_name, + std::initializer_list column_names, + int batch_size, + bool with_copy = false) + : txn_(txn) + , table_name_(table_name) + , column_names_(column_names) + , batch_size_(batch_size) + , with_copy_(with_copy) + { + initializeStream(); + } + + void setConflictDoNothing() { + if (with_copy_) { + throw std::runtime_error("ON CONFLICT not supported with COPY mode"); + } + conflict_action_ = ConflictAction::DoNothing; + buildConflictClause(); + } + + void setConflictDoUpdate(std::initializer_list conflict_columns, std::string_view update_condition) { + if (with_copy_) { + throw std::runtime_error("ON CONFLICT not supported with COPY mode"); + } + conflict_action_ = ConflictAction::DoUpdate; + conflict_columns_ = std::move(conflict_columns); + update_condition_ = std::move(update_condition); + buildConflictClause(); + } + +private: + void buildConflictClause() { + std::ostringstream conflict_stream; + + if (conflict_action_ != ConflictAction::None) { + conflict_stream << " ON CONFLICT "; + + if (conflict_columns_.size()) { + conflict_stream << "("; + auto it = conflict_columns_.begin(); + conflict_stream << *it++; + for(; it != conflict_columns_.end(); ++it) { + conflict_stream << ", " << *it; + } + conflict_stream << ") "; + } + + if (conflict_action_ == ConflictAction::DoNothing) { + conflict_stream << "DO NOTHING"; + } else if (conflict_action_ == ConflictAction::DoUpdate) { + conflict_stream << "DO UPDATE SET "; + bool first = true; + for (const auto& col : column_names_) { + if (!first) conflict_stream << ", "; + conflict_stream << col << " = EXCLUDED." << col; + first = false; + } + if (!update_condition_.empty()) { + conflict_stream << " WHERE " << update_condition_; + } + } + } + + conflict_clause_ = conflict_stream.str(); + } + + void initializeStream() { + if (with_copy_) { + copy_stream_.emplace(pqxx::stream_to::table(txn_, {table_name_}, column_names_)); + return; + } + + insert_stream_.str(""); + insert_stream_.clear(); + is_first_row_ = true; + + // Build INSERT part + insert_stream_ << "INSERT INTO " << table_name_ << " ("; + bool first = true; + for (const auto& col : column_names_) { + if (!first) insert_stream_ << ", "; + insert_stream_ << col; + first = false; + } + insert_stream_ << ") VALUES "; + } + +public: + template + void insert_row(std::tuple row) { + if (std::tuple_size::value != column_names_.size()) { + throw std::runtime_error("row size doesn't match column names size"); + } + if (with_copy_) { + copy_stream_->write_row(row); + return; + } + + if (conflict_clause_added_) { + throw std::runtime_error("can't insert row after conflict clause"); + } + + if (!is_first_row_) { + insert_stream_ << ","; + } + is_first_row_ = false; + + insert_stream_ << "("; + bool first = true; + std::apply([&](const auto&... args) { + ((insert_stream_ << (first ? "" : ",") << txn_.quote(args), first = false), ...); + }, row); + insert_stream_ << ")"; + } + + std::string get_str() { + if (with_copy_) { + throw std::runtime_error("get_str not supported with COPY mode"); + } + if (is_first_row_) { + return ""; + } + if (!conflict_clause_added_) { + insert_stream_ << conflict_clause_ << ";"; + conflict_clause_added_ = true; + } + return insert_stream_.str(); + } + + void finish() { + if (with_copy_) { + copy_stream_->complete(); + return; + } + if (is_first_row_) { + return; + } + + txn_.exec0(get_str()); + } +}; std::string content_to_json_string(const std::map &content) { td::JsonBuilder jetton_content_json; @@ -84,7 +374,6 @@ std::unordered_set msg_bodies_in_progress; std::mutex messages_in_progress_mutex; std::mutex latest_account_states_update_mutex; - // // InsertBatchPostgres // @@ -93,28 +382,21 @@ void InsertBatchPostgres::start_up() { alarm(); } - void InsertBatchPostgres::alarm() { try { pqxx::connection c(connection_string_); - if (!c.is_open()) { - promise_.set_error(td::Status::Error(ErrorCode::DB_ERROR, "Failed to open database")); - return; - } - // update account states pqxx::work txn(c); - - // prepare queries + insert_blocks(txn, with_copy_); + insert_shard_state(txn, with_copy_); + insert_transactions(txn, with_copy_); + insert_messages(txn, with_copy_); + insert_account_states(txn, with_copy_); + insert_jetton_transfers(txn, with_copy_); + insert_jetton_burns(txn, with_copy_); + insert_nft_transfers(txn, with_copy_); + insert_traces(txn, with_copy_); std::string insert_under_mutex_query; - insert_blocks(txn); - insert_shard_state(txn); - insert_transactions(txn); - insert_messages(txn); - insert_account_states(txn); - insert_jetton_transfers(txn); - insert_jetton_burns(txn); - insert_nft_transfers(txn); insert_under_mutex_query += insert_jetton_masters(txn); insert_under_mutex_query += insert_jetton_wallets(txn); insert_under_mutex_query += insert_nft_collections(txn); @@ -122,9 +404,7 @@ void InsertBatchPostgres::alarm() { insert_under_mutex_query += insert_getgems_nft_auctions(txn); insert_under_mutex_query += insert_getgems_nft_sales(txn); insert_under_mutex_query += insert_latest_account_states(txn); - insert_under_mutex_query += insert_traces(txn); - - // execute queries + { std::lock_guard guard(latest_account_states_update_mutex); txn.exec0(insert_under_mutex_query); @@ -136,15 +416,17 @@ void InsertBatchPostgres::alarm() { } promise_.set_value(td::Unit()); stop(); - return; + } catch (const pqxx::integrity_constraint_violation &e) { + LOG(WARNING) << "Error COPY to PG: " << e.what(); + LOG(WARNING) << "Apparently this block already exists in the database. Nevertheless we retry with INSERT ... ON CONFLICT ..."; + with_copy_ = false; + alarm_timestamp() = td::Timestamp::now(); } catch (const std::exception &e) { LOG(ERROR) << "Error inserting to PG: " << e.what(); - ++retry_count_; + alarm_timestamp() = td::Timestamp::in(1.0); } - alarm_timestamp() = td::Timestamp::in(10.0); } - std::string InsertBatchPostgres::stringify(schema::ComputeSkipReason compute_skip_reason) { switch (compute_skip_reason) { case schema::ComputeSkipReason::cskip_no_state: return "no_state"; @@ -155,7 +437,6 @@ std::string InsertBatchPostgres::stringify(schema::ComputeSkipReason compute_ski UNREACHABLE(); } - std::string InsertBatchPostgres::stringify(schema::AccStatusChange acc_status_change) { switch (acc_status_change) { case schema::AccStatusChange::acst_unchanged: return "unchanged"; @@ -165,7 +446,6 @@ std::string InsertBatchPostgres::stringify(schema::AccStatusChange acc_status_ch UNREACHABLE(); } - std::string InsertBatchPostgres::stringify(schema::AccountStatus account_status) { switch (account_status) { @@ -186,605 +466,482 @@ std::string InsertBatchPostgres::stringify(schema::Trace::State state) { UNREACHABLE(); } -std::string InsertBatchPostgres::insert_blocks(pqxx::work &txn) { - std::ostringstream query; - query << "INSERT INTO blocks (workchain, shard, seqno, root_hash, file_hash, mc_block_workchain, " - "mc_block_shard, mc_block_seqno, global_id, version, after_merge, before_split, " - "after_split, want_merge, want_split, key_block, vert_seqno_incr, flags, gen_utime, start_lt, " - "end_lt, validator_list_hash_short, gen_catchain_seqno, min_ref_mc_seqno, " - "prev_key_block_seqno, vert_seqno, master_ref_seqno, rand_seed, created_by, tx_count, prev_blocks) VALUES "; - - bool is_first = true; +void InsertBatchPostgres::insert_blocks(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = { + "workchain", "shard", "seqno", "root_hash", "file_hash", "mc_block_workchain", "mc_block_shard", "mc_block_seqno", + "global_id", "version", "after_merge", "before_split", "after_split", "want_merge", "want_split", "key_block", + "vert_seqno_incr", "flags", "gen_utime", "start_lt", "end_lt", "validator_list_hash_short", "gen_catchain_seqno", + "min_ref_mc_seqno", "prev_key_block_seqno", "vert_seqno", "master_ref_seqno", "rand_seed", "created_by", "tx_count", "prev_blocks" + }; + PopulateTableStream stream(txn, "blocks", columns, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoNothing(); + } - int count = 0; + // Prepare data for (const auto& task : insert_tasks_) { for (const auto& block : task.parsed_block_->blocks_) { - td::StringBuilder prev_blocks_str; - prev_blocks_str << "{"; - bool first_prev_block = true; - for(const auto &prev : block.prev_blocks) { - if (first_prev_block) { - first_prev_block = false; - } else { - prev_blocks_str << ", "; - } - prev_blocks_str << "\"(" << prev.workchain << ", " << prev.shard << ", " << prev.seqno << ")\""; - } - prev_blocks_str << "}"; - - if (is_first) { - is_first = false; - } else { - query << ", "; - } - ++count; - query << "(" - << block.workchain << "," - << block.shard << "," - << block.seqno << "," - << txn.quote(block.root_hash) << "," - << txn.quote(block.file_hash) << "," - << TO_SQL_OPTIONAL(block.mc_block_workchain) << "," - << TO_SQL_OPTIONAL(block.mc_block_shard) << "," - << TO_SQL_OPTIONAL(block.mc_block_seqno) << "," - << block.global_id << "," - << block.version << "," - << TO_SQL_BOOL(block.after_merge) << "," - << TO_SQL_BOOL(block.before_split) << "," - << TO_SQL_BOOL(block.after_split) << "," - << TO_SQL_BOOL(block.want_merge) << "," - << TO_SQL_BOOL(block.want_split) << "," - << TO_SQL_BOOL(block.key_block) << "," - << TO_SQL_BOOL(block.vert_seqno_incr) << "," - << block.flags << "," - << block.gen_utime << "," - << block.start_lt << "," - << block.end_lt << "," - << block.validator_list_hash_short << "," - << block.gen_catchain_seqno << "," - << block.min_ref_mc_seqno << "," - << block.prev_key_block_seqno << "," - << block.vert_seqno << "," - << TO_SQL_OPTIONAL(block.master_ref_seqno) << "," - << txn.quote(block.rand_seed) << "," - << txn.quote(block.created_by) << "," - << block.transactions.size() << "," - << txn.quote(prev_blocks_str.as_cslice().str()) - << ")"; + auto tuple = std::make_tuple( + block.workchain, + block.shard, + block.seqno, + block.root_hash, + block.file_hash, + block.mc_block_workchain, + block.mc_block_shard, + block.mc_block_seqno, + block.global_id, + block.version, + block.after_merge, + block.before_split, + block.after_split, + block.want_merge, + block.want_split, + block.key_block, + block.vert_seqno_incr, + block.flags, + block.gen_utime, + block.start_lt, + block.end_lt, + block.validator_list_hash_short, + block.gen_catchain_seqno, + block.min_ref_mc_seqno, + block.prev_key_block_seqno, + block.vert_seqno, + block.master_ref_seqno, + block.rand_seed, + block.created_by, + block.transactions.size(), + block.prev_blocks + ); + stream.insert_row(std::move(tuple)); } } - if (is_first) { - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - txn.exec0(query.str()); - - // LOG(DEBUG) << "Running SQL query: " << query.str(); - // LOG(INFO) << "Blocks query size: " << double(query.str().length()) / 1024 / 1024; - return ""; + stream.finish(); } -std::string InsertBatchPostgres::insert_shard_state(pqxx::work &txn) { - std::ostringstream query; - query << "INSERT INTO shard_state (mc_seqno, workchain, shard, seqno) VALUES "; +void InsertBatchPostgres::insert_shard_state(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = { + "mc_seqno", "workchain", "shard", "seqno" + }; + PopulateTableStream stream(txn, "shard_state", columns, 1000); + if (!with_copy) { + stream.setConflictDoNothing(); + } - bool is_first = true; for (const auto& task : insert_tasks_) { for (const auto& shard : task.parsed_block_->shard_state_) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - query << "(" - << shard.mc_seqno << "," - << shard.workchain << "," - << shard.shard << "," - << shard.seqno - << ")"; + auto tuple = std::make_tuple( + shard.mc_seqno, + shard.workchain, + shard.shard, + shard.seqno + ); + stream.insert_row(std::move(tuple)); } } - if (is_first) { - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - txn.exec0(query.str()); - - // LOG(DEBUG) << "Running SQL query: " << query.str(); - return ""; + stream.finish(); } -std::string InsertBatchPostgres::insert_transactions(pqxx::work &txn) { - std::ostringstream query; - query << "INSERT INTO transactions (account, hash, lt, block_workchain, block_shard, block_seqno, " - "mc_block_seqno, trace_id, prev_trans_hash, prev_trans_lt, now, " - "orig_status, end_status, total_fees, total_fees_extra_currencies, " - "account_state_hash_before, account_state_hash_after, descr, aborted, destroyed, " - "credit_first, is_tock, installed, storage_fees_collected, " - "storage_fees_due, storage_status_change, credit_due_fees_collected, " - "credit, credit_extra_currencies, compute_skipped, skipped_reason, compute_success, " - "compute_msg_state_used, compute_account_activated, compute_gas_fees, " - "compute_gas_used, compute_gas_limit, compute_gas_credit, compute_mode, " - "compute_exit_code, compute_exit_arg, compute_vm_steps, " - "compute_vm_init_state_hash, compute_vm_final_state_hash, action_success, " - "action_valid, action_no_funds, action_status_change, action_total_fwd_fees, " - "action_total_action_fees, action_result_code, action_result_arg, " - "action_tot_actions, action_spec_actions, action_skipped_actions, " - "action_msgs_created, action_action_list_hash, action_tot_msg_size_cells, " - "action_tot_msg_size_bits, bounce, bounce_msg_size_cells, bounce_msg_size_bits, " - "bounce_req_fwd_fees, bounce_msg_fees, bounce_fwd_fees, split_info_cur_shard_pfx_len, " - "split_info_acc_split_depth, split_info_this_addr, split_info_sibling_addr) VALUES "; - - auto store_storage_ph = [&](const schema::TrStoragePhase& storage_ph) { - query << storage_ph.storage_fees_collected << "," - << TO_SQL_OPTIONAL_REFINT(storage_ph.storage_fees_due) << "," - << txn.quote(stringify(storage_ph.status_change)) << ","; +template +using tuple_cat_t = decltype(std::tuple_cat(std::declval()...)); + +void InsertBatchPostgres::insert_transactions(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = { + "account", "hash", "lt", "block_workchain", "block_shard", "block_seqno", "mc_block_seqno", "trace_id", + "prev_trans_hash", "prev_trans_lt", "now", "orig_status", "end_status", "total_fees", "total_fees_extra_currencies", + "account_state_hash_before", "account_state_hash_after", "descr", "aborted", "destroyed", "credit_first", "is_tock", + "installed", "storage_fees_collected", "storage_fees_due", "storage_status_change", "credit_due_fees_collected", + "credit", "credit_extra_currencies", "compute_skipped", "skipped_reason", "compute_success", "compute_msg_state_used", + "compute_account_activated", "compute_gas_fees", "compute_gas_used", "compute_gas_limit", "compute_gas_credit", + "compute_mode", "compute_exit_code", "compute_exit_arg", "compute_vm_steps", "compute_vm_init_state_hash", + "compute_vm_final_state_hash", "action_success", "action_valid", "action_no_funds", "action_status_change", + "action_total_fwd_fees", "action_total_action_fees", "action_result_code", "action_result_arg", "action_tot_actions", + "action_spec_actions", "action_skipped_actions", "action_msgs_created", "action_action_list_hash", + "action_tot_msg_size_cells", "action_tot_msg_size_bits", "bounce", "bounce_msg_size_cells", "bounce_msg_size_bits", + "bounce_req_fwd_fees", "bounce_msg_fees", "bounce_fwd_fees", "split_info_cur_shard_pfx_len", "split_info_acc_split_depth", + "split_info_this_addr", "split_info_sibling_addr" + }; + + PopulateTableStream stream(txn, "transactions", columns, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoNothing(); + } + + using storage_ph_tuple = std::tuple, std::optional, std::optional>; + auto store_storage_ph = [&](const schema::TrStoragePhase& storage_ph) -> storage_ph_tuple { + return {storage_ph.storage_fees_collected, storage_ph.storage_fees_due, stringify(storage_ph.status_change)}; }; - auto store_empty_storage_ph = [&]() { - query << "NULL, NULL, NULL,"; + auto store_empty_storage_ph = [&]() -> storage_ph_tuple { + return {std::nullopt, std::nullopt, std::nullopt}; }; - auto store_credit_ph = [&](const schema::TrCreditPhase& credit_ph) { - query << TO_SQL_OPTIONAL_REFINT(credit_ph.due_fees_collected) << "," - << credit_ph.credit.grams << "," - << txn.quote(extra_currencies_to_json_string(credit_ph.credit.extra_currencies)) << ","; + + using credit_ph_tuple = std::tuple, std::optional, std::optional>; + auto store_credit_ph = [](const schema::TrCreditPhase& credit_ph) -> credit_ph_tuple { + return {credit_ph.due_fees_collected, credit_ph.credit.grams, extra_currencies_to_json_string(credit_ph.credit.extra_currencies)}; }; - auto store_empty_credit_ph = [&]() { - query << "NULL,NULL,NULL,"; + auto store_empty_credit_ph = [&]() -> credit_ph_tuple { + return {std::nullopt, std::nullopt, std::nullopt}; }; - auto store_compute_ph = [&](const schema::TrComputePhase& compute_ph) { - if (auto* v = std::get_if(&compute_ph)) { - query << "TRUE," - << txn.quote(stringify(v->reason)) << "," - << "NULL,NULL,NULL,NULL,NULL,NULL,NULL," - << "NULL,NULL,NULL,NULL,NULL,NULL,"; - } - else if (auto* v = std::get_if(&compute_ph)) { - query << "FALSE," - << "NULL," - << TO_SQL_BOOL(v->success) << "," - << TO_SQL_BOOL(v->msg_state_used) << "," - << TO_SQL_BOOL(v->account_activated) << "," - << v->gas_used << "," - << v->gas_fees << "," - << v->gas_limit << "," - << TO_SQL_OPTIONAL(v->gas_credit) << "," - << std::to_string(v->mode) << "," - << v->exit_code << "," - << TO_SQL_OPTIONAL(v->exit_arg) << "," - << v->vm_steps << "," - << txn.quote(td::base64_encode(v->vm_init_state_hash.as_slice())) << "," - << txn.quote(td::base64_encode(v->vm_final_state_hash.as_slice())) << ","; + + using compute_ph_tuple = std::tuple, std::optional, std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, std::optional, + std::optional, std::optional>; + auto store_compute_ph = [&](const schema::TrComputePhase& compute_ph) -> compute_ph_tuple { + return std::visit([&](auto&& arg) -> compute_ph_tuple { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return {true, stringify(arg.reason), std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt}; + } else if constexpr (std::is_same_v) { + return {false, std::nullopt, arg.success, arg.msg_state_used, arg.account_activated, + arg.gas_used, arg.gas_fees, arg.gas_limit, arg.gas_credit, + arg.mode, arg.exit_code, arg.exit_arg, arg.vm_steps, + arg.vm_init_state_hash, arg.vm_final_state_hash}; + } else { + UNREACHABLE(); } + }, compute_ph); }; - auto store_empty_compute_ph = [&]() { - query << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL,"; + auto store_empty_compute_ph = [&]() -> compute_ph_tuple { + return {std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt}; }; - auto store_action_ph = [&](const schema::TrActionPhase& action) { - query << TO_SQL_BOOL(action.success) << "," - << TO_SQL_BOOL(action.valid) << "," - << TO_SQL_BOOL(action.no_funds) << "," - << txn.quote(stringify(action.status_change)) << "," - << TO_SQL_OPTIONAL_REFINT(action.total_fwd_fees) << "," - << TO_SQL_OPTIONAL_REFINT(action.total_action_fees) << "," - << action.result_code << "," - << TO_SQL_OPTIONAL(action.result_arg) << "," - << action.tot_actions << "," - << action.spec_actions << "," - << action.skipped_actions << "," - << action.msgs_created << "," - << txn.quote(td::base64_encode(action.action_list_hash.as_slice())) << "," - << action.tot_msg_size.cells << "," - << action.tot_msg_size.bits << ","; + + using action_ph_tuple = std::tuple, std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional>; + auto store_action_ph = [&](const schema::TrActionPhase& action) -> action_ph_tuple { + return {action.success, action.valid, action.no_funds, stringify(action.status_change), + action.total_fwd_fees, action.total_action_fees, action.result_code, action.result_arg, + action.tot_actions, action.spec_actions, action.skipped_actions, action.msgs_created, + action.action_list_hash, action.tot_msg_size.cells, action.tot_msg_size.bits}; }; - auto store_empty_action_ph = [&]() { - query << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL,"; + auto store_empty_action_ph = [&]() -> action_ph_tuple { + return {std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt}; }; - auto store_bounce_ph = [&](const schema::TrBouncePhase& bounce) { - if(auto* v = std::get_if(&bounce)) { - query << "'negfunds'," - << "NULL,NULL,NULL,NULL,NULL,"; - } else if (auto* v = std::get_if(&bounce)) { - query << "'nofunds'," - << v->msg_size.cells << "," - << v->msg_size.bits << "," - << v->req_fwd_fees << "," - << "NULL,NULL,"; - } else if (auto* v = std::get_if(&bounce)) { - query << "'ok'," - << v->msg_size.cells << "," - << v->msg_size.bits << "," - << "NULL," - << v->msg_fees << "," - << v->fwd_fees << ","; + + using bounce_ph_tuple = std::tuple, std::optional, std::optional, + std::optional, std::optional, std::optional>; + auto store_bounce_ph = [&](const schema::TrBouncePhase& bounce) -> bounce_ph_tuple { + return std::visit([&](auto&& arg) -> bounce_ph_tuple { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return {"negfunds", std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}; + } else if constexpr (std::is_same_v) { + return {"nofunds", arg.msg_size.cells, arg.msg_size.bits, arg.req_fwd_fees, std::nullopt, std::nullopt}; + } else if constexpr (std::is_same_v) { + return {"ok", arg.msg_size.cells, arg.msg_size.bits, std::nullopt, arg.msg_fees, arg.fwd_fees}; + } else { + UNREACHABLE(); } + }, bounce); }; - auto store_empty_bounce_ph = [&]() { - query << "NULL,NULL,NULL,NULL,NULL,NULL,"; + auto store_empty_bounce_ph = [&]() -> bounce_ph_tuple { + return {std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}; }; - auto store_split_info = [&](const schema::SplitMergeInfo& split_info) { - query << split_info.cur_shard_pfx_len << "," - << split_info.acc_split_depth << "," - << txn.quote(td::base64_encode(split_info.this_addr.as_slice())) << "," - << txn.quote(td::base64_encode(split_info.sibling_addr.as_slice())) << ""; + + using split_info_tuple = std::tuple, std::optional, std::optional, std::optional>; + auto store_split_info = [&](const schema::SplitMergeInfo& split_info) -> split_info_tuple { + return {split_info.cur_shard_pfx_len, split_info.acc_split_depth, split_info.this_addr, split_info.sibling_addr}; }; - auto store_empty_split_info = [&]() { - query << "NULL,NULL,NULL,NULL"; + auto store_empty_split_info = [&]() -> split_info_tuple { + return {std::nullopt, std::nullopt, std::nullopt, std::nullopt}; + }; + + using transaction_descr_begin_tuple = std::tuple, std::optional, + std::optional, std::optional, std::optional>; + + using transaction_descr = tuple_cat_t; + + auto store_transaction_descr_ord = [&](const schema::TransactionDescr_ord& descr) -> transaction_descr { + auto begin_tuple = std::make_tuple("ord", descr.aborted, descr.destroyed, descr.credit_first, std::nullopt, std::nullopt); + auto storage_ph = descr.storage_ph ? store_storage_ph(descr.storage_ph.value()) : store_empty_storage_ph(); + auto credit_ph = descr.credit_ph ? store_credit_ph(descr.credit_ph.value()) : store_empty_credit_ph(); + auto compute_ph = store_compute_ph(descr.compute_ph); + auto action_ph = descr.action ? store_action_ph(descr.action.value()) : store_empty_action_ph(); + auto bounce_ph = descr.bounce ? store_bounce_ph(descr.bounce.value()) : store_empty_bounce_ph(); + auto split_info = store_empty_split_info(); + return std::tuple_cat(std::move(begin_tuple), std::move(storage_ph), std::move(credit_ph), + std::move(compute_ph), std::move(action_ph), std::move(bounce_ph), std::move(split_info)); + }; + auto store_transaction_descr_storage = [&](const schema::TransactionDescr_storage& descr) -> transaction_descr { + auto begin_tuple = std::make_tuple("storage", std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt); + auto storage_ph = store_storage_ph(descr.storage_ph); + auto credit_ph = store_empty_credit_ph(); + auto compute_ph = store_empty_compute_ph(); + auto action_ph = store_empty_action_ph(); + auto bounce_ph = store_empty_bounce_ph(); + auto split_info = store_empty_split_info(); + return std::tuple_cat(std::move(begin_tuple), std::move(storage_ph), std::move(credit_ph), + std::move(compute_ph), std::move(action_ph), std::move(bounce_ph), std::move(split_info)); + }; + auto store_transaction_descr_tick_tock = [&](const schema::TransactionDescr_tick_tock& descr) -> transaction_descr { + auto begin_tuple = std::make_tuple("tick_tock", descr.aborted, descr.destroyed, std::nullopt, descr.is_tock, std::nullopt); + auto storage_ph = store_storage_ph(descr.storage_ph); + auto credit_ph = store_empty_credit_ph(); + auto compute_ph = store_compute_ph(descr.compute_ph); + auto action_ph = descr.action ? store_action_ph(descr.action.value()) : store_empty_action_ph(); + auto bounce_ph = store_empty_bounce_ph(); + auto split_info = store_empty_split_info(); + return std::tuple_cat(std::move(begin_tuple), std::move(storage_ph), std::move(credit_ph), + std::move(compute_ph), std::move(action_ph), std::move(bounce_ph), std::move(split_info)); + }; + auto store_transaction_descr_split_prepare = [&](const schema::TransactionDescr_split_prepare& descr) -> transaction_descr { + auto begin_tuple = std::make_tuple("split_prepare", descr.aborted, descr.destroyed, std::nullopt, std::nullopt, std::nullopt); + auto storage_ph = descr.storage_ph ? store_storage_ph(descr.storage_ph.value()) : store_empty_storage_ph(); + auto credit_ph = store_empty_credit_ph(); + auto compute_ph = store_compute_ph(descr.compute_ph); + auto action_ph = descr.action ? store_action_ph(descr.action.value()) : store_empty_action_ph(); + auto bounce_ph = store_empty_bounce_ph(); + auto split_info = store_split_info(descr.split_info); + return std::tuple_cat(std::move(begin_tuple), std::move(storage_ph), std::move(credit_ph), + std::move(compute_ph), std::move(action_ph), std::move(bounce_ph), std::move(split_info)); + }; + auto store_transaction_descr_split_install = [&](const schema::TransactionDescr_split_install& descr) -> transaction_descr { + auto begin_tuple = std::make_tuple("split_install", std::nullopt, std::nullopt, std::nullopt, std::nullopt, descr.installed); + auto storage_pt = store_empty_storage_ph(); + auto credit_ph = store_empty_credit_ph(); + auto compute_ph = store_empty_compute_ph(); + auto action_ph = store_empty_action_ph(); + auto bounce_ph = store_empty_bounce_ph(); + auto split_info = store_split_info(descr.split_info); + return std::tuple_cat(std::move(begin_tuple), std::move(storage_pt), std::move(credit_ph), + std::move(compute_ph), std::move(action_ph), std::move(bounce_ph), std::move(split_info)); + }; + auto store_transaction_descr_merge_prepare = [&](const schema::TransactionDescr_merge_prepare& descr) -> transaction_descr { + auto begin_tuple = std::make_tuple("merge_prepare", descr.aborted, std::nullopt, std::nullopt, std::nullopt, std::nullopt); + auto storage_ph = store_storage_ph(descr.storage_ph); + auto credit_ph = store_empty_credit_ph(); + auto compute_ph = store_empty_compute_ph(); + auto action_ph = store_empty_action_ph(); + auto bounce_ph = store_empty_bounce_ph(); + auto split_info = store_split_info(descr.split_info); + return std::tuple_cat(std::move(begin_tuple), std::move(storage_ph), std::move(credit_ph), + std::move(compute_ph), std::move(action_ph), std::move(bounce_ph), std::move(split_info)); + }; + auto store_transaction_descr_merge_install = [&](const schema::TransactionDescr_merge_install& descr) -> transaction_descr { + auto begin_tuple = std::make_tuple("merge_install", descr.aborted, descr.destroyed, std::nullopt, std::nullopt, std::nullopt); + auto storage_ph = descr.storage_ph ? store_storage_ph(descr.storage_ph.value()) : store_empty_storage_ph(); + auto credit_ph = descr.credit_ph ? store_credit_ph(descr.credit_ph.value()) : store_empty_credit_ph(); + auto compute_ph = store_compute_ph(descr.compute_ph); + auto action_ph = descr.action ? store_action_ph(descr.action.value()) : store_empty_action_ph(); + auto bounce_ph = store_empty_bounce_ph(); + auto split_info = store_split_info(descr.split_info); + return std::tuple_cat(std::move(begin_tuple), std::move(storage_ph), std::move(credit_ph), + std::move(compute_ph), std::move(action_ph), std::move(bounce_ph), std::move(split_info)); }; - bool is_first = true; for (const auto& task : insert_tasks_) { for (const auto &blk : task.parsed_block_->blocks_) { for (const auto& transaction : blk.transactions) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - query << "(" - << txn.quote(convert::to_raw_address(transaction.account)) << "," - << txn.quote(td::base64_encode(transaction.hash.as_slice())) << "," - << transaction.lt << "," - << blk.workchain << "," - << blk.shard << "," - << blk.seqno << "," - << TO_SQL_OPTIONAL(blk.mc_block_seqno) << "," - << txn.quote(td::base64_encode(transaction.trace_id.as_slice())) << "," - << txn.quote(td::base64_encode(transaction.prev_trans_hash.as_slice())) << "," - << transaction.prev_trans_lt << "," - << transaction.now << "," - << txn.quote(stringify(transaction.orig_status)) << "," - << txn.quote(stringify(transaction.end_status)) << "," - << transaction.total_fees.grams << "," - << txn.quote(extra_currencies_to_json_string(transaction.total_fees.extra_currencies)) << "," - << txn.quote(td::base64_encode(transaction.account_state_hash_before.as_slice())) << "," - << txn.quote(td::base64_encode(transaction.account_state_hash_after.as_slice())) << ","; - // insert description - if (auto* v = std::get_if(&transaction.description)) { - query << "'ord'," - << TO_SQL_BOOL(v->aborted) << "," - << TO_SQL_BOOL(v->destroyed) << "," - << TO_SQL_BOOL(v->credit_first) << "," - << "NULL," - << "NULL,"; - store_storage_ph(v->storage_ph); - store_credit_ph(v->credit_ph); - store_compute_ph(v->compute_ph); - if (v->action) { - store_action_ph(v->action.value()); - } else { - store_empty_action_ph(); - } - if (v->bounce) { - store_bounce_ph(v->bounce.value()); - } else { - store_empty_bounce_ph(); - } - store_empty_split_info(); - } - else if (auto* v = std::get_if(&transaction.description)) { - query << "'storage'," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << "NULL,"; - store_storage_ph(v->storage_ph); - store_empty_credit_ph(); - store_empty_compute_ph(); - store_empty_action_ph(); - store_empty_bounce_ph(); - store_empty_split_info(); - } - else if (auto* v = std::get_if(&transaction.description)) { - query << "'tick_tock'," - << TO_SQL_BOOL(v->aborted) << "," - << TO_SQL_BOOL(v->destroyed) << "," - << "NULL," - << TO_SQL_BOOL(v->is_tock) << "," - << "NULL,"; - store_storage_ph(v->storage_ph); - store_empty_credit_ph(); - store_compute_ph(v->compute_ph); - if (v->action) { - store_action_ph(v->action.value()); - } else { - store_empty_action_ph(); - } - store_empty_bounce_ph(); - store_empty_split_info(); - } - else if (auto* v = std::get_if(&transaction.description)) { - query << "'split_prepare'," - << TO_SQL_BOOL(v->aborted) << "," - << TO_SQL_BOOL(v->destroyed) << "," - << "NULL," - << "NULL," - << "NULL,"; - if (v->storage_ph) { - store_storage_ph(v->storage_ph.value()); - } else { - store_empty_storage_ph(); - } - store_empty_credit_ph(); - store_compute_ph(v->compute_ph); - if (v->action) { - store_action_ph(v->action.value()); - } else { - store_empty_action_ph(); - } - store_empty_bounce_ph(); - store_split_info(v->split_info); - } - else if (auto* v = std::get_if(&transaction.description)) { - query << "'split_install'," - << "NULL," - << "NULL," - << "NULL," - << "NULL," - << TO_SQL_BOOL(v->installed) << ","; - store_empty_storage_ph(); - store_empty_credit_ph(); - store_empty_compute_ph(); - store_empty_action_ph(); - store_empty_bounce_ph(); - store_split_info(v->split_info); - } - else if (auto* v = std::get_if(&transaction.description)) { - query << "'merge_prepare'," - << TO_SQL_BOOL(v->aborted) << "," - << "NULL," - << "NULL," - << "NULL," - << "NULL,"; - store_storage_ph(v->storage_ph); - store_empty_credit_ph(); - store_empty_compute_ph(); - store_empty_action_ph(); - store_empty_bounce_ph(); - store_split_info(v->split_info); - } - else if (auto* v = std::get_if(&transaction.description)) { - query << "'merge_install'," - << TO_SQL_BOOL(v->aborted) << "," - << TO_SQL_BOOL(v->destroyed) << "," - << "NULL," - << "NULL," - << "NULL,"; - if (v->storage_ph) { - store_storage_ph(v->storage_ph.value()); - } else { - store_empty_storage_ph(); - } - if (v->credit_ph) { - store_credit_ph(v->credit_ph.value()); - } else { - store_empty_credit_ph(); - } - store_compute_ph(v->compute_ph); - if (v->action) { - store_action_ph(v->action.value()); + auto tx_common_tuple = std::make_tuple( + transaction.account, + transaction.hash, + transaction.lt, + blk.workchain, + blk.shard, + blk.seqno, + blk.mc_block_seqno, + transaction.trace_id, + transaction.prev_trans_hash, + transaction.prev_trans_lt, + transaction.now, + stringify(transaction.orig_status), + stringify(transaction.end_status), + transaction.total_fees.grams, + extra_currencies_to_json_string(transaction.total_fees.extra_currencies), + transaction.account_state_hash_before, + transaction.account_state_hash_after + ); + auto descr_tuple = std::visit([&](auto&& arg) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return store_transaction_descr_ord(arg); + } else if constexpr (std::is_same_v) { + return store_transaction_descr_storage(arg); + } else if constexpr (std::is_same_v) { + return store_transaction_descr_tick_tock(arg); + } else if constexpr (std::is_same_v) { + return store_transaction_descr_split_prepare(arg); + } else if constexpr (std::is_same_v) { + return store_transaction_descr_split_install(arg); + } else if constexpr (std::is_same_v) { + return store_transaction_descr_merge_prepare(arg); + } else if constexpr (std::is_same_v) { + return store_transaction_descr_merge_install(arg); } else { - store_empty_action_ph(); + UNREACHABLE(); } - store_empty_bounce_ph(); - store_split_info(v->split_info); - } - query << ")"; + }, transaction.description); + auto row_tuple = std::tuple_cat(std::move(tx_common_tuple), std::move(descr_tuple)); + stream.insert_row(std::move(row_tuple)); } } } - if (is_first) { - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - txn.exec0(query.str()); - // LOG(INFO) << "Transactions query size: " << double(query.str().length()) / 1024 / 1024; - return ""; + stream.finish(); } +void InsertBatchPostgres::insert_messages(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = {"tx_hash", "tx_lt", "msg_hash", "direction", "trace_id", "source", "destination", + "value", "value_extra_currencies", "fwd_fee", "ihr_fee", "created_lt", "created_at", + "opcode", "ihr_disabled", "bounce", "bounced", "import_fee", "body_hash", "init_state_hash"}; + PopulateTableStream stream(txn, "messages", columns, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoNothing(); + } -std::string InsertBatchPostgres::insert_messages(pqxx::work &txn) { - std::vector> msg_bodies; - { - std::ostringstream query; - query << "INSERT INTO messages (tx_hash, tx_lt, msg_hash, direction, trace_id, source, " - "destination, value, value_extra_currencies, fwd_fee, ihr_fee, created_lt, " - "created_at, opcode, ihr_disabled, bounce, bounced, " - "import_fee, body_hash, init_state_hash) VALUES "; - bool is_first = true; - auto store_message = [&](const schema::Transaction& tx, const schema::Message& msg, std::string direction) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - // msg.import_fee is defined by user and can be too large for bigint, so we need to check it - // and if it is too large, we will insert NULL. - // TODO: change bigint to numeric - std::optional import_fee_val; - if (msg.import_fee) { - import_fee_val = msg.import_fee.value()->to_long(); - if (import_fee_val.value() == (~0ULL << 63)) { - LOG(WARNING) << "Import fee of msg " << msg.hash.to_hex() << " is too large for bigint: " << msg.import_fee.value(); - import_fee_val = std::nullopt; - } - } - query << "(" - << txn.quote(td::base64_encode(tx.hash.as_slice())) << "," - << tx.lt << "," - << txn.quote(td::base64_encode(msg.hash.as_slice())) << "," - << txn.quote(direction) << "," - << txn.quote(td::base64_encode(msg.trace_id.as_slice())) << "," - << TO_SQL_OPTIONAL_STRING(msg.source, txn) << "," - << TO_SQL_OPTIONAL_STRING(msg.destination, txn) << "," - << (msg.value ? msg.value->grams->to_dec_string() : "NULL") << "," - << (msg.value ? txn.quote(extra_currencies_to_json_string(msg.value->extra_currencies)) : "NULL") << "," - << TO_SQL_OPTIONAL_REFINT(msg.fwd_fee) << "," - << TO_SQL_OPTIONAL_REFINT(msg.ihr_fee) << "," - << TO_SQL_OPTIONAL(msg.created_lt) << "," - << TO_SQL_OPTIONAL(msg.created_at) << "," - << TO_SQL_OPTIONAL(msg.opcode) << "," - << TO_SQL_OPTIONAL_BOOL(msg.ihr_disabled) << "," - << TO_SQL_OPTIONAL_BOOL(msg.bounce) << "," - << TO_SQL_OPTIONAL_BOOL(msg.bounced) << "," - << TO_SQL_OPTIONAL(import_fee_val) << "," - << txn.quote(td::base64_encode(msg.body->get_hash().as_slice())) << "," - << (msg.init_state.not_null() ? txn.quote(td::base64_encode(msg.init_state->get_hash().as_slice())) : "NULL") - << ")"; - // collect unique message contents - { - std::lock_guard guard(messages_in_progress_mutex); - td::Bits256 body_hash = msg.body->get_hash().bits(); - if (msg_bodies_in_progress.find(body_hash) == msg_bodies_in_progress.end()) { - msg_bodies.push_back({body_hash, msg.body_boc}); - msg_bodies_in_progress.insert(body_hash); - } - if (msg.init_state_boc) { - td::Bits256 init_state_hash = msg.init_state->get_hash().bits(); - if (msg_bodies_in_progress.find(init_state_hash) == msg_bodies_in_progress.end()) { - msg_bodies.push_back({init_state_hash, msg.init_state_boc.value()}); - msg_bodies_in_progress.insert(init_state_hash); - } - } + auto store_message = [&](const schema::Transaction& tx, const schema::Message& msg, const std::string_view& direction) { + // msg.import_fee is defined by user and can be too large for bigint, so we need to check it + // and if it is too large, we will insert NULL. + // TODO: change bigint to numeric + std::optional import_fee_val; + if (msg.import_fee) { + import_fee_val = msg.import_fee.value()->to_long(); + if (import_fee_val.value() == (~0ULL << 63)) { + LOG(WARNING) << "Import fee of msg " << msg.hash.to_hex() << " is too large for bigint: " << msg.import_fee.value(); + import_fee_val = std::nullopt; } - }; + } + auto tuple = std::make_tuple( + tx.hash, + tx.lt, + msg.hash, + direction, + msg.trace_id, + msg.source, + msg.destination, + msg.value ? std::make_optional(msg.value->grams) : std::nullopt, + msg.value ? std::make_optional(extra_currencies_to_json_string(msg.value->extra_currencies)) : std::nullopt, + msg.fwd_fee, + msg.ihr_fee, + msg.created_lt, + msg.created_at, + msg.opcode, + msg.ihr_disabled, + msg.bounce, + msg.bounced, + import_fee_val, + msg.body->get_hash(), + msg.init_state.not_null() ? std::make_optional(msg.init_state->get_hash()) : std::nullopt + ); + stream.insert_row(std::move(tuple)); + }; + + std::vector> msg_bodies; + // we lock the message bodies to prevent multiple parallel queries for the same message + // otherwise it causes deadlocks + auto lock_msg_body = [&](const td::Bits256& body_hash, const std::string& body_boc) { + if (msg_bodies_in_progress.find(body_hash) == msg_bodies_in_progress.end()) { + msg_bodies.push_back({body_hash, body_boc}); + msg_bodies_in_progress.insert(body_hash); + } + }; + + { + std::lock_guard guard(messages_in_progress_mutex); for (const auto& task : insert_tasks_) { for (const auto &blk : task.parsed_block_->blocks_) { for (const auto& transaction : blk.transactions) { - if(transaction.in_msg.has_value()) { - store_message(transaction, transaction.in_msg.value(), "in"); + if (transaction.in_msg) { + lock_msg_body(transaction.in_msg->body->get_hash().bits(), transaction.in_msg->body_boc); + if (transaction.in_msg->init_state_boc) { + lock_msg_body(transaction.in_msg->init_state->get_hash().bits(), transaction.in_msg->init_state_boc.value()); + } } for (const auto& msg : transaction.out_msgs) { - store_message(transaction, msg, "out"); + lock_msg_body(msg.body->get_hash().bits(), msg.body_boc); + if (msg.init_state_boc) { + lock_msg_body(msg.init_state->get_hash().bits(), msg.init_state_boc.value()); + } } } } } - if (is_first) { - LOG(INFO) << "WFT???"; - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - // LOG(INFO) << "Messages query size: " << double(query.str().length()) / 1024 / 1024; - txn.exec0(query.str()); } - // insert message contents - { - // LOG(INFO) << "Insert " << msg_bodies.size() << " msg bodies. In progress: " << msg_bodies_in_progress.size(); - std::ostringstream query; - query << "INSERT INTO message_contents (hash, body) VALUES "; - bool is_first = true; - for(const auto& [body_hash, body] : msg_bodies) { - if (is_first) { - is_first = false; - } else { - query << ", "; + for (const auto& task : insert_tasks_) { + for (const auto &blk : task.parsed_block_->blocks_) { + for (const auto& transaction : blk.transactions) { + if(transaction.in_msg.has_value()) { + store_message(transaction, transaction.in_msg.value(), "in"); + } + for (const auto& msg : transaction.out_msgs) { + store_message(transaction, msg, "out"); + } } - query << "(" - << txn.quote(td::base64_encode(body_hash.as_slice())) << "," - << txn.quote(body) - << ")"; - } - if (!is_first) { - query << " ON CONFLICT DO NOTHING;\n"; - txn.exec0(query.str()); - } else { - LOG(WARNING) << "No message bodies in batch!"; } } + stream.finish(); + + PopulateTableStream bodies_stream(txn, "message_contents", {"hash", "body"}, 1000, false); + bodies_stream.setConflictDoNothing(); + + for (const auto& [body_hash, body] : msg_bodies) { + auto tuple = std::make_tuple(body_hash, body); + bodies_stream.insert_row(std::move(tuple)); + } + bodies_stream.finish(); - // unlock messages + // unlock message bodies { std::lock_guard guard(messages_in_progress_mutex); for (const auto& [body_hash, body] : msg_bodies) { msg_bodies_in_progress.erase(body_hash); } } - - return ""; } -std::string InsertBatchPostgres::insert_account_states(pqxx::work &txn) { - std::ostringstream query; - query << "INSERT INTO account_states (hash, account, balance, balance_extra_currencies, account_status, frozen_hash, code_hash, data_hash) VALUES "; - bool is_first = true; +void InsertBatchPostgres::insert_account_states(pqxx::work &txn, bool with_copy) { + PopulateTableStream stream(txn, "account_states", { + "hash", "account", "balance", "balance_extra_currencies", "account_status", "frozen_hash", "code_hash", "data_hash" + }, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoNothing(); + } + for (const auto& task : insert_tasks_) { for (const auto& account_state : task.parsed_block_->account_states_) { if (account_state.account_status == "nonexist") { // nonexist account state is inserted on DB initialization continue; } - if (is_first) { - is_first = false; - } else { - query << ", "; - } - std::optional frozen_hash; - if (account_state.frozen_hash) { - frozen_hash = td::base64_encode(account_state.frozen_hash.value().as_slice()); - } - std::optional code_hash; - if (account_state.code_hash) { - code_hash = td::base64_encode(account_state.code_hash.value().as_slice()); - } - std::optional data_hash; - if (account_state.data_hash) { - data_hash = td::base64_encode(account_state.data_hash.value().as_slice()); - } - query << "(" - << txn.quote(td::base64_encode(account_state.hash.as_slice())) << "," - << txn.quote(convert::to_raw_address(account_state.account)) << "," - << account_state.balance.grams << "," - << txn.quote(extra_currencies_to_json_string(account_state.balance.extra_currencies)) << "," - << txn.quote(account_state.account_status) << "," - << TO_SQL_OPTIONAL_STRING(frozen_hash, txn) << "," - << TO_SQL_OPTIONAL_STRING(code_hash, txn) << "," - << TO_SQL_OPTIONAL_STRING(data_hash, txn) - << ")"; + auto tuple = std::make_tuple( + account_state.hash, + account_state.account, + account_state.balance.grams, + extra_currencies_to_json_string(account_state.balance.extra_currencies), + account_state.account_status, + account_state.frozen_hash, + account_state.code_hash, + account_state.data_hash + ); + stream.insert_row(std::move(tuple)); } } - if (is_first) { - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - // LOG(DEBUG) << "Running SQL query: " << query.str(); - // LOG(INFO) << "Account states query size: " << double(query.str().length()) / 1024 / 1024; - txn.exec0(query.str()); - - return ""; + + stream.finish(); } std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { + std::initializer_list columns = { + "account", "account_friendly", "hash", "balance", "balance_extra_currencies", "account_status", "timestamp", + "last_trans_hash", "last_trans_lt", "frozen_hash", "data_hash", "code_hash", "data_boc", "code_boc" + }; + PopulateTableStream stream(txn, "latest_account_states", columns, 1000, false); + stream.setConflictDoUpdate({"account"}, "latest_account_states.last_trans_lt < EXCLUDED.last_trans_lt"); + std::unordered_map latest_account_states; for (const auto& task : insert_tasks_) { for (const auto& account_state : task.parsed_block_->account_states_) { @@ -799,21 +956,9 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { } } - std::ostringstream query; - bool is_first = true; - for (auto i = latest_account_states.begin(); i != latest_account_states.end(); ++i) { - auto& account_state = i->second; - if (is_first) { - query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, balance_extra_currencies, " - "account_status, timestamp, last_trans_hash, last_trans_lt, " - "frozen_hash, data_hash, code_hash, " - "data_boc, code_boc) VALUES "; - is_first = false; - } else { - query << ", "; - } - std::string code_str = "NULL"; - std::string data_str = "NULL"; + for (const auto& [_, account_state] : latest_account_states) { + std::optional code_str = std::nullopt; + std::optional data_str = std::nullopt; if (max_data_depth_ >= 0 && account_state.data.not_null() && (max_data_depth_ == 0 || account_state.data->get_depth() <= max_data_depth_)){ auto data_res = vm::std_boc_serialize(account_state.data); @@ -831,57 +976,29 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { if (code_res.is_ok()){ code_str = txn.quote(td::base64_encode(code_res.move_as_ok().as_slice().str())); } - if (code_str.length() > 128000) { - LOG(ERROR) << "Large account code:" << account_state.account; + if (code_str->length() > 128000) { + LOG(WARNING) << "Large account code: " << account_state.account; } } - std::optional frozen_hash; - if (account_state.frozen_hash) { - frozen_hash = td::base64_encode(account_state.frozen_hash.value().as_slice()); - } - std::optional code_hash; - if (account_state.code_hash) { - code_hash = td::base64_encode(account_state.code_hash.value().as_slice()); - } - std::optional data_hash; - if (account_state.data_hash) { - data_hash = td::base64_encode(account_state.data_hash.value().as_slice()); - } - query << "(" - << txn.quote(convert::to_raw_address(account_state.account)) << "," - << "NULL," - << txn.quote(td::base64_encode(account_state.hash.as_slice())) << "," - << account_state.balance.grams << "," - << txn.quote(extra_currencies_to_json_string(account_state.balance.extra_currencies)) << "," - << txn.quote(account_state.account_status) << "," - << account_state.timestamp << "," - << txn.quote(td::base64_encode(account_state.last_trans_hash.as_slice())) << "," - << account_state.last_trans_lt << "," - << TO_SQL_OPTIONAL_STRING(frozen_hash, txn) << "," - << TO_SQL_OPTIONAL_STRING(data_hash, txn) << "," - << TO_SQL_OPTIONAL_STRING(code_hash, txn) << "," - << data_str << "," - << code_str << ")"; - } - if (!is_first) { - query << " ON CONFLICT (account) DO UPDATE SET " - << "account_friendly = EXCLUDED.account_friendly, " - << "hash = EXCLUDED.hash, " - << "balance = EXCLUDED.balance, " - << "balance_extra_currencies = EXCLUDED.balance_extra_currencies, " - << "account_status = EXCLUDED.account_status, " - << "timestamp = EXCLUDED.timestamp, " - << "last_trans_hash = EXCLUDED.last_trans_hash, " - << "last_trans_lt = EXCLUDED.last_trans_lt, " - << "frozen_hash = EXCLUDED.frozen_hash, " - << "data_hash = EXCLUDED.data_hash, " - << "code_hash = EXCLUDED.code_hash, " - << "data_boc = EXCLUDED.data_boc, " - << "code_boc = EXCLUDED.code_boc " - << "WHERE latest_account_states.last_trans_lt < EXCLUDED.last_trans_lt;\n"; + auto tuple = std::make_tuple( + account_state.account, + std::nullopt, + account_state.hash, + account_state.balance.grams, + extra_currencies_to_json_string(account_state.balance.extra_currencies), + account_state.account_status, + account_state.timestamp, + account_state.last_trans_hash, + account_state.last_trans_lt, + account_state.frozen_hash, + account_state.data_hash, + account_state.code_hash, + data_str, + code_str + ); + stream.insert_row(std::move(tuple)); } - - return query.str(); + return stream.get_str(); } std::string InsertBatchPostgres::insert_jetton_masters(pqxx::work &txn) { @@ -900,44 +1017,34 @@ std::string InsertBatchPostgres::insert_jetton_masters(pqxx::work &txn) { } } - std::ostringstream query; - query << "INSERT INTO jetton_masters (address, total_supply, mintable, admin_address, jetton_content, jetton_wallet_code_hash, last_transaction_lt, code_hash, data_hash) VALUES "; - bool is_first = true; + std::initializer_list columns = { + "address", "total_supply", "mintable", "admin_address", "jetton_content", + "jetton_wallet_code_hash", "last_transaction_lt", "code_hash", "data_hash" + }; + + PopulateTableStream stream(txn, "jetton_masters", columns, 1000, false); + stream.setConflictDoUpdate({"address"}, "jetton_masters.last_transaction_lt < EXCLUDED.last_transaction_lt"); + for (const auto& [addr, jetton_master] : jetton_masters) { - if (is_first) { - is_first = false; - } else { - query << ", "; + std::optional jetton_content_str = std::nullopt; + if (jetton_master.jetton_content) { + jetton_content_str = content_to_json_string(jetton_master.jetton_content.value()); } - std::optional raw_admin_address; - if (jetton_master.admin_address) { - raw_admin_address = convert::to_raw_address(jetton_master.admin_address.value()); - } - query << "(" - << txn.quote(convert::to_raw_address(jetton_master.address)) << "," - << jetton_master.total_supply << "," - << TO_SQL_BOOL(jetton_master.mintable) << "," - << TO_SQL_OPTIONAL_STRING(raw_admin_address, txn) << "," - << (jetton_master.jetton_content ? txn.quote(content_to_json_string(jetton_master.jetton_content.value())) : "NULL") << "," - << txn.quote(td::base64_encode(jetton_master.jetton_wallet_code_hash.as_slice())) << "," - << jetton_master.last_transaction_lt << "," - << txn.quote(td::base64_encode(jetton_master.code_hash.as_slice())) << "," - << txn.quote(td::base64_encode(jetton_master.data_hash.as_slice())) - << ")"; - } - if (is_first) { - return ""; - } - query << " ON CONFLICT (address) DO UPDATE SET " - << "total_supply = EXCLUDED.total_supply, " - << "mintable = EXCLUDED.mintable, " - << "admin_address = EXCLUDED.admin_address, " - << "jetton_content = EXCLUDED.jetton_content, " - << "jetton_wallet_code_hash = EXCLUDED.jetton_wallet_code_hash, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_hash = EXCLUDED.code_hash, " - << "data_hash = EXCLUDED.data_hash WHERE jetton_masters.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; - return query.str(); + + auto tuple = std::make_tuple( + jetton_master.address, + jetton_master.total_supply, + jetton_master.mintable, + jetton_master.admin_address, + jetton_content_str, + jetton_master.jetton_wallet_code_hash, + jetton_master.last_transaction_lt, + jetton_master.code_hash, + jetton_master.data_hash + ); + stream.insert_row(std::move(tuple)); + } + return stream.get_str(); } std::string InsertBatchPostgres::insert_jetton_wallets(pqxx::work &txn) { @@ -957,55 +1064,43 @@ std::string InsertBatchPostgres::insert_jetton_wallets(pqxx::work &txn) { std::unordered_set known_mintless_masters; - std::ostringstream query; - query << "INSERT INTO jetton_wallets (balance, address, owner, jetton, last_transaction_lt, code_hash, data_hash, mintless_is_claimed) VALUES "; - bool is_first = true; + std::initializer_list columns = { + "balance", "address", "owner", "jetton", "last_transaction_lt", "code_hash", "data_hash", "mintless_is_claimed" + }; + + PopulateTableStream stream(txn, "jetton_wallets", columns, 1000, false); + stream.setConflictDoUpdate({"address"}, "jetton_wallets.last_transaction_lt < EXCLUDED.last_transaction_lt"); + for (const auto& [addr, jetton_wallet] : jetton_wallets) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - query << "(" - << (jetton_wallet.balance.not_null() ? jetton_wallet.balance->to_dec_string() : "NULL") << "," - << txn.quote(convert::to_raw_address(jetton_wallet.address)) << "," - << txn.quote(convert::to_raw_address(jetton_wallet.owner)) << "," - << txn.quote(convert::to_raw_address(jetton_wallet.jetton)) << "," - << jetton_wallet.last_transaction_lt << "," - << txn.quote(td::base64_encode(jetton_wallet.code_hash.as_slice())) << "," - << txn.quote(td::base64_encode(jetton_wallet.data_hash.as_slice())) << "," - << TO_SQL_OPTIONAL_BOOL(jetton_wallet.mintless_is_claimed) - << ")"; + auto tuple = std::make_tuple( + jetton_wallet.balance, + jetton_wallet.address, + jetton_wallet.owner, + jetton_wallet.jetton, + jetton_wallet.last_transaction_lt, + jetton_wallet.code_hash, + jetton_wallet.data_hash, + jetton_wallet.mintless_is_claimed + ); + stream.insert_row(std::move(tuple)); if (jetton_wallet.mintless_is_claimed.has_value()) { known_mintless_masters.insert(jetton_wallet.jetton); } } - if (is_first) { - return ""; - } - query << " ON CONFLICT (address) DO UPDATE SET " - << "balance = EXCLUDED.balance, " - << "owner = EXCLUDED.owner, " - << "jetton = EXCLUDED.jetton, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_hash = EXCLUDED.code_hash, " - << "data_hash = EXCLUDED.data_hash, " - << "mintless_is_claimed = EXCLUDED.mintless_is_claimed WHERE jetton_wallets.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; - + + std::string result = stream.get_str(); if (!known_mintless_masters.empty()) { - bool is_first = true; - query << "INSERT INTO mintless_jetton_masters(address, is_indexed) VALUES "; + PopulateTableStream mintless_stream(txn, "mintless_jetton_masters", {"address", "is_indexed"}, 1000, false); + mintless_stream.setConflictDoNothing(); + for (const auto &addr : known_mintless_masters) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - query << "(" << txn.quote(convert::to_raw_address(addr)) << ", FALSE)"; + auto tuple = std::make_tuple(addr, false); + mintless_stream.insert_row(std::move(tuple)); } - query << " ON CONFLICT (address) DO NOTHING;\n"; + result += mintless_stream.get_str(); } - return query.str(); + + return result; } std::string InsertBatchPostgres::insert_nft_collections(pqxx::work &txn) { @@ -1023,40 +1118,30 @@ std::string InsertBatchPostgres::insert_nft_collections(pqxx::work &txn) { } } - std::ostringstream query; - query << "INSERT INTO nft_collections (address, next_item_index, owner_address, collection_content, last_transaction_lt, code_hash, data_hash) VALUES "; - bool is_first = true; + std::initializer_list columns = { + "address", "next_item_index", "owner_address", "collection_content", "last_transaction_lt", "code_hash", "data_hash" + }; + + PopulateTableStream stream(txn, "nft_collections", columns, 1000, false); + stream.setConflictDoUpdate({"address"}, "nft_collections.last_transaction_lt < EXCLUDED.last_transaction_lt"); + for (const auto& [addr, nft_collection] : nft_collections) { - if (is_first) { - is_first = false; - } else { - query << ", "; + std::optional collection_content_str = std::nullopt; + if (nft_collection.collection_content) { + collection_content_str = content_to_json_string(nft_collection.collection_content.value()); } - std::optional raw_owner_address; - if (nft_collection.owner_address) { - raw_owner_address = convert::to_raw_address(nft_collection.owner_address.value()); - } - query << "(" - << txn.quote(convert::to_raw_address(nft_collection.address)) << "," - << nft_collection.next_item_index << "," - << TO_SQL_OPTIONAL_STRING(raw_owner_address, txn) << "," - << (nft_collection.collection_content ? txn.quote(content_to_json_string(nft_collection.collection_content.value())) : "NULL") << "," - << nft_collection.last_transaction_lt << "," - << txn.quote(td::base64_encode(nft_collection.code_hash.as_slice())) << "," - << txn.quote(td::base64_encode(nft_collection.data_hash.as_slice())) - << ")"; - } - if (is_first) { - return ""; - } - query << " ON CONFLICT (address) DO UPDATE SET " - << "next_item_index = EXCLUDED.next_item_index, " - << "owner_address = EXCLUDED.owner_address, " - << "collection_content = EXCLUDED.collection_content, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_hash = EXCLUDED.code_hash, " - << "data_hash = EXCLUDED.data_hash WHERE nft_collections.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; - return query.str(); + auto tuple = std::make_tuple( + nft_collection.address, + nft_collection.next_item_index, + nft_collection.owner_address, + collection_content_str, + nft_collection.last_transaction_lt, + nft_collection.code_hash, + nft_collection.data_hash + ); + stream.insert_row(std::move(tuple)); + } + return stream.get_str(); } std::string InsertBatchPostgres::insert_nft_items(pqxx::work &txn) { @@ -1074,103 +1159,58 @@ std::string InsertBatchPostgres::insert_nft_items(pqxx::work &txn) { } } - std::ostringstream query; - bool is_first = true; + std::initializer_list columns = { + "address", "init", "index", "collection_address", "owner_address", "content", "last_transaction_lt", "code_hash", "data_hash" + }; + + PopulateTableStream stream(txn, "nft_items", columns, 1000, false); + stream.setConflictDoUpdate({"address"}, "nft_items.last_transaction_lt < EXCLUDED.last_transaction_lt"); + for (const auto& [addr, nft_item] : nft_items) { - if (is_first) { - query << "INSERT INTO nft_items (address, init, index, collection_address, owner_address, content, last_transaction_lt, code_hash, data_hash) VALUES "; - is_first = false; - } else { - query << ", "; + std::optional content_str = std::nullopt; + if (nft_item.content) { + content_str = content_to_json_string(nft_item.content.value()); } - std::optional raw_collection_address; - if (nft_item.collection_address) { - raw_collection_address = convert::to_raw_address(nft_item.collection_address.value()); - } - std::optional raw_owner_address; - if (nft_item.owner_address) { - raw_owner_address = convert::to_raw_address(nft_item.owner_address.value()); - } - query << "(" - << txn.quote(convert::to_raw_address(nft_item.address)) << "," - << TO_SQL_BOOL(nft_item.init) << "," - << nft_item.index << "," - << TO_SQL_OPTIONAL_STRING(raw_collection_address, txn) << "," - << TO_SQL_OPTIONAL_STRING(raw_owner_address, txn) << "," - << (nft_item.content ? txn.quote(content_to_json_string(nft_item.content.value())) : "NULL") << "," - << nft_item.last_transaction_lt << "," - << txn.quote(td::base64_encode(nft_item.code_hash.as_slice())) << "," - << txn.quote(td::base64_encode(nft_item.data_hash.as_slice())) - << ")"; - } - if (!is_first) { - query << " ON CONFLICT (address) DO UPDATE SET " - << "init = EXCLUDED.init, " - << "index = EXCLUDED.index, " - << "collection_address = EXCLUDED.collection_address, " - << "owner_address = EXCLUDED.owner_address, " - << "content = EXCLUDED.content, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_hash = EXCLUDED.code_hash, " - << "data_hash = EXCLUDED.data_hash WHERE nft_items.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; - } - - is_first = true; + auto tuple = std::make_tuple( + nft_item.address, + nft_item.init, + nft_item.index, + nft_item.collection_address, + nft_item.owner_address, + content_str, + nft_item.last_transaction_lt, + nft_item.code_hash, + nft_item.data_hash + ); + stream.insert_row(std::move(tuple)); + } + auto result = stream.get_str(); + + std::initializer_list dns_columns = { + "nft_item_address", "nft_item_owner", "domain", "dns_next_resolver", "dns_wallet", "dns_site_adnl", "dns_storage_bag_id", "last_transaction_lt" + }; + PopulateTableStream dns_stream(txn, "dns_entries", dns_columns, 1000, false); + dns_stream.setConflictDoUpdate({"nft_item_address"}, "dns_entries.last_transaction_lt < EXCLUDED.last_transaction_lt"); + for (const auto& [addr, nft_item] : nft_items) { if (!nft_item.dns_entry) { continue; } - if (is_first) { - query << "INSERT INTO dns_entries (nft_item_address, nft_item_owner, domain, dns_next_resolver, dns_wallet, dns_site_adnl, dns_storage_bag_id, last_transaction_lt) VALUES "; - is_first = false; - } else { - query << ", "; - } - - std::optional raw_owner_address; - if (nft_item.owner_address) { - raw_owner_address = convert::to_raw_address(nft_item.owner_address.value()); - } - std::optional raw_dns_next_resolver; - if (nft_item.dns_entry->next_resolver) { - raw_dns_next_resolver = convert::to_raw_address(nft_item.dns_entry->next_resolver.value()); - } - std::optional raw_dns_wallet; - if (nft_item.dns_entry->wallet) { - raw_dns_wallet = convert::to_raw_address(nft_item.dns_entry->wallet.value()); - } - std::optional raw_dns_site; - if (nft_item.dns_entry->site_adnl) { - raw_dns_site = nft_item.dns_entry->site_adnl->to_hex(); - } - std::optional raw_dns_storage_bag_id; - if (nft_item.dns_entry->storage_bag_id) { - raw_dns_storage_bag_id = nft_item.dns_entry->storage_bag_id->to_hex(); - } - query << "(" - << txn.quote(convert::to_raw_address(nft_item.address)) << "," - << TO_SQL_OPTIONAL_STRING(raw_owner_address, txn) << "," - << txn.quote(nft_item.dns_entry->domain) << "," - << TO_SQL_OPTIONAL_STRING(raw_dns_next_resolver, txn) << "," - << TO_SQL_OPTIONAL_STRING(raw_dns_wallet, txn) << "," - << TO_SQL_OPTIONAL_STRING(raw_dns_site, txn) << "," - << TO_SQL_OPTIONAL_STRING(raw_dns_storage_bag_id, txn) << "," - << nft_item.last_transaction_lt - << ")"; - } - if (!is_first) { - query << " ON CONFLICT (nft_item_address) DO UPDATE SET " - << "nft_item_owner = EXCLUDED.nft_item_owner, " - << "domain = EXCLUDED.domain, " - << "dns_next_resolver = EXCLUDED.dns_next_resolver, " - << "dns_wallet = EXCLUDED.dns_wallet, " - << "dns_site_adnl = EXCLUDED.dns_site_adnl, " - << "dns_storage_bag_id = EXCLUDED.dns_storage_bag_id, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt " - << "WHERE dns_entries.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; - } - - return query.str(); + + auto tuple = std::make_tuple( + nft_item.address, + nft_item.owner_address, + nft_item.dns_entry->domain, + nft_item.dns_entry->next_resolver, + nft_item.dns_entry->wallet, + nft_item.dns_entry->site_adnl, + nft_item.dns_entry->storage_bag_id, + nft_item.last_transaction_lt + ); + dns_stream.insert_row(std::move(tuple)); + }; + result += dns_stream.get_str(); + return result; } std::string InsertBatchPostgres::insert_getgems_nft_sales(pqxx::work &txn) { @@ -1188,54 +1228,33 @@ std::string InsertBatchPostgres::insert_getgems_nft_sales(pqxx::work &txn) { } } - std::ostringstream query; - query << "INSERT INTO getgems_nft_sales (address, is_complete, created_at, marketplace_address, nft_address, nft_owner_address, full_price, marketplace_fee_address, marketplace_fee, royalty_address, royalty_amount, last_transaction_lt, code_hash, data_hash) VALUES "; - bool is_first = true; + std::initializer_list columns = { + "address", "is_complete", "created_at", "marketplace_address", "nft_address", "nft_owner_address", "full_price", + "marketplace_fee_address", "marketplace_fee", "royalty_address", "royalty_amount", "last_transaction_lt", "code_hash", "data_hash" + }; + PopulateTableStream stream(txn, "getgems_nft_sales", columns, 1000, false); + stream.setConflictDoUpdate({"address"}, "getgems_nft_sales.last_transaction_lt < EXCLUDED.last_transaction_lt"); + for (const auto& [addr, nft_sale] : nft_sales) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - std::optional raw_nft_owner_address; - if (nft_sale.nft_owner_address) { - raw_nft_owner_address = convert::to_raw_address(nft_sale.nft_owner_address.value()); - } - query << "(" - << txn.quote(convert::to_raw_address(nft_sale.address)) << "," - << TO_SQL_BOOL(nft_sale.is_complete) << "," - << nft_sale.created_at << "," - << txn.quote(convert::to_raw_address(nft_sale.marketplace_address)) << "," - << txn.quote(convert::to_raw_address(nft_sale.nft_address)) << "," - << TO_SQL_OPTIONAL_STRING(raw_nft_owner_address, txn) << "," - << (nft_sale.full_price.not_null() ? nft_sale.full_price->to_dec_string() : "NULL") << "," - << txn.quote(convert::to_raw_address(nft_sale.marketplace_fee_address)) << "," - << (nft_sale.marketplace_fee.not_null() ? nft_sale.marketplace_fee->to_dec_string() : "NULL") << "," - << txn.quote(convert::to_raw_address(nft_sale.royalty_address)) << "," - << (nft_sale.royalty_amount.not_null() ? nft_sale.royalty_amount->to_dec_string() : "NULL") << "," - << nft_sale.last_transaction_lt << "," - << txn.quote(td::base64_encode(nft_sale.code_hash.as_slice())) << "," - << txn.quote(td::base64_encode(nft_sale.data_hash.as_slice())) - << ")"; - } - if (is_first) { - return ""; - } - query << " ON CONFLICT (address) DO UPDATE SET " - << "is_complete = EXCLUDED.is_complete, " - << "created_at = EXCLUDED.created_at, " - << "marketplace_address = EXCLUDED.marketplace_address, " - << "nft_address = EXCLUDED.nft_address, " - << "nft_owner_address = EXCLUDED.nft_owner_address, " - << "full_price = EXCLUDED.full_price, " - << "marketplace_fee_address = EXCLUDED.marketplace_fee_address, " - << "marketplace_fee = EXCLUDED.marketplace_fee, " - << "royalty_address = EXCLUDED.royalty_address, " - << "royalty_amount = EXCLUDED.royalty_amount, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_hash = EXCLUDED.code_hash, " - << "data_hash = EXCLUDED.data_hash WHERE getgems_nft_sales.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; - return query.str(); + auto tuple = std::make_tuple( + nft_sale.address, + nft_sale.is_complete, + nft_sale.created_at, + nft_sale.marketplace_address, + nft_sale.nft_address, + nft_sale.nft_owner_address, + nft_sale.full_price, + nft_sale.marketplace_fee_address, + nft_sale.marketplace_fee, + nft_sale.royalty_address, + nft_sale.royalty_amount, + nft_sale.last_transaction_lt, + nft_sale.code_hash, + nft_sale.data_hash + ); + stream.insert_row(std::move(tuple)); + } + return stream.get_str(); } std::string InsertBatchPostgres::insert_getgems_nft_auctions(pqxx::work &txn) { @@ -1253,226 +1272,170 @@ std::string InsertBatchPostgres::insert_getgems_nft_auctions(pqxx::work &txn) { } } - std::ostringstream query; - query << "INSERT INTO getgems_nft_auctions (address, end_flag, end_time, mp_addr, nft_addr, nft_owner, last_bid, last_member, min_step, mp_fee_addr, mp_fee_factor, mp_fee_base, royalty_fee_addr, royalty_fee_factor, royalty_fee_base, max_bid, min_bid, created_at, last_bid_at, is_canceled, last_transaction_lt, code_hash, data_hash) VALUES "; - bool is_first = true; + std::initializer_list columns = {"address", "end_flag", "end_time", "mp_addr", "nft_addr", "nft_owner", + "last_bid", "last_member", "min_step", "mp_fee_addr", "mp_fee_factor", "mp_fee_base", "royalty_fee_addr", "royalty_fee_factor", + "royalty_fee_base", "max_bid", "min_bid", "created_at", "last_bid_at", "is_canceled", "last_transaction_lt", "code_hash", "data_hash" + }; + PopulateTableStream stream(txn, "getgems_nft_auctions", columns, 1000, false); + stream.setConflictDoUpdate({"address"}, "getgems_nft_auctions.last_transaction_lt < EXCLUDED.last_transaction_lt"); + for (const auto& [addr, nft_auction] : nft_auctions) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - std::optional raw_nft_owner; - if (nft_auction.nft_owner) { - raw_nft_owner = convert::to_raw_address(nft_auction.nft_owner.value()); - } - std::optional raw_last_member; - if (nft_auction.last_member) { - raw_last_member = convert::to_raw_address(nft_auction.last_member.value()); - } - query << "(" - << txn.quote(convert::to_raw_address(nft_auction.address)) << "," - << TO_SQL_BOOL(nft_auction.end) << "," - << nft_auction.end_time << "," - << txn.quote(convert::to_raw_address(nft_auction.mp_addr)) << "," - << txn.quote(convert::to_raw_address(nft_auction.nft_addr)) << "," - << TO_SQL_OPTIONAL_STRING(raw_nft_owner, txn) << "," - << (nft_auction.last_bid.not_null() ? nft_auction.last_bid->to_dec_string() : "NULL") << "," - << TO_SQL_OPTIONAL_STRING(raw_last_member, txn) << "," - << nft_auction.min_step << "," - << txn.quote(convert::to_raw_address(nft_auction.mp_fee_addr)) << "," - << nft_auction.mp_fee_factor << "," - << nft_auction.mp_fee_base << "," - << txn.quote(convert::to_raw_address(nft_auction.royalty_fee_addr)) << "," - << nft_auction.royalty_fee_factor << "," - << nft_auction.royalty_fee_base << "," - << (nft_auction.max_bid.not_null() ? nft_auction.max_bid->to_dec_string() : "NULL") << "," - << (nft_auction.min_bid.not_null() ? nft_auction.min_bid->to_dec_string() : "NULL") << "," - << nft_auction.created_at << "," - << nft_auction.last_bid_at << "," - << TO_SQL_BOOL(nft_auction.is_canceled) << "," - << nft_auction.last_transaction_lt << "," - << txn.quote(td::base64_encode(nft_auction.code_hash.as_slice())) << "," - << txn.quote(td::base64_encode(nft_auction.data_hash.as_slice())) - << ")"; - } - if (is_first) { - return ""; - } - query << " ON CONFLICT (address) DO UPDATE SET " - << "end_flag = EXCLUDED.end_flag, " - << "end_time = EXCLUDED.end_time, " - << "mp_addr = EXCLUDED.mp_addr, " - << "nft_addr = EXCLUDED.nft_addr, " - << "nft_owner = EXCLUDED.nft_owner, " - << "last_bid = EXCLUDED.last_bid, " - << "last_member = EXCLUDED.last_member, " - << "min_step = EXCLUDED.min_step, " - << "mp_fee_addr = EXCLUDED.mp_fee_addr, " - << "mp_fee_factor = EXCLUDED.mp_fee_factor, " - << "mp_fee_base = EXCLUDED.mp_fee_base, " - << "royalty_fee_addr = EXCLUDED.royalty_fee_addr, " - << "royalty_fee_factor = EXCLUDED.royalty_fee_factor, " - << "royalty_fee_base = EXCLUDED.royalty_fee_base, " - << "max_bid = EXCLUDED.max_bid, " - << "min_bid = EXCLUDED.min_bid, " - << "created_at = EXCLUDED.created_at, " - << "last_bid_at = EXCLUDED.last_bid_at, " - << "is_canceled = EXCLUDED.is_canceled, " - << "last_transaction_lt = EXCLUDED.last_transaction_lt, " - << "code_hash = EXCLUDED.code_hash, " - << "data_hash = EXCLUDED.data_hash WHERE getgems_nft_auctions.last_transaction_lt < EXCLUDED.last_transaction_lt;\n"; - return query.str(); + auto tuple = std::make_tuple( + nft_auction.address, + nft_auction.end, + nft_auction.end_time, + nft_auction.mp_addr, + nft_auction.nft_addr, + nft_auction.nft_owner, + nft_auction.last_bid, + nft_auction.last_member, + nft_auction.min_step, + nft_auction.mp_fee_addr, + nft_auction.mp_fee_factor, + nft_auction.mp_fee_base, + nft_auction.royalty_fee_addr, + nft_auction.royalty_fee_factor, + nft_auction.royalty_fee_base, + nft_auction.max_bid, + nft_auction.min_bid, + nft_auction.created_at, + nft_auction.last_bid_at, + nft_auction.is_canceled, + nft_auction.last_transaction_lt, + nft_auction.code_hash, + nft_auction.data_hash + ); + stream.insert_row(std::move(tuple)); + } + return stream.get_str(); } -std::string InsertBatchPostgres::insert_jetton_transfers(pqxx::work &txn) { - std::ostringstream query; - query << "INSERT INTO jetton_transfers (tx_hash, tx_lt, tx_now, tx_aborted, query_id, amount, source, " - "destination, jetton_wallet_address, jetton_master_address, response_destination, " - "custom_payload, forward_ton_amount, forward_payload, trace_id) VALUES "; - bool is_first = true; +void InsertBatchPostgres::insert_jetton_transfers(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = { + "tx_hash", "tx_lt", "tx_now", "tx_aborted", "query_id", "amount", "source", "destination", "jetton_wallet_address", + "jetton_master_address", "response_destination", "custom_payload", "forward_ton_amount", "forward_payload", "trace_id" + }; + PopulateTableStream stream(txn, "jetton_transfers", columns, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoNothing(); + } + for (const auto& task : insert_tasks_) { for (const auto& transfer : task.parsed_block_->get_events()) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; - query << "(" - << txn.quote(td::base64_encode(transfer.transaction_hash.as_slice())) << "," - << transfer.transaction_lt << "," - << transfer.transaction_now << "," - << TO_SQL_BOOL(transfer.transaction_aborted) << "," - << transfer.query_id << "," - << (transfer.amount.not_null() ? transfer.amount->to_dec_string() : "NULL") << "," - << txn.quote(transfer.source) << "," - << txn.quote(transfer.destination) << "," - << txn.quote(transfer.jetton_wallet) << "," - << txn.quote(transfer.jetton_master) << "," - << txn.quote(transfer.response_destination) << "," - << TO_SQL_OPTIONAL_STRING(custom_payload_boc, txn) << "," - << (transfer.forward_ton_amount.not_null() ? transfer.forward_ton_amount->to_dec_string() : "NULL") << "," - << TO_SQL_OPTIONAL_STRING(forward_payload_boc, txn) << "," - << txn.quote(td::base64_encode(transfer.trace_id.as_slice())) - << ")"; + auto tuple = std::make_tuple( + transfer.transaction_hash, + transfer.transaction_lt, + transfer.transaction_now, + transfer.transaction_aborted, + transfer.query_id, + transfer.amount, + transfer.source, + transfer.destination, + transfer.jetton_wallet, + transfer.jetton_master, + transfer.response_destination, + custom_payload_boc, + transfer.forward_ton_amount, + forward_payload_boc, + transfer.trace_id + ); + stream.insert_row(std::move(tuple)); } } - if (is_first) { - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - - // LOG(DEBUG) << "Running SQL query: " << query.str(); - // LOG(INFO) << "Jetton transfers query size: " << double(query.str().length()) / 1024 / 1024; - txn.exec0(query.str()); - return ""; + stream.finish(); } -std::string InsertBatchPostgres::insert_jetton_burns(pqxx::work &txn) { - std::ostringstream query; - query << "INSERT INTO jetton_burns (tx_hash, tx_lt, tx_now, tx_aborted, query_id, owner, jetton_wallet_address, jetton_master_address, amount, response_destination, custom_payload, trace_id) VALUES "; - bool is_first = true; +void InsertBatchPostgres::insert_jetton_burns(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = { + "tx_hash", "tx_lt", "tx_now", "tx_aborted", "query_id", "owner", "jetton_wallet_address", "jetton_master_address", + "amount", "response_destination", "custom_payload", "trace_id" + }; + PopulateTableStream stream(txn, "jetton_burns", columns, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoNothing(); + } + for (const auto& task : insert_tasks_) { for (const auto& burn : task.parsed_block_->get_events()) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } - auto custom_payload_boc_r = convert::to_bytes(burn.custom_payload); auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; - query << "(" - << txn.quote(td::base64_encode(burn.transaction_hash.as_slice())) << "," - << burn.transaction_lt << "," - << burn.transaction_now << "," - << TO_SQL_BOOL(burn.transaction_aborted) << "," - << burn.query_id << "," - << txn.quote(burn.owner) << "," - << txn.quote(burn.jetton_wallet) << "," - << txn.quote(burn.jetton_master) << "," - << (burn.amount.not_null() ? burn.amount->to_dec_string() : "NULL") << "," - << txn.quote(burn.response_destination) << "," - << TO_SQL_OPTIONAL_STRING(custom_payload_boc, txn) << "," - << txn.quote(td::base64_encode(burn.trace_id.as_slice())) - << ")"; + auto tuple = std::make_tuple( + burn.transaction_hash, + burn.transaction_lt, + burn.transaction_now, + burn.transaction_aborted, + burn.query_id, + burn.owner, + burn.jetton_wallet, + burn.jetton_master, + burn.amount, + burn.response_destination, + custom_payload_boc, + burn.trace_id + ); + + stream.insert_row(std::move(tuple)); } } - if (is_first) { - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - - // LOG(DEBUG) << "Running SQL query: " << query.str(); - // LOG(INFO) << "Jetton burns query size: " << double(query.str().length()) / 1024 / 1024; - txn.exec0(query.str()); - return ""; + stream.finish(); } -std::string InsertBatchPostgres::insert_nft_transfers(pqxx::work &txn) { - std::ostringstream query; - query << "INSERT INTO nft_transfers (tx_hash, tx_lt, tx_now, tx_aborted, query_id, nft_item_address, nft_item_index, nft_collection_address, old_owner, new_owner, response_destination, custom_payload, forward_amount, forward_payload, trace_id) VALUES "; - bool is_first = true; +void InsertBatchPostgres::insert_nft_transfers(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = { + "tx_hash", "tx_lt", "tx_now", "tx_aborted", "query_id", "nft_item_address", "nft_item_index", "nft_collection_address", + "old_owner", "new_owner", "response_destination", "custom_payload", "forward_amount", "forward_payload", "trace_id" + }; + PopulateTableStream stream(txn, "nft_transfers", columns, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoNothing(); + } + for (const auto& task : insert_tasks_) { for (const auto& transfer : task.parsed_block_->get_events()) { - if (is_first) { - is_first = false; - } else { - query << ", "; - } auto custom_payload_boc_r = convert::to_bytes(transfer.custom_payload); auto custom_payload_boc = custom_payload_boc_r.is_ok() ? custom_payload_boc_r.move_as_ok() : std::nullopt; auto forward_payload_boc_r = convert::to_bytes(transfer.forward_payload); auto forward_payload_boc = forward_payload_boc_r.is_ok() ? forward_payload_boc_r.move_as_ok() : std::nullopt; - query << "(" - << txn.quote(td::base64_encode(transfer.transaction_hash.as_slice())) << "," - << transfer.transaction_lt << "," - << transfer.transaction_now << "," - << TO_SQL_BOOL(transfer.transaction_aborted) << "," - << transfer.query_id << "," - << txn.quote(convert::to_raw_address(transfer.nft_item)) << "," - << (transfer.nft_item_index.not_null() ? transfer.nft_item_index->to_dec_string() : "NULL") << "," - << txn.quote(transfer.nft_collection) << "," - << txn.quote(transfer.old_owner) << "," - << txn.quote(transfer.new_owner) << "," - << txn.quote(transfer.response_destination) << "," - << TO_SQL_OPTIONAL_STRING(custom_payload_boc, txn) << "," - << (transfer.forward_amount.not_null() ? transfer.forward_amount->to_dec_string() : "NULL") << "," - << TO_SQL_OPTIONAL_STRING(forward_payload_boc, txn) << "," - << txn.quote(td::base64_encode(transfer.trace_id.as_slice())) - << ")"; + auto tuple = std::make_tuple( + transfer.transaction_hash, + transfer.transaction_lt, + transfer.transaction_now, + transfer.transaction_aborted, + transfer.query_id, + transfer.nft_item, + transfer.nft_item_index, + transfer.nft_collection, + transfer.old_owner, + transfer.new_owner, + transfer.response_destination, + custom_payload_boc, + transfer.forward_amount, + forward_payload_boc, + transfer.trace_id + ); + stream.insert_row(std::move(tuple)); } } - if (is_first) { - return ""; - } - query << " ON CONFLICT DO NOTHING;\n"; - - // LOG(DEBUG) << "Running SQL query: " << query.str(); - // LOG(INFO) << "NFT transfers query size: " << double(query.str().length()) / 1024 / 1024; - txn.exec0(query.str()); - - return ""; + stream.finish(); } -#define B64HASH(x) (td::base64_encode((x).as_slice())) - -std::string InsertBatchPostgres::insert_traces(pqxx::work &txn) { - std::ostringstream traces_query; +void InsertBatchPostgres::insert_traces(pqxx::work &txn, bool with_copy) { + std::initializer_list columns = { "trace_id", "external_hash", "mc_seqno_start", "mc_seqno_end", + "start_lt", "start_utime", "end_lt", "end_utime", "state", "pending_edges_", "edges_", "nodes_" }; - traces_query << "INSERT INTO traces (trace_id, external_hash, mc_seqno_start, mc_seqno_end, " - "start_lt, start_utime, end_lt, end_utime, state, pending_edges_, edges_, nodes_) VALUES "; - - bool is_first_trace = true; + PopulateTableStream stream(txn, "traces", columns, 1000, with_copy); + if (!with_copy) { + stream.setConflictDoUpdate({"trace_id"}, "traces.end_lt < EXCLUDED.end_lt"); + } std::unordered_map traces_map; for (const auto& task : insert_tasks_) { @@ -1488,40 +1451,23 @@ std::string InsertBatchPostgres::insert_traces(pqxx::work &txn) { } } for(auto &[_, trace] : traces_map) { - // trace - if(is_first_trace) { - is_first_trace = false; - } else { - traces_query << ", "; - } - traces_query << "(" - << txn.quote(B64HASH(trace.trace_id)) << "," - << (trace.external_hash.has_value() ? txn.quote(B64HASH(trace.external_hash.value())) : "NULL" ) << "," - << trace.mc_seqno_start << "," - << trace.mc_seqno_end << "," - << trace.start_lt << "," - << trace.start_utime << "," - << trace.end_lt << "," - << trace.end_utime << "," - << txn.quote(stringify(trace.state)) << "," - << trace.pending_edges_ << "," - << trace.edges_ << "," - << trace.nodes_ - << ")"; - } - if (!is_first_trace) { - traces_query << " ON CONFLICT (trace_id) DO UPDATE SET " - << "mc_seqno_end = EXCLUDED.mc_seqno_end, " - << "end_lt = EXCLUDED.end_lt, " - << "end_utime = EXCLUDED.end_utime, " - << "state = EXCLUDED.state, " - << "pending_edges_ = EXCLUDED.pending_edges_, " - << "edges_ = EXCLUDED.edges_, " - << "nodes_ = EXCLUDED.nodes_ " - << "WHERE traces.end_lt < EXCLUDED.end_lt;\n"; - } - - return traces_query.str(); + auto tuple = std::make_tuple( + trace.trace_id, + trace.external_hash, + trace.mc_seqno_start, + trace.mc_seqno_end, + trace.start_lt, + trace.start_utime, + trace.end_lt, + trace.end_utime, + stringify(trace.state), + trace.pending_edges_, + trace.edges_, + trace.nodes_ + ); + stream.insert_row(std::move(tuple)); + } + stream.finish(); } // diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.h b/ton-index-postgres-v2/src/InsertManagerPostgres.h index a6d04ab3..f2a827a8 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.h +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.h @@ -54,27 +54,27 @@ class InsertBatchPostgres: public td::actor::Actor { std::vector insert_tasks_; td::Promise promise_; std::int32_t max_data_depth_; - std::int32_t retry_count_{0}; + bool with_copy_{true}; std::string stringify(schema::ComputeSkipReason compute_skip_reason); std::string stringify(schema::AccStatusChange acc_status_change); std::string stringify(schema::AccountStatus account_status); std::string stringify(schema::Trace::State state); - std::string insert_blocks(pqxx::work &txn); - std::string insert_shard_state(pqxx::work &txn); - std::string insert_transactions(pqxx::work &txn); - std::string insert_messages(pqxx::work &txn); - std::string insert_account_states(pqxx::work &txn); + void insert_blocks(pqxx::work &txn, bool with_copy); + void insert_shard_state(pqxx::work &txn, bool with_copy); + void insert_transactions(pqxx::work &txn, bool with_copy); + void insert_messages(pqxx::work &txn, bool with_copy); + void insert_account_states(pqxx::work &txn, bool with_copy); std::string insert_latest_account_states(pqxx::work &txn); - std::string insert_jetton_transfers(pqxx::work &txn); - std::string insert_jetton_burns(pqxx::work &txn); - std::string insert_nft_transfers(pqxx::work &txn); + void insert_jetton_transfers(pqxx::work &txn, bool with_copy); + void insert_jetton_burns(pqxx::work &txn, bool with_copy); + void insert_nft_transfers(pqxx::work &txn, bool with_copy); std::string insert_jetton_masters(pqxx::work &txn); std::string insert_jetton_wallets(pqxx::work &txn); std::string insert_nft_collections(pqxx::work &txn); std::string insert_nft_items(pqxx::work &txn); std::string insert_getgems_nft_auctions(pqxx::work &txn); std::string insert_getgems_nft_sales(pqxx::work &txn); - std::string insert_traces(pqxx::work &txn); + void insert_traces(pqxx::work &txn, bool with_copy); }; diff --git a/tondb-scanner/src/IndexData.h b/tondb-scanner/src/IndexData.h index 08412bea..dfa180c6 100644 --- a/tondb-scanner/src/IndexData.h +++ b/tondb-scanner/src/IndexData.h @@ -119,8 +119,8 @@ struct SplitMergeInfo { struct TransactionDescr_ord { bool credit_first; - TrStoragePhase storage_ph; - TrCreditPhase credit_ph; + std::optional storage_ph; + std::optional credit_ph; TrComputePhase compute_ph; std::optional action; bool aborted; From cdab7ca2b1195dad2c4035d71a79c70b0ff5cb9e Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 18 Jan 2025 13:59:04 +0100 Subject: [PATCH 10/28] Fix RefInt256 nullness (#101) --- ton-index-postgres-v2/src/InsertManagerPostgres.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index 164078de..2adec4d0 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -34,7 +34,16 @@ template<> struct string_traits } }; -template<> struct nullness : pqxx::no_null {}; +template<> struct nullness +{ + static constexpr bool has_null{true}; + + static constexpr bool always_null{false}; + + static bool is_null(td::RefInt256 const &value) { + return value.is_null(); + } +}; template<> struct string_traits { From dd267b05eca2ac6c986573272646cbb060b8ae6f Mon Sep 17 00:00:00 2001 From: Marat S Date: Sat, 18 Jan 2025 13:53:21 +0000 Subject: [PATCH 11/28] build fixes --- ton-index-clickhouse/src/IndexScheduler.cpp | 7 ++++++- .../src/InsertManagerClickhouse.cpp | 13 ++++++++++--- ton-index-postgres/src/IndexScheduler.cpp | 6 +++++- ton-index-postgres/src/InsertManagerPostgres.cpp | 8 ++++++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ton-index-clickhouse/src/IndexScheduler.cpp b/ton-index-clickhouse/src/IndexScheduler.cpp index 76912159..a448caf6 100644 --- a/ton-index-clickhouse/src/IndexScheduler.cpp +++ b/ton-index-clickhouse/src/IndexScheduler.cpp @@ -132,7 +132,12 @@ void IndexScheduler::seqno_fetched(std::uint32_t mc_seqno, MasterchainBlockDataS } td::actor::send_closure(SelfId, &IndexScheduler::seqno_parsed, mc_seqno, R.move_as_ok()); }); - td::actor::send_closure(parse_manager_, &ParseManager::parse, mc_seqno, std::move(block_data_state), std::move(P)); + + td::actor::send_closure(db_scanner_, &DbScanner::get_cell_db_reader, + [SelfId = actor_id(this), parse_manager = parse_manager_, mc_seqno, block_data_state, P = std::move(P)](td::Result> cell_db_reader) mutable { + CHECK(cell_db_reader.is_ok()); + td::actor::send_closure(parse_manager, &ParseManager::parse, mc_seqno, std::move(block_data_state), cell_db_reader.move_as_ok(), std::move(P)); + }); } void IndexScheduler::seqno_parsed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block) { diff --git a/ton-index-clickhouse/src/InsertManagerClickhouse.cpp b/ton-index-clickhouse/src/InsertManagerClickhouse.cpp index 7c972ac5..ff73a81f 100644 --- a/ton-index-clickhouse/src/InsertManagerClickhouse.cpp +++ b/ton-index-clickhouse/src/InsertManagerClickhouse.cpp @@ -860,9 +860,16 @@ void InsertBatchClickhouse::insert_transactions(clickhouse::Client& client) { ord__credit_first_col->Append(v->credit_first); tick_tock__is_tock_col->Append(std::nullopt); split_install__installed_col->Append(std::nullopt); - - store_storage_ph(v->storage_ph); - store_credit_ph(v->credit_ph); + if (v->storage_ph) { + store_storage_ph(v->storage_ph.value()); + } else { + store_empty_storage_ph(); + } + if (v->credit_ph) { + store_credit_ph(v->credit_ph.value()); + } else { + store_empty_credit_ph(); + } store_compute_ph(v->compute_ph); if (v->action) { store_action_ph(v->action.value()); diff --git a/ton-index-postgres/src/IndexScheduler.cpp b/ton-index-postgres/src/IndexScheduler.cpp index 25e05b3a..9674aefd 100644 --- a/ton-index-postgres/src/IndexScheduler.cpp +++ b/ton-index-postgres/src/IndexScheduler.cpp @@ -130,7 +130,11 @@ void IndexScheduler::seqno_fetched(uint32_t mc_seqno, MasterchainBlockDataState } td::actor::send_closure(SelfId, &IndexScheduler::seqno_parsed, mc_seqno, R.move_as_ok()); }); - td::actor::send_closure(parse_manager_, &ParseManager::parse, mc_seqno, std::move(block_data_state), std::move(P)); + td::actor::send_closure(db_scanner_, &DbScanner::get_cell_db_reader, + [SelfId = actor_id(this), parse_manager = parse_manager_, mc_seqno, block_data_state, P = std::move(P)](td::Result> cell_db_reader) mutable { + CHECK(cell_db_reader.is_ok()); + td::actor::send_closure(parse_manager, &ParseManager::parse, mc_seqno, std::move(block_data_state), cell_db_reader.move_as_ok(), std::move(P)); + }); } void IndexScheduler::seqno_parsed(uint32_t mc_seqno, ParsedBlockPtr parsed_block) { diff --git a/ton-index-postgres/src/InsertManagerPostgres.cpp b/ton-index-postgres/src/InsertManagerPostgres.cpp index 65a3575c..df257948 100644 --- a/ton-index-postgres/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres/src/InsertManagerPostgres.cpp @@ -327,8 +327,12 @@ std::string InsertBatchPostgres::jsonify(schema::TransactionDescr descr) { const auto& ord = std::get(descr); obj("type", "ord"); obj("credit_first", td::JsonBool(ord.credit_first)); - obj("storage_ph", td::JsonRaw(jsonify(ord.storage_ph))); - obj("credit_ph", td::JsonRaw(jsonify(ord.credit_ph))); + if (ord.storage_ph.has_value()) { + obj("storage_ph", td::JsonRaw(jsonify(ord.storage_ph.value()))); + } + if (ord.credit_ph.has_value()) { + obj("credit_ph", td::JsonRaw(jsonify(ord.credit_ph.value()))); + } obj("compute_ph", td::JsonRaw(jsonify(ord.compute_ph))); if (ord.action.has_value()) { obj("action", td::JsonRaw(jsonify(ord.action.value()))); From 4afeda1cb43820e39a5b9951ac512546a04699cf Mon Sep 17 00:00:00 2001 From: Marat S Date: Wed, 29 Jan 2025 15:36:41 +0000 Subject: [PATCH 12/28] If TA state not found, start indexing from smaller seqno. --- ton-index-postgres-v2/src/IndexScheduler.cpp | 23 +++++++++++++------- tondb-scanner/src/TraceAssembler.cpp | 7 +++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ton-index-postgres-v2/src/IndexScheduler.cpp b/ton-index-postgres-v2/src/IndexScheduler.cpp index 0cc9432d..fbba2d60 100644 --- a/ton-index-postgres-v2/src/IndexScheduler.cpp +++ b/ton-index-postgres-v2/src/IndexScheduler.cpp @@ -76,11 +76,11 @@ void IndexScheduler::got_existing_seqnos(td::Result> } ton::BlockSeqno next_seqno = from_seqno_; - std::vector seqnos{R.move_as_ok()}; - if (seqnos.size()) { - std::sort(seqnos.begin(), seqnos.end()); - next_seqno = seqnos[0]; - for (auto value : seqnos) { + auto existing_db_seqnos = R.move_as_ok(); + if (existing_db_seqnos.size()) { + std::sort(existing_db_seqnos.begin(), existing_db_seqnos.end()); + next_seqno = existing_db_seqnos[0]; + for (auto value : existing_db_seqnos) { if (value == next_seqno) { ++next_seqno; } else { @@ -88,13 +88,20 @@ void IndexScheduler::got_existing_seqnos(td::Result> } } size_t accepted_cnt = next_seqno - from_seqno_; - LOG(INFO) << "Accepted " << accepted_cnt << " of " << seqnos.size() << " existing seqnos. Next seqno: " << next_seqno; + LOG(INFO) << "Accepted " << accepted_cnt << " of " << existing_db_seqnos.size() + << " existing seqnos in DB (continuous increasing sequence)"; + LOG(INFO) << "Next seqno: " << next_seqno; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), next_seqno](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), from_seqno = from_seqno_, next_seqno](td::Result R) mutable { if (R.is_error()) { LOG(WARNING) << "TraceAssembler state not found for seqno " << next_seqno - 1; - LOG(WARNING) << "Traces that started before block " << next_seqno << " will be marked as broken and not inserted."; + if (next_seqno > from_seqno) { + // to make sure we don't miss any traces (we assume no trace lasts for longer than 50 mc blocks) + auto to_start_from = std::max(from_seqno, static_cast(next_seqno) - 50); + next_seqno = static_cast(to_start_from); + } + LOG(WARNING) << "Traces that started before block " << next_seqno << " and will be marked as broken and not inserted."; td::actor::send_closure(SelfId, &IndexScheduler::got_trace_assembler_last_state_seqno, next_seqno - 1); } else { LOG(INFO) << "Restored TraceAssembler state for seqno " << R.ok(); diff --git a/tondb-scanner/src/TraceAssembler.cpp b/tondb-scanner/src/TraceAssembler.cpp index a3ac541a..5aa8da17 100644 --- a/tondb-scanner/src/TraceAssembler.cpp +++ b/tondb-scanner/src/TraceAssembler.cpp @@ -115,12 +115,17 @@ void TraceAssembler::alarm() { } }, td::Timestamp::now()); - LOG(INFO) << " Pending traces: " << pending_traces_.size() + LOG(INFO) << "Expected seqno: " << expected_seqno_ + << " Pending traces: " << pending_traces_.size() << " Pending edges: " << pending_edges_.size() << " Broken traces: " << broken_count_; } void TraceAssembler::assemble(ton::BlockSeqno seqno, ParsedBlockPtr block, td::Promise promise) { + if (seqno < expected_seqno_) { + LOG(FATAL) << "TraceAssembler received seqno " << seqno << " that is lower than expected " << expected_seqno_; + return; + } queue_.emplace(seqno, Task{seqno, std::move(block), std::move(promise)}); process_queue(); From 3431d125276d641174729a901977f6787ae51f9a Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:18:23 +0100 Subject: [PATCH 13/28] index extra currencies in ton-smc-scanner (#105) --- ton-smc-scanner/src/PostgreSQLInserter.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ton-smc-scanner/src/PostgreSQLInserter.cpp b/ton-smc-scanner/src/PostgreSQLInserter.cpp index 6c89df0a..8a07ecab 100644 --- a/ton-smc-scanner/src/PostgreSQLInserter.cpp +++ b/ton-smc-scanner/src/PostgreSQLInserter.cpp @@ -23,6 +23,17 @@ std::string content_to_json_string(const std::map &con return jetton_content_json.string_builder().as_cslice().str(); } +std::string extra_currencies_to_json_string(const std::map &extra_currencies) { + td::JsonBuilder extra_currencies_json; + auto obj = extra_currencies_json.enter_object(); + for (auto ¤cy : extra_currencies) { + obj(std::to_string(currency.first), currency.second->to_dec_string()); + } + obj.leave(); + + return extra_currencies_json.string_builder().as_cslice().str(); +} + void PostgreSQLInserter::start_up() { try { pqxx::connection c(connection_string_); @@ -63,7 +74,7 @@ void PostgreSQLInserter::insert_latest_account_states(pqxx::work &transaction) { } } std::ostringstream query; - query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, " + query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, balance_extra_currencies" "account_status, timestamp, last_trans_hash, last_trans_lt, " "frozen_hash, data_hash, code_hash, " "data_boc, code_boc) VALUES "; @@ -103,6 +114,7 @@ void PostgreSQLInserter::insert_latest_account_states(pqxx::work &transaction) { << "NULL," << transaction.quote(td::base64_encode(account_state.hash.as_slice())) << "," << account_state.balance.grams << "," + << transaction.quote(extra_currencies_to_json_string(account_state.balance.extra_currencies)) << "," << transaction.quote(account_state.account_status) << "," << account_state.timestamp << "," << transaction.quote(td::base64_encode(account_state.last_trans_hash.as_slice())) << "," From 45f17aa308211e1bfae4f1969da0c9429b301ace Mon Sep 17 00:00:00 2001 From: K-Dimentional Tree Date: Thu, 30 Jan 2025 23:56:58 +0000 Subject: [PATCH 14/28] Fixed missing comma --- ton-smc-scanner/src/PostgreSQLInserter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ton-smc-scanner/src/PostgreSQLInserter.cpp b/ton-smc-scanner/src/PostgreSQLInserter.cpp index 8a07ecab..e5564746 100644 --- a/ton-smc-scanner/src/PostgreSQLInserter.cpp +++ b/ton-smc-scanner/src/PostgreSQLInserter.cpp @@ -74,7 +74,7 @@ void PostgreSQLInserter::insert_latest_account_states(pqxx::work &transaction) { } } std::ostringstream query; - query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, balance_extra_currencies" + query << "INSERT INTO latest_account_states (account, account_friendly, hash, balance, balance_extra_currencies, " "account_status, timestamp, last_trans_hash, last_trans_lt, " "frozen_hash, data_hash, code_hash, " "data_boc, code_boc) VALUES "; From c72f556d851ac6fd67afab4bb8573df8e178ce0f Mon Sep 17 00:00:00 2001 From: K-Dimentional Tree Date: Sun, 2 Feb 2025 07:16:53 +0000 Subject: [PATCH 15/28] Create indexes and run migrations by default --- .../src/InsertManagerPostgres.cpp | 57 +++++++++++-------- .../src/InsertManagerPostgres.h | 5 +- ton-index-postgres-v2/src/main.cpp | 16 ++++-- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index 2adec4d0..fda36023 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -2101,37 +2101,44 @@ void InsertManagerPostgres::start_up() { LOG(ERROR) << "Error while creating indexes in database: " << e.what(); std::_Exit(1); } + } else { + LOG(WARNING) << "Skipping creation of indexes!"; } // some migrations - LOG(INFO) << "Running some migrations..."; - try { - pqxx::connection c(credential_.get_connection_string()); - pqxx::work txn(c); + if (run_migrations_) { + LOG(INFO) << "Running some migrations..."; + try { + pqxx::connection c(credential_.get_connection_string()); + pqxx::work txn(c); - std::string query = ""; - - query += ( - "alter table jetton_wallets add column if not exists mintless_is_claimed boolean;\n" - "alter table jetton_wallets add column if not exists mintless_amount numeric;\n" - "alter table jetton_wallets add column if not exists mintless_start_from bigint;\n" - "alter table jetton_wallets add column if not exists mintless_expire_at bigint;\n" - "alter table mintless_jetton_masters add column if not exists custom_payload_api_uri varchar[];\n" - - "alter table transactions add column if not exists total_fees_extra_currencies jsonb;\n" - "alter table transactions add column if not exists credit_extra_currencies jsonb;\n" - "alter table messages add column if not exists value_extra_currencies jsonb;\n" - "alter table account_states add column if not exists balance_extra_currencies jsonb;\n" - "alter table latest_account_states add column if not exists balance_extra_currencies jsonb;\n" - ); + std::string query = ""; + + query += ( + "alter table jetton_wallets add column if not exists mintless_is_claimed boolean;\n" + "alter table jetton_wallets add column if not exists mintless_amount numeric;\n" + "alter table jetton_wallets add column if not exists mintless_start_from bigint;\n" + "alter table jetton_wallets add column if not exists mintless_expire_at bigint;\n" + "alter table mintless_jetton_masters add column if not exists custom_payload_api_uri varchar[];\n" + + "alter table transactions add column if not exists total_fees_extra_currencies jsonb;\n" + "alter table transactions add column if not exists credit_extra_currencies jsonb;\n" + "alter table messages add column if not exists value_extra_currencies jsonb;\n" + "alter table account_states add column if not exists balance_extra_currencies jsonb;\n" + "alter table latest_account_states add column if not exists balance_extra_currencies jsonb;\n" + ); - LOG(DEBUG) << query; - txn.exec0(query); - txn.commit(); - } catch (const std::exception &e) { - LOG(ERROR) << "Error while running some migrations in database: " << e.what(); - std::_Exit(1); + LOG(DEBUG) << query; + txn.exec0(query); + txn.commit(); + } catch (const std::exception &e) { + LOG(ERROR) << "Error while running some migrations in database: " << e.what(); + std::_Exit(1); + } + } else { + LOG(WARNING) << "Skipping migrations!"; } + LOG(INFO) << "Database is ready!"; // if success alarm_timestamp() = td::Timestamp::in(1.0); diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.h b/ton-index-postgres-v2/src/InsertManagerPostgres.h index f2a827a8..7b79f567 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.h +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.h @@ -21,11 +21,12 @@ class InsertManagerPostgres: public InsertManagerBase { InsertManagerPostgres::Credential credential_; bool custom_types_{false}; bool create_indexes_{false}; + bool run_migrations_{false}; std::int32_t max_data_depth_{0}; std::int32_t out_of_sync_seqno_{0}; public: - InsertManagerPostgres(InsertManagerPostgres::Credential credential, bool custom_types, bool create_indexes) : - credential_(credential), custom_types_(custom_types), create_indexes_(create_indexes) {} + InsertManagerPostgres(InsertManagerPostgres::Credential credential, bool custom_types, bool create_indexes, bool run_migrations) : + credential_(credential), custom_types_(custom_types), create_indexes_(create_indexes), run_migrations_(run_migrations) {} void start_up() override; diff --git a/ton-index-postgres-v2/src/main.cpp b/ton-index-postgres-v2/src/main.cpp index b231da85..0ebd6c6a 100644 --- a/ton-index-postgres-v2/src/main.cpp +++ b/ton-index-postgres-v2/src/main.cpp @@ -37,7 +37,8 @@ int main(int argc, char *argv[]) { td::uint32 to_seqno = 0; bool force_index = false; bool custom_types = false; - bool create_indexes = false; + bool create_indexes = true; + bool run_migrations = true; InsertManagerPostgres::Credential credential; bool testnet = false; @@ -94,9 +95,14 @@ int main(int argc, char *argv[]) { LOG(WARNING) << "Using pgton extension!"; }); - p.add_option('\0', "create-indexes", "Create indexes in database", [&]() { - create_indexes = true; - LOG(WARNING) << "Indexes will be created on launch. It may take several hours!"; + p.add_option('\0', "no-create-indexes", "Do not create indexes in database", [&]() { + create_indexes = false; + LOG(WARNING) << "Indexes will not be created on launch."; + }); + + p.add_option('\0', "no-migrations", "Do not run migrations", [&]() { + run_migrations = false; + LOG(WARNING) << "Migrations will not be executed on launch."; }); p.add_option('\0', "testnet", "Use for testnet. It is used for correct indexing of .ton DNS entries (in testnet .ton collection has a different address)", [&]() { @@ -312,7 +318,7 @@ int main(int argc, char *argv[]) { }); td::actor::Scheduler scheduler({td::actor::Scheduler::NodeInfo{threads, io_workers}}); - scheduler.run_in_context([&] { insert_manager_ = td::actor::create_actor("insertmanager", credential, custom_types, create_indexes); }); + scheduler.run_in_context([&] { insert_manager_ = td::actor::create_actor("insertmanager", credential, custom_types, create_indexes, run_migrations); }); scheduler.run_in_context([&] { parse_manager_ = td::actor::create_actor("parsemanager"); }); scheduler.run_in_context([&] { db_scanner_ = td::actor::create_actor("scanner", db_root, dbs_secondary, working_dir + "/secondary_logs"); }); From 1366ddf4b433a0bf16d975706dd316ea6f11e1fb Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:17:06 +0100 Subject: [PATCH 16/28] Fill empty dns_entry in case it was deleted (#106) --- ton-smc-scanner/src/SmcScanner.cpp | 41 +++++++++------------ ton-smc-scanner/src/SmcScanner.h | 9 ++--- tondb-scanner/src/smc-interfaces/Tokens.cpp | 13 +++++-- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/ton-smc-scanner/src/SmcScanner.cpp b/ton-smc-scanner/src/SmcScanner.cpp index a1c871fc..322b3cd5 100644 --- a/ton-smc-scanner/src/SmcScanner.cpp +++ b/ton-smc-scanner/src/SmcScanner.cpp @@ -30,15 +30,16 @@ ShardStateScanner::ShardStateScanner(td::Ref shard_state, MasterchainB void ShardStateScanner::schedule_next() { vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(shard_state_data_->sstate_.accounts), 256, block::tlb::aug_ShardAccounts}; - int count = 0; - allow_same = true; - while (!finished && count < 10000) { - td::Ref shard_account_csr = accounts_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr_.bits(), 256, true, allow_same); + std::vector> batch; + batch.reserve(options_.batch_size_); + + while (batch.size() < options_.batch_size_ && !finished_) { + td::Ref shard_account_csr = accounts_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr_.bits(), 256, true, allow_same_); if (shard_account_csr.is_null()) { - finished = true; + finished_ = true; break; } - allow_same = false; + allow_same_ = false; shard_account_csr = accounts_dict.extract_value(shard_account_csr); block::gen::ShardAccount::Record acc_info; if(!tlb::csr_unpack(shard_account_csr, acc_info)) { @@ -46,25 +47,20 @@ void ShardStateScanner::schedule_next() { continue; } - ++count; - queue_.push_back(std::make_pair(cur_addr_, std::move(acc_info))); - if (queue_.size() > options_.batch_size_) { - // LOG(INFO) << "Dispatched batch of " << queue_.size() << " account states"; - std::vector> batch_; - std::copy(queue_.begin(), queue_.end(), std::back_inserter(batch_)); - queue_.clear(); - - in_progress_.fetch_add(1); - td::actor::create_actor("parser", std::move(batch_), shard_state_data_, actor_id(this), options_).release(); - } + batch.push_back(std::make_pair(cur_addr_, std::move(acc_info))); } - processed_ += count; + + LOG(INFO) << shard_.to_str() << ": Dispatched batch of " << batch.size() << " account states"; + processed_ += batch.size(); + + in_progress_++; + td::actor::create_actor("parser", std::move(batch), shard_state_data_, actor_id(this), options_).release(); td::actor::send_closure(options_.insert_manager_, &PostgreSQLInsertManager::checkpoint, shard_, cur_addr_); - if(!finished) { + if(!finished_) { alarm_timestamp() = td::Timestamp::in(0.1); } else { - LOG(INFO) << "Shard " << shard_.to_str() << " is finished!"; + LOG(INFO) << "Shard " << shard_.to_str() << " is finished with " << processed_ << " account states"; stop(); } } @@ -115,7 +111,7 @@ void ShardStateScanner::alarm() { } void ShardStateScanner::batch_inserted() { - in_progress_.fetch_sub(1); + in_progress_--; } void ShardStateScanner::got_checkpoint(td::Bits256 cur_addr) { @@ -214,9 +210,6 @@ void StateBatchParser::process_account_states(std::vector } void StateBatchParser::start_up() { - // if (cur_addr_.to_hex() != "E753CF93EAEDD2EC01B5DE8F49A334622BD630A8728806ABA65F1443EB7C8FD7") { - // continue; - // } std::vector state_list; for (auto &[addr_, acc_info] : data_) { int account_tag = block::gen::t_Account.get_tag(vm::load_cell_slice(acc_info.account)); diff --git a/ton-smc-scanner/src/SmcScanner.h b/ton-smc-scanner/src/SmcScanner.h index bd319b2e..9751ad2e 100644 --- a/ton-smc-scanner/src/SmcScanner.h +++ b/ton-smc-scanner/src/SmcScanner.h @@ -53,15 +53,14 @@ class ShardStateScanner: public td::actor::Actor { ShardStateDataPtr shard_state_data_; Options options_; - std::vector> queue_; td::Bits256 cur_addr_{td::Bits256::zero()}; ton::ShardIdFull shard_; - bool allow_same{true}; - bool finished{false}; - std::atomic_uint32_t in_progress_{0}; - std::int32_t processed_{0}; + bool allow_same_{true}; + bool finished_{false}; + uint32_t in_progress_{0}; + uint32_t processed_{0}; std::unordered_map no_interface_count_; std::unordered_set code_hashes_to_skip_; diff --git a/tondb-scanner/src/smc-interfaces/Tokens.cpp b/tondb-scanner/src/smc-interfaces/Tokens.cpp index 159a050c..35f128cb 100644 --- a/tondb-scanner/src/smc-interfaces/Tokens.cpp +++ b/tondb-scanner/src/smc-interfaces/Tokens.cpp @@ -409,8 +409,14 @@ td::Result NftItemDetectorR::get_dns_entry_d auto zero_byte_slice = vm::load_cell_slice_ref(zero_byte_cell); td::RefInt256 categories{true, 0}; - TRY_RESULT(stack, execute_smc_method<2>(address_, code_cell_, data_cell_, config_, "dnsresolve", - {vm::StackEntry(zero_byte_slice), vm::StackEntry(categories)}, {vm::StackEntry::Type::t_int, vm::StackEntry::Type::t_cell})); + TRY_RESULT(stack, execute_smc_method(address_, code_cell_, data_cell_, config_, "dnsresolve", + {vm::StackEntry(zero_byte_slice), vm::StackEntry(categories)})); + if (stack.size() != 2) { + return td::Status::Error("dnsresolve returned unexpected stack size"); + } + if (stack[0].type() != vm::StackEntry::Type::t_int) { + return td::Status::Error("dnsresolve returned unexpected stack type at index 0"); + } auto resolved_bits_cnt = stack[0].as_int()->to_long(); if (resolved_bits_cnt != 8) { @@ -419,7 +425,8 @@ td::Result NftItemDetectorR::get_dns_entry_d auto recordset_cell = stack[1].as_cell(); if (recordset_cell.is_null()) { - return td::Status::Error("recordset is null"); + // recordset is null + return Result::DNSEntry{}; } vm::Dictionary records{recordset_cell, 256}; From 64bdf8f5d1a73260ce149e087734d572a6b61b03 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Thu, 6 Feb 2025 14:17:55 +0100 Subject: [PATCH 17/28] Implement measuring statistics (#107) * statistics wip * make it threadlocal + add metrics --- celldb-migrate/CMakeLists.txt | 2 +- ton-index-clickhouse/CMakeLists.txt | 2 +- ton-index-postgres-v2/CMakeLists.txt | 2 +- .../src/BlockInterfacesDetector.h | 4 + ton-index-postgres-v2/src/IndexScheduler.cpp | 21 ++- ton-index-postgres-v2/src/IndexScheduler.h | 3 + .../src/InsertManagerPostgres.cpp | 18 ++ ton-index-postgres/CMakeLists.txt | 2 +- ton-integrity-checker/CMakeLists.txt | 2 +- ton-smc-scanner/CMakeLists.txt | 2 +- ton-smc-scanner/src/SmcScanner.cpp | 2 +- ton-trace-emulator/CMakeLists.txt | 4 +- tondb-scanner/CMakeLists.txt | 3 +- tondb-scanner/src/DataParser.cpp | 22 ++- tondb-scanner/src/DataParser.h | 4 +- tondb-scanner/src/DbScanner.cpp | 10 +- tondb-scanner/src/EventProcessor.h | 4 + tondb-scanner/src/Statistics.cpp | 147 ++++++++++++++++ tondb-scanner/src/Statistics.h | 163 ++++++++++++++++++ tondb-scanner/src/TraceAssembler.cpp | 10 +- tondb-scanner/src/smc-interfaces/Tokens.cpp | 4 +- 21 files changed, 409 insertions(+), 22 deletions(-) create mode 100644 tondb-scanner/src/Statistics.cpp create mode 100644 tondb-scanner/src/Statistics.h diff --git a/celldb-migrate/CMakeLists.txt b/celldb-migrate/CMakeLists.txt index 24bbe017..58c2b250 100644 --- a/celldb-migrate/CMakeLists.txt +++ b/celldb-migrate/CMakeLists.txt @@ -6,7 +6,7 @@ add_executable(celldb-migrate target_include_directories(celldb-migrate PUBLIC src ) -target_compile_features(celldb-migrate PRIVATE cxx_std_17) +target_compile_features(celldb-migrate PRIVATE cxx_std_20) target_link_libraries(celldb-migrate tondb-scanner) install(TARGETS celldb-migrate RUNTIME DESTINATION bin) diff --git a/ton-index-clickhouse/CMakeLists.txt b/ton-index-clickhouse/CMakeLists.txt index ed770fb2..d4ca7c5d 100644 --- a/ton-index-clickhouse/CMakeLists.txt +++ b/ton-index-clickhouse/CMakeLists.txt @@ -17,7 +17,7 @@ target_link_directories(ton-index-clickhouse PUBLIC external/ton PUBLIC external/clickhouse-cpp ) -target_compile_features(ton-index-clickhouse PRIVATE cxx_std_17) +target_compile_features(ton-index-clickhouse PRIVATE cxx_std_20) target_link_libraries(ton-index-clickhouse tondb-scanner overlay tdutils tdactor adnl tl_api dht ton_crypto catchain validatorsession validator-disk ton_validator validator-disk smc-envelope clickhouse-cpp-lib) diff --git a/ton-index-postgres-v2/CMakeLists.txt b/ton-index-postgres-v2/CMakeLists.txt index 111aa340..adc9574b 100644 --- a/ton-index-postgres-v2/CMakeLists.txt +++ b/ton-index-postgres-v2/CMakeLists.txt @@ -18,7 +18,7 @@ target_link_directories(ton-index-postgres-v2 PUBLIC external/libpqxx ) -target_compile_features(ton-index-postgres-v2 PRIVATE cxx_std_17) +target_compile_features(ton-index-postgres-v2 PRIVATE cxx_std_20) target_link_libraries(ton-index-postgres-v2 tondb-scanner overlay tdutils tdactor adnl tl_api dht ton_crypto catchain validatorsession validator-disk ton_validator validator-disk smc-envelope pqxx) target_link_options(ton-index-postgres-v2 PUBLIC -rdynamic) diff --git a/ton-index-postgres-v2/src/BlockInterfacesDetector.h b/ton-index-postgres-v2/src/BlockInterfacesDetector.h index abe6f201..8efd272f 100644 --- a/ton-index-postgres-v2/src/BlockInterfacesDetector.h +++ b/ton-index-postgres-v2/src/BlockInterfacesDetector.h @@ -2,17 +2,20 @@ #include #include #include "IndexData.h" +#include "Statistics.h" class BlockInterfaceProcessor: public td::actor::Actor { private: ParsedBlockPtr block_; td::Promise promise_; std::unordered_map, AddressHasher> interfaces_{}; + td::Timer timer_{true}; public: BlockInterfaceProcessor(ParsedBlockPtr block, td::Promise promise) : block_(std::move(block)), promise_(std::move(promise)) {} void start_up() override { + timer_.resume(); std::unordered_map account_states_to_detect; for (const auto& account_state : block_->account_states_) { if (!account_state.code_hash || !account_state.data_hash) { @@ -165,6 +168,7 @@ class BlockInterfaceProcessor: public td::actor::Actor { block_->account_interfaces_ = std::move(interfaces_); promise_.set_result(std::move(block_)); } + g_statistics.record_time(DETECT_INTERFACES_SEQNO, timer_.elapsed() * 1e3); stop(); } }; \ No newline at end of file diff --git a/ton-index-postgres-v2/src/IndexScheduler.cpp b/ton-index-postgres-v2/src/IndexScheduler.cpp index fbba2d60..2fb9167d 100644 --- a/ton-index-postgres-v2/src/IndexScheduler.cpp +++ b/ton-index-postgres-v2/src/IndexScheduler.cpp @@ -3,6 +3,9 @@ #include "td/utils/StringBuilder.h" #include #include "BlockInterfacesDetector.h" +#include "Statistics.h" +#include "td/utils/filesystem.h" +#include "common/delay.h" void IndexScheduler::start_up() { @@ -45,6 +48,18 @@ void IndexScheduler::alarm() { print_stats(); next_print_stats_ = td::Timestamp::in(stats_timeout_); } + if (next_statistics_flush_.is_in_past()) { + ton::delay_action([working_dir = this->working_dir_]() { + auto stats = g_statistics.generate_report_and_reset(); + auto path = working_dir + "/" + "stats.txt"; + auto status = td::atomic_write_file(path, std::move(stats)); + if (status.is_error()) { + LOG(ERROR) << "Failed to write statistics to " << path << ": " << status.error(); + } + }, td::Timestamp::now()); + + next_statistics_flush_ = td::Timestamp::in(60.0); + } auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R){ R.ensure(); @@ -145,6 +160,7 @@ void IndexScheduler::schedule_seqno(std::uint32_t mc_seqno) { LOG(DEBUG) << "Scheduled seqno " << mc_seqno; processing_seqnos_.insert(mc_seqno); + timers_[mc_seqno] = td::Timer(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), mc_seqno](td::Result R) { if (R.is_error()) { LOG(ERROR) << "Failed to fetch seqno " << mc_seqno << ": " << R.move_as_error(); @@ -227,12 +243,13 @@ void IndexScheduler::seqno_interfaces_processed(std::uint32_t mc_seqno, ParsedBl void IndexScheduler::seqno_actions_processed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block) { LOG(DEBUG) << "Actions processed for seqno " << mc_seqno; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), mc_seqno](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), mc_seqno, timer = td::Timer{}](td::Result R) { if (R.is_error()) { LOG(ERROR) << "Failed to insert seqno " << mc_seqno << ": " << R.move_as_error(); td::actor::send_closure(SelfId, &IndexScheduler::reschedule_seqno, mc_seqno); return; } + g_statistics.record_time(INSERT_SEQNO, timer.elapsed() * 1e3); td::actor::send_closure(SelfId, &IndexScheduler::seqno_inserted, mc_seqno, R.move_as_ok()); }); auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), mc_seqno](td::Result R){ @@ -281,6 +298,8 @@ void IndexScheduler::seqno_inserted(std::uint32_t mc_seqno, td::Unit result) { if (mc_seqno > last_indexed_seqno_) { last_indexed_seqno_ = mc_seqno; } + g_statistics.record_time(PROCESS_SEQNO, timers_[mc_seqno].elapsed() * 1e3); + timers_.erase(mc_seqno); } void IndexScheduler::schedule_next_seqnos() { diff --git a/ton-index-postgres-v2/src/IndexScheduler.h b/ton-index-postgres-v2/src/IndexScheduler.h index f948c643..4fa09af2 100644 --- a/ton-index-postgres-v2/src/IndexScheduler.h +++ b/ton-index-postgres-v2/src/IndexScheduler.h @@ -43,6 +43,9 @@ class IndexScheduler: public td::actor::Actor { std::int32_t stats_timeout_{10}; td::Timestamp next_print_stats_; + td::Timestamp next_statistics_flush_; + + std::unordered_map timers_; public: IndexScheduler(td::actor::ActorId db_scanner, td::actor::ActorId insert_manager, td::actor::ActorId parse_manager, std::string working_dir, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0, bool force_index = false, diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index fda36023..101579b5 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -2,6 +2,7 @@ #include "td/utils/JsonBuilder.h" #include "InsertManagerPostgres.h" #include "convert-utils.h" +#include "Statistics.h" namespace pqxx @@ -393,9 +394,13 @@ void InsertBatchPostgres::start_up() { void InsertBatchPostgres::alarm() { try { + td::Timer connect_timer; pqxx::connection c(connection_string_); + connect_timer.pause(); + td::Timer data_timer; pqxx::work txn(c); + insert_blocks(txn, with_copy_); insert_shard_state(txn, with_copy_); insert_transactions(txn, with_copy_); @@ -405,6 +410,8 @@ void InsertBatchPostgres::alarm() { insert_jetton_burns(txn, with_copy_); insert_nft_transfers(txn, with_copy_); insert_traces(txn, with_copy_); + data_timer.pause(); + td::Timer states_timer; std::string insert_under_mutex_query; insert_under_mutex_query += insert_jetton_masters(txn); insert_under_mutex_query += insert_jetton_wallets(txn); @@ -414,21 +421,32 @@ void InsertBatchPostgres::alarm() { insert_under_mutex_query += insert_getgems_nft_sales(txn); insert_under_mutex_query += insert_latest_account_states(txn); + td::Timer commit_timer{true}; { std::lock_guard guard(latest_account_states_update_mutex); txn.exec0(insert_under_mutex_query); + states_timer.pause(); + commit_timer.resume(); txn.commit(); + commit_timer.pause(); } for(auto& task : insert_tasks_) { task.promise_.set_value(td::Unit()); } promise_.set_value(td::Unit()); + + g_statistics.record_time(INSERT_BATCH_CONNECT, connect_timer.elapsed() * 1e3); + g_statistics.record_time(INSERT_BATCH_EXEC_DATA, data_timer.elapsed() * 1e3); + g_statistics.record_time(INSERT_BATCH_EXEC_STATES, states_timer.elapsed() * 1e3); + g_statistics.record_time(INSERT_BATCH_COMMIT, commit_timer.elapsed() * 1e3); + stop(); } catch (const pqxx::integrity_constraint_violation &e) { LOG(WARNING) << "Error COPY to PG: " << e.what(); LOG(WARNING) << "Apparently this block already exists in the database. Nevertheless we retry with INSERT ... ON CONFLICT ..."; with_copy_ = false; + g_statistics.record_count(INSERT_CONFLICT); alarm_timestamp() = td::Timestamp::now(); } catch (const std::exception &e) { LOG(ERROR) << "Error inserting to PG: " << e.what(); diff --git a/ton-index-postgres/CMakeLists.txt b/ton-index-postgres/CMakeLists.txt index 67f8d860..da4b7eac 100644 --- a/ton-index-postgres/CMakeLists.txt +++ b/ton-index-postgres/CMakeLists.txt @@ -18,7 +18,7 @@ target_link_directories(ton-index-postgres PUBLIC external/libpqxx ) -target_compile_features(ton-index-postgres PRIVATE cxx_std_17) +target_compile_features(ton-index-postgres PRIVATE cxx_std_20) target_link_libraries(ton-index-postgres tondb-scanner overlay tdutils tdactor adnl tl_api dht ton_crypto catchain validatorsession validator-disk ton_validator validator-disk smc-envelope pqxx) target_link_options(ton-index-postgres PUBLIC -rdynamic) diff --git a/ton-integrity-checker/CMakeLists.txt b/ton-integrity-checker/CMakeLists.txt index 23972ee3..83f7b560 100644 --- a/ton-integrity-checker/CMakeLists.txt +++ b/ton-integrity-checker/CMakeLists.txt @@ -17,7 +17,7 @@ target_link_directories(ton-integrity-checker PUBLIC external/libpqxx ) -target_compile_features(ton-integrity-checker PRIVATE cxx_std_17) +target_compile_features(ton-integrity-checker PRIVATE cxx_std_20) target_link_libraries(ton-integrity-checker tondb-scanner overlay tdutils tdactor adnl tl_api dht ton_crypto catchain validatorsession validator-disk ton_validator validator-disk smc-envelope pqxx) diff --git a/ton-smc-scanner/CMakeLists.txt b/ton-smc-scanner/CMakeLists.txt index d29933d0..7fe9085c 100644 --- a/ton-smc-scanner/CMakeLists.txt +++ b/ton-smc-scanner/CMakeLists.txt @@ -18,7 +18,7 @@ target_link_directories(ton-smc-scanner PUBLIC external/libpqxx ) -target_compile_features(ton-smc-scanner PRIVATE cxx_std_17) +target_compile_features(ton-smc-scanner PRIVATE cxx_std_20) target_link_libraries(ton-smc-scanner tondb-scanner overlay tdutils tdactor adnl tl_api dht ton_crypto catchain validatorsession validator-disk ton_validator validator-disk smc-envelope pqxx) diff --git a/ton-smc-scanner/src/SmcScanner.cpp b/ton-smc-scanner/src/SmcScanner.cpp index 322b3cd5..6ac2b54f 100644 --- a/ton-smc-scanner/src/SmcScanner.cpp +++ b/ton-smc-scanner/src/SmcScanner.cpp @@ -2,7 +2,7 @@ #include "convert-utils.h" void SmcScanner::start_up() { - auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R){ + auto P = td::PromiseCreator::lambda([=, this, SelfId = actor_id(this)](td::Result R){ if (R.is_error()) { LOG(ERROR) << "Failed to get seqno " << options_.seqno_ << ": " << R.move_as_error(); stop(); diff --git a/ton-trace-emulator/CMakeLists.txt b/ton-trace-emulator/CMakeLists.txt index 3d5418ea..90d384cc 100644 --- a/ton-trace-emulator/CMakeLists.txt +++ b/ton-trace-emulator/CMakeLists.txt @@ -15,7 +15,7 @@ target_link_directories(ton-trace-emulator-core PUBLIC external/ton ) target_link_libraries(ton-trace-emulator-core tondb-scanner overlay ton_validator emulator) -target_compile_features(ton-trace-emulator-core PRIVATE cxx_std_17) +target_compile_features(ton-trace-emulator-core PRIVATE cxx_std_20) add_executable(ton-trace-emulator src/main.cpp @@ -37,7 +37,7 @@ target_link_directories(ton-trace-emulator PUBLIC external/redis-plus-plus PUBLIC external/msgpack-c ) -target_compile_features(ton-trace-emulator PRIVATE cxx_std_17) +target_compile_features(ton-trace-emulator PRIVATE cxx_std_20) target_link_libraries(ton-trace-emulator ton_validator ton-trace-emulator-core hiredis redis++ msgpack-cxx) target_link_options(ton-trace-emulator PUBLIC -rdynamic) install(TARGETS ton-trace-emulator RUNTIME DESTINATION bin) diff --git a/tondb-scanner/CMakeLists.txt b/tondb-scanner/CMakeLists.txt index db81bc45..7d14caec 100644 --- a/tondb-scanner/CMakeLists.txt +++ b/tondb-scanner/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(tondb-scanner STATIC src/parse_token_data.cpp src/convert-utils.cpp src/tokens.cpp + src/Statistics.cpp src/smc-interfaces/Tokens.cpp src/smc-interfaces/NftSale.cpp src/smc-interfaces/execute-smc.cpp @@ -29,7 +30,7 @@ target_link_directories(tondb-scanner PUBLIC external/libpqxx PUBLIC external/msgpack-c ) -target_compile_features(tondb-scanner PRIVATE cxx_std_17) +target_compile_features(tondb-scanner PRIVATE cxx_std_20) target_link_libraries(tondb-scanner overlay tdutils tdactor adnl tl_api dht catchain validatorsession validator-disk ton_validator validator-disk smc-envelope pqxx msgpack-cxx) diff --git a/tondb-scanner/src/DataParser.cpp b/tondb-scanner/src/DataParser.cpp index 2745c4b0..371fa835 100644 --- a/tondb-scanner/src/DataParser.cpp +++ b/tondb-scanner/src/DataParser.cpp @@ -8,11 +8,15 @@ #include "validator/interfaces/block.h" #include "validator/interfaces/shard.h" #include "convert-utils.h" +#include "Statistics.h" using namespace ton::validator; //TODO: remove this void ParseQuery::start_up() { + td::Timer timer; auto status = parse_impl(); + g_statistics.record_time(PARSE_SEQNO, timer.elapsed() * 1e3); + if(status.is_error()) { promise_.set_error(status.move_as_error()); } @@ -25,6 +29,7 @@ void ParseQuery::start_up() { td::Status ParseQuery::parse_impl() { td::optional mc_block; for (auto &block_ds : mc_block_.shard_blocks_diff_) { + td::Timer timer; // common block info block::gen::Block::Record blk; block::gen::BlockInfo::Record info; @@ -40,39 +45,46 @@ td::Status ParseQuery::parse_impl() { } // transactions and messages + td::Timer transactions_timer; std::map account_states_to_get; TRY_RESULT_ASSIGN(schema_block.transactions, parse_transactions(block_ds.block_data->block_id(), blk, info, extra, account_states_to_get)); + g_statistics.record_time(PARSE_TRANSACTION, transactions_timer.elapsed() * 1e6, schema_block.transactions.size()); + td::Timer acc_states_timer; TRY_RESULT(account_states_fast, parse_account_states_new(schema_block.workchain, schema_block.gen_utime, account_states_to_get)); + g_statistics.record_time(PARSE_ACCOUNT_STATE, acc_states_timer.elapsed() * 1e6, account_states_fast.size()); for (auto &acc : account_states_fast) { - result->account_states_.push_back(acc); + result->account_states_.push_back(std::move(acc)); } // config if (block_ds.block_data->block_id().is_masterchain()) { + td::Timer config_timer; TRY_RESULT_ASSIGN(mc_block_.config_, block::ConfigInfo::extract_config(block_ds.block_state, block::ConfigInfo::needCapabilities | block::ConfigInfo::needLibraries)); + g_statistics.record_time(PARSE_CONFIG, config_timer.elapsed() * 1e6); } result->blocks_.push_back(schema_block); + g_statistics.record_time(PARSE_BLOCK, timer.elapsed() * 1e3); } // shard details - for (auto &block_ds : mc_block_.shard_blocks_) { + for (const auto &block_ds : mc_block_.shard_blocks_) { auto shard_block = parse_shard_state(mc_block.value(), block_ds.block_data->block_id()); - result->shard_state_.push_back(shard_block); + result->shard_state_.push_back(std::move(shard_block)); } result->mc_block_ = mc_block_; return td::Status::OK(); } -schema::MasterchainBlockShard ParseQuery::parse_shard_state(schema::Block mc_block, const ton::BlockIdExt& shard_blk_id) { +schema::MasterchainBlockShard ParseQuery::parse_shard_state(const schema::Block& mc_block, const ton::BlockIdExt& shard_blk_id) { return {mc_block.seqno, mc_block.start_lt, mc_block.gen_utime, shard_blk_id.id.workchain, static_cast(shard_blk_id.id.shard), shard_blk_id.id.seqno}; } schema::Block ParseQuery::parse_block(const td::Ref& root_cell, const ton::BlockIdExt& blk_id, block::gen::Block::Record& blk, const block::gen::BlockInfo::Record& info, - const block::gen::BlockExtra::Record& extra, td::optional &mc_block) { + const block::gen::BlockExtra::Record& extra, const td::optional &mc_block) { schema::Block block; block.workchain = blk_id.id.workchain; block.shard = static_cast(blk_id.id.shard); diff --git a/tondb-scanner/src/DataParser.h b/tondb-scanner/src/DataParser.h index 6b8a360e..fbfa251a 100644 --- a/tondb-scanner/src/DataParser.h +++ b/tondb-scanner/src/DataParser.h @@ -19,8 +19,8 @@ class ParseQuery: public td::actor::Actor { td::Status parse_impl(); schema::Block parse_block(const td::Ref& root_cell, const ton::BlockIdExt& blk_id, block::gen::Block::Record& blk, const block::gen::BlockInfo::Record& info, - const block::gen::BlockExtra::Record& extra, td::optional &mc_block); - schema::MasterchainBlockShard parse_shard_state(schema::Block mc_block, const ton::BlockIdExt& shard_blk_id); + const block::gen::BlockExtra::Record& extra, const td::optional &mc_block); + schema::MasterchainBlockShard parse_shard_state(const schema::Block& mc_block, const ton::BlockIdExt& shard_blk_id); static td::Result parse_currency_collection(td::Ref csr); td::Result parse_message(td::Ref msg_cell); td::Result parse_tr_storage_phase(vm::CellSlice& cs); diff --git a/tondb-scanner/src/DbScanner.cpp b/tondb-scanner/src/DbScanner.cpp index 120255ba..8ebfa793 100644 --- a/tondb-scanner/src/DbScanner.cpp +++ b/tondb-scanner/src/DbScanner.cpp @@ -2,6 +2,7 @@ #include "validator/interfaces/block.h" #include "validator/interfaces/shard.h" #include "td/actor/MultiPromise.h" +#include "Statistics.h" using namespace ton::validator; @@ -71,6 +72,7 @@ class IndexQuery: public td::actor::Actor { const int mc_seqno_; td::actor::ActorId db_; td::Promise promise_; + td::Timer timer_{true}; td::Ref mc_block_data_; td::Ref mc_block_state_; @@ -93,6 +95,7 @@ class IndexQuery: public td::actor::Actor { } void start_up() override { + timer_.resume(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { td::actor::send_closure(SelfId, &IndexQuery::got_mc_block_handle, std::move(R)); }); @@ -267,11 +270,13 @@ class IndexQuery: public td::actor::Actor { void finish() { // LOG(INFO) << "For seqno " << mc_seqno_ << " got " << result_.shard_blocks_.size() << " shard blocks and " << result_.shard_blocks_diff_.size() << " shard blocks diff"; promise_.set_value(std::move(result_)); + g_statistics.record_time(FETCH_SEQNO, timer_.elapsed() * 1e3); stop(); } void error(td::Status error) { promise_.set_error(std::move(error)); + g_statistics.record_count(SEQNO_FETCH_ERROR); stop(); } }; @@ -311,9 +316,12 @@ void DbScanner::get_cell_db_reader(td::Promise } void DbScanner::catch_up_with_primary() { - auto R = td::PromiseCreator::lambda([SelfId = actor_id(this), this](td::Result R) { + auto R = td::PromiseCreator::lambda([SelfId = actor_id(this), this, timer = td::Timer{}](td::Result R) mutable { R.ensure(); this->is_ready_ = true; + timer.pause(); + g_statistics.record_time(CATCH_UP_WITH_PRIMARY, timer.elapsed() * 1e3); + LOG(INFO) << "Catch up with primary took " << timer.elapsed() * 1e3 << " micros"; }); td::actor::send_closure(db_, &RootDb::try_catch_up_with_primary, std::move(R)); } diff --git a/tondb-scanner/src/EventProcessor.h b/tondb-scanner/src/EventProcessor.h index 0a89e981..40f20db6 100644 --- a/tondb-scanner/src/EventProcessor.h +++ b/tondb-scanner/src/EventProcessor.h @@ -1,5 +1,6 @@ #pragma once #include "InterfaceDetectors.hpp" +#include "Statistics.h" // Detects special cases of Actions like - Jetton transfers and burns, NFT transfers @@ -7,16 +8,19 @@ class ActionDetector: public td::actor::Actor { private: ParsedBlockPtr block_; td::Promise promise_; + td::Timer timer_{true}; public: ActionDetector(ParsedBlockPtr block, td::Promise promise): block_(block), promise_(std::move(promise)) { } void start_up() override { + timer_.resume(); for (const auto& block : block_->blocks_) { for (const auto& transaction : block.transactions) { process_tx(transaction); } } + g_statistics.record_time(DETECT_ACTIONS_SEQNO, timer_.elapsed() * 1e6); promise_.set_value(std::move(block_)); stop(); } diff --git a/tondb-scanner/src/Statistics.cpp b/tondb-scanner/src/Statistics.cpp new file mode 100644 index 00000000..3411c420 --- /dev/null +++ b/tondb-scanner/src/Statistics.cpp @@ -0,0 +1,147 @@ +#include "Statistics.h" +#include + + +void HistogramImpl::add(uint64_t duration) { + count_.fetch_add(1, std::memory_order_relaxed); + sum_.fetch_add(duration, std::memory_order_relaxed); + update_max(max_, duration); + + size_t index = bucketMapper.IndexForValue(duration); + buckets_[index].fetch_add(1, std::memory_order_relaxed); +} + +uint64_t HistogramImpl::get_count() const { + return count_.load(std::memory_order_relaxed); +} + +uint64_t HistogramImpl::get_sum() const { + return sum_.load(std::memory_order_relaxed); +} + +double HistogramImpl::compute_percentile(double percentile) const { + std::lock_guard lock(mutex_); + uint64_t total = get_count(); + if (total == 0) return 0.0; + + double threshold = total * (percentile / 100.0); + uint64_t accumulated = 0; + size_t bucket_idx = 0; + + for (; bucket_idx < bucketCount(); ++bucket_idx) { + uint64_t bucket_count = buckets_[bucket_idx].load(std::memory_order_relaxed); + if (accumulated + bucket_count >= threshold) break; + accumulated += bucket_count; + } + + uint64_t lower = 0, upper = 0; + if (bucket_idx == 0) { + upper = bucketMapper.FirstValue(); + } else if (bucket_idx < bucketCount()) { + lower = bucketMapper.BucketLimit(bucket_idx - 1) + 1; + upper = bucketMapper.BucketLimit(bucket_idx); + } else { + lower = bucketMapper.LastValue() + 1; + upper = max_.load(std::memory_order_relaxed); + } + + uint64_t actual_max = max_.load(std::memory_order_relaxed); + if (upper > actual_max) { + upper = actual_max; + } + + uint64_t bucket_count = buckets_[bucket_idx].load(std::memory_order_relaxed); + if (bucket_count == 0) return upper; + + double fraction = (threshold - accumulated) / bucket_count; + return lower + fraction * (upper - lower); +} + +uint64_t HistogramImpl::get_max() const { + return max_.load(std::memory_order_relaxed); +} + +void HistogramImpl::merge(const HistogramImpl &other) { + std::lock_guard lock(mutex_); + count_.fetch_add(other.count_.load(std::memory_order_relaxed), std::memory_order_relaxed); + sum_.fetch_add(other.sum_.load(std::memory_order_relaxed), std::memory_order_relaxed); + update_max(max_, other.max_.load(std::memory_order_relaxed)); + update_min(min_, other.min_.load(std::memory_order_relaxed)); + + for (size_t i = 0; i < bucketCount(); ++i) { + buckets_[i].fetch_add(other.buckets_[i].load(std::memory_order_relaxed), std::memory_order_relaxed); + } +} + +void HistogramImpl::reset() { + std::lock_guard lock(mutex_); + count_.store(0, std::memory_order_relaxed); + sum_.store(0, std::memory_order_relaxed); + max_.store(0, std::memory_order_relaxed); + min_.store(0, std::memory_order_relaxed); + + for (size_t i = 0; i < bucketCount(); ++i) { + buckets_[i].store(0, std::memory_order_relaxed); + } +} + +void HistogramImpl::update_max(std::atomic& max, uint64_t value) { + uint64_t current = max.load(std::memory_order_relaxed); + while (value > current && !max.compare_exchange_weak(current, value, std::memory_order_relaxed)); +} + +void HistogramImpl::update_min(std::atomic& min, uint64_t value) { + uint64_t current = min.load(std::memory_order_relaxed); + while (value < current && !min.compare_exchange_weak(current, value, std::memory_order_relaxed)); +} + +void Statistics::record_time(Histogram hist, uint64_t duration, uint32_t count) { + if (count > 1) { + // for batch events, we average the duration + duration /= count; + } + per_core_stats_.get().histograms_[hist].add(duration); +} + +void Statistics::record_count(Ticker ticker, uint64_t count) { + per_core_stats_.get().tickers_[ticker].fetch_add(count, std::memory_order_relaxed); +} + +std::string Statistics::generate_report_and_reset() { + std::array aggTickers = {}; + std::array aggHist; + + per_core_stats_.for_each([&](StatisticsData &data) { + // Merge tickers. + for (uint32_t i = 0; i < TICKERS_COUNT; ++i) { + uint64_t value = data.tickers_[i].load(std::memory_order_relaxed); + aggTickers[i] += value; + data.tickers_[i].store(0, std::memory_order_relaxed); + } + + // Merge histograms. + for (uint32_t h = 0; h < HISTOGRAMS_COUNT; ++h) { + aggHist[h].merge(data.histograms_[h]); + data.histograms_[h].reset(); + } + }); + + std::ostringstream oss; + oss << std::setprecision(3) << std::fixed; + for (uint32_t i = 0; i < TICKERS_COUNT; ++i) { + oss << ticker_names.at(i) << " COUNT : " << aggTickers[i] << std::endl; + } + + for (uint32_t h = 0; h < HISTOGRAMS_COUNT; ++h) { + oss << histogram_names.at(h) << " P50 : " << aggHist[h].compute_percentile(50.0) + << " P95 : " << aggHist[h].compute_percentile(95.0) + << " P99 : " << aggHist[h].compute_percentile(99.0) + << " P100 : " << aggHist[h].get_max() + << " COUNT : " << aggHist[h].get_count() + << " SUM : " << aggHist[h].get_sum() << std::endl; + } + + return oss.str(); +} + +Statistics g_statistics; \ No newline at end of file diff --git a/tondb-scanner/src/Statistics.h b/tondb-scanner/src/Statistics.h new file mode 100644 index 00000000..68e2d1aa --- /dev/null +++ b/tondb-scanner/src/Statistics.h @@ -0,0 +1,163 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include "td/utils/ThreadLocalStorage.h" + +constexpr size_t bucketCount() { + size_t count = 2; // 1 and 2 + double bucket_val = 2.0; + constexpr double max = static_cast(std::numeric_limits::max()); + while ((bucket_val = 1.5 * bucket_val) <= max) { + ++count; + } + return count; +}; + +consteval auto bucketValues() { + std::array bucketValues; + bucketValues[0] = 1; + bucketValues[1] = 2; + size_t i = 1; + double bucket_val = static_cast(bucketValues[i++]); + constexpr double max = static_cast(std::numeric_limits::max()); + while ((bucket_val = 1.5 * bucket_val) <= max) { + bucketValues[i++] = static_cast(bucket_val); + } + if (bucketCount() != i) { + throw "bucketValues() does not match bucketCount()"; + } + return bucketValues; +} + +class HistogramBucketMapper +{ +public: + consteval HistogramBucketMapper() : bucketValues_(bucketValues()) {} + + size_t IndexForValue(const uint64_t value) const { + auto beg = bucketValues_.begin(); + auto end = bucketValues_.end(); + if (value >= LastValue()) + return end - beg - 1; // bucketValues_.size() - 1 + else + return std::lower_bound(beg, end, value) - beg; + } + + constexpr uint64_t LastValue() const { return bucketValues_.back(); } + constexpr uint64_t FirstValue() const { return bucketValues_.front(); } + constexpr uint64_t BucketCount() const { return bucketValues_.size(); } + + uint64_t BucketLimit(const size_t bucketNumber) const { + assert(bucketNumber < BucketCount()); + return bucketValues_[bucketNumber]; + } + +private: + std::result_of::type bucketValues_; +}; + +constexpr HistogramBucketMapper bucketMapper; + +class HistogramImpl { +public: + HistogramImpl() = default; + void add(uint64_t duration); + uint64_t get_count() const; + uint64_t get_sum() const; + double compute_percentile(double percentile) const; + uint64_t get_max() const; + void merge(const HistogramImpl &other); + void reset(); + +private: + std::atomic count_{0}; + std::atomic sum_{0}; + std::atomic min_{0}; + std::atomic max_{0}; + std::vector bucket_limits_; + std::atomic buckets_[bucketCount()]; + mutable std::mutex mutex_; + + static void update_max(std::atomic &max, uint64_t value); + static void update_min(std::atomic &min, uint64_t value); +}; + +enum Ticker : uint32_t { + SEQNO_FETCH_ERROR = 0, + INSERT_CONFLICT, + TICKERS_COUNT +}; + +enum Histogram : uint32_t { + PROCESS_SEQNO = 0, + + CATCH_UP_WITH_PRIMARY, + FETCH_SEQNO, + + PARSE_SEQNO, + PARSE_BLOCK, + PARSE_TRANSACTION, + PARSE_ACCOUNT_STATE, + PARSE_CONFIG, + + TRACE_ASSEMBLER_GC_STATE, + TRACE_ASSEMBLER_SAVE_STATE, + TRACE_ASSEMBLER_PROCESS_BLOCK, + + DETECT_INTERFACES_SEQNO, + DETECT_ACTIONS_SEQNO, + + INSERT_SEQNO, + INSERT_BATCH_CONNECT, + INSERT_BATCH_EXEC_DATA, + INSERT_BATCH_EXEC_STATES, + INSERT_BATCH_COMMIT, + + HISTOGRAMS_COUNT +}; + +const std::unordered_map ticker_names = { + {SEQNO_FETCH_ERROR, "indexer.seqno.fetch.error"}, + {INSERT_CONFLICT, "indexer.insert.conflict"}, +}; +const std::unordered_map histogram_names = { + {PROCESS_SEQNO, "indexer.index.seqno.millis"}, + {CATCH_UP_WITH_PRIMARY, "indexer.catch_up_with_primary.millis"}, + {FETCH_SEQNO, "indexer.fetch.seqno.millis"}, + {PARSE_SEQNO, "indexer.parse.seqno.millis"}, + {PARSE_BLOCK, "indexer.parse.block.millis"}, + {PARSE_TRANSACTION, "indexer.parse.transaction.micros"}, + {PARSE_ACCOUNT_STATE, "indexer.parse.account_state.micros"}, + {PARSE_CONFIG, "indexer.parse.config.micros"}, + {TRACE_ASSEMBLER_GC_STATE, "indexer.traceassembler.gc_state.micros"}, + {TRACE_ASSEMBLER_SAVE_STATE, "indexer.traceassembler.save_state.micros"}, + {TRACE_ASSEMBLER_PROCESS_BLOCK, "indexer.traceassembler.process_block.micros"}, + {DETECT_INTERFACES_SEQNO, "indexer.interfaces.seqno.millis"}, + {DETECT_ACTIONS_SEQNO, "indexer.actions.seqno.micros"}, + {INSERT_SEQNO, "indexer.insert.seqno.millis"}, + {INSERT_BATCH_CONNECT, "indexer.insert.batch.connect.millis"}, + {INSERT_BATCH_EXEC_DATA, "indexer.insert.batch.exec_data.millis"}, + {INSERT_BATCH_EXEC_STATES, "indexer.insert.batch.exec_states.millis"}, + {INSERT_BATCH_COMMIT, "indexer.insert.batch.commit.millis"}, +}; + +class Statistics { +public: + void record_time(Histogram hist, uint64_t duration, uint32_t count = 1); + void record_count(Ticker ticker, uint64_t count = 1); + std::string generate_report_and_reset(); + +private: + struct StatisticsData { + std::atomic_uint_fast64_t tickers_[TICKERS_COUNT] = {{0}}; + HistogramImpl histograms_[HISTOGRAMS_COUNT]; + }; + + td::ThreadLocalStorage per_core_stats_; +}; + +extern Statistics g_statistics; diff --git a/tondb-scanner/src/TraceAssembler.cpp b/tondb-scanner/src/TraceAssembler.cpp index 5aa8da17..121f54da 100644 --- a/tondb-scanner/src/TraceAssembler.cpp +++ b/tondb-scanner/src/TraceAssembler.cpp @@ -6,6 +6,7 @@ #include "td/utils/port/path.h" #include "common/delay.h" #include "convert-utils.h" +#include "Statistics.h" namespace fs = std::filesystem; @@ -49,6 +50,7 @@ void TraceAssembler::start_up() { } td::Status gc_states(std::string db_path, ton::BlockSeqno current_seqno, size_t keep_last) { + td::Timer timer; std::map> fileMap; for (const auto& entry : fs::directory_iterator(db_path)) { @@ -80,18 +82,22 @@ td::Status gc_states(std::string db_path, ton::BlockSeqno current_seqno, size_t count++; } } + g_statistics.record_time(TRACE_ASSEMBLER_GC_STATE, timer.elapsed() * 1e6); return td::Status::OK(); } td::Status save_state(std::string db_path, ton::BlockSeqno seqno, std::unordered_map pending_traces, std::unordered_map pending_edges) { + td::Timer timer; std::stringstream buffer; msgpack::pack(buffer, pending_traces); msgpack::pack(buffer, pending_edges); auto path = db_path + "/" + std::to_string(seqno) + ".tastate"; - return td::atomic_write_file(path, buffer.str()); + auto result = td::atomic_write_file(path, buffer.str()); + g_statistics.record_time(TRACE_ASSEMBLER_SAVE_STATE, timer.elapsed() * 1e6); + return result; } void TraceAssembler::alarm() { @@ -202,7 +208,9 @@ td::Result TraceAssembler::restore_state(ton::BlockSeqno seqno) void TraceAssembler::process_queue() { auto it = queue_.find(expected_seqno_); while(it != queue_.end()) { + td::Timer timer; process_block(it->second.seqno_, it->second.block_); + g_statistics.record_time(TRACE_ASSEMBLER_PROCESS_BLOCK, timer.elapsed() * 1e6); it->second.promise_.set_result(it->second.block_); // block processed diff --git a/tondb-scanner/src/smc-interfaces/Tokens.cpp b/tondb-scanner/src/smc-interfaces/Tokens.cpp index 35f128cb..72482705 100644 --- a/tondb-scanner/src/smc-interfaces/Tokens.cpp +++ b/tondb-scanner/src/smc-interfaces/Tokens.cpp @@ -126,7 +126,7 @@ void JettonWalletDetectorR::start_up() { data.mintless_is_claimed = std::nullopt; } - auto R = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result account_state_r) mutable { + auto R = td::PromiseCreator::lambda([=, this, SelfId = actor_id(this)](td::Result account_state_r) mutable { if (account_state_r.is_error()) { promise_.set_error(account_state_r.move_as_error()); stop(); @@ -302,7 +302,7 @@ void NftItemDetectorR::start_up() { stop(); } else { auto ind_content = stack[4].as_cell(); - auto R = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result account_state_r) mutable { + auto R = td::PromiseCreator::lambda([=, this, SelfId = actor_id(this)](td::Result account_state_r) mutable { if (account_state_r.is_error()) { promise_.set_error(account_state_r.move_as_error()); stop(); From b4c7e8047f45f2f4bf8ef577e3f31ca33e23123b Mon Sep 17 00:00:00 2001 From: Marat S Date: Fri, 7 Feb 2025 16:09:54 +0000 Subject: [PATCH 18/28] fix statistics add by batch --- tondb-scanner/src/DbScanner.cpp | 1 - tondb-scanner/src/Statistics.cpp | 8 ++++---- tondb-scanner/src/Statistics.h | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tondb-scanner/src/DbScanner.cpp b/tondb-scanner/src/DbScanner.cpp index 8ebfa793..0ca7994b 100644 --- a/tondb-scanner/src/DbScanner.cpp +++ b/tondb-scanner/src/DbScanner.cpp @@ -321,7 +321,6 @@ void DbScanner::catch_up_with_primary() { this->is_ready_ = true; timer.pause(); g_statistics.record_time(CATCH_UP_WITH_PRIMARY, timer.elapsed() * 1e3); - LOG(INFO) << "Catch up with primary took " << timer.elapsed() * 1e3 << " micros"; }); td::actor::send_closure(db_, &RootDb::try_catch_up_with_primary, std::move(R)); } diff --git a/tondb-scanner/src/Statistics.cpp b/tondb-scanner/src/Statistics.cpp index 3411c420..4fb0df5b 100644 --- a/tondb-scanner/src/Statistics.cpp +++ b/tondb-scanner/src/Statistics.cpp @@ -2,13 +2,13 @@ #include -void HistogramImpl::add(uint64_t duration) { - count_.fetch_add(1, std::memory_order_relaxed); +void HistogramImpl::add(uint64_t duration, size_t count = 1) { + count_.fetch_add(count, std::memory_order_relaxed); sum_.fetch_add(duration, std::memory_order_relaxed); update_max(max_, duration); size_t index = bucketMapper.IndexForValue(duration); - buckets_[index].fetch_add(1, std::memory_order_relaxed); + buckets_[index].fetch_add(count, std::memory_order_relaxed); } uint64_t HistogramImpl::get_count() const { @@ -100,7 +100,7 @@ void Statistics::record_time(Histogram hist, uint64_t duration, uint32_t count) // for batch events, we average the duration duration /= count; } - per_core_stats_.get().histograms_[hist].add(duration); + per_core_stats_.get().histograms_[hist].add(duration, count); } void Statistics::record_count(Ticker ticker, uint64_t count) { diff --git a/tondb-scanner/src/Statistics.h b/tondb-scanner/src/Statistics.h index 68e2d1aa..c3b57d3c 100644 --- a/tondb-scanner/src/Statistics.h +++ b/tondb-scanner/src/Statistics.h @@ -65,7 +65,7 @@ constexpr HistogramBucketMapper bucketMapper; class HistogramImpl { public: HistogramImpl() = default; - void add(uint64_t duration); + void add(uint64_t duration, size_t count = 1); uint64_t get_count() const; uint64_t get_sum() const; double compute_percentile(double percentile) const; From 977890a001819cd5b9dd9daa100f9de79033ea78 Mon Sep 17 00:00:00 2001 From: Marat S Date: Fri, 7 Feb 2025 16:53:58 +0000 Subject: [PATCH 19/28] bf --- tondb-scanner/src/Statistics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tondb-scanner/src/Statistics.cpp b/tondb-scanner/src/Statistics.cpp index 4fb0df5b..7e71aa2a 100644 --- a/tondb-scanner/src/Statistics.cpp +++ b/tondb-scanner/src/Statistics.cpp @@ -2,7 +2,7 @@ #include -void HistogramImpl::add(uint64_t duration, size_t count = 1) { +void HistogramImpl::add(uint64_t duration, size_t count) { count_.fetch_add(count, std::memory_order_relaxed); sum_.fetch_add(duration, std::memory_order_relaxed); update_max(max_, duration); From ca47d6d215c3c4cfc66713f6768717358de36097 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:50:27 +0100 Subject: [PATCH 20/28] Fix excessive quoting of code and data in latest_account_states table (#109) --- ton-index-postgres-v2/src/InsertManagerPostgres.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp index 101579b5..f3cd29e2 100644 --- a/ton-index-postgres-v2/src/InsertManagerPostgres.cpp +++ b/ton-index-postgres-v2/src/InsertManagerPostgres.cpp @@ -990,7 +990,7 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { if (max_data_depth_ >= 0 && account_state.data.not_null() && (max_data_depth_ == 0 || account_state.data->get_depth() <= max_data_depth_)){ auto data_res = vm::std_boc_serialize(account_state.data); if (data_res.is_ok()){ - data_str = txn.quote(td::base64_encode(data_res.move_as_ok().as_slice().str())); + data_str = td::base64_encode(data_res.move_as_ok()); } } else { if (account_state.data.not_null()) { @@ -1001,7 +1001,7 @@ std::string InsertBatchPostgres::insert_latest_account_states(pqxx::work &txn) { { auto code_res = vm::std_boc_serialize(account_state.code); if (code_res.is_ok()){ - code_str = txn.quote(td::base64_encode(code_res.move_as_ok().as_slice().str())); + code_str = td::base64_encode(code_res.move_as_ok()); } if (code_str->length() > 128000) { LOG(WARNING) << "Large account code: " << account_state.account; From 5674c8829801595c7570d23faa1929405e1e5656 Mon Sep 17 00:00:00 2001 From: Pavel Shuvalov Date: Tue, 18 Feb 2025 19:14:18 +0400 Subject: [PATCH 21/28] Add anycast parsing support (#104) * Add anycast parsing support * Add support for std address as well * rewrite pfx without direct access to raw pointers --- tondb-scanner/CMakeLists.txt | 14 ++++++++++ tondb-scanner/src/convert-utils.cpp | 23 +++++++++++++++ tondb-scanner/test/tests.cpp | 43 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/tondb-scanner/CMakeLists.txt b/tondb-scanner/CMakeLists.txt index 7d14caec..9dca54c7 100644 --- a/tondb-scanner/CMakeLists.txt +++ b/tondb-scanner/CMakeLists.txt @@ -50,3 +50,17 @@ add_custom_command( add_custom_target(tlb_generate_tokens DEPENDS ${TLB_TOKENS}) add_dependencies(tondb-scanner tlb_generate_tokens) + +set(TONDB_SCANNER_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/tests.cpp + PARENT_SCOPE +) + +add_executable(test-tondb test/tests.cpp ${TONDB_SCANNER_TEST_SOURCE}) +target_include_directories(test-tondb + PUBLIC external/ton + PUBLIC src/ +) + +target_link_libraries(test-tondb PRIVATE tdutils tdactor smc-envelope tondb-scanner ton_block) +add_dependencies(test-tondb test-tdutils) \ No newline at end of file diff --git a/tondb-scanner/src/convert-utils.cpp b/tondb-scanner/src/convert-utils.cpp index fbf91637..3b2b828c 100644 --- a/tondb-scanner/src/convert-utils.cpp +++ b/tondb-scanner/src/convert-utils.cpp @@ -1,5 +1,6 @@ #include "td/actor/actor.h" #include "vm/cells/Cell.h" +#include "vm/cells/CellSlice.h" #include "vm/stack.hpp" #include "common/refcnt.hpp" #include "smc-envelope/SmartContract.h" @@ -21,6 +22,17 @@ td::Result convert::to_raw_address(td::Ref cs) { if (!tlb::csr_unpack(cs, addr)) { return td::Status::Error("Failed to unpack MsgAddressInt"); } + if (addr.anycast.not_null()) { + if (addr.anycast->bit_at(0) == 1) { + auto anycast_slice = vm::CellSlice(*addr.anycast); + anycast_slice.advance(1); // skip maybe bit + block::gen::Anycast::Record anycast; + if (!tlb::unpack_exact(anycast_slice, anycast)) { + return td::Status::Error("Failed to unpack Anycast"); + } + addr.address.bits().copy_from(anycast.rewrite_pfx->cbits(), anycast.depth); + } + } return std::to_string(addr.workchain_id) + ":" + addr.address.to_hex(); } default: @@ -52,6 +64,17 @@ td::Result convert::to_std_address(td::Ref cs) if (!tlb::csr_unpack(cs, addr)) { return td::Status::Error("Failed to unpack addr_std"); } + if (addr.anycast.not_null()) { + if (addr.anycast->bit_at(0) == 1) { + auto anycast_slice = vm::CellSlice(*addr.anycast); + anycast_slice.advance(1); // skip maybe bit + block::gen::Anycast::Record anycast; + if (!tlb::unpack_exact(anycast_slice, anycast)) { + return td::Status::Error("Failed to unpack Anycast"); + } + addr.address.bits().copy_from(anycast.rewrite_pfx->cbits(), anycast.depth); + } + } return block::StdAddress(addr.workchain_id, addr.address); } default: diff --git a/tondb-scanner/test/tests.cpp b/tondb-scanner/test/tests.cpp index b937899d..afbc86f7 100644 --- a/tondb-scanner/test/tests.cpp +++ b/tondb-scanner/test/tests.cpp @@ -5,14 +5,51 @@ #include "td/utils/check.h" #include "crypto/vm/cp0.h" +#include "crypto/vm/boc.h" #include "td/utils/tests.h" #include "td/actor/actor.h" #include "td/utils/base64.h" +#include "crypto/block/block.h" +#include "vm/cells/Cell.h" +#include "convert-utils.h" // #include "InterfaceDetector.hpp" +TEST(convert, to_raw_address) { + auto address = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6ccgEBAQEAJAAAQ4ARhy+Ifz/haSyza6FGBWNSde+ZjHy+uBieS1O4PxaPVnA=")).move_as_ok()).move_as_ok(); + auto raw_address_serialized = convert::to_raw_address(vm::load_cell_slice_ref(address)); + CHECK(raw_address_serialized.is_ok()); + ASSERT_EQ("0:8C397C43F9FF0B49659B5D0A302B1A93AF7CCC63E5F5C0C4F25A9DC1F8B47AB3", raw_address_serialized.move_as_ok()); +} + +TEST(convert, to_raw_address_with_anycast) { + // tx hash 692263ed0c02006a42c2570c1526dc0968e9ef36849086e7888599f5f7745f3b + auto address = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6ccgEBAQEAKAAAS74kmhnMAhgIba/WWH4XFusx+cERuqOcvI+CfNEkWIYKL2ECy/pq")).move_as_ok()).move_as_ok(); + auto raw_address_serialized = convert::to_raw_address(vm::load_cell_slice_ref(address)); + CHECK(raw_address_serialized.is_ok()); + ASSERT_EQ("0:249A19CFF5961F85C5BACC7E70446EA8E72F23E09F34491621828BD840B2FE9A", raw_address_serialized.move_as_ok()); +} + +TEST(convert, to_std_address) { + auto address = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6ccgEBAQEAJAAAQ4ARhy+Ifz/haSyza6FGBWNSde+ZjHy+uBieS1O4PxaPVnA=")).move_as_ok()).move_as_ok(); + auto std_address_serialized = convert::to_std_address(vm::load_cell_slice_ref(address)); + CHECK(std_address_serialized.is_ok()); + td::Bits256 addr; + addr.from_hex(td::Slice("8C397C43F9FF0B49659B5D0A302B1A93AF7CCC63E5F5C0C4F25A9DC1F8B47AB3")); + ASSERT_EQ(block::StdAddress(0, addr), std_address_serialized.move_as_ok()); +} + +TEST(convert, to_std_address_with_anycast) { + auto address = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6ccgEBAQEAKAAAS74kmhnMAhgIba/WWH4XFusx+cERuqOcvI+CfNEkWIYKL2ECy/pq")).move_as_ok()).move_as_ok(); + auto std_address_serialized = convert::to_std_address(vm::load_cell_slice_ref(address)); + CHECK(std_address_serialized.is_ok()); + td::Bits256 addr; + addr.from_hex(td::Slice("249A19CFF5961F85C5BACC7E70446EA8E72F23E09F34491621828BD840B2FE9A")); + ASSERT_EQ(block::StdAddress(0, addr), std_address_serialized.move_as_ok()); +} + // TEST(TonDbScanner, JettonWalletDetector) { // td::actor::Scheduler scheduler({1}); @@ -37,3 +74,9 @@ +int main(int argc, char **argv) { + td::set_default_failure_signal_handler().ensure(); + auto &runner = td::TestsRunner::get_default(); + runner.run_all(); + return 0; +} \ No newline at end of file From 4fa6a647cd2a086caf29b47af262f3f37908a347 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Tue, 18 Feb 2025 16:14:37 +0100 Subject: [PATCH 22/28] Fix macos build (#110) * fix macos build * fix ton * update libpqxx to 7.10.0 --- .gitignore | 8 +++----- external/libpqxx | 2 +- ton-index-postgres-v2/CMakeLists.txt | 5 ----- tondb-scanner/CMakeLists.txt | 16 ++++------------ tondb-scanner/src/EventProcessor.cpp | 2 +- tondb-scanner/src/InterfaceDetectors.hpp | 2 +- tondb-scanner/src/Statistics.cpp | 1 + tondb-scanner/src/Statistics.h | 2 +- tondb-scanner/src/parse_token_data.cpp | 2 +- tondb-scanner/src/smc-interfaces/Tokens.cpp | 2 +- 10 files changed, 14 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 1c091ed9..1bfdacad 100644 --- a/.gitignore +++ b/.gitignore @@ -225,9 +225,7 @@ sandbox/ # vs code .vscode/ +.DS_Store -tondb-scanner/src/tokens.cpp -tondb-scanner/src/tokens.h - -ton-index-worker/src/tokens.cpp -ton-index-worker/src/tokens.h +tondb-scanner/src/tokens-tlb.cpp +tondb-scanner/src/tokens-tlb.h diff --git a/external/libpqxx b/external/libpqxx index b7cdeb93..d280c6c9 160000 --- a/external/libpqxx +++ b/external/libpqxx @@ -1 +1 @@ -Subproject commit b7cdeb9327cf8ac817355cd2ae76b063e9bdde02 +Subproject commit d280c6c9a6d063638b5bb0d2450bc15ed641c27c diff --git a/ton-index-postgres-v2/CMakeLists.txt b/ton-index-postgres-v2/CMakeLists.txt index adc9574b..cc083cd2 100644 --- a/ton-index-postgres-v2/CMakeLists.txt +++ b/ton-index-postgres-v2/CMakeLists.txt @@ -13,11 +13,6 @@ target_include_directories(ton-index-postgres-v2 PUBLIC src/ ) -target_link_directories(ton-index-postgres-v2 - PUBLIC external/ton - PUBLIC external/libpqxx -) - target_compile_features(ton-index-postgres-v2 PRIVATE cxx_std_20) target_link_libraries(ton-index-postgres-v2 tondb-scanner overlay tdutils tdactor adnl tl_api dht ton_crypto catchain validatorsession validator-disk ton_validator validator-disk smc-envelope pqxx) diff --git a/tondb-scanner/CMakeLists.txt b/tondb-scanner/CMakeLists.txt index 9dca54c7..8bbb421d 100644 --- a/tondb-scanner/CMakeLists.txt +++ b/tondb-scanner/CMakeLists.txt @@ -11,7 +11,7 @@ add_library(tondb-scanner STATIC src/queue_state.cpp src/parse_token_data.cpp src/convert-utils.cpp - src/tokens.cpp + src/tokens-tlb.cpp src/Statistics.cpp src/smc-interfaces/Tokens.cpp src/smc-interfaces/NftSale.cpp @@ -19,30 +19,22 @@ add_library(tondb-scanner STATIC ) target_include_directories(tondb-scanner - PUBLIC external/ton - PUBLIC external/libpqxx - PUBLIC external/msgpack-c PUBLIC src/ ) -target_link_directories(tondb-scanner - PUBLIC external/ton - PUBLIC external/libpqxx - PUBLIC external/msgpack-c -) target_compile_features(tondb-scanner PRIVATE cxx_std_20) target_link_libraries(tondb-scanner overlay tdutils tdactor adnl tl_api dht catchain validatorsession validator-disk ton_validator validator-disk smc-envelope pqxx msgpack-cxx) set(TLB_TOKENS - ${CMAKE_CURRENT_SOURCE_DIR}/src/tokens.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/tokens.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/tokens-tlb.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/tokens-tlb.h ) add_custom_command( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src - COMMAND tlbc -o tokens -n tokens::gen -z tlb/tokens.tlb + COMMAND tlbc -o tokens-tlb -n tokens::gen -z tlb/tokens.tlb COMMENT "Generate tokes tlb source files" OUTPUT ${TLB_TOKENS} DEPENDS tlbc src/tlb/tokens.tlb diff --git a/tondb-scanner/src/EventProcessor.cpp b/tondb-scanner/src/EventProcessor.cpp index 4af2ffb6..47fd7463 100644 --- a/tondb-scanner/src/EventProcessor.cpp +++ b/tondb-scanner/src/EventProcessor.cpp @@ -9,7 +9,7 @@ #include "IndexData.h" #include "td/actor/MultiPromise.h" #include "convert-utils.h" -#include "tokens.h" +#include "tokens-tlb.h" // process ParsedBlock and try detect master and wallet interfaces diff --git a/tondb-scanner/src/InterfaceDetectors.hpp b/tondb-scanner/src/InterfaceDetectors.hpp index c05aacc0..8f47a19c 100644 --- a/tondb-scanner/src/InterfaceDetectors.hpp +++ b/tondb-scanner/src/InterfaceDetectors.hpp @@ -16,7 +16,7 @@ #include "parse_token_data.h" #include "convert-utils.h" -#include "tokens.h" +#include "tokens-tlb.h" #include "InsertManager.h" #include "IndexData.h" #include "DataParser.h" diff --git a/tondb-scanner/src/Statistics.cpp b/tondb-scanner/src/Statistics.cpp index 7e71aa2a..fa542d9e 100644 --- a/tondb-scanner/src/Statistics.cpp +++ b/tondb-scanner/src/Statistics.cpp @@ -1,4 +1,5 @@ #include "Statistics.h" +#include #include diff --git a/tondb-scanner/src/Statistics.h b/tondb-scanner/src/Statistics.h index c3b57d3c..1f7787f5 100644 --- a/tondb-scanner/src/Statistics.h +++ b/tondb-scanner/src/Statistics.h @@ -57,7 +57,7 @@ class HistogramBucketMapper } private: - std::result_of::type bucketValues_; + std::invoke_result_t bucketValues_; }; constexpr HistogramBucketMapper bucketMapper; diff --git a/tondb-scanner/src/parse_token_data.cpp b/tondb-scanner/src/parse_token_data.cpp index c122258b..afd69e2d 100644 --- a/tondb-scanner/src/parse_token_data.cpp +++ b/tondb-scanner/src/parse_token_data.cpp @@ -1,5 +1,5 @@ #include "InsertManager.h" -#include "tokens.h" +#include "tokens-tlb.h" #include "common/checksum.h" diff --git a/tondb-scanner/src/smc-interfaces/Tokens.cpp b/tondb-scanner/src/smc-interfaces/Tokens.cpp index 72482705..ffacfdde 100644 --- a/tondb-scanner/src/smc-interfaces/Tokens.cpp +++ b/tondb-scanner/src/smc-interfaces/Tokens.cpp @@ -6,7 +6,7 @@ #include "Tokens.h" #include "parse_token_data.h" #include "smc-interfaces/execute-smc.h" -#include "tokens.h" +#include "tokens-tlb.h" #include "common/checksum.h" From d56732580380a1d1beef5e6b5a302aff1518aa2c Mon Sep 17 00:00:00 2001 From: Pavel Shuvalov Date: Wed, 26 Feb 2025 10:30:11 +0400 Subject: [PATCH 23/28] Add test for Jetton burn without custom payload parser --- tondb-scanner/test/tests.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tondb-scanner/test/tests.cpp b/tondb-scanner/test/tests.cpp index afbc86f7..988abc90 100644 --- a/tondb-scanner/test/tests.cpp +++ b/tondb-scanner/test/tests.cpp @@ -13,7 +13,7 @@ #include "crypto/block/block.h" #include "vm/cells/Cell.h" #include "convert-utils.h" -// #include "InterfaceDetector.hpp" +#include "InterfaceDetectors.hpp" @@ -50,6 +50,30 @@ TEST(convert, to_std_address_with_anycast) { ASSERT_EQ(block::StdAddress(0, addr), std_address_serialized.move_as_ok()); } +TEST(TonDbScanner, JettonWalletDetector) { + td::actor::Scheduler scheduler({1}); + auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + + block::StdAddress addr(std::string("EQDKC7jQ_tIJuYyrWfI4FIAN-hFHakG3GrATpOiqBVtsGOd5")); + auto code_cell = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6cckECEgEAAzEAART/APSkE/S88sgLAQIBYgIDAgLLBAUAG6D2BdqJofQB9IH0gahhAgEgBgcCAWILDAIBSAgJAfH4Hpn/0AfSAQ+AH2omh9AH0gfSBqGCibUKkVY4L5cWCUYX/5cWEqGiE4KhAJqgoB5CgCfQEsZ4sA54tmZJFkZYCJegB6AGWAZJB8gDg6ZGWBZQPl/+ToAn0gegIY/QAQa6ThAHlxYjvADGRlgqgEZ4s4fQEL5bWJ5kCgC3QgxwCSXwTgAdDTAwFxsJUTXwPwEuD6QPpAMfoAMXHXIfoAMfoAMALTHyGCEA+KfqW6lTE0WfAP4CGCEBeNRRm6ljFERAPwEOA1ghBZXwe8upNZ8BHgXwSED/LwgAEV+kQwcLry4U2ACughAXjUUZyMsfGcs/UAf6AiLPFlAGzxYl+gJQA88WyVAFzCORcpFx4lAIqBOgggiYloCqAIIImJaAoKAUvPLixQTJgED7ABAjyFAE+gJYzxYBzxbMye1UAgEgDQ4AgUgCDXIe1E0PoA+kD6QNQwBNMfIYIQF41FGboCghB73ZfeuhKx8uLF0z8x+gAwE6BQI8hQBPoCWM8WAc8WzMntVIA/c7UTQ+gD6QPpA1DAI0z/6AFFRoAX6QPpAU1vHBVRzbXBUIBNUFAPIUAT6AljPFgHPFszJIsjLARL0APQAywDJ+QBwdMjLAsoHy//J0FANxwUcsfLiwwr6AFGooYIImJaAggiYloAStgihggiYloCgGKEn4w8l1wsBwwAjgDxARAOM7UTQ+gD6QPpA1DAH0z/6APpA9AQwUWKhUkrHBfLiwSjC//LiwoIImJaAqgAXoBe88uLDghB73ZfeyMsfyz9QBfoCIc8WUAPPFvQAyXGAGMjLBSTPFnD6AstqzMmAQPsAQBPIUAT6AljPFgHPFszJ7VSAAcFJ5oBihghBzYtCcyMsfUjDLP1j6AlAHzxZQB88WyXGAEMjLBSTPFlAG+gIVy2oUzMlx+wAQJBAjAA4QSRA4N18EAHbCALCOIYIQ1TJ223CAEMjLBVAIzxZQBPoCFstqEssfEss/yXL7AJM1bCHiA8hQBPoCWM8WAc8WzMntVLp4DOo=")).move_as_ok()).move_as_ok(); + auto data_cell = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6cckECEwEAA3sAAY0xKctoASFZDXpO6Q6ZgXHilrBvG9KSTVMUJk1CMXwYaoCc9JirAC61IQRl0/la95t27xhIpjxZt32vl1QQVF2UgTNuvD18YAEBFP8A9KQT9LzyyAsCAgFiAwQCAssFBgAboPYF2omh9AH0gfSBqGECASAHCAIBYgwNAgFICQoB8fgemf/QB9IBD4AfaiaH0AfSB9IGoYKJtQqRVjgvlxYJRhf/lxYSoaITgqEAmqCgHkKAJ9ASxniwDni2ZkkWRlgIl6AHoAZYBkkHyAODpkZYFlA+X/5OgCfSB6Ahj9ABBrpOEAeXFiO8AMZGWCqARnizh9AQvltYnmQLALdCDHAJJfBOAB0NMDAXGwlRNfA/AS4PpA+kAx+gAxcdch+gAx+gAwAtMfIYIQD4p+pbqVMTRZ8A/gIYIQF41FGbqWMUREA/AQ4DWCEFlfB7y6k1nwEeBfBIQP8vCAARX6RDBwuvLhTYAK6CEBeNRRnIyx8Zyz9QB/oCIs8WUAbPFiX6AlADzxbJUAXMI5FykXHiUAioE6CCCJiWgKoAggiYloCgoBS88uLFBMmAQPsAECPIUAT6AljPFgHPFszJ7VQCASAODwCBSAINch7UTQ+gD6QPpA1DAE0x8hghAXjUUZugKCEHvdl966ErHy4sXTPzH6ADAToFAjyFAE+gJYzxYBzxbMye1UgD9ztRND6APpA+kDUMAjTP/oAUVGgBfpA+kBTW8cFVHNtcFQgE1QUA8hQBPoCWM8WAc8WzMkiyMsBEvQA9ADLAMn5AHB0yMsCygfL/8nQUA3HBRyx8uLDCvoAUaihggiYloCCCJiWgBK2CKGCCJiWgKAYoSfjDyXXCwHDACOAQERIA4ztRND6APpA+kDUMAfTP/oA+kD0BDBRYqFSSscF8uLBKML/8uLCggiYloCqABegF7zy4sOCEHvdl97Iyx/LP1AF+gIhzxZQA88W9ADJcYAYyMsFJM8WcPoCy2rMyYBA+wBAE8hQBPoCWM8WAc8WzMntVIABwUnmgGKGCEHNi0JzIyx9SMMs/WPoCUAfPFlAHzxbJcYAQyMsFJM8WUAb6AhXLahTMyXH7ABAkECMADhBJEDg3XwQAdsIAsI4hghDVMnbbcIAQyMsFUAjPFlAE+gIWy2oSyx8Syz/JcvsAkzVsIeIDyFAE+gJYzxYBzxbMye1U8/HTGA==")).move_as_ok()).move_as_ok(); + auto P = td::PromiseCreator::lambda([](td::Result R) { + CHECK(R.is_ok()); + LOG(INFO) << R.move_as_ok().jetton; + }); + scheduler.run_in_context([&] { + td::actor::ActorId insert_manager; + // auto interface_manager = td::actor::create_actor("interface_manager"); + // auto jetton_master_detector = td::actor::create_actor("jetton_master_detector"); + // auto jetton_wallet_detector = td::actor::create_actor("jetton_wallet_detector", jetton_master_detector.get(), interface_manager.get(), insert_manager); + // td::actor::send_closure(jetton_wallet_detector, &JettonWalletDetector::detect, block::StdAddress(), code_cell, data_cell, 0, std::move(P)); + watcher.reset(); + }); + + scheduler.run(); + +} + // TEST(TonDbScanner, JettonWalletDetector) { // td::actor::Scheduler scheduler({1}); From 045f2bbf1241897030680629d76d7af2c7eeead4 Mon Sep 17 00:00:00 2001 From: Pavel Shuvalov Date: Wed, 26 Feb 2025 17:42:44 +0400 Subject: [PATCH 24/28] Add jetton burn test --- tondb-scanner/src/InterfaceDetectors.hpp | 10 +++++ tondb-scanner/test/tests.cpp | 54 ++++++++++++++++++------ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/tondb-scanner/src/InterfaceDetectors.hpp b/tondb-scanner/src/InterfaceDetectors.hpp index 8f47a19c..94255ccc 100644 --- a/tondb-scanner/src/InterfaceDetectors.hpp +++ b/tondb-scanner/src/InterfaceDetectors.hpp @@ -125,6 +125,8 @@ class InterfaceDetector: public td::actor::Actor { template class InterfaceStorage { public: + InterfaceStorage() {}; + InterfaceStorage(std::unordered_map cache) : cache_(cache) {}; std::unordered_map cache_{}; void check(block::StdAddress address, td::Promise promise) { @@ -351,6 +353,14 @@ class JettonWalletDetector: public InterfaceDetector { : jetton_master_detector_(jetton_master_detector) , interface_manager_(interface_manager) { } + JettonWalletDetector(td::actor::ActorId jetton_master_detector, + td::actor::ActorId interface_manager, + td::actor::ActorId insert_manager, + std::unordered_map cache) + : jetton_master_detector_(jetton_master_detector) + , interface_manager_(interface_manager) + , storage_(cache) { + } void detect(block::StdAddress address, td::Ref code_cell, td::Ref data_cell, uint64_t last_tx_lt, uint32_t last_tx_now, const MasterchainBlockDataState& blocks_ds, td::Promise promise) override { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), address, code_cell, data_cell, last_tx_lt, last_tx_now, blocks_ds, promise = std::move(promise)](td::Result code_hash_is_wallet) mutable { diff --git a/tondb-scanner/test/tests.cpp b/tondb-scanner/test/tests.cpp index 988abc90..5f61d4cc 100644 --- a/tondb-scanner/test/tests.cpp +++ b/tondb-scanner/test/tests.cpp @@ -14,7 +14,7 @@ #include "vm/cells/Cell.h" #include "convert-utils.h" #include "InterfaceDetectors.hpp" - +#include "IndexData.h" TEST(convert, to_raw_address) { @@ -50,27 +50,53 @@ TEST(convert, to_std_address_with_anycast) { ASSERT_EQ(block::StdAddress(0, addr), std_address_serialized.move_as_ok()); } -TEST(TonDbScanner, JettonWalletDetector) { +TEST(TonDbScanner, ParseBurnWithCustomPayload) { td::actor::Scheduler scheduler({1}); auto watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); block::StdAddress addr(std::string("EQDKC7jQ_tIJuYyrWfI4FIAN-hFHakG3GrATpOiqBVtsGOd5")); - auto code_cell = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6cckECEgEAAzEAART/APSkE/S88sgLAQIBYgIDAgLLBAUAG6D2BdqJofQB9IH0gahhAgEgBgcCAWILDAIBSAgJAfH4Hpn/0AfSAQ+AH2omh9AH0gfSBqGCibUKkVY4L5cWCUYX/5cWEqGiE4KhAJqgoB5CgCfQEsZ4sA54tmZJFkZYCJegB6AGWAZJB8gDg6ZGWBZQPl/+ToAn0gegIY/QAQa6ThAHlxYjvADGRlgqgEZ4s4fQEL5bWJ5kCgC3QgxwCSXwTgAdDTAwFxsJUTXwPwEuD6QPpAMfoAMXHXIfoAMfoAMALTHyGCEA+KfqW6lTE0WfAP4CGCEBeNRRm6ljFERAPwEOA1ghBZXwe8upNZ8BHgXwSED/LwgAEV+kQwcLry4U2ACughAXjUUZyMsfGcs/UAf6AiLPFlAGzxYl+gJQA88WyVAFzCORcpFx4lAIqBOgggiYloCqAIIImJaAoKAUvPLixQTJgED7ABAjyFAE+gJYzxYBzxbMye1UAgEgDQ4AgUgCDXIe1E0PoA+kD6QNQwBNMfIYIQF41FGboCghB73ZfeuhKx8uLF0z8x+gAwE6BQI8hQBPoCWM8WAc8WzMntVIA/c7UTQ+gD6QPpA1DAI0z/6AFFRoAX6QPpAU1vHBVRzbXBUIBNUFAPIUAT6AljPFgHPFszJIsjLARL0APQAywDJ+QBwdMjLAsoHy//J0FANxwUcsfLiwwr6AFGooYIImJaAggiYloAStgihggiYloCgGKEn4w8l1wsBwwAjgDxARAOM7UTQ+gD6QPpA1DAH0z/6APpA9AQwUWKhUkrHBfLiwSjC//LiwoIImJaAqgAXoBe88uLDghB73ZfeyMsfyz9QBfoCIc8WUAPPFvQAyXGAGMjLBSTPFnD6AstqzMmAQPsAQBPIUAT6AljPFgHPFszJ7VSAAcFJ5oBihghBzYtCcyMsfUjDLP1j6AlAHzxZQB88WyXGAEMjLBSTPFlAG+gIVy2oUzMlx+wAQJBAjAA4QSRA4N18EAHbCALCOIYIQ1TJ223CAEMjLBVAIzxZQBPoCFstqEssfEss/yXL7AJM1bCHiA8hQBPoCWM8WAc8WzMntVLp4DOo=")).move_as_ok()).move_as_ok(); - auto data_cell = vm::std_boc_deserialize(td::base64_decode(td::Slice("te6cckECEwEAA3sAAY0xKctoASFZDXpO6Q6ZgXHilrBvG9KSTVMUJk1CMXwYaoCc9JirAC61IQRl0/la95t27xhIpjxZt32vl1QQVF2UgTNuvD18YAEBFP8A9KQT9LzyyAsCAgFiAwQCAssFBgAboPYF2omh9AH0gfSBqGECASAHCAIBYgwNAgFICQoB8fgemf/QB9IBD4AfaiaH0AfSB9IGoYKJtQqRVjgvlxYJRhf/lxYSoaITgqEAmqCgHkKAJ9ASxniwDni2ZkkWRlgIl6AHoAZYBkkHyAODpkZYFlA+X/5OgCfSB6Ahj9ABBrpOEAeXFiO8AMZGWCqARnizh9AQvltYnmQLALdCDHAJJfBOAB0NMDAXGwlRNfA/AS4PpA+kAx+gAxcdch+gAx+gAwAtMfIYIQD4p+pbqVMTRZ8A/gIYIQF41FGbqWMUREA/AQ4DWCEFlfB7y6k1nwEeBfBIQP8vCAARX6RDBwuvLhTYAK6CEBeNRRnIyx8Zyz9QB/oCIs8WUAbPFiX6AlADzxbJUAXMI5FykXHiUAioE6CCCJiWgKoAggiYloCgoBS88uLFBMmAQPsAECPIUAT6AljPFgHPFszJ7VQCASAODwCBSAINch7UTQ+gD6QPpA1DAE0x8hghAXjUUZugKCEHvdl966ErHy4sXTPzH6ADAToFAjyFAE+gJYzxYBzxbMye1UgD9ztRND6APpA+kDUMAjTP/oAUVGgBfpA+kBTW8cFVHNtcFQgE1QUA8hQBPoCWM8WAc8WzMkiyMsBEvQA9ADLAMn5AHB0yMsCygfL/8nQUA3HBRyx8uLDCvoAUaihggiYloCCCJiWgBK2CKGCCJiWgKAYoSfjDyXXCwHDACOAQERIA4ztRND6APpA+kDUMAfTP/oA+kD0BDBRYqFSSscF8uLBKML/8uLCggiYloCqABegF7zy4sOCEHvdl97Iyx/LP1AF+gIhzxZQA88W9ADJcYAYyMsFJM8WcPoCy2rMyYBA+wBAE8hQBPoCWM8WAc8WzMntVIABwUnmgGKGCEHNi0JzIyx9SMMs/WPoCUAfPFlAHzxbJcYAQyMsFJM8WUAb6AhXLahTMyXH7ABAkECMADhBJEDg3XwQAdsIAsI4hghDVMnbbcIAQyMsFUAjPFlAE+gIWy2oSyx8Syz/JcvsAkzVsIeIDyFAE+gJYzxYBzxbMye1U8/HTGA==")).move_as_ok()).move_as_ok(); - auto P = td::PromiseCreator::lambda([](td::Result R) { - CHECK(R.is_ok()); - LOG(INFO) << R.move_as_ok().jetton; - }); + // message payload for tx xiOZW3mVbHkCtgLxQqXVAg4DIgxrTE3j9lHw6H3P/Yg=, correct layout + auto message_payload = vm::load_cell_slice_ref(vm::std_boc_deserialize(td::base64_decode( + td::Slice("te6cckEBAgEAOQABZllfB7xUbeTvz/ieq1AezMgZqAEPdvSWQKq0LY5UE5PZzQsh8ADqxh1H03XLREV/xoLz4QEAAaAxtO6I")).move_as_ok()).move_as_ok()); + + auto transaction = schema::Transaction(); + transaction.account = block::StdAddress(std::string("EQCk6s76oduqQH_3Y3O7fxjWVoUXtD3Ev6-NbHjjDmfG1drE")); // jetton wallet + transaction.in_msg = std::make_optional(schema::Message()); + transaction.in_msg->source = "0:87BB7A4B20555A16C72A09C9ECE68590F80075630EA3E9BAE5A222BFE34179F0"; // owner + + td::actor::ActorId insert_manager; + td::actor::ActorOwn jetton_wallet_detector; + td::actor::ActorOwn interface_manager; + td::actor::ActorOwn jetton_master_detector; + + // prepare jetton metadata + std::unordered_map cache; + JettonWalletData jetton_master; + jetton_master.jetton = "0:BDF3FA8098D129B54B4F73B5BAC5D1E1FD91EB054169C3916DFC8CCD536D1000"; + cache.emplace(std::string("0:A4EACEFAA1DBAA407FF76373BB7F18D6568517B43DC4BFAF8D6C78E30E67C6D5"), jetton_master); + scheduler.run_in_context([&] { - td::actor::ActorId insert_manager; - // auto interface_manager = td::actor::create_actor("interface_manager"); - // auto jetton_master_detector = td::actor::create_actor("jetton_master_detector"); - // auto jetton_wallet_detector = td::actor::create_actor("jetton_wallet_detector", jetton_master_detector.get(), interface_manager.get(), insert_manager); - // td::actor::send_closure(jetton_wallet_detector, &JettonWalletDetector::detect, block::StdAddress(), code_cell, data_cell, 0, std::move(P)); + interface_manager = td::actor::create_actor("interface_manager", insert_manager); + jetton_master_detector = td::actor::create_actor("jetton_master_detector", interface_manager.get(), insert_manager); + + jetton_wallet_detector = td::actor::create_actor("jetton_wallet_detector", + jetton_master_detector.get(), interface_manager.get(), insert_manager, cache); + + auto P = td::PromiseCreator::lambda([&transaction, &jetton_master](td::Result R) { + CHECK(R.is_ok()); + auto burn = R.move_as_ok(); + ASSERT_EQ(transaction.in_msg->source.value(), burn.owner); + ASSERT_EQ(convert::to_raw_address(transaction.account), burn.jetton_wallet); + ASSERT_EQ(jetton_master.jetton, burn.jetton_master); + ASSERT_EQ(6083770390284902059, burn.query_id); + CHECK(td::BigIntG<257>(8267792794) == **burn.amount.get()); + }); + td::actor::send_closure(jetton_wallet_detector, &JettonWalletDetector::parse_burn, transaction, message_payload, std::move(P)); watcher.reset(); }); - scheduler.run(); + scheduler.run(10); + scheduler.stop(); } From 1fe5c1fb2ce93865eb61c40f599cde5f08d249c4 Mon Sep 17 00:00:00 2001 From: Pavel Shuvalov Date: Wed, 26 Feb 2025 17:46:21 +0400 Subject: [PATCH 25/28] Add github workflow script --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b65b0aac..63d7cabd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.16) project(ton-index-cpp) +set(CMAKE_BUILD_TYPE Debug) + option(PGTON "Enable adding the pgton subdirectory" OFF) option(TON_USE_JEMALLOC "Use \"ON\" to enable JeMalloc." OFF) From 8795821f6289029d0a72a0b244285587d68d86d9 Mon Sep 17 00:00:00 2001 From: Pavel Shuvalov Date: Wed, 26 Feb 2025 17:48:14 +0400 Subject: [PATCH 26/28] Fix cmake file --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 63d7cabd..b65b0aac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,8 +2,6 @@ cmake_minimum_required(VERSION 3.16) project(ton-index-cpp) -set(CMAKE_BUILD_TYPE Debug) - option(PGTON "Enable adding the pgton subdirectory" OFF) option(TON_USE_JEMALLOC "Use \"ON\" to enable JeMalloc." OFF) From f16fd050c4acf884baeb4dcec8b2b75ea5c9b5f1 Mon Sep 17 00:00:00 2001 From: Pavel Shuvalov Date: Wed, 26 Feb 2025 17:52:49 +0400 Subject: [PATCH 27/28] Add gh workflow --- .github/workflows/tests.yml | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..96bbcf70 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,41 @@ +name: Test pull request + +on: + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +env: + BUILD_TYPE: Release + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build + + - name: Build + # Build your program with the given configuration + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + + - name: Test + working-directory: ${{github.workspace}}/build + run: ./tondb-scanner/test-tondb From 3409b8ed63ee4b87697155c8adfce69e9aefc67a Mon Sep 17 00:00:00 2001 From: Pavel Shuvalov Date: Wed, 26 Feb 2025 18:16:17 +0400 Subject: [PATCH 28/28] Testing workflow --- tondb-scanner/test/tests.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tondb-scanner/test/tests.cpp b/tondb-scanner/test/tests.cpp index 5f61d4cc..579095d3 100644 --- a/tondb-scanner/test/tests.cpp +++ b/tondb-scanner/test/tests.cpp @@ -89,6 +89,7 @@ TEST(TonDbScanner, ParseBurnWithCustomPayload) { ASSERT_EQ(convert::to_raw_address(transaction.account), burn.jetton_wallet); ASSERT_EQ(jetton_master.jetton, burn.jetton_master); ASSERT_EQ(6083770390284902059, burn.query_id); + CHECK(td::BigIntG<257>(8267792794) == **burn.amount.get()); }); td::actor::send_closure(jetton_wallet_detector, &JettonWalletDetector::parse_burn, transaction, message_payload, std::move(P));