From 87ab1666ce052311d8415ad0933b17e911d1a8c1 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 16:17:17 +0300 Subject: [PATCH 01/19] Add overlay certificate checks --- catchain/catchain-receiver.cpp | 2 +- overlay/overlay-broadcast.cpp | 45 ++++++++++++++++++-- overlay/overlay-broadcast.hpp | 20 +++++---- overlay/overlay-fec-broadcast.cpp | 11 ++++- overlay/overlay-fec-broadcast.hpp | 4 +- overlay/overlay-manager.cpp | 67 ++++++++++++++++++++++-------- overlay/overlay.cpp | 43 ++++++++++++------- overlay/overlay.hpp | 5 ++- overlay/overlays.h | 48 +++++++++++++++++---- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 64168 -> 64644 bytes validator/full-node-shard.cpp | 27 +++++++++--- validator/full-node-shard.hpp | 3 ++ validator/manager-disk.hpp | 3 ++ validator/manager-hardfork.hpp | 3 ++ validator/manager.hpp | 5 +++ validator/validator.h | 6 +-- 17 files changed, 232 insertions(+), 62 deletions(-) diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index b9b0373f0..e7f5c019f 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -486,7 +486,7 @@ void CatChainReceiverImpl::start_up() { } td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), - make_callback(), overlay::OverlayPrivacyRules{0, std::move(root_keys)}); + make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}); CHECK(root_block_); diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index bade50e70..1c05f7056 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -17,8 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-broadcast.hpp" +#include "adnl/adnl-node-id.hpp" +#include "common/util.h" #include "overlay.hpp" #include "keys/encryptor.h" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/port/Stat.h" namespace ton { @@ -33,7 +39,13 @@ td::Status BroadcastSimple::check_duplicate() { } td::Status BroadcastSimple::check_source() { - return overlay_->check_source_eligible(source_, cert_.get(), data_size()); + auto r = overlay_->check_source_eligible(source_, cert_.get(), data_size(), false); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + is_valid_ = r == BroadcastCheckResult::Allowed; + return td::Status::OK(); } td::BufferSlice BroadcastSimple::to_sign() { @@ -66,6 +78,14 @@ td::Status BroadcastSimple::distribute() { return td::Status::OK(); } +void BroadcastSimple::broadcast_checked(td::Result R) { + if (R.is_error()) { + return; + } + is_valid_ = true; + run_continue().ignore(); +} + tl_object_ptr BroadcastSimple::tl() const { return create_tl_object(source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), flags_, data_.clone(), date_, signature_.clone()); @@ -75,6 +95,25 @@ td::BufferSlice BroadcastSimple::serialize() { return serialize_tl_object(tl(), true); } +td::Status BroadcastSimple::run_continue() { + TRY_STATUS(distribute()); + deliver(); + return td::Status::OK(); +} + +td::Status BroadcastSimple::run() { + TRY_STATUS(run_checks()); + if (!is_valid_) { + auto P = td::PromiseCreator::lambda( + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result R) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(R)); + }); + overlay_->check_broadcast(source_.compute_short_id(), data_.clone(), std::move(P)); + return td::Status::OK(); + } + return run_continue(); +} + td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); @@ -86,7 +125,7 @@ td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr(broadcast_hash, src, std::move(cert), broadcast->flags_, std::move(broadcast->data_), broadcast->date_, - std::move(broadcast->signature_), overlay); + std::move(broadcast->signature_), false, overlay); TRY_STATUS(B->run()); overlay->register_simple_broadcast(std::move(B)); return td::Status::OK(); @@ -100,7 +139,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, nullptr); + td::BufferSlice{}, false, nullptr); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 665c269d2..da29695a4 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -18,9 +18,16 @@ */ #pragma once +#include "adnl/adnl-local-id.h" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "common/refcnt.hpp" #include "overlay/overlay.h" +#include "td/actor/PromiseFuture.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" namespace ton { @@ -38,6 +45,7 @@ class BroadcastSimple : public td::ListNode { td::BufferSlice data_; td::uint32 date_; td::BufferSlice signature_; + bool is_valid_{false}; OverlayImpl *overlay_; @@ -52,7 +60,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, - td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, + td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, OverlayImpl *overlay) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) @@ -61,6 +69,7 @@ class BroadcastSimple : public td::ListNode { , data_(std::move(data)) , date_(date) , signature_(std::move(signature)) + , is_valid_(is_valid) , overlay_(overlay) { } @@ -80,17 +89,14 @@ class BroadcastSimple : public td::ListNode { } void deliver(); - td::Status run() { - TRY_STATUS(run_checks()); - TRY_STATUS(distribute()); - deliver(); - return td::Status::OK(); - } + td::Status run(); + td::Status run_continue(); tl_object_ptr tl() const; td::BufferSlice serialize(); void update_overlay(OverlayImpl *overlay); + void broadcast_checked(td::Result R); static td::Status create(OverlayImpl *overlay, tl_object_ptr broadcast); static td::Status create_new(td::actor::ActorId overlay, td::actor::ActorId keyring, diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index e368c6208..6ab038001 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -54,7 +54,16 @@ td::Status OverlayFecBroadcastPart::check_duplicate() { } td::Status OverlayFecBroadcastPart::check_source() { - TRY_STATUS(overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_)); + auto r = overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_, true); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + // FIXME + if (r == BroadcastCheckResult::NeedCheck) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + if (bcast_) { TRY_STATUS(bcast_->is_eligible_sender(source_)); } diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 6d30d47cc..0ce9c2ff7 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -185,6 +185,9 @@ class BroadcastFec : public td::ListNode { } } + void broadcast_checked(td::Result R) { + } + private: bool ready_ = false; @@ -311,4 +314,3 @@ class OverlayFecBroadcastPart : public td::ListNode { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index ae7c3dd5a..059ad4e21 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-manager.h" +#include "auto/tl/ton_api.h" #include "overlay.h" #include "adnl/utils.hpp" @@ -268,17 +269,21 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o db_.set(key, create_serialize_tl_object(std::move(obj))); } -Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } -Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } @@ -290,9 +295,19 @@ void Certificate::set_issuer(PublicKey issuer) { issued_by_ = issuer; } +constexpr td::uint32 cert_default_flags(td::uint32 max_size) { + return (max_size > Overlays::max_simple_broadcast_size() ? CertificateFlags::AllowFec : 0) | + CertificateFlags::Trusted; +} + td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const { - return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, - max_size_); + if (flags_ == cert_default_flags(max_size_)) { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_); + } else { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_, flags_); + } } const PublicKeyHash Certificate::issuer_hash() const { @@ -307,32 +322,48 @@ const PublicKey &Certificate::issuer() const { td::Result> Certificate::create(tl_object_ptr cert) { std::shared_ptr res; - ton_api::downcast_call(*cert.get(), td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, - [&](ton_api::overlay_certificate &obj) { - res = std::make_shared( - PublicKey{obj.issued_by_}, obj.expire_at_, - static_cast(obj.max_size_), - std::move(obj.signature_)); - })); + ton_api::downcast_call(*cert.get(), + td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, + [&](ton_api::overlay_certificate &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + cert_default_flags(obj.max_size_), + std::move(obj.signature_)); + }, + [&](ton_api::overlay_certificateV2 &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + static_cast(obj.flags_), + std::move(obj.signature_)); + })); return std::move(res); } -td::Status Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, - td::uint32 size) const { +BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, + td::uint32 size, bool is_fec) const { if (size > max_size_) { - return td::Status::Error(ErrorCode::protoviolation, "too big broadcast size"); + return BroadcastCheckResult::Forbidden; } if (unix_time > expire_at_) { - return td::Status::Error(ErrorCode::protoviolation, "too old certificate"); + return BroadcastCheckResult::Forbidden; + } + if (is_fec && !(flags_ & CertificateFlags::AllowFec)) { + return BroadcastCheckResult::Forbidden; } - TRY_RESULT(E, issued_by_.get().create_encryptor()); + auto R1 = issued_by_.get().create_encryptor(); + if (R1.is_error()) { + return BroadcastCheckResult::Forbidden; + } + auto E = R1.move_as_ok(); auto B = to_sign(overlay_id, node); - TRY_STATUS(E->check_signature(B.as_slice(), signature_.as_slice())); + if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { + return BroadcastCheckResult::Forbidden; + } - return td::Status::OK(); + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } tl_object_ptr Certificate::tl() const { diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 47019c16c..21ec9363c 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -391,25 +391,21 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } -td::Status OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size) { +BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, + bool is_fec) { if (size == 0) { - return td::Status::Error(ErrorCode::protoviolation, "empty broadcast"); + return BroadcastCheckResult::Forbidden; } auto short_id = source.compute_short_id(); - auto r = rules_.max_size(source.compute_short_id()); - if (r >= size) { - return td::Status::OK(); - } - if (!cert) { - return td::Status::Error(ErrorCode::protoviolation, "source is not eligible"); + auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + if (!cert || r == BroadcastCheckResult::Allowed) { + return r; } - TRY_STATUS(cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size)); - auto issuer_short = cert->issuer_hash(); - if (rules_.max_size(issuer_short) < size) { - return td::Status::Error(ErrorCode::protoviolation, "bad certificate"); - } - return td::Status::OK(); + + auto r2 = cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); + return broadcast_check_result_max(r, r2); } td::Status OverlayImpl::check_delivered(BroadcastHash hash) { @@ -539,6 +535,25 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { rules_ = std::move(rules); } +void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { + callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); +} + +void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { + { + auto it = broadcasts_.find(hash); + if (it != broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } + { + auto it = fec_broadcasts_.find(hash); + if (it != fec_broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index b087fc754..cbddd6544 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -144,9 +144,12 @@ class OverlayImpl : public Overlay { void print(td::StringBuilder &sb) override; td::Status check_date(td::uint32 date); - td::Status check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size); + BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); td::Status check_delivered(BroadcastHash hash); + void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); + void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise); + BroadcastFec *get_fec_broadcast(BroadcastHash hash); void register_fec_broadcast(std::unique_ptr bcast); void register_simple_broadcast(std::unique_ptr bcast); diff --git a/overlay/overlays.h b/overlay/overlays.h index 90f3877ed..1ad554530 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -21,7 +21,11 @@ #include "adnl/adnl.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" #include @@ -80,41 +84,64 @@ class OverlayIdFull { td::BufferSlice name_; }; +struct CertificateFlags { + enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; +}; + +enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; + +inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::max(static_cast(l), static_cast(r))); +} +inline BroadcastCheckResult broadcast_check_result_min(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::min(static_cast(l), static_cast(r))); +} + class OverlayPrivacyRules { public: OverlayPrivacyRules() { } OverlayPrivacyRules(td::uint32 size) : max_unath_size_(size) { } - OverlayPrivacyRules(td::uint32 max_size, std::map authorized_keys) - : max_unath_size_(max_size), authorized_keys_(std::move(authorized_keys)) { + OverlayPrivacyRules(td::uint32 max_size, td::uint32 flags, std::map authorized_keys) + : max_unath_size_(max_size), flags_(flags), authorized_keys_(std::move(authorized_keys)) { } - td::uint32 max_size(PublicKeyHash hash) { + BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { - return max_unath_size_; + if (size > max_unath_size_) { + return BroadcastCheckResult::Forbidden; + } + if (!(flags_ & CertificateFlags::AllowFec) && is_fec) { + return BroadcastCheckResult::Forbidden; + } + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } else { - return it->second; + return it->second >= size ? BroadcastCheckResult::Allowed : BroadcastCheckResult::Forbidden; } } private: td::uint32 max_unath_size_{0}; + td::uint32 flags_{0}; std::map authorized_keys_; }; class Certificate { public: - Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); - Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); + Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); + Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); Certificate() { } void set_signature(td::BufferSlice signature); void set_issuer(PublicKey issuer); td::BufferSlice to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const; - td::Status check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size) const; + BroadcastCheckResult check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size, + bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; const PublicKeyHash issuer_hash() const; @@ -126,6 +153,7 @@ class Certificate { td::Variant issued_by_; td::int32 expire_at_; td::uint32 max_size_; + td::uint32 flags_; td::SharedSlice signature_; }; @@ -137,6 +165,10 @@ class Overlays : public td::actor::Actor { virtual void receive_query(adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) = 0; virtual void receive_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data) = 0; + virtual void check_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) { + promise.set_value(td::Unit()); + } virtual ~Callback() = default; }; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 57ed9ddcb..b226ca7f1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -219,9 +219,11 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; +overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; overlay.certificateId overlay_id:int256 node:int256 expire_at:int max_size:int = overlay.CertificateId; +overlay.certificateIdV2 overlay_id:int256 node:int256 expire_at:int max_size:int flags:int = overlay.CertificateId; overlay.unicast data:bytes = overlay.Broadcast; overlay.broadcast src:PublicKey certificate:overlay.Certificate flags:int data:bytes date:int signature:bytes = overlay.Broadcast; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 823219b5455c649d4cc8f04b6231dcdd439c8c16..32f42b3c4843343c874d1a1a2696af299ab58f1c 100644 GIT binary patch delta 298 zcmZ4SmAU07^9Bt`$=dJLX~OwssYN-7m3q#pMJ1VOnaPPIsmv1{#aZMw9V(t2BPlV_ zL1J@{WCa(a#b#EO_4%wIO$?J6Gu000^VMZXe5KCXq=$|a0 R!8(~YQyFa3=9ZZ%i2&9tb(jDE delta 162 zcmZqq$-Lq#^9Bt`iGSHHmxS}nQj2mDEA^aHi%K%nGLsWaQYSh{u promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } @@ -95,6 +101,17 @@ void FullNodeShardImpl::create_overlay() { } } +void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); + } + + auto q = B.move_as_ok(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, + std::move(q->message_->data_), std::move(promise)); +} + void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; @@ -804,8 +821,9 @@ void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { return; } - ton::overlay::Certificate cert{sign_by, static_cast(td::Clocks::system() + 3600), - overlay::Overlays::max_fec_broadcast_size(), td::BufferSlice{}}; + ton::overlay::Certificate cert{ + sign_by, static_cast(td::Clocks::system() + 3600), overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; auto to_sign = cert.to_sign(overlay_id_, local_id_); auto P = td::PromiseCreator::lambda( @@ -845,7 +863,7 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size()); } - rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)}; + rules_ = overlay::OverlayPrivacyRules{1 << 14, 0, std::move(authorized_keys)}; td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_); if (update_cert) { @@ -949,8 +967,7 @@ void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, do } } -void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, - td::BufferSlice data) { +void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data) { auto it = neighbours_.find(adnl_id); if (it == neighbours_.end()) { return; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 4aad637b1..2c4683747 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -19,6 +19,8 @@ #pragma once #include "full-node-shard.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" namespace ton { @@ -139,6 +141,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 71818e6e4..fa365ccf4 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -124,6 +124,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 506ee0367..9938bcb2b 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -144,6 +144,9 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 672085d03..08de5060a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -20,6 +20,8 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -325,6 +327,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + promise.set_value(td::Unit()); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; diff --git a/validator/validator.h b/validator/validator.h index 54c35ee76..0cbdbe00e 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -94,9 +94,8 @@ struct ValidatorManagerOptions : public td::CntObject { BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 300, - double block_ttl = 86400 * 7, double state_ttl = 3600, - double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + bool allow_blockchain_init = false, double sync_blocks_before = 300, double block_ttl = 86400 * 7, + double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, bool initial_sync_disabled = false); }; @@ -176,6 +175,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void write_handle(BlockHandle handle, td::Promise promise) = 0; virtual void new_external_message(td::BufferSlice data) = 0; + virtual void check_external_message(td::BufferSlice data, td::Promise promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; From df076491f13f999f769b5532ddc00ed7ddcb41d2 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 17:15:19 +0300 Subject: [PATCH 02/19] Add mempool messages cap --- validator-engine/validator-engine.cpp | 7 +++++++ validator-engine/validator-engine.hpp | 4 ++++ validator/manager.cpp | 3 +++ validator/manager.hpp | 3 +++ validator/validator-options.cpp | 3 ++- validator/validator-options.hpp | 10 +++++++++- validator/validator.h | 3 +++ 7 files changed, 31 insertions(+), 2 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 16f71d5d3..9c9d4a984 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1309,6 +1309,9 @@ td::Status ValidatorEngine::load_global_config() { if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } + if (max_mempool_num_ != 0) { + validator_options_.write().set_max_mempool_num(max_mempool_num_); + } if (block_ttl_ != 0) { validator_options_.write().set_block_ttl(block_ttl_); } @@ -3336,6 +3339,10 @@ int main(int argc, char *argv[]) { auto v = td::to_double(fname); acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); }); + p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { + auto v = td::to_double(fname); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); + }); p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 8ce56ab4a..07078f954 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -191,6 +191,7 @@ class ValidatorEngine : public td::actor::Actor { std::map control_permissions_; double state_ttl_ = 0; + double max_mempool_num_ = 0; double block_ttl_ = 0; double sync_ttl_ = 0; double archive_ttl_ = 0; @@ -223,6 +224,9 @@ class ValidatorEngine : public td::actor::Actor { void set_state_ttl(double t) { state_ttl_ = t; } + void set_max_mempool_num(double t) { + max_mempool_num_ = t; + } void set_block_ttl(double t) { block_ttl_ = t; } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3abea6e80..1741502d9 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -366,6 +366,9 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { if (!is_validator()) { return; } + if( ext_messages_.size() > max_mempool_num() ) { + return; + } auto R = create_ext_message(std::move(data)); if (R.is_error()) { VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 08de5060a..c2e41a30e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -580,6 +580,9 @@ class ValidatorManagerImpl : public ValidatorManager { double block_ttl() const { return opts_->block_ttl(); } + double max_mempool_num() const { + return opts_->max_mempool_num(); + } private: std::map> shard_client_waiters_; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 30bb050be..93fe05e6c 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -27,10 +27,11 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, + max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 92597156c..e794166f2 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -50,6 +50,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double state_ttl() const override { return state_ttl_; } + double max_mempool_num() const override { + return max_mempool_num_; + } double archive_ttl() const override { return archive_ttl_; } @@ -130,6 +133,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_state_ttl(double value) override { state_ttl_ = value; } + void set_max_mempool_num(double value) override { + max_mempool_num_ = value; + } void set_archive_ttl(double value) override { archive_ttl_ = value; } @@ -163,7 +169,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, + double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) @@ -173,6 +179,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) , state_ttl_(state_ttl) + , max_mempool_num_(max_mempool_num) , archive_ttl_(archive_ttl) , key_proof_ttl_(key_proof_ttl) , initial_sync_disabled_(initial_sync_disabled) { @@ -186,6 +193,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double sync_blocks_before_; double block_ttl_; double state_ttl_; + double max_mempool_num_; double archive_ttl_; double key_proof_ttl_; bool initial_sync_disabled_; diff --git a/validator/validator.h b/validator/validator.h index 0cbdbe00e..349824d67 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -56,6 +56,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; virtual double state_ttl() const = 0; + virtual double max_mempool_num() const = 0; virtual double archive_ttl() const = 0; virtual double key_proof_ttl() const = 0; virtual bool initial_sync_disabled() const = 0; @@ -81,6 +82,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; virtual void set_state_ttl(double value) = 0; + virtual void set_max_mempool_num(double value) = 0; virtual void set_archive_ttl(double value) = 0; virtual void set_key_proof_ttl(double value) = 0; virtual void set_initial_sync_disabled(bool value) = 0; @@ -96,6 +98,7 @@ struct ValidatorManagerOptions : public td::CntObject { ShardCheckMode) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 300, double block_ttl = 86400 * 7, double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; From f70b27d6d4b78bdc6689e9efab1f722e0b5167c9 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 23:28:06 +0300 Subject: [PATCH 03/19] Add stub for external message checks --- validator/fabric.h | 2 ++ validator/impl/external-message.cpp | 21 +++++++++++++++++++++ validator/impl/external-message.hpp | 9 +++++++++ validator/impl/fabric.cpp | 5 +++++ validator/manager.cpp | 4 ++++ validator/manager.hpp | 5 ++--- 6 files changed, 43 insertions(+), 3 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index a10251dd7..6ee0ea00a 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -46,6 +46,8 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 6af721b20..fb5c3d961 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -16,12 +16,14 @@ Copyright 2017-2020 Telegram Systems LLP */ + #include "external-message.hpp" #include "vm/boc.h" #include "block/block-parse.h" #include "block/block-auto.h" #include "block/block-db.h" + namespace ton { namespace validator { @@ -73,5 +75,24 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat return Ref{true, std::move(data), std::move(ext_msg), dest_prefix}; } +void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise) { + auto R = create_ext_message(std::move(data)); + if (R.is_error()) { + return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); + } + auto M = R.move_as_ok(); + auto root = M->root_cell(); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + tlb::unpack_cell_inexact(root, info); // checked in create message + ton::StdSmcAddress addr; + ton::WorkchainId wc; + if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { + return promise.set_error(td::Status::Error(PSLICE() << "Can't parse destination address")); + } + promise.set_value(td::Unit()); +} + + } // namespace validator } // namespace ton diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index 5bb52b11a..c22dc18cd 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -18,6 +18,8 @@ */ #pragma once + +#include "interfaces/validator-manager.h" #include "validator/interfaces/external-message.h" #include "auto/tl/ton_api.h" #include "adnl/utils.hpp" @@ -49,6 +51,13 @@ class ExtMessageQ : public ExtMessage { } ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard); static td::Result> create_ext_message(td::BufferSlice data); + static void run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise); + static void reject_message(/*td::Promise promise*/ td::Status reason); + static void continue_run_message_with_mc_state(td::actor::ActorId manager, + td::Ref mc_state, BlockIdExt blkid, + td::Promise promise, + ton::StdSmcAddress addr, ton::WorkchainId wc); }; } // namespace validator diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index ac296eb48..5e7ccd7c8 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -116,6 +116,11 @@ td::Result> create_ext_message(td::BufferSlice data) { return std::move(res); } +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) { + ExtMessageQ::run_message(std::move(data), std::move(manager), std::move(promise)); + //promise.set_value(td::Unit()); +} + td::Result> create_ihr_message(td::BufferSlice data) { TRY_RESULT(res, IhrMessageQ::create_ihr_message(std::move(data))); return std::move(res); diff --git a/validator/manager.cpp b/validator/manager.cpp index 1741502d9..3e5428296 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -382,6 +382,10 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { } } +void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise promise) { + run_check_external_message(std::move(data), actor_id(this), std::move(promise)); +} + void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { if (!is_validator()) { return; diff --git a/validator/manager.hpp b/validator/manager.hpp index c2e41a30e..d72ced9db 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -327,9 +327,8 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; - void check_external_message(td::BufferSlice data, td::Promise promise) override { - promise.set_value(td::Unit()); - } + void check_external_message(td::BufferSlice data, td::Promise promise) override; + void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; From dcfde952845867802a174b95011c9e6981a77ce3 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sun, 14 Nov 2021 00:10:32 +0300 Subject: [PATCH 04/19] Allow local liteserver directly inject external messages --- validator/impl/external-message.hpp | 6 ------ validator/manager.cpp | 12 ++++++++---- validator/manager.hpp | 1 + 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index c22dc18cd..e69abe917 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -18,7 +18,6 @@ */ #pragma once - #include "interfaces/validator-manager.h" #include "validator/interfaces/external-message.h" #include "auto/tl/ton_api.h" @@ -53,11 +52,6 @@ class ExtMessageQ : public ExtMessage { static td::Result> create_ext_message(td::BufferSlice data); static void run_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); - static void reject_message(/*td::Promise promise*/ td::Status reason); - static void continue_run_message_with_mc_state(td::actor::ActorId manager, - td::Ref mc_state, BlockIdExt blkid, - td::Promise promise, - ton::StdSmcAddress addr, ton::WorkchainId wc); }; } // namespace validator diff --git a/validator/manager.cpp b/validator/manager.cpp index 3e5428296..8f8db1299 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -374,14 +374,17 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); return; } - auto M = std::make_unique>(R.move_as_ok()); - auto id = M->ext_id(); + add_external_message(R.move_as_ok()); +} + +void ValidatorManagerImpl::add_external_message(td::Ref msg) { + auto message = std::make_unique>(msg); + auto id = message->ext_id(); if (ext_messages_hashes_.count(id.hash) == 0) { - ext_messages_.emplace(id, std::move(M)); + ext_messages_.emplace(id, std::move(message)); ext_messages_hashes_.emplace(id.hash, id); } } - void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise promise) { run_check_external_message(std::move(data), actor_id(this), std::move(promise)); } @@ -1356,6 +1359,7 @@ void ValidatorManagerImpl::send_get_next_key_blocks_request(BlockIdExt block_id, void ValidatorManagerImpl::send_external_message(td::Ref message) { callback_->send_ext_message(message->shard(), message->serialize()); + add_external_message(std::move(message)); } void ValidatorManagerImpl::send_ihr_message(td::Ref message) { diff --git a/validator/manager.hpp b/validator/manager.hpp index d72ced9db..ab3c701de 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -327,6 +327,7 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void add_external_message(td::Ref message); void check_external_message(td::BufferSlice data, td::Promise promise) override; void new_ihr_message(td::BufferSlice data) override; From 1ae8af60bcfae8f316418df95d75110d1f25a956 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Mon, 15 Nov 2021 18:27:44 +0300 Subject: [PATCH 05/19] Add checks for external messages --- validator/fabric.h | 2 + validator/impl/external-message.cpp | 133 +++++++++++++++++++++++++++- validator/impl/external-message.hpp | 6 ++ validator/impl/fabric.cpp | 6 +- validator/impl/liteserver.cpp | 46 +++++++++- validator/impl/liteserver.hpp | 12 +++ 6 files changed, 199 insertions(+), 6 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index 6ee0ea00a..67b6ae9ee 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -83,6 +83,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); +void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index fb5c3d961..84c4f9659 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -22,7 +22,10 @@ #include "block/block-parse.h" #include "block/block-auto.h" #include "block/block-db.h" - +#include "fabric.h" +#include "td/actor/actor.h" +#include "td/utils/Random.h" +#include "crypto/openssl/rand.hpp" namespace ton { @@ -76,7 +79,7 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat } void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId manager, - td::Promise promise) { + td::Promise promise) { auto R = create_ext_message(std::move(data)); if (R.is_error()) { return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); @@ -90,9 +93,133 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId,UnixTime,LogicalTime,std::unique_ptr>> res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); + } else { + auto tuple = res.move_as_ok(); + block::Account acc; + auto shard_acc = std::get<0>(tuple); + auto utime = std::get<1>(tuple); + auto lt = std::get<2>(tuple); + auto config = std::move(std::get<3>(tuple)); + if(!acc.unpack(shard_acc, {}, utime, false)) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); + } + if(run_message_on_account(wc, acc, utime, lt + 1, msg_root, std::move(config))) { + promise.set_value(td::Unit()); + } else { + promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted")); + } + } + } + ); } +bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc, + block::Account acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config) { + + std::vector storage_prices_; + td::BitArray<256> rand_seed_; + block::ComputePhaseConfig compute_phase_cfg_; + block::ActionPhaseConfig action_phase_cfg_; + { + auto res = config->get_storage_prices(); + if (res.is_error()) { + LOG(DEBUG) << "Can not unpack storage prices"; + return false; + } + storage_prices_ = res.move_as_ok(); + } + block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; + { + // generate rand seed + prng::rand_gen().strong_rand_bytes(rand_seed_.data(), 32); + LOG(DEBUG) << "block random seed set to " << rand_seed_.to_hex(); + } + { + // compute compute_phase_cfg / storage_phase_cfg + auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); + if (cell.is_null()) { + LOG(DEBUG) << "cannot fetch current gas prices and limits from masterchain configuration"; + return false; + } + if (!compute_phase_cfg_.parse_GasLimitsPrices(std::move(cell), storage_phase_cfg_.freeze_due_limit, + storage_phase_cfg_.delete_due_limit)) { + LOG(DEBUG) <<"cannot unpack current gas prices and limits from masterchain configuration"; + return false; + } + compute_phase_cfg_.block_rand_seed = rand_seed_; + compute_phase_cfg_.libraries = std::make_unique(config->get_libraries_root(), 256); + compute_phase_cfg_.global_config = config->get_root_cell(); + } + { + // compute action_phase_cfg + block::gen::MsgForwardPrices::Record rec; + auto cell = config->get_config_param(24); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + LOG(DEBUG) << "cannot fetch masterchain message transfer prices from masterchain configuration"; + return false; + } + action_phase_cfg_.fwd_mc = + block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, + (unsigned)rec.first_frac, (unsigned)rec.next_frac}; + cell = config->get_config_param(25); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + LOG(DEBUG) << "cannot fetch standard message transfer prices from masterchain configuration"; + return false; + } + action_phase_cfg_.fwd_std = + block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, + (unsigned)rec.first_frac, (unsigned)rec.next_frac}; + action_phase_cfg_.workchains = &config->get_workchain_list(); + action_phase_cfg_.bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0); + } + + std::unique_ptr trans = + std::make_unique(acc, block::Transaction::tr_ord, lt, utime, msg_root); + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { + // inbound external message was not accepted + LOG(DEBUG) << "inbound external message rejected by account" + << " before smart-contract execution"; + return false; + } + + if (!trans->prepare_storage_phase(storage_phase_cfg_, true, true)) { + LOG(DEBUG) << "cannot create storage phase of a new transaction for smart contract "; + return false; + } + if (!trans->prepare_compute_phase(compute_phase_cfg_)) { + LOG(DEBUG) << "cannot create compute phase of a new transaction for smart contract "; + return false; + } + if (!trans->compute_phase->accepted) { + // inbound external message was not accepted + LOG(DEBUG) << "inbound external message rejected by transaction "; + return false; + } + if (trans->compute_phase->success && !trans->prepare_action_phase(action_phase_cfg_)) { + LOG(DEBUG) << "cannot create action phase of a new transaction for smart contract "; + return false; + } + if (!trans->serialize()) { + LOG(DEBUG) << "cannot serialize new transaction for smart contract "; + return false; + } + auto trans_root = trans->commit(acc); + if (trans_root.is_null()) { + LOG(DEBUG) << "cannot commit new transaction for smart contract "; + return false; + } + return true; + +} } // namespace validator } // namespace ton diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index e69abe917..3c758e504 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -22,6 +22,7 @@ #include "validator/interfaces/external-message.h" #include "auto/tl/ton_api.h" #include "adnl/utils.hpp" +#include "block/transaction.h" namespace ton { @@ -52,6 +53,11 @@ class ExtMessageQ : public ExtMessage { static td::Result> create_ext_message(td::BufferSlice data); static void run_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + static bool run_message_on_account(ton::WorkchainId wc, + block::Account acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config); }; } // namespace validator diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 5e7ccd7c8..196595aed 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -118,7 +118,6 @@ td::Result> create_ext_message(td::BufferSlice data) { void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) { ExtMessageQ::run_message(std::move(data), std::move(manager), std::move(promise)); - //promise.set_value(td::Unit()); } td::Result> create_ihr_message(td::BufferSlice data) { @@ -242,6 +241,11 @@ void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + LiteQuery::fetch_account_state(wc, addr, std::move(manager), std::move(promise)); +} + void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index b3c0f3eb5..13536fb5e 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -58,17 +58,31 @@ void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId("litequery", std::move(data), std::move(manager), std::move(promise)).release(); } +void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + td::actor::create_actor("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release(); +} + LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) : query_(std::move(data)), manager_(std::move(manager)), promise_(std::move(promise)) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } +LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) + : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) { + timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); +} + void LiteQuery::abort_query(td::Status reason) { LOG(INFO) << "aborted liteserver query: " << reason.to_string(); if (promise_) { promise_.set_error(std::move(reason)); } + if (acc_state_promise_) { + acc_state_promise_.set_error(std::move(reason)); + } stop(); } @@ -111,6 +125,11 @@ bool LiteQuery::finish_query(td::BufferSlice result) { void LiteQuery::start_up() { alarm_timestamp() = timeout_; + if(acc_state_promise_) { + td::actor::send_closure_later(actor_id(this),&LiteQuery::perform_fetchAccountState); + return; + } + auto F = fetch_tl_object(std::move(query_), true); if (F.is_error()) { abort_query(F.move_as_error()); @@ -205,17 +224,23 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { } td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this), mode](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); - td::actor::send_closure_later(Self, &LiteQuery::continue_getMasterchainInfo, std::move(pair.first), + auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo; + td::actor::send_closure_later(Self, func, std::move(pair.first), pair.second, mode); } }); } +void LiteQuery::gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, + int mode) { + perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000); +} + void LiteQuery::continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode) { LOG(INFO) << "obtained data for getMasterchainInfo() : last block = " << blkid.to_str(); @@ -702,6 +727,10 @@ void LiteQuery::continue_getAccountState_0(Ref request_mc_block_data(blkid); } +void LiteQuery::perform_fetchAccountState() { + perform_getMasterchainInfo(-1); +} + void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params) { LOG(INFO) << "started a runSmcMethod(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", " @@ -1010,6 +1039,19 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { } vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); + if (mode_ & 0x80000000) { + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + if (config.is_error()) { + fatal_error(config.move_as_error()); + return; + } + auto rconfig = config.move_as_ok(); + acc_state_promise_.set_value(std::make_tuple( + std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) + )); + return; + } + Ref acc_root; if (acc_csr.not_null()) { acc_root = acc_csr->prefetch_ref(); diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index f6353c41d..243920855 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -26,6 +26,8 @@ #include "block.hpp" #include "shard.hpp" #include "proof.hpp" +#include "block/block-auto.h" + namespace ton { @@ -37,6 +39,9 @@ class LiteQuery : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise promise_; + + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; + int pending_{0}; int mode_{0}; WorkchainId acc_workchain_; @@ -71,9 +76,14 @@ class LiteQuery : public td::actor::Actor { }; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); static void run_query(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + private: bool fatal_error(td::Status error); bool fatal_error(std::string err_msg, int err_code = -400); @@ -87,6 +97,7 @@ class LiteQuery : public td::actor::Actor { void perform_getVersion(); void perform_getMasterchainInfo(int mode); void continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode); + void gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, int mode); void perform_getBlock(BlockIdExt blkid); void continue_getBlock(BlockIdExt blkid, Ref block); void perform_getBlockHeader(BlockIdExt blkid, int mode); @@ -99,6 +110,7 @@ class LiteQuery : public td::actor::Actor { void continue_getAccountState_0(Ref mc_state, BlockIdExt blkid); void continue_getAccountState(); void finish_getAccountState(td::BufferSlice shard_proof); + void perform_fetchAccountState(); void perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params); void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref acc_root, From d63fdcd9c94a7841c75ecc7bbd8e7b54c98b3094 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Wed, 17 Nov 2021 00:03:32 +0300 Subject: [PATCH 06/19] Move create_ordinary_transaction logic to static method --- validator/impl/collator-impl.h | 21 +++ validator/impl/collator.cpp | 214 ++++++++++++++---------- validator/impl/external-message.cpp | 141 +++++----------- validator/impl/external-message.hpp | 14 +- validator/impl/liteserver.cpp | 6 +- validator/interfaces/external-message.h | 2 + validator/manager.cpp | 17 +- validator/manager.hpp | 4 + 8 files changed, 221 insertions(+), 198 deletions(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 87aa54941..01c613684 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -103,6 +103,27 @@ class Collator final : public td::actor::Actor { return 2; } + static std::pair,td::Status> + impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc); + + static std::pair,td::Status> + impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + LogicalTime after_lt); + private: void start_up() override; void alarm() override; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 9ca4a6cc5..084066edc 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1556,68 +1556,92 @@ bool Collator::init_lt() { } bool Collator::fetch_config_params() { - old_mparams_ = config_->get_config_param(9); + auto tuple = impl_fetch_config_params(std::move(config_), + &old_mparams_, &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, + &basechain_create_fee_, &basechain_create_fee_, + workchain() + ); + config_ = std::move(tuple.first); + if (tuple.second.is_error()) { + return fatal_error(tuple.second.move_as_error()); + } + return true; +} + +std::pair,td::Status> + Collator::impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc) { + *old_mparams = config->get_config_param(9); { - auto res = config_->get_storage_prices(); + auto res = config->get_storage_prices(); if (res.is_error()) { - return fatal_error(res.move_as_error()); + return std::make_pair(std::move(config),res.move_as_error()); } - storage_prices_ = res.move_as_ok(); + *storage_prices = res.move_as_ok(); } { // generate rand seed - prng::rand_gen().strong_rand_bytes(rand_seed_.data(), 32); - LOG(DEBUG) << "block random seed set to " << rand_seed_.to_hex(); + prng::rand_gen().strong_rand_bytes(rand_seed->data(), 32); + LOG(DEBUG) << "block random seed set to " << rand_seed->to_hex(); } { // compute compute_phase_cfg / storage_phase_cfg - auto cell = config_->get_config_param(is_masterchain() ? 20 : 21); + auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); if (cell.is_null()) { - return fatal_error("cannot fetch current gas prices and limits from masterchain configuration"); + return std::make_pair(std::move(config),td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration")); } - if (!compute_phase_cfg_.parse_GasLimitsPrices(std::move(cell), storage_phase_cfg_.freeze_due_limit, - storage_phase_cfg_.delete_due_limit)) { - return fatal_error("cannot unpack current gas prices and limits from masterchain configuration"); + if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit, + storage_phase_cfg->delete_due_limit)) { + return std::make_pair(std::move(config),td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration")); } - compute_phase_cfg_.block_rand_seed = rand_seed_; - compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); - compute_phase_cfg_.global_config = config_->get_root_cell(); + compute_phase_cfg->block_rand_seed = *rand_seed; + compute_phase_cfg->libraries = std::make_unique(config->get_libraries_root(), 256); + compute_phase_cfg->global_config = config->get_root_cell(); } { // compute action_phase_cfg block::gen::MsgForwardPrices::Record rec; - auto cell = config_->get_config_param(24); + auto cell = config->get_config_param(24); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch masterchain message transfer prices from masterchain configuration"); + return std::make_pair(std::move(config),td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration")); } - action_phase_cfg_.fwd_mc = + action_phase_cfg->fwd_mc = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - cell = config_->get_config_param(25); + cell = config->get_config_param(25); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch standard message transfer prices from masterchain configuration"); + return std::make_pair(std::move(config),td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration")); } - action_phase_cfg_.fwd_std = + action_phase_cfg->fwd_std = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - action_phase_cfg_.workchains = &config_->get_workchain_list(); - action_phase_cfg_.bounce_msg_body = (config_->has_capability(ton::capBounceMsgBody) ? 256 : 0); + action_phase_cfg->workchains = &config->get_workchain_list(); + action_phase_cfg->bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0); } { // fetch block_grams_created - auto cell = config_->get_config_param(14); + auto cell = config->get_config_param(14); if (cell.is_null()) { - basechain_create_fee_ = masterchain_create_fee_ = td::zero_refint(); + *basechain_create_fee = *masterchain_create_fee = td::zero_refint(); } else { block::gen::BlockCreateFees::Record create_fees; if (!(tlb::unpack_cell(cell, create_fees) && - block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, masterchain_create_fee_) && - block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, basechain_create_fee_))) { - return fatal_error("cannot unpack BlockCreateFees from configuration parameter #14"); + block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) && + block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) { + return std::make_pair(std::move(config),td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14")); } } } - return true; + return std::make_pair(std::move(config),td::Status::OK()); } bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { @@ -2218,91 +2242,105 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { } block::Account* acc = acc_res.move_as_ok(); assert(acc); - if (acc->last_trans_end_lt_ >= start_lt && acc->transactions.empty()) { - fatal_error(PSTRING() << "last transaction time in the state of account " << workchain() << ":" << addr.to_hex() - << " is too large"); + + + auto res_tuple = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + external? last_proc_int_msg_.first : 0 + ); + if(res_tuple.second.is_error()) { + fatal_error(res_tuple.second.move_as_error()); + return {}; + } + std::unique_ptr trans = std::move(res_tuple.first); + + if (!trans->update_limits(*block_limit_status_)) { + fatal_error("cannot update block limit status to include the new transaction"); + return {}; + } + auto trans_root = trans->commit(*acc); + if (trans_root.is_null()) { + fatal_error("cannot commit new transaction for smart contract "s + addr.to_hex()); return {}; } - auto trans_min_lt = start_lt; + + register_new_msgs(*trans); + update_max_lt(acc->last_trans_end_lt_); + // temporary patch to stop producing dangerous block + if (acc->status == block::Account::acc_nonexist) { + block_full_ = true; + } + return trans_root; +} + +std::pair,td::Status> Collator::impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + LogicalTime after_lt) { + if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { + return std::make_pair(nullptr, + td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() + << " is too large")); + } + auto trans_min_lt = lt; + bool external = bool(after_lt); if (external) { // transactions processing external messages must have lt larger than all processed internal messages - trans_min_lt = std::max(trans_min_lt, last_proc_int_msg_.first); + trans_min_lt = std::max(trans_min_lt, after_lt); } + std::unique_ptr trans = - std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, now_, msg_root); + std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { if (external) { // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by account " << addr.to_hex() + LOG(DEBUG) << "inbound external message rejected by account " << acc->addr.to_hex() << " before smart-contract execution"; - return {}; - } - fatal_error("cannot unpack input message for a new transaction"); - return {}; + } + return std::make_pair(nullptr,td::Status::Error(-669,"cannot unpack input message for a new transaction")); } if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return std::make_pair(nullptr,td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex())); + } if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } + return std::make_pair(nullptr,td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex())); + } } else { if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->prepare_storage_phase(storage_phase_cfg_, true, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } + return std::make_pair(nullptr,td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex())); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return std::make_pair(nullptr,td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex())); + } } - if (!trans->prepare_compute_phase(compute_phase_cfg_)) { - fatal_error("cannot create compute phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { + return std::make_pair(nullptr,td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex())); } if (!trans->compute_phase->accepted) { if (external) { // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by transaction " << addr.to_hex(); - return {}; - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - fatal_error("new ordinary transaction for smart contract "s + addr.to_hex() + - " has not been accepted by the smart contract (?)"); - return {}; - } + LOG(DEBUG) << "inbound external message rejected by transaction " << acc->addr.to_hex(); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return std::make_pair(nullptr,td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)")); + } } - if (trans->compute_phase->success && !trans->prepare_action_phase(action_phase_cfg_)) { - fatal_error("cannot create action phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { + return std::make_pair(nullptr,td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex())); } - if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(action_phase_cfg_)) { - fatal_error("cannot create bounce phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; + if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(*action_phase_cfg)) { + return std::make_pair(nullptr,td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex())); } if (!trans->serialize()) { - fatal_error("cannot serialize new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->update_limits(*block_limit_status_)) { - fatal_error("cannot update block limit status to include the new transaction"); - return {}; + return std::make_pair(nullptr,td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + acc->addr.to_hex())); } - auto trans_root = trans->commit(*acc); - if (trans_root.is_null()) { - fatal_error("cannot commit new transaction for smart contract "s + addr.to_hex()); - return {}; - } - register_new_msgs(*trans); - update_max_lt(acc->last_trans_end_lt_); - // temporary patch to stop producing dangerous block - if (acc->status == block::Account::acc_nonexist) { - block_full_ = true; - } - return trans_root; + return std::make_pair(std::move(trans),td::Status::OK()); } void Collator::update_max_lt(ton::LogicalTime lt) { diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 84c4f9659..3c1a82e90 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -18,6 +18,7 @@ */ #include "external-message.hpp" +#include "collator-impl.h" #include "vm/boc.h" #include "block/block-parse.h" #include "block/block-auto.h" @@ -32,8 +33,8 @@ namespace ton { namespace validator { using td::Ref; -ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix) - : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)) { +ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix, ton::WorkchainId wc, ton::StdSmcAddress addr) + : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)), wc_(wc), addr_(addr) { hash_ = block::compute_file_hash(data_); } @@ -75,7 +76,13 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat if (!dest_prefix.is_valid()) { return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); } - return Ref{true, std::move(data), std::move(ext_msg), dest_prefix}; + ton::StdSmcAddress addr; + ton::WorkchainId wc; + if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { + return td::Status::Error(PSLICE() << "Can't parse destination address"); + } + + return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; } void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId manager, @@ -88,11 +95,8 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorIdroot_cell(); block::gen::CommonMsgInfo::Record_ext_in_msg_info info; tlb::unpack_cell_inexact(root, info); // checked in create message - ton::StdSmcAddress addr; - ton::WorkchainId wc; - if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { - return promise.set_error(td::Status::Error(PSLICE() << "Can't parse destination address")); - } + ton::StdSmcAddress addr = M->addr(); + ton::WorkchainId wc = M->wc(); run_fetch_account_state(wc, addr, manager, [promise = std::move(promise), msg_root = root, wc = wc](td::Result,UnixTime,LogicalTime,std::unique_ptr>> res) mutable { @@ -100,12 +104,12 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId(tuple); + block::Account *acc; + auto shard_acc = std::move(std::get<0>(tuple)); auto utime = std::get<1>(tuple); auto lt = std::get<2>(tuple); auto config = std::move(std::get<3>(tuple)); - if(!acc.unpack(shard_acc, {}, utime, false)) { + if(!acc->unpack(shard_acc, {}, utime, false)) { promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); } if(run_message_on_account(wc, acc, utime, lt + 1, msg_root, std::move(config))) { @@ -119,106 +123,41 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId msg_root, std::unique_ptr config) { + Ref old_mparams; std::vector storage_prices_; + block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; td::BitArray<256> rand_seed_; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; - { - auto res = config->get_storage_prices(); - if (res.is_error()) { - LOG(DEBUG) << "Can not unpack storage prices"; - return false; - } - storage_prices_ = res.move_as_ok(); - } - block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; - { - // generate rand seed - prng::rand_gen().strong_rand_bytes(rand_seed_.data(), 32); - LOG(DEBUG) << "block random seed set to " << rand_seed_.to_hex(); - } - { - // compute compute_phase_cfg / storage_phase_cfg - auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); - if (cell.is_null()) { - LOG(DEBUG) << "cannot fetch current gas prices and limits from masterchain configuration"; - return false; - } - if (!compute_phase_cfg_.parse_GasLimitsPrices(std::move(cell), storage_phase_cfg_.freeze_due_limit, - storage_phase_cfg_.delete_due_limit)) { - LOG(DEBUG) <<"cannot unpack current gas prices and limits from masterchain configuration"; - return false; - } - compute_phase_cfg_.block_rand_seed = rand_seed_; - compute_phase_cfg_.libraries = std::make_unique(config->get_libraries_root(), 256); - compute_phase_cfg_.global_config = config->get_root_cell(); - } - { - // compute action_phase_cfg - block::gen::MsgForwardPrices::Record rec; - auto cell = config->get_config_param(24); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - LOG(DEBUG) << "cannot fetch masterchain message transfer prices from masterchain configuration"; - return false; - } - action_phase_cfg_.fwd_mc = - block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, - (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - cell = config->get_config_param(25); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - LOG(DEBUG) << "cannot fetch standard message transfer prices from masterchain configuration"; - return false; - } - action_phase_cfg_.fwd_std = - block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, - (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - action_phase_cfg_.workchains = &config->get_workchain_list(); - action_phase_cfg_.bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0); - } - - std::unique_ptr trans = - std::make_unique(acc, block::Transaction::tr_ord, lt, utime, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by account" - << " before smart-contract execution"; - return false; - } - - if (!trans->prepare_storage_phase(storage_phase_cfg_, true, true)) { - LOG(DEBUG) << "cannot create storage phase of a new transaction for smart contract "; - return false; - } - if (!trans->prepare_compute_phase(compute_phase_cfg_)) { - LOG(DEBUG) << "cannot create compute phase of a new transaction for smart contract "; - return false; - } - if (!trans->compute_phase->accepted) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by transaction "; - return false; - } - if (trans->compute_phase->success && !trans->prepare_action_phase(action_phase_cfg_)) { - LOG(DEBUG) << "cannot create action phase of a new transaction for smart contract "; + td::RefInt256 masterchain_create_fee, basechain_create_fee; + + auto res = Collator::impl_fetch_config_params(std::move(config), &old_mparams, + &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &masterchain_create_fee, + &basechain_create_fee, wc); + auto res_tuple = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + lt); + if(res_tuple.second.is_error()) { + //LOG(DEBUG)? + //fatal_error(res_tuple.second.move_as_error()); return false; - } - if (!trans->serialize()) { - LOG(DEBUG) << "cannot serialize new transaction for smart contract "; - return false; - } - auto trans_root = trans->commit(acc); - if (trans_root.is_null()) { - LOG(DEBUG) << "cannot commit new transaction for smart contract "; - return false; - } - return true; + } + std::unique_ptr trans = std::move(res_tuple.first); + auto trans_root = trans->commit(*acc); + if (trans_root.is_null()) { + LOG(DEBUG) << "cannot commit new transaction for smart contract "; + return false; + } + return true; } } // namespace validator diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index 3c758e504..230258cd5 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -33,6 +33,8 @@ class ExtMessageQ : public ExtMessage { AccountIdPrefixFull addr_prefix_; td::BufferSlice data_; Hash hash_; + ton::WorkchainId wc_; + ton::StdSmcAddress addr_; public: static constexpr unsigned max_ext_msg_size = 65535; @@ -49,12 +51,20 @@ class ExtMessageQ : public ExtMessage { Hash hash() const override { return hash_; } - ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard); + ton::WorkchainId wc() const override { + return wc_; + } + + ton::StdSmcAddress addr() const override { + return addr_; + } + + ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard, ton::WorkchainId wc, ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data); static void run_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); static bool run_message_on_account(ton::WorkchainId wc, - block::Account acc, + block::Account* acc, UnixTime utime, LogicalTime lt, td::Ref msg_root, std::unique_ptr config); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 13536fb5e..0552f1d1e 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -77,12 +77,12 @@ LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId void LiteQuery::abort_query(td::Status reason) { LOG(INFO) << "aborted liteserver query: " << reason.to_string(); - if (promise_) { - promise_.set_error(std::move(reason)); - } if (acc_state_promise_) { acc_state_promise_.set_error(std::move(reason)); } + if (promise_) { + promise_.set_error(std::move(reason)); + } stop(); } diff --git a/validator/interfaces/external-message.h b/validator/interfaces/external-message.h index f6d9d600d..871c624a4 100644 --- a/validator/interfaces/external-message.h +++ b/validator/interfaces/external-message.h @@ -35,6 +35,8 @@ class ExtMessage : public td::CntObject { virtual td::BufferSlice serialize() const = 0; virtual td::Ref root_cell() const = 0; virtual Hash hash() const = 0; + virtual ton::WorkchainId wc() const = 0; + virtual ton::StdSmcAddress addr() const = 0; }; } // namespace validator diff --git a/validator/manager.cpp b/validator/manager.cpp index 8f8db1299..7fcf0274b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -380,9 +380,14 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { void ValidatorManagerImpl::add_external_message(td::Ref msg) { auto message = std::make_unique>(msg); auto id = message->ext_id(); - if (ext_messages_hashes_.count(id.hash) == 0) { - ext_messages_.emplace(id, std::move(message)); - ext_messages_hashes_.emplace(id.hash, id); + auto address = message->address(); + unsigned long per_address_limit = 256; + if(ext_addr_messages_.count(address) < per_address_limit) { + if (ext_messages_hashes_.count(id.hash) == 0) { + ext_messages_.emplace(id, std::move(message)); + ext_messages_hashes_.emplace(id.hash, id); + ext_addr_messages_[address].emplace(id.hash, id); + } } } void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise promise) { @@ -766,6 +771,7 @@ void ValidatorManagerImpl::get_external_messages(ShardIdFull shard, break; } if (it->second->expired()) { + ext_addr_messages_[it->second->address()].erase(it->first.hash); ext_messages_hashes_.erase(it->first.hash); it = ext_messages_.erase(it); continue; @@ -814,17 +820,20 @@ void ValidatorManagerImpl::complete_external_messages(std::vectorsecond]->address()].erase(it->first); CHECK(ext_messages_.erase(it->second)); ext_messages_hashes_.erase(it); } } + unsigned long soft_mempool_limit = 1024; for (auto &hash : to_delay) { auto it = ext_messages_hashes_.find(hash); if (it != ext_messages_hashes_.end()) { auto it2 = ext_messages_.find(it->second); - if (it2->second->can_postpone()) { + if ((ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) { it2->second->postpone(); } else { + ext_addr_messages_[it2->second->address()].erase(it2->first.hash); ext_messages_.erase(it2); ext_messages_hashes_.erase(it); } diff --git a/validator/manager.hpp b/validator/manager.hpp index ab3c701de..19ee4cdde 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -74,6 +74,9 @@ class MessageExt { auto hash() const { return message_->hash(); } + auto address() const { + return std::make_pair(message_->wc(), message_->addr()); + } bool is_active() { if (!active_) { if (reactivate_at_.is_in_past()) { @@ -210,6 +213,7 @@ class ValidatorManagerImpl : public ValidatorManager { // DATA FOR COLLATOR std::map> shard_blocks_; std::map, std::unique_ptr>> ext_messages_; + std::map, std::map>> ext_addr_messages_; std::map> ext_messages_hashes_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; From a9eafdb62fdd3daae6c3a7586b90dd3ab9eeb875 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Wed, 17 Nov 2021 10:02:46 +0300 Subject: [PATCH 07/19] Check external messages fron liteclients --- validator/impl/liteserver.cpp | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 0552f1d1e..312f07dab 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -456,15 +456,25 @@ void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) { void LiteQuery::perform_sendMessage(td::BufferSlice data) { LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query"; - auto res = ton::validator::create_ext_message(std::move(data)); - if (res.is_error()) { - abort_query(res.move_as_error()); - return; - } - LOG(INFO) << "sending an external message to validator manager"; - td::actor::send_closure_later(manager_, &ValidatorManager::send_external_message, res.move_as_ok()); - auto b = ton::create_serialize_tl_object(1); - finish_query(std::move(b)); + td::actor::send_closure_later( + manager_, &ValidatorManager::check_external_message, data.clone(), + [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result res) { + if(res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot apply external message to current state : "s)); + } else { + auto crm = ton::validator::create_ext_message(data.clone()); + if (crm.is_error()) { + //UNREACHABLE, checks in check_external_message, + td::actor::send_closure(Self, &LiteQuery::abort_query, + crm.move_as_error()); + } + LOG(INFO) << "sending an external message to validator manager"; + td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, crm.move_as_ok()); + auto b = ton::create_serialize_tl_object(1); + td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b)); + } + }); } bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { From 1cd58ed442f06d001d4cc2af833ad4e098ab007e Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Wed, 17 Nov 2021 19:50:24 +0300 Subject: [PATCH 08/19] Fix various bugs --- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 27 +++++++++++++++++---------- validator/impl/external-message.cpp | 10 +++++----- validator/impl/liteserver.cpp | 3 +-- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 01c613684..f84490bc3 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -122,7 +122,7 @@ class Collator final : public td::actor::Actor { block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, - LogicalTime after_lt); + bool external, LogicalTime after_lt); private: void start_up() override; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 084066edc..a9d297443 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1559,7 +1559,7 @@ bool Collator::fetch_config_params() { auto tuple = impl_fetch_config_params(std::move(config_), &old_mparams_, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, - &basechain_create_fee_, &basechain_create_fee_, + &masterchain_create_fee_, &basechain_create_fee_, workchain() ); config_ = std::move(tuple.first); @@ -2247,10 +2247,16 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { auto res_tuple = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, &action_phase_cfg_, - external? last_proc_int_msg_.first : 0 + external, last_proc_int_msg_.first ); if(res_tuple.second.is_error()) { - fatal_error(res_tuple.second.move_as_error()); + auto error = res_tuple.second.move_as_error(); + if(error.code() == -701) { + // ignorable errors + LOG(DEBUG) << error.message(); + return {}; + } + fatal_error(std::move(error)); return {}; } std::unique_ptr trans = std::move(res_tuple.first); @@ -2274,33 +2280,34 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { return trans_root; } +// If td::status::error_code == 669 - Fatal Error block can not be produced +// if td::status::error_code == 701 - Transaction can not be included into block, but it's ok (external or too early internal) std::pair,td::Status> Collator::impl_create_ordinary_transaction(Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, - LogicalTime after_lt) { + bool external, LogicalTime after_lt) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return std::make_pair(nullptr, td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() << " is too large")); } auto trans_min_lt = lt; - bool external = bool(after_lt); if (external) { // transactions processing external messages must have lt larger than all processed internal messages trans_min_lt = std::max(trans_min_lt, after_lt); } - + std::unique_ptr trans = std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); bool ihr_delivered = false; // FIXME if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { if (external) { // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by account " << acc->addr.to_hex() - << " before smart-contract execution"; + return std::make_pair(nullptr,td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution")); } return std::make_pair(nullptr,td::Status::Error(-669,"cannot unpack input message for a new transaction")); } @@ -2325,9 +2332,9 @@ std::pair,td::Status> Collator::impl_create_ if (!trans->compute_phase->accepted) { if (external) { // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by transaction " << acc->addr.to_hex(); + return std::make_pair(nullptr,td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex())); } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - return std::make_pair(nullptr,td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + + return std::make_pair(nullptr,td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + " has not been accepted by the smart contract (?)")); } } diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 3c1a82e90..496794f31 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -104,15 +104,15 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId(tuple)); auto utime = std::get<1>(tuple); auto lt = std::get<2>(tuple); auto config = std::move(std::get<3>(tuple)); - if(!acc->unpack(shard_acc, {}, utime, false)) { + if(!acc.unpack(shard_acc, {}, utime, false)) { promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); } - if(run_message_on_account(wc, acc, utime, lt + 1, msg_root, std::move(config))) { + if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) { promise.set_value(td::Unit()); } else { promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted")); @@ -144,9 +144,9 @@ bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc, auto res_tuple = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, &action_phase_cfg_, - lt); + true, lt); if(res_tuple.second.is_error()) { - //LOG(DEBUG)? + LOG(DEBUG) << "Cannot run message on account" << res_tuple.second.message(); //fatal_error(res_tuple.second.move_as_error()); return false; } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 312f07dab..510f49061 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -79,8 +79,7 @@ void LiteQuery::abort_query(td::Status reason) { LOG(INFO) << "aborted liteserver query: " << reason.to_string(); if (acc_state_promise_) { acc_state_promise_.set_error(std::move(reason)); - } - if (promise_) { + } else if (promise_) { promise_.set_error(std::move(reason)); } stop(); From 8dd936b52fbd6644ce4842c4bc8632da2c7865d1 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Thu, 18 Nov 2021 17:18:18 +0300 Subject: [PATCH 09/19] Use td::Result instead pair --- validator/impl/collator-impl.h | 4 +- validator/impl/collator.cpp | 69 ++++++++++++++--------------- validator/impl/external-message.cpp | 18 +++++--- 3 files changed, 48 insertions(+), 43 deletions(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index f84490bc3..2dced7495 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -103,7 +103,7 @@ class Collator final : public td::actor::Actor { return 2; } - static std::pair,td::Status> + static td::Result> impl_fetch_config_params(std::unique_ptr config, Ref* old_mparams, std::vector* storage_prices, @@ -115,7 +115,7 @@ class Collator final : public td::actor::Actor { td::RefInt256* basechain_create_fee, WorkchainId wc); - static std::pair,td::Status> + static td::Result> impl_create_ordinary_transaction(Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index a9d297443..6b1c72a6a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1556,20 +1556,20 @@ bool Collator::init_lt() { } bool Collator::fetch_config_params() { - auto tuple = impl_fetch_config_params(std::move(config_), + auto res = impl_fetch_config_params(std::move(config_), &old_mparams_, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, &masterchain_create_fee_, &basechain_create_fee_, workchain() ); - config_ = std::move(tuple.first); - if (tuple.second.is_error()) { - return fatal_error(tuple.second.move_as_error()); + if (res.is_error()) { + return fatal_error(res.move_as_error()); } + config_ = res.move_as_ok(); return true; } -std::pair,td::Status> +td::Result> Collator::impl_fetch_config_params(std::unique_ptr config, Ref* old_mparams, std::vector* storage_prices, @@ -1584,7 +1584,7 @@ std::pair,td::Status> { auto res = config->get_storage_prices(); if (res.is_error()) { - return std::make_pair(std::move(config),res.move_as_error()); + return res.move_as_error(); } *storage_prices = res.move_as_ok(); } @@ -1597,11 +1597,11 @@ std::pair,td::Status> // compute compute_phase_cfg / storage_phase_cfg auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); if (cell.is_null()) { - return std::make_pair(std::move(config),td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration")); + return td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration"); } if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit, storage_phase_cfg->delete_due_limit)) { - return std::make_pair(std::move(config),td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration")); + return td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration"); } compute_phase_cfg->block_rand_seed = *rand_seed; compute_phase_cfg->libraries = std::make_unique(config->get_libraries_root(), 256); @@ -1612,14 +1612,14 @@ std::pair,td::Status> block::gen::MsgForwardPrices::Record rec; auto cell = config->get_config_param(24); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return std::make_pair(std::move(config),td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration")); + return td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration"); } action_phase_cfg->fwd_mc = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; cell = config->get_config_param(25); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return std::make_pair(std::move(config),td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration")); + return td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration"); } action_phase_cfg->fwd_std = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, @@ -1637,11 +1637,11 @@ std::pair,td::Status> if (!(tlb::unpack_cell(cell, create_fees) && block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) && block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) { - return std::make_pair(std::move(config),td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14")); + return td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14"); } } } - return std::make_pair(std::move(config),td::Status::OK()); + return std::move(config); } bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { @@ -2244,13 +2244,13 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { assert(acc); - auto res_tuple = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, + auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, &action_phase_cfg_, external, last_proc_int_msg_.first ); - if(res_tuple.second.is_error()) { - auto error = res_tuple.second.move_as_error(); + if(res.is_error()) { + auto error = res.move_as_error(); if(error.code() == -701) { // ignorable errors LOG(DEBUG) << error.message(); @@ -2259,7 +2259,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { fatal_error(std::move(error)); return {}; } - std::unique_ptr trans = std::move(res_tuple.first); + std::unique_ptr trans = res.move_as_ok(); if (!trans->update_limits(*block_limit_status_)) { fatal_error("cannot update block limit status to include the new transaction"); @@ -2282,7 +2282,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { // If td::status::error_code == 669 - Fatal Error block can not be produced // if td::status::error_code == 701 - Transaction can not be included into block, but it's ok (external or too early internal) -std::pair,td::Status> Collator::impl_create_ordinary_transaction(Ref msg_root, +td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, block::StoragePhaseConfig* storage_phase_cfg, @@ -2290,9 +2290,8 @@ std::pair,td::Status> Collator::impl_create_ block::ActionPhaseConfig* action_phase_cfg, bool external, LogicalTime after_lt) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { - return std::make_pair(nullptr, - td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() - << " is too large")); + return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() + << " is too large"); } auto trans_min_lt = lt; if (external) { @@ -2306,48 +2305,48 @@ std::pair,td::Status> Collator::impl_create_ if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { if (external) { // inbound external message was not accepted - return std::make_pair(nullptr,td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + - " before smart-contract execution")); + return td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); } - return std::make_pair(nullptr,td::Status::Error(-669,"cannot unpack input message for a new transaction")); + return td::Status::Error(-669,"cannot unpack input message for a new transaction"); } if (trans->bounce_enabled) { if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (!external && !trans->prepare_credit_phase()) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); } } else { if (!external && !trans->prepare_credit_phase()) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); } } if (!trans->prepare_compute_phase(*compute_phase_cfg)) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (!trans->compute_phase->accepted) { if (external) { // inbound external message was not accepted - return std::make_pair(nullptr,td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex())); + return td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex()); } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - return std::make_pair(nullptr,td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + - " has not been accepted by the smart contract (?)")); + return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); } } if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(*action_phase_cfg)) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } if (!trans->serialize()) { - return std::make_pair(nullptr,td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + acc->addr.to_hex())); + return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } - return std::make_pair(std::move(trans),td::Status::OK()); + return std::move(trans); } void Collator::update_max_lt(ton::LogicalTime lt) { diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 496794f31..3e0be15b9 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -136,21 +136,27 @@ bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc, block::ActionPhaseConfig action_phase_cfg_; td::RefInt256 masterchain_create_fee, basechain_create_fee; - auto res = Collator::impl_fetch_config_params(std::move(config), &old_mparams, + auto fetch_res = Collator::impl_fetch_config_params(std::move(config), &old_mparams, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, &masterchain_create_fee, &basechain_create_fee, wc); - auto res_tuple = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, + if(fetch_res.is_error()) { + auto error = fetch_res.move_as_error(); + LOG(DEBUG) << "Cannot fetch config params" << error.message(); + return false; + } + + auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, &action_phase_cfg_, true, lt); - if(res_tuple.second.is_error()) { - LOG(DEBUG) << "Cannot run message on account" << res_tuple.second.message(); - //fatal_error(res_tuple.second.move_as_error()); + if(res.is_error()) { + auto error = res.move_as_error(); + LOG(DEBUG) << "Cannot run message on account" << error.message(); return false; } - std::unique_ptr trans = std::move(res_tuple.first); + std::unique_ptr trans = res.move_as_ok(); auto trans_root = trans->commit(*acc); if (trans_root.is_null()) { From 4c1ccdee9231b2f003de6203fb3975114697666b Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Fri, 19 Nov 2021 15:16:03 +0300 Subject: [PATCH 10/19] Stop checking external message if account is empty --- validator/impl/external-message.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 3e0be15b9..c5f2d0974 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -111,11 +111,12 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId(tuple)); if(!acc.unpack(shard_acc, {}, utime, false)) { promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); - } - if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) { - promise.set_value(td::Unit()); } else { - promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted")); + if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) { + promise.set_value(td::Unit()); + } else { + promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted")); + } } } } From 33ebdf396ebc6c26d51802cbb0af70a509c5fb88 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Tue, 30 Nov 2021 15:11:14 +0300 Subject: [PATCH 11/19] Add methods to sign and import certificates --- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 64644 -> 64852 bytes .../validator-engine-console-query.cpp | 104 ++++++++++++++++++ .../validator-engine-console-query.h | 62 +++++++++++ .../validator-engine-console.cpp | 2 + validator-engine/validator-engine.cpp | 41 +++++++ validator-engine/validator-engine.hpp | 2 + 7 files changed, 213 insertions(+) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index b226ca7f1..95c7a4fdf 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -663,6 +663,8 @@ engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; engine.validator.controlQuery data:bytes = Object; +engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 32f42b3c4843343c874d1a1a2696af299ab58f1c..0febe41583dde313e36f1989237f8d6f7c3a4a71 100644 GIT binary patch delta 115 zcmZqq$$aG(^9GR%j8U6KFI0=_tPcr!sg#dhV=hww delta 21 dcmccei@D_|^9GR%jFFo~FI0 @@ -720,3 +721,106 @@ td::Status CheckDhtServersQuery::receive(td::BufferSlice data) { } return td::Status::OK(); } + +td::Status SignCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(signer_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status SignCertificateQuery::send() { + auto cid = ton::create_serialize_tl_object(overlay_, id_, expire_at_, max_size_); + auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); + auto pub = ton::create_serialize_tl_object(signer_.tl()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); + return td::Status::OK(); +} + +void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if (f.is_error()) { + handle_error(f.move_as_error_prefix("Failed to get pubkey: ")); + } + pubkey_ = f.move_as_ok(); + has_pubkey_ = true; + if(has_signature_) { + save_certificate(); + } +} + + +td::Status SignCertificateQuery::receive(td::BufferSlice data) { + UNREACHABLE(); +} + +void SignCertificateQuery::receive_signature(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if(f.is_error()){ + handle_error(f.move_as_error_prefix("Failed to get signature: ")); + } + signature_ = std::move(f.move_as_ok()->signature_); + if(has_pubkey_) { + save_certificate(); + } +} + +void SignCertificateQuery::save_certificate() { + auto c = ton::create_serialize_tl_object( + std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto w = td::write_file(out_file_, c.as_slice()); + if(w.is_error()) { + handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); + } + td::TerminalIO::out() << "saved certificate\n"; + stop(); +} + +td::Status ImportCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(kh_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status ImportCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object( + overlay_, + ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), + std::move(cert) + ); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + + +td::Status ImportCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} + diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index e010d76fe..42d7db484 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -903,3 +903,65 @@ class CheckDhtServersQuery : public Query { private: ton::PublicKeyHash id_; }; + +class SignCertificateQuery : public Query { + public: + SignCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signcert"; + } + static std::string get_help() { + return "signcert \tsign overlay certificate by key"; + } + std::string name() const override { + return get_name(); + } + + void receive_pubkey(td::BufferSlice R); + void receive_signature(td::BufferSlice R); + + + private: + void save_certificate(); + + td::Bits256 overlay_; + td::Bits256 id_; + td::int32 expire_at_; + td::uint32 max_size_; + std::string out_file_; + ton::PublicKeyHash signer_; + td::BufferSlice signature_; + std::unique_ptr pubkey_; + bool has_signature_{0}; + bool has_pubkey_{0}; +}; + +class ImportCertificateQuery : public Query { + public: + ImportCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importcert"; + } + static std::string get_help() { + return "importcert \timport overlay certificate for specific key"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 overlay_; + td::Bits256 id_; + ton::PublicKeyHash kh_; + std::string in_file_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 43a02dc61..3f58cb7de 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -134,6 +134,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 9c9d4a984..ccbdaabe8 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3143,6 +3143,47 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_createCom .release(); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + //TODO force Overlays::update_certificate to return result + /*auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + */ + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.key_->key_hash_}, + r.move_as_ok()); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true) + ); +} + + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 07078f954..2108a0271 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -395,6 +395,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_createComplaintVote &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { From bce61bc499ea50917ee3c6d5393faaf325f514ed Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sun, 5 Dec 2021 12:06:53 +0300 Subject: [PATCH 12/19] Add signShardOverlayCertificate and importShardOverlayCertificate commands --- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 64852 -> 65348 bytes .../validator-engine-console-query.cpp | 60 ++++++++++++++++++ .../validator-engine-console-query.h | 56 ++++++++++++++++ validator-engine/validator-engine.cpp | 54 +++++++++++++++- validator-engine/validator-engine.hpp | 5 ++ validator/full-node-shard.cpp | 32 ++++++++++ validator/full-node-shard.h | 4 ++ validator/full-node-shard.hpp | 4 ++ validator/full-node.cpp | 20 ++++++ validator/full-node.h | 7 ++ validator/full-node.hpp | 8 +++ 12 files changed, 252 insertions(+), 2 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 95c7a4fdf..fe2726ac4 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -663,7 +663,9 @@ engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; engine.validator.controlQuery data:bytes = Object; -engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; +engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 0febe41583dde313e36f1989237f8d6f7c3a4a71..890e0187bc78072cc0c2318eb89a9293f4122733 100644 GIT binary patch delta 205 zcmccei}}bu<_#hj7-KezUZ@sjIe(zZX7ZG$H9TC!ndy0{De>8H;5sa& zmztNJnU|_pmY9>7l30>oq?egnkY7|1oRL_R;$N0pl#^IFdCgOq$q`8|=qC6~KJn~> zYM9OR7#$2#Kz1RSF}ZQ3_~e##K9b0eU;qLR+qQ##OOG95m>gTHJy~If%H#$Oj>!cx L1UK(|K3fz3NV-!+ delta 47 zcmV+~0MP%${sYwd1F#g(0a~*a(0dgHuT)g&B9m?A7n7{zWRo~E43i}0tdp?k@Uy7t FoE0ap7yJMK diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 77915eeb6..901009a67 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -759,6 +759,7 @@ void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { auto f = ton::fetch_tl_object(R.as_slice(), true); if (f.is_error()) { handle_error(f.move_as_error_prefix("Failed to get pubkey: ")); + return; } pubkey_ = f.move_as_ok(); has_pubkey_ = true; @@ -776,6 +777,7 @@ void SignCertificateQuery::receive_signature(td::BufferSlice R) { auto f = ton::fetch_tl_object(R.as_slice(), true); if(f.is_error()){ handle_error(f.move_as_error_prefix("Failed to get signature: ")); + return; } signature_ = std::move(f.move_as_ok()->signature_); if(has_pubkey_) { @@ -789,6 +791,7 @@ void SignCertificateQuery::save_certificate() { auto w = td::write_file(out_file_, c.as_slice()); if(w.is_error()) { handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); + return; } td::TerminalIO::out() << "saved certificate\n"; stop(); @@ -824,3 +827,60 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { return td::Status::OK(); } + +td::Status SignShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::send() { + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect cert: "); + auto w = td::write_file(out_file_, data.as_slice()); + if(w.is_error()) { + return w.move_as_error_prefix("Failed to write certificate to file: "); + } + td::TerminalIO::out() << "saved certificate\n"; + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} + diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 42d7db484..bfd6b3e37 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -965,3 +965,59 @@ class ImportCertificateQuery : public Query { ton::PublicKeyHash kh_; std::string in_file_; }; + +class SignShardOverlayCertificateQuery : public Query { + public: + SignShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signshardoverlaycert"; + } + static std::string get_help() { + return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + td::int32 expire_at_; + ton::PublicKeyHash key_; + td::uint32 max_size_; + std::string out_file_; +}; + + +class ImportShardOverlayCertificateQuery : public Query { + public: + ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importshardoverlaycert"; + } + static std::string get_help() { + return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + ton::PublicKeyHash key_; + std::string in_file_; +}; + diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index ccbdaabe8..091f62213 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3176,13 +3176,65 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, ton::overlay::OverlayIdShort{query.overlay_id_}, - ton::PublicKeyHash{query.key_->key_hash_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok()); promise.set_value( ton::serialize_tl_object(ton::create_tl_object(), true) ); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(promise)); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 2108a0271..6466c1841 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -397,6 +397,11 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 769ef8f39..1dac76fcd 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -848,6 +848,38 @@ void FullNodeShardImpl::signed_new_certificate(ton::overlay::Certificate cert) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } +void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expire_at, td::uint32 max_size, td::Promise promise) { + auto sign_by = sign_cert_by_; + if (sign_by.is_zero()) { + promise.set_error(td::Status::Error("Node has no key with signing authority")); + return; + } + + ton::overlay::Certificate cert{ + sign_by, static_cast(expire_at), max_size, + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; + auto to_sign = cert.to_sign(overlay_id_, signed_key); + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), expire_at = expire_at, max_size = max_size, promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to create certificate: failed to sign: ")); + } else { + auto p = R.move_as_ok(); + auto c = ton::create_serialize_tl_object(p.second.tl(), static_cast(expire_at), max_size, std::move(p.first)); + promise.set_value(std::move(c)); + } + }); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, sign_by, std::move(to_sign), + std::move(P)); +} + +void FullNodeShardImpl::import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::update_certificate, + adnl_id_, overlay_id_, signed_key, cert); + promise.set_value( td::Unit() ); +} + void FullNodeShardImpl::update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) { if (!client_.empty()) { return; diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 40a422e09..c1712baf2 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -42,6 +42,10 @@ class FullNodeShard : public td::actor::Actor { virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; + virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; + + virtual void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 2c4683747..a2dd5cc46 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -170,6 +170,10 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; + + void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; + void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; + void sign_new_certificate(PublicKeyHash sign_by); void signed_new_certificate(ton::overlay::Certificate cert); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 691a8b362..7a5f0e38d 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -76,6 +76,26 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr promise.set_value(td::Unit()); } +void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + } + td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); +} + +void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + } + td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); +} + void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { adnl_id_ = adnl_id; diff --git a/validator/full-node.h b/validator/full-node.h index d2c2d58dd..cdf39d6fa 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -53,6 +53,13 @@ class FullNode : public td::actor::Actor { virtual void add_permanent_key(PublicKeyHash key, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; + virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) = 0; + virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; static constexpr td::uint32 max_block_size() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index e9b64f573..6d57f4a82 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -42,6 +42,14 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) override; + void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) override; + + void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; void add_shard(ShardIdFull shard); From b0d16ce282bb74a162b57d4d563b7d8525e3643b Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sun, 5 Dec 2021 13:10:32 +0300 Subject: [PATCH 13/19] Add getOverlayStats command --- overlay/overlay-manager.cpp | 51 ++++++++++++++ overlay/overlay-manager.h | 1 + overlay/overlay.cpp | 15 +++++ overlay/overlay.h | 2 +- overlay/overlay.hpp | 2 + overlay/overlays.h | 1 + tdutils/td/utils/DecTree.h | 20 ++++++ tl/generate/scheme/ton_api.tl | 7 ++ tl/generate/scheme/ton_api.tlo | Bin 65348 -> 65956 bytes .../validator-engine-console-query.cpp | 31 ++++++++- .../validator-engine-console-query.h | 20 +++++- .../validator-engine-console.cpp | 4 +- validator-engine/validator-engine.cpp | 63 ++++++++++++++---- validator-engine/validator-engine.hpp | 4 +- 14 files changed, 203 insertions(+), 18 deletions(-) diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 059ad4e21..1d1336734 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -21,13 +21,18 @@ #include "overlay.h" #include "adnl/utils.hpp" +#include "td/actor/actor.h" +#include "td/actor/common.h" #include "td/utils/Random.h" #include "td/db/RocksDb.h" +#include "td/utils/Status.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/Poll.h" +#include namespace ton { @@ -269,6 +274,52 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o db_.set(key, create_serialize_tl_object(std::move(obj))); } +void OverlayManager::get_stats(td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + Cb(td::Promise> promise) : promise_(std::move(promise)) { + } + void incr_pending() { + pending_++; + } + void decr_pending() { + if (!--pending_) { + promise_.set_result(create_tl_object(std::move(res_))); + stop(); + } + } + void receive_answer(tl_object_ptr res) { + if (res) { + res_.push_back(std::move(res)); + } + decr_pending(); + } + + private: + std::vector> res_; + size_t pending_{1}; + td::Promise> promise_; + }; + + auto act = td::actor::create_actor("overlaysstatsmerger", std::move(promise)).release(); + + for (auto &a : overlays_) { + for (auto &b : a.second) { + td::actor::send_closure(act, &Cb::incr_pending); + td::actor::send_closure(b.second, &Overlay::get_stats, + [act](td::Result> R) { + if (R.is_ok()) { + td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); + } else { + td::actor::send_closure(act, &Cb::receive_answer, nullptr); + } + }); + } + } + + td::actor::send_closure(act, &Cb::decr_pending); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 574d0f914..c6afefaff 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -91,6 +91,7 @@ class OverlayManager : public Overlays { void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::actor::ActorOwn overlay); + void get_stats(td::Promise> promise) override; struct PrintId {}; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 21ec9363c..ec13fcd3e 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "auto/tl/ton_api.h" #include "td/utils/Random.h" #include "adnl/utils.hpp" @@ -25,6 +26,7 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/StringBuilder.h" namespace ton { @@ -554,6 +556,19 @@ void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result> promise) { + auto res = create_tl_object(); + res->adnl_id_ = local_id_.bits256_value(); + res->overlay_id_ = overlay_id_.bits256_value(); + res->overlay_id_full_ = id_full_.pubkey().tl(); + peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { res->nodes_.push_back(key.tl()); }); + + res->stats_.push_back( + create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + + promise.set_value(std::move(res)); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.h b/overlay/overlay.h index cba151453..cce24ed81 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -63,6 +63,7 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void get_stats(td::Promise> promise) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; }; @@ -70,4 +71,3 @@ class Overlay : public td::actor::Actor { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index cbddd6544..76d17aabe 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -190,6 +190,8 @@ class OverlayImpl : public Overlay { std::shared_ptr get_certificate(PublicKeyHash local_id); td::Result get_encryptor(PublicKey source); + void get_stats(td::Promise> promise) override; + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { diff --git a/overlay/overlays.h b/overlay/overlays.h index 1ad554530..aba83ab13 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -231,6 +231,7 @@ class Overlays : public td::actor::Actor { virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; + virtual void get_stats(td::Promise> promise) = 0; }; } // namespace overlay diff --git a/tdutils/td/utils/DecTree.h b/tdutils/td/utils/DecTree.h index 80bd0e21c..1ebb6cd63 100644 --- a/tdutils/td/utils/DecTree.h +++ b/tdutils/td/utils/DecTree.h @@ -46,6 +46,17 @@ class DecTree { } } + template + void iterate(const FuncT &cb) { + if (left_) { + left_->iterate(cb); + } + cb(key_, value_); + if (right_) { + right_->iterate(cb); + } + } + Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) { } }; @@ -223,6 +234,15 @@ class DecTree { bool exists(const KeyType &key) const { return get_node(root_, key) != nullptr; } + + template + void iterate(const FuncT &cb) { + if (size() == 0) { + return; + } else { + root_->iterate(cb); + } + } }; } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index fe2726ac4..0be70540c 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -620,6 +620,11 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 nodes:(vector adnl.id.short) + stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; + + ---functions--- engine.validator.getTime = engine.validator.Time; @@ -661,6 +666,8 @@ engine.validator.createComplaintVote election_id:int vote:bytes = engine.validat engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; +engine.validator.getOverlaysStats = engine.validator.OverlaysStats; + engine.validator.controlQuery data:bytes = Object; engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 890e0187bc78072cc0c2318eb89a9293f4122733..dbdd73b3db01254391b73828527be4872531d40a 100644 GIT binary patch delta 378 zcmX@|k9kQm3-6=Z`c@23aAYIzd{M>pU-D~YQ}fa@^HTN75_2+B5=-)n^!&?Gi*gbx zgG&-iCMHTN*Q;^Pk;A2^7^JFrvOkB^=Cz{Xe2i9`SykRAYhX7xA7OAY0|Q9Se3?MZ(j|usJOp$(eUy}nWJ-HxIK>}nT*m$61ku#ePhi%(IkcQ1K+I2h_V>gRl j_$`c@23aA+g%e9_G+ViA0d7Mp8T-Y0M7=u+@tjM=Pt>9fe@jOR8| E0ML>WVE_OC diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 901009a67..4b5802c64 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -26,6 +26,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "validator-engine-console-query.h" +#include "auto/tl/ton_api.h" +#include "td/utils/StringBuilder.h" #include "validator-engine-console.h" #include "terminal/terminal.h" #include "td/utils/filesystem.h" @@ -818,6 +820,34 @@ td::Status ImportCertificateQuery::send() { td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } +td::Status GetOverlaysStatsQuery::run() { + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + for (auto &s : f->overlays_) { + td::StringBuilder sb; + sb << "overlay_id=" << s->overlay_id_ << " adnl_id=" << s->adnl_id_ << "\n"; + sb << " nodes:"; + for (auto &n : s->nodes_) { + sb << " " << n->id_ << "\n"; + } + sb << " stats:\n"; + for (auto &t : s->stats_) { + sb << " " << t->key_ << "\t" << t->value_ << "\n"; + } + td::TerminalIO::output(sb.as_cslice()); + } + return td::Status::OK(); +} td::Status ImportCertificateQuery::receive(td::BufferSlice data) { @@ -883,4 +913,3 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; return td::Status::OK(); } - diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index bfd6b3e37..769f2052d 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -904,6 +904,25 @@ class CheckDhtServersQuery : public Query { ton::PublicKeyHash id_; }; +class GetOverlaysStatsQuery : public Query { + public: + GetOverlaysStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getoverlaysstats"; + } + static std::string get_help() { + return "getoverlaysstats\tgets stats for all overlays"; + } + std::string name() const override { + return get_name(); + } +}; + class SignCertificateQuery : public Query { public: SignCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -921,7 +940,6 @@ class SignCertificateQuery : public Query { std::string name() const override { return get_name(); } - void receive_pubkey(td::BufferSlice R); void receive_signature(td::BufferSlice R); diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 3f58cb7de..c47aa893d 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -136,6 +136,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { @@ -260,7 +261,8 @@ int main(int argc, char* argv[]) { std::exit(2); }); p.add_option('V', "version", "shows validator-engine-console build information", [&]() { - std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_checked_option('a', "address", "server address", [&](td::Slice arg) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 091f62213..303868c33 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,6 +27,11 @@ */ #include "validator-engine.hpp" +#include "auto/tl/ton_api.h" +#include "overlay-manager.h" +#include "td/actor/actor.h" +#include "tl-utils/tl-utils.hpp" +#include "tl/TlObject.h" #include "ton/ton-types.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -3236,6 +3241,35 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShard } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::get_stats, + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value(create_control_query_error( + td::Status::Error(ton::ErrorCode::notready, "overlay manager not ready"))); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3387,7 +3421,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows validator-engine build information", [&]() { - std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { @@ -3467,18 +3502,20 @@ int main(int argc, char *argv[]) { acts.push_back([&x, seq]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain, seq); }); return td::Status::OK(); }); - p.add_checked_option( - 'F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", [&](td::Slice params) { - auto pos1 = params.find(':'); - TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); - params = params.substr(++pos1, params.size()); - auto pos2 = params.find(':'); - TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); - params = params.substr(++pos2, params.size()); - auto h = std::stoi(params.substr(0, params.size()).str()); - acts.push_back([&x, b_seq, cc_seq, h]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); }); - return td::Status::OK(); - }); + p.add_checked_option('F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", + [&](td::Slice params) { + auto pos1 = params.find(':'); + TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); + params = params.substr(++pos1, params.size()); + auto pos2 = params.find(':'); + TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); + params = params.substr(++pos2, params.size()); + auto h = std::stoi(params.substr(0, params.size()).str()); + acts.push_back([&x, b_seq, cc_seq, h]() { + td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 6466c1841..88ba802df 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -28,6 +28,7 @@ #pragma once #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "rldp/rldp.h" #include "dht/dht.h" #include "validator/manager.h" @@ -401,7 +402,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - + void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { From 5e3e6f2b7da86e997884e823f2e9d80e8029d2ee Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sun, 5 Dec 2021 15:18:26 +0300 Subject: [PATCH 14/19] Reuse adnl_id as pubkey --- .../validator-engine-console.cpp | 2 ++ validator/full-node-shard.cpp | 1 + validator/full-node.cpp | 12 +++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index c47aa893d..899cd53fb 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -137,6 +137,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 1dac76fcd..72684151e 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -115,6 +115,7 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; + local_id_ = adnl_id_.pubkey_hash(); create_overlay(); } diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 7a5f0e38d..6606f2152 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -82,6 +82,7 @@ void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKe auto it = shards_.find(shard_id); if(it == shards_.end()) { promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; } td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); } @@ -106,6 +107,7 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise Date: Sun, 5 Dec 2021 17:41:22 +0300 Subject: [PATCH 15/19] Add tests and fixes for modpow2 --- CMakeLists.txt | 5 + crypto/CMakeLists.txt | 5 + crypto/common/bigint.hpp | 45 +- crypto/fift/lib/Asm.fif | 2 + crypto/test/modbigint.cpp | 931 ++++++++++++++++++++++++++++++++++++ crypto/test/test-bigint.cpp | 398 +++++++++++++++ rldp2/CMakeLists.txt | 2 +- 7 files changed, 1367 insertions(+), 21 deletions(-) create mode 100644 crypto/test/modbigint.cpp create mode 100644 crypto/test/test-bigint.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a97a9c960..da1945ff2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -423,6 +423,9 @@ target_link_libraries(test-vm PRIVATE ton_crypto fift-lib) add_executable(test-smartcont test/test-td-main.cpp ${SMARTCONT_TEST_SOURCE}) target_link_libraries(test-smartcont PRIVATE smc-envelope fift-lib ton_db) +add_executable(test-bigint ${BIGINT_TEST_SOURCE}) +target_link_libraries(test-bigint PRIVATE ton_crypto) + add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE}) target_link_libraries(test-cells PRIVATE ton_crypto) @@ -523,6 +526,7 @@ if (HAS_PARENT) ${FEC_TEST_SOURCE} ${ED25519_TEST_SOURCE} ${TONDB_TEST_SOURCE} + ${BIGNUM_TEST_SOURCE} ${CELLS_TEST_SOURCE} # ${TONVM_TEST_SOURCE} ${FIFT_TEST_SOURCE} ${TONLIB_ONLINE_TEST_SOURCE} PARENT_SCOPE) endif() @@ -536,6 +540,7 @@ set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests separate_arguments(TEST_OPTIONS) add_test(test-ed25519-crypto crypto/test-ed25519-crypto) add_test(test-ed25519 test-ed25519) +add_test(test-bigint test-bigint) add_test(test-vm test-vm ${TEST_OPTIONS}) add_test(test-fift test-fift ${TEST_OPTIONS}) add_test(test-cells test-cells ${TEST_OPTIONS}) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 79406ffca..88a0272b6 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -259,6 +259,11 @@ set(FIFT_TEST_SOURCE PARENT_SCOPE ) +set(BIGINT_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/test-bigint.cpp + PARENT_SCOPE +) + add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) target_include_directories(ton_crypto PUBLIC $ diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index e78f4892e..72d2ccd81 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -314,8 +314,15 @@ class BigIntG { digits[0] = x; } BigIntG(Normalize, word_t x) : n(1) { - digits[0] = x; - normalize_bool(); + if (x >= -Tr::Half && x < Tr::Half) { + digits[0] = x; + } else if (len <= 1) { + digits[0] = x; + normalize_bool(); + } else { + digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half; + digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); + } } BigIntG(const BigIntG& x) : n(x.n) { std::memcpy(digits, x.digits, n * sizeof(word_t)); @@ -757,7 +764,7 @@ bool AnyIntView::add_pow2_any(int exponent, int factor) { while (size() <= k) { digits[inc_size()] = 0; } - digits[k] += (factor << dm.rem); + digits[k] += ((word_t)factor << dm.rem); return true; } @@ -1087,12 +1094,16 @@ int AnyIntView::cmp_any(const AnyIntView& yp) const { template int AnyIntView::cmp_any(word_t y) const { - if (size() > 1) { - return top_word() < 0 ? -1 : 1; - } else if (size() == 1) { + if (size() == 1) { return digits[0] < y ? -1 : (digits[0] > y ? 1 : 0); - } else { + } else if (!size()) { return 0x80000000; + } else if (size() == 2 && (y >= Tr::Half || y < -Tr::Half)) { + word_t x0 = digits[0] & (Tr::Base - 1), y0 = y & (Tr::Base - 1); + word_t x1 = digits[1] + (digits[0] >> Tr::word_shift), y1 = (y >> Tr::word_shift); + return x1 < y1 ? -1 : (x1 > y1 ? 1 : (x0 < y0 ? -1 : (x0 > y0 ? 1 : 0))); + } else { + return top_word() < 0 ? -1 : 1; } } @@ -1312,10 +1323,8 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, if (k > quot.max_size()) { return invalidate_bool(); } - quot.set_size(max(k,1)); - for(int qi=0; qi< max(k,1); qi++) { - quot.digits[qi]=0; - } + quot.set_size(max(k, 1)); + quot.digits[0] = 0; } else { if (k >= quot.max_size()) { return invalidate_bool(); @@ -1466,21 +1475,17 @@ bool AnyIntView::mod_pow2_any(int exponent) { digits[size() - 1] = 0; digits[inc_size()] = ((word_t)1 << (q - word_shift)); } - if (q - word_shift == -1 && size() < max_size() - 1) { + if (q - word_shift == -1 && size() < max_size()) { digits[size() - 1] = -Tr::Half; digits[inc_size()] = 1; } else { digits[size() - 1] = pow; } return true; - } else if (v >= Tr::Half) { - if (size() == max_size() - 1) { - return invalidate_bool(); - } else { - digits[size() - 1] = v | -Tr::Half; - digits[inc_size()] = ((word_t)1 << (q - word_shift)); - return true; - } + } else if (v >= Tr::Half && size() < max_size()) { + digits[size() - 1] = v & (Tr::Base - 1); + digits[inc_size()] = (v >> word_shift); + return true; } else { digits[size() - 1] = v; return true; diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index d61923c12..bfd8da492 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -334,6 +334,8 @@ x{A926} @Defop RSHIFTC x{A935} @Defop(8u+1) RSHIFTR# x{A936} @Defop(8u+1) RSHIFTC# x{A938} @Defop(8u+1) MODPOW2# +x{A939} @Defop(8u+1) MODPOW2R# +x{A93A} @Defop(8u+1) MODPOW2C# x{A984} @Defop MULDIV x{A985} @Defop MULDIVR x{A98C} @Defop MULDIVMOD diff --git a/crypto/test/modbigint.cpp b/crypto/test/modbigint.cpp new file mode 100644 index 000000000..c73bdc0c5 --- /dev/null +++ b/crypto/test/modbigint.cpp @@ -0,0 +1,931 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include + +namespace modint { + +enum { mod_cnt = 32 }; + +// mod_cnt = 9 => integers -2^268 .. 2^268 +// mod_cnt = 18 => integers -2^537 .. 2^537 +// mod_cnt = 32 => integers -2^955 .. 2^955 +constexpr int mod[mod_cnt] = { +999999937, 999999929, 999999893, 999999883, 999999797, 999999761, 999999757, 999999751, +999999739, 999999733, 999999677, 999999667, 999999613, 999999607, 999999599, 999999587, +999999541, 999999527, 999999503, 999999491, 999999487, 999999433, 999999391, 999999353, +99999337, 999999323, 999999229, 999999223, 999999197, 999999193, 999999191, 999999181 +}; + +// invm[i][j] = mod[i]^(-1) modulo mod[j] +int invm[mod_cnt][mod_cnt]; + +int gcdx (int a, int b, int& u, int& v); + +template +struct ModArray; + +template +struct MixedRadix; + +template +struct ArrayRawDumpRef; + +template +std::ostream& raw_dump_array(std::ostream& os, const std::array& arr) { + os << '['; + for (auto x : arr) { + os << ' ' << x; + } + return os << " ]"; +} + +template +struct MixedRadix { + enum { n = N }; + std::array a; + MixedRadix(int v) { set_int(v); } + MixedRadix() = default; + MixedRadix(const MixedRadix&) = default; + MixedRadix(std::initializer_list l) : a(l) {} + MixedRadix(const std::array& arr) : a(arr) {} + template + MixedRadix(const MixedRadix& other) { assert (M >= N); memcpy(a.data(), other.a.data(), N * sizeof(int)); } + MixedRadix(const ModArray& other); + MixedRadix(const ModArray& other, bool sgnd); + + MixedRadix& set_zero() { a.fill(0); return *this; } + MixedRadix& set_one() { a.fill(0); a[0] = 1; return *this; } + + MixedRadix& set_int(int v) { a.fill(0); a[0] = v; return *this; } + MixedRadix copy() const { return MixedRadix{*this}; } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + int sgn() const { + int i = N - 1; + while (i >= 0 && !a[i]) { --i; } + return i < 0 ? 0 : (a[i] > 0 ? 1 : -1); + } + + int cmp(const MixedRadix& other) const { + int i = N - 1; + while (i >= 0 && a[i] == other.a[i]) { --i; } + return i < 0 ? 0 : (a[i] > other.a[i] ? 1 : -1); + } + + bool is_small() const { + return !a[N - 1] || a[N - 1] == -1; + } + + bool operator==(const MixedRadix& other) const { + return a == other.a; + } + + bool operator!=(const MixedRadix& other) const { + return a != other.a; + } + + bool operator<(const MixedRadix& other) const { + return cmp(other) < 0; + } + + bool operator<=(const MixedRadix& other) const { + return cmp(other) <= 0; + } + + bool operator>(const MixedRadix& other) const { + return cmp(other) > 0; + } + + bool operator>=(const MixedRadix& other) const { + return cmp(other) >= 0; + } + + MixedRadix& negate() { + int i = 0; + while (i < N - 1 && !a[i]) { i++; } + a[i]--; + for (; i < N; i++) { + a[i] = mod[i] - a[i] - 1; + } + a[N - 1] -= mod[N - 1]; + return *this; + } + + static const MixedRadix& pow2(int power); + static MixedRadix negpow2(int power) { + return -pow2(power); + } + + template + const MixedRadix& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& import_mod_array(const int* data, bool sgnd = true) { + for (int i = 0; i < N; i++) { + a[i] = data[i] % mod[i]; + } + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + for (int j = i + 1; j < N; j++) { + a[j] = (int)((long long)(a[j] - a[i]) * invm[i][j] % mod[j]); + } + } + if (sgnd && a[N - 1] > (mod[N - 1] >> 1)) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator=(const MixedRadix&) = default; + + template + MixedRadix& operator=(const MixedRadix& other) { + static_assert (M >= N); + memcpy (a.data(), other.a.data(), N * sizeof (int)); + } + + MixedRadix& import_mod_array(const ModArray& other, bool sgnd = true); + + MixedRadix& operator=(const ModArray& other) { + return import_mod_array(other); + } + + MixedRadix& set_sum(const MixedRadix& x, const MixedRadix& y, int factor = 1) { + long long carry = 0; + for (int i = 0; i < N; i++) { + long long acc = x.a[i] + carry + (long long)factor * y.a[i]; + carry = acc / mod[i]; + a[i] = (int)(acc - carry * mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + --carry; + } + } + if (a[N - 1] >= 0 && carry == -1) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator+=(const MixedRadix& other) { + return set_sum(*this, other); + } + + MixedRadix& operator-=(const MixedRadix& other) { + return set_sum(*this, other, -1); + } + + static const MixedRadix& zero(); + static const MixedRadix& one(); + + MixedRadix& operator*=(int factor) { + return set_sum(zero(), *this, factor); + } + + MixedRadix operator-() const { + MixedRadix copy{*this}; + copy.negate(); + return copy; + } + + MixedRadix operator+(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other); + return res; + } + + MixedRadix operator-(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other, -1); + return res; + } + + MixedRadix operator*(int factor) const { + MixedRadix res; + res.set_sum(zero(), *this, factor); + return res; + } + + int operator%(int b) const { + int x = a[N - 1] % b; + for (int i = N - 2; i >= 0; --i) { + x = ((long long)x * mod[i] + a[i]) % b; + } + return ((x ^ b) < 0 && x) ? x + b : x; + } + + explicit operator double() const { + double acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + explicit operator long long() const { + long long acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + MixedRadix& to_base(int base) { + int k = N - 1; + while (k > 0 && !a[k]) { --k; } + if (k <= 0) { + return *this; + } + for (int i = k - 1; i >= 0; --i) { + // a[i..k] := a[i+1..k] * mod[i] + a[i] + long long carry = a[i]; + for (int j = i; j < k; j++) { + long long t = (long long)a[j + 1] * mod[i] + carry; + carry = t / base; + a[j] = (int)(t - carry * base); + } + a[k] = (int)carry; + } + return *this; + } + + std::ostream& print_dec_destroy(std::ostream& os) { + int s = sgn(); + if (s < 0) { + os << '-'; + negate(); + } else if (!s) { + os << '0'; + return os; + } + to_base(1000000000); + int i = N - 1; + while (!a[i] && i > 0) { --i; } + os << a[i]; + while (--i >= 0) { + char buff[12]; + sprintf (buff, "%09d", a[i]); + os << buff; + } + return os; + } + + std::ostream& print_dec(std::ostream& os) const& { + MixedRadix copy{*this}; + return copy.print_dec_destroy(os); + } + + std::ostream& print_dec(std::ostream& os) && { + return print_dec_destroy(os); + } + + std::string to_dec_string_destroy() { + std::ostringstream os; + print_dec_destroy(os); + return std::move(os).str(); + } + + std::string to_dec_string() const& { + MixedRadix copy{*this}; + return copy.to_dec_string_destroy(); + } + + std::string to_dec_string() && { + return to_dec_string_destroy(); + } + + bool to_binary_destroy(unsigned char *arr, int size, bool sgnd = true) { + if (size <= 0) { + return false; + } + int s = (sgnd ? sgn() : 1); + memset (arr, 0, size); + if (s < 0) { + negate(); + } else if (!s) { + return true; + } + to_base(1 << 30); + long long acc = 0; + int bits = 0, j = size; + for (int i = 0; i < N; i++) { + if (!j && a[i]) { + return false; + } + acc += ((long long) a[i] << bits); + bits += 30; + while (bits >= 8 && j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + bits -= 8; + acc >>= 8; + } + } + while (j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + if (acc) { + return false; + } + if (!sgnd) { + return true; + } + if (s >= 0) { + return arr[0] <= 0x7f; + } + j = size - 1; + while (j >= 0 && !arr[j]) { --j; } + assert (j >= 0); + arr[j] = (unsigned char)(-arr[j]); + while (--j >= 0) { arr[j] = (unsigned char)~arr[j]; } + return arr[0] >= 0x80; + } + + bool to_binary(unsigned char *arr, int size, bool sgnd = true) const& { + MixedRadix copy{*this}; + return copy.to_binary_destroy(arr, size, sgnd); + } + + bool to_binary(unsigned char *arr, int size, bool sgnd = true) && { + return to_binary_destroy(arr, size, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +struct ModArray { + enum { n = N }; + std::array a; + ModArray(int v) { set_int(v); } + ModArray(long long v) { set_long(v); } + ModArray(long v) { set_long(v); } + ModArray() = default; + ModArray(const ModArray&) = default; + ModArray(std::initializer_list l) : a(l) {} + ModArray(const std::array& arr) : a(arr) {} + template + ModArray(const ModArray& other) { static_assert (M >= N); memcpy(a.data(), other.a.data(), N * sizeof(int)); } + ModArray(const int* p) : a(p) {} + ModArray(std::string str) { assert(from_dec_string(str) && "not a decimal number"); } + + ModArray& set_zero() { a.fill(0); return *this; } + ModArray& set_one() { a.fill(1); return *this; } + + ModArray& set_int(int v) { + if (v >= 0) { + a.fill(v); + } else { + for (int i = 0; i < N; i++) { + a[i] = mod[i] + v; + } + } + return *this; + } + + ModArray& set_long(long long v) { + for (int i = 0; i < N; i++) { + a[i] = v % mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray copy() const { + return ModArray{*this}; + } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + static const ModArray& zero(); + static const ModArray& one(); + + ModArray& operator=(const ModArray&) = default; + + template + ModArray& operator=(const ModArray& other) { + assert (M >= N); + memcpy (a.data(), other.a.data(), N * sizeof (int)); + return *this; + } + + ModArray& negate() { + for (int i = 0; i < N; i++) { + a[i] = (a[i] ? mod[i] - a[i] : 0); + } + return *this; + } + + ModArray& norm_neg() { + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& normalize() { + for (int i = 0; i < N; i++) { + a[i] %= mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + bool operator==(const ModArray& other) const { + return a == other.a; + } + + bool operator!=(const ModArray& other) const { + return a != other.a; + } + + ModArray& set_sum(const ModArray& x, const ModArray& y) { + for (int i = 0; i < N; i++) { + a[i] = x.a[i] + y.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] += other.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + v) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] -= other.a[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(long long v) { + return (operator+=)(-v); + } + + ModArray& mul_arr(const int other[]) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other[i]) % mod[i]); + } + return *this; + } + + ModArray& operator*=(const ModArray& other) { + return mul_arr(other.a.data()); + } + + template + ModArray& operator*=(const ModArray& other) { + assert (M >= N); + return mul_arr(other.a.data()); + } + + ModArray& operator*=(int v) { + for (int i = 0; i < N; i++) { + a[i] = ((long long)a[i] * v) % mod[i]; + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& operator*=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * (v % mod[i])) % mod[i]); + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& mul_add(int v, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * v + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& mul_add(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other.a[i] + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& lshift_add(int shift, long long w) { + return mul_add(pow2(shift), w); + } + + ModArray operator+(const ModArray& other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(const ModArray& other) const { + ModArray copy{*this}; + copy -= other; + return copy; + } + + ModArray operator+(long long other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(long long other) const { + ModArray copy{*this}; + copy += -other; + return copy; + } + + ModArray operator-() const { + ModArray copy{*this}; + copy.negate(); + return copy; + } + + ModArray operator*(const ModArray& other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + ModArray operator*(long long other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + bool invert() { + for (int i = 0; i < N; i++) { + int t; + if (gcdx(a[i], mod[i], a[i], t) != 1) { + return false; + } + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + bool try_divide(const ModArray& other) { + for (int i = 0; i < N; i++) { + int q, t; + if (gcdx(other.a[i], mod[i], q, t) != 1) { + return false; + } + a[i] = (long long)a[i] * q % mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + ModArray& operator/=(const ModArray& other) { + assert (try_divide(other) && "division by zero?"); + return *this; + } + + ModArray operator/(const ModArray& other) { + ModArray copy{*this}; + copy /= other; + return copy; + } + + ModArray& operator<<=(int lshift); + + ModArray operator<<(int lshift) const { + ModArray copy{*this}; + copy <<= lshift; + return copy; + } + + static const ModArray& pow2(int power); + static const ModArray& negpow2(int power); + + template + const ModArray& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& to_mixed_radix(MixedRadix& dest, bool sgnd = true) const { + return dest.import_mod_array(a.data(), sgnd); + } + + MixedRadix to_mixed_radix(bool sgnd = true) const { + return MixedRadix(*this, sgnd); + } + + int operator%(int div) const { + return to_mixed_radix() % div; + } + + explicit operator double() const { + return (double)to_mixed_radix(); + } + + std::string to_dec_string() const { + return MixedRadix(*this).to_dec_string(); + } + + std::ostream& print_dec(std::ostream& os, bool sgnd = true) const { + return MixedRadix(*this, sgnd).print_dec(os); + } + + bool to_binary(unsigned char *arr, int size, bool sgnd = true) const { + return MixedRadix(*this, sgnd).to_binary(arr, size, sgnd); + } + + template + bool to_binary(std::array& arr, bool sgnd = true) const { + return to_binary(arr.data(), M, sgnd); + } + + bool from_dec_string(const char *start, const char *end) { + set_zero(); + if (start >= end) { + return false; + } + bool sgn = (*start == '-'); + if (sgn && ++start == end) { + return false; + } + int acc = 0, pow = 1; + while (start < end) { + if (*start < '0' || *start > '9') { + return false; + } + acc = acc * 10 + (*start++ - '0'); + pow *= 10; + if (pow >= 1000000000) { + mul_add(pow, acc); + pow = 1; + acc = 0; + } + } + if (pow > 1) { + mul_add(pow, acc); + } + if (sgn) { + negate(); + } + return true; + } + + bool from_dec_string(std::string str) { + return from_dec_string(str.data(), str.data() + str.size()); + } + + ModArray& from_binary(const unsigned char *arr, int size, bool sgnd = true) { + set_zero(); + if (size <= 0) { + return *this; + } + int i = 0, pow = 0; + long long acc = (sgnd && arr[0] >= 0x80 ? -1 : 0); + while (i < size && arr[i] == (unsigned char)acc) { + i++; + } + for (; i < size; i++) { + pow += 8; + acc = (acc << 8) + arr[i]; + if (pow >= 56) { + lshift_add(pow, acc); + acc = pow = 0; + } + } + if (pow || acc) { + lshift_add(pow, acc); + } + return *this; + } + + template + ModArray& from_binary(const std::array& arr, bool sgnd = true) { + return from_binary(arr.data(), M, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +MixedRadix::MixedRadix(const ModArray& other) { + import_mod_array(other.a.data()); +} + +template +MixedRadix::MixedRadix(const ModArray& other, bool sgnd) { + import_mod_array(other.a.data(), sgnd); +} + +template +MixedRadix& MixedRadix::import_mod_array(const ModArray& other, bool sgnd) { + return import_mod_array(other.a.data(), sgnd); +} + +template +std::ostream& operator<<(std::ostream& os, const ModArray& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, const MixedRadix& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, MixedRadix&& x) { + return x.print_dec_destroy(os); +} + +template +struct ArrayRawDumpRef { + const std::array& ref; + ArrayRawDumpRef(const std::array& _ref) : ref(_ref) {}; +}; + +template +std::ostream& operator<<(std::ostream& os, ArrayRawDumpRef rd_ref) { + return raw_dump_array(os, rd_ref.ref); +}; + +constexpr int pow2_cnt = 1001; + +ModArray Zero(0), One(1), Pow2[pow2_cnt], NegPow2[pow2_cnt]; +MixedRadix Zero_mr(0), One_mr(1), Pow2_mr[pow2_cnt], NegPow2_mr[pow2_cnt]; + +template +ModArray& ModArray::operator<<=(int lshift) { + assert (lshift >= 0 && lshift < pow2_cnt); + return (operator*=)(Pow2[lshift]); +} + +template +const MixedRadix& MixedRadix::pow2(int power) { + return Pow2_mr[power].as_shorter(); +} + +/* +template +const MixedRadix& MixedRadix::negpow2(int power) { + return NegPow2_mr[power].as_shorter(); +} +*/ + +template +const ModArray& ModArray::pow2(int power) { + return Pow2[power].as_shorter(); +} + +template +const ModArray& ModArray::negpow2(int power) { + return NegPow2[power].as_shorter(); +} + +template +const ModArray& ModArray::zero() { + return Zero.as_shorter(); +} + +template +const ModArray& ModArray::one() { + return One.as_shorter(); +} + +template +const MixedRadix& MixedRadix::zero() { + return Zero_mr.as_shorter(); +} + +template +const MixedRadix& MixedRadix::one() { + return One_mr.as_shorter(); +} + +void init_pow2() { + Pow2[0].set_one(); + Pow2_mr[0].set_one(); + for (int i = 1; i < pow2_cnt; i++) { + Pow2[i].set_sum(Pow2[i - 1], Pow2[i - 1]); + Pow2_mr[i].set_sum(Pow2_mr[i - 1], Pow2_mr[i - 1]); + } + for (int i = 0; i < pow2_cnt; i++) { + NegPow2[i] = -Pow2[i]; + NegPow2_mr[i] = -Pow2_mr[i]; + } +} + +int gcdx (int a, int b, int& u, int& v) { + int a1 = 1, a2 = 0, b1 = 0, b2 = 1; + while (b) { + int q = a / b; + int t = a - q * b; a = b; b = t; + t = a1 - q * b1; a1 = b1; b1 = t; + t = a2 - q * b2; a2 = b2; b2 = t; + } + u = a1; v = a2; + return a; +} + +void init_invm() { + for (int i = 0; i < mod_cnt; i++) { + assert (mod[i] > 0 && mod[i] <= (1 << 30)); + for (int j = 0; j < i; j++) { + assert (gcdx (mod[i], mod[j], invm[i][j], invm[j][i]) == 1); + if (invm[i][j] < 0) { + invm[i][j] += mod[j]; + } + if (invm[j][i] < 0) { + invm[j][i] += mod[i]; + } + } + } +} + +void init() { + init_invm(); + init_pow2(); +} + +} // namespace modint diff --git a/crypto/test/test-bigint.cpp b/crypto/test/test-bigint.cpp new file mode 100644 index 000000000..94c6ed07e --- /dev/null +++ b/crypto/test/test-bigint.cpp @@ -0,0 +1,398 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include +#include "common/refcnt.hpp" +#include "common/bigint.hpp" +#include "common/refint.h" +#include "modbigint.cpp" + +#include "td/utils/tests.h" + +using BInt = modint::ModArray<18>; // integers up to 2^537 +using MRInt = modint::MixedRadix<18>; // auxiliary integer representation for printing, comparing etc + +MRInt p2_256, np2_256, p2_63, np2_63; +constexpr long long ll_min = -2 * (1LL << 62), ll_max = ~ll_min; + +int mkint_chk_mode = -1, res_chk_mode = 0; + +bool mr_in_range(const MRInt& x) { + return x < p2_256 && x >= np2_256; +} + +bool mr_is_small(const MRInt& x) { + return x < p2_63 && x >= np2_63; +} + +bool mr_fits_bits(const MRInt& x, int bits) { + if (bits > 0) { + return x < MRInt::pow2(bits - 1) && x >= MRInt::negpow2(bits - 1); + } else { + return !bits && !x.sgn(); + } +} + +bool mr_ufits_bits(const MRInt& x, int bits) { + return bits >= 0 && x.sgn() >= 0 && x < MRInt::pow2(bits); +} + +bool extract_value_any_bool(BInt& val, const td::AnyIntView& x, bool chk_norm = true) { + int n = x.size(); + if (n <= 0 || n > x.max_size() || (!x.digits[n - 1] && n > 1)) { + return false; + } + assert (n == 1 || x.digits[n - 1] != 0); + val.set_zero(); + for (int i = n - 1; i >= 0; --i) { + val.lshift_add(td::BigIntInfo::word_shift, x.digits[i]); + if (chk_norm && (x.digits[i] < - td::BigIntInfo::Half || x.digits[i] >= td::BigIntInfo::Half)) { + return false; // unnormalized + } + } + return true; +} + +template +bool extract_value_bool(BInt& val, const T& x, bool chk_norm = true) { + return extract_value_any_bool(val, x.as_any_int(), chk_norm); +} + +BInt extract_value_any(const td::AnyIntView& x, bool chk_norm = true) { + BInt res; + CHECK(extract_value_any_bool(res, x, chk_norm)); + return res; +} + +template +BInt extract_value(const T& x, bool chk_norm = true) { + return extract_value_any(x.as_any_int(), chk_norm); +} + +template +BInt extract_value_alt(const T& x) { + BInt res; + const int *md = res.mod_array(); + for (int i = 0; i < res.n / 2; i++) { + T copy{x}; + int m1 = md[2 * i], m2 = md[2 * i + 1]; + long long rem = copy.divmod_short((long long)m1 * m2); + res.a[2 * i] = (int)(rem % m1); + res.a[2 * i + 1] = (int)(rem % m2); + } + if (res.n & 1) { + T copy{x}; + res.a[res.n - 1] = (int)copy.divmod_short(md[res.n - 1]); + } + return res; +} + +constexpr int min_spec_int = -0xfd08, max_spec_int = 0xfd07; +// x = sgn*(ord*256+a*16+b) => sgn*((32+a)*2^(ord-2) + b - 8) +// x = -0xfd08 => -2^256 ... x = 0xfd07 => 2^256 - 1 +td::RefInt256 make_special_int(int x, BInt *ptr = nullptr, unsigned char bin[64] = nullptr) { + bool sgn = (x < 0); + if (sgn) { x = -x; } + int ord = (x >> 8) - 2, a = 32 + ((x >> 4) & 15), b = (x & 15) - 8; + if (ord < 0) { + a >>= -ord; + ord = 0; + } + if (sgn) { a = -a; b = -b; } + if (ptr) { + ptr->set_int(a); + *ptr <<= ord; + *ptr += b; + } + if (bin) { + int acc = b, r = ord; + for (int i = 63; i >= 0; --i) { + if (r < 8) { + acc += (a << r); + r = 1024; + } + r -= 8; + bin[i] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + } + return (td::make_refint(a) << ord) + b; +} + +void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt *valptr = nullptr, const unsigned char bin[64] = nullptr) { + CHECK(x.not_null() && (in_range <= -2 || x->is_valid())); + if (!x->is_valid()) { + // not much to check when x is a NaN + unsigned char bytes[64]; + if (valptr) { + // check that the true answer at `valptr` is out of range + CHECK(!mr_in_range(valptr->to_mixed_radix())); + if (mode & 0x200) { + // check BInt binary export + valptr->to_binary(bytes, 64); + if (bin) { + // check that the two true answers match + CHECK(!memcmp(bin, bytes, 64)); + } else { + bin = bytes; + } + } + } + if (bin) { + // check that the true answer in `bin` is out of range + int i = 0, sgn = (bin[0] >= 0x80 ? -1 : 0); + while (i < 32 && bin[i] == (unsigned char)sgn); + CHECK(i < 32); + if (valptr && (mode & 0x100)) { + // check BInt binary export + BInt val2; + val2.from_binary(bin, 64); + CHECK(*valptr == val2); + } + } + return; + } + unsigned char bytes[64]; + CHECK(x->export_bytes(bytes, 64)); + if (bin) { + CHECK(!memcmp(bytes, bin, 64)); + } + BInt val = extract_value(*x); + if (valptr) { + CHECK(val == *valptr); + } + if (mode & 1) { + BInt val2 = extract_value_alt(*x); + CHECK(val == val2); + } + MRInt mval(val); + bool val_in_range = mr_in_range(mval); + CHECK(x->fits_bits(257) == val_in_range); + if (in_range >= 0) { + CHECK((int)val_in_range == in_range); + } + if (mode & 2) { + // check binary import + td::BigInt256 y; + y.import_bytes(bytes, 64); + CHECK(y == *x); + } + if (mode & 0x100) { + // check binary import for BInt + BInt val2; + val2.from_binary(bytes, 64); + CHECK(val == val2); + } + if (mode & 0x200) { + // check binary export for BInt + unsigned char bytes2[64]; + mval.to_binary(bytes2, 64); + CHECK(!memcmp(bytes, bytes2, 64)); + } + // check sign + int sgn = mval.sgn(); + CHECK(x->sgn() == sgn); + // check if small (fits into 64 bits) + bool is_small = mr_is_small(mval); + CHECK(is_small == x->fits_bits(64)); + if (is_small) { + // special check for small (64-bit) values + long long xval = (long long)mval; + CHECK(x->to_long() == xval); + CHECK((long long)__builtin_bswap64(*(long long *)(bytes + 64 - 8)) == xval); + // check comparison with long long + CHECK(x == xval); + CHECK(!cmp(x, xval)); + // check constructor from long long + CHECK(!cmp(x, td::make_refint(xval))); + if (xval != ll_min) { + CHECK(x > xval - 1); + CHECK(x > td::make_refint(xval - 1)); + } + if (xval != ll_max) { + CHECK(x < xval + 1); + CHECK(x < td::make_refint(xval + 1)); + } + } + if (mode & 0x10) { + // check decimal export + std::string dec = mval.to_dec_string(); + CHECK(x->to_dec_string() == dec); + // check decimal import + td::BigInt256 y; + int l = y.parse_dec(dec); + CHECK((std::size_t)l == dec.size() && y == *x); + if (mode & 0x1000) { + // check decimal import for BInt + BInt val2; + CHECK(val2.from_dec_string(dec) && val2 == val); + } + } + if (mode & 0x20) { + // check binary bit size + int sz = x->bit_size(); + CHECK(sz >= 0 && sz <= 300); + CHECK(x->fits_bits(sz) && (!sz || !x->fits_bits(sz - 1))); + CHECK(mr_fits_bits(mval, sz) && !mr_fits_bits(mval, sz - 1)); + int usz = x->bit_size(false); + CHECK(sgn >= 0 || usz == 0x7fffffff); + if (sgn >= 0) { + CHECK(x->unsigned_fits_bits(usz) && (!usz || !x->unsigned_fits_bits(usz - 1))); + CHECK(mr_ufits_bits(mval, usz) && !mr_ufits_bits(mval, usz - 1)); + } else { + CHECK(!x->unsigned_fits_bits(256) && !x->unsigned_fits_bits(300)); + } + } +} + +void check_special_int(int idx) { + BInt b; + unsigned char binary[64]; + td::RefInt256 x = make_special_int(idx, &b, binary); + check_one_int_repr(x, mkint_chk_mode, idx >= min_spec_int && idx <= max_spec_int, &b, binary); +} + +void init_aux() { + np2_256 = p2_256 = MRInt::pow2(256); + np2_256.negate(); + CHECK(np2_256 == MRInt::negpow2(256)); + p2_63 = np2_63 = MRInt::pow2(63); + np2_63.negate(); + CHECK(np2_63 == MRInt::negpow2(63)); +} + +std::vector SpecInt; +BInt SpecIntB[max_spec_int - min_spec_int + 1]; + +void init_check_special_ints() { + std::cerr << "check_special_ints" << std::endl; + BInt b; + unsigned char binary[64]; + for (int idx = min_spec_int - 512; idx <= max_spec_int + 512; idx++) { + td::RefInt256 x = make_special_int(idx, &b, binary); + check_one_int_repr(x, mkint_chk_mode, idx >= min_spec_int && idx <= max_spec_int, &b, binary); + if (idx >= min_spec_int && idx <= max_spec_int) { + SpecIntB[idx - min_spec_int] = b; + SpecInt.push_back(std::move(x)); + } + } +} + +void check_res(td::RefInt256 y, const BInt& yv) { + check_one_int_repr(std::move(y), res_chk_mode, -2, &yv); +} + +void check_unary_ops_on(td::RefInt256 x, const BInt& xv) { + // NEGATE + BInt yv = -xv; + check_res(-x, yv); + // NOT + check_res(~x, yv -= 1); +} + +void check_unary_ops() { + std::cerr << "check unary ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + check_unary_ops_on(SpecInt[idx - min_spec_int], SpecIntB[idx - min_spec_int]); + } +} + +void check_pow2_ops(int shift) { + // POW2 + td::RefInt256 r{true}; + r.unique_write().set_pow2(shift); + check_res(r, BInt::pow2(shift)); + // POW2DEC + r.unique_write().set_pow2(shift).add_tiny(-1).normalize(); + check_res(r, BInt::pow2(shift) - 1); + // NEGPOW2 + r.unique_write().set_pow2(shift).negate().normalize(); + check_res(r, -BInt::pow2(shift)); +} + +void check_pow2_ops() { + std::cerr << "check power-2 ops" << std::endl; + for (int i = 0; i <= 256; i++) { + check_pow2_ops(i); + } +} + +void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv) { + // LSHIFT + check_res(x << shift, xv << shift); + // FITS + MRInt mval(xv); + CHECK(x->fits_bits(shift) == mr_fits_bits(mval, shift)); + // UFITS + CHECK(x->unsigned_fits_bits(shift) == mr_ufits_bits(mval, shift)); + // ADDPOW2 / SUBPOW2 + auto y = x; + y.write().add_pow2(shift).normalize(); + check_res(std::move(y), xv + BInt::pow2(shift)); + y = x; + y.write().sub_pow2(shift).normalize(); + check_res(std::move(y), xv - BInt::pow2(shift)); + // RSHIFT, MODPOW2 + for (int round_mode = -1; round_mode <= 1; round_mode++) { + auto r = x, q = td::rshift(x, shift, round_mode); // RSHIFT + CHECK(q.not_null() && q->is_valid()); + r.write().mod_pow2(shift, round_mode).normalize(); // MODPOW2 + CHECK(r.not_null() && r->is_valid()); + if (round_mode < 0) { + CHECK(!cmp(x >> shift, q)); // operator>> should be equivalent to td::rshift + } + BInt qv = extract_value(*q), rv = extract_value(*r); + // check main division equality (q << shift) + r == x + CHECK((qv << shift) + rv == xv); + MRInt rval(rv); + // check remainder range + switch (round_mode) { + case 1: + rval.negate(); // fallthrough + case -1: + CHECK(mr_ufits_bits(rval, shift)); + break; + case 0: + CHECK(mr_fits_bits(rval, shift)); + } + } +} + +void check_shift_ops() { + std::cerr << "check left/right shift ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + //for (int idx = -52239; idx <= max_spec_int; idx++) { + std::cerr << "idx=" << idx << " : " << SpecIntB[idx - min_spec_int] << std::endl; + for (int i = 0; i <= 256; i++) { + check_shift_ops_on(i, SpecInt[idx - min_spec_int], SpecIntB[idx - min_spec_int]); + } + } +} + +int main() { + modint::init(); + init_aux(); + init_check_special_ints(); + check_pow2_ops(); + check_unary_ops(); + check_shift_ops(); + return 0; +} diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index f93f6e1e0..33ae5b672 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -51,7 +51,7 @@ target_include_directories(rldp PUBLIC ${OPENSSL_INCLUDE_DIR} ) if (GSL_FOUND) - target_link_libraries(rldp2 PRIVATE GSL::gsl) + target_link_libraries(rldp2 PRIVATE gsl) target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1) endif() target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api) From 875f5b04e9d3aa5a0e7d35ce3b7ab4a689613be7 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sun, 5 Dec 2021 18:44:17 +0300 Subject: [PATCH 16/19] Fix error handling in validator-engine::sign_shard_overlay_cert --- validator-engine/validator-engine.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 303868c33..59279bef7 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3236,8 +3236,15 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShard return; } ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value(R.move_as_ok()); + } + }); td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(promise)); + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); } From e4324e803b7f0c26e44bc131ac59aba05adaa852 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Wed, 22 Dec 2021 22:08:24 +0300 Subject: [PATCH 17/19] Fix issues in muldivmod operations --- crypto/common/bigint.hpp | 32 +- crypto/common/refint.cpp | 8 +- crypto/fift/lib/Asm.fif | 3 + crypto/test/modbigint.cpp | 485 ++++++++++++++++++---------- crypto/test/test-bigint.cpp | 628 +++++++++++++++++++++++++++++++----- crypto/vm/arithops.cpp | 26 +- doc/tvm.tex | 5 +- 7 files changed, 902 insertions(+), 285 deletions(-) diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index 72d2ccd81..94da3a131 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -264,7 +264,7 @@ class AnyIntView { return digits[size() - 1]; } double top_double() const { - return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * (1.0 / Tr::Base) + return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * Tr::InvBase : (double)digits[size() - 1]; } bool is_odd_any() const { @@ -1323,15 +1323,14 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, if (k > quot.max_size()) { return invalidate_bool(); } - quot.set_size(max(k, 1)); + quot.set_size(std::max(k, 1)); quot.digits[0] = 0; } else { if (k >= quot.max_size()) { return invalidate_bool(); } quot.set_size(k + 1); - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv * Tr::InvBase); + word_t q = std::llrint(top_double() * y_inv * Tr::InvBase); quot.digits[k] = q; int i = yp.size() - 1; word_t hi = 0; @@ -1346,8 +1345,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, quot.digits[0] = 0; } while (--k >= 0) { - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv); + word_t q = std::llrint(top_double() * y_inv); quot.digits[k] = q; for (int i = yp.size() - 1; i >= 0; --i) { Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); @@ -1355,15 +1353,18 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, dec_size(); digits[size() - 1] += (digits[size()] << word_shift); } - if (size() >= yp.size()) { - assert(size() == yp.size()); - double x_top = top_double(); - double t = x_top * y_inv * Tr::InvBase; + if (size() >= yp.size() - 1) { + assert(size() <= yp.size()); + bool grow = (size() < yp.size()); + double t = top_double() * y_inv * (grow ? Tr::InvBase * Tr::InvBase : Tr::InvBase); if (round_mode >= 0) { t += (round_mode ? 1 : 0.5); } word_t q = std::llrint(std::floor(t)); if (q) { + if (grow) { + digits[inc_size()] = 0; + } for (int i = 0; i < size(); i++) { digits[i] -= q * yp.digits[i]; } @@ -1420,6 +1421,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, return normalize_bool_any(); } +// works for almost-normalized numbers (digits -Base+1 .. Base-1, top non-zero), result also almost-normalized template bool AnyIntView::mod_pow2_any(int exponent) { if (!is_valid()) { @@ -1471,11 +1473,10 @@ bool AnyIntView::mod_pow2_any(int exponent) { if (exponent >= max_size() * word_shift) { return invalidate_bool(); } - if (q - word_shift >= 0) { + if (q - word_shift >= 0) { // original top digit was a non-zero multiple of Base, impossible(?) digits[size() - 1] = 0; digits[inc_size()] = ((word_t)1 << (q - word_shift)); - } - if (q - word_shift == -1 && size() < max_size()) { + } else if (q - word_shift == -1 && size() < max_size()) { digits[size() - 1] = -Tr::Half; digits[inc_size()] = 1; } else { @@ -1483,8 +1484,9 @@ bool AnyIntView::mod_pow2_any(int exponent) { } return true; } else if (v >= Tr::Half && size() < max_size()) { - digits[size() - 1] = v & (Tr::Base - 1); - digits[inc_size()] = (v >> word_shift); + word_t w = (((v >> (word_shift - 1)) + 1) >> 1); + digits[size() - 1] = v - (w << word_shift); + digits[inc_size()] = w; return true; } else { digits[size() - 1] = v; diff --git a/crypto/common/refint.cpp b/crypto/common/refint.cpp index b79750ce8..8e06da5fe 100644 --- a/crypto/common/refint.cpp +++ b/crypto/common/refint.cpp @@ -128,12 +128,10 @@ RefInt256 muldiv(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { } std::pair muldivmod(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); - RefInt256 quot{true}; - tmp.mod_div(*z, quot.unique_write(), round_mode); - quot.write().normalize(); - return std::make_pair(std::move(quot), td::make_refint(tmp)); + tmp.mod_div(*z, quot, round_mode); + return std::make_pair(td::make_refint(quot.normalize()), td::make_refint(tmp)); } RefInt256 operator&(RefInt256 x, RefInt256 y) { diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index bfd8da492..76381b85b 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -338,7 +338,10 @@ x{A939} @Defop(8u+1) MODPOW2R# x{A93A} @Defop(8u+1) MODPOW2C# x{A984} @Defop MULDIV x{A985} @Defop MULDIVR +x{A988} @Defop MULMOD x{A98C} @Defop MULDIVMOD +x{A98D} @Defop MULDIVMODR +x{A98E} @Defop MULDIVMODC x{A9A4} @Defop MULRSHIFT x{A9A5} @Defop MULRSHIFTR x{A9A6} @Defop MULRSHIFTC diff --git a/crypto/test/modbigint.cpp b/crypto/test/modbigint.cpp index c73bdc0c5..851f0f9be 100644 --- a/crypto/test/modbigint.cpp +++ b/crypto/test/modbigint.cpp @@ -28,17 +28,15 @@ enum { mod_cnt = 32 }; // mod_cnt = 9 => integers -2^268 .. 2^268 // mod_cnt = 18 => integers -2^537 .. 2^537 // mod_cnt = 32 => integers -2^955 .. 2^955 -constexpr int mod[mod_cnt] = { -999999937, 999999929, 999999893, 999999883, 999999797, 999999761, 999999757, 999999751, -999999739, 999999733, 999999677, 999999667, 999999613, 999999607, 999999599, 999999587, -999999541, 999999527, 999999503, 999999491, 999999487, 999999433, 999999391, 999999353, -99999337, 999999323, 999999229, 999999223, 999999197, 999999193, 999999191, 999999181 -}; +constexpr int mod[mod_cnt] = {999999937, 999999929, 999999893, 999999883, 999999797, 999999761, 999999757, 999999751, + 999999739, 999999733, 999999677, 999999667, 999999613, 999999607, 999999599, 999999587, + 999999541, 999999527, 999999503, 999999491, 999999487, 999999433, 999999391, 999999353, + 99999337, 999999323, 999999229, 999999223, 999999197, 999999193, 999999191, 999999181}; // invm[i][j] = mod[i]^(-1) modulo mod[j] int invm[mod_cnt][mod_cnt]; -int gcdx (int a, int b, int& u, int& v); +int gcdx(int a, int b, int& u, int& v); template struct ModArray; @@ -61,22 +59,46 @@ std::ostream& raw_dump_array(std::ostream& os, const std::array& arr) { template struct MixedRadix { enum { n = N }; - std::array a; - MixedRadix(int v) { set_int(v); } + int a[N]; + MixedRadix(int v) { + set_int(v); + } MixedRadix() = default; MixedRadix(const MixedRadix&) = default; - MixedRadix(std::initializer_list l) : a(l) {} - MixedRadix(const std::array& arr) : a(arr) {} + MixedRadix(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + MixedRadix(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } template - MixedRadix(const MixedRadix& other) { assert (M >= N); memcpy(a.data(), other.a.data(), N * sizeof(int)); } + MixedRadix(const MixedRadix& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } MixedRadix(const ModArray& other); MixedRadix(const ModArray& other, bool sgnd); - - MixedRadix& set_zero() { a.fill(0); return *this; } - MixedRadix& set_one() { a.fill(0); a[0] = 1; return *this; } - - MixedRadix& set_int(int v) { a.fill(0); a[0] = v; return *this; } - MixedRadix copy() const { return MixedRadix{*this}; } + + MixedRadix& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + MixedRadix& set_one() { + a[0] = 1; + std::fill(a + 1, a + N, 0); + return *this; + } + MixedRadix& set_int(int v) { + a[0] = v; + std::fill(a + 1, a + N, 0); + return *this; + } + + MixedRadix copy() const { + return MixedRadix{*this}; + } static const int* mod_array() { return mod; @@ -88,13 +110,17 @@ struct MixedRadix { int sgn() const { int i = N - 1; - while (i >= 0 && !a[i]) { --i; } + while (i >= 0 && !a[i]) { + --i; + } return i < 0 ? 0 : (a[i] > 0 ? 1 : -1); } - + int cmp(const MixedRadix& other) const { int i = N - 1; - while (i >= 0 && a[i] == other.a[i]) { --i; } + while (i >= 0 && a[i] == other.a[i]) { + --i; + } return i < 0 ? 0 : (a[i] > other.a[i] ? 1 : -1); } @@ -103,11 +129,11 @@ struct MixedRadix { } bool operator==(const MixedRadix& other) const { - return a == other.a; + return std::equal(a, a + N, other.a); } - + bool operator!=(const MixedRadix& other) const { - return a != other.a; + return !std::equal(a, a + N, other.a); } bool operator<(const MixedRadix& other) const { @@ -126,9 +152,19 @@ struct MixedRadix { return cmp(other) >= 0; } + explicit operator bool() const { + return sgn(); + } + + bool operator!() const { + return !sgn(); + } + MixedRadix& negate() { int i = 0; - while (i < N - 1 && !a[i]) { i++; } + while (i < N - 1 && !a[i]) { + i++; + } a[i]--; for (; i < N; i++) { a[i] = mod[i] - a[i] - 1; @@ -136,12 +172,12 @@ struct MixedRadix { a[N - 1] -= mod[N - 1]; return *this; } - + static const MixedRadix& pow2(int power); static MixedRadix negpow2(int power) { return -pow2(power); } - + template const MixedRadix& as_shorter() const { static_assert(M <= N); @@ -154,10 +190,10 @@ struct MixedRadix { } for (int i = 0; i < N; i++) { if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } for (int j = i + 1; j < N; j++) { - a[j] = (int)((long long)(a[j] - a[i]) * invm[i][j] % mod[j]); + a[j] = (int)((long long)(a[j] - a[i]) * invm[i][j] % mod[j]); } } if (sgnd && a[N - 1] > (mod[N - 1] >> 1)) { @@ -165,13 +201,13 @@ struct MixedRadix { } return *this; } - + MixedRadix& operator=(const MixedRadix&) = default; - + template MixedRadix& operator=(const MixedRadix& other) { - static_assert (M >= N); - memcpy (a.data(), other.a.data(), N * sizeof (int)); + static_assert(M >= N); + std::copy(other.a, other.a + N, a); } MixedRadix& import_mod_array(const ModArray& other, bool sgnd = true); @@ -179,7 +215,7 @@ struct MixedRadix { MixedRadix& operator=(const ModArray& other) { return import_mod_array(other); } - + MixedRadix& set_sum(const MixedRadix& x, const MixedRadix& y, int factor = 1) { long long carry = 0; for (int i = 0; i < N; i++) { @@ -187,8 +223,8 @@ struct MixedRadix { carry = acc / mod[i]; a[i] = (int)(acc - carry * mod[i]); if (a[i] < 0) { - a[i] += mod[i]; - --carry; + a[i] += mod[i]; + --carry; } } if (a[N - 1] >= 0 && carry == -1) { @@ -196,28 +232,28 @@ struct MixedRadix { } return *this; } - + MixedRadix& operator+=(const MixedRadix& other) { return set_sum(*this, other); } - + MixedRadix& operator-=(const MixedRadix& other) { return set_sum(*this, other, -1); } - + static const MixedRadix& zero(); static const MixedRadix& one(); - + MixedRadix& operator*=(int factor) { return set_sum(zero(), *this, factor); } - + MixedRadix operator-() const { MixedRadix copy{*this}; copy.negate(); return copy; } - + MixedRadix operator+(const MixedRadix& other) const { MixedRadix res; res.set_sum(*this, other); @@ -243,7 +279,7 @@ struct MixedRadix { } return ((x ^ b) < 0 && x) ? x + b : x; } - + explicit operator double() const { double acc = 0.; for (int i = N - 1; i >= 0; --i) { @@ -262,7 +298,9 @@ struct MixedRadix { MixedRadix& to_base(int base) { int k = N - 1; - while (k > 0 && !a[k]) { --k; } + while (k > 0 && !a[k]) { + --k; + } if (k <= 0) { return *this; } @@ -270,15 +308,15 @@ struct MixedRadix { // a[i..k] := a[i+1..k] * mod[i] + a[i] long long carry = a[i]; for (int j = i; j < k; j++) { - long long t = (long long)a[j + 1] * mod[i] + carry; - carry = t / base; - a[j] = (int)(t - carry * base); + long long t = (long long)a[j + 1] * mod[i] + carry; + carry = t / base; + a[j] = (int)(t - carry * base); } a[k] = (int)carry; } return *this; } - + std::ostream& print_dec_destroy(std::ostream& os) { int s = sgn(); if (s < 0) { @@ -290,16 +328,18 @@ struct MixedRadix { } to_base(1000000000); int i = N - 1; - while (!a[i] && i > 0) { --i; } + while (!a[i] && i > 0) { + --i; + } os << a[i]; while (--i >= 0) { char buff[12]; - sprintf (buff, "%09d", a[i]); + sprintf(buff, "%09d", a[i]); os << buff; } return os; } - + std::ostream& print_dec(std::ostream& os) const& { MixedRadix copy{*this}; return copy.print_dec_destroy(os); @@ -314,22 +354,22 @@ struct MixedRadix { print_dec_destroy(os); return std::move(os).str(); } - + std::string to_dec_string() const& { MixedRadix copy{*this}; return copy.to_dec_string_destroy(); } - + std::string to_dec_string() && { return to_dec_string_destroy(); } - bool to_binary_destroy(unsigned char *arr, int size, bool sgnd = true) { + bool to_binary_destroy(unsigned char* arr, int size, bool sgnd = true) { if (size <= 0) { return false; } int s = (sgnd ? sgn() : 1); - memset (arr, 0, size); + memset(arr, 0, size); if (s < 0) { negate(); } else if (!s) { @@ -340,14 +380,14 @@ struct MixedRadix { int bits = 0, j = size; for (int i = 0; i < N; i++) { if (!j && a[i]) { - return false; + return false; } - acc += ((long long) a[i] << bits); + acc += ((long long)a[i] << bits); bits += 30; while (bits >= 8 && j > 0) { - arr[--j] = (unsigned char)(acc & 0xff); - bits -= 8; - acc >>= 8; + arr[--j] = (unsigned char)(acc & 0xff); + bits -= 8; + acc >>= 8; } } while (j > 0) { @@ -364,26 +404,30 @@ struct MixedRadix { return arr[0] <= 0x7f; } j = size - 1; - while (j >= 0 && !arr[j]) { --j; } - assert (j >= 0); + while (j >= 0 && !arr[j]) { + --j; + } + assert(j >= 0); arr[j] = (unsigned char)(-arr[j]); - while (--j >= 0) { arr[j] = (unsigned char)~arr[j]; } + while (--j >= 0) { + arr[j] = (unsigned char)~arr[j]; + } return arr[0] >= 0x80; } - bool to_binary(unsigned char *arr, int size, bool sgnd = true) const& { + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const& { MixedRadix copy{*this}; return copy.to_binary_destroy(arr, size, sgnd); } - bool to_binary(unsigned char *arr, int size, bool sgnd = true) && { + bool to_binary(unsigned char* arr, int size, bool sgnd = true) && { return to_binary_destroy(arr, size, sgnd); } std::ostream& raw_dump(std::ostream& os) const { return raw_dump_array(os, a); } - + ArrayRawDumpRef dump() const { return {a}; } @@ -392,28 +436,52 @@ struct MixedRadix { template struct ModArray { enum { n = N }; - std::array a; - ModArray(int v) { set_int(v); } - ModArray(long long v) { set_long(v); } - ModArray(long v) { set_long(v); } + int a[N]; + ModArray(int v) { + set_int(v); + } + ModArray(long long v) { + set_long(v); + } + ModArray(long v) { + set_long(v); + } ModArray() = default; ModArray(const ModArray&) = default; - ModArray(std::initializer_list l) : a(l) {} - ModArray(const std::array& arr) : a(arr) {} + ModArray(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + ModArray(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } template - ModArray(const ModArray& other) { static_assert (M >= N); memcpy(a.data(), other.a.data(), N * sizeof(int)); } - ModArray(const int* p) : a(p) {} - ModArray(std::string str) { assert(from_dec_string(str) && "not a decimal number"); } + ModArray(const ModArray& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + ModArray(const int* p) : a(p) { + } + ModArray(std::string str) { + assert(from_dec_string(str) && "not a decimal number"); + } - ModArray& set_zero() { a.fill(0); return *this; } - ModArray& set_one() { a.fill(1); return *this; } + ModArray& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + ModArray& set_one() { + std::fill(a, a + N, 1); + return *this; + } ModArray& set_int(int v) { if (v >= 0) { - a.fill(v); + std::fill(a, a + N, v); } else { for (int i = 0; i < N; i++) { - a[i] = mod[i] + v; + a[i] = mod[i] + v; } } return *this; @@ -423,12 +491,12 @@ struct ModArray { for (int i = 0; i < N; i++) { a[i] = v % mod[i]; if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return *this; } - + ModArray copy() const { return ModArray{*this}; } @@ -445,11 +513,11 @@ struct ModArray { static const ModArray& one(); ModArray& operator=(const ModArray&) = default; - + template ModArray& operator=(const ModArray& other) { - assert (M >= N); - memcpy (a.data(), other.a.data(), N * sizeof (int)); + static_assert(M >= N); + std::copy(other.a, other.a + N, a); return *this; } @@ -459,39 +527,83 @@ struct ModArray { } return *this; } - + ModArray& norm_neg() { for (int i = 0; i < N; i++) { if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return *this; } - + ModArray& normalize() { for (int i = 0; i < N; i++) { a[i] %= mod[i]; if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return *this; } - + + bool is_zero() const { + for (int i = 0; i < N; i++) { + if (a[i]) { + return false; + } + } + return true; + } + + explicit operator bool() const { + return !is_zero(); + } + + bool operator!() const { + return is_zero(); + } + bool operator==(const ModArray& other) const { - return a == other.a; + return std::equal(a, a + N, other.a); } - + bool operator!=(const ModArray& other) const { - return a != other.a; + return !std::equal(a, a + N, other.a); + } + + bool operator==(long long val) const { + for (int i = 0; i < N; i++) { + int r = (int)(val % mod[i]); + if (a[i] != (r < 0 ? r + mod[i] : r)) { + return false; + } + } + return true; + } + + bool operator!=(long long val) const { + return !operator==(val); + } + + long long try_get_long() const { + return (long long)(MixedRadix<3>(*this)); + } + + bool fits_long() const { + return operator==(try_get_long()); + } + + explicit operator long long() const { + auto v = try_get_long(); + return operator==(v) ? v : -0x8000000000000000; } ModArray& set_sum(const ModArray& x, const ModArray& y) { for (int i = 0; i < N; i++) { a[i] = x.a[i] + y.a[i]; if (a[i] >= mod[i]) { - a[i] -= mod[i]; + a[i] -= mod[i]; } } return *this; @@ -501,7 +613,7 @@ struct ModArray { for (int i = 0; i < N; i++) { a[i] += other.a[i]; if (a[i] >= mod[i]) { - a[i] -= mod[i]; + a[i] -= mod[i]; } } return *this; @@ -511,7 +623,7 @@ struct ModArray { for (int i = 0; i < N; i++) { a[i] = (int)((a[i] + v) % mod[i]); if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return *this; @@ -521,7 +633,7 @@ struct ModArray { for (int i = 0; i < N; i++) { a[i] -= other.a[i]; if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return *this; @@ -530,7 +642,7 @@ struct ModArray { ModArray& operator-=(long long v) { return (operator+=)(-v); } - + ModArray& mul_arr(const int other[]) { for (int i = 0; i < N; i++) { a[i] = (int)(((long long)a[i] * other[i]) % mod[i]); @@ -539,144 +651,170 @@ struct ModArray { } ModArray& operator*=(const ModArray& other) { - return mul_arr(other.a.data()); + return mul_arr(other.a); } - + template ModArray& operator*=(const ModArray& other) { - assert (M >= N); - return mul_arr(other.a.data()); + static_assert(M >= N); + return mul_arr(other.a); } ModArray& operator*=(int v) { for (int i = 0; i < N; i++) { - a[i] = ((long long)a[i] * v) % mod[i]; + a[i] = (int)(((long long)a[i] * v) % mod[i]); } return (v >= 0 ? *this : norm_neg()); } - + ModArray& operator*=(long long v) { for (int i = 0; i < N; i++) { a[i] = (int)(((long long)a[i] * (v % mod[i])) % mod[i]); } return (v >= 0 ? *this : norm_neg()); } - + ModArray& mul_add(int v, long long w) { for (int i = 0; i < N; i++) { a[i] = (int)(((long long)a[i] * v + w) % mod[i]); if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return *this; } - + + // *this = (*this * other) + w ModArray& mul_add(const ModArray& other, long long w) { for (int i = 0; i < N; i++) { a[i] = (int)(((long long)a[i] * other.a[i] + w) % mod[i]); if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return *this; } + // *this = (*this << shift) + w ModArray& lshift_add(int shift, long long w) { return mul_add(pow2(shift), w); } + // *this = *this + other * w + ModArray& add_mul(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + other.a[i] * w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this += w << shift + ModArray& add_lshift(int shift, long long w) { + return add_mul(pow2(shift), w); + } + ModArray operator+(const ModArray& other) const { ModArray copy{*this}; copy += other; return copy; } - + ModArray operator-(const ModArray& other) const { ModArray copy{*this}; copy -= other; return copy; } - + ModArray operator+(long long other) const { ModArray copy{*this}; copy += other; return copy; } - + ModArray operator-(long long other) const { ModArray copy{*this}; copy += -other; return copy; } - + ModArray operator-() const { ModArray copy{*this}; copy.negate(); return copy; } - + ModArray operator*(const ModArray& other) const { ModArray copy{*this}; copy *= other; return copy; } - + ModArray operator*(long long other) const { ModArray copy{*this}; copy *= other; return copy; } - + bool invert() { for (int i = 0; i < N; i++) { int t; if (gcdx(a[i], mod[i], a[i], t) != 1) { - return false; + return false; } if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return true; } - + bool try_divide(const ModArray& other) { for (int i = 0; i < N; i++) { int q, t; if (gcdx(other.a[i], mod[i], q, t) != 1) { - return false; + return false; } - a[i] = (long long)a[i] * q % mod[i]; + a[i] = (int)((long long)a[i] * q % mod[i]); if (a[i] < 0) { - a[i] += mod[i]; + a[i] += mod[i]; } } return true; } ModArray& operator/=(const ModArray& other) { - assert (try_divide(other) && "division by zero?"); + assert(try_divide(other) && "division by zero?"); return *this; } - + ModArray operator/(const ModArray& other) { ModArray copy{*this}; copy /= other; return copy; } - - ModArray& operator<<=(int lshift); - ModArray operator<<(int lshift) const { - ModArray copy{*this}; - copy <<= lshift; - return copy; - } - static const ModArray& pow2(int power); static const ModArray& negpow2(int power); - + + ModArray& operator<<=(int lshift) { + return operator*=(pow2(lshift)); + } + + ModArray operator<<(int lshift) const { + return operator*(pow2(lshift)); + } + + ModArray& operator>>=(int rshift) { + return operator/=(pow2(rshift)); + } + + ModArray operator>>(int rshift) const { + return operator/(pow2(rshift)); + } + template const ModArray& as_shorter() const { static_assert(M <= N); @@ -684,21 +822,21 @@ struct ModArray { } MixedRadix& to_mixed_radix(MixedRadix& dest, bool sgnd = true) const { - return dest.import_mod_array(a.data(), sgnd); + return dest.import_mod_array(a, sgnd); } - + MixedRadix to_mixed_radix(bool sgnd = true) const { return MixedRadix(*this, sgnd); } - + int operator%(int div) const { return to_mixed_radix() % div; } - + explicit operator double() const { return (double)to_mixed_radix(); } - + std::string to_dec_string() const { return MixedRadix(*this).to_dec_string(); } @@ -706,17 +844,17 @@ struct ModArray { std::ostream& print_dec(std::ostream& os, bool sgnd = true) const { return MixedRadix(*this, sgnd).print_dec(os); } - - bool to_binary(unsigned char *arr, int size, bool sgnd = true) const { + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const { return MixedRadix(*this, sgnd).to_binary(arr, size, sgnd); } - + template bool to_binary(std::array& arr, bool sgnd = true) const { return to_binary(arr.data(), M, sgnd); } - bool from_dec_string(const char *start, const char *end) { + bool from_dec_string(const char* start, const char* end) { set_zero(); if (start >= end) { return false; @@ -728,14 +866,14 @@ struct ModArray { int acc = 0, pow = 1; while (start < end) { if (*start < '0' || *start > '9') { - return false; + return false; } acc = acc * 10 + (*start++ - '0'); pow *= 10; if (pow >= 1000000000) { - mul_add(pow, acc); - pow = 1; - acc = 0; + mul_add(pow, acc); + pow = 1; + acc = 0; } } if (pow > 1) { @@ -746,12 +884,12 @@ struct ModArray { } return true; } - + bool from_dec_string(std::string str) { return from_dec_string(str.data(), str.data() + str.size()); } - - ModArray& from_binary(const unsigned char *arr, int size, bool sgnd = true) { + + ModArray& from_binary(const unsigned char* arr, int size, bool sgnd = true) { set_zero(); if (size <= 0) { return *this; @@ -765,8 +903,8 @@ struct ModArray { pow += 8; acc = (acc << 8) + arr[i]; if (pow >= 56) { - lshift_add(pow, acc); - acc = pow = 0; + lshift_add(pow, acc); + acc = pow = 0; } } if (pow || acc) { @@ -774,16 +912,16 @@ struct ModArray { } return *this; } - + template ModArray& from_binary(const std::array& arr, bool sgnd = true) { return from_binary(arr.data(), M, sgnd); } - + std::ostream& raw_dump(std::ostream& os) const { return raw_dump_array(os, a); } - + ArrayRawDumpRef dump() const { return {a}; } @@ -791,17 +929,17 @@ struct ModArray { template MixedRadix::MixedRadix(const ModArray& other) { - import_mod_array(other.a.data()); + import_mod_array(other.a); } template MixedRadix::MixedRadix(const ModArray& other, bool sgnd) { - import_mod_array(other.a.data(), sgnd); + import_mod_array(other.a, sgnd); } template MixedRadix& MixedRadix::import_mod_array(const ModArray& other, bool sgnd) { - return import_mod_array(other.a.data(), sgnd); + return import_mod_array(other.a, sgnd); } template @@ -822,7 +960,7 @@ std::ostream& operator<<(std::ostream& os, MixedRadix&& x) { template struct ArrayRawDumpRef { const std::array& ref; - ArrayRawDumpRef(const std::array& _ref) : ref(_ref) {}; + ArrayRawDumpRef(const std::array& _ref) : ref(_ref){}; }; template @@ -835,12 +973,6 @@ constexpr int pow2_cnt = 1001; ModArray Zero(0), One(1), Pow2[pow2_cnt], NegPow2[pow2_cnt]; MixedRadix Zero_mr(0), One_mr(1), Pow2_mr[pow2_cnt], NegPow2_mr[pow2_cnt]; -template -ModArray& ModArray::operator<<=(int lshift) { - assert (lshift >= 0 && lshift < pow2_cnt); - return (operator*=)(Pow2[lshift]); -} - template const MixedRadix& MixedRadix::pow2(int power) { return Pow2_mr[power].as_shorter(); @@ -896,28 +1028,35 @@ void init_pow2() { } } -int gcdx (int a, int b, int& u, int& v) { +int gcdx(int a, int b, int& u, int& v) { int a1 = 1, a2 = 0, b1 = 0, b2 = 1; while (b) { int q = a / b; - int t = a - q * b; a = b; b = t; - t = a1 - q * b1; a1 = b1; b1 = t; - t = a2 - q * b2; a2 = b2; b2 = t; - } - u = a1; v = a2; + int t = a - q * b; + a = b; + b = t; + t = a1 - q * b1; + a1 = b1; + b1 = t; + t = a2 - q * b2; + a2 = b2; + b2 = t; + } + u = a1; + v = a2; return a; } void init_invm() { for (int i = 0; i < mod_cnt; i++) { - assert (mod[i] > 0 && mod[i] <= (1 << 30)); + assert(mod[i] > 0 && mod[i] <= (1 << 30)); for (int j = 0; j < i; j++) { - assert (gcdx (mod[i], mod[j], invm[i][j], invm[j][i]) == 1); + assert(gcdx(mod[i], mod[j], invm[i][j], invm[j][i]) == 1); if (invm[i][j] < 0) { - invm[i][j] += mod[j]; + invm[i][j] += mod[j]; } if (invm[j][i] < 0) { - invm[j][i] += mod[i]; + invm[j][i] += mod[i]; } } } @@ -928,4 +1067,4 @@ void init() { init_pow2(); } -} // namespace modint +} // namespace modint diff --git a/crypto/test/test-bigint.cpp b/crypto/test/test-bigint.cpp index 94c6ed07e..bf85e2ad0 100644 --- a/crypto/test/test-bigint.cpp +++ b/crypto/test/test-bigint.cpp @@ -16,10 +16,12 @@ */ #include #include +#include #include #include #include #include +#include #include "common/refcnt.hpp" #include "common/bigint.hpp" #include "common/refint.h" @@ -27,13 +29,30 @@ #include "td/utils/tests.h" -using BInt = modint::ModArray<18>; // integers up to 2^537 +int mkint_chk_mode = -1, res_chk_mode = 0; +long long iterations = 100000, cur_iteration = -1, debug_iteration = -2; +#define IFDEBUG if (cur_iteration == debug_iteration || debug_iteration == -3) + +using BInt = modint::ModArray<18>; // integers up to 2^537 using MRInt = modint::MixedRadix<18>; // auxiliary integer representation for printing, comparing etc MRInt p2_256, np2_256, p2_63, np2_63; constexpr long long ll_min = -2 * (1LL << 62), ll_max = ~ll_min; +constexpr double dbl_pow256 = 1.1579208923731619542e77 /* 0x1p256 */; // 2^256 -int mkint_chk_mode = -1, res_chk_mode = 0; +std::mt19937_64 Random(666); + +template +bool equal(td::RefInt256 x, T y) { + return !td::cmp(x, y); +} + +bool equal_or_nan(td::RefInt256 x, td::RefInt256 y) { + return equal(x, y) || (!x->is_valid() && !y->fits_bits(257)) || (!y->is_valid() && !x->fits_bits(257)); +} + +#define CHECK_EQ(__x, __y) CHECK(equal(__x, __y)) +#define CHECK_EQ_NAN(__x, __y) CHECK(equal_or_nan(__x, __y)) bool mr_in_range(const MRInt& x) { return x < p2_256 && x >= np2_256; @@ -55,17 +74,53 @@ bool mr_ufits_bits(const MRInt& x, int bits) { return bits >= 0 && x.sgn() >= 0 && x < MRInt::pow2(bits); } -bool extract_value_any_bool(BInt& val, const td::AnyIntView& x, bool chk_norm = true) { +struct ShowBin { + unsigned char* data; + ShowBin(unsigned char _data[64]) : data(_data) { + } +}; + +std::ostream& operator<<(std::ostream& os, ShowBin bin) { + int i = 0, s = bin.data[0]; + if (s == 0 || s == 0xff) { + while (i < 64 && bin.data[i] == s) { + i++; + } + } + if (i >= 3) { + os << (s ? "ff..ff" : "00..00"); + } else { + i = 0; + } + constexpr static char hex_digits[] = "0123456789abcdef"; + while (i < 64) { + int t = bin.data[i++]; + os << hex_digits[t >> 4] << hex_digits[t & 15]; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const td::AnyIntView& x) { + os << '['; + for (int i = 0; i < x.size(); i++) { + os << ' ' << x.digits[i]; + } + os << " ]"; + return os; +} + +template +bool extract_value_any_bool(BInt& val, const td::AnyIntView& x, bool chk_norm = true) { int n = x.size(); if (n <= 0 || n > x.max_size() || (!x.digits[n - 1] && n > 1)) { return false; } - assert (n == 1 || x.digits[n - 1] != 0); + assert(n == 1 || x.digits[n - 1] != 0); val.set_zero(); for (int i = n - 1; i >= 0; --i) { - val.lshift_add(td::BigIntInfo::word_shift, x.digits[i]); - if (chk_norm && (x.digits[i] < - td::BigIntInfo::Half || x.digits[i] >= td::BigIntInfo::Half)) { - return false; // unnormalized + val.lshift_add(T::word_shift, x.digits[i]); + if (chk_norm && (x.digits[i] < -T::Half || x.digits[i] >= T::Half)) { + return false; // unnormalized } } return true; @@ -90,12 +145,12 @@ BInt extract_value(const T& x, bool chk_norm = true) { template BInt extract_value_alt(const T& x) { BInt res; - const int *md = res.mod_array(); + const int* md = res.mod_array(); for (int i = 0; i < res.n / 2; i++) { T copy{x}; int m1 = md[2 * i], m2 = md[2 * i + 1]; long long rem = copy.divmod_short((long long)m1 * m2); - res.a[2 * i] = (int)(rem % m1); + res.a[2 * i] = (int)(rem % m1); res.a[2 * i + 1] = (int)(rem % m2); } if (res.n & 1) { @@ -108,15 +163,20 @@ BInt extract_value_alt(const T& x) { constexpr int min_spec_int = -0xfd08, max_spec_int = 0xfd07; // x = sgn*(ord*256+a*16+b) => sgn*((32+a)*2^(ord-2) + b - 8) // x = -0xfd08 => -2^256 ... x = 0xfd07 => 2^256 - 1 -td::RefInt256 make_special_int(int x, BInt *ptr = nullptr, unsigned char bin[64] = nullptr) { +td::RefInt256 make_special_int(int x, BInt* ptr = nullptr, unsigned char bin[64] = nullptr) { bool sgn = (x < 0); - if (sgn) { x = -x; } + if (sgn) { + x = -x; + } int ord = (x >> 8) - 2, a = 32 + ((x >> 4) & 15), b = (x & 15) - 8; if (ord < 0) { a >>= -ord; ord = 0; } - if (sgn) { a = -a; b = -b; } + if (sgn) { + a = -a; + b = -b; + } if (ptr) { ptr->set_int(a); *ptr <<= ord; @@ -126,8 +186,8 @@ td::RefInt256 make_special_int(int x, BInt *ptr = nullptr, unsigned char bin[64] int acc = b, r = ord; for (int i = 63; i >= 0; --i) { if (r < 8) { - acc += (a << r); - r = 1024; + acc += (a << r); + r = 1024; } r -= 8; bin[i] = (unsigned char)(acc & 0xff); @@ -137,7 +197,103 @@ td::RefInt256 make_special_int(int x, BInt *ptr = nullptr, unsigned char bin[64] return (td::make_refint(a) << ord) + b; } -void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt *valptr = nullptr, const unsigned char bin[64] = nullptr) { +int rand_int(int min, int max) { + return min + (int)(Random() % (max - min + 1)); +} + +unsigned randu() { + return (unsigned)(Random() << 16); +} + +bool coin() { + return Random() & (1 << 28); +} + +// returns 0 with probability 1/2, 1 with prob. 1/4, ..., k with prob. 1/2^(k+1) +int randexp(int max = 63, int min = 0) { + return min + __builtin_clzll(Random() | (1ULL << (63 - max + min))); +} + +void bin_add_small(unsigned char bin[64], long long val, int shift = 0) { + val <<= shift & 7; + for (int i = 63 - (shift >> 3); i >= 0 && val; --i) { + val += bin[i]; + bin[i] = (unsigned char)val; + val >>= 8; + } +} + +// adds sgn * (random number less than 2^(ord - ord2)) * 2^ord2 +td::RefInt256 add_random_bits(td::RefInt256 x, BInt& val, unsigned char bin[64], int ord2, int ord, int sgn = 1) { + int t; + do { + t = std::max((ord - 1) & -16, ord2); + int a = sgn * rand_int(0, (1 << (ord - t)) - 1); + // add a << t + val.add_lshift(t, a); + x += td::make_refint(a) << t; + bin_add_small(bin, a, t); + ord = t; + } while (t > ord2); + return x; +} + +// generates a random integer in range -2^256 .. 2^256-1 (and sometimes outside) +// distribution is skewed towards +/- 2^n +/- 2^n +/- smallint, but completely random integers are also generated +td::RefInt256 make_random_int0(BInt& val, unsigned char bin[64]) { + memset(bin, 0, 64); + int ord = rand_int(-257, 257); + if (ord <= 2 && ord >= -2) { + // -2..2 represent themselves + val.set_int(ord); + bin_add_small(bin, ord); + return td::make_refint(ord); + } + int sgn = (ord < 0 ? -1 : 1); + ord = sgn * ord - 1; + int f = std::min(ord, randexp(15)), a = sgn * rand_int(1 << f, (2 << f) - 1); + ord -= f; + // first summand is a << ord + auto res = td::make_refint(a) << ord; + val.set_int(a); + val <<= ord; + bin_add_small(bin, a, ord); + if (!ord) { + // all bits ready + return res; + } + for (int s = 0; s < 2 && ord; s++) { + // decide whether we want an intermediate order (50%), and whether we want randomness above/below that order + int ord2 = (s ? 0 : std::max(0, rand_int(~ord, ord - 1))); + if (!rand_int(0, 4)) { // 20% + // random bits between ord2 and ord + res = add_random_bits(std::move(res), val, bin, ord2, ord, sgn); + } + if (rand_int(0, 4)) { // 80% + // non-zero adjustment + f = randexp(15); + a = rand_int(-(2 << f) + 1, (2 << f) - 1); + ord = std::max(ord2 - f, 0); + // add a << ord + val.add_lshift(ord, a); + res += (td::make_refint(a) << ord); + bin_add_small(bin, a, ord); + } + } + return res; +} + +td::RefInt256 make_random_int(BInt& val, unsigned char bin[64]) { + while (true) { + auto res = make_random_int0(val, bin); + if (res->fits_bits(257)) { + return res; + } + } +} + +void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt* valptr = nullptr, + const unsigned char bin[64] = nullptr) { CHECK(x.not_null() && (in_range <= -2 || x->is_valid())); if (!x->is_valid()) { // not much to check when x is a NaN @@ -146,23 +302,24 @@ void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt *val // check that the true answer at `valptr` is out of range CHECK(!mr_in_range(valptr->to_mixed_radix())); if (mode & 0x200) { - // check BInt binary export - valptr->to_binary(bytes, 64); - if (bin) { - // check that the two true answers match - CHECK(!memcmp(bin, bytes, 64)); - } else { - bin = bytes; - } + // check BInt binary export + valptr->to_binary(bytes, 64); + if (bin) { + // check that the two true answers match + CHECK(!memcmp(bin, bytes, 64)); + } else { + bin = bytes; + } } } if (bin) { // check that the true answer in `bin` is out of range int i = 0, sgn = (bin[0] >= 0x80 ? -1 : 0); - while (i < 32 && bin[i] == (unsigned char)sgn); + while (i < 32 && bin[i] == (unsigned char)sgn) + ; CHECK(i < 32); if (valptr && (mode & 0x100)) { - // check BInt binary export + // check BInt binary export BInt val2; val2.from_binary(bin, 64); CHECK(*valptr == val2); @@ -177,18 +334,16 @@ void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt *val } BInt val = extract_value(*x); if (valptr) { + if (val != *valptr) { + std::cerr << "extracted " << val << " from " << x << ' ' << x->as_any_int() << ", expected " << *valptr + << std::endl; + } CHECK(val == *valptr); } if (mode & 1) { BInt val2 = extract_value_alt(*x); CHECK(val == val2); } - MRInt mval(val); - bool val_in_range = mr_in_range(mval); - CHECK(x->fits_bits(257) == val_in_range); - if (in_range >= 0) { - CHECK((int)val_in_range == in_range); - } if (mode & 2) { // check binary import td::BigInt256 y; @@ -201,37 +356,56 @@ void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt *val val2.from_binary(bytes, 64); CHECK(val == val2); } - if (mode & 0x200) { - // check binary export for BInt - unsigned char bytes2[64]; - mval.to_binary(bytes2, 64); - CHECK(!memcmp(bytes, bytes2, 64)); - } - // check sign - int sgn = mval.sgn(); - CHECK(x->sgn() == sgn); // check if small (fits into 64 bits) - bool is_small = mr_is_small(mval); + long long xval = (long long)val; + bool is_small = (xval != ll_min || val == xval); CHECK(is_small == x->fits_bits(64)); if (is_small) { // special check for small (64-bit) values - long long xval = (long long)mval; CHECK(x->to_long() == xval); - CHECK((long long)__builtin_bswap64(*(long long *)(bytes + 64 - 8)) == xval); + CHECK((long long)__builtin_bswap64(*(long long*)(bytes + 64 - 8)) == xval); + CHECK(in_range); + // check sign + CHECK(x->sgn() == (xval > 0 ? 1 : (xval < 0 ? -1 : 0))); // check comparison with long long CHECK(x == xval); CHECK(!cmp(x, xval)); - // check constructor from long long - CHECK(!cmp(x, td::make_refint(xval))); - if (xval != ll_min) { - CHECK(x > xval - 1); - CHECK(x > td::make_refint(xval - 1)); + if (mode & 4) { + // check constructor from long long + CHECK(!cmp(x, td::make_refint(xval))); + if (xval != ll_min) { + CHECK(x > xval - 1); + CHECK(x > td::make_refint(xval - 1)); + } + if (xval != ll_max) { + CHECK(x < xval + 1); + CHECK(x < td::make_refint(xval + 1)); + } } - if (xval != ll_max) { - CHECK(x < xval + 1); - CHECK(x < td::make_refint(xval + 1)); + if (!(mode & ~0x107)) { + return; // fast check for small ints in this case } } + + MRInt mval(val); // somewhat slow + bool val_in_range = mr_in_range(mval); + CHECK(x->fits_bits(257) == val_in_range); + if (in_range >= 0) { + CHECK((int)val_in_range == in_range); + } + if (mode & 0x200) { + // check binary export for BInt + unsigned char bytes2[64]; + mval.to_binary(bytes2, 64); + CHECK(!memcmp(bytes, bytes2, 64)); + } + // check sign + int sgn = mval.sgn(); + CHECK(x->sgn() == sgn); + CHECK(is_small == mr_is_small(mval)); + if (is_small) { + CHECK((long long)mval == xval); + } if (mode & 0x10) { // check decimal export std::string dec = mval.to_dec_string(); @@ -263,13 +437,6 @@ void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt *val } } -void check_special_int(int idx) { - BInt b; - unsigned char binary[64]; - td::RefInt256 x = make_special_int(idx, &b, binary); - check_one_int_repr(x, mkint_chk_mode, idx >= min_spec_int && idx <= max_spec_int, &b, binary); -} - void init_aux() { np2_256 = p2_256 = MRInt::pow2(256); np2_256.negate(); @@ -283,7 +450,7 @@ std::vector SpecInt; BInt SpecIntB[max_spec_int - min_spec_int + 1]; void init_check_special_ints() { - std::cerr << "check_special_ints" << std::endl; + std::cerr << "check special ints" << std::endl; BInt b; unsigned char binary[64]; for (int idx = min_spec_int - 512; idx <= max_spec_int + 512; idx++) { @@ -304,7 +471,7 @@ void check_unary_ops_on(td::RefInt256 x, const BInt& xv) { // NEGATE BInt yv = -xv; check_res(-x, yv); - // NOT + // NOT check_res(~x, yv -= 1); } @@ -335,11 +502,10 @@ void check_pow2_ops() { } } -void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv) { +void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv, const MRInt& mval) { // LSHIFT check_res(x << shift, xv << shift); // FITS - MRInt mval(xv); CHECK(x->fits_bits(shift) == mr_fits_bits(mval, shift)); // UFITS CHECK(x->unsigned_fits_bits(shift) == mr_ufits_bits(mval, shift)); @@ -352,7 +518,7 @@ void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv) { check_res(std::move(y), xv - BInt::pow2(shift)); // RSHIFT, MODPOW2 for (int round_mode = -1; round_mode <= 1; round_mode++) { - auto r = x, q = td::rshift(x, shift, round_mode); // RSHIFT + auto r = x, q = td::rshift(x, shift, round_mode); // RSHIFT CHECK(q.not_null() && q->is_valid()); r.write().mod_pow2(shift, round_mode).normalize(); // MODPOW2 CHECK(r.not_null() && r->is_valid()); @@ -365,13 +531,13 @@ void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv) { MRInt rval(rv); // check remainder range switch (round_mode) { - case 1: - rval.negate(); // fallthrough - case -1: - CHECK(mr_ufits_bits(rval, shift)); - break; - case 0: - CHECK(mr_fits_bits(rval, shift)); + case 1: + rval.negate(); // fallthrough + case -1: + CHECK(mr_ufits_bits(rval, shift)); + break; + case 0: + CHECK(mr_fits_bits(rval, shift)); } } } @@ -379,20 +545,332 @@ void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv) { void check_shift_ops() { std::cerr << "check left/right shift ops" << std::endl; for (int idx = min_spec_int; idx <= max_spec_int; idx++) { - //for (int idx = -52239; idx <= max_spec_int; idx++) { - std::cerr << "idx=" << idx << " : " << SpecIntB[idx - min_spec_int] << std::endl; + //for (int idx : {-52240, -52239, -52238, -3, -2, -1, 0, 1, 2, 3, 52238, 52239, 52240}) { + const auto& xv = SpecIntB[idx - min_spec_int]; + MRInt mval(xv); + if (!(idx % 1000)) { + std::cerr << "# " << idx << " : " << mval << std::endl; + } for (int i = 0; i <= 256; i++) { - check_shift_ops_on(i, SpecInt[idx - min_spec_int], SpecIntB[idx - min_spec_int]); + check_shift_ops_on(i, SpecInt[idx - min_spec_int], xv, mval); + } + } +} + +void check_remainder_range(BInt& rv, const BInt& dv, int rmode = -1) { + if (rmode > 0) { + rv.negate(); + } else if (!rmode) { + rv *= 2; + } + MRInt d(dv), r(rv); + int ds = d.sgn(), rs = r.sgn(); + //std::cerr << "rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + if (!rs) { + return; + } + if (rmode) { + // must have 0 < r < d or 0 > r > d + //if (rs != ds) std::cerr << "iter=" << cur_iteration << " : rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + CHECK(rs == ds); + CHECK(ds * r.cmp(d) < 0); + } else { + // must have -d <= r < d or -d >= r > d + if (rs == -ds) { + r.negate(); + CHECK(ds * r.cmp(d) <= 0); + } else { + CHECK(ds * r.cmp(d) < 0); + } + } +} + +void check_divmod(td::RefInt256 x, const BInt& xv, long long xl, td::RefInt256 y, const BInt& yv, long long yl, + int rmode = -2) { + if (rmode < -1) { + //IFDEBUG std::cerr << " divide " << x << " / " << y << std::endl; + for (rmode = -1; rmode <= 1; rmode++) { + check_divmod(x, xv, xl, y, yv, yl, rmode); + } + return; + } + auto dm = td::divmod(x, y, rmode); + auto q = std::move(dm.first), r = std::move(dm.second); + if (!yl) { + // division by zero + CHECK(q.not_null() && !q->is_valid() && r.not_null() && !r->is_valid()); + return; + } + CHECK(q.not_null() && q->is_valid() && r.not_null() && r->is_valid()); + CHECK_EQ(x, y * q + r); + BInt qv = extract_value(*q), rv = extract_value(*r); + CHECK(xv == yv * qv + rv); + //IFDEBUG std::cerr << " quot=" << q << " rem=" << r << std::endl; + check_remainder_range(rv, yv, rmode); + if (yl != ll_min && rmode == -1) { + // check divmod_short() + auto qq = x; + auto rem = qq.write().divmod_short(yl); + qq.write().normalize(); + CHECK(qq->is_valid()); + CHECK_EQ(qq, q); + CHECK(r == rem); + if (xl != ll_min) { + auto dm = std::lldiv(xl, yl); + if (dm.rem && (dm.rem ^ yl) < 0) { + dm.rem += yl; + dm.quot--; + } + CHECK(q == dm.quot); + CHECK(r == dm.rem); + } + } +} + +void check_binary_ops_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv) { + bool x_small = x->fits_bits(62), y_small = y->fits_bits(62); // not 63 + long long xl = x_small ? x->to_long() : ll_min, yl = y_small ? y->to_long() : ll_min; + if (x_small) { + CHECK(x == xl); + } + if (y_small) { + CHECK(y == yl); + } + // ADD, ADDR + auto z = x + y, w = y + x; + CHECK_EQ(z, w); + check_res(z, xv + yv); + // ADDCONST + if (y_small) { + CHECK_EQ(z, x + yl); + } + if (x_small) { + CHECK_EQ(z, y + xl); + } + if (x_small && y_small) { + CHECK_EQ(z, xl + yl); + } + // SUB + z = x - y; + check_res(z, xv - yv); + // SUBCONST + if (y_small) { + CHECK_EQ(z, x - yl); + if (x_small) { + CHECK_EQ(z, xl - yl); + } + } + // SUBR + z = y - x; + check_res(z, yv - xv); + if (x_small) { + CHECK_EQ(z, y - xl); + if (y_small) { + CHECK_EQ(z, yl - xl); } } + // CMP + MRInt xmr(xv), ymr(yv); + int cmpv = xmr.cmp(ymr); + CHECK(td::cmp(x, y) == cmpv); + CHECK(td::cmp(y, x) == -cmpv); + if (y_small) { + CHECK(td::cmp(x, yl) == cmpv); + } + if (x_small) { + CHECK(td::cmp(y, xl) == -cmpv); + } + if (x_small && y_small) { + CHECK(cmpv == (xl < yl ? -1 : (xl > yl ? 1 : 0))); + } + // MUL + z = x * y; + BInt zv = xv * yv; + check_res(z, zv); + CHECK_EQ(z, y * x); + // MULCONST + if (y_small) { + CHECK_EQ_NAN(z, x * yl); + } + if (x_small) { + CHECK_EQ_NAN(z, y * xl); + } + if (x_small && y_small && (!yl || std::abs(xl) <= ll_max / std::abs(yl))) { + CHECK_EQ(z, xl * yl); + } + // DIVMOD + if (z->fits_bits(257)) { + int adj = 2 * rand_int(-2, 2) - (int)z->is_odd(); + z += adj; + z >>= 1; + zv += adj; + zv >>= 1; + // z is approximately x * y / 2; divide by y + check_divmod(z, zv, z->fits_bits(62) ? z->to_long() : ll_min, y, yv, yl); + } + check_divmod(x, xv, xl, y, yv, yl); +} + +void finish_check_muldivmod(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, td::RefInt256 q, td::RefInt256 r, int rmode) { + static constexpr double eps = 1e-14; + CHECK(q.not_null() && r.not_null()); + //std::cerr << " muldivmod: " << xv << " * " << yv << " / " << zv << " (round " << rmode << ") = " << q << " " << r << std::endl; + if (!zv) { + // division by zero + CHECK(!q->is_valid() && !r->is_valid()); + return; + } + CHECK(r->is_valid()); // remainder always exists if y != 0 + BInt xyv = xv * yv, rv = extract_value(*r); + MRInt xy_mr(xyv), z_mr(zv); + double q0 = (double)xy_mr / (double)z_mr; + if (std::abs(q0) < 1.01 * dbl_pow256) { + // result more or less in range + CHECK(q->is_valid()); + } else if (!q->is_valid()) { + // result out of range, NaN is an acceptable answer + // check that x * y - r is divisible by z + xyv -= rv; + xyv /= zv; + xy_mr = xyv; + double q1 = (double)xy_mr; + CHECK(std::abs(q1 - q0) < eps * std::abs(q0)); + } else { + BInt qv = extract_value(*q); + // must have x * y = z * q + r + CHECK(xv * yv == zv * qv + rv); + } + // check that r is in correct range [0, z) or [0, -z) or [-z/2, z/2) + check_remainder_range(rv, zv, rmode); +} + +void check_muldivmod_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_muldivmod_on(x, xv, y, yv, z, zv, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULDIVMOD + auto qr = td::muldivmod(x, y, z, rmode); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, std::move(z), zv, std::move(qr.first), + std::move(qr.second), rmode); } -int main() { +void check_mul_rshift_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_mul_rshift_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULRSHIFTMOD + typename td::BigInt256::DoubleInt tmp{0}; + tmp.add_mul(*x, *y); + typename td::BigInt256::DoubleInt tmp2{tmp}; + tmp2.rshift(shift, rmode).normalize(); + tmp.normalize().mod_pow2(shift, rmode).normalize(); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, {}, BInt::pow2(shift), td::make_refint(tmp2), + td::make_refint(tmp), rmode); +} + +void check_lshift_div_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_lshift_div_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // LSHIFTDIV + typename td::BigInt256::DoubleInt tmp{*x}, quot; + tmp <<= shift; + tmp.mod_div(*y, quot, rmode); + quot.normalize(); + finish_check_muldivmod(std::move(x), xv, {}, BInt::pow2(shift), std::move(y), yv, td::make_refint(quot), + td::make_refint(tmp), rmode); +} + +void check_random_ops() { + constexpr long long chk_it = 100000; + std::cerr << "check random ops (" << iterations << " iterations)" << std::endl; + BInt xv, yv, zv; + unsigned char xbin[64], ybin[64], zbin[64]; + for (cur_iteration = 0; cur_iteration < iterations; cur_iteration++) { + auto x = make_random_int0(xv, xbin); + if (!(cur_iteration % 10000)) { + std::cerr << "#" << cur_iteration << ": check on " << xv << " = " << ShowBin(xbin) << " = " << x->as_any_int() + << std::endl; + } + check_one_int_repr(x, cur_iteration < chk_it ? -1 : 0, -1, &xv, xbin); + MRInt xmr(xv); + if (!x->fits_bits(257)) { + continue; + } + check_unary_ops_on(x, xv); + for (int j = 0; j < 10; j++) { + int shift = rand_int(0, 256); + //std::cerr << "check shift by " << shift << std::endl; + check_shift_ops_on(shift, x, xv, xmr); + auto y = make_random_int(yv, ybin); + //std::cerr << " y = " << y << " = " << yv << " = " << ShowBin(ybin) << " = " << y->as_any_int() << std::endl; + check_one_int_repr(y, 0, 1, &yv, ybin); + check_binary_ops_on(x, xv, y, yv); + //std::cerr << " *>> " << shift << std::endl; + check_mul_rshift_on(x, xv, y, yv, shift); + //std::cerr << " <as_any_int() << std::endl; + check_muldivmod_on(x, xv, y, yv, z, zv); + } + } +} + +void check_special() { + std::cerr << "run special tests" << std::endl; + check_divmod((td::make_refint(-1) << 207) - 1, BInt::negpow2(207) - 1, ll_min, (td::make_refint(1) << 207) - 1, + BInt::pow2(207) - 1, ll_min); +} + +int main(int argc, char* const argv[]) { + bool do_check_shift_ops = false; + int i; + while ((i = getopt(argc, argv, "hSs:i:")) != -1) { + switch (i) { + case 'S': + do_check_shift_ops = true; + break; + case 's': + Random.seed(atoll(optarg)); + break; + case 'i': + iterations = atoll(optarg); + break; + default: + std::cerr << "unknown option: " << (char)i << std::endl; + // fall through + case 'h': + std::cerr << "usage:\t" << argv[0] << " [-S] [-i] [-s]" << std::endl; + return 2; + } + } modint::init(); init_aux(); init_check_special_ints(); check_pow2_ops(); check_unary_ops(); - check_shift_ops(); + if (do_check_shift_ops) { + check_shift_ops(); + } + check_special(); + check_random_ops(); return 0; } diff --git a/crypto/vm/arithops.cpp b/crypto/vm/arithops.cpp index 823b44080..2831944bc 100644 --- a/crypto/vm/arithops.cpp +++ b/crypto/vm/arithops.cpp @@ -387,18 +387,16 @@ int exec_muldivmod(VmState* st, unsigned args, int quiet) { auto z = stack.pop_int(); auto y = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); + tmp.mod_div(*z, quot, round_mode); switch ((args >> 2) & 3) { case 1: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); break; case 3: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); // fallthrough case 2: stack.push_int_quiet(td::make_refint(tmp), quiet); @@ -459,7 +457,7 @@ int exec_mulshrmod(VmState* st, unsigned args, int mode) { } // fallthrough case 2: - tmp.mod_pow2(z, round_mode).normalize(); + tmp.normalize().mod_pow2(z, round_mode).normalize(); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } @@ -520,21 +518,17 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) { } auto z = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{*x}; + typename td::BigInt256::DoubleInt tmp{*x}, quot; tmp <<= y; switch ((args >> 2) & 3) { case 1: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); break; } case 3: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } diff --git a/doc/tvm.tex b/doc/tvm.tex index 4f887ac23..71aa7abce 100644 --- a/doc/tvm.tex +++ b/doc/tvm.tex @@ -1531,6 +1531,7 @@ \section*{Introduction} \item {\tt A934$tt$} --- same as {\tt RSHIFT $tt+1$}: ($x$ -- $\lfloor x\cdot 2^{-tt-1}\rfloor$). \item {\tt A938$tt$} --- {\tt MODPOW2 $tt+1$}: ($x$ -- $x\bmod 2^{tt+1}$). \item {\tt A985} --- {\tt MULDIVR} ($x$ $y$ $z$ -- $q'$), where $q'=\lfloor xy/z+1/2\rfloor$. +\item {\tt A988} --- {\tt MULMOD} ($x$ $y$ $z$ -- $r$), where $r=xy\bmod z=xy-qz$, $q=\lfloor xy/z\rfloor$. This operation always succeeds for $z\neq0$ and returns the correct value of~$r$, even if the intermediate result $xy$ or the quotient $q$ do not fit into 257 bits. \item {\tt A98C} --- {\tt MULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$ (same as {\tt */MOD} in Forth). \item {\tt A9A4} --- {\tt MULRSHIFT} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}\rfloor$) for $0\leq z\leq 256$. \item {\tt A9A5} --- {\tt MULRSHIFTR} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}+1/2\rfloor$) for $0\leq z\leq 256$. @@ -1576,10 +1577,12 @@ \section*{Introduction} \begin{itemize} \item {\tt B7xx} --- {\tt QUIET} prefix, transforming any arithmetic operation into its ``quiet'' variant, indicated by prefixing a {\tt Q} to its mnemonic. Such operations return {\tt NaN}s instead of throwing integer overflow exceptions if the results do not fit in {\it Integer\/}s, or if one of their arguments is a {\tt NaN}. Notice that this does not extend to shift amounts and other parameters that must be within a small range (e.g., 0--1023). Also notice that this does not disable type-checking exceptions if a value of a type other than {\it Integer\/} is supplied. \item {\tt B7A0} --- {\tt QADD} ($x$ $y$ -- $x+y$), always works if $x$ and $y$ are {\it Integer\/}s, but returns a {\tt NaN} if the addition cannot be performed. +\item {\tt B7A8} --- {\tt QMUL} ($x$ $y$ -- $xy$), returns the product of $x$ and $y$ if $-2^{256}\leq xy<2^{256}$. Otherwise returns a {\tt NaN}, even if $x=0$ and $y$ is a {\tt NaN}. \item {\tt B7A904} --- {\tt QDIV} ($x$ $y$ -- $\lfloor x/y\rfloor$), returns a {\tt NaN} if $y=0$, or if $y=-1$ and $x=-2^{256}$, or if either of $x$ or $y$ is a {\tt NaN}. +\item {\tt B7A98C} --- {\tt QMULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$. If $z=0$, or if at least one of $x$, $y$, or $z$ is a {\tt NaN}, both $q$ and $r$ are set to {\tt NaN}. Otherwise the correct value of $r$ is always returned, but $q$ is replaced with {\tt NaN} if $q<-2^{256}$ or $q\geq2^{256}$. \item {\tt B7B0} --- {\tt QAND} ($x$ $y$ -- $x\&y$), bitwise ``and'' (similar to {\tt AND}), but returns a {\tt NaN} if either $x$ or $y$ is a {\tt NaN} instead of throwing an integer overflow exception. However, if one of the arguments is zero, and the other is a {\tt NaN}, the result is zero. \item {\tt B7B1} --- {\tt QOR} ($x$ $y$ -- $x\vee y$), bitwise ``or''. If $x=-1$ or $y=-1$, the result is always $-1$, even if the other argument is a {\tt NaN}. -\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte). +\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte or a {\tt NaN}). \end{itemize} \mysubsection{Comparison primitives} From 4280b5a1bc9b371414b4e87e0c115b9bb1fa0201 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Thu, 23 Dec 2021 10:39:14 +0300 Subject: [PATCH 18/19] Fix infinity max_gas_limit for get methods --- crypto/smc-envelope/SmartContract.cpp | 2 +- validator/impl/liteserver.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index f3485be9b..5f284b57a 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -207,7 +207,7 @@ SmartContract::Answer SmartContract::run_get_method(Args args) const { args.c7 = prepare_vm_c7(now, args.balance); } if (!args.limits) { - args.limits = vm::GasLimits{1000000}; + args.limits = vm::GasLimits{1000000, 1000000}; } if (!args.stack) { args.stack = td::Ref(true); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 510f49061..6b0b90089 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1150,7 +1150,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice long long gas_limit = client_method_gas_limit; LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** - vm::GasLimits gas{gas_limit}; + vm::GasLimits gas{gas_limit, gas_limit}; vm::VmState vm{std::move(code), std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null()}; auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance); vm.set_c7(c7); // tuple with SmartContractInfo From dbb666056391ea49174d15f9902355c97c36cadc Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Fri, 31 Dec 2021 18:48:51 +0300 Subject: [PATCH 19/19] Deactivate accounts upon destruction in block --- crypto/block/transaction.cpp | 52 ++++++++++++++++++++++--------- crypto/block/transaction.h | 11 ++++--- validator/impl/collator.cpp | 9 ------ validator/impl/validate-query.cpp | 3 +- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 3668a45f3..e5cd701d7 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -271,7 +271,7 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, } bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { - if (split_depth_set_ || !created || !set_split_depth(split_depth)) { + if (split_depth_set_ || !set_split_depth(split_depth)) { return false; } addr_orig = addr; @@ -304,15 +304,8 @@ bool Account::unpack(Ref shard_account, Ref extra, total_state = orig_total_state = account; auto acc_cs = load_cell_slice(std::move(account)); if (block::gen::t_Account.get_tag(acc_cs) == block::gen::Account::account_none) { - status = acc_nonexist; - last_paid = 0; - last_trans_end_lt_ = 0; is_special = special; - if (workchain != ton::workchainInvalid) { - addr_orig = addr; - addr_rewrite = addr.cbits(); - } - return compute_my_addr() && acc_cs.size_ext() == 1; + return acc_cs.size_ext() == 1 && init_new(now); } block::gen::Account::Record_account acc; block::gen::AccountStorage::Record storage; @@ -328,6 +321,7 @@ bool Account::unpack(Ref shard_account, Ref extra, case block::gen::AccountState::account_uninit: status = orig_status = acc_uninit; state_hash = addr; + forget_split_depth(); break; case block::gen::AccountState::account_frozen: status = orig_status = acc_frozen; @@ -396,10 +390,42 @@ bool Account::init_new(ton::UnixTime now) { state_hash = addr_orig; status = orig_status = acc_nonexist; split_depth_set_ = false; - created = true; return true; } +bool Account::forget_split_depth() { + split_depth_set_ = false; + split_depth_ = 0; + addr_orig = addr; + my_addr = my_addr_exact; + addr_rewrite = addr.bits(); + return true; +} + +bool Account::deactivate() { + if (status == acc_active) { + return false; + } + // forget special (tick/tock) info + tick = tock = false; + if (status == acc_nonexist || status == acc_uninit) { + // forget split depth and address rewriting info + forget_split_depth(); + // forget specific state hash for deleted or uninitialized accounts (revert to addr) + state_hash = addr; + } + // forget code and data (only active accounts remember these) + code.clear(); + data.clear(); + library.clear(); + // if deleted, balance must be zero + if (status == acc_nonexist && !balance.is_zero()) { + return false; + } + return true; +} + + bool Account::belongs_to_shard(ton::ShardIdFull shard) const { return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr); } @@ -2214,7 +2240,7 @@ Ref Transaction::commit(Account& acc) { CHECK((const void*)&acc == (const void*)&account); // export all fields modified by the Transaction into original account // NB: this is the only method that modifies account - if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status == Account::acc_nonexist && + if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active && acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() << " with split_depth=" << new_split_depth @@ -2243,9 +2269,7 @@ Ref Transaction::commit(Account& acc) { acc.tick = new_tick; acc.tock = new_tock; } else { - acc.tick = acc.tock = false; - acc.split_depth_set_ = false; - acc.created = true; + CHECK(acc.deactivate()); } end_lt = 0; acc.push_transaction(root, start_lt); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index c1cfbc545..398e27319 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -213,17 +213,16 @@ struct Account { bool is_special{false}; bool tick{false}; bool tock{false}; - bool created{false}; bool split_depth_set_{false}; unsigned char split_depth_{0}; int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; td::BitArray<32> addr_rewrite; // rewrite (anycast) data, split_depth bits - ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`) - ton::StdSmcAddress addr_orig; // address indicated in smart-contract data - Ref my_addr; // address as stored in the smart contract (MsgAddressInt) - Ref my_addr_exact; // exact address without anycast info + ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts + ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) + Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info + Ref my_addr_exact; // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info ton::LogicalTime last_trans_end_lt_; ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; @@ -250,6 +249,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, Ref extra, ton::UnixTime now, bool special = false); bool init_new(ton::UnixTime now); + bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { @@ -268,6 +268,7 @@ struct Account { friend struct Transaction; bool set_split_depth(int split_depth); bool check_split_depth(int split_depth) const; + bool forget_split_depth(); bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite); private: diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6b1c72a6a..5820a2ce3 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1944,7 +1944,6 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr } auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -2273,10 +2272,6 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { register_new_msgs(*trans); update_max_lt(acc->last_trans_end_lt_); - // temporary patch to stop producing dangerous block - if (acc->status == block::Account::acc_nonexist) { - block_full_ = true; - } return trans_root; } @@ -2495,10 +2490,6 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!insert_out_msg(cb.finalize())) { return -1; } - // 6.5. check for temporary patch can be left here - if (block_full_) { - return 3; - } // 7. check whether the block is full now if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fc90be1de..02a16f83a 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -4073,7 +4073,6 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr Ref extra) { auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -4308,7 +4307,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } if (is_first && is_masterchain() && account.is_special && account.tick && - (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && !account.created) { + (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && account.orig_status == block::Account::acc_active) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is the first transaction for this special tick account in this block, but the " "transaction is not a tick transaction");