Deduplicate cells in collated data during one validator session

create-hardfork/create-hardfork.cpp

@@ -246,8 +246,9 @@ class HardforkCreator : public td::actor::Actor {
       }
       void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override {
       }
-      void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
-                                td::BufferSlice data, int mode) override {
+      void send_block_candidate_broadcast(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno,
+                                          td::uint32 validator_set_hash, td::BufferSlice data,
+                                          td::optional<td::BufferSlice> collated_data, int mode) override {
       }
       void send_broadcast(ton::BlockBroadcast broadcast, int mode) override {
       }
@@ -277,6 +278,9 @@ class HardforkCreator : public td::actor::Actor {
           ton::ShardIdFull dst_shard, std::vector<ton::BlockIdExt> blocks, block::ImportedMsgQueueLimits limits,
           td::Timestamp timeout, td::Promise<std::vector<td::Ref<ton::validator::OutMsgQueueProof>>> promise) override {
       }
+      void download_block_candidate(ton::BlockIdExt block_id, bool only_collated_data, td::Timestamp timeout,
+                                    td::Promise<std::pair<td::BufferSlice, td::BufferSlice>> promise) override {
+      }
 
       void new_key_block(ton::validator::BlockHandle handle) override {
       }

crypto/block/block.tlb

@@ -451,7 +451,7 @@ block_info#9bc7a987 version:uint32
   after_split:(## 1) 
   want_split:Bool want_merge:Bool
   key_block:Bool vert_seqno_incr:(## 1)
-  flags:(## 8) { flags <= 1 }
+  flags:(## 8) { flags <= 3 }
   seq_no:# vert_seq_no:# { vert_seq_no >= vert_seqno_incr } 
   { prev_seq_no:# } { ~prev_seq_no + 1 = seq_no } 
   shard:ShardIdent gen_utime:uint32
@@ -464,6 +464,7 @@ block_info#9bc7a987 version:uint32
   master_ref:not_master?^BlkMasterInfo 
   prev_ref:^(BlkPrevInfo after_merge)
   prev_vert_ref:vert_seqno_incr?^(BlkPrevInfo 0)
+  collated_data_hash:flags . 1?bits256
   = BlockInfo;
 
 prev_blk_info$_ prev:ExtBlkRef = BlkPrevInfo 0;
@@ -772,7 +773,7 @@ consensus_config_v3#d8 flags:(## 7) { flags = 0 } new_catchain_ids:Bool
   max_block_bytes:uint32 max_collated_bytes:uint32 
   proto_version:uint16 = ConsensusConfig;
 
-consensus_config_v4#d9 flags:(## 7) { flags = 0 } new_catchain_ids:Bool
+consensus_config_v4#d9 flags:(## 6) { flags = 0 } merge_collated_data:Bool new_catchain_ids:Bool
   round_candidates:(## 8) { round_candidates >= 1 }
   next_candidate_delay_ms:uint32 consensus_timeout_ms:uint32
   fast_attempts:uint32 attempt_duration:uint32 catchain_max_deps:uint32
@@ -857,6 +858,9 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures)
 //
 top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet;
 account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof;
+collated_data_root_state#4b2f36ec hash:bits256 = CollatedDataRootState;
+collated_data_root_storage_dict#796eaeb6 hash:bits256 = CollatedDataRootStorageDict;
+collated_data_separator#fa8b2b92 = CollatedDataSeparator;
 
 //
 //  VALIDATOR MISBEHAVIOR COMPLAINTS

crypto/block/mc-config.cpp

@@ -336,6 +336,7 @@ ton::ValidatorSessionConfig Config::get_consensus_config() const {
       td::uint64 catchain_lifetime = std::max(catchain_config.mc_cc_lifetime, catchain_config.shard_cc_lifetime);
       c.catchain_opts.max_block_height_coeff = catchain_lifetime * max_blocks_coeff;
     }
+    c.merge_collated_data = r.merge_collated_data;
   };
   if (cc.not_null()) {
     block::gen::ConsensusConfig::Record_consensus_config_v4 r4;

crypto/common/refcnt.hpp

@@ -285,14 +285,14 @@ class Ref {
   Ref& operator=(Ref<S>&& r);
   const typename RefValue<T>::Type* operator->() const {
     if (!ptr) {
-      CHECK(ptr && "deferencing null Ref");
+      LOG_CHECK(ptr) << "dereferencing null Ref<" << typeid(T).name() << ">";
       throw NullRef{};
     }
     return RefValue<T>::make_const_ptr(ptr);
   }
   const typename RefValue<T>::Type& operator*() const {
     if (!ptr) {
-      CHECK(ptr && "deferencing null Ref");
+      LOG_CHECK(ptr) << "dereferencing null Ref<" << typeid(T).name() << ">";
       throw NullRef{};
     }
     return RefValue<T>::make_const_ref(ptr);
@@ -308,7 +308,7 @@ class Ref {
   }
   bool is_unique() const {
     if (!ptr) {
-      CHECK(ptr && "defererencing null Ref");
+      LOG_CHECK(ptr) << "dereferencing null Ref<" << typeid(T).name() << ">";
       throw NullRef{};
     }
     return ptr->is_unique();

crypto/vm/boc-compression.cpp

@@ -26,6 +26,7 @@
 #include "vm/cellslice.h"
 #include "td/utils/Slice-decl.h"
 #include "td/utils/lz4.h"
+#include "ton/ton-types.h"
 
 namespace vm {
 
@@ -57,7 +58,7 @@ td::Result<std::vector<td::Ref<vm::Cell>>> boc_decompress_baseline_lz4(td::Slice
   }
 
   TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size));
-  TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed));
+  TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed, ton::max_collated_data_roots + 1, true));
   return roots;
 }
 

crypto/vm/boc.cpp

@@ -20,6 +20,9 @@
 #include <iomanip>
 #include <algorithm>
 #include "vm/boc.h"
+
+#include "cells/MerkleProof.h"
+#include "cells/PrunnedCell.h"
 #include "vm/boc-writers.h"
 #include "vm/cells.h"
 #include "vm/cellslice.h"
@@ -973,7 +976,7 @@ td::Result<Ref<Cell>> std_boc_deserialize(td::Slice data, bool can_be_empty, boo
   return std::move(root);
 }
 
-td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data, int max_roots) {
+td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data, int max_roots, bool allow_nonzero_level) {
   if (data.empty()) {
     return std::vector<Ref<Cell>>{};
   }
@@ -989,7 +992,7 @@ td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data, int
     if (root.is_null()) {
       return td::Status::Error("bag of cells has a null root cell (?)");
     }
-    if (root->get_level() != 0) {
+    if (!allow_nonzero_level && root->get_level() != 0) {
       return td::Status::Error("bag of cells has a root with non-zero level");
     }
     roots.emplace_back(std::move(root));
@@ -1265,46 +1268,122 @@ bool VmStorageStat::add_storage(const CellSlice& cs) {
 }
 
 void ProofStorageStat::add_loaded_cell(const Ref<DataCell>& cell, td::uint8 max_level) {
-  max_level = std::min<td::uint32>(max_level, Cell::max_level);
-  auto& [status, size] = cells_[cell->get_hash(max_level)];
-  if (status == c_loaded) {
+  max_level = std::min<td::uint8>(max_level, Cell::max_level);
+  auto& info = cells_[cell->get_hash(max_level)];
+  if (info.status == c_loaded) {
     return;
   }
-  proof_size_ -= size;
-  status = c_loaded;
-  proof_size_ += size = estimate_serialized_size(cell);
+  proof_size_ -= info.serialized_size;
+  info.status = c_loaded;
+  info.cell = cell;
+  info.cell_max_level = max_level;
+  proof_size_ += info.serialized_size = estimate_serialized_size(cell);
   max_level += (cell->special_type() == CellTraits::SpecialType::MerkleProof ||
                 cell->special_type() == CellTraits::SpecialType::MerkleUpdate);
   for (unsigned i = 0; i < cell->size_refs(); ++i) {
-    auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash(max_level)];
-    if (child_status == c_none) {
-      child_status = c_prunned;
-      proof_size_ += child_size = estimate_prunned_size();
+    auto& child = cells_[cell->get_ref(i)->get_hash(max_level)];
+    if (child.status == c_none) {
+      child.status = c_prunned;
+      proof_size_ += child.serialized_size = estimate_prunned_size();
     }
   }
 }
 
 void ProofStorageStat::add_loaded_cells(const ProofStorageStat& other) {
-  for (const auto& [hash, x] : other.cells_) {
-    const auto& [new_status, new_size] = x;
-    auto& [old_status, old_size] = cells_[hash];
-    if (old_status >= new_status) {
+  for (const auto& [hash, new_info] : other.cells_) {
+    auto& old_info = cells_[hash];
+    if (old_info.status >= new_info.status) {
       continue;
     }
-    proof_size_ -= old_size;
-    old_status = new_status;
-    proof_size_ += old_size = new_size;
+    proof_size_ -= old_info.serialized_size;
+    old_info = new_info;
+    proof_size_ += old_info.serialized_size;
   }
 }
 
-
 td::uint64 ProofStorageStat::estimate_proof_size() const {
   return proof_size_;
 }
 
 ProofStorageStat::CellStatus ProofStorageStat::get_cell_status(const Cell::Hash& hash) const {
   auto it = cells_.find(hash);
-  return it == cells_.end() ? c_none : it->second.first;
+  return it == cells_.end() ? c_none : it->second.status;
+}
+
+std::vector<Ref<Cell>> ProofStorageStat::build_collated_data(std::vector<Ref<Cell>> skip_roots) const {
+  struct Cache {
+    Ref<Cell> result;
+    bool is_root = true;
+    bool skip = false;
+    bool skip_visited = false;
+  };
+  std::map<Cell::Hash, Cache> cache;
+
+  std::function<void(const Ref<Cell>&)> dfs_skip = [&](const Ref<Cell>& cell) {
+    Cell::Hash hash = cell->get_hash();
+    Cache& entry = cache[hash];
+    if (entry.skip_visited) {
+      return;
+    }
+    entry.skip_visited = entry.skip = true;
+    CellSlice cs{NoVm{}, cell};
+    if (cs.special_type() != CellTraits::SpecialType::PrunnedBranch) {
+      for (unsigned i = 0; i < cell->get_level(); ++i) {
+        cache[cell->get_hash(i)].skip = true;
+      }
+    }
+    for (unsigned i = 0; i < cs.size_refs(); ++i) {
+      dfs_skip(cs.prefetch_ref(i));
+    }
+  };
+  for (const auto& cell : skip_roots) {
+    if (cell.not_null()) {
+      dfs_skip(cell);
+    }
+  }
+
+  std::function<Cache&(const CellInfo&)> dfs = [&](const CellInfo& info) -> Cache& {
+    Cell::Hash hash = info.cell->get_hash(info.cell_max_level);
+    Cache& entry = cache[hash];
+    if (entry.result.not_null()) {
+      return entry;
+    }
+    CellBuilder cb;
+    cb.store_bits(info.cell->get_data(), info.cell->size());
+    td::uint8 child_max_level = info.cell_max_level;
+    if (info.cell->special_type() == CellTraits::SpecialType::MerkleProof ||
+        info.cell->special_type() == CellTraits::SpecialType::MerkleUpdate) {
+      ++child_max_level;
+    }
+    for (unsigned i = 0; i < info.cell->size_refs(); ++i) {
+      Ref<Cell> child = info.cell->get_ref(i);
+      Cell::Hash child_hash = child->get_hash(child_max_level);
+      auto it = cells_.find(child_hash);
+      if (it == cells_.end() || it->second.status != c_loaded || cache[child_hash].skip) {
+        cb.store_ref(CellBuilder::create_pruned_branch(child, Cell::max_level, child_max_level));
+      } else {
+        Cache& child_result = dfs(it->second);
+        child_result.is_root = false;
+        cb.store_ref(child_result.result);
+      }
+    }
+    Cache& entry2 = cache[hash];
+    entry2.result = cb.finalize(info.cell->is_special());
+    CHECK(entry2.result->get_hash(std::min<int>(info.cell->get_level(), info.cell_max_level)) == hash);
+    return entry2;
+  };
+  for (auto& [hash, info] : cells_) {
+    if (info.status == c_loaded && !cache[hash].skip) {
+      dfs(info);
+    }
+  }
+  std::vector<Ref<Cell>> result;
+  for (auto& [_, entry] : cache) {
+    if (entry.result.not_null() && entry.is_root) {
+      result.push_back(std::move(entry.result));
+    }
+  }
+  return result;
 }
 
 td::uint64 ProofStorageStat::estimate_prunned_size() {

crypto/vm/boc.h

@@ -177,11 +177,19 @@ class ProofStorageStat {
     return get_cell_status(hash) == c_loaded;
   }
 
+  std::vector<Ref<Cell>> build_collated_data(std::vector<Ref<Cell>> skip_roots = {}) const;
+
   static td::uint64 estimate_prunned_size();
   static td::uint64 estimate_serialized_size(const Ref<DataCell>& cell);
 
  private:
-  td::HashMap<Cell::Hash, std::pair<CellStatus, td::uint64>> cells_;
+  struct CellInfo {
+    Ref<DataCell> cell;                          // only for c_loaded
+    td::uint8 cell_max_level = Cell::max_level;  // only for c_loaded
+    CellStatus status = c_none;
+    td::uint64 serialized_size = 0;
+  };
+  td::HashMap<Cell::Hash, CellInfo> cells_;
   td::uint64 proof_size_ = 0;
 };
 
@@ -400,7 +408,8 @@ td::Result<Ref<Cell>> std_boc_deserialize(td::Slice data, bool can_be_empty = fa
 td::Result<td::BufferSlice> std_boc_serialize(Ref<Cell> root, int mode = 0);
 
 td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data,
-                                                             int max_roots = BagOfCells::default_max_roots);
+                                                             int max_roots = BagOfCells::default_max_roots,
+                                                             bool allow_nonzero_level = false);
 td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> root, int mode = 0);
 
 td::Status std_boc_serialize_to_file(Ref<Cell> root, td::FileFd& fd, int mode = 0,

crypto/vm/cells/ExtCell.h

@@ -68,6 +68,18 @@ class ExtCell : public Cell {
   Ref<PrunnedCell<ExtraT>> get_prunned_cell() const {
     return prunned_cell_.load();
   }
+  td::Status set_inner_cell(Ref<DataCell> new_cell) const {
+    auto prunned_cell = prunned_cell_.load();
+    if (prunned_cell.is_null()) {
+      return td::Status::OK();
+    }
+    TRY_STATUS(prunned_cell->check_equals_unloaded(new_cell));
+    if (data_cell_.store_if_empty(new_cell)) {
+      prunned_cell_.store({});
+      get_thread_safe_counter_unloaded().add(-1);
+    }
+    return td::Status::OK();
+  }
 
  private:
   mutable td::AtomicRef<DataCell> data_cell_;

overlay/overlay-peers.cpp

@@ -508,12 +508,16 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers,
                                            td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) {
   std::vector<adnl::AdnlNodeIdShort> v;
   auto t = td::Clocks::system();
-  while (v.size() < max_peers && v.size() < peer_list_.peers_.size() - peer_list_.bad_peers_.size()) {
+  td::uint32 iters = 0;
+  while (v.size() < max_peers && v.size() < peer_list_.peers_.size() - peer_list_.bad_peers_.size() &&
+         iters <= peer_list_.peers_.size() * 2) {
+    ++iters;
     auto P = peer_list_.peers_.get_random();
     if (!P->is_permanent_member() && (P->get_version() + 3600 < t || P->certificate()->is_expired(t))) {
       VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id();
       del_peer(P->get_id());
-    } else if (P->is_alive()) {
+    } else if (P->is_alive() && P->get_id() != local_id_ &&
+               !(P->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts)) {
       bool dup = false;
       for (auto &n : v) {
         if (n == P->get_id()) {

tdutils/td/utils/LRUCache.h

@@ -86,6 +86,17 @@ class LRUCache {
     return result;
   }
 
+  void remove(const K& key) {
+    auto it = cache_.find(key);
+    if (it == cache_.end()) {
+      return;
+    }
+    total_weight_ -= (*it)->weight;
+    (*it)->remove();
+    cache_.erase(it);
+  }
+
+
  private:
   struct Entry : ListNode {
     Entry(K key, uint64 weight) : key(std::move(key)), weight(weight) {

tdutils/td/utils/Status.h

@@ -74,7 +74,7 @@
 #define TRY_RESULT_ASSIGN(name, result) TRY_RESULT_IMPL(TD_CONCAT(r_response, __LINE__), name, result)
 
 #define TRY_RESULT_PROMISE_ASSIGN(promise_name, name, result) \
-  TRY_RESULT_PROMISE_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result)
+  TRY_RESULT_PROMISE_IMPL(promise_name, TD_CONCAT(r_response, __LINE__), name, result)
 
 #define TRY_RESULT_PREFIX(name, result, prefix) \
   TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix)
@@ -86,7 +86,7 @@
   TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix)
 
 #define TRY_RESULT_PROMISE_PREFIX_ASSIGN(promise_name, name, result, prefix) \
-  TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix)
+  TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(r_response, __LINE__), name, result, prefix)
 
 #define TRY_RESULT_IMPL(r_name, name, result) \
   auto r_name = (result);                     \

tdutils/td/utils/optional.h

@@ -89,6 +89,13 @@ class optional {
     return {};
   }
 
+  td::optional<T> clone() const {
+    if (*this) {
+      return value().clone();
+    }
+    return {};
+  }
+
   template <class... ArgsT>
   void emplace(ArgsT &&... args) {
     impl_.emplace(std::forward<ArgsT>(args)...);