Merge remote-tracking branch 'origin/testnet' into testnet

Author: neodix42

Changelog.md

@@ -1,3 +1,17 @@
+## 2024.12 Update
+
+1. FunC 0.4.6: Fix in try/catch handling, fixing pure flag for functions stored in variables
+2. Merging parts of Accelerator: support of specific shard monitoring, archive/liteserver slice format, support for partial liteservers, proxy liteserver, on-demand neighbour queue loading
+3. Fix of asynchronous cell loading
+4. Various improvements: caching certificates checks, better block overloading detection, `_malloc` in emulator
+5. Introduction of telemetry in overlays
+6. Use non-null local-id for tonlib-LS interaction - mitigates MitM attack.
+7. Adding `SECP256K1_XONLY_PUBKEY_TWEAK_ADD`, `SETCONTCTRMANY` instructions to TVM (activated by `Config8.version >= 9`)
+8. Private keys export via validator-engine-console - required for better backups
+9. Fix proof checking in tonlib, `hash` in `raw.Message` in tonlib_api
+
+Besides the work of the core team, this update is based on the efforts of OtterSec and LayerZero (FunC), tg:@throwunless (FunC), Aviv Frenkel and Dima Kogan from Fordefi (LS MitM), @hacker-volodya (Tonlib), OKX team (async cell loading), @krigga (emulator)
+
 ## 2024.10 Update
 
 1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks.

create-hardfork/create-hardfork.cpp

@@ -272,6 +272,10 @@ class HardforkCreator : public td::actor::Actor {
       void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir,
                             td::Timestamp timeout, td::Promise<std::string> promise) override {
       }
+      void download_out_msg_queue_proof(
+          ton::ShardIdFull dst_shard, std::vector<ton::BlockIdExt> blocks, block::ImportedMsgQueueLimits limits,
+          td::Timestamp timeout, td::Promise<std::vector<td::Ref<ton::validator::OutMsgQueueProof>>> promise) override {
+      }
 
       void new_key_block(ton::validator::BlockHandle handle) override {
       }

crypto/block/block.cpp

@@ -660,6 +660,12 @@ bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const {
          hash_ == key + 96;
 }
 
+bool ImportedMsgQueueLimits::deserialize(vm::CellSlice& cs) {
+  return cs.fetch_ulong(8) == 0xd3           // imported_msg_queue_limits#d3
+         && cs.fetch_uint_to(32, max_bytes)  // max_bytes:#
+         && cs.fetch_uint_to(32, max_msgs);  // max_msgs:#
+}
+
 bool ParamLimits::deserialize(vm::CellSlice& cs) {
   return cs.fetch_ulong(8) == 0xc3            // param_limits#c3
          && cs.fetch_uint_to(32, limits_[0])  // underload:uint32

crypto/block/block.h

@@ -216,6 +216,16 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC
   return proc_coll.print(os);
 }
 
+struct ImportedMsgQueueLimits {
+  // Default values
+  td::uint32 max_bytes = 1 << 16;
+  td::uint32 max_msgs = 30;
+  bool deserialize(vm::CellSlice& cs);
+  ImportedMsgQueueLimits operator*(td::uint32 x) const {
+    return {max_bytes * x, max_msgs * x};
+  }
+};
+
 struct ParamLimits {
   enum { limits_cnt = 4 };
   enum { cl_underload = 0, cl_normal = 1, cl_soft = 2, cl_medium = 3, cl_hard = 4 };

crypto/common/bitstring.h

@@ -554,11 +554,7 @@ class BitArray {
     set_same(0);
   }
   void set_zero_s() {
-    volatile uint8* p = data();
-    auto x = m;
-    while (x--) {
-      *p++ = 0;
-    }
+    as_slice().fill_zero_secure();
   }
   void set_ones() {
     set_same(1);

crypto/vm/db/DynamicBagOfCellsDb.cpp

@@ -111,14 +111,16 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
   }
   void load_cell_async(td::Slice hash, std::shared_ptr<AsyncExecutor> executor,
                        td::Promise<Ref<DataCell>> promise) override {
+    auto promise_ptr = std::make_shared<td::Promise<Ref<DataCell>>>(std::move(promise));
     auto info = hash_table_.get_if_exists(hash);
     if (info && info->sync_with_db) {
-      TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell());
-      promise.set_result(loaded_cell.data_cell);
+      executor->execute_async([promise = std::move(promise_ptr), cell = info->cell]() mutable {
+        TRY_RESULT_PROMISE((*promise), loaded_cell, cell->load_cell());
+        promise->set_result(loaded_cell.data_cell);
+      });
       return;
     }
     SimpleExtCellCreator ext_cell_creator(cell_db_reader_);
-    auto promise_ptr = std::make_shared<td::Promise<Ref<DataCell>>>(std::move(promise));
     executor->execute_async(
         [executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this,
          ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable {

emulator/CMakeLists.txt

@@ -47,8 +47,8 @@ endif()
 if (USE_EMSCRIPTEN)
   add_executable(emulator-emscripten ${EMULATOR_EMSCRIPTEN_SOURCE})
   target_link_libraries(emulator-emscripten PUBLIC emulator)
-  target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=_malloc,free,UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8)
-  target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version)
+  target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8)
+  target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_malloc,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version)
   target_link_options(emulator-emscripten PRIVATE -sEXPORT_NAME=EmulatorModule)
   target_link_options(emulator-emscripten PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0)
   target_link_options(emulator-emscripten PRIVATE -Oz)

keyring/keyring.cpp

@@ -28,7 +28,7 @@ namespace ton {
 namespace keyring {
 
 KeyringImpl::PrivateKeyDescr::PrivateKeyDescr(PrivateKey private_key, bool is_temp)
-    : public_key(private_key.compute_public_key()), is_temp(is_temp) {
+    : public_key(private_key.compute_public_key()), private_key(private_key), is_temp(is_temp) {
   auto D = private_key.create_decryptor_async();
   D.ensure();
   decryptor_sign = D.move_as_ok();
@@ -190,6 +190,16 @@ void KeyringImpl::decrypt_message(PublicKeyHash key_hash, td::BufferSlice data,
   }
 }
 
+void KeyringImpl::export_all_private_keys(td::Promise<std::vector<PrivateKey>> promise) {
+  std::vector<PrivateKey> keys;
+  for (auto& [_, descr] : map_) {
+    if (!descr->is_temp && descr->private_key.exportable()) {
+      keys.push_back(descr->private_key);
+    }
+  }
+  promise.set_value(std::move(keys));
+}
+
 td::actor::ActorOwn<Keyring> Keyring::create(std::string db_root) {
   return td::actor::create_actor<KeyringImpl>("keyring", db_root);
 }

keyring/keyring.h

@@ -44,6 +44,8 @@ class Keyring : public td::actor::Actor {
 
   virtual void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise<td::BufferSlice> promise) = 0;
 
+  virtual void export_all_private_keys(td::Promise<std::vector<PrivateKey>> promise) = 0;
+
   static td::actor::ActorOwn<Keyring> create(std::string db_root);
 };
 

keyring/keyring.hpp

@@ -33,6 +33,7 @@ class KeyringImpl : public Keyring {
     td::actor::ActorOwn<DecryptorAsync> decryptor_sign;
     td::actor::ActorOwn<DecryptorAsync> decryptor_decrypt;
     PublicKey public_key;
+    PrivateKey private_key;
     bool is_temp;
     PrivateKeyDescr(PrivateKey private_key, bool is_temp);
   };
@@ -56,6 +57,8 @@ class KeyringImpl : public Keyring {
 
   void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise<td::BufferSlice> promise) override;
 
+  void export_all_private_keys(td::Promise<std::vector<PrivateKey>> promise) override;
+
   KeyringImpl(std::string db_root) : db_root_(db_root) {
   }