Merge branch 'ton-blockchain:testnet' into testnet

neodix42 · Jun 9, 2024 · 0c0bc47 · 0c0bc47
2 parents 30091a0 + 7841d75
commit 0c0bc47
Show file tree

Hide file tree

Showing 15 changed files with 168 additions and 130 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -1,3 +1,16 @@
+## 2024.06 Update
+
+1. Make Jemalloc default allocator
+2. Add candidate broadcasting and caching
+3. Limit per address speed for external messages broadcast by reasonably large number 
+4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks
+5. Extended statistics and logs for celldb usage, session stats, persistent state serialization
+6. Tonlib and explorer fixes
+7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all`
+8. Add valiator-console command to stop persistent state serialization
+9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only.
+
+
 ## 2024.04 Update
 
 1. Emulator: Single call optimized runGetMethod added

diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh
@@ -15,8 +15,6 @@ while getopts 't' flag; do
 done
 
 cp assembly/nix/linux-arm64* .
-cp assembly/nix/microhttpd.nix .
-cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
 if [ "$with_tests" = true ]; then
@@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
+
 nix-build linux-arm64-tonlib.nix
+
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp ./result/lib/fift/* artifacts/lib/

diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh
@@ -15,8 +15,6 @@ while getopts 't' flag; do
 done
 
 cp assembly/nix/linux-x86-64* .
-cp assembly/nix/microhttpd.nix .
-cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
 if [ "$with_tests" = true ]; then
@@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
+
 nix-build linux-x86-64-tonlib.nix
+
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp ./result/lib/fift/* artifacts/lib/

diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh
@@ -28,7 +28,9 @@ cp ./result-bin/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result-bin
+
 nix-build macos-tonlib.nix
+
 cp ./result/lib/libtonlibjson.dylib artifacts/
 cp ./result/lib/libemulator.dylib artifacts/
 cp ./result/lib/fift/* artifacts/lib/

diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix
@@ -6,41 +6,57 @@
 , testing ? false
 }:
 let
-  microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+  jemallocStatic = (staticOptions pkgs.jemalloc);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
+
 in
-with import microhttpdmy;
 stdenv.mkDerivation {
   pname = "ton";
   version = "dev-bin";
 
   src = ./.;
 
   nativeBuildInputs = with pkgs;
-    [
-      cmake ninja git pkg-config
-    ];
+    [  cmake ninja git pkg-config ];
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      jemallocStatic
+      secp256k1Static
+      libsodiumStatic.dev
+      glibc.static
     ];
 
-  makeStatic = true;
-  doCheck = testing;
-
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
     "-DBUILD_SHARED_LIBS=OFF"
     "-DCMAKE_LINK_SEARCH_START_STATIC=ON"
     "-DCMAKE_LINK_SEARCH_END_STATIC=ON"
-    "-DMHD_FOUND=1"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
-    "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800"
+    "-DTON_USE_JEMALLOC=ON"
   ];
 
+  makeStatic = true;
+  doCheck = testing;
+
   LDFLAGS = [
-     "-static-libgcc" "-static-libstdc++" "-static"
+    "-static-libgcc" "-static-libstdc++" "-static"
   ];
 }
diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix
@@ -5,9 +5,21 @@
 , stdenv ? pkgs.stdenv
 }:
 let
-     microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
 in
-with import microhttpdmy;
 pkgs.llvmPackages_16.stdenv.mkDerivation {
   pname = "ton";
   version = "dev-lib";
@@ -21,17 +33,19 @@ pkgs.llvmPackages_16.stdenv.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      secp256k1Static
+      libsodiumStatic.dev
     ];
 
   dontAddStaticConfigureFlags = false;
 
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
-    "-DMHD_FOUND=1"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
   ];
 
   LDFLAGS = [

diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix
@@ -6,41 +6,57 @@
 , testing ? false
 }:
 let
-  microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+  jemallocStatic = (staticOptions pkgs.jemalloc);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
+
 in
-with import microhttpdmy;
 stdenv.mkDerivation {
   pname = "ton";
   version = "dev-bin";
 
   src = ./.;
 
   nativeBuildInputs = with pkgs;
-    [
-      cmake ninja git pkg-config
-    ];
+    [  cmake ninja git pkg-config ];
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      jemallocStatic
+      secp256k1Static
+      libsodiumStatic.dev
+      glibc.static
     ];
 
-  makeStatic = true;
-  doCheck = testing;
-
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
     "-DBUILD_SHARED_LIBS=OFF"
     "-DCMAKE_LINK_SEARCH_START_STATIC=ON"
     "-DCMAKE_LINK_SEARCH_END_STATIC=ON"
-    "-DMHD_FOUND=1"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
-    "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800"
+    "-DTON_USE_JEMALLOC=ON"
   ];
 
+  makeStatic = true;
+  doCheck = testing;
+
   LDFLAGS = [
-     "-static-libgcc" "-static-libstdc++" "-static"
+    "-static-libgcc" "-static-libstdc++" "-fPIC"
   ];
 }
diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix
@@ -7,20 +7,35 @@
 , stdenv ? pkgs.stdenv
 }:
 let
-      system = builtins.currentSystem;
-
-          nixos1909 = (import (builtins.fetchTarball {
-            url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz";
-            sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy";
-          }) { inherit system; });
-          glibc227 = nixos1909.glibc // { pname = "glibc"; };
-          stdenv227 = let
-            cc = pkgs.wrapCCWith {
-              cc = nixos1909.buildPackages.gcc-unwrapped;
-              libc = glibc227;
-              bintools = pkgs.binutils.override { libc = glibc227; };
-            };
-          in (pkgs.overrideCC pkgs.stdenv cc);
+  system = builtins.currentSystem;
+
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
+
+  nixos1909 = (import (builtins.fetchTarball {
+    url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz";
+    sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy";
+  }) { inherit system; });
+  glibc227 = nixos1909.glibc // { pname = "glibc"; };
+  stdenv227 = let
+    cc = pkgs.wrapCCWith {
+      cc = nixos1909.buildPackages.gcc-unwrapped;
+      libc = glibc227;
+      bintools = pkgs.binutils.override { libc = glibc227; };
+    };
+  in (pkgs.overrideCC pkgs.stdenv cc);
 
 in
 stdenv227.mkDerivation {
@@ -34,7 +49,12 @@ stdenv227.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      secp256k1Static
+      libsodiumStatic.dev
     ];
 
   dontAddStaticConfigureFlags = false;

diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix
@@ -17,7 +17,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
 
   buildInputs = with pkgs;
    lib.forEach [
-        secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext (gnutls.override { withP11-kit = false; }).dev
+        secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext jemalloc (gnutls.override { withP11-kit = false; }).dev
       ]
       (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; }))
     ++ [
@@ -38,13 +38,13 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
+    "-DTON_USE_JEMALLOC=ON"
     "-DCMAKE_CROSSCOMPILING=OFF"
     "-DCMAKE_LINK_SEARCH_START_STATIC=ON"
     "-DCMAKE_LINK_SEARCH_END_STATIC=ON"
     "-DBUILD_SHARED_LIBS=OFF"
     "-DCMAKE_CXX_FLAGS=-stdlib=libc++"
     "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3"
-    "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800"
   ];
 
   LDFLAGS = [

diff --git a/assembly/nix/microhttpd.nix b/assembly/nix/microhttpd.nix