Merge branch 'develop' of https://github.com/dashpay/dash into develop

ci/test/05_before_script.sh

@@ -33,7 +33,7 @@ if [ -z "$NO_DEPENDS" ]; then
   else
     SHELL_OPTS="CONFIG_SHELL="
   fi
-  DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+  DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1
 fi
 if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
   DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"

contrib/guix/manifest.scm

@@ -590,6 +590,7 @@ parse, modify and abstract ELF, PE and MachO formats.")
         zlib
         (list zlib "static")
         ;; Build tools
+        cmake-minimal
         gnu-make
         libtool-2.4.7
         autoconf-2.71
@@ -615,5 +616,5 @@ parse, modify and abstract ELF, PE and MachO formats.")
           ((string-contains target "-linux-")
            (list (make-bitcoin-cross-toolchain target)))
           ((string-contains target "darwin")
-           (list clang-toolchain-10 binutils cmake-minimal xorriso python-signapple))
+           (list clang-toolchain-10 binutils xorriso python-signapple))
           (else '())))))

depends/Makefile

@@ -42,6 +42,7 @@ NO_UPNP ?=
 NO_USDT ?=
 NO_NATPMP ?=
 MULTIPROCESS ?=
+LTO ?=
 FALLBACK_DOWNLOAD_PATH ?= http://dash-depends-sources.s3-website-us-west-2.amazonaws.com
 
 BUILD = $(shell ./config.guess)
@@ -140,8 +141,8 @@ include packages/packages.mk
 #     2. Before including packages/*.mk (excluding packages/packages.mk), since
 #        they rely on the build_id variables
 #
-build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
-$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
+build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
+$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
 
 qrencode_packages_$(NO_QR) = $(qrencode_$(host_os)_packages)
 
@@ -239,6 +240,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@no_usdt@|$(NO_USDT)|' \
             -e 's|@no_natpmp@|$(NO_NATPMP)|' \
             -e 's|@multiprocess@|$(MULTIPROCESS)|' \
+            -e 's|@lto@|$(LTO)|' \
             -e 's|@debug@|$(DEBUG)|' \
             $< > $@
 	touch $@
@@ -271,7 +273,7 @@ clean-all: clean
 	@rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x*
 
 clean:
-	@rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD)
+	@rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) *.log
 
 install: check-packages $(host_prefix)/share/config.site
 

depends/README.md

@@ -41,7 +41,7 @@ Common `host-platform-triplet`s for cross compilation are:
 - `aarch64-linux-android` for Android ARM 64 bit
 - `x86_64-linux-android` for Android x86 64 bit
 
-The paths are automatically configured and no other options are needed unless targeting [Android](#Android).
+The paths are automatically configured and no other options are needed unless targeting [Android](../doc/build-android.md).
 
 ### Install the required dependencies: Ubuntu & Debian
 
@@ -116,7 +116,11 @@ The following can be set when running make: `make FOO=bar`
 - `BUILD_ID_SALT`: Optional salt to use when generating build package ids
 - `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the
   system's `$PATH` rather than the default prebuilt release of Clang
-  from llvm.org. Clang 8 or later is required.
+  from llvm.org. Clang 8 or later is required
+- `LOG`: Use file-based logging for individual packages. During a package build its log file
+  resides in the `depends` directory, and the log file is printed out automatically in case
+  of build error. After successful build log files are moved along with package archives
+- `LTO`: Use LTO when building packages.
 
 If some packages are not built, for example `make NO_WALLET=1`, the appropriate
 options will be passed to Dash Core's configure. In this case, `--disable-wallet`.
@@ -129,18 +133,6 @@ options will be passed to Dash Core's configure. In this case, `--disable-wallet
     download-linux: run 'make download-linux' to fetch all sources needed for linux builds
 
 
-### Android
-
-Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level).
-In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set.
-
-API levels from 24 to 29 have been tested to work.
-
-If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted.
-This is an example command for a default build with no disabled dependencies:
-
-    ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin
-
 ### Other documentation
 
 - [description.md](description.md): General description of the depends system

depends/config.site.in

@@ -78,6 +78,10 @@ if test "@host_os@" = darwin; then
   BREW=no
 fi
 
+if test -z "$enable_lto" && test -n "@lto@"; then
+  enable_lto=yes
+fi
+
 PATH="${depends_prefix}/native/bin:${PATH}"
 PKG_CONFIG="$(which pkg-config) --static"
 

depends/funcs.mk

@@ -67,6 +67,7 @@ $(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_bui
 $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id)
 $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/
 $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz
+$(1)_build_log:=$(BASEDIR)/$(1)-$($(1)_version)-$($(1)_build_id).log
 $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources)
 
 #stamps
@@ -85,7 +86,7 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path))
 # The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior
 $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash))
 $(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash)  $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash &&  $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_TAR) --no-same-owner --strip-components=1 -xf $$($(1)_source)
-$(1)_preprocess_cmds ?=
+$(1)_preprocess_cmds ?= true
 $(1)_build_cmds ?=
 $(1)_config_cmds ?=
 $(1)_stage_cmds ?=
@@ -174,7 +175,7 @@ $(1)_cmake=env CC="$$($(1)_cc)" \
                CXX="$$($(1)_cxx)" \
                CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \
                LDFLAGS="$$($(1)_ldflags)" \
-             cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_cmake_opts)
+             cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_config_opts)
 ifeq ($($(1)_type),build)
 $(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib"
 else
@@ -187,6 +188,10 @@ endif
 endef
 
 define int_add_cmds
+ifneq ($(LOG),)
+$(1)_logging = >>$$($(1)_build_log) 2>&1 || { if test -f $$($(1)_build_log); then cat $$($(1)_build_log); fi; exit 1; }
+endif
+
 $($(1)_fetched):
 	mkdir -p $$(@D) $(SOURCES_PATH)
 	rm -f $$@
@@ -203,23 +208,23 @@ $($(1)_preprocessed): | $($(1)_extracted)
 	echo Preprocessing $(1)...
 	mkdir -p $$(@D) $($(1)_patch_dir)
 	$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;)
-	cd $$(@D); $($(1)_preprocess_cmds)
+	{ cd $$(@D); $($(1)_preprocess_cmds); } $$($(1)_logging)
 	touch $$@
 $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed)
 	echo Configuring $(1)...
 	rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), $(build_TAR) --no-same-owner -xf $($(package)_cached); )
 	mkdir -p $$(@D)
-	+cd $$(@D); $($(1)_config_env) $($(1)_config_cmds)
+	+{ cd $$(@D); $($(1)_config_env) $($(1)_config_cmds); } $$($(1)_logging)
 	touch $$@
 $($(1)_built): | $($(1)_configured)
 	echo Building $(1)...
 	mkdir -p $$(@D)
-	+cd $$(@D); $($(1)_build_env) $($(1)_build_cmds)
+	+{ cd $$(@D); $($(1)_build_env) $($(1)_build_cmds); } $$($(1)_logging)
 	touch $$@
 $($(1)_staged): | $($(1)_built)
 	echo Staging $(1)...
 	mkdir -p $($(1)_staging_dir)/$(host_prefix)
-	cd $($(1)_build_dir); $($(1)_stage_env) $($(1)_stage_cmds)
+	+{ cd $($(1)_build_dir); $($(1)_stage_env) $($(1)_stage_cmds); } $$($(1)_logging)
 	rm -rf $($(1)_extract_dir)
 	touch $$@
 $($(1)_postprocessed): | $($(1)_staged)
@@ -233,6 +238,7 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed)
 	rm -rf $$(@D) && mkdir -p $$(@D)
 	mv $$($(1)_staging_dir)/$$(@F) $$(@)
 	rm -rf $($(1)_staging_dir)
+	if test -f $($(1)_build_log); then mv $($(1)_build_log) $$(@D); fi
 $($(1)_cached_checksum): $($(1)_cached)
 	cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@)
 

depends/gen_id

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Usage: env [ CC=... ] [ CXX=... ] [ AR=... ] [ RANLIB=... ] [ STRIP=... ] \
-#            [ DEBUG=... ] ./build-id [ID_SALT]...
+#            [ DEBUG=... ] [ LTO=... ] ./build-id [ID_SALT]...
 #
 # Prints to stdout a SHA256 hash representing the current toolset, used by
 # depends/Makefile as a build id for caching purposes (detecting when the
@@ -63,6 +63,10 @@
     env | grep '^STRIP_'
     echo "END STRIP"
 
+    echo "BEGIN LTO"
+    echo "LTO=${LTO}"
+    echo "END LTO"
+
     echo "END ALL"
 ) | if [ -n "$DEBUG" ] && command -v tee > /dev/null 2>&1; then
         # When debugging and `tee` is available, output the preimage to stderr

depends/hosts/android.mk

@@ -1,12 +1,17 @@
 ifeq ($(HOST),armv7a-linux-android)
-android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar
 android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++
 android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang
-android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib
 else
-android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar
 android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++
 android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang
-android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib
 endif
+
+ifneq ($(LTO),)
+android_CFLAGS += -flto
+android_LDFLAGS += -flto
+endif
+
+android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar
+android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib
+
 android_cmake_system=Android

depends/hosts/darwin.mk

@@ -113,6 +113,12 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
                -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 
 darwin_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+darwin_CFLAGS += -flto
+darwin_LDFLAGS += -flto
+endif
+
 darwin_CXXFLAGS=$(darwin_CFLAGS)
 
 darwin_release_CFLAGS=-O2

depends/hosts/freebsd.mk

@@ -1,4 +1,10 @@
 freebsd_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+freebsd_CFLAGS += -flto
+freebsd_LDFLAGS += -flto
+endif
+
 freebsd_CXXFLAGS=$(freebsd_CFLAGS)
 
 freebsd_release_CFLAGS=-O2

depends/hosts/linux.mk

@@ -1,4 +1,10 @@
 linux_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+linux_CFLAGS += -flto
+linux_LDFLAGS += -flto
+endif
+
 linux_CXXFLAGS=$(linux_CFLAGS)
 
 linux_release_CFLAGS=-O2

depends/hosts/mingw32.mk

@@ -3,6 +3,12 @@ mingw32_CXX := $(host)-g++-posix
 endif
 
 mingw32_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+mingw32_CFLAGS += -flto
+mingw32_LDFLAGS += -flto
+endif
+
 mingw32_CXXFLAGS=$(mingw32_CFLAGS)
 
 mingw32_release_CFLAGS=-O2

depends/hosts/netbsd.mk

@@ -1,4 +1,10 @@
 netbsd_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+netbsd_CFLAGS += -flto
+netbsd_LDFLAGS += -flto
+endif
+
 netbsd_CXXFLAGS=$(netbsd_CFLAGS)
 
 netbsd_release_CFLAGS=-O2

depends/hosts/openbsd.mk

@@ -1,6 +1,11 @@
 openbsd_CFLAGS=-pipe
 openbsd_CXXFLAGS=$(openbsd_CFLAGS)
 
+ifneq ($(LTO),)
+openbsd_CFLAGS += -flto
+openbsd_LDFLAGS += -flto
+endif
+
 openbsd_release_CFLAGS=-O2
 openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS)
 

depends/packages/libevent.mk

@@ -3,23 +3,27 @@ $(package)_version=2.1.12-stable
 $(package)_download_path=https://github.com/libevent/libevent/releases/download/release-$($(package)_version)/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb
+$(package)_patches=cmake_fixups.patch
+$(package)_patches+=fix_mingw_link.patch
+$(package)_build_subdir=build
 
 # When building for Windows, we set _WIN32_WINNT to target the same Windows
 # version as we do in configure. Due to quirks in libevents build system, this
 # is also required to enable support for ipv6. See #19375.
 define $(package)_set_vars
-  $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples
-  $(package)_config_opts += --disable-dependency-tracking --enable-option-checking
-  $(package)_config_opts_release=--disable-debug-mode
+  $(package)_config_opts=-DEVENT__DISABLE_BENCHMARK=ON -DEVENT__DISABLE_OPENSSL=ON
+  $(package)_config_opts+=-DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON
+  $(package)_config_opts+=-DEVENT__DISABLE_TESTS=ON -DEVENT__LIBRARY_TYPE=STATIC
   $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601
 endef
 
 define $(package)_preprocess_cmds
-  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux
+  patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch && \
+  patch -p1 < $($(package)_patch_dir)/fix_mingw_link.patch
 endef
 
 define $(package)_config_cmds
-  $($(package)_autoconf)
+  $($(package)_cmake) -S .. -B .
 endef
 
 define $(package)_build_cmds
@@ -31,7 +35,7 @@ define $(package)_stage_cmds
 endef
 
 define $(package)_postprocess_cmds
-  rm lib/*.la && \
+  rm bin/event_rpcgen.py && \
   rm include/ev*.h && \
   rm include/event2/*_compat.h
 endef

depends/packages/libmultiprocess.mk

@@ -10,8 +10,8 @@ endif
 
 define $(package)_set_vars :=
 ifneq ($(host),$(build))
-$(package)_cmake_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp"
-$(package)_cmake_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++"
+$(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp"
+$(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++"
 endif
 endef
 

depends/packages/qrencode.mk

@@ -3,20 +3,22 @@ $(package)_version=4.1.1
 $(package)_download_path=https://fukuchi.org/works/qrencode/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=e455d9732f8041cf5b9c388e345a641fd15707860f928e94507b1961256a6923
+$(package)_patches=cmake_fixups.patch
 
 define $(package)_set_vars
-$(package)_config_opts=--disable-shared --without-tools --without-tests --without-png
-$(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap
-$(package)_config_opts += --disable-dependency-tracking --enable-option-checking
+$(package)_config_opts := -DWITH_TOOLS=NO -DWITH_TESTS=NO -DGPROF=OFF -DCOVERAGE=OFF
+$(package)_config_opts += -DCMAKE_DISABLE_FIND_PACKAGE_PNG=TRUE -DWITHOUT_PNG=ON
+$(package)_config_opts += -DCMAKE_DISABLE_FIND_PACKAGE_ICONV=TRUE
 $(package)_cflags += -Wno-int-conversion -Wno-implicit-function-declaration
 endef
 
 define $(package)_preprocess_cmds
-  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub use
+  patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch
 endef
 
+
 define $(package)_config_cmds
-  $($(package)_autoconf)
+  $($(package)_cmake) -S . -B .
 endef
 
 define $(package)_build_cmds
@@ -26,7 +28,3 @@ endef
 define $(package)_stage_cmds
   $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef
-
-define $(package)_postprocess_cmds
-  rm lib/*.la
-endef