Merge branch 'master' into docs/update-arch-process

Author: jfrader

Cargo.lock

@@ -100,17 +100,22 @@ ENV LIGHTNINGD_VERSION=master
 
 RUN dpkg --add-architecture ${target_arch_dpkg}
 
-#TODO: python3-dev needs QEMU for post install scripts. find a workaround to not use QEMU
+# Install architecture-independent libraries
 RUN apt-get update && \
     apt-get install -qq -y --no-install-recommends \
-        pkg-config:${target_arch_dpkg} \
-        libffi-dev:${target_arch_dpkg} \
-        python3-dev:${target_arch_dpkg} \
-        libicu-dev:${target_arch_dpkg} \
-        zlib1g-dev:${target_arch_dpkg} \
-        libsqlite3-dev:${target_arch_dpkg} \
-        libpq-dev:${target_arch_dpkg} \
-        crossbuild-essential-${target_arch_dpkg}
+        python3-dev \
+        lowdown
+
+# Install target-arch libraries
+RUN apt-get install -qq -y --no-install-recommends \
+    pkg-config:${target_arch_dpkg} \
+    libffi-dev:${target_arch_dpkg} \
+    libicu-dev:${target_arch_dpkg} \
+    zlib1g-dev:${target_arch_dpkg} \
+    libsqlite3-dev:${target_arch_dpkg} \
+    libpq-dev:${target_arch_dpkg} \
+    libsodium-dev:${target_arch_dpkg} \
+    crossbuild-essential-${target_arch_dpkg}
 
 ARG AR=${target_arch}-ar
 ARG AS=${target_arch}-as
@@ -139,6 +144,8 @@ RUN ./install-uv.sh -q
 RUN ./install-rust.sh -y -q --profile minimal --component rustfmt --target ${target_arch_rust}
 
 ENV PATH="/root/.cargo/bin:/root/.local/bin:${PATH}"
+ENV PKG_CONFIG_PATH=/usr/lib/${target_arch}/pkgconfig
+ENV PKG_CONFIG_LIBDIR=/usr/lib/${target_arch}/pkgconfig
 
 WORKDIR /opt/lightningd
 
@@ -161,7 +168,9 @@ RUN apt-get update && \
         inotify-tools \
         socat \
         jq \
-        libpq5 && \
+        libpq5 \
+        libsqlite3-0 \
+        libsodium23 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

Dockerfile

@@ -263,14 +263,26 @@ man8dir = $(mandir)/man8
 ifeq ("$(OS)-$(ARCH)", "Darwin-arm64")
 CPATH := /opt/homebrew/include
 LIBRARY_PATH := /opt/homebrew/lib
-LDFLAGS := -L/opt/homebrew/opt/sqlite/lib -L/opt/homebrew/opt/openssl@3/lib
-CPPFLAGS := -I/opt/homebrew/opt/sqlite/include -I/opt/homebrew/opt/openssl@3/include
-PKG_CONFIG_PATH=/opt/homebrew/opt/sqlite/lib/pkgconfig
 else
 CPATH := /usr/local/include
 LIBRARY_PATH := /usr/local/lib
 endif
 
+# Detect OpenSSL and SQLite paths dynamically using brew --prefix
+ifeq ("$(OS)", "Darwin")
+OPENSSL_PREFIX := $(shell brew --prefix openssl@3 2>/dev/null || brew --prefix openssl 2>/dev/null || echo "")
+SQLITE_PREFIX := $(shell brew --prefix sqlite 2>/dev/null || echo "")
+ifneq ("$(OPENSSL_PREFIX)", "")
+LDFLAGS += -L$(OPENSSL_PREFIX)/lib
+CPPFLAGS += -I$(OPENSSL_PREFIX)/include
+endif
+ifneq ("$(SQLITE_PREFIX)", "")
+LDFLAGS += -L$(SQLITE_PREFIX)/lib
+CPPFLAGS += -I$(SQLITE_PREFIX)/include
+PKG_CONFIG_PATH := $(SQLITE_PREFIX)/lib/pkgconfig:$(PKG_CONFIG_PATH)
+endif
+endif
+
 CPPFLAGS += -DCLN_NEXT_VERSION="\"$(CLN_NEXT_VERSION)\"" -DPKGLIBEXECDIR="\"$(pkglibexecdir)\"" -DBINDIR="\"$(bindir)\"" -DPLUGINDIR="\"$(plugindir)\"" -DCCAN_TAL_NEVER_RETURN_NULL=1
 CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(SODIUM_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS)
 
@@ -392,7 +404,12 @@ include cln-grpc/Makefile
 endif
 include plugins/Makefile
 include tests/plugins/Makefile
+
+# Only include fuzz tests if OpenSSL >= 3.0, will be disabled on ubuntu focal
+OPENSSL_VERSION := $(shell openssl version | sed -n 's/OpenSSL \([0-9]\+\)\..*/\1/p')
+ifneq ($(shell test $(OPENSSL_VERSION) -ge 3 && echo yes),)
 include tests/fuzz/Makefile
+endif
 
 ifneq ($V,1)
 MSGGEN_ARGS := -s

Makefile

@@ -11,7 +11,6 @@
 static void *cln_wally_tal(size_t size)
 {
 	assert(wally_tal_ctx);
-	assert(tal_check(wally_tal_ctx, "cln_wally_tal ctx check"));
 	return tal_arr_label(wally_tal_ctx, u8, size, "cln_wally_tal");
 }
 

common/setup.c

@@ -20,7 +20,17 @@ RUN dnf update -y && \
 		jq \
 		xz \
 		zlib-devel \
-		cargo && \
+		cargo \
+		libsodium-devel && \
+	wget https://github.com/kristapsdz/lowdown/archive/refs/tags/VERSION_1_0_2.tar.gz && \
+	tar -xzf VERSION_1_0_2.tar.gz && \
+	cd lowdown-VERSION_1_0_2 && \
+	./configure && \
+	make && \
+	make install && \
+	ldconfig && \
+	cd /tmp && \
+	rm -rf VERSION_1_0_2.tar.gz lowdown-VERSION_1_0_2 && \
 	dnf clean all
 
 RUN wget https://storage.googleapis.com/c-lightning-tests/bitcoind/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz \

contrib/docker/Dockerfile.builder.fedora

@@ -27137,7 +27137,7 @@
             "description": [
               "It specifies the type of address wanted; currently *bech32* (e.g. `tb1qu9j4lg5f9rgjyfhvfd905vw46eg39czmktxqgg` on bitcoin testnet or `bc1qwqdg6squsna38e46795at95yu9atm8azzmyvckulcc7kytlcckxswvvzej` on bitcoin mainnet), or *p2tr* taproot addresses. The special value *all* generates all known address types for the same underlying key."
             ],
-            "default": "*bech32* address",
+            "default": "*p2tr* address",
             "enum": [
               "bech32",
               "p2tr",

contrib/msggen/msggen/schema.json

@@ -1106,7 +1106,7 @@ def restart(self, timeout=10, clean=True):
     def fundchannel(self, l2, amount=FUNDAMOUNT, wait_for_active=True,
                     announce_channel=True, **kwargs):
         # Give yourself some funds to work with
-        addr = self.rpc.newaddr()['bech32']
+        addr = self.rpc.newaddr('bech32')['bech32']
 
         def has_funds_on_addr(addr):
             """Check if the given address has funds in the internal wallet.
@@ -1737,7 +1737,7 @@ def join_nodes(self, nodes, fundchannel=True, fundamount=FUNDAMOUNT, wait_for_an
         bitcoind = nodes[0].bitcoin
         # If we got here, we want to fund channels
         for src, dst in connections:
-            addr = src.rpc.newaddr()['bech32']
+            addr = src.rpc.newaddr('bech32')['bech32']
             bitcoind.rpc.sendtoaddress(addr, (fundamount + 1000000) / 10**8)
 
         bitcoind.generate_block(1)

contrib/pyln-testing/pyln/testing/utils.py

@@ -11,22 +11,32 @@ RUN sed -i '/updates/d' /etc/apt/sources.list && \
     sed -i '/security/d' /etc/apt/sources.list
 
 RUN apt-get update \
-	&& apt-get install -y --no-install-recommends \
-	autoconf \
-	build-essential \
-	ca-certificates \
-	file \
-	gettext \
-	git \
-	libpq-dev \
-	libsodium23 \
-	libtool \
-	m4 \
-	sudo \
-	unzip \
-	wget \
-	git \
-	zip
+    && apt-get install -y --no-install-recommends \
+    autoconf \
+    build-essential \
+    ca-certificates \
+    file \
+    gettext \
+    git \
+    libpq-dev \
+    libsodium23 \
+    libsodium-dev \
+    libtool \
+    m4 \
+    sudo \
+    unzip \
+    wget \
+    zip \
+    && cd /tmp \
+    && wget https://github.com/kristapsdz/lowdown/archive/refs/tags/VERSION_1_0_2.tar.gz \
+    && tar -xzf VERSION_1_0_2.tar.gz \
+    && cd lowdown-VERSION_1_0_2 \
+    && ./configure \
+    && make \
+    && make install \
+    && ldconfig \
+    && cd / \
+    && rm -rf /tmp/VERSION_1_0_2.tar.gz /tmp/lowdown-VERSION_1_0_2
 
 # Ensure correct ownership
 RUN chown root:root /etc/sudoers

contrib/reprobuild/Dockerfile.focal

@@ -10,24 +10,35 @@ ENV PROTOC_VERSION=29.4
 RUN sed -i '/updates/d' /etc/apt/sources.list && \
     sed -i '/security/d' /etc/apt/sources.list
 
-RUN apt-get update \
-	&& apt-get install -y --no-install-recommends \
-	autoconf \
-	build-essential \
-	ca-certificates \
-	file \
-	gettext \
-	git \
-	libsqlite3-dev \
-	libpq-dev \
-	libsodium23 \
-	libtool \
-	m4 \
-	sudo \
-	unzip \
-	wget \
-	jq \
-	zip
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        autoconf \
+        build-essential \
+        ca-certificates \
+        file \
+        gettext \
+        git \
+        libsqlite3-dev \
+        libpq-dev \
+        libsodium23 \
+        libsodium-dev \
+        libtool \
+        m4 \
+        sudo \
+        unzip \
+        wget \
+        jq \
+        zip \
+    && cd /tmp \
+    && wget https://github.com/kristapsdz/lowdown/archive/refs/tags/VERSION_1_0_2.tar.gz \
+    && tar -xzf VERSION_1_0_2.tar.gz \
+    && cd lowdown-VERSION_1_0_2 \
+    && ./configure \
+    && make \
+    && make install \
+    && ldconfig \
+    && cd / \
+    && rm -rf /tmp/VERSION_1_0_2.tar.gz /tmp/lowdown-VERSION_1_0_2
 
 # Ensure correct ownership
 RUN chown root:root /etc/sudoers

contrib/reprobuild/Dockerfile.jammy

@@ -22,6 +22,8 @@ RUN apt-get update \
     libsqlite3-dev \
 	libpq-dev \
 	libsodium23 \
+    libsodium-dev \
+    lowdown \
 	libtool \
 	m4 \
 	sudo \