Fix ssl issue and improve musl builds

Signed-off-by: Heinz N. Gies <[email protected]>

.cargo/config

@@ -1,2 +1,11 @@
 [build]
 rustflags = ["-C", "target-cpu=native"]
+
+[target.x86_64-unknown-linux-musl]
+rustflags = [
+        "-C", "target-cpu=native",
+        "-C", "link-arg=-lc",
+        "-C", "link-arg=-lstdc++",
+        "-C", "link-arg=-lgcc",
+        "-C", "link-arg=-lgcc_eh",
+    ]

.depignore

@@ -0,0 +1 @@
+openssl

.github/checks/deps.sh

@@ -9,7 +9,7 @@ fi
 
 cnt=0
 
-for d in $(remarshal -i $path/Cargo.toml -of json | jq -r '.dependencies | keys []')
+for d in $(remarshal -i $path/Cargo.toml -of json | jq -r '.dependencies | keys []' | grep -v -f .depignore)
 do
     dep=$(echo $d | sed -e 's/-/_/g')
     if ! rg "use $dep(::|;| )" $path -trust > /dev/null

CHANGELOG.md

@@ -7,6 +7,8 @@
 * Default to thin-lto for all builds (prior this was only done in docker)
 * Automatically generate rpms and tarballs for releases.
 * Update rust to 1.49.0
+* Build deb packages
+* Statically link openssl
 
 ## 0.9.4
 

Cargo.toml

@@ -89,8 +89,8 @@ postgres-protocol = "0.6"
 tokio-postgres = "0.7"
 
 # kafka. cmake is the encouraged way to build this and also the one that works on windows/with musl.
-rdkafka = {version = "0.24", features = ["cmake-build", "libz"], default-features = false}
-rdkafka-sys = {version = "2.0.0", features = ["cmake-build", "libz"]}# tracking the version rdkafka depends on
+rdkafka = {version = "0.24", features = ["cmake-build", "libz-static"], default-features = false}
+rdkafka-sys = {version = "2.0.0", features = ["cmake-build", "libz-static"]}# tracking the version rdkafka depends on
 
 # crononome
 cron = "0.7.0"
@@ -100,7 +100,7 @@ grok = "1"
 
 # not used directly in tremor codebase, but present here so that we can turn
 # on features for these (see static-ssl feature here)
-#openssl = { version = "0.10" }
+openssl = { version = "0.10", features = ["vendored"] }
 
 # rest onramp
 tide = "0.15"

Makefile

@@ -27,7 +27,7 @@ RELEASE_TARGETS := \
 # packaging using debian image itself (might also need to have separate build for
 # archive packaging too).
 #RELEASE_FORMATS_x86_64-unknown-linux-gnu := archive,deb,rpm
-RELEASE_FORMATS_x86_64-unknown-linux-gnu := archive,rpm
+RELEASE_FORMATS_x86_64-unknown-linux-gnu := archive,rpm,deb
 RELEASE_FORMATS_x86_64-alpine-linux-musl := archive
 RELEASE_FORMATS_x86_64-unknown-linux-musl := archive
 

packaging/builder-images/Dockerfile.x86_64-debian-linux-gnu

@@ -0,0 +1,12 @@
+ARG RUST_VERSION
+
+# update the version here as needed
+# may bring updates to things like glibc so proceed carefully.
+# via https://hub.docker.com/_/rust
+FROM rust:${RUST_VERSION}-buster
+
+COPY shared/install_dependencies_debian.sh /
+RUN /install_dependencies_debian.sh
+
+COPY shared/entrypoint.sh /
+ENTRYPOINT [ "/entrypoint.sh" ]

packaging/builder-images/Dockerfile.x86_64-unknown-linux-musl

@@ -2,7 +2,7 @@ ARG RUST_VERSION
 
 # update the version here as needed
 # via https://hub.docker.com/_/rust
-FROM rust:${RUST_VERSION}-buster
+FROM rustembedded/cross:x86_64-unknown-linux-musl-0.2.1
 
 # install musl
 # this setup borrowed from https://github.com/rust-embedded/cross/blob/v0.2.0/docker/musl.sh#L44
@@ -16,21 +16,33 @@ RUN temp_dir=$(mktemp -d) \
   && cd $temp_dir \
   && curl -L https://github.com/richfelker/musl-cross-make/archive/v0.9.9.tar.gz | tar --strip-components=1 -xz \
   && make install -j$(nproc) \
-    GCC_VER=9.2.0 \
-    MUSL_VER=1.2.0 \
-    DL_CMD="curl -C - -L -o" \
-    OUTPUT=/usr/local/ \
-    TARGET=x86_64-linux-musl \
+  GCC_VER=9.2.0 \
+  MUSL_VER=1.2.0 \
+  DL_CMD="curl -C - -L -o" \
+  OUTPUT=/usr/local/ \
+  TARGET=x86_64-linux-musl \
   && rm -rf $temp_dir
 
-ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=x86_64-linux-musl-gcc \
-    CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc \
-    CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++
 
-RUN rustup target add x86_64-unknown-linux-musl
+RUN apt-get update; \
+  apt-get install -y \
+  cmake        `# for building C deps` \
+  libclang-dev `# for onig_sys (via the regex crate)` \
+  libssl-dev   `# for openssl (via surf)` \
+  libsasl2-dev `# for librdkafka` \
+  libzstd-dev  `# for librdkafka`; \
+  apt-get autoremove -y \
+  && apt-get clean -y \
+  && rm -rf /var/lib/apt/lists/*
+
+#COPY shared/entrypoint.sh /
+#ENTRYPOINT [ "/entrypoint.sh" ]
+
+
+ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=x86_64-linux-musl-gcc \
+  CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc \
+  CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++
 
-COPY shared/install_dependencies_debian.sh /
-RUN /install_dependencies_debian.sh
 
-COPY shared/entrypoint.sh /
-ENTRYPOINT [ "/entrypoint.sh" ]
+# COPY shared/entrypoint.sh /
+# ENTRYPOINT [ "/entrypoint.sh" ]

packaging/builder-images/publish_images.sh

@@ -14,7 +14,10 @@ set -o errexit
 # catch exit status for piped commands
 set -o pipefail
 
-RUST_TOOLCHAIN_FILE="../../rust-toolchain"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+
+RUST_TOOLCHAIN_FILE="${SCRIPT_DIR}/../../rust-toolchain"
 RUST_VERSION=$(<"$RUST_TOOLCHAIN_FILE")
 
 IMAGES=(
@@ -28,4 +31,3 @@ IMAGES=(
 for image in "${IMAGES[@]}"; do
   docker push "tremorproject/${image}"
 done
-./publish_images

packaging/builder-images/shared/install_dependencies_debian.sh

@@ -11,9 +11,11 @@ apt-get update
 apt-get install -y \
   cmake        `# for building C deps` \
   libclang-dev `# for onig_sys (via the regex crate)` \
-  libssl-dev   `# for openssl (via surf)`
+  libssl-dev   `# for openssl (via surf)` \
+  libsasl2-dev `# for librdkafka` \
+  libzstd-dev  `# for librdkafka`
 
 # cleanup
-echo apt-get autoremove -y \
+apt-get autoremove -y \
   && apt-get clean -y \
   && rm -rf /var/lib/apt/lists/*

packaging/cross_build.sh

@@ -102,6 +102,8 @@ fi
 export RUSTFLAGS="${RUSTFLAGS} ${CUSTOM_RUSTFLAGS[@]}"
 echo "RUSTFLAGS set to: ${RUSTFLAGS}"
 
+cross -V
+
 cross build -p tremor-cli "${BUILD_ARGS[@]}"
 
 TARGET_BIN="${ROOT_DIR}/target/${TARGET}/${BUILD_MODE}/${BIN_NAME}"