From 5fa38f2444b1127dfe29c938a16369ed1d845134 Mon Sep 17 00:00:00 2001 From: Dainius Jocas Date: Tue, 25 May 2021 15:38:34 +0300 Subject: [PATCH] chore: linux static image compilation (#13) --- Makefile | 23 +++++++--- README.md | 4 +- deps.edn | 38 ++++++++++++++++- dockerfiles/Dockerfile.executable-builder | 42 ++++++++++++++++--- resources/KET_VERSION | 1 + resources/logback.xml | 1 + script/compile | 51 +++++++++++++++++++++++ script/setup-musl | 32 ++++++++++++++ src/core.clj | 27 +++++++----- 9 files changed, 196 insertions(+), 23 deletions(-) create mode 100644 resources/KET_VERSION create mode 100755 script/compile create mode 100755 script/setup-musl diff --git a/Makefile b/Makefile index 5d06ac8..5ecf477 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,13 @@ include dockerfiles/docker.mk +.PHONY: pom.xml +pom.xml: + clojure -Spom + +.PHONY: uberjar +uberjar: pom.xml + clojure -X:uberjar :jar target/ket-uber.jar :main-class core + .PHONY: lint lint: clojure -M:clj-kondo @@ -27,8 +35,13 @@ run-integration-tests: docker-compose $(ES_TEST) build docker-compose $(ES_TEST) up --remove-orphans --abort-on-container-exit --exit-code-from tools-test -build-ket: - docker build -f dockerfiles/Dockerfile.executable-builder -t ket-native-image . - docker rm ket-native-image-build || true - docker create --name ket-native-image-build ket-native-image - docker cp ket-native-image-build:/usr/src/app/ket ket +docker_build = (docker build --build-arg $1 --build-arg $2 -f dockerfiles/Dockerfile.executable-builder -t ket-native-image .; \ + docker rm ket-native-image-build || true; \ + docker create --name ket-native-image-build ket-native-image; \ + docker cp ket-native-image-build:/usr/src/app/ket ket) + +build: + clojure -M:native-image + +build-linux-static: + $(call docker_build, KET_STATIC=true, KET_MUSL=true) diff --git a/README.md b/README.md index 06749f2..fb8d63c 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,9 @@ Also, in the same fashion the script can be provided: ## Logging -Logging is controlled by the [logback](http://logback.qos.ch/) library. The output layout is JSON (you can query it with `jq` or collect logs with logstash or beats). +Logging is controlled by the [logback](http://logback.qos.ch/) library. +Logger logs to `System.err`. +The output layout is JSON (you can query it with `jq` or collect logs with logstash). Default logging level is `INFO`. When executed as a binary, i.e. `./ket OPERATION`, then logging levels are controlled by an environment variable called: `ROOT_LOGGER_LEVEL`, e.g. `ROOT_LOGGER_LEVEL=WARN ./ket operation -f config.json` Acceptable values of the `ROOT_LOGGER_LEVEL` are: `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `FATAL`. diff --git a/deps.edn b/deps.edn index ac920ec..bdb76e6 100644 --- a/deps.edn +++ b/deps.edn @@ -42,8 +42,15 @@ {:main-opts ["-m" "clj-kondo.main" "--lint" "src" "test"] :extra-deps {clj-kondo/clj-kondo {:mvn/version "2021.03.03"}} :jvm-opts ["-Dclojure.main.report=stderr"]} - :native-ket + :uberjar + {:replace-deps {com.github.seancorfield/depstar {:mvn/version "2.0.216"}} + :exec-fn hf.depstar/uberjar + :exec-args {:aot true}} + :native-image-linux-static-musl {:main-opts ["-m" "clj.native-image" "core" + "--static" + "--libc=musl" + "--verbose" "--enable-https" "--no-fallback" "--language:js" @@ -60,6 +67,35 @@ "-H:IncludeResources=logback.xml" "-H:Name=ket"] :jvm-opts ["-Dclojure.compiler.direct-linking=true"] + :extra-deps {org.jboss.logging/commons-logging-jboss-logging + {:mvn/version "1.0.0.Final"} + borkdude/clj-reflector-graal-java11-fix + {:mvn/version "0.0.1-graalvm-20.3.0" + :exclusions [org.graalvm.nativeimage/svm]} + clj.native-image/clj.native-image + {:git/url "https://github.com/taylorwood/clj.native-image.git" + :exclusions [commons-logging/commons-logging + org.slf4j/slf4j-nop] + :sha "f3e40672d5c543b80a2019c1f07b2d3fe785962c"}}} + :native-image + {:main-opts ["-m" "clj.native-image" "core" + "--enable-https" + "--no-fallback" + "--language:js" + "--verbose" + "--allow-incomplete-classpath" + "--initialize-at-build-time" + "--enable-all-security-services" + "--initialize-at-run-time=org.httpkit.client.HttpClient" + "--initialize-at-run-time=org.httpkit.client.SslContextFactory" + "--report-unsupported-elements-at-runtime" + "-J-Dclojure.compiler.direct-linking=true" + "-H:ReflectionConfigurationFiles=graalvm/reflect-config.json" + ;; optional native image name override + "-H:+ReportExceptionStackTraces" + "-H:IncludeResources=logback.xml" + "-H:Name=ket"] + :jvm-opts ["-Dclojure.compiler.direct-linking=true"] :extra-deps {org.jboss.logging/commons-logging-jboss-logging {:mvn/version "1.0.0.Final"} borkdude/clj-reflector-graal-java11-fix diff --git a/dockerfiles/Dockerfile.executable-builder b/dockerfiles/Dockerfile.executable-builder index e88ccaf..e846d35 100644 --- a/dockerfiles/Dockerfile.executable-builder +++ b/dockerfiles/Dockerfile.executable-builder @@ -1,11 +1,12 @@ -FROM findepi/graalvm:java11-native as BUILDER +FROM findepi/graalvm:21.0.0.2-java11-all as BUILDER ENV GRAALVM_HOME=/graalvm ENV JAVA_HOME=/graalvm ENV CLOJURE_VERSION=1.10.3.814 -RUN apt-get install -y curl \ - && gu install native-image \ +RUN apt-get update && apt-get install -y curl git make + +RUN gu install native-image \ && gu install ruby \ && gu install python \ && gu install r \ @@ -14,13 +15,44 @@ RUN apt-get install -y curl \ && ./linux-install-$CLOJURE_VERSION.sh \ && rm linux-install-$CLOJURE_VERSION.sh +ENV MUSL_DIR=${HOME}/.musl +ENV MUSL_VERSION=1.2.2 +ENV ZLIB_VERSION=1.2.11 + +RUN mkdir $MUSL_DIR \ + && curl https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz -o musl-${MUSL_VERSION}.tar.gz \ + && tar zxvf musl-${MUSL_VERSION}.tar.gz \ + && cd musl-${MUSL_VERSION} \ + && ./configure --disable-shared --prefix=${MUSL_DIR} \ + && make \ + && make install \ + && curl https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz -o zlib-${ZLIB_VERSION}.tar.gz \ + && tar zxvf zlib-${ZLIB_VERSION}.tar.gz \ + && cd zlib-${ZLIB_VERSION} \ + && ./configure --static --prefix=${MUSL_DIR} \ + && make \ + && make install + RUN mkdir -p /usr/src/app WORKDIR /usr/src/app +ENV PATH=$PATH:${MUSL_DIR}/bin + COPY deps.edn /usr/src/app/ -RUN clojure -P -M:native-ket +RUN clojure -P && clojure -P -M:uberjar COPY resources/ /usr/src/app/resources COPY graalvm/ /usr/src/app/graalvm/ COPY src/ /usr/src/app/src +COPY script/ /usr/src/app/script + +RUN clojure -Spom +RUN clojure -X:uberjar :jar target/ket-uber.jar :main-class core + +ARG KET_STATIC +ENV KET_STATIC=$KET_STATIC + +ARG KET_MUSL +ENV KET_MUSL=$KET_MUSL -RUN clojure -M:native-ket +# RUN script/compile +RUN clojure -M:native-image-linux-static-musl diff --git a/resources/KET_VERSION b/resources/KET_VERSION new file mode 100644 index 0000000..e38f400 --- /dev/null +++ b/resources/KET_VERSION @@ -0,0 +1 @@ +v2021.05.24 diff --git a/resources/logback.xml b/resources/logback.xml index 47dad76..56a178c 100644 --- a/resources/logback.xml +++ b/resources/logback.xml @@ -1,6 +1,7 @@ + ${LOGGER_TARGET:-System.err} diff --git a/script/compile b/script/compile new file mode 100755 index 0000000..0b8edf9 --- /dev/null +++ b/script/compile @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eou pipefail + +KET_XMX=${KET_XMX:-"-J-Xmx48500m"} + +if [ -z "$GRAALVM_HOME" ]; then + echo "Please set GRAALVM_HOME" + exit 1 +fi + +KET_JAR=${KET_JAR:-"target/ket-uber.jar"} + +if [[ ! -f "$KET_JAR" ]] +then + echo "Please run make uberjar first." + exit 1 +fi + +args=( "-jar" + "$KET_JAR" + "-H:Name=ket" + "-H:IncludeResources=KET_VERSION" + "--verbose" + "--no-server" + "--enable-https" + "--no-fallback" + "--language:js" + "--allow-incomplete-classpath" + "--initialize-at-build-time" + "--enable-all-security-services" + "--initialize-at-run-time=org.httpkit.client.HttpClient" + "--initialize-at-run-time=org.httpkit.client.SslContextFactory" + "--report-unsupported-elements-at-runtime" + "-J-Dclojure.compiler.direct-linking=true" + "-H:ReflectionConfigurationFiles=graalvm/reflect-config.json" + "-H:+ReportExceptionStackTraces" + "-H:IncludeResources=logback.xml" + "$KET_XMX") + +KET_STATIC=${KET_STATIC:-} +KET_MUSL=${KET_MUSL:-} + +if [ "$KET_STATIC" = "true" ]; then + args+=("--static") + if [ "$KET_MUSL" = "true" ]; then + args+=("--libc=musl") + fi +fi + +"$GRAALVM_HOME/bin/native-image" "${args[@]}" diff --git a/script/setup-musl b/script/setup-musl new file mode 100755 index 0000000..a187a13 --- /dev/null +++ b/script/setup-musl @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +MUSL_DIR=${HOME}/.musl + +MUSL_GCC_COMPILER=${HOME}/.musl/bin/musl-gcc + +if [ -f "$MUSL_GCC_COMPILER" ]; then + echo "MUSL is already setup at ${MUSL_DIR}" + exit 0 +fi + +MUSL_VERSION=1.2.2 +ZLIB_VERSION=1.2.11 + +mkdir $MUSL_DIR || true +cd $MUSL_DIR +curl https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz -o musl-${MUSL_VERSION}.tar.gz \ + && tar zxvf musl-${MUSL_VERSION}.tar.gz \ + && cd musl-${MUSL_VERSION} \ + && ./configure --disable-shared --prefix=${MUSL_DIR} \ + && make \ + && make install \ + && curl https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz -o zlib-${ZLIB_VERSION}.tar.gz \ + && tar zxvf zlib-${ZLIB_VERSION}.tar.gz \ + && cd zlib-${ZLIB_VERSION} \ + && ./configure --static --prefix=${MUSL_DIR} \ + && make \ + && make install + +export PATH=$PATH:${MUSL_DIR}/bin diff --git a/src/core.clj b/src/core.clj index fddd2b4..62a6110 100644 --- a/src/core.clj +++ b/src/core.clj @@ -12,6 +12,8 @@ (:import (org.slf4j LoggerFactory) (ch.qos.logback.classic Logger Level))) +(def version (str/trim (slurp (io/resource "KET_VERSION")))) + (defn find-operation [operation-name cli-operations] (first (filter (fn [op] (= (name operation-name) (:name op))) cli-operations))) @@ -35,16 +37,14 @@ resp (cond (true? (:docs options)) (:docs operation) (true? (:defaults options)) (:defaults operation))] - (println - (json/encode - (if-let [msg (if (empty? resp) - (if (empty? options) - (throw (Exception. (format "Configuration for the operation '%s' is bad: '%s'" - (name operation-name) options))) - ((:handler-fn operation) options)) - resp)] - msg - (format "Operation '%s' is finished" (name operation-name)))))) + (if-let [msg (if (empty? resp) + (if (empty? options) + (throw (Exception. (format "Configuration for the operation '%s' is bad: '%s'" + (name operation-name) options))) + ((:handler-fn operation) options)) + resp)] + (println (json/encode msg)) + (log/infof "Operation '%s' is finished" (name operation-name)))) (throw (Exception. "Operation name was not provided")))) (defn handle-subcommand [{:keys [options] :as cli-opts} cli-operations] @@ -78,6 +78,11 @@ (def cli-operations (concat ops/operations ops-overrides/cli)) +(defn print-summary-msg [summary] + (println (format "ket %s" version)) + (println "Supported options:") + (println summary)) + (defn handle-cli [args] (let [{:keys [options summary errors arguments] :as cli-opts} (cli/recursive-parse args cli-operations)] (if errors @@ -87,7 +92,7 @@ (if (or (get options :help) (and (empty? options) (empty? arguments)) (empty? args)) - (println summary) + (print-summary-msg summary) (handle-subcommand cli-opts cli-operations))))) (comment