Refactor docker image build scripts

All versions of docker images are controlled by docker/version.sh. Now, it builds images with specific tags.
pedro-lino · Jul 28, 2020 · 1bdb6b3 · 1bdb6b3
1 parent 4b565a0
commit 1bdb6b3
Show file tree

Hide file tree

Showing 28 changed files with 423 additions and 279 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -8,16 +8,12 @@ stages:
   - cleanup
 
 variables:
-  ALPINE_VERSION: "3.10"
-  GOLANG_VERSION: "1.12.14"
-  PYTHON_VERSION: "3.7.5"
-  JAVA_VERSION: "11.0.4"
-  REPO_GO_DEPS: "goloop/go-deps/${CI_COMMIT_REF_SLUG}"
-  REPO_PY_DEPS: "goloop/py-deps/${CI_COMMIT_REF_SLUG}"
-  REPO_JAVA_DEPS: "goloop/java-deps/${CI_COMMIT_REF_SLUG}"
-  REPO_GOLOOP: "goloop/${CI_COMMIT_REF_SLUG}"
-  REPO_GOLOOP_PY: "goloop-py/${CI_COMMIT_REF_SLUG}"
-  REPO_GOLOOP_JAVA: "goloop-java/${CI_COMMIT_REF_SLUG}"
+  IMAGE_GO_DEPS: "goloop/go-deps:${CI_COMMIT_REF_NAME}"
+  IMAGE_PY_DEPS: "goloop/py-deps/${CI_COMMIT_REF_NAME}"
+  IMAGE_JAVA_DEPS: "goloop/java-deps:${CI_COMMIT_REF_NAME}"
+  IMAGE_GOLOOP: "goloop:${CI_COMMIT_REF_NAME}"
+  IMAGE_GOLOOP_PY: "goloop-py:${CI_COMMIT_REF_NAME}"
+  IMAGE_GOLOOP_JAVA: "goloop-java:${CI_COMMIT_REF_NAME}"
   REGISTRY: "ci.arch.iconloop.com:80"
 
 before_script:
@@ -48,15 +44,15 @@ deps:java:
     - docker-build
 
 unit:go:
-  image: "${REPO_GO_DEPS}"
+  image: "${IMAGE_GO_DEPS}"
   stage: unit
   script:
     - make test
   tags:
     - docker
 
 unit:py:
-  image: "${REPO_PY_DEPS}"
+  image: "${IMAGE_PY_DEPS}"
   stage: unit
   script:
     - cd pyee
@@ -65,7 +61,7 @@ unit:py:
     - docker
 
 unit:java:
-  image: "${REPO_JAVA_DEPS}"
+  image: "${IMAGE_JAVA_DEPS}"
   stage: unit
   script:
     - cd javaee
@@ -74,7 +70,7 @@ unit:java:
     - docker
 
 build:go:
-  image: "${REPO_GO_DEPS}"
+  image: "${IMAGE_GO_DEPS}"
   stage: build
   script:
     - make goloop
@@ -87,21 +83,21 @@ build:go:
     - docker
 
 build:pyee:
-  image: "${REPO_PY_DEPS}"
+  image: "${IMAGE_PY_DEPS}"
   stage: build
   script:
     - cd pyee
     - pip3 install wheel
-    - python3 setup.py bdist_wheel
+    - python3 setup.py bdist_wheel -d ../build/pyee/dist
   artifacts:
     expire_in: 1 day
     paths:
-      - pyee/dist/
+      - build/pyee/dist/
   tags:
     - docker
 
 build:java:
-  image: "${REPO_JAVA_DEPS}"
+  image: "${IMAGE_JAVA_DEPS}"
   stage: build
   script:
     - cd javaee
@@ -404,19 +400,19 @@ pages:IT:goloop:interscore:
 push:goloop:py:
   <<: *push
   variables:
-    REPO: "${REPO_GOLOOP_PY}"
+    REPO: "${IMAGE_GOLOOP_PY}"
     REPO_TAG: "goloop-py:${CI_COMMIT_REF_NAME}"
 
 push:goloop:java:
   <<: *push
   variables:
-    REPO: "${REPO_GOLOOP_JAVA}"
+    REPO: "${IMAGE_GOLOOP_JAVA}"
     REPO_TAG: "goloop-java:${CI_COMMIT_REF_NAME}"
 
 push:goloop:
   <<: *push
   variables:
-    REPO: "${REPO_GOLOOP}"
+    REPO: "${IMAGE_GOLOOP}"
     REPO_TAG: "goloop:${CI_COMMIT_REF_NAME}"
 
 .cleanup:integration:

diff --git a/Makefile b/Makefile
@@ -130,31 +130,20 @@ javarun-% : javadeps-image
 goloop-image: pyrun-pyexec gorun-goloop-linux javarun-javaexec
 	@ echo "[#] Building image $(GOLOOP_IMAGE) for $(GL_VERSION)"
 	@ rm -rf $(GOLOOP_DOCKER_DIR)
-	@ mkdir -p $(GOLOOP_DOCKER_DIR)/dist/pyee
-	@ mkdir -p $(GOLOOP_DOCKER_DIR)/dist/bin
-	@ cp $(BUILD_ROOT)/docker/goloop/* $(GOLOOP_DOCKER_DIR)
-	@ cp $(PYEE_DIST_DIR)/*.whl $(GOLOOP_DOCKER_DIR)/dist/pyee
-	@ cp $(LINUX_BIN_DIR)/goloop $(GOLOOP_DOCKER_DIR)/dist/bin
-	@ cp $(BUILD_ROOT)/javaee/app/execman/build/distributions/*.zip $(GOLOOP_DOCKER_DIR)/dist
-	@ docker build -t $(GOLOOP_IMAGE) \
-	    --build-arg TAG_PY_DEPS=$(GL_TAG) \
-	    --build-arg GOLOOP_VERSION=$(GL_VERSION) \
-	    --build-arg JAVAEE_VERSION=$(JAVAEE_VERSION) \
-	    $(GOLOOP_DOCKER_DIR)
+	@ \
+	BIN_DIR=$(BUILD_ROOT)/linux \
+	IMAGE_PY_DEPS=$(PYDEPS_IMAGE) \
+	GOLOOP_VERSION=$(GL_VERSION) \
+	$(BUILD_ROOT)/docker/goloop/update.sh $(GOLOOP_IMAGE) $(BUILD_ROOT) $(GOLOOP_DOCKER_DIR)
 
 gochain-image: pyrun-pyexec gorun-gochain-linux javarun-javaexec
 	@ echo "[#] Building image $(GOCHAIN_IMAGE) for $(GL_VERSION)"
 	@ rm -rf $(GOCHAIN_DOCKER_DIR)
-	@ mkdir -p $(GOCHAIN_DOCKER_DIR)/dist
-	@ cp $(BUILD_ROOT)/docker/gochain/* $(GOCHAIN_DOCKER_DIR)
-	@ cp $(PYEE_DIST_DIR)/*.whl $(GOCHAIN_DOCKER_DIR)/dist
-	@ cp $(LINUX_BIN_DIR)/gochain $(GOCHAIN_DOCKER_DIR)/dist
-	@ cp $(BUILD_ROOT)/javaee/app/execman/build/distributions/*.zip $(GOCHAIN_DOCKER_DIR)/dist
-	@ docker build -t $(GOCHAIN_IMAGE) \
-	    --build-arg TAG_PY_DEPS=$(GL_TAG) \
-	    --build-arg GOCHAIN_VERSION=$(GL_VERSION) \
-	    --build-arg JAVAEE_VERSION=$(JAVAEE_VERSION) \
-	    $(GOCHAIN_DOCKER_DIR)
+	@ \
+	BIN_DIR=$(BUILD_ROOT)/linux \
+	IMAGE_PY_DEPS=$(PYDEPS_IMAGE) \
+	GOCHAIN_VERSION=$(GL_VERSION) \
+	$(BUILD_ROOT)/docker/gochain/update.sh $(GOCHAIN_IMAGE) $(BUILD_ROOT) $(GOCHAIN_DOCKER_DIR)
 
 .PHONY: test
 

diff --git a/docker/function.sh b/docker/function.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+get_hash_of_files() {
+    cat "$@" | sha1sum | cut -d " " -f1
+}
+
+get_label_of_image() {
+    local LABEL=$1
+    local TAG=$2
+    docker image inspect -f "{{.Config.Labels.${LABEL}}}" ${TAG} 2> /dev/null || echo 'none'
+}
+
+get_id_with_hash() {
+    local REPO=$1
+    local LABEL=$2
+    local HASH=$3
+    docker images --filter="reference=${REPO}" --filter="label=${LABEL}=${HASH}" --format="{{.ID}}" | head -n 1
+}
diff --git a/docker/go-deps/Dockerfile b/docker/go-deps/Dockerfile
@@ -1,5 +1,5 @@
-ARG GOLANG_VERSION=1.12.14
-ARG ALPINE_VERSION=3.10
+ARG GOLANG_VERSION
+ARG ALPINE_VERSION
 FROM golang:${GOLANG_VERSION}-alpine${ALPINE_VERSION}
 RUN apk add make git
 ENV GO111MODULE on
@@ -10,4 +10,4 @@ LABEL GOLOOP_GOMOD_SHA="$GOLOOP_GOMOD_SHA"
 ADD go.mod go.sum /goloop/
 WORKDIR /goloop
 
-RUN echo "go mod download $GOMOD_SHA" && go mod download
+RUN echo "go mod download $GOLOOP_GOMOD_SHA" && go mod download
diff --git a/docker/go-deps/build.sh b/docker/go-deps/build.sh
@@ -5,14 +5,8 @@ PRE_PWD=$(pwd)
 WORKDIR=$(dirname "$(readlink -f ${0})")
 cd $WORKDIR
 
-REPO_GO_DEPS=${REPO_GO_DEPS:-goloop/go-deps}
+IMAGE_GO_DEPS=${IMAGE_GO_DEPS:-goloop/go-deps:latest}
 
-./update.sh "${REPO_GO_DEPS}" ../..
-
-if [ "${TAG_GO_DEPS}" != "" ] && [ "${TAG_GO_DEPS}" != "latest" ];then
-  TAG_SLUG=${TAG_GO_DEPS//\//__}
-  echo "Tag image ${REPO_GO_DEPS} to ${REPO_GO_DEPS}:${TAG_SLUG} for TAG_GO_DEPS:${TAG_GO_DEPS}"
-  docker tag ${REPO_GO_DEPS} ${REPO_GO_DEPS}:${TAG_SLUG}
-fi
+./update.sh "${IMAGE_GO_DEPS}" ../..
 
 cd $PRE_PWD
diff --git a/docker/go-deps/update.sh b/docker/go-deps/update.sh
@@ -1,64 +1,72 @@
 #!/bin/sh
 
 BASE_DIR=$(dirname $0)
-GOLANG_VERSION=${GOLANG_VERSION:-1.12.14}
+. ${BASE_DIR}/../version.sh
+. ${BASE_DIR}/../function.sh
+
+LABEL="GOLOOP_GOMOD_SHA"
 
 get_hash_of_dir() {
     local SRC_DIR=$1
-    local SUM=$(sha1sum "${SRC_DIR}/go.sum" | cut -d ' ' -f 1)
-    echo "${GOLANG_VERSION}-${SUM}"
-}
-
-get_hash_of_image() {
-    local TAG=$1
-    docker image inspect -f '{{.Config.Labels.GOLOOP_GOMOD_SHA}}' ${TAG} 2> /dev/null || echo 'none'
+    local SUM=$(get_hash_of_files \
+        "${SRC_DIR}/go.mod" "${SRC_DIR}/go.sum" \
+        "${SRC_DIR}/docker/go-deps/Dockerfile" )
+    echo "${GOLANG_VERSION}-alpine${ALPINE_VERSION}-${SUM}"
 }
 
 update_image() {
     if [ $# -lt 1 ] ; then
-	echo "Usage: $0 <image_name> [<src_dir>] [<build_dir>]"
-	return 1
+        echo "Usage: $0 <image_name> [<src_dir>] [<build_dir>]"
+        return 1
     fi
 
-    local TAG=$1
+    local TARGET_IMAGE=$1
+    local TARGET_REPO=${TARGET_IMAGE%%:*}
     local SRC_DIR=$2
     if [ -z "${SRC_DIR}" ] ; then
-	SRC_DIR="."
+        SRC_DIR="."
     fi
     local BUILD_DIR=$3
 
-    local GOLOOP_GOMOD_SHA=$(get_hash_of_dir ${SRC_DIR})
-    local IMAGE_GOMOD_SHA=$(get_hash_of_image ${TAG})
+    local HASH_OF_DIR=$(get_hash_of_dir ${SRC_DIR})
+    local HASH_OF_IMAGE=$(get_label_of_image ${LABEL} ${TARGET_IMAGE})
 
-    if [ "${GOLOOP_GOMOD_SHA}" != "${IMAGE_GOMOD_SHA}" ] ; then
-	# Prepare build directory if it's set
-	if [ "${BUILD_DIR}" != "" ] ; then
-	    rm -rf ${BUILD_DIR}
-	    mkdir -p ${BUILD_DIR}
-	    cp ${BASE_DIR}/* ${BUILD_DIR}
-	else
-	    BUILD_DIR=${BASE_DIR}
-	fi
+    if [ "${HASH_OF_DIR}" != "${HASH_OF_IMAGE}" ] ; then
+        local IMAGE_ID=$(get_id_with_hash ${TARGET_REPO} ${LABEL} ${HASH_OF_DIR})
+        if [ "${IMAGE_ID}" != "" ]; then
+            echo "Tagging image ${IMAGE_ID} as ${TARGET_IMAGE}"
+            docker tag ${IMAGE_ID} ${TARGET_IMAGE}
+            return $?
+        fi
 
-	cp ${SRC_DIR}/go.sum ${SRC_DIR}/go.mod ${BUILD_DIR}/
+        # Prepare build directory if it's set
+        if [ "${BUILD_DIR}" != "" ] ; then
+            rm -rf ${BUILD_DIR}
+            mkdir -p ${BUILD_DIR}
+            cp ${BASE_DIR}/* ${BUILD_DIR}
+        else
+            BUILD_DIR=${BASE_DIR}
+        fi
 
-        CDIR=$(pwd)
-	cd ${BUILD_DIR}
+        cp ${SRC_DIR}/go.sum ${SRC_DIR}/go.mod ${BUILD_DIR}/
 
-	echo "Building image ${TAG} for ${GOLOOP_GOMOD_SHA}"
-	docker build \
-	    --build-arg GOLOOP_GOMOD_SHA=${GOLOOP_GOMOD_SHA} \
-	    --build-arg GOLANG_VERSION=${GOLANG_VERSION} \
-	    --tag ${TAG} .
-	local result=$?
+        CDIR=$(pwd)
+        cd ${BUILD_DIR}
 
-	rm -f go.sum go.mod
-	cd ${CDIR}
+        echo "Building image ${TARGET_IMAGE} for ${HASH_OF_DIR}"
+        docker build \
+            --build-arg ${LABEL}=${HASH_OF_DIR} \
+            --build-arg GOLANG_VERSION=${GOLANG_VERSION} \
+            --build-arg ALPINE_VERSION=${ALPINE_VERSION} \
+            --tag ${TARGET_IMAGE} .
+        local result=$?
 
-	return $result
+        rm -f go.sum go.mod
+        cd ${CDIR}
+        return $result
     else
-	echo "Already exist image ${TAG} for ${GOLOOP_GOMOD_SHA}"
-	return 0
+        echo "Reuse image ${TARGET_IMAGE} for ${HASH_OF_DIR}"
+        return 0
     fi
     return 0
 }

diff --git a/docker/gochain/Dockerfile b/docker/gochain/Dockerfile
@@ -1,6 +1,5 @@
-ARG REPO_PY_DEPS=goloop/py-deps
-ARG TAG_PY_DEPS=latest
-FROM ${REPO_PY_DEPS}:${TAG_PY_DEPS}
+ARG IMAGE_PY_DEPS
+FROM ${IMAGE_PY_DEPS}
 LABEL MAINTAINER="[email protected]"
 
 # install JRE

diff --git a/docker/gochain/build.sh b/docker/gochain/build.sh
@@ -5,44 +5,10 @@ PRE_PWD=$(pwd)
 WORKDIR=$(dirname "$(readlink -f ${0})")
 cd $WORKDIR
 
-PYTHON_VERSION=${PYTHON_VERSION:-3.7.5}
-SHASUM=$(cat ../../pyee/requirements.txt \
-             ../../docker/py-deps/Dockerfile \
-         | sha1sum | cut -d ' ' -f 1)
-PYDEP_SHA=${PYTHON_VERSION}-${SHASUM}
-REPO_PY_DEPS=${REPO_PY_DEPS:-goloop/py-deps}
-TAG_PY_DEPS=${TAG_PY_DEPS:-$(docker images --filter="reference=$REPO_PY_DEPS" --filter="label=GOLOOP_PYDEP_SHA=${PYDEP_SHA}" --format="{{.Tag}}" | head -n 1)}
-if [ "${TAG_PY_DEPS}" != "" ]; then
-  TAG_SLUG=${TAG_PY_DEPS//\//__}
-  BUILD_ARG_TAG_PY_DEPS="--build-arg=TAG_PY_DEPS=${TAG_SLUG}"
-fi
+export GOCHAIN_VERSION=${GOCHAIN_VERSION:-$(git describe --always --tags --dirty)}
+export IMAGE_PY_DEPS=${IMAGE_PY_DEPS:-goloop/py-deps:latest}
+IMAGE_GOCHAIN=${IMAGE_GOCHAIN:-goloop/gochain:latest}
 
-GOCHAIN_VERSION=${GOCHAIN_VERSION:-$(git describe --always --tags --dirty)}
-REPO_GOCHAIN=${REPO_GOCHAIN:-goloop/gochain}
-PRE_GOCHAIN_VERSION=$(docker image inspect ${REPO_GOCHAIN} -f "{{.Config.Labels.GOCHAIN_VERSION}}" || echo "none")
-if [ "${GOCHAIN_VERSION}" != "${PRE_GOCHAIN_VERSION}" ]
-then
-  echo "Build image ${REPO_GOCHAIN} using ${REPO_PY_DEPS} with TAG_PY_DEPS:${TAG_PY_DEPS}"
-  JAVAEE_VERSION=$(grep "^VERSION=" ../../javaee/gradle.properties | cut -d= -f2)
-  mkdir dist
-  cp ../../pyee/dist/pyexec-*.whl ./dist/
-  cp ../../bin/gochain ./dist/
-  cp ../../javaee/app/execman/build/distributions/execman-*.zip ./dist/
-  docker build \
-    --build-arg REPO_PY_DEPS=${REPO_PY_DEPS} \
-    ${BUILD_ARG_TAG_PY_DEPS} \
-    --build-arg GOCHAIN_VERSION=${GOCHAIN_VERSION} \
-    --build-arg JAVAEE_VERSION=${JAVAEE_VERSION} \
-    --tag ${REPO_GOCHAIN} .
-  rm -rf dist
-else
-  echo "Already exists image ${REPO_GOCHAIN}"
-fi
-
-if [ "${TAG_GOCHAIN}" != "" ] && [ "${TAG_GOCHAIN}" != "latest" ]; then
-  TAG_SLUG=${TAG_GOCHAIN//\//__}
-  echo "Tag image ${REPO_GOCHAIN} to ${REPO_GOCHAIN}:${TAG_SLUG} for TAG_GOCHAIN:${TAG_GOCHAIN}"
-  docker tag ${REPO_GOCHAIN} ${REPO_GOCHAIN}:${TAG_SLUG}
-fi
+./update.sh "${IMAGE_GOCHAIN}" ../..
 
 cd $PRE_PWD