From 59f7d352bfd2ff3c1fdcd26f48d764268fba6329 Mon Sep 17 00:00:00 2001
From: Jean Humann <jean.humann@cleyrop.com>
Date: Thu, 14 Mar 2024 18:18:12 +0100
Subject: [PATCH] ci(docker): add build for ARM64

---
 .github/workflows/publish-registry-pr.yml |  2 ++
 .github/workflows/publish-registry.yml    | 20 +++++++++---
 .github/workflows/release.yml             |  2 +-
 Dockerfile                                |  5 ++-
 scripts/setup_spark.sh                    | 38 +++++++++++++++++++++++
 5 files changed, 60 insertions(+), 7 deletions(-)
 create mode 100755 scripts/setup_spark.sh

diff --git a/.github/workflows/publish-registry-pr.yml b/.github/workflows/publish-registry-pr.yml
index 5129c1e0..1fef9601 100644
--- a/.github/workflows/publish-registry-pr.yml
+++ b/.github/workflows/publish-registry-pr.yml
@@ -11,6 +11,8 @@ jobs:
     strategy:
       matrix:
         include:
+          - spark_version: 3.3.4
+            hadoop_version: 3
           - spark_version: 3.4.2
             hadoop_version: 3
           - spark_version: 3.5.1
diff --git a/.github/workflows/publish-registry.yml b/.github/workflows/publish-registry.yml
index 53a369db..48519237 100644
--- a/.github/workflows/publish-registry.yml
+++ b/.github/workflows/publish-registry.yml
@@ -1,6 +1,10 @@
 name: Publish image to Github container registry
 on:
   workflow_call: { }
+env:
+  TARGET_PLATFORMS: linux/amd64,linux/arm64
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   publish-registry:
@@ -8,21 +12,27 @@ jobs:
     strategy:
       matrix:
         include:
+          - spark_version: 3.3.4
+            hadoop_version: 3
           - spark_version: 3.4.2
             hadoop_version: 3
           - spark_version: 3.5.1
             hadoop_version: 3
     steps:
-      - uses: docker/setup-buildx-action@v2
-      - uses: docker/login-action@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
-          registry: ghcr.io
+          registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ github.token }}
-      - uses: docker/build-push-action@v3
+      - uses: docker/build-push-action@v5
         with:
           push: true
           build-args: |
             SPARK_VERSION=${{ matrix.spark_version }}
             HADOOP_VERSION=${{ matrix.hadoop_version }}
-          tags: ghcr.io/exacaster/lighter:${{ github.event.release.tag_name }}-spark${{ matrix.spark_version }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}-spark${{ matrix.spark_version }}
+          platforms: ${{ env.TARGET_PLATFORMS }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e912b75d..06aaa457 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -6,4 +6,4 @@ on:
 
 jobs:
   publish-registry:
-    uses: exacaster/lighter/.github/workflows/publish-registry.yml@master
+    uses: ./.github/workflows/publish-registry.yml
diff --git a/Dockerfile b/Dockerfile
index 0cb153ad..0943d78c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,7 +18,10 @@ ENV APP_BASE_URL='/lighter'
 
 WORKDIR /home/app/
 COPY frontend/ ./frontend/
-RUN wget "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -O - | tar -xz
+COPY scripts/setup_spark.sh spark.sh
+RUN apk update; \
+    apk add bash curl; \
+    wget "$(./spark.sh ${SPARK_VERSION})/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -O - | tar -xz
 
 WORKDIR /home/app/frontend/
 RUN yarn install && yarn build
diff --git a/scripts/setup_spark.sh b/scripts/setup_spark.sh
new file mode 100755
index 00000000..5cfc72be
--- /dev/null
+++ b/scripts/setup_spark.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Required env variable: SPARK_VERSION
+
+# Function to get all the references for a given webpage
+get_all_refs() {
+	curl -s "$1" | grep -o '<a href=['"'"'"][^"'"'"']*['"'"'"]' | sed -e 's/^<a href=["'"'"']//' -e 's/["'"'"']$//' -e 's/\///'
+}
+
+# Function to get stable versions of Spark
+get_spark_stable_versions() {
+	all_refs=$(get_all_refs "https://dlcdn.apache.org/spark/")
+	echo -n "${all_refs}" | grep -o 'spark-[0-9]\+\.[0-9]\+\.[0-9]\+' | sed -e 's/spark-//' | sort -r
+}
+
+get_last_minors() {
+	refs=$(get_spark_stable_versions)
+	versions=()
+	minors=$(echo -n "${refs}" | grep -o '[0-9]\+\.[0-9]\+' | uniq | head -n 2)
+	# shellcheck disable=SC2068
+	for minor in ${minors[@]}; do
+		versions+=("$(echo -n "${refs}" | grep -o "${minor}\.[0-9]\+" | head -n 1)")
+	done
+	echo -n "${versions[@]}"
+}
+
+# Function to get the download URL for a given Spark version
+spark_url() {
+	stable_versions=$(get_spark_stable_versions)
+	if echo -n "${stable_versions}" | grep -q "$1"; then
+		echo "https://dlcdn.apache.org/spark"
+	else
+		echo "https://archive.apache.org/dist/spark"
+	fi
+}
+
+# Main
+spark_url "$1"