ci(docker): add build for ARM64

.github/workflows/publish-registry-pr.yml

@@ -11,6 +11,8 @@ jobs:
     strategy:
       matrix:
         include:
+          - spark_version: 3.3.4
+            hadoop_version: 3
           - spark_version: 3.4.2
             hadoop_version: 3
           - spark_version: 3.5.1

.github/workflows/publish-registry.yml

@@ -1,28 +1,38 @@
 name: Publish image to Github container registry
 on:
   workflow_call: { }
+env:
+  TARGET_PLATFORMS: linux/amd64,linux/arm64
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   publish-registry:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
+          - spark_version: 3.3.4
+            hadoop_version: 3
           - spark_version: 3.4.2
             hadoop_version: 3
           - spark_version: 3.5.1
             hadoop_version: 3
     steps:
-      - uses: docker/setup-buildx-action@v2
-      - uses: docker/login-action@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
-          registry: ghcr.io
+          registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ github.token }}
-      - uses: docker/build-push-action@v3
+      - uses: docker/build-push-action@v5
         with:
           push: true
           build-args: |
             SPARK_VERSION=${{ matrix.spark_version }}
             HADOOP_VERSION=${{ matrix.hadoop_version }}
-          tags: ghcr.io/exacaster/lighter:${{ github.event.release.tag_name }}-spark${{ matrix.spark_version }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}-spark${{ matrix.spark_version }}
+          platforms: ${{ env.TARGET_PLATFORMS }}

.github/workflows/release.yml

@@ -6,4 +6,4 @@ on:
 
 jobs:
   publish-registry:
-    uses: exacaster/lighter/.github/workflows/publish-registry.yml@master
+    uses: ./.github/workflows/publish-registry.yml

Dockerfile

@@ -18,7 +18,10 @@ ENV APP_BASE_URL='/lighter'
 
 WORKDIR /home/app/
 COPY frontend/ ./frontend/
-RUN wget "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -O - | tar -xz
+COPY scripts/setup_spark.sh spark.sh
+RUN apk update; \
+    apk add bash curl; \
+    wget "$(./spark.sh ${SPARK_VERSION})/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -O - | tar -xz
 
 WORKDIR /home/app/frontend/
 RUN yarn install && yarn build

scripts/setup_spark.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Required env variable: SPARK_VERSION
+
+# Function to get all the references for a given webpage
+get_all_refs() {
+	curl -s "$1" | grep -o '<a href=['"'"'"][^"'"'"']*['"'"'"]' | sed -e 's/^<a href=["'"'"']//' -e 's/["'"'"']$//' -e 's/\///'
+}
+
+# Function to get stable versions of Spark
+get_spark_stable_versions() {
+	all_refs=$(get_all_refs "https://dlcdn.apache.org/spark/")
+	echo -n "${all_refs}" | grep -o 'spark-[0-9]\+\.[0-9]\+\.[0-9]\+' | sed -e 's/spark-//' | sort -r
+}
+
+get_last_minors() {
+	refs=$(get_spark_stable_versions)
+	versions=()
+	minors=$(echo -n "${refs}" | grep -o '[0-9]\+\.[0-9]\+' | uniq | head -n 2)
+	# shellcheck disable=SC2068
+	for minor in ${minors[@]}; do
+		versions+=("$(echo -n "${refs}" | grep -o "${minor}\.[0-9]\+" | head -n 1)")
+	done
+	echo -n "${versions[@]}"
+}
+
+# Function to get the download URL for a given Spark version
+spark_url() {
+	stable_versions=$(get_spark_stable_versions)
+	if echo -n "${stable_versions}" | grep -q "$1"; then
+		echo "https://dlcdn.apache.org/spark"
+	else
+		echo "https://archive.apache.org/dist/spark"
+	fi
+}
+
+# Main
+spark_url "$1"