feat(docker): allow for docker release builds to be multi-platform (#…

…27055)
apache · Feb 9, 2024 · 13915bb · 13915bb
1 parent 5951f6c
commit 13915bb
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 53 deletions.
diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
@@ -2,7 +2,7 @@ name: Docker Publish Release
 
 on:
   release:
-    types: [published]
+    types: [published, edited]
 
   # Can be triggered manually
   workflow_dispatch:
@@ -43,14 +43,6 @@ jobs:
     strategy:
       matrix:
         build_preset: ["dev", "lean", "py310", "websocket", "dockerize"]
-        platform: ["linux/amd64", "linux/arm64"]
-        exclude:
-          # disabling because slow! no python wheels for arm/py39 and
-          # QEMU is slow!
-          - build_preset: "dev"
-            platform: "linux/arm64"
-          - build_preset: "lean"
-            platform: "linux/arm64"
       fail-fast: false
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -88,8 +80,10 @@ jobs:
             EVENT="release"
           fi
           pip install click
+          # Make a multi-platform image
           ./scripts/build_docker.py \
             ${{ matrix.build_preset }} \
             "$EVENT" \
-            --build_context_ref "$RELEASE" \
-            --platform ${{ matrix.platform }} $FORCE_LATEST
+            --build_context_ref "$RELEASE" $FORCE_LATEST \
+            --platform "linux/arm64" \
+            --platform "linux/amd64"
diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml
@@ -143,10 +143,9 @@ jobs:
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY: superset-ci
-          SHA: ${{ steps.get-sha.outputs.sha }}
           IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
         run: |
-          docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$SHA
+          docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
           docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
 
   ephemeral-env-up:
@@ -181,7 +180,7 @@ jobs:
         aws ecr describe-images \
         --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
         --repository-name superset-ci \
-        --image-ids imageTag=${{ steps.get-sha.outputs.sha }}
+        --image-ids imageTag=pr-${{ github.event.issue.number }}-ci
 
     - name: Fail on missing container image
       if: steps.check-image.outcome == 'failure'
@@ -204,7 +203,7 @@ jobs:
       with:
         task-definition: .github/workflows/ecs-task-definition.json
         container-name: superset-ci
-        image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}
+        image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
 
     - name: Update env vars in the Amazon ECS task definition
       run: |

diff --git a/scripts/build_docker.py b/scripts/build_docker.py
@@ -40,6 +40,10 @@ def run_cmd(command: str) -> str:
             output += line
 
     process.wait()  # Wait for the subprocess to finish
+
+    if process.returncode != 0:
+        raise subprocess.CalledProcessError(process.returncode, command, output)
+
     return output
 
 
@@ -79,7 +83,7 @@ def make_docker_tag(l: list[str]) -> str:
 
 def get_docker_tags(
     build_preset: str,
-    build_platform: str,
+    build_platforms: list[str],
     sha: str,
     build_context: str,
     build_context_ref: str,
@@ -91,17 +95,18 @@ def get_docker_tags(
     tags: set[str] = set()
     tag_chunks: list[str] = []
 
-    short_build_platform = build_platform.replace("linux/", "").replace("64", "")
-
     is_latest = is_latest_release(build_context_ref)
 
     if build_preset != "lean":
         # Always add the preset_build name if different from default (lean)
         tag_chunks += [build_preset]
 
-    if short_build_platform != "amd":
-        # Always a platform indicator if different from default (amd)
-        tag_chunks += [short_build_platform]
+    if len(build_platforms) == 1:
+        build_platform = build_platforms[0]
+        short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+        if short_build_platform != "amd":
+            # Always a platform indicator if different from default (amd)
+            tag_chunks += [short_build_platform]
 
     # Always craft a tag for the SHA
     tags.add(make_docker_tag([sha] + tag_chunks))
@@ -123,7 +128,7 @@ def get_docker_tags(
 
 def get_docker_command(
     build_preset: str,
-    build_platform: str,
+    build_platforms: list[str],
     is_authenticated: bool,
     sha: str,
     build_context: str,
@@ -160,7 +165,7 @@ def get_docker_command(
 
     tags = get_docker_tags(
         build_preset,
-        build_platform,
+        build_platforms,
         sha,
         build_context,
         build_context_ref,
@@ -170,8 +175,14 @@ def get_docker_command(
 
     docker_args = "--load" if not is_authenticated else "--push"
     target_argument = f"--target {build_target}" if build_target else ""
-    short_build_platform = build_platform.replace("linux/", "").replace("64", "")
-    cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+
+    cache_ref = f"{CACHE_REPO}:{py_ver}"
+    if len(build_platforms) == 1:
+        build_platform = build_platforms[0]
+        short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+        cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+    platform_arg = "--platform " + ",".join(build_platforms)
+
     cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
     cache_to_arg = (
         f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
@@ -187,7 +198,7 @@ def get_docker_command(
         {cache_from_arg} \\
         {cache_to_arg} \\
         {build_arg} \\
-        --platform {build_platform} \\
+        {platform_arg} \\
         --label sha={sha} \\
         --label target={build_target} \\
         --label build_trigger={build_context} \\
@@ -206,20 +217,23 @@ def get_docker_command(
 @click.option(
     "--platform",
     type=click.Choice(["linux/arm64", "linux/amd64"]),
-    default="linux/amd64",
+    default=["linux/amd64"],
+    multiple=True,
 )
 @click.option("--build_context_ref", help="a reference to the pr, release or branch")
 @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
+@click.option("--verbose", is_flag=True, help="Print more info")
 @click.option(
     "--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
 )
 def main(
     build_preset: str,
     build_context: str,
     build_context_ref: str,
-    platform: str,
+    platform: list[str],
     dry_run: bool,
     force_latest: bool,
+    verbose: bool,
 ) -> None:
     """
     This script executes docker build and push commands based on given arguments.
@@ -262,6 +276,8 @@ def main(
                 """
             )
         script = script + docker_build_command
+        if verbose:
+            run_cmd("cat Dockerfile")
         stdout = run_cmd(script)
     else:
         print("Dry Run - Docker Build Command:")