Wip

Author: welpaolo

.github/workflows/build.yaml

@@ -36,14 +36,14 @@ jobs:
           sudo snap install yq
           sudo snap install rockcraft --classic --edge    
 
-      # - name: Build image
-      #   run: sudo make build
+      - name: Build image
+        run: sudo make build
 
-      # - name: Build image (Jupyter)
-      #   run: sudo make build FLAVOUR=jupyter
+      - name: Build image (Jupyter)
+        run: sudo make build FLAVOUR=jupyter
 
-      # - name: Build image (Kyuubi)
-      #   run: sudo make build FLAVOUR=kyuubi
+      - name: Build image (Kyuubi)
+        run: sudo make build FLAVOUR=kyuubi
 
       - name: Build image (GPU)
         run: sudo make build FLAVOUR=spark-gpu
@@ -61,16 +61,15 @@ jobs:
           echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Change artifact permissions
-        run: sudo chmod a+r  ${{ steps.artifact.outputs.gpu_artifact_name }}
-        # run: sudo chmod a+r  ${{ steps.artifact.outputs.base_artifact_name }} ${{ steps.artifact.outputs.jupyter_artifact_name }} ${{ steps.artifact.outputs.kyuubi_artifact_name }} ${{ steps.artifact.outputs.gpu_artifact_name }}
+        run: sudo chmod a+r  ${{ steps.artifact.outputs.base_artifact_name }} ${{ steps.artifact.outputs.jupyter_artifact_name }} ${{ steps.artifact.outputs.kyuubi_artifact_name }} ${{ steps.artifact.outputs.gpu_artifact_name }}
 
       - name: Upload locally built artifact
         uses: actions/upload-artifact@v4
         with:
           name: charmed-spark
-          # ${{ steps.artifact.outputs.base_artifact_name }}
-          # ${{ steps.artifact.outputs.jupyter_artifact_name }}
-          # ${{ steps.artifact.outputs.kyuubi_artifact_name }}
-          path: |  
+          path: |
+            ${{ steps.artifact.outputs.base_artifact_name }}
+            ${{ steps.artifact.outputs.jupyter_artifact_name }}
+            ${{ steps.artifact.outputs.kyuubi_artifact_name }}
             ${{ steps.artifact.outputs.gpu_artifact_name }}
 

.github/workflows/integration-gpu.yaml

@@ -26,7 +26,7 @@ jobs:
         uses: machulav/[email protected]
         with:
           mode: start
-          github-token: ${{ secrets.GH_TOKEN }}
+          github-token: ${{ secrets.GHCR_TOKEN }}
           ec2-image-id: ${{ vars.AWS_EC2_IMAGE_ID }}
           ec2-instance-type: ${{ vars.AWS_EC2_INSTANCE_TYPE }}
           subnet-id: ${{ vars.AWS_DEFAULT_SUBNET_ID }}

.github/workflows/publish.yaml

@@ -74,6 +74,8 @@ jobs:
           echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
           KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
           echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+          echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Download artifact
         uses: actions/download-artifact@v4
@@ -185,6 +187,42 @@ jobs:
 
             docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG}
 
+            echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"
+            docker push ${IMAGE_NAME}:${VERSION_TAG}
+          fi
+      - name: Publish Charmed Spark GPU Image to Channel
+        run: |
+          # Unpack artifact
+          mv charmed-spark/${{ steps.artifact.outputs.gpu_artifact_name }} .
+          rmdir charmed-spark
+
+          REPOSITORY="ghcr.io/canonical/"
+          RISK=${{ needs.release_checks.outputs.risk }}
+          TRACK=${{ needs.release_checks.outputs.track }}
+          if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi
+      
+          # Import artifact into docker with new tag
+          sudo make docker-import \
+            FLAVOUR=spark-gpu \
+            REPOSITORY=${REPOSITORY} \
+            TAG=${TAG} \
+            -o ${{ steps.artifact.outputs.gpu_artifact_name }}
+
+          # Add relevant labels
+          COMMIT_ID=$(git log -1 --format=%H)
+          DESCRIPTION=$(yq .description images/charmed-spark-gpu/rockcraft.yaml | xargs)
+          
+          echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" -
+
+          echo "Publishing ${IMAGE_NAME}:${TAG}"
+          docker push ${IMAGE_NAME}:${TAG}
+      
+          if [[ "$RISK" == "edge" ]]; then
+            VERSION_LONG=$(make help FLAVOUR=spark-gpu | grep "Tag\:" | cut -d ":" -f2  | xargs)
+            VERSION_TAG="${VERSION_LONG}-${{ needs.release_checks.outputs.base }}_edge"
+      
+            docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG}
+      
             echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"
             docker push ${IMAGE_NAME}:${VERSION_TAG}
           fi

images/charmed-spark-gpu/rockcraft.yaml

@@ -1,10 +1,11 @@
 name: charmed-spark-gpu
-summary: Charmed Apache Spark ROCK
+summary: Charmed Apache Spark ROCK with Spark Rapids support
 description: |
   This is an OCI image that bundles Apache Spark binaries together with other
   tools of its ecosystem in order to be used in Charmed Operators, providing
   an automated and seamless experience to deploy, operate, manage and monitor
-  SparkJob on K8s cluster.
+  SparkJob on K8s cluster. 
+  This image includes support for Nvidia Spark Rapids.
   It is an open source, end-to-end, production ready data platform on top of
   cloud native technologies.