[DPE-4628] Add image with spark-rapids support (#93)

canonical · Jul 4, 2024 · be00442 · be00442
1 parent 5773781
commit be00442
Show file tree

Hide file tree

Showing 16 changed files with 983 additions and 9 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -16,6 +16,8 @@ jobs:
         run:  |
           yamllint -d "{extends: relaxed, rules: {line-length: {max: 250}}}" \
             --no-warnings images/charmed-spark/rockcraft.yaml
+          yamllint -d "{extends: relaxed, rules: {line-length: {max: 250}}}" \
+            --no-warnings images/charmed-spark-gpu/rockcraft.yaml
 
   build:
     runs-on: ubuntu-latest
@@ -55,7 +57,7 @@ jobs:
 
       - name: Change artifact permissions
         run: sudo chmod a+r  ${{ steps.artifact.outputs.base_artifact_name }} ${{ steps.artifact.outputs.jupyter_artifact_name }} ${{ steps.artifact.outputs.kyuubi_artifact_name }}
-
+          
       - name: Upload locally built artifact
         uses: actions/upload-artifact@v4
         with:
@@ -65,3 +67,4 @@ jobs:
             ${{ steps.artifact.outputs.jupyter_artifact_name }}
             ${{ steps.artifact.outputs.kyuubi_artifact_name }}
 
+
diff --git a/.github/workflows/build_gpu.yaml b/.github/workflows/build_gpu.yaml
@@ -0,0 +1,55 @@
+name: Build GPU ROCK
+
+on:
+  workflow_call:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install yamllint
+        run: python3 -m pip install yamllint
+      - name: YAML Lint
+        run:  |
+          yamllint -d "{extends: relaxed, rules: {line-length: {max: 250}}}" \
+            --no-warnings images/charmed-spark-gpu/rockcraft.yaml
+
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs:
+      - lint
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup LXD
+        uses: canonical/setup-lxd@main
+
+      - name: Install dependencies
+        run: |
+          sudo snap install yq
+          sudo snap install rockcraft --classic --edge    
+      
+      - name: Build image (GPU)
+        run: sudo make build FLAVOUR=spark-gpu
+
+      - name: Get Artifact Name
+        id: artifact
+        run: |
+          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+          echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+
+      - name: Change artifact permissions
+        run: sudo chmod a+r  ${{ steps.artifact.outputs.gpu_artifact_name }}
+
+      - name: Upload locally built artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: charmed-spark-gpu
+          path: |
+            ${{ steps.artifact.outputs.gpu_artifact_name }}
+
diff --git a/.github/workflows/integration-gpu.yaml b/.github/workflows/integration-gpu.yaml
@@ -0,0 +1,118 @@
+name: GPU integration CI pipeline
+
+on:
+  pull_request:
+
+jobs:
+  build-rock:
+    uses: ./.github/workflows/build_gpu.yaml
+
+  start-runner:
+    name: Start self-hosted EC2 runner
+    needs: [build-rock]
+    runs-on: ubuntu-22.04
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/[email protected]
+        with:
+          mode: start
+          github-token: ${{ secrets.WELPAOLO_GITHUB_TOKEN }}
+          ec2-image-id: ${{ vars.AWS_EC2_IMAGE_ID }}
+          ec2-instance-type: ${{ vars.AWS_EC2_INSTANCE_TYPE }}
+          subnet-id: ${{ vars.AWS_DEFAULT_SUBNET_ID }}
+          security-group-id: ${{ vars.AWS_SECURITY_GROUP_ID }}
+
+  integration-tests:
+    name: Run integration tests
+    needs: [start-runner]
+    runs-on: ${{ needs.start-runner.outputs.label }} 
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Setup environment
+      run: |
+        sudo -E bash ./tests/integration/setup-environment.sh
+      env:
+        MICROK8S_CHANNEL: 1.28/stable
+
+    - name: Setup microk8s
+      run: sudo -E /bin/bash ./tests/integration/config-microk8s-gpu.sh
+      env:
+        MICROK8S_ADDONS: "storage dns rbac gpu minio"
+
+    - name: Configure microk8s
+      run: |
+        sudo microk8s.status
+        sudo snap alias microk8s.kubectl kubectl
+        mkdir -p /home/ubuntu/.kube
+        sudo chown -f -R ubuntu /home/ubuntu/.kube
+        sudo microk8s config | tee /home/ubuntu/.kube/config
+    
+    - name: Install required packages/snaps
+      run: |
+        sudo snap install yq
+        sudo apt-get install build-essential -yqq
+
+    - name: Get Artifact Name
+      id: artifact
+      run: |
+        GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+        echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: charmed-spark-gpu
+        path: charmed-spark
+
+    - name: Load image
+      run: |
+        # Unpack Artifact
+        mv charmed-spark/${{ steps.artifact.outputs.gpu_artifact_name }} .
+        touch .make_cache/k8s.tag
+        # Import artifact into microk8s to be used in integration tests
+        sudo make microk8s-import FLAVOUR=spark-gpu PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
+          -o ${{ steps.artifact.outputs.gpu_artifact_name }}
+
+    - name: Run integration tests on GPU
+      run: |
+        echo "Start test..."
+        /bin/bash ./tests/integration/setup-aws-cli.sh
+        /bin/bash ./tests/integration/integration-tests-gpu.sh
+        echo "End of the test :)"
+      env:
+        KUBECONFIG: "/home/ubuntu/.kube/config"
+        USER: "ubuntu"
+
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner 
+      - integration-tests
+    runs-on: ubuntu-22.04
+    if: ${{ always() }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/[email protected]
+        with:
+          mode: stop
+          github-token: ${{ secrets.WELPAOLO_GITHUB_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -36,6 +36,8 @@ jobs:
           echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
           KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
           echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+          echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Install and configure microk8s
         run: |

diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -74,6 +74,8 @@ jobs:
           echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
           KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
           echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+          echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Download artifact
         uses: actions/download-artifact@v4
@@ -185,6 +187,42 @@ jobs:
 
             docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG}
 
+            echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"
+            docker push ${IMAGE_NAME}:${VERSION_TAG}
+          fi
+      - name: Publish Charmed Spark GPU Image to Channel
+        run: |
+          # Unpack artifact
+          mv charmed-spark/${{ steps.artifact.outputs.gpu_artifact_name }} .
+          rmdir charmed-spark
+
+          REPOSITORY="ghcr.io/canonical/"
+          RISK=${{ needs.release_checks.outputs.risk }}
+          TRACK=${{ needs.release_checks.outputs.track }}
+          if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi
+      
+          # Import artifact into docker with new tag
+          sudo make docker-import \
+            FLAVOUR=spark-gpu \
+            REPOSITORY=${REPOSITORY} \
+            TAG=${TAG} \
+            -o ${{ steps.artifact.outputs.gpu_artifact_name }}
+
+          # Add relevant labels
+          COMMIT_ID=$(git log -1 --format=%H)
+          DESCRIPTION=$(yq .description images/charmed-spark-gpu/rockcraft.yaml | xargs)
+
+          echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" -
+
+          echo "Publishing ${IMAGE_NAME}:${TAG}"
+          docker push ${IMAGE_NAME}:${TAG}
+
+          if [[ "$RISK" == "edge" ]]; then
+            VERSION_LONG=$(make help FLAVOUR=spark-gpu | grep "Tag\:" | cut -d ":" -f2  | xargs)
+            VERSION_TAG="${VERSION_LONG}-${{ needs.release_checks.outputs.base }}_edge"
+
+            docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG}
+
             echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"
             docker push ${IMAGE_NAME}:${VERSION_TAG}
           fi