[DPE-5150] Update spark-metrics library #5
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish | |
| on: | |
| workflow_call: | |
| workflow_dispatch: | |
| pull_request: | |
| jobs: | |
| release_checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Extract branch metadata | |
| shell: bash | |
| run: | | |
| BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| echo "branch=${BRANCH}" >> $GITHUB_OUTPUT | |
| echo "risk=${BRANCH##*\/}" >> $GITHUB_OUTPUT | |
| echo "track=${BRANCH%*\/*}" >> $GITHUB_OUTPUT | |
| id: branch_metadata | |
| - name: Extract ROCK metadata | |
| shell: bash | |
| run: | | |
| VERSION=$(yq '(.version|split("-"))[0]' images/charmed-spark/rockcraft.yaml) | |
| BASE=$(yq '(.base|split("@"))[1]' images/charmed-spark/rockcraft.yaml) | |
| echo "version=${VERSION}" >> $GITHUB_OUTPUT | |
| echo "base=${BASE}" >> $GITHUB_OUTPUT | |
| id: rock_metadata | |
| # - name: Check consistency between metadata and release branch | |
| # run: | | |
| # MAJOR_MINOR_VERSION=$(echo ${{ steps.rock_metadata.outputs.version }} | sed -n "s/\(^[0-9]*\.[0-9]*\).*/\1/p") | |
| # BASE=${{ steps.rock_metadata.outputs.base }} | |
| # if [ "${MAJOR_MINOR_VERSION}-${BASE}" != "${{ steps.branch_metadata.outputs.track }}" ]; then exit 1; fi | |
| # continue-on-error: false | |
| outputs: | |
| branch: ${{ steps.branch_metadata.outputs.branch }} | |
| track: ${{ steps.branch_metadata.outputs.track }} | |
| risk: ${{ steps.branch_metadata.outputs.risk }} | |
| base: ${{ steps.rock_metadata.outputs.base }} | |
| version: ${{ steps.rock_metadata.outputs.version }} | |
| tests: | |
| uses: ./.github/workflows/integration.yaml | |
| secrets: inherit | |
| # tests-gpu: | |
| # uses: ./.github/workflows/integration-gpu.yaml | |
| # secrets: inherit | |
| publish: | |
| needs: [tests] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| sudo snap install yq | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_USER }} | |
| password: ${{ secrets.GHCR_TOKEN }} | |
| - name: Get Artifact Name | |
| id: artifact | |
| run: | | |
| BASE_ARTIFACT=$(make help | grep 'Artifact: ') | |
| echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT | |
| JUPYTER_ARTIFACT=$(make help FLAVOUR=jupyter | grep 'Artifact: ') | |
| echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT | |
| KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ') | |
| echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT | |
| GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ') | |
| echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: charmed-spark | |
| path: charmed-spark | |
| - name: Publish Image to Channel | |
| run: | | |
| # Unpack artifact | |
| mv charmed-spark/${{ steps.artifact.outputs.base_artifact_name }} . | |
| ls | |
| REPOSITORY="ghcr.io/welpaolo/" | |
| RISK="edge" | |
| TRACK="4.0.0-preview1" | |
| # Remove | |
| echo "REPOSITORY: $REPOSITORY" | |
| echo "RISK $RISK" | |
| echo "TRACK: $TRACK" | |
| if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi | |
| IMAGE_NAME=$(make help REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2 | xargs) | |
| echo "IMAGE_NAME $IMAGE_NAME" | |
| # Import artifact into docker with new tag | |
| sudo make docker-import REPOSITORY=${REPOSITORY} TAG=${TAG}\ | |
| -o ${{ steps.artifact.outputs.base_artifact_name }} | |
| echo "After import" | |
| # Add relevant labels | |
| COMMIT_ID=$(git log -1 --format=%H) | |
| DESCRIPTION=$(yq .description images/charmed-spark/rockcraft.yaml | xargs) | |
| echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" - | |
| echo "Publishing ${IMAGE_NAME}:${TAG}" | |
| docker push ${IMAGE_NAME}:${TAG} | |
| # if [[ "$RISK" == "edge" ]]; then | |
| # VERSION_TAG="${{ needs.release_checks.outputs.version }}-${{ needs.release_checks.outputs.base }}_edge" | |
| # docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} | |
| # echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}" | |
| # docker push ${IMAGE_NAME}:${VERSION_TAG} | |
| # fi | |
| # - name: Publish JupyterLab Image to Channel | |
| # run: | | |
| # # Unpack artifact | |
| # mv charmed-spark/${{ steps.artifact.outputs.jupyter_artifact_name }} . | |
| # REPOSITORY="ghcr.io/welpaolo/" | |
| # RISK=${{ needs.release_checks.outputs.risk }} | |
| # TRACK=${{ needs.release_checks.outputs.track }} | |
| # if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi | |
| # # Import artifact into docker with new tag | |
| # sudo make docker-import \ | |
| # FLAVOUR=jupyter \ | |
| # REPOSITORY=${REPOSITORY} \ | |
| # TAG=${TAG} \ | |
| # -o ${{ steps.artifact.outputs.jupyter_artifact_name }} | |
| # IMAGE_NAME=$(make help FLAVOUR=jupyter REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2 | xargs) | |
| # DESCRIPTION=$(yq .flavours.jupyter.image_description images/metadata.yaml | xargs) | |
| # echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" - | |
| # echo "Publishing ${IMAGE_NAME}:${TAG}" | |
| # docker push ${IMAGE_NAME}:${TAG} | |
| # if [[ "$RISK" == "edge" ]]; then | |
| # VERSION_LONG=$(make help FLAVOUR=jupyter | grep "Tag\:" | cut -d ":" -f2 | xargs) | |
| # VERSION_TAG="${VERSION_LONG}-${{ needs.release_checks.outputs.base }}_edge" | |
| # docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} | |
| # echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}" | |
| # docker push ${IMAGE_NAME}:${VERSION_TAG} | |
| # fi | |
| # - name: Publish Kyuubi Image to Channel | |
| # run: | | |
| # # Unpack artifact | |
| # mv charmed-spark/${{ steps.artifact.outputs.kyuubi_artifact_name }} . | |
| # rmdir charmed-spark | |
| # REPOSITORY="ghcr.io/canonical/" | |
| # RISK=${{ needs.release_checks.outputs.risk }} | |
| # TRACK=${{ needs.release_checks.outputs.track }} | |
| # if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi | |
| # # Import artifact into docker with new tag | |
| # sudo make docker-import \ | |
| # FLAVOUR=kyuubi \ | |
| # REPOSITORY=${REPOSITORY} \ | |
| # TAG=${TAG} \ | |
| # -o ${{ steps.artifact.outputs.kyuubi_artifact_name }} | |
| # IMAGE_NAME=$(make help FLAVOUR=kyuubi REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2 | xargs) | |
| # DESCRIPTION=$(yq .flavours.kyuubi.image_description images/metadata.yaml | xargs) | |
| # echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" - | |
| # echo "Publishing ${IMAGE_NAME}:${TAG}" | |
| # docker push ${IMAGE_NAME}:${TAG} | |
| # if [[ "$RISK" == "edge" ]]; then | |
| # VERSION_LONG=$(make help FLAVOUR=kyuubi | grep "Tag\:" | cut -d ":" -f2 | xargs) | |
| # VERSION_TAG="${VERSION_LONG}-${{ needs.release_checks.outputs.base }}_edge" | |
| # docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} | |
| # echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}" | |
| # docker push ${IMAGE_NAME}:${VERSION_TAG} | |
| # fi | |
| # - name: Download gpu artifact | |
| # uses: actions/download-artifact@v4 | |
| # with: | |
| # name: charmed-spark-gpu | |
| # path: charmed-spark-gpu | |
| # - name: Publish Charmed Spark GPU Image to Channel | |
| # run: | | |
| # # Unpack artifact | |
| # mv charmed-spark-gpu/${{ steps.artifact.outputs.gpu_artifact_name }} . | |
| # rmdir charmed-spark-gpu | |
| # REPOSITORY="ghcr.io/canonical/" | |
| # RISK=${{ needs.release_checks.outputs.risk }} | |
| # TRACK=${{ needs.release_checks.outputs.track }} | |
| # if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi | |
| # IMAGE_NAME=$(make help REPOSITORY=${REPOSITORY} TAG=${TAG} FLAVOUR=spark-gpu help | grep "Image\:" | cut -d ":" -f2 | xargs) | |
| # # Import artifact into docker with new tag | |
| # sudo make docker-import \ | |
| # FLAVOUR=spark-gpu \ | |
| # REPOSITORY=${REPOSITORY} \ | |
| # TAG=${TAG} \ | |
| # -o ${{ steps.artifact.outputs.gpu_artifact_name }} | |
| # # Add relevant labels | |
| # COMMIT_ID=$(git log -1 --format=%H) | |
| # DESCRIPTION=$(yq .description images/charmed-spark-gpu/rockcraft.yaml | xargs) | |
| # echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" - | |
| # echo "Publishing ${IMAGE_NAME}:${TAG}" | |
| # docker push ${IMAGE_NAME}:${TAG} | |
| # if [[ "$RISK" == "edge" ]]; then | |
| # VERSION_LONG=$(make help FLAVOUR=spark-gpu | grep "Tag\:" | cut -d ":" -f2 | xargs) | |
| # VERSION_TAG="${VERSION_LONG}-${{ needs.release_checks.outputs.base }}_edge" | |
| # docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} | |
| # echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}" | |
| # docker push ${IMAGE_NAME}:${VERSION_TAG} | |
| # fi |