[DPE-3104] Create Jupyter notebook image

canonical · Dec 11, 2023 · 3073393 · 3073393
1 parent 73ba1cf
commit 3073393
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 14 deletions.
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -52,5 +52,9 @@ jobs:
           # Import artifact into microk8s to be used in integration tests
           sudo make import TARGET=microk8s PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
             -o $(find .make_cache -name "*.tag")
+
+          # Import artifact into docker with new tag
+          sudo make jupyter TARGET=docker REPOSITORY=ghcr.io/canonical/ PREFIX=test- \
+            -o $(find .make_cache -name "*.tag")
           
           sg microk8s -c "make tests"
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -22,7 +22,7 @@ jobs:
       shell: bash
       run: |
         VERSION=$(yq '(.version|split("-"))[0]' rockcraft.yaml)
-        BASE=$(yq '(.base|split(":"))[1]' rockcraft.yaml)
+        BASE=$(yq '(.base|split("@"))[1]' rockcraft.yaml)
         echo "version=${VERSION}" >> $GITHUB_OUTPUT
         echo "base=${BASE}" >> $GITHUB_OUTPUT
       id: rock_metadata
@@ -84,22 +84,50 @@ jobs:
           RISK=${{ needs.release_checks.outputs.risk }}
           TRACK=${{ needs.release_checks.outputs.track }}
           if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi
-          
+
+          IMAGE_NAME=$(make REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2  | xargs)
+                    
           # Import artifact into docker with new tag
           sudo make import TARGET=docker REPOSITORY=${REPOSITORY} TAG=${TAG}\
             -o ${{ steps.artifact.outputs.name }} 
 
-          IMAGE_NAME="${REPOSITORY}charmed-spark"
-          
           echo "Publishing ${IMAGE_NAME}:${TAG}"          
           docker push ${IMAGE_NAME}:${TAG}
           
           if [[ "$RISK" == "edge" ]]; then           
-            TAG="${{ needs.release_checks.outputs.version }}-${{ needs.release_checks.outputs.base }}_edge"
+            VERSION_TAG="${{ needs.release_checks.outputs.version }}-${{ needs.release_checks.outputs.base }}_edge"
             
-            sudo make import TARGET=docker REPOSITORY=${REPOSITORY} TAG=${TAG}\
-              -o ${{ steps.artifact.outputs.name }} 
+            docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} 
+ 
+            echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"          
+            docker push ${IMAGE_NAME}:${VERSION_TAG}
+          fi
+
+      - name: Publish JupyterLab Image to Channel
+        run: |
+          
+          REPOSITORY="ghcr.io/canonical/"
+          RISK=${{ needs.release_checks.outputs.risk }}
+          TRACK=${{ needs.release_checks.outputs.track }}
+          if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi
+                    
+          # Import artifact into docker with new tag
+          sudo make jupyter REPOSITORY=${REPOSITORY} TAG=${TAG}\
+            -o $(find .make_cache -name "*.tag")
+
+          IMAGE_NAME=$(make REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Jupyter\:" | cut -d ":" -f2  | xargs)
+
+          echo "Publishing ${IMAGE_NAME}:${TAG}"          
+          docker push ${IMAGE_NAME}:${TAG}
+          
+          if [[ "$RISK" == "edge" ]]; then           
+            VERSION_TAG="${{ needs.release_checks.outputs.version }}-${{ needs.release_checks.outputs.base }}_edge"
+
+            docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} 
             
-            echo "Publishing ${IMAGE_NAME}:${TAG}"          
-            docker push ${IMAGE_NAME}:${TAG}
-          fi
+            echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"          
+            docker push ${IMAGE_NAME}:${VERSION_TAG}
+          fi
+
+
+
diff --git a/Makefile b/Makefile
@@ -40,14 +40,20 @@ _TMP_OCI_TAG := $(_MAKE_DIR)/$(_TMP_OCI_NAME)/$(TAG).tag
 CHARMED_OCI_FULL_NAME=$(REPOSITORY)$(PREFIX)$(IMAGE_NAME)
 CHARMED_OCI_TAG := $(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)/$(TAG).tag
 
+CHARMED_OCI_JUPYTER=$(CHARMED_OCI_FULL_NAME)-jupyterlab4
+CHARMED_OCI_JUPYTER_TAG := $(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)/$(TAG).tag
+
 help:
 	@echo "---------------HELP-----------------"
-	@echo "Image: $(IMAGE_NAME)"
+	@echo "Name: $(IMAGE_NAME)"
 	@echo "Version: $(VERSION)"
 	@echo "Platform: $(PLATFORM)"
 	@echo " "
 	@echo "Artifact: $(BASE_NAME)"
 	@echo " "
+	@echo "Image: $(CHARMED_OCI_FULL_NAME)"
+	@echo "Jupyter: $(CHARMED_OCI_JUPYTER)"
+	@echo " "
 	@echo "Type 'make' followed by one of these keywords:"
 	@echo " "
 	@echo "  - build for creating the OCI Images"
@@ -69,8 +75,8 @@ $(_TMP_OCI_TAG): $(_ROCK_OCI)
 	if [ ! -d "$(_MAKE_DIR)/$(_TMP_OCI_NAME)" ]; then mkdir -p "$(_MAKE_DIR)/$(_TMP_OCI_NAME)"; fi
 	touch $(_TMP_OCI_TAG)
 
-$(CHARMED_OCI_TAG): $(_TMP_OCI_TAG)
-	docker build - -t "$(CHARMED_OCI_FULL_NAME):$(TAG)" --build-arg BASE_IMAGE="$(_TMP_OCI_NAME):$(TAG)" < Dockerfile
+$(CHARMED_OCI_TAG): $(_TMP_OCI_TAG) build/Dockerfile
+	docker build -t "$(CHARMED_OCI_FULL_NAME):$(TAG)" --build-arg BASE_IMAGE="$(_TMP_OCI_NAME):$(TAG)" -f build/Dockerfile .
 	if [ ! -d "$(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)" ]; then mkdir -p "$(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)"; fi
 	touch $(CHARMED_OCI_TAG)
 
@@ -85,12 +91,21 @@ microk8s: $(K8S_TAG)
 $(_MAKE_DIR)/%/$(TAG).tar: $(_MAKE_DIR)/%/$(TAG).tag
 	docker save $*:$(TAG) > $(_MAKE_DIR)/$*/$(TAG).tar
 
+$(CHARMED_OCI_JUPYTER_TAG): $(CHARMED_OCI_TAG) build/Dockerfile.jupyter
+	docker build -t "$(CHARMED_OCI_JUPYTER):$(TAG)" --build-arg BASE_IMAGE="$(CHARMED_OCI_FULL_NAME):$(TAG)" -f build/Dockerfile.jupyter .
+	if [ ! -d "$(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)" ]; then mkdir -p "$(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)"; fi
+	touch $(CHARMED_OCI_JUPYTER_TAG)
+
 $(BASE_NAME): $(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)/$(TAG).tar
 	@echo "=== Creating $(BASE_NAME) OCI archive ==="
 	cp $(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)/$(TAG).tar $(BASE_NAME)
 
 build: $(BASE_NAME)
 
+jupyter: $(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)/$(TAG).tar
+	@echo "=== Creating $(BASE_NAME) OCI jupyter archive ==="
+	cp $(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)/$(TAG).tar $(IMAGE_NAME)-jupyter_$(VERSION)_$(PLATFORM).tar
+
 ifeq ($(TARGET), docker)
 import: build
 	@echo "=== Importing image $(CHARMED_OCI_FULL_NAME):$(TAG) into docker ==="

diff --git a/README.md b/README.md
@@ -71,6 +71,29 @@ Charmed Spark Rock Image is delivered with Pebble already included in order to m
 docker run ghcr.io/canonical/charmed-spark:3.4.1-22.04_edge \; start history-server
 ```
 
+### Running Jupyter Lab
+
+In the Charmed Spark bundle we also provide the `charmed-spark-jupyter` image, 
+specifically built for running JupterLab server integrated with Spark where any notebook will also 
+start dedicated executors and inject a SparkSession and/or SparkContext within the notebook.
+
+To start a JupyterLab server using the `charmed-spark-jupyter` image, use
+
+```shell
+docker run \
+  -v /path/to/kube/config:/var/lib/spark/.kube/config \
+  -p <port>:8888
+  ghcr.io/canonical/charmed-spark-jupyter:3.4.1-22.04_edge \
+  --username <spark-service-account> --namespace <spark-namespace>
+```
+
+Make sure to have created the `<spark-service-account>` in the `<spark-namespace>` with the `spark8t` CLI beforehand.
+You should be able to access the jupyter server at `http://0.0.0.0:<port>`.
+
+You can provide extra-arguments to further configure the spark-executors by providing more `spark8t`
+commands. The mount of the local `kubeconfig` file is necessary to provide the ability to the 
+JupyterLab server to act as a Spark driver and request resources on the K8s cluster. 
+
 ## Developers and Contributing
 
 Please see the [CONTRIBUTING.md](https://github.com/canonical/charmed-spark-rock/blob/3.4-22.04/edge/CONTRIBUTING.md) for guidelines and for developer guidance.

diff --git a/Dockerfile → build/Dockerfile b/Dockerfile → build/Dockerfile
diff --git a/build/Dockerfile.jupyter b/build/Dockerfile.jupyter
@@ -0,0 +1,13 @@
+ARG BASE_IMAGE=base-charmed-spark:latest
+FROM $BASE_IMAGE
+
+USER root
+RUN rm /var/lib/pebble/default/layers/*.yaml
+
+RUN python3 -m pip install "jupyterlab~=4.0"
+COPY ./files/jupyter/pebble/layers.yaml /var/lib/pebble/default/layers/001-charmed-jupyter.yaml
+
+USER _daemon_
+
+# Provide Default Entrypoint for Pebble
+ENTRYPOINT [ "/bin/pebble", "enter", "--verbose", "--args", "jupyter" ]
diff --git a/files/jupyter/pebble/layers.yaml b/files/jupyter/pebble/layers.yaml
@@ -0,0 +1,9 @@
+services:
+  jupyter:
+      command: "spark-client.pyspark"
+      summary: "This is the Spark-powered Jupyter service"
+      override: replace
+      startup: enabled
+      environment:
+        PYSPARK_DRIVER_PYTHON: jupyter
+        PYSPARK_DRIVER_PYTHON_OPTS: "lab --no-browser --port=8888 --ip=0.0.0.0 --NotebookApp.token='' --notebook-dir=/var/lib/spark/notebook"
diff --git a/rockcraft.yaml b/rockcraft.yaml
@@ -4,7 +4,7 @@ description: Spark ROCK
 license: Apache-2.0
 
 version: "3.4.1"
-base: ubuntu:22.04
+base: ubuntu@22.04
 platforms:
   amd64: