updates

build/Dockerfile.jupyter

@@ -1,13 +1,16 @@
 ARG BASE_IMAGE=base-charmed-spark:latest
+ARG JUPYTERLAB_VERSION=4.0.11
 FROM $BASE_IMAGE
 
+ARG JUPYTERLAB_VERSION
 USER root
-RUN rm /var/lib/pebble/default/layers/*.yaml
 
-RUN python3 -m pip install "jupyterlab~=4.0"
+RUN rm /var/lib/pebble/default/layers/*.yaml /opt/pebble/*.sh
+
+RUN python3 -m pip install "jupyterlab==$JUPYTERLAB_VERSION"
 COPY ./files/jupyter/pebble/layers.yaml /var/lib/pebble/default/layers/001-charmed-jupyter.yaml
-COPY ./files/jupyter/bin/kf-nb.sh /opt/spark/bin/kf-nb.sh
-RUN chown _daemon_:_daemon_ /opt/spark/bin/kf-nb.sh
+COPY ./files/jupyter/bin/jupyterlab-server.sh /opt/pebble/jupyterlab-server.sh
+RUN chown _daemon_:_daemon_ /opt/pebble/jupyterlab-server.sh
 
 USER _daemon_
 

files/jupyter/bin/jupyterlab-server.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+sleep 5
+
+export PYSPARK_DRIVER_PYTHON=jupyter
+
+# This variable is injected when running a notebook from Kubeflow.
+if [ -z "${NB_PREFIX}" ]; then
+  NB_PREFIX_ARG="--NotebookApp.base_url '${NB_PREFIX}'"
+fi
+
+export PYSPARK_DRIVER_PYTHON_OPTS="lab --no-browser --port=8888 ${NB_PREFIX_ARG} --ip=0.0.0.0 --NotebookApp.token='' --notebook-dir=/var/lib/spark/notebook"
+
+spark-client.pyspark $*

files/jupyter/pebble/layers.yaml

@@ -1,6 +1,6 @@
 services:
   jupyter:
-      command: "/opt/spark/bin/kf-nb.sh"
+      command: "/opt/pebble/jupyterlab-server.sh"
       summary: "This is the Spark-powered Jupyter service"
       override: replace
       startup: enabled

rockcraft.yaml

@@ -133,7 +133,7 @@ parts:
       - python3-pip
     overlay-script: |
       mkdir -p $CRAFT_PART_INSTALL/opt/spark8t/python/dist
-      pip install --target=${CRAFT_PART_INSTALL}/opt/spark8t/python/dist  git+https://github.com/canonical/spark-k8s-toolkit-py.git@wip-incluster-config
+      pip install --target=${CRAFT_PART_INSTALL}/opt/spark8t/python/dist  git+https://github.com/canonical/spark-k8s-toolkit-py.git@dpe-3398-fix-service-account-permission
       rm usr/bin/pip*
     stage:
       - opt/spark8t/python/dist

tests/integration/integration-tests.sh

@@ -118,12 +118,8 @@ setup_test_pod() {
     SLEEP_TIME=$(expr $SLEEP_TIME \* 2);
   done
 
-  MY_KUBE_CONFIG=$(cat /home/${USER}/.kube/config)
   TEST_POD_TEMPLATE=$(cat tests/integration/resources/podTemplate.yaml)
 
-  kubectl exec testpod -- /bin/bash -c 'mkdir -p ~/.kube'
-  kubectl exec testpod -- env KCONFIG="$MY_KUBE_CONFIG" /bin/bash -c 'echo "$KCONFIG" > ~/.kube/config'
-  kubectl exec testpod -- /bin/bash -c 'cat ~/.kube/config'
   kubectl exec testpod -- /bin/bash -c 'cp -r /opt/spark/python /var/lib/spark/'
   kubectl exec testpod -- env PTEMPLATE="$TEST_POD_TEMPLATE" /bin/bash -c 'echo "$PTEMPLATE" > /etc/spark/conf/podTemplate.yaml'
 }