[DPE-4487] Add Integration Tests for Azure Storage (#89)

canonical · Jun 20, 2024 · 69d493c · 69d493c
1 parent bbdeabb
commit 69d493c
Show file tree

Hide file tree

Showing 10 changed files with 435 additions and 223 deletions.
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -20,6 +20,9 @@ jobs:
         env: [integration]
         k8s_version: ["1.28/stable","1.29/stable", "1.30/beta" ]
       fail-fast: false
+    env:
+      AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+      AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -49,10 +52,6 @@ jobs:
           # Unpack Artifact
           mv charmed-spark/${{ steps.artifact.outputs.base_artifact_name }} .
 
-          # Import artifact into docker with new tag
-          sudo make docker-import REPOSITORY=ghcr.io/canonical/ PREFIX=test- \
-            -o ${{ steps.artifact.outputs.base_artifact_name }} 
-
           # Import artifact into microk8s to be used in integration tests
           sudo make microk8s-import PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
             -o ${{ steps.artifact.outputs.base_artifact_name }}
@@ -75,6 +74,9 @@ jobs:
           sg microk8s -c "make tests FLAVOUR=jupyter"
 
       - name: Run tests (Kyuubi)
+        env:
+          AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+          AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
         run: |
           # Unpack Artifact
           mv charmed-spark/${{ steps.artifact.outputs.kyuubi_artifact_name }} .

diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -94,7 +94,7 @@ jobs:
                     
           # Import artifact into docker with new tag
           sudo make docker-import REPOSITORY=${REPOSITORY} TAG=${TAG}\
-            -o ${{ steps.artifact.outputs.name }} 
+            -o ${{ steps.artifact.outputs.base_artifact_name }} 
 
           # Add relevant labels
           COMMIT_ID=$(git log -1 --format=%H)

diff --git a/Makefile b/Makefile
@@ -27,6 +27,10 @@ FLAVOUR := "spark"
 # The channel of `microk8s` snap to be used for testing
 MICROK8S_CHANNEL := "1.28/stable"
 
+# The Azure credentials supplied as environment variables
+AZURE_STORAGE_ACCOUNT := ${AZURE_STORAGE_ACCOUNT}
+AZURE_STORAGE_KEY := ${AZURE_STORAGE_KEY}
+
 # ======================
 # INTERNAL VARIABLES
 # ======================
@@ -105,6 +109,7 @@ JUPYTER_MARKER=$(_MAKE_DIR)/jupyter-$(JUPYTER_VERSION).tag
 KYUUBI_MARKER=$(_MAKE_DIR)/kyuubi-$(KYUUBI_VERSION).tag
 K8S_MARKER=$(_MAKE_DIR)/k8s.tag
 AWS_MARKER=$(_MAKE_DIR)/aws.tag
+AZURE_MARKER=$(_MAKE_DIR)/azure.tag
 
 
 # The names of different flavours of the image in the docker container registry
@@ -249,14 +254,18 @@ microk8s-import: $(ARTIFACT) $(K8S_MARKER)
 
 
 # Recipe that runs the integration tests
-tests: $(K8S_MARKER) $(AWS_MARKER)
+tests: $(K8S_MARKER) $(AWS_MARKER) $(AZURE_MARKER)
 	@echo "=== Running Integration Tests ==="
 ifeq ($(FLAVOUR), jupyter)
 	/bin/bash ./tests/integration/integration-tests-jupyter.sh
 else ifeq ($(FLAVOUR), kyuubi)
-	/bin/bash ./tests/integration/integration-tests-kyuubi.sh
+	@export AZURE_STORAGE_ACCOUNT=$(AZURE_STORAGE_ACCOUNT) \
+			AZURE_STORAGE_KEY=$(AZURE_STORAGE_KEY) \
+	&& /bin/bash ./tests/integration/integration-tests-kyuubi.sh
 else
-	/bin/bash ./tests/integration/integration-tests.sh
+	@export AZURE_STORAGE_ACCOUNT=$(AZURE_STORAGE_ACCOUNT) \
+			AZURE_STORAGE_KEY=$(AZURE_STORAGE_KEY) \
+	&& /bin/bash ./tests/integration/integration-tests.sh
 endif
 
 
@@ -266,6 +275,8 @@ microk8s-setup: $(K8S_MARKER)
 # Shorthand recipe for setup and configuration of AWS CLI.
 aws-cli-setup: $(AWS_MARKER)
 
+# Shorthand recipe for setup and configuration of Azure CLI.
+azure-cli-setup: $(AZURE_MARKER)
 
 # Recipe for setting up and configuring the K8s cluster. 
 $(K8S_MARKER):
@@ -281,3 +292,10 @@ $(AWS_MARKER): $(K8S_MARKER)
 	@echo "=== Setting up and configure AWS CLI ==="
 	/bin/bash ./tests/integration/setup-aws-cli.sh
 	touch $(AWS_MARKER)
+
+
+# Recipe for setting up and configuring the Azure CLI and credentials. 
+$(AZURE_MARKER):
+	@echo "=== Setup and configure Azure CLI ==="
+	/bin/bash ./tests/integration/setup-azure-cli.sh
+	touch $(AZURE_MARKER)
diff --git a/tests/integration/integration-tests-jupyter.sh b/tests/integration/integration-tests-jupyter.sh
@@ -12,11 +12,16 @@
 # of the Spark user (service accounts and secrets) at the beginning of each test, and they are destroyed at the
 # end of the test. 
 
+
+# Import reusable utilities
+source ./tests/integration/utils/k8s-utils.sh
+
+# Global Variables
 NAMESPACE=tests
+ADMIN_POD_NAME=testpod-admin
 
 get_spark_version(){
-  SPARK_VERSION=$(yq '(.version)' images/charmed-spark/rockcraft.yaml)
-  echo "$SPARK_VERSION"
+  yq '(.version)' images/charmed-spark/rockcraft.yaml
 }
 
 
@@ -31,8 +36,8 @@ setup_jupyter() {
   USERNAME=$1
   NAMESPACE=$2
 
-  kubectl -n $NAMESPACE exec testpod-admin -- env UU="$USERNAME" NN="$NAMESPACE" \
-                /bin/bash -c 'spark-client.service-account-registry create --username $UU --namespace $NN'
+  # Create service account using the admin pod
+  create_serviceaccount_using_pod $USERNAME $NAMESPACE $ADMIN_POD_NAME
 
   IMAGE=$(spark_image)
   echo $IMAGE
@@ -84,53 +89,9 @@ cleanup_user_failure() {
   cleanup_user 1 spark $NAMESPACE
 }
 
-wait_for_pod() {
-
-  POD=$1
-  NAMESPACE=$2
-
-  SLEEP_TIME=1
-  for i in {1..5}
-  do
-    pod_status=$(kubectl -n ${NAMESPACE} get pod ${POD} | awk '{ print $3 }' | tail -n 1)
-    echo $pod_status
-    if [[ "${pod_status}" == "Running" ]]
-    then
-        echo "testpod is Running now!"
-        break
-    elif [[ "${i}" -le "5" ]]
-    then
-        echo "Waiting for the pod to come online..."
-        sleep $SLEEP_TIME
-    else
-        echo "testpod did not come up. Test Failed!"
-        exit 3
-    fi
-    SLEEP_TIME=$(expr $SLEEP_TIME \* 2);
-  done
-}
-
-setup_admin_test_pod() {
-  kubectl create ns $NAMESPACE
-
-  echo "Creating admin test-pod"
-
-  # Create a pod with admin service account
-  yq ea '.spec.containers[0].env[0].name = "KUBECONFIG" | .spec.containers[0].env[0].value = "/var/lib/spark/.kube/config" | .metadata.name = "testpod-admin"' \
-    ./tests/integration/resources/testpod.yaml | \
-    kubectl -n tests apply -f -
-
-  wait_for_pod testpod-admin $NAMESPACE
-
-  MY_KUBE_CONFIG=$(cat /home/${USER}/.kube/config)
-
-  kubectl -n $NAMESPACE exec testpod-admin -- /bin/bash -c 'mkdir -p ~/.kube'
-  kubectl -n $NAMESPACE exec testpod-admin -- env KCONFIG="$MY_KUBE_CONFIG" /bin/bash -c 'echo "$KCONFIG" > ~/.kube/config'
-}
 
 teardown_test_pod() {
   kubectl -n $NAMESPACE delete pod testpod-admin
-  kubectl delete namespace $NAMESPACE
 }
 
 get_status_code() {
@@ -173,7 +134,8 @@ echo -e "##################################"
 echo -e "SETUP TEST POD"
 echo -e "##################################"
 
-setup_admin_test_pod
+kubectl create namespace $NAMESPACE
+setup_admin_pod $ADMIN_POD_NAME $(spark_image) $NAMESPACE
 
 echo -e "##################################"
 echo -e "START JUPYTER SERVICE"
@@ -186,6 +148,7 @@ echo -e "TEARDOWN ADMIN POD"
 echo -e "##################################"
 
 teardown_test_pod
+kubectl delete namespace $NAMESPACE
 
 echo -e "##################################"
 echo -e "END OF THE TEST"

diff --git a/tests/integration/integration-tests-kyuubi.sh b/tests/integration/integration-tests-kyuubi.sh
@@ -15,6 +15,7 @@
 
 # Import reusable utilities
 source ./tests/integration/utils/s3-utils.sh
+source ./tests/integration/utils/azure-utils.sh
 source ./tests/integration/utils/k8s-utils.sh
 
 
@@ -23,8 +24,8 @@ NAMESPACE=tests
 SERVICE_ACCOUNT=spark
 ADMIN_POD_NAME=testpod-admin
 USER_POD_NAME=kyuubi-test
-S3_BUCKET=kyuubi
-
+S3_BUCKET=kyuubi-$(uuidgen)
+AZURE_CONTAINER=$S3_BUCKET
 
 get_spark_version(){
   # Fetch Spark version from images/charmed-spark/rockcraft.yaml
@@ -38,8 +39,8 @@ kyuubi_image(){
 }
 
 
-setup_kyuubi_pod() {
-  # Setup Kyuubi pod for testing
+setup_kyuubi_pod_with_s3() {
+  # Setup Kyuubi pod for testing, using S3 as object storage
   #
   # Arguments:
   # $1: The service account to be used for creating Kyuubi pod
@@ -68,20 +69,82 @@ setup_kyuubi_pod() {
   s3_secret_key=$(get_s3_secret_key)
 
   # Write Spark configs inside the Kyuubi container
-  kubectl -n $NAMESPACE exec kyuubi-test -- env IMG="$image"                /bin/bash -c 'echo spark.kubernetes.container.image=$IMG  > /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test -- env NN="$NAMESPACE"             /bin/bash -c 'echo spark.kubernetes.namespace=$NN         >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test -- env UU="$USERNAME"              /bin/bash -c 'echo spark.kubernetes.authenticate.driver.serviceAccountName=$UU >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test -- env ENDPOINT="$s3_endpoint"     /bin/bash -c 'echo spark.hadoop.fs.s3a.endpoint=$ENDPOINT >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test -- env ACCESS_KEY="$s3_access_key" /bin/bash -c 'echo spark.hadoop.fs.s3a.access.key=$ACCESS_KEY >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test -- env SECRET_KEY="$s3_secret_key" /bin/bash -c 'echo spark.hadoop.fs.s3a.secret.key=$SECRET_KEY >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test --                                 /bin/bash -c 'echo spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test --                                 /bin/bash -c 'echo spark.hadoop.fs.s3a.connection.ssl.enabled=false >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test --                                 /bin/bash -c 'echo spark.hadoop.fs.s3a.path.style.access=true       >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test -- env BUCKET="$S3_BUCKET"         /bin/bash -c 'echo spark.sql.warehouse.dir=s3a://$BUCKET/warehouse  >> /etc/spark8t/conf/spark-defaults.conf'
-  kubectl -n $NAMESPACE exec kyuubi-test -- env BUCKET="$S3_BUCKET"         /bin/bash -c 'echo spark.kubernetes.file.upload.path=s3a://$BUCKET  >> /etc/spark8t/conf/spark-defaults.conf'
-
-  # Wait some time for the server to be up and running
-  sleep 10
+  # Add relevant Spark configurations in the service account and write the config
+  # to spark-defaults.conf file inside the container
+  kubectl -n $NAMESPACE exec kyuubi-test -- \
+      env IMG="$image" \
+          UU="$SERVICE_ACCOUNT" \
+          NN="$NAMESPACE" \
+          ENDPOINT="$s3_endpoint" \
+          ACCESS_KEY="$s3_access_key" \
+          SECRET_KEY="$s3_secret_key" \
+          BUCKET="$S3_BUCKET" \
+      /bin/bash -c '\
+        spark-client.service-account-registry add-config --username $UU --namespace $NN \
+          --conf spark.kubernetes.container.image=$IMG \
+          --conf spark.hadoop.fs.s3a.endpoint=$ENDPOINT \
+          --conf spark.hadoop.fs.s3a.access.key=$ACCESS_KEY \
+          --conf spark.hadoop.fs.s3a.secret.key=$SECRET_KEY \
+          --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider \
+          --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
+          --conf spark.hadoop.fs.s3a.path.style.access=true \
+          --conf spark.sql.warehouse.dir=s3a://$BUCKET/warehouse \
+          --conf spark.kubernetes.file.upload.path=s3a://$BUCKET \
+        && \
+        spark-client.service-account-registry get-config --username $UU --namespace $NN > /etc/spark8t/conf/spark-defaults.conf'
+
+}
+
+
+setup_kyuubi_pod_with_azure_abfss() {
+  # Setup Kyuubi pod for testing, using Azure blob storage as object storage and ABFSS as protocol
+  #
+  # Arguments:
+  # $1: The service account to be used for creating Kyuubi pod
+  # $1: The namespace to be used for creating Kyuubi pod
+
+  # Create service account using the admin pod
+  create_serviceaccount_using_pod $SERVICE_ACCOUNT $NAMESPACE $ADMIN_POD_NAME
+
+  image=$(kyuubi_image)
+
+  # Create the pod with the newly created service account
+  sed -e "s%<IMAGE>%${image}%g" \
+      -e "s/<SERVICE_ACCOUNT>/${SERVICE_ACCOUNT}/g" \
+      -e "s/<NAMESPACE>/${NAMESPACE}/g" \
+      -e "s/<POD_NAME>/${USER_POD_NAME}/g" \
+      ./tests/integration/resources/kyuubi.yaml | \
+    kubectl -n tests apply -f -
+
+  wait_for_pod $USER_POD_NAME $NAMESPACE
+
+  # Create Azure storage container
+  create_azure_container $AZURE_CONTAINER
+
+  storage_account_name=$(get_azure_storage_account_name)
+  storage_account_key=$(get_azure_storage_secret_key)
+  warehouse_path=$(construct_resource_uri $AZURE_CONTAINER warehouse abfss)
+  file_upload_path=$(construct_resource_uri $AZURE_CONTAINER "" abfss)
+
+  # Add relevant Spark configurations in the service account and write the config
+  # to spark-defaults.conf file inside the container
+  kubectl -n $NAMESPACE exec kyuubi-test -- \
+      env IMG="$image" \
+          UU="$SERVICE_ACCOUNT" \
+          NN="$NAMESPACE" \
+          ACCOUNT_NAME="$storage_account_name" \
+          SECRET_KEY="$storage_account_key" \
+          WAREHOUSE="$warehouse_path" \
+          UPLOAD_PATH="$file_upload_path" \
+        /bin/bash -c '\
+          spark-client.service-account-registry add-config --username $UU --namespace $NN \
+            --conf spark.kubernetes.container.image=$IMG \
+            --conf spark.hadoop.fs.azure.account.key.$ACCOUNT_NAME.dfs.core.windows.net=$SECRET_KEY \
+            --conf spark.sql.warehouse.dir=$WAREHOUSE \
+            --conf spark.kubernetes.file.upload.path=$UPLOAD_PATH \
+          && \
+          spark-client.service-account-registry get-config --username $UU --namespace $NN > /etc/spark8t/conf/spark-defaults.conf'
+
 }
 
 
@@ -103,7 +166,10 @@ cleanup_user() {
                   /bin/bash -c 'spark-client.service-account-registry delete --username $UU --namespace $NN'  
 
   # Delete S3 bucket
-  delete_s3_bucket kyuubi
+  delete_s3_bucket $S3_BUCKET || true
+
+  # Delete Azure container
+  delete_azure_container $AZURE_CONTAINER || true
 
   # Verify deletion of service account
   output=$(kubectl -n $NAMESPACE exec $ADMIN_POD_NAME -- /bin/bash -c 'spark-client.service-account-registry list')
@@ -169,10 +235,16 @@ kubectl create namespace $NAMESPACE
 setup_admin_pod $ADMIN_POD_NAME $(kyuubi_image) $NAMESPACE
 
 echo -e "##################################"
-echo -e "START KYUUBI POD AND BEGIN TESTING"
+echo -e "START KYUUBI POD AND BEGIN TESTING (USING S3)"
+echo -e "##################################"
+
+(setup_kyuubi_pod_with_s3 && test_jdbc_connection && cleanup_user_success) || cleanup_user_failure
+
+echo -e "##################################"
+echo -e "START KYUUBI POD AND BEGIN TESTING (USING Azure Storage)"
 echo -e "##################################"
 
-(setup_kyuubi_pod && test_jdbc_connection && cleanup_user_success) || cleanup_user_failure
+(setup_kyuubi_pod_with_azure_abfss && test_jdbc_connection && cleanup_user_success) || cleanup_user_failure
 
 echo -e "##################################"
 echo -e "TEARDOWN ADMIN POD"