Backporting changes from multiple PRs from 3.4-22.04/edge branch

.github/workflows/build.yaml

@@ -2,6 +2,10 @@ name: Build ROCK
 
 on:
   workflow_call:
+    inputs:
+      branch:
+        type: string
+        default: ''
 
 jobs:
   lint:
@@ -27,6 +31,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
 
       - name: Setup LXD
         uses: canonical/setup-lxd@main

.github/workflows/trivy.yml

@@ -4,17 +4,29 @@ on:
     branches:
       - 3.4-22.04/edge
       - 3.5-22.04/edge
+      - 3.4-22.04/stable
+      - 3.5-22.04/stable
   pull_request:
+  workflow_call:
+    inputs:
+      branch:
+        type: string
+        default: ''
 jobs:
   build:
     uses: ./.github/workflows/build.yaml
+    with:
+      branch: ${{ inputs.branch }}
   scan:
-    name: Trivy scan
+    name: Trivy scan and sbom generation
     needs: build
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+
       - name: Install rockcraft (for skopeo)
         run: |
           sudo snap install rockcraft --classic --edge
@@ -36,7 +48,7 @@ jobs:
             docker-archive:${{ steps.artifact.outputs.base_artifact_name }} \
             docker-daemon:trivy/charmed-spark:test
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.28.0
         with:
           image-ref: 'trivy/charmed-spark:test'
           format: 'sarif'
@@ -47,4 +59,23 @@ jobs:
         uses: github/codeql-action/upload-sarif@v2
         if: always()
         with:
-          sarif_file: 'trivy-results.sarif'
+          sarif_file: 'trivy-results.sarif'
+          ref: ${{ inputs.branch }}
+
+      - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph
+        uses: aquasecurity/[email protected]
+        with:
+          scan-type: 'image'
+          format: 'spdx-json'
+          output: 'dependency-results.sbom.json'
+          image-ref: 'trivy/charmed-spark:test'
+          github-pat: ${{ secrets.GITHUB_TOKEN }}
+          severity: "MEDIUM,HIGH,CRITICAL"
+          scanners: "vuln"
+
+      - name: Upload trivy report as a Github artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: trivy-sbom-report
+          path: '${{ github.workspace }}/dependency-results.sbom.json'
+          retention-days: 90

images/charmed-spark/bin/sparkd.sh

@@ -1,5 +1,29 @@
 #!/bin/bash
 
+function get_log_layer {
+  local loki_url=$1
+  local log_layer_file=${2-"/opt/pebble/log-layer.yaml"}
+  sed -e "s/\$LOKI_URL/$loki_url/g" \
+      -e "s/\$FLAVOUR/$FLAVOUR/g" \
+      -e "s/\$SPARK_APPLICATION_ID/$SPARK_APPLICATION_ID/g" \
+      -e "s/\$SPARK_USER/$SPARK_USER/g" \
+      -e "s/\$HOSTNAME/$HOSTNAME/g" \
+      $log_layer_file
+}
+
+function log_forwarding {
+  # We need to escape special characters from URL to be able to use with template.
+  local loki_url="$(<<< "$LOKI_URL" sed -e 's`[][\\/.*^$]`\\&`g')"
+  if [ ! -z "$loki_url" ]; then
+      echo "Log-forwarding to Loki is enabled."
+      local rendered_log_layer=$(get_log_layer $loki_url)
+      echo "$rendered_log_layer" | tee /tmp/rendered_log_layer.yaml
+      pebble add logging /tmp/rendered_log_layer.yaml
+  else
+      echo "Log-forwarding to Loki is disabled."
+  fi
+}
+
 function finish {
   if [ $? -ne 0 ]
   then
@@ -9,13 +33,13 @@ function finish {
 }
 trap finish EXIT
 
-
 FLAVOUR=$1
 
 echo "Running script with ${FLAVOUR} flavour"
 
 case "${FLAVOUR}" in
   driver|executor)
+    log_forwarding
     pushd /opt/spark
     ./entrypoint.sh "$@"
     ;;

images/charmed-spark/conf/jmx_prometheus.yaml

@@ -0,0 +1,2 @@
+rules:
+- pattern: ".*"

images/charmed-spark/pebble/log-layer.yaml

@@ -0,0 +1,13 @@
+log-targets:
+  grafana-agent-k8s:
+    override: replace
+    type: loki
+    location: $LOKI_URL
+    services: [all]
+    labels:
+      product: charmed-spark
+      role: $FLAVOUR
+      app: spark
+      spark_job_id: $SPARK_APPLICATION_ID
+      user: $SPARK_USER
+      pod: $HOSTNAME

images/charmed-spark/rockcraft.yaml

@@ -100,14 +100,17 @@ parts:
       ICEBERG_VERSION='1.4.3'
       SPARK_METRICS_VERSION='3.4-1.0.1'
       SERVLET_FILTERS_VERSION='0.0.1'
+      PROMETHEUS_JMX_EXPORTER_VERSION='0.15.0'
       SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df'
       SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='493cf77133cbf03e96fb848121ce10ac16e6f907f595df637649b98b42118e57d6b6e1bdab71bfee3394eb369637c5b4f6b05dd8fa30a1ff6899e74069c972ce'
       SHA512SUM_SPARK_SERVLET_FILTER_JAR='ffeb809d58ef0151d513b09d4c2bfd5cc064b0b888ca45899687aed2f42bcb1ce9834be9709290dd70bd9df84049f02cbbff6c2d5ec3c136c278c93f167c8096'
+      SHA1SUM_PROMETHEUS_JMX_EXPORTER='a1061f29088ac2709da076a97736de575a872538'
 
       JARS=(
       "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/LIB_VERSION/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-LIB_VERSION.jar $ICEBERG_VERSION sha1sum $SHA1SUM_ICEBERG_JAR"
       "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-LIB_VERSION/spark-metrics-assembly-LIB_VERSION.jar $SPARK_METRICS_VERSION sha512sum $SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR"
       "https://github.com/canonical/central-uploader/releases/download/servlet-filters-LIB_VERSION/servlet-filters-LIB_VERSION.jar $SERVLET_FILTERS_VERSION sha512sum $SHA512SUM_SPARK_SERVLET_FILTER_JAR"
+      "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/LIB_VERSION/jmx_prometheus_javaagent-LIB_VERSION.jar $PROMETHEUS_JMX_EXPORTER_VERSION sha1sum $SHA1SUM_PROMETHEUS_JMX_EXPORTER"
       )
       for ENTRY in "${JARS[@]}"; do
         echo "$ENTRY"
@@ -171,22 +174,26 @@ parts:
     source: .
     organize:
       conf/spark-defaults.conf: etc/spark8t/conf/spark-defaults.conf
+      conf/jmx_prometheus.yaml: etc/spark/conf/jmx_prometheus.yaml
       bin/sparkd.sh: opt/pebble/sparkd.sh
       bin/history-server.sh: opt/pebble/history-server.sh
       bin/spark-client.pyspark: opt/spark-client/python/bin/spark-client.pyspark
       bin/spark-client.spark-sql: opt/spark-client/python/bin/spark-client.spark-sql
       bin/spark-client.service-account-registry: opt/spark-client/python/bin/spark-client.service-account-registry
       bin/spark-client.spark-shell: opt/spark-client/python/bin/spark-client.spark-shell
       bin/spark-client.spark-submit: opt/spark-client/python/bin/spark-client.spark-submit
+      pebble/log-layer.yaml: opt/pebble/log-layer.yaml
     stage:
       - etc/spark8t/conf/
+      - etc/spark/conf/
       - opt/pebble/sparkd.sh
       - opt/pebble/history-server.sh
       - opt/spark-client/python/bin/spark-client.pyspark
       - opt/spark-client/python/bin/spark-client.spark-sql
       - opt/spark-client/python/bin/spark-client.service-account-registry
       - opt/spark-client/python/bin/spark-client.spark-shell
       - opt/spark-client/python/bin/spark-client.spark-submit
+      - opt/pebble/log-layer.yaml
 
   user-setup:
     plugin: nil

tests/integration/integration-tests.sh

@@ -53,6 +53,20 @@ validate_metrics() {
   fi
 }
 
+validate_logs() {
+  log=$1
+  if [ $(grep -Ri "Log-forwarding to Loki is enabled." $log | wc -l) -lt 3 ]; then
+    echo "ERROR: Could not validate logs."
+    echo "DEBUG: Log file:\n$(cat $log)"
+    exit 1
+  fi
+  if [ $(grep -Ri 'Layer \\\\"logging\\\\" added successfully from \\\\"/tmp/rendered_log_layer.yaml\\\\"' $log | wc -l) -lt 3 ]; then
+    echo "ERROR: Could not validate logs."
+    echo "DEBUG: Log file:\n$(cat $log)"
+    exit 1
+  fi
+}
+
 setup_user() {
   echo "setup_user() ${1} ${2}"
 
@@ -412,6 +426,56 @@ run_example_job_in_pod_with_metrics() {
 }
 
 
+run_example_job_in_pod_with_log_forwarding() {
+  NAMESPACE=${1-$NAMESPACE}
+  USERNAME=${2-spark}
+  SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"
+
+  PREVIOUS_JOB=$(kubectl -n $NAMESPACE get pods --sort-by=.metadata.creationTimestamp | grep driver | tail -n 1 | cut -d' ' -f1)
+  # start simple http server
+  LOG_FILE="/tmp/server-loki.log"
+  SERVER_PORT=9091
+  python3 tests/integration/resources/test_web_server.py $SERVER_PORT > $LOG_FILE &
+  HTTP_SERVER_PID=$!
+  # get ip address
+  IP_ADDRESS=$(hostname -I | cut -d ' ' -f 1)
+  echo "IP: $IP_ADDRESS"
+
+  kubectl -n $NAMESPACE exec testpod -- env PORT="$SERVER_PORT" IP="$IP_ADDRESS" UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
+                  /bin/bash -c 'spark-client.spark-submit \
+                  --username $UU --namespace $NN \
+                  --conf spark.kubernetes.driver.request.cores=100m \
+                  --conf spark.kubernetes.executor.request.cores=100m \
+                  --conf spark.kubernetes.container.image=$IM \
+                  --conf spark.executorEnv.LOKI_URL="http://$IP:$PORT" \
+                  --conf spark.kubernetes.driverEnv.LOKI_URL="http://$IP:$PORT" \
+                  --class org.apache.spark.examples.SparkPi \
+                  local:///opt/spark/examples/jars/$JJ 1000'
+
+  # kubectl --kubeconfig=${KUBE_CONFIG} get pods
+  DRIVER_PODS=$(kubectl get pods --sort-by=.metadata.creationTimestamp -n ${NAMESPACE} | grep driver )
+  DRIVER_JOB=$(kubectl get pods --sort-by=.metadata.creationTimestamp -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1)
+
+  if [[ "${DRIVER_JOB}" == "${PREVIOUS_JOB}" ]]
+  then
+    echo "ERROR: Sample job has not run!"
+    exit 1
+  fi
+
+  # Check job output
+  # Sample output
+  # "Pi is roughly 3.13956232343"
+  pi=$(kubectl logs $(kubectl get pods --sort-by=.metadata.creationTimestamp -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1)  -n ${NAMESPACE} | grep 'Pi is roughly' | rev | cut -d' ' -f1 | rev | cut -c 1-3)
+  echo -e "Spark Pi Job Output: \n ${pi}"
+
+  validate_pi_value $pi
+  validate_logs $LOG_FILE
+
+  # kill http server
+  kill $HTTP_SERVER_PID
+}
+
+
 run_example_job_with_error_in_pod() {
   SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"
 
@@ -647,6 +711,12 @@ echo -e "########################################"
 
 (setup_user_context && test_example_job_in_pod_with_metrics && cleanup_user_success) || cleanup_user_failure_in_pod
 
+echo -e "########################################"
+echo -e "RUN EXAMPLE JOB WITH LOG FORWARDING"
+echo -e "########################################"
+
+(setup_user_context && run_example_job_in_pod_with_log_forwarding && cleanup_user_success) || cleanup_user_failure_in_pod
+
 echo -e "########################################"
 echo -e "RUN EXAMPLE JOB WITH ERRORS"
 echo -e "########################################"

tests/integration/setup-aws-cli.sh

@@ -2,14 +2,41 @@
 
 # Install AWS CLI
 sudo snap install aws-cli --classic
-
+set -x
 
 get_s3_endpoint(){
-    # Get S3 endpoint from MinIO
-    kubectl get service minio -n minio-operator -o jsonpath='{.spec.clusterIP}' 
+  # Print the endpoint where the S3 bucket is exposed on.
+  kubectl get service minio -n minio-operator -o jsonpath='{.spec.clusterIP}'
+}
+
+
+get_s3_access_key(){
+  # Print the S3 Access Key by reading it from K8s secret or by outputting the default value
+    kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null
+    if [ $? -eq 0 ]; then
+        # echo "Use access-key from secret"
+        access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d)
+    else
+        # echo "use default access-key"
+        access_key="minio"
+    fi
+    echo "$access_key"
 }
 
 
+get_s3_secret_key(){
+  # Print the S3 Secret Key by reading it from K8s secret or by outputting the default value
+    kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null
+    if [ $? -eq 0 ]; then
+      # echo "Use access-key from secret"
+      secret_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d)
+    else
+      # echo "use default access-key"
+      secret_key="minio123"
+    fi
+    echo "$secret_key"
+}
+
 wait_and_retry(){
     # Retry a command for a number of times by waiting a few seconds.
 
@@ -37,8 +64,8 @@ wait_and_retry get_s3_endpoint
 
 S3_ENDPOINT=$(get_s3_endpoint)
 DEFAULT_REGION="us-east-2"
-ACCESS_KEY=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d)
-SECRET_KEY=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d)
+ACCESS_KEY=$(get_s3_access_key)
+SECRET_KEY=$(get_s3_secret_key)
 
 # Configure AWS CLI credentials
 aws configure set aws_access_key_id $ACCESS_KEY

tests/integration/utils/s3-utils.sh

@@ -20,14 +20,30 @@ get_s3_endpoint(){
 
 
 get_s3_access_key(){
-  # Print the S3 Access Key by reading it from K8s secret
-  kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d
+  # Print the S3 Access Key by reading it from K8s secret or by outputting the default value
+    kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null
+    if [ $? -eq 0 ]; then
+        # echo "Use access-key from secret"
+        access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d)
+    else
+        # echo "use default access-key"
+        access_key="minio"
+    fi
+    echo "$access_key"
 }
 
 
 get_s3_secret_key(){
-  # Print the S3 Secret Key by reading it from K8s secret
-  kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d
+  # Print the S3 Secret Key by reading it from K8s secret or by outputting the default value
+    kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null
+    if [ $? -eq 0 ]; then
+      # echo "Use access-key from secret"
+      secret_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d)
+    else
+      # echo "use default access-key"
+      secret_key="minio123"
+    fi
+    echo "$secret_key"
 }