Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DPE-3017] Patch Pebble issue on shutdown on failure #54

Merged
merged 3 commits into from
Nov 16, 2023
+65 −1
Merged

[DPE-3017] Patch Pebble issue on shutdown on failure #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0ab1990
Add test for validating errors in pod
welpaolo Nov 10, 2023
add9b41
Try to fix exiting
welpaolo Nov 10, 2023
f7f3263
Fix capture of error
welpaolo Nov 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion files/spark/bin/charmed-spark-entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
#!/bin/bash

function finish {
if [ $? -ne 0 ]
then
kill -1 1
sleep 1
fi
}
trap finish EXIT


FLAVOUR=$1

echo "Running script with ${FLAVOUR} flavour"
Expand All @@ -17,4 +27,4 @@ case "${FLAVOUR}" in
echo "Component \"${FLAVOUR}\" unknown"
exit 1
;;
esac
esac
54 changes: 54 additions & 0 deletions tests/integration/integration-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,54 @@ run_example_job_in_pod_with_metrics() {
}


run_example_job_with_error_in_pod() {
SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"

PREVIOUS_JOB=$(kubectl get pods | grep driver | tail -n 1 | cut -d' ' -f1)
NAMESPACE=$1
USERNAME=$2

kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
/bin/bash -c 'spark-client.spark-submit \
--username $UU --namespace $NN \
--conf spark.kubernetes.driver.request.cores=100m \
--conf spark.kubernetes.executor.request.cores=100m \
--conf spark.kubernetes.container.image=$IM \
--class org.apache.spark.examples.SparkPi \
local:///opt/spark/examples/jars/$JJ -1'

# kubectl --kubeconfig=${KUBE_CONFIG} get pods
DRIVER_JOB=$(kubectl get pods -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1)

if [[ "${DRIVER_JOB}" == "${PREVIOUS_JOB}" ]]
then
echo "ERROR: Sample job has not run!"
exit 1
fi

# Check job output
res=$(kubectl logs $(kubectl get pods -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1) -n ${NAMESPACE} | grep 'Exception in thread' | wc -l)
echo -e "Number of errors: \n ${res}"
if [ "${res}" != "1" ]; then
echo "ERROR: Error is not captured."
exit 1
fi
status=$(kubectl get pod $(kubectl get pods -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1) -n ${NAMESPACE} | tail -1 | cut -d " " -f 9)
if [ "${status}" = "Completed" ]; then
echo "ERROR: Status should not be set to Completed."
exit 1
fi
if [ "${status}" = "Error" ]; then
echo "Status is correctly set to ERROR!"
fi

}

test_example_job_in_pod_with_errors() {
run_example_job_with_error_in_pod tests spark
}


test_example_job_in_pod_with_templates() {
run_example_job_in_pod_with_pod_templates tests spark
}
Expand Down Expand Up @@ -371,6 +419,12 @@ echo -e "########################################"

(setup_user_admin_context && test_example_job_in_pod_with_metrics && cleanup_user_success) || cleanup_user_failure_in_pod


echo -e "########################################"
echo -e "RUN EXAMPLE JOB WITH ERRORS"
echo -e "########################################"

(setup_user_admin_context && test_example_job_in_pod_with_errors && cleanup_user_success) || cleanup_user_failure_in_pod
echo -e "##################################"
echo -e "TEARDOWN TEST POD"
echo -e "##################################"
Expand Down