From 9eeb3da734d4c2d421dfce69a783b9c3db66db83 Mon Sep 17 00:00:00 2001 From: Saad Khan Date: Tue, 1 Oct 2024 10:11:12 +0530 Subject: [PATCH 1/3] add test script for authentication Signed-off-by: Saad Khan --- tests/scripts/common/common_functions.sh | 3 +- tests/scripts/functional_tests.sh | 1 + .../authentication_tests.sh | 147 ++++++++++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 tests/scripts/local_monitoring_tests/authentication_tests.sh diff --git a/tests/scripts/common/common_functions.sh b/tests/scripts/common/common_functions.sh index 3333305af..263a9f79a 100755 --- a/tests/scripts/common/common_functions.sh +++ b/tests/scripts/common/common_functions.sh @@ -46,7 +46,8 @@ TEST_SUITE_ARRAY=("app_autotune_yaml_tests" "kruize_layer_id_tests" "em_standalone_tests" "remote_monitoring_tests" -"local_monitoring_tests") +"local_monitoring_tests" +"authentication_tests") modify_kruize_layer_tests=("add_new_tunable" "apply_null_tunable" diff --git a/tests/scripts/functional_tests.sh b/tests/scripts/functional_tests.sh index bfea136ab..39edbd91a 100755 --- a/tests/scripts/functional_tests.sh +++ b/tests/scripts/functional_tests.sh @@ -33,6 +33,7 @@ SCRIPTS_DIR="${CURRENT_DIR}" . ${SCRIPTS_DIR}/em/em_standalone_tests.sh . ${SCRIPTS_DIR}/remote_monitoring_tests/remote_monitoring_tests.sh . ${SCRIPTS_DIR}/local_monitoring_tests/local_monitoring_tests.sh +. ${SCRIPTS_DIR}/local_monitoring_tests/authentication_tests.sh # Iterate through the commandline options while getopts i:o:r:-: gopts diff --git a/tests/scripts/local_monitoring_tests/authentication_tests.sh b/tests/scripts/local_monitoring_tests/authentication_tests.sh new file mode 100644 index 000000000..e765f2d31 --- /dev/null +++ b/tests/scripts/local_monitoring_tests/authentication_tests.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Copyright (c) 2024, 2024 Red Hat, IBM Corporation and others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Get the absolute path of current directory +CURRENT_DIR="$(dirname "$(realpath "$0")")" +LOCAL_MONITORING_TEST_DIR="${CURRENT_DIR}/local_monitoring_tests" + + +# Source the common functions scripts +. ${LOCAL_MONITORING_TEST_DIR}/../common/common_functions.sh + +NAMESPACE="openshift-tuning" +APP_DEPLOYMENT="kruize" +DB_DEPLOYMENT="kruize-db-deployment" +DB_PVC="kruize-db-pv-claim" +SECRET_NAME="custom-token-secret" # TODO: to be updated +AUTOTUNE_IMAGE="quay.io/kruize/autotune_operator:0.0.25_mvp" +# Configuration +AUTH_TOKEN_PATH="/var/run/secrets/kubernetes.io/serviceaccount/token" +DATASOURCE_URL="https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091" +YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml" + +# Tests to validate authentication types in Kruize +function authentication_tests() { + TEST_SUITE_DIR="${RESULTS}/authentication_tests" + mkdir -p ${TEST_SUITE_DIR} 2>&1 + for token_type in "${!tokens[@]}"; do + deploy_and_check_pod $token_type + done + +# Define token scenarios +declare -A tokens +tokens=( + ["valid"]="/var/run/secrets/kubernetes.io/serviceaccount/token" + ["expired"]="EXPIRED_TOKEN" + ["invalid"]="random-invalid-token-string" + ["empty"]="" +) + + +# Update the YAML file with the token +update_yaml_with_token() { + local token_value=$1 + # Escape special characters in the new token to avoid sed issues + new_token_escaped=$(printf '%s\n' "$new_token" | sed -e 's/[\/&]/\\&/g') + + sed -i.bak 's/\("tokenFilePath": \)"[^"]*"/\1"'"$new_token_escaped"'"/' $YAML_FILE +} + +# Deploy app and check pod status +deploy_and_check_pod() { + local token_type=$1 + echo "**********************************" + echo "Testing with $token_type token..." + echo "**********************************" + + LOG="${TEST_SUITE_DIR}/${token_type}.log" + echo "***********************************" + echo "Terminating any existing instance of kruize..." + echo "***********************************" + kruize_terminate > /dev/null + sleep 10 + + # Update the secret with the appropriate token + echo "*************************************" + echo "Updating the yaml with $token_type token..." + echo "*************************************" + update_yaml_with_token "${tokens[$token_type]}" + echo "" + + # Restart the app and db pod (if it's already running) +# kubectl rollout restart deployment/$APP_DEPLOYMENT -n $NAMESPACE :TODO: to be used once the code is fixed + # Run the deployment script again + echo "**********************" + echo "Redeploying kruize..." + echo "**********************" + ${LOCAL_MONITORING_TEST_DIR}/../../../deploy.sh -c ${cluster_type} -i ${AUTOTUNE_IMAGE} -m crc > /dev/null + # Wait for the pod to be ready or fail + kubectl wait --for=condition=Ready pod -l app=$APP_DEPLOYMENT -n $NAMESPACE --timeout=120s >> ${LOG} #2> /dev/null + local pod_status=$? + # Check pod logs for errors + if [ $pod_status -ne 0 ]; then + echo "$token_type token: Pod failed to start as expected." + kubectl logs -l app=$APP_DEPLOYMENT -n $NAMESPACE --tail=20 + else + echo "$token_type token: Pod started successfully (unexpected for invalid tokens)." + kubectl logs -l app=$APP_DEPLOYMENT -n $NAMESPACE --tail=20 + fi + + # Check pod logs for errors + echo "Checking logs for the pod..." + pod_logs=$(kubectl logs -l app=$APP_DEPLOYMENT -n $NAMESPACE --tail=100) + + # Check if the log contains the error message + if echo "$pod_logs" | grep -q "Datasource is not serviceable."; then + echo "$token_type token: Failure detected in logs (as expected for invalid tokens)." + else + echo "$token_type token: No failure detected in logs (as expected for valid tokens)." + fi + # Restore original YAML file + mv ${YAML_FILE}.bak $YAML_FILE + +} + +function kruize_terminate() { + ${LOCAL_MONITORING_TEST_DIR}/../../../deploy.sh -c ${cluster_type} -i ${AUTOTUNE_IMAGE} -m crc -t + # Wait for the pod to terminate + while true; do + # Get the status of the pod + pod_name=$(kubectl get pod -l app=kruize -o jsonpath="{.items[0].metadata.name}" 2>/dev/null) + namespace_status=$(kubectl get namespace $NAMESPACE --no-wait -o jsonpath='{.status.phase}' 2>/dev/null) + + + # Check if the pod exists + if [ -z "$pod_name" ]; then + echo "Pod has fully terminated." + break + fi + + # Get the pod phase (Running, Succeeded, Failed, etc.) + pod_phase=$(kubectl get pod $pod_name -o jsonpath='{.status.phase}' 2>/dev/null) + + # Check the pod phase + if [ "$pod_phase" == "Succeeded" ] || [ "$pod_phase" == "Failed" ]; then + echo "Pod has terminated with status: $pod_phase." + break + fi + + echo "Waiting for pod to terminate..." + sleep 5 + done +} From 2525a6b1dec6912dac00e78e06839e97712082ce Mon Sep 17 00:00:00 2001 From: Saad Khan Date: Mon, 25 Nov 2024 03:02:23 +0530 Subject: [PATCH 2/3] update test scripts and readme Signed-off-by: Saad Khan --- .../Local_monitoring_tests.md | 10 + .../authentication_tests.sh | 214 ++++++++++-------- 2 files changed, 134 insertions(+), 90 deletions(-) diff --git a/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md b/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md index 3444b66d3..15504c488 100644 --- a/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md +++ b/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md @@ -182,3 +182,13 @@ Else, you can change the workload name and namespace name in the test to match w Note: The test will fail if it's run as is if there are no matching workloads that the test looks for. This test result can be ignored in case of a non-gpu workload +### Authentication Test: + +Kruize 0.1.1 supports the authentication which provides the user an option to pass authentication details in the yaml for the service they are using. + +The authentication test is part of functional bucket and has a separate script similar to local_monitoring tests. It contains various valid and invalid scenarios for testing. + +It can be run as shown in the example below: + +`/test_autotune.sh -c -i -r benchmarks/ --testsuite=authentication_tests` + diff --git a/tests/scripts/local_monitoring_tests/authentication_tests.sh b/tests/scripts/local_monitoring_tests/authentication_tests.sh index e765f2d31..ecce7c41b 100644 --- a/tests/scripts/local_monitoring_tests/authentication_tests.sh +++ b/tests/scripts/local_monitoring_tests/authentication_tests.sh @@ -20,28 +20,10 @@ CURRENT_DIR="$(dirname "$(realpath "$0")")" LOCAL_MONITORING_TEST_DIR="${CURRENT_DIR}/local_monitoring_tests" - # Source the common functions scripts . ${LOCAL_MONITORING_TEST_DIR}/../common/common_functions.sh -NAMESPACE="openshift-tuning" APP_DEPLOYMENT="kruize" -DB_DEPLOYMENT="kruize-db-deployment" -DB_PVC="kruize-db-pv-claim" -SECRET_NAME="custom-token-secret" # TODO: to be updated -AUTOTUNE_IMAGE="quay.io/kruize/autotune_operator:0.0.25_mvp" -# Configuration -AUTH_TOKEN_PATH="/var/run/secrets/kubernetes.io/serviceaccount/token" -DATASOURCE_URL="https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091" -YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml" - -# Tests to validate authentication types in Kruize -function authentication_tests() { - TEST_SUITE_DIR="${RESULTS}/authentication_tests" - mkdir -p ${TEST_SUITE_DIR} 2>&1 - for token_type in "${!tokens[@]}"; do - deploy_and_check_pod $token_type - done # Define token scenarios declare -A tokens @@ -51,97 +33,149 @@ tokens=( ["invalid"]="random-invalid-token-string" ["empty"]="" ) +# Tests to validate authentication types in Kruize +function authentication_tests() { + start_time=$(get_date) + FAILED_CASES=() + TESTS=0 + failed=0 + ((TOTAL_TEST_SUITES++)) + TEST_SUITE_DIR="${RESULTS}/authentication_tests" + mkdir -p "${TEST_SUITE_DIR}" 2>&1 + KRUIZE_SETUP_LOG="${TEST_SUITE_DIR}/kruize_setup.log" + KRUIZE_POD_LOG="${TEST_SUITE_DIR}/kruize_pod.log" + target="crc" + echo "" + echo "Setting up kruize..." | tee -a ${LOG} + echo "${KRUIZE_SETUP_LOG}" + setup "${KRUIZE_POD_LOG}" >> "${KRUIZE_SETUP_LOG}" 2>&1 + echo "Setting up kruize...Done" | tee -a ${LOG} + sleep 15 + if [ "$cluster_type" == "minikube" ] || [ "$cluster_type" == "kind" ]; then + NAMESPACE="monitoring" + YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml" + elif [ "$cluster_type" == "openshift" ]; then + NAMESPACE="openshift-tuning" + YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml" + else + echo "Invalid cluster type found: ${cluster_type}" + fi + kubectl_cmd="kubectl -n ${NAMESPACE}" -# Update the YAML file with the token -update_yaml_with_token() { - local token_value=$1 - # Escape special characters in the new token to avoid sed issues - new_token_escaped=$(printf '%s\n' "$new_token" | sed -e 's/[\/&]/\\&/g') + echo "" + echo "******************* Executing test suite ${FUNCNAME} ****************" + echo "" + for token_type in "${!tokens[@]}"; + do + echo "" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo " Running Test ${token_type}-token" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + + deploy_and_check_pod "$token_type" + +# check for success and failed cases here + if [ "${TESTS_FAILED}" -ne "0" ]; then + FAILED_CASES+=(${test}) + fi + done - sed -i.bak 's/\("tokenFilePath": \)"[^"]*"/\1"'"$new_token_escaped"'"/' $YAML_FILE + TESTS=$(($TESTS_PASSED + $TESTS_FAILED)) + TOTAL_TESTS_FAILED=${TESTS_FAILED} + TOTAL_TESTS_PASSED=${TESTS_PASSED} + TOTAL_TESTS=${TESTS} + + if [ "${TESTS_FAILED}" -ne "0" ]; then + FAILED_TEST_SUITE+=(${FUNCNAME}) + fi + + end_time=$(get_date) + elapsed_time=$(time_diff "${start_time}" "${end_time}") + + # Remove the duplicates + FAILED_CASES=( $(printf '%s\n' "${FAILED_CASES[@]}" | uniq ) ) + + # print the testsuite summary + testsuitesummary ${FUNCNAME} "${elapsed_time}" ${FAILED_CASES} } # Deploy app and check pod status deploy_and_check_pod() { local token_type=$1 - echo "**********************************" - echo "Testing with $token_type token..." - echo "**********************************" - - LOG="${TEST_SUITE_DIR}/${token_type}.log" - echo "***********************************" - echo "Terminating any existing instance of kruize..." - echo "***********************************" - kruize_terminate > /dev/null - sleep 10 + POD_LOG="${TEST_SUITE_DIR}/${token_type}-pod.log" - # Update the secret with the appropriate token + # Update the yaml with the appropriate token echo "*************************************" - echo "Updating the yaml with $token_type token..." + echo "Updating the yaml with $token_type token and restarting kruize..." echo "*************************************" update_yaml_with_token "${tokens[$token_type]}" echo "" - # Restart the app and db pod (if it's already running) -# kubectl rollout restart deployment/$APP_DEPLOYMENT -n $NAMESPACE :TODO: to be used once the code is fixed - # Run the deployment script again - echo "**********************" - echo "Redeploying kruize..." - echo "**********************" - ${LOCAL_MONITORING_TEST_DIR}/../../../deploy.sh -c ${cluster_type} -i ${AUTOTUNE_IMAGE} -m crc > /dev/null - # Wait for the pod to be ready or fail - kubectl wait --for=condition=Ready pod -l app=$APP_DEPLOYMENT -n $NAMESPACE --timeout=120s >> ${LOG} #2> /dev/null - local pod_status=$? - # Check pod logs for errors - if [ $pod_status -ne 0 ]; then - echo "$token_type token: Pod failed to start as expected." - kubectl logs -l app=$APP_DEPLOYMENT -n $NAMESPACE --tail=20 - else - echo "$token_type token: Pod started successfully (unexpected for invalid tokens)." - kubectl logs -l app=$APP_DEPLOYMENT -n $NAMESPACE --tail=20 - fi - + # re-apply the yaml to update the auth config + $kubectl_cmd apply -f "$YAML_FILE" > /dev/null + # get the kruize pod name + POD_NAME=$($kubectl_cmd get pods | grep 'kruize' | grep -v -E 'kruize-db|kruize-ui' | awk 'NR==1{print $1}') + # Check if POD_NAME is not empty + if [ -n "$POD_NAME" ]; then + # Delete the pod + $kubectl_cmd delete pod "$POD_NAME" + else + echo "No matching pod found to delete." + fi + + # Wait for the new pod to be ready or fail + $kubectl_cmd wait --for=condition=Ready pod -l app=$APP_DEPLOYMENT --timeout=120s > /dev/null # Check pod logs for errors echo "Checking logs for the pod..." - pod_logs=$(kubectl logs -l app=$APP_DEPLOYMENT -n $NAMESPACE --tail=100) - - # Check if the log contains the error message - if echo "$pod_logs" | grep -q "Datasource is not serviceable."; then - echo "$token_type token: Failure detected in logs (as expected for invalid tokens)." + POD_NAME=$($kubectl_cmd get pods | grep 'kruize' | grep -v -E 'kruize-db|kruize-ui' | awk 'NR==1{print $1}') + echo "$kubectl_cmd logs -f ${POD_NAME} > ${POD_LOG} 2>&1 &" + $kubectl_cmd logs -f "${POD_NAME}" > "${POD_LOG}" 2>&1 & + sleep 10 + echo "" + # Determine the test outcome based on logs + if [[ $(grep -i "Datasource connection refused or timed out" ${POD_LOG}) ]]; then + if [ "$token_type" == "valid" ]; then + echo "$token_type token: Unexpected failure detected in logs." + ((TESTS_FAILED++)) # Increment the global TESTS_FAILED + else + echo "$token_type token: Failure detected in logs (as expected for invalid tokens)." + ((TESTS_PASSED++)) # Increment the global TESTS_PASSED + fi else - echo "$token_type token: No failure detected in logs (as expected for valid tokens)." + if [ "$token_type" == "valid" ]; then + echo "$token_type token: No failure detected in logs (as expected for valid tokens)." + ((TESTS_PASSED++)) # Increment the global TESTS_PASSED + else + echo "$token_type token: Unexpected success detected in logs." + ((TESTS_FAILED++)) # Increment the global TESTS_FAILED + fi fi # Restore original YAML file - mv ${YAML_FILE}.bak $YAML_FILE - + mv "${YAML_FILE}".token.bak "$YAML_FILE" + mv "${YAML_FILE}".image.bak "$YAML_FILE" } -function kruize_terminate() { - ${LOCAL_MONITORING_TEST_DIR}/../../../deploy.sh -c ${cluster_type} -i ${AUTOTUNE_IMAGE} -m crc -t - # Wait for the pod to terminate - while true; do - # Get the status of the pod - pod_name=$(kubectl get pod -l app=kruize -o jsonpath="{.items[0].metadata.name}" 2>/dev/null) - namespace_status=$(kubectl get namespace $NAMESPACE --no-wait -o jsonpath='{.status.phase}' 2>/dev/null) - - - # Check if the pod exists - if [ -z "$pod_name" ]; then - echo "Pod has fully terminated." - break - fi - - # Get the pod phase (Running, Succeeded, Failed, etc.) - pod_phase=$(kubectl get pod $pod_name -o jsonpath='{.status.phase}' 2>/dev/null) - - # Check the pod phase - if [ "$pod_phase" == "Succeeded" ] || [ "$pod_phase" == "Failed" ]; then - echo "Pod has terminated with status: $pod_phase." - break - fi - - echo "Waiting for pod to terminate..." - sleep 5 - done +# Update the YAML file with the token +update_yaml_with_token() { + local token_value=$1 + + # Escape special characters in the new token to avoid sed issues + new_token_escaped=$(printf '%s\n' "$token_value" | sed -e 's/[\/&]/\\&/g') + + # Update the tokenFilePath + sed -i.token.bak 's/\("tokenFilePath": \)"[^"]*"/\1"'"$new_token_escaped"'"/' "$YAML_FILE" + echo "Token updated" + + # Update the image in the Deployment YAML + sed -i.image.bak ' + /kind: Deployment/,/kind:/{ + /name: kruize$/,/containers:/{ + /^ - name: kruize$/{ + n + s|image: .*|image: '"$AUTOTUNE_IMAGE"'| + } + } + }' "$YAML_FILE" + echo "Updated image in YAML to $AUTOTUNE_IMAGE" } From f484a5a49b16cfef211fc0b40c985e2d69973cb5 Mon Sep 17 00:00:00 2001 From: Saad Khan Date: Mon, 25 Nov 2024 12:12:21 +0530 Subject: [PATCH 3/3] update readme and token scenario as per the review Signed-off-by: Saad Khan --- .../local_monitoring_tests/Local_monitoring_tests.md | 10 +++++++++- .../local_monitoring_tests/authentication_tests.sh | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md b/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md index 15504c488..5fa8d3f9f 100644 --- a/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md +++ b/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md @@ -184,7 +184,7 @@ Note: The test will fail if it's run as is if there are no matching workloads th ### Authentication Test: -Kruize 0.1.1 supports the authentication which provides the user an option to pass authentication details in the yaml for the service they are using. +Kruize 0.2 supports the authentication which provides the user an option to pass authentication details in the yaml for the service they are using. The authentication test is part of functional bucket and has a separate script similar to local_monitoring tests. It contains various valid and invalid scenarios for testing. @@ -192,3 +192,11 @@ It can be run as shown in the example below: `/test_autotune.sh -c -i -r benchmarks/ --testsuite=authentication_tests` +#### Scenarios +**_valid_**: a valid path to the token + +**_expired_**: an expired token value + +**_invalid_**: an invalid path to the token + +**_empty_**: a blank input in place of the token file path diff --git a/tests/scripts/local_monitoring_tests/authentication_tests.sh b/tests/scripts/local_monitoring_tests/authentication_tests.sh index ecce7c41b..623376879 100644 --- a/tests/scripts/local_monitoring_tests/authentication_tests.sh +++ b/tests/scripts/local_monitoring_tests/authentication_tests.sh @@ -30,7 +30,7 @@ declare -A tokens tokens=( ["valid"]="/var/run/secrets/kubernetes.io/serviceaccount/token" ["expired"]="EXPIRED_TOKEN" - ["invalid"]="random-invalid-token-string" + ["invalid"]="/var/run/secrets/kubernetes.io/serviceaccount/token2" ["empty"]="" ) # Tests to validate authentication types in Kruize