Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Included grpc post experiment and post experiment result tests #85

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Included grpc post experiment and post experiment result tests #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 157 additions & 2 deletions tests/scripts/common/common_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ function deploy_hpo() {
./deploy_hpo.sh -c ${cluster_type} -o ${HPO_CONTAINER_IMAGE} -n ${namespace}
elif [ ${cluster_type} == "openshift" ]; then
namespace="openshift-tuning"
namespace="default"
cmd="./deploy_hpo.sh -c ${cluster_type} -o ${HPO_CONTAINER_IMAGE} -n ${namespace}"
echo "Command to deploy hpo - ${cmd}"
./deploy_hpo.sh -c ${cluster_type} -o ${HPO_CONTAINER_IMAGE} -n ${namespace}
Expand Down Expand Up @@ -135,11 +136,12 @@ function deploy_hpo() {
function terminate_hpo() {
cluster_type=$1

namespace="default"
pushd ${HPO_REPO} > /dev/null
echo "Terminating hpo..."
cmd="./deploy_hpo.sh -c ${cluster_type} -t"
cmd="./deploy_hpo.sh -c ${cluster_type} -n ${namespace} -t"
echo "CMD = ${cmd}"
./deploy_hpo.sh -c ${cluster_type} -t
./deploy_hpo.sh -c ${cluster_type} -n ${namespace} -t
popd > /dev/null
echo "done"
}
Expand Down Expand Up @@ -535,6 +537,7 @@ function form_hpo_api_url {
;;
openshift)
hpo_ns="openshift-tuning"
hpo_ns="default"
SERVER_IP=$(oc -n ${hpo_ns} get pods -l=app=hpo -o wide -o=custom-columns=NODE:.spec.nodeName --no-headers)
PORT=$(oc get svc hpo --no-headers -o=custom-columns=PORT:.spec.ports[*].nodePort)
;;
Expand Down Expand Up @@ -734,3 +737,155 @@ function post_experiment_json() {
echo "http_code is $http_code Response is ${response}"
}

# Post a JSON object to HPO (Hyper Parameter Optimization) module
# input: JSON object
# output: Create the command with given JSON and get the result
function post_grpc_experiment_json() {
json_array_=$1
echo ""
echo "******************************************"
echo "json array = ${json_array_}"
echo "******************************************"

grpc_client="../src/grpc_client.py"
search_space="${TESTS_}/searchspace.json"

echo "${json_array_}" > ${search_space}

python ${grpc_client} new --file ${search_space} > ${TESTS_}/output.log 2>&1
post_exp_status=$?

post_experiment_cmd="python ${grpc_client} new --file ${search_space} > ${TESTS_}/output.log 2>&1"

echo "" | tee -a ${LOG_} ${LOG}
echo "Command used to post the experiment = ${post_experiment_cmd}" | tee -a ${LOG_} ${LOG}
echo "" | tee -a ${LOG_} ${LOG}

cat "${TESTS_}/output.log"
}

# The test does the following:
# In case of hpo_post_experiment test, Post valid and invalid experiments to HPO /experiment_trials API and validate the reslut
# In case of hpo_post_exp_result test, Post valid and invalid experiments results to HPO /experiment_trials API and validate the result
# input: Test name
function run_grpc_post_tests(){
hpo_test_name=$1

if [ "${hpo_test_name}" == "hpo_grpc_post_experiment" ]; then
exp_tests=("${run_grpc_post_experiment_tests[@]}")
else
exp_tests=("${run_grpc_post_exp_result_tests[@]}")
fi

SERV_LOG="${TEST_DIR}/service.log"
# Deploy hpo
if [ ${cluster_type} == "native" ]; then
deploy_hpo ${cluster_type} ${SERV_LOG}
else
deploy_hpo ${cluster_type} ${HPO_CONTAINER_IMAGE} ${SERV_LOG}
fi

# Check if HPO services are started
check_server_status "${SERV_LOG}"

for post_test in "${exp_tests[@]}"
do

# Get the length of the service log before the test
log_length_before_test=$(cat ${SERV_LOG} | wc -l)

TESTS_="${TEST_DIR}/${post_test}"
mkdir -p ${TESTS_}
LOG_="${TEST_DIR}/${post_test}.log"
TEST_SERV_LOG="${TESTS_}/service.log"

echo "************************************* ${post_test} Test ****************************************" | tee -a ${LOG_} ${LOG}
echo "" | tee -a ${LOG_} ${LOG}

exp="${post_test}"

experiment_name=""
if [ "${hpo_test_name}" == "hpo_grpc_post_exp_result" ]; then
exp="valid-experiment"

# Get the experiment id from search space JSON
experiment_name=$(echo ${hpo_grpc_post_experiment_json[${exp}]} | jq '.experiment_name')

# Post the experiment JSON to HPO /experiment_trials API
post_grpc_experiment_json "${hpo_grpc_post_experiment_json[${exp}]}"

# Post the experiment result to HPO /experiment_trials API
echo "Posting result - $post_test"
echo "${hpo_post_exp_result[$post_test]}"
post_grpc_experiment_result "${hpo_post_exp_result_json[${post_test}]}"

expected_log_msg="${hpo_grpc_exp_result_error_messages[$post_test]}"
else
# Get the experiment id from search space JSON
experiment_name=$(echo ${hpo_grpc_post_experiment_json[${post_test}]} | jq '.experiment_name')
# Post the experiment JSON to HPO /experiment_trials API
post_grpc_experiment_json "${hpo_grpc_post_experiment_json[$post_test]}"

expected_log_msg="${hpo_grpc_error_messages[$post_test]}"

fi

#if [[ "${post_test}" == valid* ]]; then
# expected_result_="200"
#else
# expected_result_="400"
# if [[ "${post_test}" == "generate-subsequent" ]]; then
# expected_result_="404"
# fi
# fi

# actual_result="${http_code}"

echo "********** post_exp_status = $post_exp_status"
should_stop_experiment=false
if [[ "${post_exp_status}" -eq "0" ]]; then
should_stop_experiment=true
fi

# Extract the lines from the service log after log_length_before_test
extract_lines=`expr ${log_length_before_test} + 1`
cat ${SERV_LOG} | tail -n +${extract_lines} > ${TEST_SERV_LOG}

echo ""
echo "log_length_before_test ${log_length_before_test}"
echo "extract_lines ${extract_lines}"
echo ""

echo ""
if [[ "${http_code}" -eq "000" ]]; then
failed=1
((TOTAL_TESTS++))
((TESTS++))
error_message "${failed}" "${post_test}"
else
echo "actual_result = $actual_result expected_result = ${expected_result_}"
compare_result "${post_test}" "${expected_result_}" "${expected_log_msg}" "${TEST_SERV_LOG}"
fi

echo ""

if [ "$should_stop_experiment" == true ]; then
echo "Stopping experiment ${experiment_name}..."
python ../src/grpc_client.py list
exp_name=$(echo $experiment_name | sed "s/\"//g")
echo "exp_name = $exp_name"
python ../src/grpc_client.py stop --name ${exp_name}
fi

echo "" | tee -a ${LOG_} ${LOG}

done

# Stop the HPO servers
echo "Terminating any running HPO servers..." | tee -a ${LOG}
terminate_hpo ${cluster_type}
echo "Terminating any running HPO servers...Done" | tee -a ${LOG}

echo "*********************************************************************************************************" | tee -a ${LOG_} ${LOG}
}

104 changes: 103 additions & 1 deletion tests/scripts/hpo/constants/hpo_api_constants.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,14 @@
# Brief description about the HPO API tests
declare -A hpo_api_test_description
hpo_api_test_description=([hpo_post_experiment]="Start the required HPO services, post an experiment json to HPO /experiment_trials API with various combinations and validate the result"
[hpo_grpc_post_experiment]="Start the required HPO services, post an experiment json to HPO using the grpc client with various combinations and validate the result"
[hpo_get_trial_json]="Start the required HPO services post a valid experiment json to HPO /experiment_trials API, query the API using different combinations of experiment id and trial_number and validate the result"
[hpo_post_exp_result]="Start the required HPO services, post a valid experiment json to HPO /experiment_trials API and then post valid and invalid combinations of experiment result to the API and validate the result"
[hpo_grpc_post_exp_result]="Start the required HPO services, post a valid experiment json to HPO using the grpc client and then post valid and invalid combinations of experiment result and validate the result"
[hpo_sanity_test]="Start the required HPO services, post a valid experiment json to HPO /experiment_trials API, get the config, post the experiment result and subsequent experiment repeatedly for the specified number of trials"
[hpo_grpc_sanity_test]="Start the required HPO services, post a valid experiment json to HPO /experiment_trials API, get the config, post the experiment result and subsequent experimentrepeatedly for the specified number of trials")
[hpo_grpc_sanity_test]="Start the required HPO services, post a valid experiment json to HPO /experiment_trials API, get the config, post the experiment result and subsequent experimentrepeatedly for the specified number of trials"
[hpo_multiple_exp_test]="Start the required HPO services, post multiple valid experiments to HPO /experiment_trials API, get the config, post the experiment result and subsequent experiment repeatedly for the specified number of trials for multiple experiments"
[hpo_grpc_multiple_exp_test]="Start the required HPO services, post multiple valid experiments to HPO using the grpc clinet, get the config, post the experiment result and subsequent experiment repeatedly for the specified number of trials for multiple experiments")

# Tests to be carried out for HPO (Hyper Parameter Optimization) module API to post an experiment
run_post_experiment_tests=(
Expand All @@ -42,7 +46,21 @@ run_post_experiment_tests=(
"generate-subsequent"
"invalid-searchspace")

# Tests to be carried out for HPO (Hyper Parameter Optimization) module API to post an experiment
run_grpc_post_experiment_tests=(
"empty-id"
"no-id"
"null-id"
"empty-name"
"no-name"
"null-name"
"valid-experiment"
"additional-field"
"invalid-searchspace"
)

other_post_experiment_tests=("post-duplicate-experiments" "operation-generate-subsequent")
other_grpc_post_experiment_tests=("post-grpc-duplicate-experiments")

# Tests to be carried out for HPO module API to get trial json
declare -A hpo_get_trial_json_tests
Expand Down Expand Up @@ -90,6 +108,29 @@ run_post_exp_result_tests=("empty-name"
"additional-field"
)

# Tests to be carried out for HPO module API to post experiment results
run_grpc_post_exp_result_tests=(
"empty-name"
"no-name"
"null-name"
"invalid-trial-number"
"no-trial-number"
"null-trial-number"
"invalid-trial-result"
"empty-trial-result"
"no-trial-result"
"null-trial-result"
"invalid-result-value-type"
"empty-result-value-type"
"no-result-value-type"
"null-result-value-type"
"invalid-result-value"
"no-result-value"
"null-result-value"
"valid-experiment-result"
"additional-field"
)

other_exp_result_post_tests=("post-duplicate-exp-result" "post-same-id-different-exp-result")

declare -A hpo_post_experiment_json=(
Expand Down Expand Up @@ -123,6 +164,26 @@ declare -A hpo_post_experiment_json=(
[invalid-searchspace]='{"operation":"EXP_TRIAL_GENERATE_NEW","search_space":{"experiment_name":"petclinic-sample-2-75884c5549-npvgd","total_trials":5,"parallel_trials":1,"experiment_id":"a123","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"xyz"}}'
)

declare -A hpo_grpc_post_experiment_json=(
[empty-id]='{"experiment_name":"petclinic-sample-2-75884c5549-npvgd","total_trials":5,"parallel_trials":1,"experiment_id":" ","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize"}'

[no-id]='{"experiment_name":"petclinic-sample-2-75884c5549-npvgd","total_trials":5,"parallel_trials":1,"value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize"}'

[null-id]='{"experiment_name":"petclinic-sample-2-75884c5549-npvgd","total_trials":5,"parallel_trials":1,"experiment_id":null,"value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize"}'

[empty-name]='{"experiment_name":" ","total_trials":5,"parallel_trials":1,"experiment_id":"a123","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize"}'

[no-name]='{"total_trials":5,"parallel_trials":1,"experiment_id":"a123","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize"}'

[null-name]='{"experiment_name":null,"total_trials":5,"parallel_trials":1,"experiment_id":"a123","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize"}'

[additional-field]='{"experiment_name":"petclinic-sample-2-75884c5549-npvgd","total_trials":5,"parallel_trials":1,"experiment_id":"a123","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize","cpu":"cputunable"}'

[valid-experiment]='{"experiment_name":"petclinic-sample-2-75884c5549-npvgd","total_trials":5,"parallel_trials":1,"experiment_id":"a123","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"minimize"}'

[invalid-searchspace]='{"experiment_name":"petclinic-sample-2-75884c5549-npvgd","total_trials":5,"parallel_trials":1,"experiment_id":"a123","value_type":"double","hpo_algo_impl":"optuna_tpe","objective_function":"transaction_response_time","tunables":[{"value_type":"double","lower_bound":150,"name":"memoryRequest","upper_bound":300,"step":1},{"value_type":"double","lower_bound":1,"name":"cpuRequest","upper_bound":3,"step":0.01}],"direction":"xyz"}'
)

declare -A hpo_error_messages
hpo_error_messages=(
[empty-id]="Parameters cannot be empty or null"
Expand All @@ -141,6 +202,19 @@ hpo_error_messages=(
[invalid-searchspace]="Direction not supported"
)

declare -A hpo_grpc_error_messages
hpo_grpc_error_messages=(
[empty-id]="Parameters cannot be empty or null"
[no-id]="'experiment_id' is a required property"
[null-id]="Parameters cannot be empty or null"
[empty-name]="Parameters cannot be empty or null"
[no-name]="'experiment_name' is a required property"
[null-name]="Parameters cannot be empty or null"
[additional-field]="Additional properties are not allowed"
[valid-experiment]="Adding new experiment"
[invalid-searchspace]="Direction not supported"
)

declare -A hpo_post_exp_result_json=([empty-name]='{"experiment_name" : " ", "trial_number": 0, "trial_result": "success", "result_value_type": "double", "result_value": 98.78, "operation" : "EXP_TRIAL_RESULT"}'
[no-name]='{"trial_number": 0, "trial_result": "success", "result_value_type": "double", "result_value": 98.78, "operation" : "EXP_TRIAL_RESULT"}'
[null-name]='{"experiment_name" : null, "trial_number": 0, "trial_result": "success", "result_value_type": "double", "result_value": 98.78, "operation" : "EXP_TRIAL_RESULT"}'
Expand Down Expand Up @@ -202,3 +276,31 @@ hpo_exp_result_error_messages=(

)

declare -A hpo_grpc_exp_result_error_messages
hpo_grpc_exp_result_error_messages=(
[empty-name]="Parameters cannot be empty or null"
[no-name]="'experiment_name' is a required property"
[null-name]="Parameters cannot be empty or null"

[invalid-trial-number]="Requested trial exceeds the completed trial limit"
[no-trial-number]="'trial_number' is a required property"
[null-trial-number]="Parameters cannot be empty or null"

[invalid-trial-result]="Trial result status is invalid"
[empty-trial-result]="Trial result status is invalid"
[no-trial-result]="'trial_result' is a required property"
[null-trial-result]="Parameters cannot be empty or null"

[invalid-result-value-type]="Unsupported value type"
[empty-result-value-type]="Unsupported value type"
[no-result-value-type]="'result_value_type' is a required property"
[null-result-value-type]="Parameters cannot be empty or null"

[invalid-result-value]="result_value cannot be negative"
[no-result-value]="'result_value' is a required property"
[null-result-value]="Parameters cannot be empty or null"

[additional-field]="Additional properties are not allowed"
[valid-experiment-result]="Starting Experiment"

)
3 changes: 2 additions & 1 deletion tests/scripts/hpo/hpo_api_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ function hpo_api_tests() {
TESTS=0
((TOTAL_TEST_SUITES++))

hpo_api_tests=("hpo_post_experiment" "hpo_get_trial_json" "hpo_post_exp_result" "hpo_sanity_test" "hpo_grpc_sanity_test" "hpo_multiple_exp_test" "hpo_grpc_multiple_exp_test")
hpo_api_tests=("hpo_post_experiment" "hpo_grpc_post_experiment" "hpo_get_trial_json" "hpo_post_exp_result" "hpo_grpc_post_exp_result" "hpo_sanity_test" "hpo_grpc_sanity_test" "hpo_multiple_exp_test"
"hpo_grpc_multiple_exp_test")

# check if the test case is supported
if [ ! -z "${testcase}" ]; then
Expand Down
Loading