Merge pull request kruize#1231 from shreyabiradar07/metric_profile_tests

Add Create and List Metric Profile tests

tests/scripts/helpers/kruize.py

@@ -364,3 +364,75 @@ def list_metadata(datasource=None, cluster_name=None, namespace=None, verbose=No
         print(response.text)
         print("\n************************************************************")
     return response
+
+
+# Description: This function creates a metric profile using the Kruize createMetricProfile API
+# Input Parameters: metric profile json
+def create_metric_profile(metric_profile_json_file):
+    json_file = open(metric_profile_json_file, "r")
+    metric_profile_json = json.loads(json_file.read())
+
+    print("\nCreating metric profile...")
+    url = URL + "/createMetricProfile"
+    print("URL = ", url)
+
+    response = requests.post(url, json=metric_profile_json)
+    print("Response status code = ", response.status_code)
+    print(response.text)
+    return response
+
+# Description: This function deletes the metric profile
+# Input Parameters: metric profile input json
+def delete_metric_profile(input_json_file, invalid_header=False):
+    json_file = open(input_json_file, "r")
+    input_json = json.loads(json_file.read())
+
+    print("\nDeleting the metric profile...")
+    url = URL + "/deleteMetricProfile"
+
+    metric_profile_name = input_json['metadata']['name']
+    query_string = f"name={metric_profile_name}"
+
+    if query_string:
+        url += "?" + query_string
+    print("URL = ", url)
+
+    headers = {'content-type': 'application/xml'}
+    if invalid_header:
+        print("Invalid header")
+        response = requests.delete(url, headers=headers)
+    else:
+        response = requests.delete(url)
+
+    print(response)
+    print("Response status code = ", response.status_code)
+    return response
+
+
+# Description: This function lists the metric profile from Kruize Autotune using GET listMetricProfiles API
+# Input Parameters: metric profile name and verbose - flag indicating granularity of data to be listed
+def list_metric_profiles(name=None, verbose=None, logging=True):
+    print("\nListing the metric profiles...")
+
+    query_params = {}
+
+    if name is not None:
+        query_params['name'] = name
+    if verbose is not None:
+        query_params['verbose'] = verbose
+
+    query_string = "&".join(f"{key}={value}" for key, value in query_params.items())
+
+    url = URL + "/listMetricProfiles"
+    if query_string:
+        url += "?" + query_string
+    print("URL = ", url)
+    print("PARAMS = ", query_params)
+    response = requests.get(url)
+
+    print("Response status code = ", response.status_code)
+    if logging:
+        print("\n************************************************************")
+        print(response.text)
+        print("\n************************************************************")
+    return response

tests/scripts/helpers/list_metric_profiles_schema.py

@@ -0,0 +1,109 @@
+"""
+Copyright (c) 2024, 2024 Red Hat, IBM Corporation and others.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+list_metric_profiles_schema = {
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "apiVersion": {
+                "type": "string"
+            },
+            "kind": {
+                "type": "string"
+            },
+            "metadata": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    }
+                },
+                "required": ["name"]
+            },
+            "profile_version": {
+                "type": "number"
+            },
+            "k8s_type": {
+                "type": "string"
+            },
+            "slo": {
+                "type": "object",
+                "properties": {
+                    "sloClass": {
+                        "type": "string"
+                    },
+                    "objective_function": {
+                        "type": "object",
+                        "properties": {
+                            "function_type": {
+                                "type": "string"
+                            }
+                        },
+                        "required": ["function_type"]
+                    },
+                    "direction": {
+                        "type": "string"
+                    },
+                    "function_variables": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": {
+                                    "type": "string"
+                                },
+                                "datasource": {
+                                    "type": "string"
+                                },
+                                "value_type": {
+                                    "type": "string"
+                                },
+                                "kubernetes_object": {
+                                    "type": "string"
+                                },
+                                "aggregation_functions": {
+                                    "type": "object",
+                                    "items": {
+                                        "type": "object",
+                                        "patternProperties": {
+                                            "^[a-zA-Z0-9_-]+$": {
+                                                "type": "object",
+                                                "properties": {
+                                                    "function": {
+                                                        "type": "string",
+                                                        "pattern": "^[a-zA-Z0-9_-]+$"
+                                                    },
+                                                    "query": {
+                                                        "type": "string"
+                                                    }
+                                                },
+                                                "required": ["function", "query"]
+                                            },
+                                        },
+                                    }
+                                },
+                            }
+                        },
+                        "required": ["name", "datasource", "value_type", "kubernetes_object", "aggregation_functions"]
+                    }
+                },
+                "required": ["sloClass", "objective_function", "direction", "function_variables"]
+            }
+        }
+    },
+    "required": ["apiVersion", "kind", "metadata", "profile_version", "k8s_type", "slo"]
+}

tests/scripts/helpers/list_metric_profiles_validate.py

@@ -0,0 +1,121 @@
+"""
+Copyright (c) 2024, 2024 Red Hat, IBM Corporation and others.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import jsonschema
+from jsonschema import FormatChecker
+from jsonschema.exceptions import ValidationError
+from helpers.list_metric_profiles_schema import list_metric_profiles_schema
+
+
+SLO_CLASSES_SUPPORTED = ("throughput", "response_time", "resource_usage")
+SLO_CLASSES_NOT_SUPPORTED = "SLO class not supported!"
+
+DIRECTIONS_SUPPORTED = ("minimize", "maximize")
+DIRECTIONS_NOT_SUPPORTED = "Directions not supported!"
+
+VALUE_TYPES_SUPPORTED = ("double", "int", "string", "categorical")
+VALUE_TYPE_NOT_SUPPORTED = "Value type not supported!"
+
+KUBERNETES_OBJECTS_TYPE_SUPPORTED = ("deployment", "pod", "container", "namespace")
+KUBERNETES_OBJECTS_TYPE_NOT_SUPPORTED = "Kubernetes objects type not supported!"
+
+FUNCTION_TYPES_SUPPORTED = ("sum", "avg", "min", "max")
+FUNCTION_TYPE_NOT_SUPPORTED = "Aggregation function type not supported!"
+
+JSON_NULL_VALUES = ("is not of type 'string'", "is not of type 'integer'", "is not of type 'number'")
+VALUE_MISSING = " cannot be empty or null!"
+
+
+def validate_list_metric_profiles_json(list_metric_profiles_json, json_schema):
+    errorMsg = ""
+    try:
+        # create a validator with the format checker
+        print("Validating json against the json schema...")
+        validator = jsonschema.Draft7Validator(json_schema, format_checker=FormatChecker())
+
+        # validate the JSON data against the schema
+        errors = ""
+        errors = list(validator.iter_errors(list_metric_profiles_json))
+        print("Validating json against the json schema...done")
+        errorMsg = validate_list_metric_profiles_json_values(list_metric_profiles_json)
+
+        if errors:
+            custom_err = ValidationError(errorMsg)
+            errors.append(custom_err)
+            return errors
+        else:
+            return errorMsg
+    except ValidationError as err:
+        print("Received a VaidationError")
+
+        # Check if the exception is due to empty or null required parameters and prepare the response accordingly
+        if any(word in err.message for word in JSON_NULL_VALUES):
+            errorMsg = "Parameters" + VALUE_MISSING
+            return errorMsg
+        # Modify the error response in case of additional properties error
+        elif str(err.message).__contains__('('):
+            errorMsg = str(err.message).split('(')
+            return errorMsg[0]
+        else:
+            return err.message
+
+def validate_list_metric_profiles_json_values(metric_profile):
+    validationErrorMsg = ""
+    slo = "slo"
+    func_var = "function_variables"
+    aggr_func = "aggregation_functions"
+
+    for key in metric_profile[0].keys():
+
+        # Check if any of the key is empty or null
+        if not (str(metric_profile[0][key]) and str(metric_profile[0][key]).strip()):
+            validationErrorMsg = ",".join([validationErrorMsg, "Parameters" + VALUE_MISSING])
+
+        if slo == key:
+            for subkey in metric_profile[0][key].keys():
+                if not (str(metric_profile[0][key][subkey]) and str(metric_profile[0][key][subkey]).strip()):
+                    print(f"FAILED - {str(metric_profile[0][key][subkey])} is empty or null")
+                    validationErrorMsg = ",".join([validationErrorMsg, "Parameters" + VALUE_MISSING])
+                elif str(subkey) == "sloClass" and (str(metric_profile[0][key][subkey]) not in SLO_CLASSES_SUPPORTED):
+                    validationErrorMsg = ",".join([validationErrorMsg, SLO_CLASSES_NOT_SUPPORTED])
+                elif str(subkey) == "direction" and (str(metric_profile[0][key][subkey]) not in DIRECTIONS_SUPPORTED):
+                    validationErrorMsg = ",".join([validationErrorMsg, DIRECTIONS_NOT_SUPPORTED])
+
+                if func_var == subkey:
+                    for func_var_object in metric_profile[0][key][subkey]:
+                        for field in func_var_object.keys():
+                            # Check if any of the key is empty or null
+                            if not (str(func_var_object.get(field)) and str(func_var_object.get(field)).strip()):
+                                print(f"FAILED - {str(func_var_object.get(field))} is empty or null")
+                                validationErrorMsg = ",".join([validationErrorMsg, "Parameters" + VALUE_MISSING])
+                            elif str(field) == "value_type" and str(func_var_object.get(field)) not in VALUE_TYPES_SUPPORTED:
+                                validationErrorMsg = ",".join([validationErrorMsg, VALUE_TYPE_NOT_SUPPORTED])
+                            elif str(field) == "kubernetes_object" and str(func_var_object.get(field)) not in KUBERNETES_OBJECTS_TYPE_SUPPORTED:
+                                validationErrorMsg = ",".join([validationErrorMsg, KUBERNETES_OBJECTS_TYPE_NOT_SUPPORTED])
+
+                            if aggr_func == field:
+                                aggr_func_obj = func_var_object.get("aggregation_functions", {})
+                                for aggr_func_object, aggr_func_value in aggr_func_obj.items():
+                                    for query in aggr_func_value.keys():
+                                        # Check if any of the key is empty or null
+                                        if not (str(aggr_func_value.get(query)) and str(aggr_func_value.get(query)).strip()):
+                                            print(f"FAILED - {str(aggr_func_value.get(query))} is empty or null")
+                                            validationErrorMsg = ",".join([validationErrorMsg, "Parameters" + VALUE_MISSING])
+                                        elif str(query) == "function" and str(aggr_func_value.get(query)) not in FUNCTION_TYPES_SUPPORTED:
+                                            validationErrorMsg = ",".join([validationErrorMsg, FUNCTION_TYPE_NOT_SUPPORTED])
+
+    return validationErrorMsg.lstrip(',')

tests/scripts/helpers/list_metric_profiles_without_parameters_schema.py

@@ -0,0 +1,28 @@
+"""
+Copyright (c) 2024, 2024 Red Hat, IBM Corporation and others.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+list_metric_profiles_without_parameters_schema = {
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "name": {
+                "type": "string"
+            }
+        },
+        "required": ["name"]
+    }
+}

tests/scripts/helpers/utils.py

@@ -28,6 +28,7 @@
 ERROR_STATUS_CODE = 400
 ERROR_409_STATUS_CODE = 409
 DUPLICATE_RECORDS_COUNT = 5
+ERROR_500_STATUS_CODE = 500
 
 SUCCESS_STATUS = "SUCCESS"
 ERROR_STATUS = "ERROR"
@@ -57,6 +58,13 @@
 LIST_METADATA_DATASOURCE_NAME_CLUSTER_NAME_ERROR_MSG = "Metadata for a given datasource name - %s, cluster_name - %s either does not exist or is not valid"
 LIST_METADATA_MISSING_DATASOURCE = "datasource is mandatory"
 IMPORT_METADATA_DATASOURCE_CONNECTION_FAILURE_MSG = "Metadata cannot be imported, datasource connection refused or timed out"
+CREATE_METRIC_PROFILE_SUCCESS_MSG = "Metric Profile : %s created successfully. View Metric Profiles at /listMetricProfiles"
+METRIC_PROFILE_EXISTS_MSG = "Validation failed: Metric Profile already exists: %s"
+METRIC_PROFILE_NOT_FOUND_MSG = "No metric profiles found!"
+INVALID_LIST_METRIC_PROFILE_INPUT_QUERY = "The query param(s) - [%s] is/are invalid"
+LIST_METRIC_PROFILES_INVALID_NAME = "Given metric profile name - %s is not valid"
+CREATE_METRIC_PROFILE_MISSING_MANDATORY_FIELD_MSG = "Validation failed: JSONObject[\"%s\"] not found."
+CREATE_METRIC_PROFILE_MISSING_MANDATORY_PARAMETERS_MSG = "Validation failed: Missing mandatory parameters: [%s] "
 
 # Kruize Recommendations Notification codes
 NOTIFICATION_CODE_FOR_RECOMMENDATIONS_AVAILABLE = "111000"

tests/scripts/local_monitoring_tests/Local_monitoring_tests.md

@@ -48,9 +48,30 @@ Here are the test scenarios:
 - List dsmetadata with datasource and namespace but without cluster_name
 - List the dsmetadata after deleting imported metadata
 
+### **Create Metric Profile API tests**
+
+Here are the test scenarios:
+
+- Create metric profile passing a valid input JSON payload with all the metric queries 
+- Post the same metric profile again - creating it twice and validate the error as metric profile name is a unique field 
+- Create multiple valid metric profiles using different jsons
+- Create Metric profile missing mandatory fields and validate error messages when the mandatory fields are missing
+
+
+### **List Metric Profile API tests**
+
+Here are the test scenarios:
+
+- List metric profiles without specifying any query parameters
+- List metric profiles specifying profile name query parameter
+- List metric profiles specifying verbose query parameter
+- List metric profiles specifying profile name and verbose query parameters
+- Test with invalid values such as blank, null or an invalid value for name query parameter in listMetricProfiles API
+- List metric profiles without creating metric profile
+
 The above tests are developed using pytest framework and the tests are run using shell script wrapper that does the following:
 - Deploys kruize in non-CRD mode using the [deploy script](https://github.com/kruize/autotune/blob/master/deploy.sh) from the autotune repo
-- Creates a resource optimization performance profile using the [createPerformanceProfile API](/design/PerformanceProfileAPI.md)
+- Creates a resource optimization metric profile using the [createMetricProfile API](/design/MetricProfileAPI.md)
 - Runs the above tests using pytest
 
 ## Prerequisites for running the tests: