From 765ec5345060a01ace255df19d51e9af9728e448 Mon Sep 17 00:00:00 2001 From: Shashank Reddy Boyapally Date: Wed, 24 Jan 2024 11:14:20 -0500 Subject: [PATCH] orion cli, pylint workflow and README (#2) --- .github/workflows/pylint.yml | 23 +++++ .gitignore | 4 + README.md | 68 +++++++++++++++ orion.py | 161 +++++++++++++++++++++++++++++++++++ requirements.txt | 14 +++ setup.py | 25 ++++++ 6 files changed, 295 insertions(+) create mode 100644 .github/workflows/pylint.yml create mode 100644 README.md create mode 100644 orion.py create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..9ded563 --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,23 @@ +name: Pylint + +on: [push,pull_request] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + - name: Analysing the code with pylint + run: | + pylint -d C0103 $(git ls-files '*.py') \ No newline at end of file diff --git a/.gitignore b/.gitignore index 68bc17f..850de9f 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,7 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +*.yaml +*.csv +.vscode/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..b85a259 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# Orion - CLI tool to find regressions +Orion stands as a powerful command-line tool designed for identifying regressions within perf-scale CPT runs, leveraging metadata provided during the process. The detection mechanism relies on [hunter](https://github.com/datastax-labs/hunter). + +Below is an illustrative example of the config and metadata that Orion can handle: + +``` +tests : + - name : aws-small-scale-cluster-density-v2 + platform: AWS + masterNodesType: m6a.xlarge + masterNodesCount: 3 + workerNodesType: m6a.xlarge + workerNodesCount: 24 + benchmark: cluster-density-v2 + ocpVersion: 4.15 + networkType: OVNKubernetes + # encrypted: true + # fips: false + # ipsec: false + + metrics : + - metric : podReadyLatency + metricType : latency + + - metric : apiserverCPU + metricType : cpu + namespace: openshift-kube-apiserver + + - metric: ovnCPU + metricType: cpu + namespace: openshift-ovn-kubernetes + + - metric: etcdCPU + metricType: cpu + namespace: openshift-ovn-kubernetes + + +``` + +## Build Orion +Building Orion is a straightforward process. Follow these commands: + +Clone the current repository using git clone. + +``` +>> git clone +>> pip install venv +>> source venv/bin/activate +>> pip install -r requirements.txt +>> export ES_SERVER = +>> pip install . +``` +## Run Orion +Executing Orion is as simple as building it. After following the build steps, run the following: +``` +>> orion +``` +At the moment, + +Orion provides flexibility in configuring its behavior by allowing users to set the path to their config file using the ```--config``` flag. + +For enhanced troubleshooting and debugging, Orion supports the ```--debug``` flag, enabling the generation of detailed debug logs. + +Additionally, users can specify a custom path for the output CSV file using the ```--output``` flag, providing control over the location where the generated CSV will be stored. + +Orion's seamless integration with metadata and hunter ensures a robust regression detection tool for perf-scale CPT runs. + + diff --git a/orion.py b/orion.py new file mode 100644 index 0000000..f40b19a --- /dev/null +++ b/orion.py @@ -0,0 +1,161 @@ +""" +This is the cli file for orion, tool to detect regressions using hunter +""" +# pylint: disable = import-error +import sys +from functools import reduce +import logging +import os + +import click +import yaml +import pandas as pd +from fmatch.matcher import Matcher + + +@click.group() +def cli(): + """ + cli function to group commands + """ + + +# pylint: disable=too-many-locals +@click.command() +@click.option("--config", default="config.yaml", help="Path to the configuration file") +@click.option("--output", default="output.csv", help="Path to save the output csv file") +@click.option("--debug", is_flag=True, help="log level ") +def orion(config, debug, output): + """Orion is the cli tool to detect regressions over the runs + + Args: + config (str): path to the config file + debug (bool): lets you log debug mode + output (str): path to the output csv file + """ + level = logging.DEBUG if debug else logging.INFO + logger = logging.getLogger("Orion") + logger = set_logging(level, logger) + + if "ES_SERVER" not in os.environ: + logger.error("ES_SERVER environment variable not set") + sys.exit(1) + + try: + with open(config, "r", encoding="utf-8") as file: + data = yaml.safe_load(file) + logger.debug("The %s file has successfully loaded", config) + except FileNotFoundError as e: + logger.error("Config file not found: %s", e) + sys.exit(1) + except Exception as e: # pylint: disable=broad-exception-caught + logger.error("An error occurred: %s", e) + sys.exit(1) + for test in data["tests"]: + metadata = get_metadata(test) + logger.info("The test %s has started", test["name"]) + match = Matcher(index="perf_scale_ci", level=level) + uuids = match.get_uuid_by_metadata(metadata) + if len(uuids) == 0: + print("No UUID present for given metadata") + sys.exit() + + runs = match.match_kube_burner(uuids) + ids = match.filter_runs(runs, runs) + metrics = test["metrics"] + dataframe_list = [] + + for metric in metrics: + logger.info("Collecting %s", metric["metric"]) + if metric["metricType"] == "latency": + if metric["metric"] == "podReadyLatency": + try: + podl = match.burner_results("", ids, "ripsaw-kube-burner*") + podl_df = match.convert_to_df( + podl, columns=["uuid", "timestamp", "P99"] + ) + dataframe_list.append(podl_df) + logger.debug(podl_df) + except Exception as e: # pylint: disable=broad-exception-caught + logger.error( + "The namespace %s does not exist, exception %s", + metric["namespace"], + e, + ) + + elif metric["metricType"] == "cpu": + try: + cpu = match.burner_cpu_results( + ids, metric["namespace"], "ripsaw-kube-burner*" + ) + cpu_df = match.convert_to_df(cpu, columns=["uuid", "cpu_avg"]) + cpu_df = cpu_df.rename( + columns={"cpu_avg": metric["metric"] + "_cpu_avg"} + ) + dataframe_list.append(cpu_df) + logger.debug(cpu_df) + except Exception as e: # pylint: disable=broad-exception-caught + logger.error( + "The namespace %s does not exist, exception %s", + metric["namespace"], + e, + ) + + merged_df = reduce( + lambda left, right: pd.merge(left, right, on="uuid", how="inner"), + dataframe_list, + ) + match.save_results(merged_df, csv_file_path=output) + + +def get_metadata(test): + """Gets metadata of the run from each test + + Args: + test (dict): test dictionary + + Returns: + dict: dictionary of the metadata + """ + metadata_columns = [ + "platform", + "masterNodesType", + "masterNodesCount", + "workerNodesType", + "workerNodesCount", + "benchmark", + "ocpVersion", + "networkType", + "encrypted", + "fips", + "ipsec", + ] + metadata = {key: test[key] for key in metadata_columns if key in test} + metadata["ocpVersion"] = str(metadata["ocpVersion"]) + return metadata + + +def set_logging(level, logger): + """sets log level and format + + Args: + level (_type_): level of the log + logger (_type_): logger object + + Returns: + logging.Logger: a formatted and level set logger + """ + logger.setLevel(level) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(level) + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + +if __name__ == "__main__": + cli.add_command(orion) + cli() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b98bd22 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +certifi==2023.11.17 +click==8.1.7 +elastic-transport==8.11.0 +elasticsearch==8.11.1 +elasticsearch7==7.13.0 +fmatch==0.0.2 +numpy==1.26.3 +pandas==2.1.4 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +six==1.16.0 +tzdata==2023.4 +urllib3==1.26.18 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..be07410 --- /dev/null +++ b/setup.py @@ -0,0 +1,25 @@ +# orion/setup.py +""" +setup.py for orion cli tool +""" +from setuptools import setup + +setup( + name='orion', + version='1.0', + py_modules=['orion'], + install_requires=[ + 'click', + 'fmatch' + ], + entry_points={ + 'console_scripts': [ + 'orion = orion:orion', + ], + }, + classifiers=[ + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + ], +)