diff --git a/.gitignore b/.gitignore index 9b495d8..393383a 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ cython_debug/ .DS_Store fmatch/main.py +*.csv +*.yaml diff --git a/fmatch/__init__.py b/fmatch/__init__.py index e69de29..4e0b627 100644 --- a/fmatch/__init__.py +++ b/fmatch/__init__.py @@ -0,0 +1,3 @@ +"""This file is the init file of the matcher package +which is used as to match and query data from elastic server""" +from fmatch.matcher import Matcher diff --git a/fmatch/matcher.py b/fmatch/matcher.py index e71e51f..955aa27 100644 --- a/fmatch/matcher.py +++ b/fmatch/matcher.py @@ -7,17 +7,22 @@ from elasticsearch.exceptions import NotFoundError # pylint: disable=import-error import pandas as pd +import yaml + + ES_URL = os.getenv("ES_SERVER") class Matcher: """ Matcher """ - def __init__(self, index="perf_scale_ci"): + def __init__(self, configpath="config.yaml", index="perf_scale_ci"): self.index = index self.es_url = ES_URL + with open(configpath, 'r',encoding='UTF-8') as file: + data = yaml.safe_load(file) self.es = Elasticsearch([self.es_url], http_auth=[ - "username", "password"]) + data['username'], data['password']], timeout=30) self.data = None def get_metadata_by_uuid(self, uuid, index=None): @@ -229,9 +234,9 @@ def convert_to_df(self, data, columns=None): _type_: _description_ """ odf = pd.json_normalize(data) + odf = odf.sort_values(by=['timestamp']) if columns is not None: odf = pd.DataFrame(odf, columns=columns) - odf = odf.sort_values(by=['timestamp']) return odf def save_results(self, df, csv_file_path="output.csv", columns=None):