From fcf08216960ca6b5a24594a32ee10fbe62f8ba8b Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Tue, 2 Jul 2024 16:55:52 +0200 Subject: [PATCH 1/2] Adds ocm jobs to cpt home page Signed-off-by: Vicente Zepeda Mas --- backend/app/api/api.py | 4 ++ backend/app/api/v1/commons/ocm.py | 51 ++++++++++++++++++ backend/app/api/v1/endpoints/cpt/cptJobs.py | 5 ++ backend/app/api/v1/endpoints/cpt/maps/ocm.py | 18 +++++++ backend/app/api/v1/endpoints/ocm/ocmJobs.py | 55 ++++++++++++++++++++ 5 files changed, 133 insertions(+) create mode 100644 backend/app/api/v1/commons/ocm.py create mode 100644 backend/app/api/v1/endpoints/cpt/maps/ocm.py create mode 100644 backend/app/api/v1/endpoints/ocm/ocmJobs.py diff --git a/backend/app/api/api.py b/backend/app/api/api.py index 562f43da..b74b8ad4 100644 --- a/backend/app/api/api.py +++ b/backend/app/api/api.py @@ -10,6 +10,7 @@ from app.api.v1.endpoints.quay import quayGraphs from app.api.v1.endpoints.telco import telcoJobs from app.api.v1.endpoints.telco import telcoGraphs +from app.api.v1.endpoints.ocm import ocmJobs router = APIRouter() @@ -35,3 +36,6 @@ # Horreum endpoint router.include_router(horreum.router, tags=['horreum']) + +# OCM endpoint +router.include_router(ocmJobs.router, tags=['ocm']) diff --git a/backend/app/api/v1/commons/ocm.py b/backend/app/api/v1/commons/ocm.py new file mode 100644 index 00000000..680628d8 --- /dev/null +++ b/backend/app/api/v1/commons/ocm.py @@ -0,0 +1,51 @@ +from datetime import date, datetime +import pandas as pd +from app.services.search import ElasticService + + +async def getData(start_datetime: date, end_datetime: date, configpath: str): + query = { + "query": { + "bool": { + "filter": { + "range": { + "metrics.earliest": { + "format": "yyyy-MM-dd" + } + } + } + } + } + } + + es = ElasticService(configpath=configpath) + response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='metrics.earliest') + await es.close() + tasks = [item['_source'] for item in response] + jobs = pd.json_normalize(tasks) + if len(jobs) == 0: + return jobs + + if 'buildUrl' not in jobs.columns: + jobs.insert(len(jobs.columns), "buildUrl", "") + jobs.fillna('', inplace=True) + jobs['ciSystem'] = jobs.apply(fillCiSystem, axis=1) + jobs['jobStatus'] = jobs.apply(convertJobStatus, axis=1) + return jobs + + +def fillCiSystem(row): + currDate = datetime.strptime(row["metrics.earliest"][:26], '%Y-%m-%dT%H:%M:%S.%f') + if currDate > datetime(2024, 6, 24): + return "Jenkins" + else: + return "Airflow" + + +def convertJobStatus(row): + if row["metrics.success"] >= 0.80: + return "success" + elif row["metrics.success"] < 0.40: + return "failure" + else: + return "unstable" diff --git a/backend/app/api/v1/endpoints/cpt/cptJobs.py b/backend/app/api/v1/endpoints/cpt/cptJobs.py index 8b8dc1bb..fab46138 100644 --- a/backend/app/api/v1/endpoints/cpt/cptJobs.py +++ b/backend/app/api/v1/endpoints/cpt/cptJobs.py @@ -10,6 +10,7 @@ from .maps.quay import quayMapper from .maps.hce import hceMapper from .maps.telco import telcoMapper +from .maps.ocm import ocmMapper from ...commons.example_responses import cpt_200_response, response_422 from fastapi.param_functions import Query @@ -20,8 +21,10 @@ "quay": quayMapper, "hce": hceMapper, "telco": telcoMapper, + "ocm": ocmMapper, } + @router.get('/api/v1/cpt/jobs', summary="Returns a job list from all the products.", description="Returns a list of jobs in the specified dates. \ @@ -69,6 +72,7 @@ async def jobs(start_date: date = Query(None, description="Start date for search jsonstring = json.dumps(response) return jsonstring + async def fetch_product_async(product, start_date, end_date): try: df = await products[product](start_date, end_date) @@ -79,5 +83,6 @@ async def fetch_product_async(product, start_date, end_date): print(f"Error in mapper for product {product}: {e}") return pd.DataFrame() + def fetch_product(product, start_date, end_date): return asyncio.run(fetch_product_async(product, start_date, end_date)) \ No newline at end of file diff --git a/backend/app/api/v1/endpoints/cpt/maps/ocm.py b/backend/app/api/v1/endpoints/cpt/maps/ocm.py new file mode 100644 index 00000000..f53bd13e --- /dev/null +++ b/backend/app/api/v1/endpoints/cpt/maps/ocm.py @@ -0,0 +1,18 @@ +from ....commons.ocm import getData +from datetime import date + + +################################################################ +# This will return a DataFrame from OCM required by the CPT endpoint +################################################################ +async def ocmMapper(start_datetime: date, end_datetime: date): + df = await getData(start_datetime, end_datetime, f'ocm.elasticsearch') + if len(df) == 0: + return df + df.insert(len(df.columns), "product", "ocm") + df.insert(len(df.columns), "releaseStream", "Nightly") + df["testName"] = df["attack"] + df["startDate"] = df["metrics.earliest"] + df["endDate"] = df["metrics.end"] + + return df diff --git a/backend/app/api/v1/endpoints/ocm/ocmJobs.py b/backend/app/api/v1/endpoints/ocm/ocmJobs.py new file mode 100644 index 00000000..bc76b7eb --- /dev/null +++ b/backend/app/api/v1/endpoints/ocm/ocmJobs.py @@ -0,0 +1,55 @@ +import json +from fastapi import Response +from datetime import datetime, timedelta, date +from fastapi import APIRouter +from ...commons.ocm import getData +from ...commons.example_responses import ocp_200_response, response_422 +from fastapi.param_functions import Query + +router = APIRouter() + + +@router.get('/api/v1/ocm/jobs', + summary="Returns a job list", + description="Returns a list of jobs in the specified dates. \ + If not dates are provided the API will default the values. \ + `startDate`: will be set to the day of the request minus 5 days.\ + `endDate`: will be set to the day of the request.", + responses={ + 200: ocp_200_response(), + 422: response_422(), + },) +async def jobs(start_date: date = Query(None, description="Start date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-10"]), + end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]), + pretty: bool = Query(False, description="Output contet in pretty format.")): + if start_date is None: + start_date = datetime.utcnow().date() + start_date = start_date - timedelta(days=5) + + if end_date is None: + end_date = datetime.utcnow().date() + + if start_date > end_date: + return Response(content=json.dumps({'error': "invalid date format, start_date must be less than end_date"}), status_code=422) + + results = await getData(start_date, end_date, 'ocm.elasticsearch') + + if len(results) >= 1: + response = { + 'startDate': start_date.__str__(), + 'endDate': end_date.__str__(), + 'results': results.to_dict('records') + } + else: + response = { + 'startDate': start_date.__str__(), + 'endDate': end_date.__str__(), + 'results': [] + } + + if pretty: + json_str = json.dumps(response, indent=4) + return Response(content=json_str, media_type='application/json') + + jsonstring = json.dumps(response) + return jsonstring From 1fdad464cecdeabe0b38e0c9802d0f0105ed5445 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Fri, 5 Jul 2024 10:03:51 +0200 Subject: [PATCH 2/2] Added validation in case of missing ciSystem col Signed-off-by: Vicente Zepeda Mas --- backend/app/api/v1/commons/ocm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/app/api/v1/commons/ocm.py b/backend/app/api/v1/commons/ocm.py index 680628d8..25870789 100644 --- a/backend/app/api/v1/commons/ocm.py +++ b/backend/app/api/v1/commons/ocm.py @@ -28,8 +28,9 @@ async def getData(start_datetime: date, end_datetime: date, configpath: str): if 'buildUrl' not in jobs.columns: jobs.insert(len(jobs.columns), "buildUrl", "") + if 'ciSystem' not in jobs.columns: + jobs.insert(len(jobs.columns), "ciSystem", "") jobs.fillna('', inplace=True) - jobs['ciSystem'] = jobs.apply(fillCiSystem, axis=1) jobs['jobStatus'] = jobs.apply(convertJobStatus, axis=1) return jobs