Merge pull request #105 from chentex/add-ocm-jobs

Adds ocm jobs to cpt home page

backend/app/api/api.py

@@ -10,6 +10,7 @@
 from app.api.v1.endpoints.quay import quayGraphs
 from app.api.v1.endpoints.telco import telcoJobs
 from app.api.v1.endpoints.telco import telcoGraphs
+from app.api.v1.endpoints.ocm import ocmJobs
 
 
 router = APIRouter()
@@ -35,3 +36,6 @@
 
 # Horreum endpoint
 router.include_router(horreum.router, tags=['horreum'])
+
+# OCM endpoint
+router.include_router(ocmJobs.router, tags=['ocm'])

backend/app/api/v1/commons/ocm.py

@@ -0,0 +1,52 @@
+from datetime import date, datetime
+import pandas as pd
+from app.services.search import ElasticService
+
+
+async def getData(start_datetime: date, end_datetime: date, configpath: str):
+    query = {
+        "query": {
+            "bool": {
+                "filter": {
+                    "range": {
+                        "metrics.earliest": {
+                            "format": "yyyy-MM-dd"
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    es = ElasticService(configpath=configpath)
+    response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='metrics.earliest')
+    await es.close()
+    tasks = [item['_source'] for item in response]
+    jobs = pd.json_normalize(tasks)
+    if len(jobs) == 0:
+        return jobs
+
+    if 'buildUrl' not in jobs.columns:
+        jobs.insert(len(jobs.columns), "buildUrl", "")
+    if 'ciSystem' not in jobs.columns:
+        jobs.insert(len(jobs.columns), "ciSystem", "")
+    jobs.fillna('', inplace=True)
+    jobs['jobStatus'] = jobs.apply(convertJobStatus, axis=1)
+    return jobs
+
+
+def fillCiSystem(row):
+    currDate = datetime.strptime(row["metrics.earliest"][:26], '%Y-%m-%dT%H:%M:%S.%f')
+    if currDate > datetime(2024, 6, 24):
+        return "Jenkins"
+    else:
+        return "Airflow"
+
+
+def convertJobStatus(row):
+    if row["metrics.success"] >= 0.80:
+        return "success"
+    elif row["metrics.success"] < 0.40:
+        return "failure"
+    else:
+        return "unstable"

backend/app/api/v1/endpoints/cpt/cptJobs.py

@@ -10,6 +10,7 @@
 from .maps.quay import quayMapper
 from .maps.hce import hceMapper
 from .maps.telco import telcoMapper
+from .maps.ocm import ocmMapper
 from ...commons.example_responses import cpt_200_response, response_422
 from fastapi.param_functions import Query
 
@@ -20,8 +21,10 @@
             "quay": quayMapper,
             "hce": hceMapper,
             "telco": telcoMapper,
+            "ocm": ocmMapper,
            }
 
+
 @router.get('/api/v1/cpt/jobs',
             summary="Returns a job list from all the products.",
             description="Returns a list of jobs in the specified dates. \
@@ -69,6 +72,7 @@ async def jobs(start_date: date = Query(None, description="Start date for search
     jsonstring = json.dumps(response)
     return jsonstring
 
+
 async def fetch_product_async(product, start_date, end_date):
     try:
         df = await products[product](start_date, end_date)
@@ -79,5 +83,6 @@ async def fetch_product_async(product, start_date, end_date):
         print(f"Error in mapper for product {product}: {e}")
         return pd.DataFrame()
 
+
 def fetch_product(product, start_date, end_date):
     return asyncio.run(fetch_product_async(product, start_date, end_date))

backend/app/api/v1/endpoints/cpt/maps/ocm.py

@@ -0,0 +1,18 @@
+from ....commons.ocm import getData
+from datetime import date
+
+
+################################################################
+# This will return a DataFrame from OCM required by the CPT endpoint
+################################################################
+async def ocmMapper(start_datetime: date, end_datetime: date):
+    df = await getData(start_datetime, end_datetime, f'ocm.elasticsearch')
+    if len(df) == 0:
+        return df
+    df.insert(len(df.columns), "product", "ocm")
+    df.insert(len(df.columns), "releaseStream", "Nightly")
+    df["testName"] = df["attack"]
+    df["startDate"] = df["metrics.earliest"]
+    df["endDate"] = df["metrics.end"]
+
+    return df

backend/app/api/v1/endpoints/ocm/ocmJobs.py

@@ -0,0 +1,55 @@
+import json
+from fastapi import Response
+from datetime import datetime, timedelta, date
+from fastapi import APIRouter
+from ...commons.ocm import getData
+from ...commons.example_responses import ocp_200_response, response_422
+from fastapi.param_functions import Query
+
+router = APIRouter()
+
+
+@router.get('/api/v1/ocm/jobs',
+            summary="Returns a job list",
+            description="Returns a list of jobs in the specified dates. \
+            If not dates are provided the API will default the values. \
+            `startDate`: will be set to the day of the request minus 5 days.\
+            `endDate`: will be set to the day of the request.",
+            responses={
+                200: ocp_200_response(),
+                422: response_422(),
+            },)
+async def jobs(start_date: date = Query(None, description="Start date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-10"]),
+               end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]),
+               pretty: bool = Query(False, description="Output contet in pretty format.")):
+    if start_date is None:
+        start_date = datetime.utcnow().date()
+        start_date = start_date - timedelta(days=5)
+
+    if end_date is None:
+        end_date = datetime.utcnow().date()
+
+    if start_date > end_date:
+        return Response(content=json.dumps({'error': "invalid date format, start_date must be less than end_date"}), status_code=422)
+
+    results = await getData(start_date, end_date, 'ocm.elasticsearch')
+
+    if len(results) >= 1:
+        response = {
+            'startDate': start_date.__str__(),
+            'endDate': end_date.__str__(),
+            'results': results.to_dict('records')
+        }
+    else:
+        response = {
+            'startDate': start_date.__str__(),
+            'endDate': end_date.__str__(),
+            'results': []
+        }
+
+    if pretty:
+        json_str = json.dumps(response, indent=4)
+        return Response(content=json_str, media_type='application/json')
+
+    jsonstring = json.dumps(response)
+    return jsonstring