From 2b7c5695cd9e30a49a18db06e1f6a322cbc1c8d7 Mon Sep 17 00:00:00 2001 From: Tullio Sebastiani Date: Thu, 3 Oct 2024 19:54:37 +0200 Subject: [PATCH] litmus and cluster events deprecation (#126) linting Signed-off-by: Tullio Sebastiani --- src/krkn_lib/k8s/krkn_kubernetes.py | 164 ------------------ src/krkn_lib/models/k8s/models.py | 76 -------- .../k8s/krkn_telemetry_kubernetes.py | 102 ----------- src/krkn_lib/tests/test_krkn_kubernetes.py | 46 ----- .../tests/test_krkn_telemetry_kubernetes.py | 49 ------ 5 files changed, 437 deletions(-) diff --git a/src/krkn_lib/k8s/krkn_kubernetes.py b/src/krkn_lib/k8s/krkn_kubernetes.py index f3a025a9..36223755 100644 --- a/src/krkn_lib/k8s/krkn_kubernetes.py +++ b/src/krkn_lib/k8s/krkn_kubernetes.py @@ -12,10 +12,8 @@ from functools import partial from queue import Queue from typing import Any, Dict, List, Optional -from krkn_lib.version import __version__ import arcaflow_lib_kubernetes -import deprecation import kubernetes import urllib3 import yaml @@ -31,10 +29,7 @@ PVC, AffectedPod, ApiRequestException, - ChaosEngine, - ChaosResult, Container, - LitmusChaosObject, Pod, PodsMonitorThread, PodsStatus, @@ -1575,82 +1570,6 @@ def get_pod_info(self, name: str, namespace: str = "default") -> Pod: ) return None - @deprecation.deprecated( - deprecated_in="3.1.0", - removed_in="3.2.0", - current_version=__version__, - details="litmus support dropped from krkn", - ) - def get_litmus_chaos_object( - self, kind: str, name: str, namespace: str = "default" - ) -> LitmusChaosObject: - """ - Retrieves Litmus Chaos CRDs - - :param kind: the custom resource type - :param name: the object name - :param namespace: the namespace (optional default `default`) - :return: data class object of a subclass of LitmusChaosObject - """ - - group = "litmuschaos.io" - version = "v1alpha1" - - if kind.lower() == "chaosengine": - plural = "chaosengines" - response = self.custom_object_client.get_namespaced_custom_object( - group=group, - plural=plural, - version=version, - namespace=namespace, - name=name, - ) - try: - engine_status = response["status"]["engineStatus"] - exp_status = response["status"]["experiments"][0]["status"] - except Exception: - engine_status = "Not Initialized" - exp_status = "Not Initialized" - custom_object = ChaosEngine( - kind="ChaosEngine", - group=group, - namespace=namespace, - name=name, - plural=plural, - version=version, - engineStatus=engine_status, - expStatus=exp_status, - ) - elif kind.lower() == "chaosresult": - plural = "chaosresults" - response = self.custom_object_client.get_namespaced_custom_object( - group=group, - plural=plural, - version=version, - namespace=namespace, - name=name, - ) - try: - verdict = response["status"]["experimentStatus"]["verdict"] - fail_step = response["status"]["experimentStatus"]["failStep"] - except Exception: - verdict = "N/A" - fail_step = "N/A" - custom_object = ChaosResult( - kind="ChaosResult", - group=group, - namespace=namespace, - name=name, - plural=plural, - version=version, - verdict=verdict, - failStep=fail_step, - ) - else: - logging.error("Invalid litmus chaos custom resource name") - custom_object = None - return custom_object - def check_if_namespace_exists(self, name: str) -> bool: """ Check if a namespace exists by parsing through @@ -2596,89 +2515,6 @@ def collect_and_parse_cluster_events( return events - @deprecation.deprecated( - deprecated_in="3.1.0", - removed_in="3.2.0", - current_version=__version__, - details="replaced by `collect_and_parse_cluster_events`", - ) - def collect_cluster_events( - self, - start_timestamp: int, - end_timestamp: int, - local_timezone: str, - cluster_timezone: str = "UTC", - limit: int = 500, - namespace: str = None, - ) -> Optional[str]: - """ - Collects cluster events querying `/api/v1/events` - filtered in a given time interval and writes them in - a temporary file in json format. - - :param start_timestamp: timestamp of the minimum date - after that the event is relevant - :param end_timestamp: timestamp of the maximum date - before that the event is relevant - :param local_timezone: timezone of the local system - :param cluster_timezone: timezone of the remote cluster - :param limit: limit of the number of events to be fetched - from the cluster - :param namespace: Namespace from which the events must be - collected, if None all-namespaces will be selected - - """ - - try: - path_params: Dict[str, str] = {} - query_params = {"limit": limit} - header_params: Dict[str, str] = {} - auth_settings = ["BearerToken"] - header_params["Accept"] = self.api_client.select_header_accept( - ["application/json"] - ) - - path = "/api/v1/events" - if namespace: - path = f"/api/v1/namespaces/{namespace}/events" - - (data) = self.api_client.call_api( - path, - "GET", - path_params, - query_params, - header_params, - response_type="str", - auth_settings=auth_settings, - ) - events = [] - json_obj = ast.literal_eval(data[0]) - events_list = reversed(json_obj["items"]) - for obj in events_list: - filtered_obj = filter_dictionary( - obj, - "firstTimestamp", - start_timestamp, - end_timestamp, - cluster_timezone, - local_timezone, - ) - if filtered_obj: - events.append(filtered_obj) - - if len(events) > 0: - file_content = json.dumps(events, indent=2) - with tempfile.NamedTemporaryFile( - delete=False, mode="w" - ) as file: - file.write(file_content) - file.flush() - return file.name - return None - except Exception as e: - logging.error(str(e)) - return None - def parse_events_from_file( self, events_filename: str ) -> Optional[list[ClusterEvent]]: diff --git a/src/krkn_lib/models/k8s/models.py b/src/krkn_lib/models/k8s/models.py index 8e1f337b..d305cc70 100644 --- a/src/krkn_lib/models/k8s/models.py +++ b/src/krkn_lib/models/k8s/models.py @@ -1,9 +1,6 @@ from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass from typing import Optional -from krkn_lib.version import __version__ - -import deprecation @dataclass(frozen=True, order=False) @@ -119,79 +116,6 @@ class Pod: """ -class LitmusChaosObject: - """ - Data class to hold information regarding - a custom object of litmus project - """ - - @deprecation.deprecated( - deprecated_in="3.1.0", - removed_in="3.2.0", - current_version=__version__, - details="litmus support removed from krkn", - ) - def __init__(self): - pass - - kind: str - """ - Litmus Object Kind - """ - group: str - """ - Api Group - """ - namespace: str - """ - Namespace where the object is deployed - """ - name: str - """ - Object name - """ - plural: str - """ - CRD plural - """ - version: str - """ - Version - """ - - -class ChaosEngine(LitmusChaosObject): - """ - Data class to hold information - regarding a ChaosEngine object - """ - - engineStatus: str - """ - Litmus Chaos engine status - """ - expStatus: str - """ - Litmus Chaos Engine experiment status - """ - - -class ChaosResult(LitmusChaosObject): - """ - Data class to hold information - regarding a ChaosResult object - """ - - verdict: str - """ - Verdict of the chaos experiment - """ - failStep: str - """ - Flag to show the failure step of the ChaosExperiment - """ - - class ApiRequestException(Exception): """ Generic API Exception raised by k8s package diff --git a/src/krkn_lib/telemetry/k8s/krkn_telemetry_kubernetes.py b/src/krkn_lib/telemetry/k8s/krkn_telemetry_kubernetes.py index 81112a5e..712c6be1 100644 --- a/src/krkn_lib/telemetry/k8s/krkn_telemetry_kubernetes.py +++ b/src/krkn_lib/telemetry/k8s/krkn_telemetry_kubernetes.py @@ -4,21 +4,18 @@ import threading import time import warnings -import deprecation from queue import Queue from typing import Optional import requests import urllib3 import yaml -from tzlocal.unix import get_localzone import krkn_lib.utils as utils from krkn_lib.k8s import KrknKubernetes from krkn_lib.models.krkn import ChaosRunAlertSummary from krkn_lib.models.telemetry import ChaosRunTelemetry, ScenarioTelemetry from krkn_lib.utils.safe_logger import SafeLogger -from krkn_lib.version import __version__ class KrknTelemetryKubernetes: @@ -562,105 +559,6 @@ def set_parameters_base64( scenario_telemetry.parameters_base64 = input_file_base64 return input_file_yaml - @deprecation.deprecated( - deprecated_in="3.1.0", - removed_in="3.2.0", - current_version=__version__, - details="Cluster events has been added to the telemetry json," - "so won't be uploaded as separated file", - ) - def put_cluster_events( - self, - request_id: str, - telemetry_config: dict, - start_timestamp: int, - end_timestamp: int, - cluster_timezone: str = "UTC", - ): - """ - Collects and puts cluster events on telemetry S3 bucket - - :param request_id: uuid of the session that will represent the - S3 folder on which the prometheus files will be stored - :param telemetry_config: telemetry section of kraken config.yaml - :param start_timestamp: timestamp of the minimum date - after that the event is relevant - :param end_timestamp: timestamp of the maximum date - before that the event is relevant - :param cluster_timezone: timezone of the remote cluster - """ - - queue = Queue() - events_backup = telemetry_config.get("events_backup") - url = telemetry_config.get("api_url") - username = telemetry_config.get("username") - password = telemetry_config.get("password") - max_retries = telemetry_config.get("max_retries") - group = telemetry_config.get("telemetry_group") - exceptions = [] - if events_backup is None: - exceptions.append("telemetry -> logs_backup flag is missing") - if url is None: - exceptions.append("telemetry -> api_url is missing") - if username is None: - exceptions.append("telemetry -> username is missing") - if password is None: - exceptions.append("telemetry -> password is missing") - if max_retries is None: - exceptions.append("telemetry -> max_retries is missing") - if not group: - group = "default" - - if not events_backup: - self.safe_logger.info( - "logs_backup is False: skipping events collection" - ) - return - if len(exceptions) > 0: - raise Exception(", ".join(exceptions)) - - events_file = self.__kubecli.collect_cluster_events( - start_timestamp, - end_timestamp, - str(get_localzone()), - cluster_timezone, - limit=500, - ) - if not events_file: - self.safe_logger.warning( - "no cluster events found in the specified time interval" - ) - return - - # this parameter has doesn't have an utility in this context - # used to match the method signature and reuse it (Poor design?) - uploaded_files = list[str]() - queue.put((0, events_file, 0)) - queue_size = queue.qsize() - self.safe_logger.info("uploading cluster events...") - - worker = threading.Thread( - target=self.generate_url_and_put_to_s3_worker, - args=( - queue, - queue_size, - request_id, - group, - f"{url}/presigned-url", - username, - password, - 0, - uploaded_files, - max_retries, - "events-", - ".json", - ), - ) - worker.daemon = True - worker.start() - queue.join() - self.safe_logger.info("cluster events successfully uploaded") - def put_critical_alerts( self, request_id: str, diff --git a/src/krkn_lib/tests/test_krkn_kubernetes.py b/src/krkn_lib/tests/test_krkn_kubernetes.py index 89e716a3..4a151dae 100644 --- a/src/krkn_lib/tests/test_krkn_kubernetes.py +++ b/src/krkn_lib/tests/test_krkn_kubernetes.py @@ -1,5 +1,4 @@ import datetime -import json import logging import os import random @@ -9,7 +8,6 @@ import unittest import uuid -import deprecation import yaml from jinja2 import Environment, FileSystemLoader from kubernetes import config @@ -856,50 +854,6 @@ def test_is_pod_running(self): result = self.lib_k8s.is_pod_running("do_not_exist", "do_not_exist") self.assertFalse(result) - @deprecation.fail_if_not_removed - def test_collect_cluster_events(self): - namespace_with_evt = "test-" + self.get_random_string(10) - namespace_no_evt = "test-" + self.get_random_string(10) - pod_name = "test-" + self.get_random_string(10) - self.deploy_namespace(namespace_with_evt, []) - self.deploy_namespace(namespace_no_evt, []) - self.deploy_delayed_readiness_pod(pod_name, namespace_with_evt, 0) - self.background_delete_pod(pod_name, namespace_with_evt) - time.sleep(10) - local_timezone = f"{get_localzone()}" - now = datetime.datetime.now() - one_hour_ago = now - datetime.timedelta(hours=1) - - event_file_namespaced = self.lib_k8s.collect_cluster_events( - int(one_hour_ago.timestamp()), - int(now.timestamp()), - local_timezone, - namespace=namespace_with_evt, - ) - - event_file_not_namespaced = self.lib_k8s.collect_cluster_events( - int(one_hour_ago.timestamp()), int(now.timestamp()), local_timezone - ) - - self.assertIsNotNone(event_file_namespaced) - self.assertIsNotNone(event_file_not_namespaced) - with open(event_file_namespaced) as file: - obj_list = json.load(file) - self.assertTrue(len(obj_list) > 0) - - with open(event_file_not_namespaced) as file: - obj_list = json.load(file) - self.assertTrue(len(obj_list) > 0) - - event_file_no_event = self.lib_k8s.collect_cluster_events( - int(one_hour_ago.timestamp()), - int(now.timestamp()), - local_timezone, - namespace=namespace_no_evt, - ) - - self.assertIsNone(event_file_no_event) - def test_collect_and_parse_cluster_events(self): namespace_with_evt = "test-" + self.get_random_string(10) pod_name = "test-" + self.get_random_string(10) diff --git a/src/krkn_lib/tests/test_krkn_telemetry_kubernetes.py b/src/krkn_lib/tests/test_krkn_telemetry_kubernetes.py index 903bbcb8..8a678b11 100644 --- a/src/krkn_lib/tests/test_krkn_telemetry_kubernetes.py +++ b/src/krkn_lib/tests/test_krkn_telemetry_kubernetes.py @@ -1,5 +1,4 @@ import base64 -import datetime import os import tempfile import time @@ -7,9 +6,7 @@ import uuid import boto3 -import deprecation import yaml -from tzlocal.unix import get_localzone from krkn_lib.models.krkn import ChaosRunAlert, ChaosRunAlertSummary from krkn_lib.models.telemetry import ChaosRunTelemetry, ScenarioTelemetry @@ -251,52 +248,6 @@ def test_send_telemetry(self): f"{request_id}/telemetry.json", ) - @deprecation.fail_if_not_removed - def test_put_cluster_events(self): - - # generate a cluster event on a namespace - - namespace_with_evt = "test-" + self.get_random_string(10) - pod_name = "test-" + self.get_random_string(10) - self.deploy_namespace(namespace_with_evt, []) - self.deploy_delayed_readiness_pod(pod_name, namespace_with_evt, 0) - self.background_delete_pod(pod_name, namespace_with_evt) - time.sleep(10) - - request_id = f"test_folder/{int(time.time())}" - telemetry_config = { - "events_backup": True, - "username": os.getenv("API_USER"), - "password": os.getenv("API_PASSWORD"), - "max_retries": 5, - "api_url": "https://9ead3157ti.execute-api.us-west-2.amazonaws.com/dev", # NOQA - "backup_threads": 6, - "telemetry_group": "default", - } - now = datetime.datetime.now() - one_hour_ago = now - datetime.timedelta(hours=1) - - self.lib_telemetry_k8s.put_cluster_events( - request_id, - telemetry_config, - int(one_hour_ago.timestamp()), - int(now.timestamp()), - str(get_localzone()), - ) - - bucket_name = os.getenv("BUCKET_NAME") - s3 = boto3.client("s3") - remote_files = s3.list_objects_v2( - Bucket=bucket_name, - Prefix=f'{telemetry_config["telemetry_group"]}/{request_id}', - ) - self.assertTrue("Contents" in remote_files.keys()) - self.assertEqual( - remote_files["Contents"][0]["Key"], - f'{telemetry_config["telemetry_group"]}/' - f"{request_id}/events-00.json", - ) - def test_put_alerts(self): request_id = f"test_folder/{int(time.time())}"