Merge pull request #1084 from chaos-genius/develop

release: v0.10.1

chaos_genius/commands.py

@@ -199,8 +199,8 @@ def run_anomaly_rca_scheduler():
 
     Note: a celery worker needs to be active for this to work.
     """
-    from chaos_genius.jobs.anomaly_tasks import anomaly_scheduler
-    res = anomaly_scheduler.delay()
+    from chaos_genius.jobs.analytics_scheduler import scheduler_wrapper
+    res = scheduler_wrapper.delay()
     res.get()
     click.echo("Completed running scheduler. Tasks should be running in the worker.")
 

chaos_genius/controllers/dashboard_controller.py

@@ -282,6 +282,7 @@ def all_dashboard_names() -> Dict[int, str]:
     return {
         row[0]: row[1]
         for row in db.session.query(Dashboard.id, Dashboard.name)
+        .filter(Dashboard.active == True)  # noqa: E712
         .order_by(Dashboard.name)
         .all()
     }

chaos_genius/controllers/data_source_controller.py

@@ -146,6 +146,7 @@ def used_data_source_types() -> List[str]:
     return [
         row[0]
         for row in db.session.query(DataSource.connection_type)
+        .filter(DataSource.active == True)  # noqa: E712
         .order_by(DataSource.connection_type)
         .distinct()
         .all()

chaos_genius/core/anomaly/controller.py

@@ -76,7 +76,11 @@ def __init__(
             self.debug = True
         if self.debug == "False":
             self.debug = False
-        self.slack = MAX_ANOMALY_SLACK_DAYS
+        self.slack = (
+            MAX_ANOMALY_SLACK_DAYS * 24
+            if self.kpi_info["anomaly_params"]["frequency"] == "H"
+            else MAX_ANOMALY_SLACK_DAYS
+        )
 
         if self.kpi_info["anomaly_params"]["frequency"] == "H":
             period = int(self.kpi_info["anomaly_params"]["anomaly_period"]) * 24

chaos_genius/core/anomaly/processor.py

@@ -3,6 +3,7 @@
 import logging
 from typing import Dict, Tuple, Union
 
+import numpy as np
 import pandas as pd
 
 from chaos_genius.core.anomaly.constants import FREQUENCY_DELTA
@@ -105,16 +106,17 @@ def _predict(self, model: AnomalyModel) -> pd.DataFrame:
         """
         input_data = self.input_data
 
-        pred_series = pd.DataFrame(
-            columns=[
-                "dt",
-                "y",
-                "yhat_lower",
-                "yhat_upper",
-                "anomaly",
-                "severity",
-                "impact",
-            ]
+        pred_series_schema = {
+            "dt": np.datetime64,
+            "y": float,
+            "yhat_lower": float,
+            "yhat_upper": float,
+            "anomaly": int,
+            "severity": float,
+            "impact": float,
+        }
+        pred_series = pd.DataFrame(columns=pred_series_schema.keys()).astype(
+            pred_series_schema
         )
 
         input_last_date = input_data["dt"].iloc[-1]
@@ -169,9 +171,7 @@ def _predict(self, model: AnomalyModel) -> pd.DataFrame:
                     )
                     prediction["y"] = df["y"].to_list()
 
-                    prediction = pd.DataFrame(prediction.iloc[-1].copy()).T.reset_index(
-                        drop=True
-                    )
+                    prediction = prediction.iloc[[-1]]
 
                     pred_series = pred_series.append(
                         self._calculate_metrics(

chaos_genius/core/utils/kpi_validation.py

@@ -6,6 +6,7 @@
 
 import pandas as pd
 from pandas.api.types import is_datetime64_any_dtype as is_datetime
+from pandas.api.types import is_float_dtype as is_float
 from pandas.api.types import is_integer_dtype as is_integer
 
 from chaos_genius.core.rca.root_cause_analysis import SUPPORTED_AGGREGATIONS
@@ -306,17 +307,21 @@ def _validate_count_column_is_number(
     df: pd.core.frame.DataFrame,
     count_column_name: Optional[str],
 ) -> Tuple[bool, str]:
-    """Validate if specified date column is parseable.
+    """Validate if specified count column is parseable.
 
     :param df: A pandas DataFrame
     :type df: pd.core.frame.DataFrame
     :param count_column_name: Name of the count column, relevant for preaggregated data
-    :type date_column_name: Optional[str]
+    :type count_column_name: Optional[str]
     :return: returns a tuple with the status as a bool and a status message
     :rtype: Tuple[bool, str]
     """
     # has to be integer if count_column_name exists, only then proceed else exit
-    if count_column_name and not (is_integer(df[count_column_name])):
+    if (
+        count_column_name
+        and not is_integer(df[count_column_name])
+        and not is_float(df[count_column_name])
+    ):
         invalid_type_err_msg = (
             "The count column is of the type"
             f" {df[count_column_name].dtype}, use 'cast' to convert to integer."

chaos_genius/jobs/analytics_scheduler.py

@@ -91,7 +91,9 @@ def _to_run_anomaly(self, kpi: Kpi, scheduled_time: datetime) -> bool:
         # anomaly is setup if model_name is set in anomaly_params
         scheduler_params = kpi.scheduler_params
         anomaly_is_setup = (
-            kpi.anomaly_params is not None and "model_name" in kpi.anomaly_params
+            (kpi.run_anomaly is True)
+            and (kpi.anomaly_params is not None)
+            and ("model_name" in kpi.anomaly_params)
         )
         anomaly_already_run = (
             scheduler_params is not None

chaos_genius/settings.py

@@ -149,7 +149,7 @@ def _make_bool(val: Union[str, bool]) -> bool:
 TASK_CHECKPOINT_LIMIT: int = int(os.getenv("TASK_CHECKPOINT_LIMIT", 1000))
 """Number of last checkpoints to retrieve in Task Monitor"""
 
-CHAOSGENIUS_VERSION_MAIN = os.getenv("CHAOSGENIUS_VERSION_MAIN", "0.10.0")
+CHAOSGENIUS_VERSION_MAIN = os.getenv("CHAOSGENIUS_VERSION_MAIN", "0.10.1")
 """ChaosGenius version - semver part only"""
 CHAOSGENIUS_VERSION_POSTFIX = os.getenv("CHAOSGENIUS_VERSION_POSTFIX", "git")
 """ChaosGenius version - postfix to identify deployment"""

chaos_genius/views/anomaly_data_view.py

@@ -4,7 +4,7 @@
 import time
 from collections import defaultdict
 from datetime import date, datetime, timedelta
-from typing import Any, Dict, List, Optional, Sequence, Tuple, cast
+from typing import Any, DefaultDict, Dict, List, Optional, Sequence, Set, Tuple, cast
 
 import pandas as pd
 from flask.blueprints import Blueprint
@@ -209,7 +209,7 @@ def kpi_anomaly_params(kpi_id: int):
 
     (This is where anomaly is setup/configured or updated).
     """
-    kpi = cast(Kpi, Kpi.get_by_id(kpi_id))
+    kpi = cast(Optional[Kpi], Kpi.get_by_id(kpi_id))
 
     if kpi is None:
         return (
@@ -336,7 +336,7 @@ def kpi_anomaly_params(kpi_id: int):
 def anomaly_settings_status(kpi_id: int):
     """Get anomaly status for a KPI."""
     logger.info(f"Retrieving anomaly settings for kpi: {kpi_id}")
-    kpi = cast(Kpi, Kpi.get_by_id(kpi_id))
+    kpi = cast(Optional[Kpi], Kpi.get_by_id(kpi_id))
 
     if kpi is None:
         return (
@@ -377,17 +377,85 @@ def anomaly_settings_status(kpi_id: int):
 def kpi_anomaly_retraining(kpi_id: int):
     """Delete all anomaly data and retrain anomaly for a KPI."""
     # delete all data in anomaly output table
-    delete_anomaly_output_for_kpi(kpi_id)
+    kpi = cast(Optional[Kpi], Kpi.get_by_id(kpi_id))
+    if kpi is not None:
+        if kpi.run_anomaly and kpi.anomaly_params is not None:
+            delete_anomaly_output_for_kpi(kpi_id)
+            # add anomaly to queue
+            from chaos_genius.jobs.anomaly_tasks import ready_anomaly_task
+            anomaly_task = ready_anomaly_task(kpi_id)
+            if anomaly_task is None:
+                message = f"retraining failed for KPI: {kpi_id}, KPI id is None"
+                status = "failure"
+            else:
+                anomaly_task.apply_async()
+                logger.info(f"Retraining started for KPI ID: {kpi_id}")
+                message = f"retraining started for KPI: {kpi_id}"
+                status = "success"
+        else:
+            message = f"Please enable anomaly for KPI ID: {kpi_id} before retraining"
+            status = "failure"
+    else:
+        message = f"KPI {kpi_id} could not be retreived."
+        status = "failure"
+    return jsonify({"msg": message, "status": status})
+
+
+@blueprint.route("/<int:kpi_id>/disable-anomaly", methods=["GET", "POST"])
+def disable_anomaly(kpi_id):
+    """API end point which disables analytics by modifying the run_anomaly flag."""
+    kpi = cast(Optional[Kpi], Kpi.get_by_id(kpi_id))
+    if kpi is not None:
+        # check if anomaly is setup
+        if kpi.anomaly_params:
+            kpi.run_anomaly = False
+            kpi.update(commit=True)
+            message = f"Disabled Analytics for KPI ID: {kpi_id}"
+            status = "success"
+        else:
+            message = f"Failed to Disable Anomaly because it is not enabled for KPI ID: {kpi_id}"
+            status = "failure"
+    else:
+        message = f"KPI {kpi_id} could not be retreived."
+        status = "failure"
 
-    # add anomaly to queue
-    from chaos_genius.jobs.anomaly_tasks import ready_anomaly_task
+    if status == "success":
+        logger.info(message)
+    else:
+        logger.error(message)
+    return jsonify({"msg": message, "status": status})
+
+
+@blueprint.route("/<int:kpi_id>/enable-anomaly", methods=["GET", "POST"])
+def enable_anomaly(kpi_id):
+    """API end point which enables analytics by modifying the run_anomaly flag."""
+    kpi = cast(Optional[Kpi], Kpi.get_by_id(kpi_id))
+    if kpi is not None:
+        if not kpi.run_anomaly:
+            kpi.run_anomaly = True
+            kpi.update(commit=True)
+            if kpi.anomaly_params is not None:
+                message = f"Enabled Analytics for KPI ID: {kpi_id}"
+                status = "success"
+            else:
+                message = f"KPI ID: {kpi_id}. Analytics enabled but is not configured. Please Configure it to run anomaly."
+                status = "success"
+                logger.warn(message)
+        else:
+            message = (
+                "Failed to Enable Anomaly because it is either already enabled"
+                f" or not set up for KPI ID: {kpi_id}"
+            )
+            status = "failure"
+    else:
+        message = f"KPI {kpi_id} could not be retreived"
+        status = "failure"
 
-    anomaly_task = ready_anomaly_task(kpi_id)
-    if anomaly_task is None:
-        return jsonify({"msg": f"retraining failed for KPI: {kpi_id}, KPI id is None"})
-    anomaly_task.apply_async()
-    logger.info(f"Retraining started for KPI ID: {kpi_id}")
-    return jsonify({"msg": f"retraining started for KPI: {kpi_id}"})
+    if status == "success":
+        logger.info(message)
+    else:
+        logger.error(message)
+    return jsonify({"msg": message, "status": status})
 
 
 def _get_dimensions_values(
@@ -426,11 +494,10 @@ def _get_dimensions_values(
 
     # series_type strings are in format {dimension1 == value1, dimension2 == value2,}
     # create a default dict mapping each dimension to a list of their values
-    dimension_values_dict = defaultdict(list)
+    dimension_values_dict: DefaultDict[str, Set[str]] = defaultdict(set)
     for dim_val_row in results:
         for dimension in dim_val_row[0].keys():
-            if dim_val_row[0][dimension] not in dimension_values_dict[dimension]:
-                dimension_values_dict[dimension].append(dim_val_row[0][dimension])
+            dimension_values_dict[dimension].add(dim_val_row[0][dimension])
 
     dimension_values_list = [
         {
@@ -443,10 +510,10 @@ def _get_dimensions_values(
                     "value": value,
                     "label_path_safe": make_path_safe(value),
                 }
-                for value in dimension_values_dict[dimension]
+                for value in sorted(dimension_values_dict[dimension])
             ],
         }
-        for dimension in dimension_values_dict
+        for dimension in sorted(dimension_values_dict)
     ]
 
     return dimension_values_list

chaos_genius/views/download_view.py

@@ -7,7 +7,6 @@
 from flask.json import jsonify
 from flask.wrappers import Response
 
-from chaos_genius.controllers.kpi_controller import get_kpi_data_from_id
 from chaos_genius.core.rca.rca_utils.api_utils import (
     kpi_line_data,
     rca_analysis,
@@ -76,7 +75,6 @@ def download_anomaly_data(kpi_id: int):
         end_date = get_anomaly_output_end_date(kpi.as_dict)
 
         is_subdim = dimension is not None and value is not None
-
         if is_subdim:
             logger.info(
                 "Downloading subdim anomaly data for KPI: %d, subdim: %s=%s",
@@ -91,7 +89,9 @@ def download_anomaly_data(kpi_id: int):
             logger.info("Downloading overall anomaly data for KPI: %d", kpi_id)
             data_points = get_anomaly_data_points(kpi_id, end_date)
 
-        if not data_points:
+        if not data_points and not kpi.run_anomaly:
+            raise Exception(f"Anomaly disabled, No data found for KPI: {kpi_id}")
+        elif not data_points:
             raise Exception(f"No anomaly data found for KPI: {kpi_id}")
 
         def row_gen(data_points: List[AnomalyDataOutput]):

chaos_genius/views/kpi_view.py

@@ -88,6 +88,7 @@ def kpi():
             count_column=data.get("count_column"),
             filters=data.get("filters"),
             dimensions=data.get("dimensions"),
+            run_anomaly=True
         )
         # Perform KPI Validation
         status, message, tz_aware = validate_kpi(
@@ -429,8 +430,7 @@ def edit_kpi(kpi_id):
                         "overall": True,
                         "subdim": True,
                     }
-
-                if kpi_obj.anomaly_params is not None and (
+                if (kpi_obj.anomaly_params is not None) and (
                     "run_optional" not in kpi_obj.anomaly_params or (
                         kpi_obj.anomaly_params["run_optional"]["subdim"]
                         != run_optional["subdim"]

docker-compose.dev-thirdparty.yml

@@ -278,6 +278,23 @@ services:
       - INTEGRATION_DATABASE=${INTEGRATION_DATABASE}
       - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
       - CELERY_BROKER_URL=${CELERY_BROKER_URL}
+      - CACHE_DEFAULT_TIMEOUT=${CACHE_DEFAULT_TIMEOUT}
+      - MULTIDIM_ANALYSIS_FOR_ANOMALY=${MULTIDIM_ANALYSIS_FOR_ANOMALY}
+      - MAX_SUBDIM_CARDINALITY=${MAX_SUBDIM_CARDINALITY}
+      - TOP_DIMENSIONS_FOR_ANOMALY_DRILLDOWN=${TOP_DIMENSIONS_FOR_ANOMALY_DRILLDOWN}
+      - MIN_DATA_IN_SUBGROUP=${MIN_DATA_IN_SUBGROUP}
+      - TOP_SUBDIMENSIONS_FOR_ANOMALY=${TOP_SUBDIMENSIONS_FOR_ANOMALY}
+      - MAX_ROWS_IN_KPI=${MAX_ROWS_IN_KPI}
+      - MAX_FILTER_SUBGROUPS_ANOMALY=${MAX_FILTER_SUBGROUPS_ANOMALY}
+      - MAX_SUMMARY_DEEPDRILLS_SLACK_DAYS=${MAX_SUMMARY_DEEPDRILLS_SLACK_DAYS}
+      - MAX_ANOMALY_SLACK_DAYS=${MAX_ANOMALY_SLACK_DAYS}
+      - DAYS_OFFSET_FOR_ANALTYICS=${DAYS_OFFSET_FOR_ANALTYICS}
+      - HOURS_OFFSET_FOR_ANALTYICS=${HOURS_OFFSET_FOR_ANALTYICS}
+      - DEEPDRILLS_HTABLE_MAX_PARENTS=${DEEPDRILLS_HTABLE_MAX_PARENTS}
+      - DEEPDRILLS_HTABLE_MAX_CHILDREN=${DEEPDRILLS_HTABLE_MAX_CHILDREN}
+      - DEEPDRILLS_HTABLE_MAX_DEPTH=${DEEPDRILLS_HTABLE_MAX_DEPTH}
+      - SUMMARY_DEEPDRILLS_ENABLED_TIME_RANGES=${SUMMARY_DEEPDRILLS_ENABLED_TIME_RANGES}
+      - DEEPDRILLS_ENABLED=${DEEPDRILLS_ENABLED}
       - REACT_APP_EVENT_ALERT=${REACT_APP_EVENT_ALERT}
       - SENTRY_DSN=${SENTRY_DSN}
       - CHAOSGENIUS_ENTERPRISE_EDITION_KEY=${CHAOSGENIUS_ENTERPRISE_EDITION_KEY}

docker-compose.dev.yml

@@ -253,6 +253,23 @@ services:
       - INTEGRATION_DATABASE=${INTEGRATION_DATABASE}
       - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
       - CELERY_BROKER_URL=${CELERY_BROKER_URL}
+      - CACHE_DEFAULT_TIMEOUT=${CACHE_DEFAULT_TIMEOUT}
+      - MULTIDIM_ANALYSIS_FOR_ANOMALY=${MULTIDIM_ANALYSIS_FOR_ANOMALY}
+      - MAX_SUBDIM_CARDINALITY=${MAX_SUBDIM_CARDINALITY}
+      - TOP_DIMENSIONS_FOR_ANOMALY_DRILLDOWN=${TOP_DIMENSIONS_FOR_ANOMALY_DRILLDOWN}
+      - MIN_DATA_IN_SUBGROUP=${MIN_DATA_IN_SUBGROUP}
+      - TOP_SUBDIMENSIONS_FOR_ANOMALY=${TOP_SUBDIMENSIONS_FOR_ANOMALY}
+      - MAX_ROWS_IN_KPI=${MAX_ROWS_IN_KPI}
+      - MAX_FILTER_SUBGROUPS_ANOMALY=${MAX_FILTER_SUBGROUPS_ANOMALY}
+      - MAX_SUMMARY_DEEPDRILLS_SLACK_DAYS=${MAX_SUMMARY_DEEPDRILLS_SLACK_DAYS}
+      - MAX_ANOMALY_SLACK_DAYS=${MAX_ANOMALY_SLACK_DAYS}
+      - DAYS_OFFSET_FOR_ANALTYICS=${DAYS_OFFSET_FOR_ANALTYICS}
+      - HOURS_OFFSET_FOR_ANALTYICS=${HOURS_OFFSET_FOR_ANALTYICS}
+      - DEEPDRILLS_HTABLE_MAX_PARENTS=${DEEPDRILLS_HTABLE_MAX_PARENTS}
+      - DEEPDRILLS_HTABLE_MAX_CHILDREN=${DEEPDRILLS_HTABLE_MAX_CHILDREN}
+      - DEEPDRILLS_HTABLE_MAX_DEPTH=${DEEPDRILLS_HTABLE_MAX_DEPTH}
+      - SUMMARY_DEEPDRILLS_ENABLED_TIME_RANGES=${SUMMARY_DEEPDRILLS_ENABLED_TIME_RANGES}
+      - DEEPDRILLS_ENABLED=${DEEPDRILLS_ENABLED}
       - REACT_APP_EVENT_ALERT=${REACT_APP_EVENT_ALERT}
       - SENTRY_DSN=${SENTRY_DSN}
       - CHAOSGENIUS_ENTERPRISE_EDITION_KEY=${CHAOSGENIUS_ENTERPRISE_EDITION_KEY}