Re-Architecture of Layered Models & Ensembles (#47)

* Fix some AutoSarima bugs.

* Harden models to granularities like MS

* Correct import path.

* Fix save/load behavior for layered models.

* Implement generic LayeredModel class.

* More helpful model config __init__ docstrings.

Create a new metaclass which allows model configs to
1. Inherit params from superclass __init__, and have those params
   included in the signature for the subclass __init__.
2. Inherit param docstrings from superclass __init__, which means that
   users don't have to rewrite lots of docstrings for superclass params.

* Various minor fixes.

Also suppresses output from Prophet while the model is training.

* Update gson from 2.8.6 to 2.8.9.

Addresses security risk.

* Order docstring params according to fn signature.

* Fix failing tests.

* Allow None models in LayeredModel.

* Make Merlion default models subclass LayeredModel.

* Allow bubbling of callable attributes in Layers.

* Simplify AutoSARIMA implementation.

* Minor fixes to default model.

* Move LayeredModel to a new file.

* Add dynamic inheritance to LayeredModel's.

* Allow AutoSarima to use a SarimaDetector.

* Add more docstrings & reduce code duplication.

* Add auto-seasonality to AutoSarima.

* Update AutoSARIMA example.

* Update tests to avoid segfault.

* Actually use approx_iter in AutoSARIMA

* Implement __reduce__ for Config & LayeredModel.

This ensures that everything is usable by multiprocessing code.
The use of _original_cls for LayeredModel.__reduce__ ensures that the
right class object is used when attempting to initialize the object, not
a dynamically defined subclass.

* change p value to 0.1

* change default regression method to c in KPSS test

* Better defined periodicity strategies.

* Add distinct AutoProphet model.

* Add AutoETS model.

* Various updates to make serialization work.

- Override setstate/getstate for ensembles
- Use a more refined method to check for unused kwargs in config init

* Remove depth field & fix _save_state() bug.

* Add comment on _original_cls

* Add example for ModelConfigMeta.

* Various AutoProphet bug fixes.

* Fix from_dict() implementations

* Simplify LayeredModel._save_state()

* Move ensemble.models from model to config.

This mirrors the changes to LayeredModel, and it greatly simplifies a
number of implementation details.

* Fix Sphinx errors & handle docstring suffixes.

* Add p-value to SeasonalityConfig.

* Move AutoETS & AutoProphet to models.automl

* Update benchmark_forecast.json

* Implement bubbling for LayeredModel.__setattr__

* Add robustness to None models in serialization.

Co-authored-by: Chenghao Liu <[email protected]>

benchmark_forecast.py

@@ -25,8 +25,7 @@
 from merlion.models.ensemble.forecast import ForecasterEnsembleConfig, ForecasterEnsemble
 from merlion.models.factory import ModelFactory
 from merlion.models.forecast.base import ForecasterBase
-from merlion.transform.resample import TemporalResample
-from merlion.utils.time_series import granularity_str_to_seconds
+from merlion.transform.resample import TemporalResample, granularity_str_to_seconds
 from merlion.utils import TimeSeries, UnivariateTimeSeries
 from merlion.utils.resample import get_gcd_timedelta
 

conf/benchmark_anomaly.json

@@ -89,13 +89,37 @@
     }
   },
 
+  "AutoETSDetector": {"alias": "AutoETS"},
+  "AutoETS": {
+    "config": {
+      "default": {
+        "model": {"name": "ETSDetector"},
+        "damped_trend": true,
+        "transform": {"name": "TemporalResample", "granularity": "1h"}
+      },
+      "IOpsCompetition": {
+        "transform": {"name": "TemporalResample", "granularity": "5min"}
+      },
+      "CASP": {
+        "transform": {"name": "TemporalResample", "granularity": "5min"}
+      }
+    }
+  },
+
   "Prophet": {"alias": "ProphetDetector"},
   "ProphetDetector": {
     "config": {
       "default": {}
     }
   },
 
+  "AutoProphetDetector": {"alias": "AutoProphet"},
+  "AutoProphet": {
+    "config": {
+      "default": {"model": {"name": "ProphetDetector"}}
+    }
+  },
+
   "StatThreshold": {
     "config": {
       "default": {}

conf/benchmark_forecast.json

@@ -37,6 +37,14 @@
     }
   },
 
+  "AutoETS": {
+    "config": {
+      "default": {
+        "damped_trend": true
+      }
+    }
+  },
+
   "MSES": {
     "config": {
       "default": {
@@ -48,18 +56,15 @@
   "Prophet": {
     "config": {
       "default": {
-        "uncertainty_samples": 0,
-        "add_seasonality": null
+        "uncertainty_samples": 0
       }
     }
   },
 
   "AutoProphet": {
-    "model_type": "Prophet",
     "config": {
       "default": {
-        "uncertainty_samples": 0,
-        "add_seasonality": "auto"
+        "uncertainty_samples": 0
       }
     }
   },

docs/source/merlion.models.automl.rst

@@ -1,3 +1,4 @@
+
 merlion.models.automl package
 ==============================
 
@@ -7,26 +8,19 @@ merlion.models.automl package
    :show-inheritance:
 
 .. autosummary::
-    layer_mixin
-    forecasting_layer_base
+    base
+    seasonality
+    autoets
+    autoprophet
     autosarima
-    seasonality_mixin
 
 Submodules
 ----------
 
-merlion.models.automl.layer_mixin module
----------------------------------------------------
-
-.. automodule:: merlion.models.automl.layer_mixin
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-merlion.models.automl.forecasting_layer_base module
----------------------------------------------------
+merlion.models.automl.base module
+---------------------------------
 
-.. automodule:: merlion.models.automl.forecasting_layer_base
+.. automodule:: merlion.models.automl.base
    :members:
    :undoc-members:
    :show-inheritance:
@@ -40,10 +34,10 @@ merlion.models.automl.autosarima module
    :show-inheritance:
 
 
-merlion.models.automl.seasonality_mixin module
-----------------------------------------------
+merlion.models.automl.seasonality module
+----------------------------------------
 
-.. automodule:: merlion.models.automl.seasonality_mixin
+.. automodule:: merlion.models.automl.seasonality
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/merlion.models.rst

@@ -58,8 +58,9 @@ Finally, we support ensembles of models in :py:mod:`merlion.models.ensemble`.
    :show-inheritance:
 
 .. autosummary::
-    base
     factory
+    base
+    layers
     defaults
     anomaly
     anomaly.change_point
@@ -86,6 +87,15 @@ Subpackages
 Submodules
 ----------
 
+merlion.models.factory module
+-----------------------------
+
+.. automodule:: merlion.models.factory
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
 merlion.models.base module
 --------------------------
 
@@ -94,15 +104,14 @@ merlion.models.base module
    :undoc-members:
    :show-inheritance:
 
-merlion.models.factory module
------------------------------
+merlion.models.layers module
+----------------------------
 
-.. automodule:: merlion.models.factory
+.. automodule:: merlion.models.layers
    :members:
    :undoc-members:
    :show-inheritance:
 
-
 merlion.models.defaults module
 ------------------------------
 

merlion/models/anomaly/base.py

@@ -20,7 +20,7 @@
 from merlion.post_process.calibrate import AnomScoreCalibrator
 from merlion.post_process.factory import PostRuleFactory
 from merlion.post_process.sequence import PostRuleSequence
-from merlion.post_process.threshold import AggregateAlarms
+from merlion.post_process.threshold import AggregateAlarms, Threshold
 from merlion.utils import TimeSeries
 
 logger = logging.getLogger(__name__)
@@ -32,6 +32,8 @@ class DetectorConfig(Config):
     """
 
     _default_threshold = AggregateAlarms(alm_threshold=3.0)
+    calibrator: AnomScoreCalibrator = None
+    threshold: Threshold = None
 
     def __init__(
         self, max_score: float = 1000, threshold=None, enable_calibrator=True, enable_threshold=True, **kwargs
@@ -73,15 +75,14 @@ def post_rule(self):
         return PostRuleSequence(rules)
 
     @classmethod
-    def from_dict(cls, config_dict: Dict[str, Any], return_unused_kwargs=False, **kwargs):
-        # Get the calibrator, but we will set it manually after the constructor
-        config_dict = copy(config_dict)
-        calibrator_config = config_dict.pop("calibrator", None)
+    def from_dict(cls, config_dict: Dict[str, Any], return_unused_kwargs=False, calibrator=None, **kwargs):
+        # Get the calibrator, but we will set it manually after the constructor by putting it in kwargs
+        calibrator = config_dict.pop("calibrator", calibrator)
         config, kwargs = super().from_dict(config_dict, return_unused_kwargs=True, **kwargs)
-        if calibrator_config is not None:
-            config.calibrator = PostRuleFactory.create(**calibrator_config)
+        if calibrator is not None:
+            calibrator = PostRuleFactory.create(**calibrator)
+            config.calibrator = calibrator
 
-        # Return unused kwargs if desired
         if len(kwargs) > 0 and not return_unused_kwargs:
             logger.warning(f"Unused kwargs: {kwargs}", stack_info=True)
         elif return_unused_kwargs:
@@ -96,6 +97,9 @@ class NoCalibrationDetectorConfig(DetectorConfig):
     """
 
     def __init__(self, enable_calibrator=False, **kwargs):
+        """
+        :param enable_calibrator: ``False`` because this config assumes calibrated outputs from the model.
+        """
         super().__init__(enable_calibrator=enable_calibrator, **kwargs)
 
     @property

merlion/models/anomaly/change_point/bocpd.py

@@ -117,7 +117,8 @@ def __init__(
     def to_dict(self, _skipped_keys=None):
         _skipped_keys = _skipped_keys if _skipped_keys is not None else set()
         config_dict = super().to_dict(_skipped_keys.union({"change_kind"}))
-        config_dict["change_kind"] = self.change_kind.name
+        if "change_kind" not in _skipped_keys:
+            config_dict["change_kind"] = self.change_kind.name
         return config_dict
 
     @property

merlion/models/anomaly/dbl.py

@@ -96,7 +96,8 @@ def determine_train_window(self):
     def to_dict(self, _skipped_keys=None):
         _skipped_keys = _skipped_keys if _skipped_keys is not None else set()
         config_dict = super().to_dict(_skipped_keys.union({"trends"}))
-        config_dict["trends"] = [t.name for t in self.trends]
+        if "trends" not in _skipped_keys:
+            config_dict["trends"] = [t.name for t in self.trends]
         return config_dict
 
 

merlion/models/anomaly/forecast_based/base.py

@@ -7,7 +7,6 @@
 """
 Base class for anomaly detectors based on forecasting models.
 """
-from abc import ABC
 import logging
 from typing import List, Optional
 
@@ -17,11 +16,12 @@
 from merlion.models.forecast.base import ForecasterBase
 from merlion.plot import Figure
 from merlion.utils import UnivariateTimeSeries, TimeSeries
+from merlion.utils.misc import AutodocABCMeta
 
 logger = logging.getLogger(__name__)
 
 
-class ForecastingDetectorBase(ForecasterBase, DetectorBase, ABC):
+class ForecastingDetectorBase(ForecasterBase, DetectorBase, metaclass=AutodocABCMeta):
     """
     Base class for a forecast-based anomaly detector.
     """

merlion/models/anomaly/forecast_based/mses.py

@@ -17,10 +17,14 @@
 
 
 class MSESDetectorConfig(MSESConfig, DetectorConfig):
+    """
+    Configuration class for an MSES forecasting model adapted for anomaly detection.
+    """
+
     _default_threshold = AggregateAlarms(alm_threshold=2)
 
-    def __init__(self, online_updates: bool = True, **kwargs):
-        super().__init__(**kwargs)
+    def __init__(self, max_forecast_steps: int, online_updates: bool = True, **kwargs):
+        super().__init__(max_forecast_steps=max_forecast_steps, **kwargs)
         self.online_updates = online_updates
 
 

merlion/models/anomaly/random_cut_forest.py

@@ -33,7 +33,7 @@ def __init__(self):
         import jpype.imports
 
         resource_dir = join(dirname(dirname(dirname(abspath(__file__)))), "resources")
-        jars = ["gson-2.8.6.jar", "randomcutforest-core-1.0.jar", "randomcutforest-serialization-json-1.0.jar"]
+        jars = ["gson-2.8.9.jar", "randomcutforest-core-1.0.jar", "randomcutforest-serialization-json-1.0.jar"]
         if not JVMSingleton._initialized:
             jpype.startJVM(classpath=[join(resource_dir, jar) for jar in jars])
             JVMSingleton._initialized = True

merlion/models/anomaly/zms.py

@@ -24,19 +24,16 @@
 
 class ZMSConfig(DetectorConfig, NormalizingConfig):
     """
-    Configuration class for `ZMS` anomaly detection model.
+    Configuration class for `ZMS` anomaly detection model. The transform of this config is actually a
+    pre-processing step, followed by the desired number of lag transforms, and a final mean/variance
+    normalization step. This full transform may be accessed as `ZMSConfig.full_transform`. Note that
+    the normalization is inherited from `NormalizingConfig`.
     """
 
     _default_transform = TemporalResample(trainable_granularity=True)
 
     def __init__(self, base: int = 2, n_lags: int = None, lag_inflation: float = 1.0, **kwargs):
         r"""
-        Configuration class for ZMS. The transform of this config is actually a
-        pre-processing step, followed by the desired number of lag transforms
-        and a final mean/variance normalization step. This full transform may be
-        accessed as `ZMSConfig.full_transform`. Note that the normalization is
-        inherited from `NormalizingConfig`.
-
         :param base: The base to use for computing exponentially distant lags.
         :param n_lags: The number of lags to be used. If None, n_lags will be
             chosen later as the maximum number of lags possible for the initial

merlion/models/automl/autoets.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2021 salesforce.com, inc.
+# All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+#
+"""
+Automatic seasonality detection for ETS.
+"""
+
+from typing import Union
+
+from merlion.models.forecast.ets import ETS
+from merlion.models.automl.seasonality import SeasonalityConfig, SeasonalityLayer
+
+
+class AutoETSConfig(SeasonalityConfig):
+    """
+    Config class for ETS with automatic seasonality detection.
+    """
+
+    def __init__(self, model: Union[ETS, dict] = None, **kwargs):
+        model = dict(name="ETS") if model is None else model
+        super().__init__(model=model, **kwargs)
+
+
+class AutoETS(SeasonalityLayer):
+    """
+    ETS with automatic seasonality detection.
+    """
+
+    config_class = AutoETSConfig