From df5dfff051532d2075aebb489d228f2cee41ec40 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 11 Oct 2022 10:53:04 +0100 Subject: [PATCH 001/247] Initial commit --- alibi_detect/od/backend/__init__.py | 4 ++ alibi_detect/od/backend/keops/knn.py | 25 +++++++ alibi_detect/od/backend/torch/knn.py | 17 +++++ alibi_detect/od/base.py | 65 +++++++++++++++++ alibi_detect/od/knn.py | 55 +++++++++++++++ alibi_detect/od/tests/test_knn.py | 100 +++++++++++++++++++++++++++ alibi_detect/od/transforms.py | 91 ++++++++++++++++++++++++ 7 files changed, 357 insertions(+) create mode 100644 alibi_detect/od/backend/__init__.py create mode 100644 alibi_detect/od/backend/keops/knn.py create mode 100644 alibi_detect/od/backend/torch/knn.py create mode 100644 alibi_detect/od/base.py create mode 100644 alibi_detect/od/knn.py create mode 100644 alibi_detect/od/tests/test_knn.py create mode 100644 alibi_detect/od/transforms.py diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py new file mode 100644 index 000000000..18ac838bc --- /dev/null +++ b/alibi_detect/od/backend/__init__.py @@ -0,0 +1,4 @@ +from alibi_detect.utils.missing_optional_dependency import import_optional + +KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) +KNNKeops = import_optional('alibi_detect.od.backend.keops.knn', ['KNNKeops']) diff --git a/alibi_detect/od/backend/keops/knn.py b/alibi_detect/od/backend/keops/knn.py new file mode 100644 index 000000000..78506f663 --- /dev/null +++ b/alibi_detect/od/backend/keops/knn.py @@ -0,0 +1,25 @@ +import numpy as np +import torch +from pykeops.torch import LazyTensor + + +def cdist(X, Y): + return ((X - Y)**2).sum(-1).sqrt() + + +class KNNKeops: + def score(X, x_ref, k, kernel=None): + ensemble = isinstance(k, (np.ndarray, list, tuple)) + X = torch.as_tensor(X) + X_keops = LazyTensor(X[:, None, :]) + x_ref_keops = LazyTensor(x_ref[None, :, :]) + K = -kernel(X_keops, x_ref_keops) if kernel else cdist(X_keops, x_ref_keops) + ks = np.array(k) if ensemble else np.array([k]) + bot_k_inds = K.argKmin(np.max(ks), dim=1) + all_knn_dists = (X[:, None, :] - x_ref[bot_k_inds][:, ks-1, :]).norm(dim=2) + all_knn_dists = all_knn_dists if ensemble else all_knn_dists[:,0] + return all_knn_dists.cpu().numpy() + + + def fit(X): + return torch.as_tensor(X) \ No newline at end of file diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py new file mode 100644 index 000000000..db846ac4c --- /dev/null +++ b/alibi_detect/od/backend/torch/knn.py @@ -0,0 +1,17 @@ +import numpy as np +import torch + + +class KNNTorch: + def score(X, x_ref, k, kernel=None): + ensemble = isinstance(k, (np.ndarray, list, tuple)) + X = torch.as_tensor(X, dtype=torch.float32) + K = -kernel(X, x_ref) if kernel else torch.cdist(X, x_ref) + ks = np.array(k) if ensemble else np.array([k]) + bot_k_dists = torch.topk(K, np.max(ks), dim=1, largest=False) + all_knn_dists = bot_k_dists.values[:,ks-1] + all_knn_dists = all_knn_dists if ensemble else all_knn_dists[:,0] + return all_knn_dists.cpu().numpy() + + def fit(X): + return torch.as_tensor(X, dtype=torch.float32) \ No newline at end of file diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py new file mode 100644 index 000000000..12d5f7ea1 --- /dev/null +++ b/alibi_detect/od/base.py @@ -0,0 +1,65 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +import numpy as np +from alibi_detect.version import __version__ +import logging +from alibi_detect.base import BaseDetector + +logger = logging.getLogger(__name__) + + +class OutlierDetector(BaseDetector, ABC): + """ Base class for outlier detection algorithms. """ + threshold_inferred = False + + @abstractmethod + def fit(self, X: np.ndarray) -> None: + pass + + + @abstractmethod + def score(self, X: np.ndarray) -> np.ndarray: + pass + + + def infer_threshold(self, X: np.ndarray, fpr: float) -> None: + """ + Infers the threshold above which only fpr% of inlying data scores. + Also saves down the scores to be later used for computing p-values + of new data points (by comparison to the empirical cdf). + For ensemble models the scores are normalised and aggregated before + saving scores and inferring threshold. + """ + self.val_scores = self.score(X) + self.val_scores = self.normaliser.fit(self.val_scores).transform(self.val_scores) \ + if getattr(self, 'normaliser') else self.val_scores + self.val_scores = self.aggregator.fit(self.val_scores).transform(self.val_scores) \ + if getattr(self, 'aggregator') else self.val_scores + self.threshold = np.quantile(self.val_scores, 1-fpr) + self.threshold_inferred = True + + + def predict(self, X: np.ndarray) -> np.ndarray: + """ + Scores the instances and then compares to pre-inferred threshold. + For ensemble models the scores from each constituent is added to the output. + p-values are also returned by comparison to validation scores (of inliers) + """ + output = {} + scores = self.score(X) + output['raw_scores'] = scores + + if getattr(self, 'normaliser') and self.normaliser.fitted: + scores = self.normaliser.transform(scores) + output['normalised_scores'] = scores + + if getattr(self, 'aggregator') and self.aggregator.fitted: + scores = self.aggregator.transform(scores) + output['aggregate_scores'] = scores + + if self.threshold_inferred: + p_vals = (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) + preds = scores > self.threshold + output.update(scores=scores, preds=preds, p_vals=p_vals) + + return output \ No newline at end of file diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py new file mode 100644 index 000000000..d58fa2563 --- /dev/null +++ b/alibi_detect/od/knn.py @@ -0,0 +1,55 @@ +from typing import Callable, Literal, Union, Optional +import numpy as np +import os + +from alibi_detect.od.base import OutlierDetector +from alibi_detect.od.transforms import BaseTransform + +from alibi_detect.od.backend import KNNTorch +from alibi_detect.od.backend import KNNKeops +from alibi_detect.utils.frameworks import BackendValidator +from alibi_detect.saving.registry import registry + +X_REF_FILENAME = 'x_ref.npy' + +backends = { + 'pytorch': KNNTorch, + 'keops': KNNKeops +} + +@registry.register('KNN') +class KNN(OutlierDetector): + CONFIG_PARAMS = ('k', 'kernel', 'aggregator', 'normaliser', 'backend') + LARGE_PARAMS = () + BASE_OBJ = True + + def __init__( + self, + k: Union[int, np.ndarray], + kernel: Optional[Callable] = None, + aggregator: Union[BaseTransform, None] = None, + normaliser: Union[BaseTransform, None] = None, + backend: Literal['pytorch', 'keops'] = 'pytorch' + ) -> None: + backend = backend.lower() + BackendValidator( + backend_options={'pytorch': ['pytorch'], + 'keops': ['keops']}, + construct_name=self.__class__.__name__ + ).verify_backend(backend) + + self.k = k + self.kernel = kernel + self.ensemble = isinstance(self.k, np.ndarray) + self.normaliser = normaliser + self.aggregator = aggregator + self.fitted = False + self.backend = backends[backend] + + def fit(self, X: np.ndarray) -> None: + self.x_ref = self.backend.fit(X) + val_scores = self.score(X) + if getattr(self, 'normaliser'): self.normaliser.fit(val_scores) + + def score(self, X: np.ndarray) -> np.ndarray: + return self.backend.score(X, self.x_ref, self.k, kernel=self.kernel) \ No newline at end of file diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py new file mode 100644 index 000000000..4f53894d7 --- /dev/null +++ b/alibi_detect/od/tests/test_knn.py @@ -0,0 +1,100 @@ +import numpy as np +import os + +from alibi_detect.od.knn import KNN +from alibi_detect.od.transforms import AverageAggregator, ShiftAndScaleNormaliser, PValNormaliser +# from alibi_detect.od.backend import KNNTorch + + +def test_knn_single(): + knn_detector = KNN(k=10) + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + x = np.array([[0, 10]]) + assert knn_detector.predict(x)['raw_scores'] > 5 + + x = np.array([[0, 0.1]]) + assert knn_detector.predict(x)['raw_scores'] < 1 + + knn_detector.infer_threshold(x_ref, 0.1) + + x = np.array([[0, 10]]) + pred = knn_detector.predict(x) + assert pred['raw_scores'] > 5 + assert pred['preds'] == True + assert pred['p_vals'] < 0.05 + + x = np.array([[0, 0.1]]) + pred = knn_detector.predict(x) + assert pred['raw_scores'] < 1 + assert pred['preds'] == False + assert pred['p_vals'] > 0.7 + + +def test_knn_ensemble(): + knn_detector = KNN( + k=[8, 9, 10], + aggregator=AverageAggregator(), + normaliser=ShiftAndScaleNormaliser() + ) + + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + x = np.array([[0, 10], [0, 0.1]]) + knn_detector.infer_threshold(x_ref, 0.1) + pred = knn_detector.predict(x) + + assert np.all(pred['normalised_scores'][0] > 1) + assert np.all(pred['normalised_scores'][1] < 0) # Is this correct? + assert np.all(pred['preds'] == [True, False]) + + knn_detector = KNN( + k=[8, 9, 10], + aggregator=AverageAggregator(), + normaliser=PValNormaliser() + ) + + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + x = np.array([[0, 10], [0, 0.1]]) + knn_detector.infer_threshold(x_ref, 0.1) + pred = knn_detector.predict(x) + + assert np.all(pred['normalised_scores'][0] > 0.8) + assert np.all(pred['normalised_scores'][1] < 0.3) + assert np.all(pred['preds'] == [True, False]) + + +def test_knn_keops(): + knn_detector = KNN( + k=[8, 9, 10], + aggregator=AverageAggregator(), + normaliser=ShiftAndScaleNormaliser(), + backend='keops' + ) + + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + x = np.array([[0, 10], [0, 0.1]]) + knn_detector.infer_threshold(x_ref, 0.1) + pred = knn_detector.predict(x) + + assert np.all(pred['normalised_scores'][0] > 1) + assert np.all(pred['normalised_scores'][1] < 0) # Is this correct? + assert np.all(pred['preds'] == [True, False]) + + knn_detector = KNN( + k=[8, 9, 10], + aggregator=AverageAggregator(), + normaliser=PValNormaliser() + ) + + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + x = np.array([[0, 10], [0, 0.1]]) + knn_detector.infer_threshold(x_ref, 0.1) + pred = knn_detector.predict(x) + + assert np.all(pred['normalised_scores'][0] > 0.8) + assert np.all(pred['normalised_scores'][1] < 0.3) + assert np.all(pred['preds'] == [True, False]) diff --git a/alibi_detect/od/transforms.py b/alibi_detect/od/transforms.py new file mode 100644 index 000000000..0a3ec7939 --- /dev/null +++ b/alibi_detect/od/transforms.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import logging +import numpy as np + +from typing import Optional +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) +from alibi_detect.saving.registry import registry + + +class BaseTransform(ABC): + fitted = False + + def fit(self, X) -> BaseTransform: + if not self.fitted and hasattr(self, '_fit'): + self._fit(X) + self.fitted = True + return self + + def _fit(self, X): + pass + + def transform(self, scores): + if not self.fitted: + raise Exception('Transform not fitted, call fit before calling transform!') + return self._transform(scores) + + @abstractmethod + def _transform(self, scores): + pass + + +class PValNormaliser(BaseTransform): + def _fit(self, val_scores: np.ndarray): + self.val_scores = val_scores + + def _transform(self, scores: np.ndarray) -> np.ndarray: + p_vals = ( + 1 + (scores[:,None,:] < self.val_scores[None,:,:]).sum(1) + )/(len(self.val_scores)+1) + return 1 - p_vals + + +class ShiftAndScaleNormaliser(BaseTransform): + def _fit(self, val_scores: np.ndarray) -> BaseTransform: + self.val_means = val_scores.mean(0)[None,:] + self.val_scales = val_scores.std(0)[None,:] + + def _transform(self, scores: np.ndarray) -> np.ndarray: + return (scores - self.val_means)/self.val_scales + + +class TopKAggregator(BaseTransform): + def __init__(self, k: Optional[int]): + self.k = k + self.fitted = True + + def _transform(self, scores: np.ndarray) -> np.ndarray: + if self.k is None: + self.k = int(np.ceil(scores.shape[1]/2)) + return np.sort(scores, 1)[:, -self.k:].mean(-1) + + +class AverageAggregator(BaseTransform): + def __init__(self, weights: Optional[np.ndarray] = None): + self.weights = weights + self.fitted = True + + def _transform(self, scores: np.ndarray) -> np.ndarray: + if self.weights is None: + m = scores.shape[-1] + self.weights = np.ones(m)/m + return scores @ self.weights + + +class MaxAggregator(BaseTransform): + def __init__(self): + self.fitted = True + + def _transform(self, scores: np.ndarray) -> np.ndarray: + return np.max(scores, axis=-1) + + +class MinAggregator(BaseTransform): + def __init__(self): + self.fitted = True + + def _transform(self, scores: np.ndarray) -> np.ndarray: + return np.min(scores, axis=-1) \ No newline at end of file From 4a4d19d00b5c9fb9962288d29f26889a51660240 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 14 Oct 2022 11:20:39 +0100 Subject: [PATCH 002/247] Add transforms --- alibi_detect/od/base.py | 21 ++++--- alibi_detect/od/knn.py | 16 ++--- alibi_detect/od/tests/test_knn.py | 78 +++++++++--------------- alibi_detect/od/tests/test_transforms.py | 52 ++++++++++++++++ alibi_detect/od/transforms.py | 64 ++++++++++++++----- 5 files changed, 148 insertions(+), 83 deletions(-) create mode 100644 alibi_detect/od/tests/test_transforms.py diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 12d5f7ea1..311c93e00 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -1,7 +1,6 @@ from __future__ import annotations from abc import ABC, abstractmethod import numpy as np -from alibi_detect.version import __version__ import logging from alibi_detect.base import BaseDetector @@ -11,17 +10,21 @@ class OutlierDetector(BaseDetector, ABC): """ Base class for outlier detection algorithms. """ threshold_inferred = False + ensemble = False + + def __init__(self): + super().__init__() + self.meta['online'] = False + self.meta['detector_type'] = 'outlier' @abstractmethod def fit(self, X: np.ndarray) -> None: pass - @abstractmethod def score(self, X: np.ndarray) -> np.ndarray: pass - def infer_threshold(self, X: np.ndarray, fpr: float) -> None: """ Infers the threshold above which only fpr% of inlying data scores. @@ -31,14 +34,14 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: saving scores and inferring threshold. """ self.val_scores = self.score(X) - self.val_scores = self.normaliser.fit(self.val_scores).transform(self.val_scores) \ - if getattr(self, 'normaliser') else self.val_scores - self.val_scores = self.aggregator.fit(self.val_scores).transform(self.val_scores) \ - if getattr(self, 'aggregator') else self.val_scores + if self.ensemble: + self.val_scores = self.normaliser.fit(self.val_scores).transform(self.val_scores) \ + if getattr(self, 'normaliser') else self.val_scores + self.val_scores = self.aggregator.fit(self.val_scores).transform(self.val_scores) \ + if getattr(self, 'aggregator') else self.val_scores self.threshold = np.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True - def predict(self, X: np.ndarray) -> np.ndarray: """ Scores the instances and then compares to pre-inferred threshold. @@ -62,4 +65,4 @@ def predict(self, X: np.ndarray) -> np.ndarray: preds = scores > self.threshold output.update(scores=scores, preds=preds, p_vals=p_vals) - return output \ No newline at end of file + return output diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index d58fa2563..b370fea4f 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,6 +1,5 @@ from typing import Callable, Literal, Union, Optional import numpy as np -import os from alibi_detect.od.base import OutlierDetector from alibi_detect.od.transforms import BaseTransform @@ -8,7 +7,6 @@ from alibi_detect.od.backend import KNNTorch from alibi_detect.od.backend import KNNKeops from alibi_detect.utils.frameworks import BackendValidator -from alibi_detect.saving.registry import registry X_REF_FILENAME = 'x_ref.npy' @@ -17,12 +15,8 @@ 'keops': KNNKeops } -@registry.register('KNN') -class KNN(OutlierDetector): - CONFIG_PARAMS = ('k', 'kernel', 'aggregator', 'normaliser', 'backend') - LARGE_PARAMS = () - BASE_OBJ = True +class KNN(OutlierDetector): def __init__( self, k: Union[int, np.ndarray], @@ -31,6 +25,7 @@ def __init__( normaliser: Union[BaseTransform, None] = None, backend: Literal['pytorch', 'keops'] = 'pytorch' ) -> None: + super().__init__() backend = backend.lower() BackendValidator( backend_options={'pytorch': ['pytorch'], @@ -40,7 +35,7 @@ def __init__( self.k = k self.kernel = kernel - self.ensemble = isinstance(self.k, np.ndarray) + self.ensemble = isinstance(self.k, (np.ndarray, list)) self.normaliser = normaliser self.aggregator = aggregator self.fitted = False @@ -49,7 +44,8 @@ def __init__( def fit(self, X: np.ndarray) -> None: self.x_ref = self.backend.fit(X) val_scores = self.score(X) - if getattr(self, 'normaliser'): self.normaliser.fit(val_scores) + if getattr(self, 'normaliser'): + self.normaliser.fit(val_scores) def score(self, X: np.ndarray) -> np.ndarray: - return self.backend.score(X, self.x_ref, self.k, kernel=self.kernel) \ No newline at end of file + return self.backend.score(X, self.x_ref, self.k, kernel=self.kernel) diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index 4f53894d7..08117e217 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -1,9 +1,9 @@ +import pytest import numpy as np -import os from alibi_detect.od.knn import KNN -from alibi_detect.od.transforms import AverageAggregator, ShiftAndScaleNormaliser, PValNormaliser -# from alibi_detect.od.backend import KNNTorch +from alibi_detect.od.transforms import AverageAggregator, TopKAggregator, MaxAggregator, MinAggregator, \ + ShiftAndScaleNormaliser, PValNormaliser def test_knn_single(): @@ -21,21 +21,23 @@ def test_knn_single(): x = np.array([[0, 10]]) pred = knn_detector.predict(x) assert pred['raw_scores'] > 5 - assert pred['preds'] == True + assert pred['preds'] assert pred['p_vals'] < 0.05 x = np.array([[0, 0.1]]) pred = knn_detector.predict(x) assert pred['raw_scores'] < 1 - assert pred['preds'] == False + assert not pred['preds'] assert pred['p_vals'] > 0.7 -def test_knn_ensemble(): +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliser, PValNormaliser, lambda: None]) +def test_knn_ensemble(aggregator, normaliser): knn_detector = KNN( - k=[8, 9, 10], - aggregator=AverageAggregator(), - normaliser=ShiftAndScaleNormaliser() + k=[8, 9, 10], + aggregator=aggregator(), + normaliser=normaliser() ) x_ref = np.random.randn(100, 2) @@ -44,32 +46,22 @@ def test_knn_ensemble(): knn_detector.infer_threshold(x_ref, 0.1) pred = knn_detector.predict(x) - assert np.all(pred['normalised_scores'][0] > 1) - assert np.all(pred['normalised_scores'][1] < 0) # Is this correct? - assert np.all(pred['preds'] == [True, False]) - - knn_detector = KNN( - k=[8, 9, 10], - aggregator=AverageAggregator(), - normaliser=PValNormaliser() - ) - - x_ref = np.random.randn(100, 2) - knn_detector.fit(x_ref) - x = np.array([[0, 10], [0, 0.1]]) - knn_detector.infer_threshold(x_ref, 0.1) - pred = knn_detector.predict(x) - - assert np.all(pred['normalised_scores'][0] > 0.8) - assert np.all(pred['normalised_scores'][1] < 0.3) assert np.all(pred['preds'] == [True, False]) + if isinstance(knn_detector.normaliser, ShiftAndScaleNormaliser): + assert np.all(pred['normalised_scores'][0] > 1) + assert np.all(pred['normalised_scores'][1] < 0) + elif isinstance(knn_detector.normaliser, PValNormaliser): + assert np.all(pred['normalised_scores'][0] > 0.8) + assert np.all(pred['normalised_scores'][1] < 0.3) -def test_knn_keops(): +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliser, PValNormaliser, lambda: None]) +def test_knn_keops(aggregator, normaliser): knn_detector = KNN( - k=[8, 9, 10], - aggregator=AverageAggregator(), - normaliser=ShiftAndScaleNormaliser(), + k=[8, 9, 10], + aggregator=aggregator(), + normaliser=normaliser(), backend='keops' ) @@ -79,22 +71,10 @@ def test_knn_keops(): knn_detector.infer_threshold(x_ref, 0.1) pred = knn_detector.predict(x) - assert np.all(pred['normalised_scores'][0] > 1) - assert np.all(pred['normalised_scores'][1] < 0) # Is this correct? - assert np.all(pred['preds'] == [True, False]) - - knn_detector = KNN( - k=[8, 9, 10], - aggregator=AverageAggregator(), - normaliser=PValNormaliser() - ) - - x_ref = np.random.randn(100, 2) - knn_detector.fit(x_ref) - x = np.array([[0, 10], [0, 0.1]]) - knn_detector.infer_threshold(x_ref, 0.1) - pred = knn_detector.predict(x) - - assert np.all(pred['normalised_scores'][0] > 0.8) - assert np.all(pred['normalised_scores'][1] < 0.3) assert np.all(pred['preds'] == [True, False]) + if isinstance(knn_detector.normaliser, ShiftAndScaleNormaliser): + assert np.all(pred['normalised_scores'][0] > 1) + assert np.all(pred['normalised_scores'][1] < 0) + elif isinstance(knn_detector.normaliser, PValNormaliser): + assert np.all(pred['normalised_scores'][0] > 0.8) + assert np.all(pred['normalised_scores'][1] < 0.3) diff --git a/alibi_detect/od/tests/test_transforms.py b/alibi_detect/od/tests/test_transforms.py new file mode 100644 index 000000000..356c22930 --- /dev/null +++ b/alibi_detect/od/tests/test_transforms.py @@ -0,0 +1,52 @@ +import numpy as np\ + +from alibi_detect.od.transforms import PValNormaliser, ShiftAndScaleNormaliser, TopKAggregator, MaxAggregator, \ + MinAggregator, AverageAggregator +import pytest + + +def test_p_val_normaliser(): + p_val_norm = PValNormaliser() + scores = np.random.normal(0, 1, (1000, 3)) * np.array([0.5, 2, 1]) + np.array([0, 1, -1]) + + with pytest.raises(ValueError): + p_val_norm.transform(np.array([[0, 0, 0]])) + + p_val_norm.fit(X=scores) + + s = np.array([[0, 0, 0]]) + np.array([0, 1, -1]) + s_transformed = p_val_norm.transform(s) + np.testing.assert_array_almost_equal( + np.array([[0.5, 0.5, 0.5]]), + s_transformed, + decimal=0.1 + ) + + s = np.array([[0, 0, 0]]) + s_transformed = p_val_norm.transform(s) + np.testing.assert_array_almost_equal( + np.array([[0.5, 0.32, 0.85]]), + s_transformed, + decimal=0.1 + ) + + +def test_shift_and_scale_normaliser(): + shift_and_scale_norm = ShiftAndScaleNormaliser() + scores = np.random.normal(0, 1, (1000, 3)) * np.array([0.5, 2, 1]) + np.array([0, 1, -1]) + shift_and_scale_norm.fit(X=scores) + s = np.array([[0, 0, 0]]) + np.array([0, 1, -1]) + s_transformed = shift_and_scale_norm.transform(s) + np.testing.assert_array_almost_equal( + np.array([[0., 0., 0.]]), + s_transformed, + decimal=0.1 + ) + + s = np.array([[0, 0, 0]]) + s_transformed = shift_and_scale_norm.transform(s) + np.testing.assert_array_almost_equal( + np.array([[0., -0.5, 1.]]), + s_transformed, + decimal=0.1 + ) diff --git a/alibi_detect/od/transforms.py b/alibi_detect/od/transforms.py index 0a3ec7939..1b8b6396d 100644 --- a/alibi_detect/od/transforms.py +++ b/alibi_detect/od/transforms.py @@ -7,46 +7,80 @@ from abc import ABC, abstractmethod logger = logging.getLogger(__name__) -from alibi_detect.saving.registry import registry class BaseTransform(ABC): + """Base Transform class. + + provides abstract methods for transform objects that map a numpy + array. + """ + def transform(self, X: np.ndarray): + return self._transform(X) + + @abstractmethod + def _transform(self, X: np.ndarray): + """Applies class transform to numpy array + + Parameters + ---------- + X + numpy array to be transformed + + Raises + ------ + NotImplementedError + if _transform is not implimented on child class raise + NotImplementedError + """ + raise NotImplementedError() + + +class BaseFittedTransform(BaseTransform): + """Base Fitted Transform class. + + Provides abstract methods for transforms that have an aditional + fit step. + """ fitted = False - def fit(self, X) -> BaseTransform: + def fit(self, X: np.ndarray) -> BaseTransform: if not self.fitted and hasattr(self, '_fit'): self._fit(X) self.fitted = True return self - def _fit(self, X): - pass + def _fit(self, X: np.ndarray): + raise NotImplementedError() - def transform(self, scores): + def transform(self, X: np.ndarray): if not self.fitted: - raise Exception('Transform not fitted, call fit before calling transform!') - return self._transform(scores) - - @abstractmethod - def _transform(self, scores): - pass + raise ValueError('Transform not fitted, call fit before calling transform!') + return self._transform(X) class PValNormaliser(BaseTransform): + """Maps scores from an ensemble of detectors to there p values. + + Needs to be fit on a reference dataset using fit. Transform counts the number of scores + in the reference dataset that are greter than the score of interest and divides by the + size of the reference dataset. Output is between 1 and 0. Small values are likely to be + outliers. + """ def _fit(self, val_scores: np.ndarray): self.val_scores = val_scores def _transform(self, scores: np.ndarray) -> np.ndarray: p_vals = ( - 1 + (scores[:,None,:] < self.val_scores[None,:,:]).sum(1) + 1 + (scores[:, None, :] < self.val_scores[None, :, :]).sum(1) )/(len(self.val_scores)+1) return 1 - p_vals class ShiftAndScaleNormaliser(BaseTransform): def _fit(self, val_scores: np.ndarray) -> BaseTransform: - self.val_means = val_scores.mean(0)[None,:] - self.val_scales = val_scores.std(0)[None,:] + self.val_means = val_scores.mean(0)[None, :] + self.val_scales = val_scores.std(0)[None, :] def _transform(self, scores: np.ndarray) -> np.ndarray: return (scores - self.val_means)/self.val_scales @@ -88,4 +122,4 @@ def __init__(self): self.fitted = True def _transform(self, scores: np.ndarray) -> np.ndarray: - return np.min(scores, axis=-1) \ No newline at end of file + return np.min(scores, axis=-1) From e7298551681b9f24de80c2dbefc2e142fef3d3cf Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 14 Nov 2022 11:28:16 +0000 Subject: [PATCH 003/247] Minor progress commit --- alibi_detect/od/knn.py | 6 ++---- alibi_detect/od/transforms.py | 10 ++++++++++ test/detectors/0/model.pt | Bin 0 -> 8346 bytes test/detectors/1/model.pt | Bin 0 -> 8382 bytes 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 test/detectors/0/model.pt create mode 100644 test/detectors/1/model.pt diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index b370fea4f..4f1f40cd1 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -21,8 +21,7 @@ def __init__( self, k: Union[int, np.ndarray], kernel: Optional[Callable] = None, - aggregator: Union[BaseTransform, None] = None, - normaliser: Union[BaseTransform, None] = None, + accumulator: Optional[]=None, backend: Literal['pytorch', 'keops'] = 'pytorch' ) -> None: super().__init__() @@ -36,8 +35,7 @@ def __init__( self.k = k self.kernel = kernel self.ensemble = isinstance(self.k, (np.ndarray, list)) - self.normaliser = normaliser - self.aggregator = aggregator + self.accumulator = accumulator self.fitted = False self.backend = backends[backend] diff --git a/alibi_detect/od/transforms.py b/alibi_detect/od/transforms.py index 1b8b6396d..46937f4c0 100644 --- a/alibi_detect/od/transforms.py +++ b/alibi_detect/od/transforms.py @@ -78,6 +78,10 @@ def _transform(self, scores: np.ndarray) -> np.ndarray: class ShiftAndScaleNormaliser(BaseTransform): + """Maps scores from an ensemble of to a set of . + + Needs to be fit on a reference dataset using fit. + """ def _fit(self, val_scores: np.ndarray) -> BaseTransform: self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] @@ -123,3 +127,9 @@ def __init__(self): def _transform(self, scores: np.ndarray) -> np.ndarray: return np.min(scores, axis=-1) + + +class Accumulator(BaseFittedTransform): + def __init__(self, normaliser: BaseFittedTransform, aggregator: BaseTransform): + self.normaliser = normaliser + self.aggregator = aggregator diff --git a/test/detectors/0/model.pt b/test/detectors/0/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dd47af59f0e47e47ed73d4071f039515e11868a GIT binary patch literal 8346 zcmbt32|Sct_hU`g>>)duEHh-uHuKmO*|(^SeH}Aqv1BPic3GnATMD5KA?A@S%9^55 zLM25=B})8fs<&6Y_4fb1@A}=D@jUmObI*S6x#q?+v=9gb1LPlr8Nvhc!@9Yn;cm`E zXSlq94hO{HAA|85oVoE9hT`_AcN9dFY5p&Gk(==<80e_~0`ETv{pLY%?)*VA6StJUm*7 zTtk2WHMb?n+o%Xhcg_KXyNS0ZSxEtxjSmB{>j5HhCV2J*GOE7+3o8B7$tLR#VzK}v_@1DP+ffvaX+B)^+Eq&1Zj;Li3saGU2t zkoxK>kQPi08W>NIvJV!4d0PzurBnUj>HGGi{RnrEXL}b?q=OoKFLWC89iMKtFdaj} z)Z-A@k8_X2f^MqbI3Y9FYro- zHKL|W9pK}lMNW;T0bQ&GVBpc+gfFXmd zrq}6b0P;N*GI^*E$<6czaEz(~wF^r?7wKSN2ATz)>RtiHJBLY%_5k=nY#BHn*#gLt z^^~~e3G6#^2&4{w2ej;oCw2Ro00PJGz=tLwlBtp)(87%d9yGNgr5lsL>MLiEG&d^% z#Q_DdSB;N^?;R&q*UkaQ1!RGdf4K?1N+s72(JhmJhvYsPpwBIHoC@sLNC34b(!sa8CP+Kh*GV%UmjNUT0bn=1ir^T_ z17)EH0M;!LAlvm$@DTeMz{H>g>Dl57=*yQPwd@Z9`4u}s4wq=~V;Bt3t}+2_PsoEQ z8ZSvi`IDNLk#!``^^2ffhXjb&&)O=(CIYB$>jSZel0lXWHVB9{0*s981Z=~~z-LWy zNV*pyEj-bgVBmW?V6*~GVn3x1$P8Hm0eD6&d(I00#V={>-0VeYAP~y@ySf$=Hl$?>yB|Fz`iVi5u5|P zF`kTiU*2Q!j5x@y$Z+TZO1L26oxL&SmpH1p13cvaQQicUM*td)qM!(J#=D`suvlLl zwdny4%4#pXJHZQ!c0+k%JlyekcQ+hO{DG~T-{5GCAqU8j4T5#?A>imXZSw10Fc=s) zxCNly2{2bIhDfIFj%PIa@?^g^fyjuXr|iv(D$a{clZuQ-9=Bx^l0R_ji@`7&o0?iv zT%y?HWY==B#712oo|_?-&J7v2?coL(%( z-DDbE6dG>E)MP5u6gpmC(Q$th6h)O}KHU9W(C#>%_y89iFL`b!Pd*sM5*f!hsHDk) zpiGJ-a*;C+1?I| z{LF8dw#zfOyyYG-t+p<~S1XzB&_1MUos*iW($AkIX_el#@bs*pz$L4>4{MGE2nRlK zO^XtRRr8i$<%5;iH6R@s%uC(-i}gyhrw+UuGJYORV?k6aj34P-eBg8@5qyCeMxRL-Z=I&pB4@wsSt1NN6QR#x&vb?yUk= zv<6QGWgBY-R!7|gTeo;*m3^pqH8w3~zm%GM_b}TBzn9cyYN{vZNmLZ|iBxLt*hO{* z(|=W;Aw=wdQy=T}-FA+%S4~h&(y=~O4#dSie;yuU31`Bq{28EKHq1qO2ZYeM- zPoO1gd*b<$_et?xfYAoMWjsDG5w#*jIvy(JS6p_ty)~P#zOb=n`tbS0=+ul0IiVA) zdtQuZAA5bTA8r=DFtIpwhBHw*+dsLc(j!o&qdV!!T4O=1ZREaM_dCggQM1HqHfdvR zkCBEw_C8s8)$=jbGQ=Yg*~FadlIS6psFUWt5l$9TIvr{}xU6WTv3M_m*~Tdo!@3nM zec8PD?jwOk{;*Ru%(!RJf~8tlrIqdZ!cOwL%n^G`Pp|K>iZUByD$~F%$C)B-@V;O; zc*d>IaDR-$CHgT|3#UQBwfWQ-(^4T#Cw`uk?j@1HgpkCtpuo480>{l9w2m*2lRVp& zb7h0~4ti-`UiOTO=5Cv?8mW#cp{I>|HvIZr%BORkWz_sL=8^2>ZP26}s;TMu5p+GC ztx*D7bBI7~gXh}F6kkIWYHJhm*vHJu@$6-3Cc z)?;L=ApWaW?1qT`+A6;KE(Qbt7LVcI$>BG0j_k*Qm9c&5IQUPA4!t zxdg}#J14yHbBjrrP}A8GS?vstny`>fiKO0;8a=^Ih zRdYgJVPk{mJGrnmEsoq$8m{^ZhOp{+iWJp`E%$VilD`j)nh^H?LW-LmiseUn zB5rB>wrOpnn|R=LP#|Qs>~z*^xfa?GxiY6>8@|$QH1qoONf+!7mt<50VLx3vj9_3B zYF@|}druA77}lKSRTAdb&Ea14+Few#p|S3|*V@UcxInl1odbvcVNS{F`q`5g&xpV0 z(ldJC6@>O#jKgDkkJ;a`KpxDhOTyQKAtACE+jLa z_&_!4<1b=;W85sQd9@?i+d4I%njaRqpJ)b#FT*Ow9=t6z8=wfzP4YMsi7dFg7qSe^G{Bvqovd)y2d1sOK%YAI)sG>>;A!Yrp?@7^$z8%F>DeQ^P@LHH0o@v zfpbLDArJJJWbLc9N|y^vs_2*Ois}RPbKJ?-QjTtIWTCy8X)z7&rggjI6%l}x)f(plM7in%5suO|gj9<8(x>4{*d%M3Vc zC?1~A@v8q!$ns%!Zr7Xk9zCRuGX-POs?4Q|vUN~jcvbtA*(#1{rI47ad=#i3WXVgxp5I+F)W8Uml!M;&Yla%_nj+{PdU68b-ro;foPKohP6 zhSSJ#R~?*%n)$JcPsdiZJZd-aVAR)(i2-4Q{`BuBq*9yiJjbg|%&a7Itq zKQ0?nPRB;23qK@IG8YmV3e!sl6V+=@o3s1+ja+iEoSL5;aXcKPaMreyqwDPHQs-(H zEh*c5y_JvUudMlB%qc9q7p_4vJDzS7+0U>sh?!g`eu6u4w*fOawJ!;!X#)qorzfvJB;^)cT7`JDi6k+?*qC zi0Ja(T_UX>yr~I^lBaE#r!FlDaLtg-Ii4Y=(lJ=zP;gxHkh6TRxm6r&Rh-wuGSA6> zdy+g+RWgmR+djP~VZYYSAlf6$W_(Gz-|da#UCfH?8syB@9@9^i{@0?wBLMT|2;+?Y5`%x$f67ej!9-~>Yb~rYCUT}h z8I=;FPmM3p*xb9If1^Ch=|sO}S@p*Sk8`peC2n;SU7tF& z_@2eLx~wpGSa5EZ(=;M?c75%XVhu#=gUV}`pqGT6tiVqRAvYa5BgMvRhzhXuo*geN zpX9z6xqRj2to*Bdg@<{!)oqVIXn;2~CehkmQF+#EftUaEMK`y1=yJ)EW82L?6m0Up zj%~lt&E`lZ`yWNJOj~2DfFZ{@ZYg%D%bYhCf^x5z+!10$T@mUvifF-cln##{uHN>+ zfXATEd3f%eDzbOCx> zA2~E9S7kA{7o+Z3fak=qWgau#`0I{l_^Y$^D@G*VHmgze^~j#kEIEgB4lW7xFZ~l^ zMVaVVU3A}i&d4psPvq$0C*IIYR}rL$G=j+seq-*IhQ-VY<+bV~yviGD4~42(@Qsfy z#;%C^?Dh@_X6j)zbnL-J!7T6hURL$NH?}uMsa{p@b`{gYXsDdhfX%hmH%i9LM!yP} zm@o26eJrA+v>}h6b;{Y$yk}5!Ey_@IZi4RP@=Q$A$9r3E&~vcL>+I&qym_fzGll=~ zPL1ShjhSMu!xNvz-q1Xt&f^`P_ZgVO$UVnot-~h!eMQ*(6tCZ8&+FwBmk?#`tq)Pj zu$tuW5=+UM_Iz0vjS5H(*5qP+SU$rr@27U(dWTYov~zARC&LqC>-}o#=JQ1y583OJ zTY0GbMLw2~T`gNgJ1Yc4#~pE;kRI<5^j^RH?)|Q2QF_mQ%r-_L2kkmB4nNg=8|&*a zPEUQ`WK0VekKEX9ws&`EEK8f`i;z>(Y`r#bQN{V*j&h^8vPL$ISnS)02R$uD0#^Rm zMU|*B7GeZVbgin&t|y^>HhTnl=24TNZs3V@19LZgqV`l1y|-V zS7+B!7g@XMNv0TSgR-XPOBJQbmYwBxGRErC_mWmu9m3yXj#iwTt2B5%vNXNZ`i^}F z@gX=}SZc=!my-8lOPMY-%TBo{)#|$*g9s<0bKGp=y$_$onhg2OKUP_(fx+%l(H@eu zy%~DwfTKfP%ROG1y8B~gh+MSK`g18P{-RZ%A=n)TAK_Nt6Qk^g=%qVX>Y=G0;hr*G zpjU*kzHmDNr6_&de=h_$y*saUl(}e&XpMmJdj}IfM-yi$W?$m~sGMp>=X0OlCH{Ot z`!?0$)zJ#l!Sq3F@0qJ={WwA{-SN6;<(#OyN*U)zIS){sZfT7yGrc`+b%7}dRTOkS zu5)s5=)UOM{++T|pH?GAt(LuZL5({DyxgBV30pfI6ITvNx}Up1^4^~us@}Aq-rS&Z zCEie8cadeMyZP}q=Bmwg6{~x^!=I&$ZSgFNo2ls5Ox_!S;*o4xv*t1TtVhlJwDpkB zC;Tom4?|3BWtz>5!yPrwMfRh1$(}(`kR?+7CqnuH%H|+l{ow1O;Y3@PbmN@oBEpsL zs8nG=k6`e2!(!BlX_)M&_vMgmjWN+?E)%M|63hYa!LU%82RK1!E*>fYKhhi0EA-~l zj`My;^6Ce>TOv^zM_VEkM=r%}K+igdw1uDv)ekrWZ*!&8#(YHf@~SW$s+nZ&*ITb% zIk!H(0Hs4ldoB`IL97p+tZy3IjS=^AGNtqm-_@Ii6G@G(oB=WBh zOk7>rfjkMHNL=9{C9#u^6M|V$oui8Zc{k|}b?jL*sd~zx;?HO2Z;)2bvEly=zO5wf zlSA}L!*oM~AcJ?5GMY2ewqV^&j^DvqHru?>r>Uaq@&O-0)ckbKDd69@jog*~pGf^(5&cYb zbE9YX4@Aw#m#M!b`mZgrpYd<*hiLwQKa=Ud!T)6e`SqALEGkIfP1N6kaH{{L8i-u2&}zBvYQ z{;+F$fxq!@ieg_k_&0*hVT9!e1lHdXd<$IW#`N^E-)QVVKE&o* literal 0 HcmV?d00001 diff --git a/test/detectors/1/model.pt b/test/detectors/1/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b45292aa63f464e4355baae8780c4165c43c497 GIT binary patch literal 8382 zcmb_h2|U!>+aJ5E8C$7jC(8_3vd&MIA!A>oGRD4)8Dn3IP_`68_AP~Ir-=Eftl24w zvQ;Ruwh-~oaBo-Ly7&FR|M$%2%<`M_JkL4L@;%Rajxmag8UmrGhy1NELAW5Ectaj8Sy-z++uSG9z*~{s+E@?R;gB8p^-m`!_+qc@Wq>2Dpy3 zBT^EBfi`zmbs?rXU>p_Pdg+ou84ORuOgYP?3A z7$`%mseu3`rzz1*FCRg3#Rh~s3f3o>i2}D!CxEl-YGA1*n|g^@4XEp>qQ+MN5ceKz z0G8Nmfw(eWM0D;$#0PB$urBT!7|(y3xVW1SAs(Cqq)ui4cZ`~dp3RxWHH9;v$m2?I zr^_Hnd1nPk3Zw*)s8M3Zv3xLlhYlcju@g*xZcWrza{{?`H6i#KD8Wy>>7d8(M6HS8 z5CWzgqn7b16ESBm04h{h6KzE#K^yM-#AD*d;M319LD?Jf2$iZ1qE%Tp*uF4>sMK-= zZ#S5$T`yJ!c5_f8#s`vsCYD^VdZPnz&j=1)n6U* zp_f#3O@9L*K2adf^>iRO85aQC$a3&-ULj~N9tcc{rGXb)mVx2MKBBBO0Dckp44jUr z10+dyieGdAjszVCDZ@Sjb%$b!EuIFzo>Shyml|H8q1-;8juQ*KsHsPYS0{iKw=W^6 zno9xME*Y?0X*bckeVABrcLq4UM-u4I?M5i>)&^CX7yz~cN$`;!rCL{$0Qf*$3h>Kf z2ekDqh>j1RAPDc`!LwsTaOuk^!mnH$flU$syrXxiyS5U5``6t-$FGUNReTB9V?IMX z{3yPz+i(atT!H|Dlgz;2g4ViwFAjiN@)v-EDskZ9(PVJxz$kI=`Z{sy>t_JL>;teF z-cf@NWrLDpM*$YPaFF#uBY2$c5@3KVM7Y#>0NT<;2zBdYKu)O$2(^y_zlOqq!{r8` z9f>Ju<`-QJ3B!9@pB;aH48O}xf&P|(FjelSn0CHSXSvgwsQ9&T&{`3~N2;=LVQ(P$V3>t^qUc69P}a`1)W9bxtu2M;HlqYrFz z0L%yD?}l@Mq5mZC#Nb@8PH1TctxXENHv<83ARcO}y)kY$(r^Mr%uz1VZ?u~a z+Sw0_MU$BZV7wjCu6Vo$fzt3Olswwi+sVfjk99=5;hdeky`3BhRIx{QZ2yNqje;B{ zVI2f-@9smO**5L1+2L?765RP=oqS*pc$_aux|279!R8ZvHy>XH0xfxNt`sq@B$*T> zKGFobZA|`U?H)KB1Ip0QoUADZ0{!MPDM?H`dCOZ;SCT;2 zKZPK%c#Q9NDbUTe%MjRInNcky860F8jwni!6iTuj*DZFOe<+G82aKR`}2e-0lG(OTYfrgV)2KGDBFqI7(RiId53%XFhSJ3lAJi%oBXADzJBSIjIMi zyvWM2*9w_I!829<%uP4wVoLw>h4qgeIrTk*W^YDdhpiufl3{lKFx7J2lF?kGMj2|A zj)rMGiHdnX+t*BIpD=hnMl!E`mw#uq+H}PZ8D=E~{9<7!lp&d~0%gKrWP?+?Z<(W! zw8zf!=EaPJ`S|KdG~v*hYoqNQ`8zzS{4KRlg;zxc92~BJ^5&*_q-5;<3VC-dZFYBq zvIZ3m?`a9B%|qgF`W=gLhH_LD5+`w~RK-uK^PWDJSdfnFjjs}tPXJ#146ySnqsw6 z_3tAOS;($(+B+hx*N?YpUaJ=xlz;a*;-O!SJauX@pX@atLsA~qY57GXg0luenWo+G z7+N(|^-#1#UgKs7H~;wFctw;(UNPH*(iw5DjBeq!F=$Ua{kG4!<}fS*}Z@=Y5IlzPwu6NcH+27yOannr?UT zs!DnJl;#slrO^8Hs5sv*XH&}%^Ylca51X5dK%hWF7|9oW|Irr?LIi*93tMrD!@>W+ zUif$H-Hg%w|F!{HoL5VvlwpbvF~pc0zSV7AwV4-i+5k~A7umwSke)rlEO+0Br@ z)D@EVE%b`b82a(W6}ZKa>J!gwsZ(OcIp2zU z&lRyp`z3y4ZsN)qW%ERbWK7l*C4czkvjy+EP&Mys<0|v2@4I}I3SCo&W))F!RF%?) zR?L!PN~zEEP$LQT1W3C4JrJrw*!~ZC+#WW}KWbF%$zhL2>N{yhU$`FI6FgmcYt|(QCbmM5m`I9NHlyBJK{6OFiH{go(BPoDLNb~H8TJ{-aJs8K~s z^~*fxh^l|sXIRc3ZdiUtyG|`UVq4?=94zZ#BRA&YiclzEHO%^S8Rrj-)4JMU_UIhJ+A{ z3u?(4K65YDy#gx@YdOJ+z4B>8_yh{3mkYd9%B=PNGropB&e$R0yYJV^?5{B@V&AdJ zDtA@QaGtxDcydQIGj(&S$ppNS$HVWkIq#r}hwJ^Rb%N+v$6(?-plRIi6&4qpl&N?V zFK6o^oOG|hvFuS1?w*jemMBDipv;7?HJrXO)$g*7U|0_Hednd%&nMV89h$A3TZtQ& za)+W6nTlj3E5$tE<&VdY-tsoRK!tw3zh7?V*%aGX-WK?%*xh8lRMvbOX|e8**#yZq+wb-!E0$UyYR4jmzt-j_b4TE$q$z_NY9e0Y%Yu zaXEP@tt~sMbA!6f|KXA{Fpmrgh|oVmb*T^I;}@MdTYA4T&?>9>9t`lH&KipEl(XM8 zccf%(6V^twaQRP3p}3orG5+4Gv>?j=g;#A4ONHO~*LO$z&6lvGHjk&qyBbq65PgRG zPaoyr&$pu;OsK56DXp>Ja(dFDl*+anD%-kG)KDZtDB!*CBoc<>+`q@<`1>c(dp4*V z&vh5x_kUXzd8{tLwl1=JC%q`>)A^0~2E))L7I)h}j7o~L{MdAKTYtx+rmvMKON`sf z`$k1Msl<3(!)OKrl!bsSBsGtcFr=99mD&|!V=|r`4AH4qCV9Q>t*@dn?Qvy*iq|O#Y6JR_{ik_gT7--dA{^{$%WnV z%GcA4**rY^Z`zxV&yMxmo(Pb+Y}p8Hx}08wsjycUwLH>Z_DcHpn%9lYyu4>&DnuP< zCi>2|sCgn=sO|eZVI$*=UwDd7`m}wX7OJj$?va}{^0d$-^mTYB_*w6R*aqKan~$-E z%S_*RO-^MFxGy(sfcLxHlq+iL6aj^X?p&MP)2hcY((T4(F|ZXuZfDbMJFi^{(!PqqNj?XylUw8T)kOlRgP4qV zCmj5!8^b-H7SYtro}tP#YHlbro0*$y&@fMI%uX?Q{Ps{WLk)QXE#XHuVAVTfl>5@V2ykGeR#NM-Ttn)btzG`lmMro|Os z=j1aay>y>?so!}%7c`T}Q{r9)8+M_oy>NR_NcI9g&*)1Sg~2t{y^cOF<9FA;B;M?k z=d{atfgFBP^%%KF2wi#nbvP1dY@*$tHX)Ju_;DZGae6xLo6%E~+|N(K3KNz? zDESRkdoCm$Ib&7+{8rRSvwd_gL+`i0?R7J&yZVvsyczp4I*4)%sn|G{v>Z5*Gk8(g z13Gn9-Cb(m>$HFn)-t|hv@v#;1HL;P`PauU&VOR|s3^`0?Atdp&2AW;HNC!eQT94S z{foi}=74uTt!e(Naly?tjS&LF*L`JR$*p@QO-HgO`)}QTH!b}>M`kd)P1*AFi~I2V z)d|#Aw-w&hns`gEZrY}8uO^2y3EP_gC1sQTHMaf2HrtU*@;^edR7(_oj}G(-rzo4~ zE%xTQfUMgFy}T^w+q~_1;dKOPQQz>1ik)AOT*wYg-^?ekT!Yw`R-YBlp0K-BzaF7B zM;ip8pnQ|bWSq_`a45-6@{OVr2ZHVuEZ(#fMw> zxvjT-{B)p}pVp-lx08X_=5VC|)Kf9X!u&zB-D{79l!^TX{SS8;=^hL@%lyb?GWg;I zYrDk~x**5RR%(DyT+OO-7QZz5qP0$MkC_*KULmrW**Baj>aL=~fsqhTi$nXkX3_jH zN>Ub_AU3<@RO)Er*PgY3sb2c_4@WGJ3;inCthuw*S4@``!|-=Q>jX8P&MFb|)aTp3 z<%TDZCz?|XGDwXvCv-Ga@~vffy$Uel2u2a1kP=JM=+imqI76{ zyi`CV`iKnYJ*JQ9A6yI3hf}45-th8do=VOXJVl3zTHM!9V|jj2+gDRQQB6^k_E_*s zS;-;hATjw3jxa@y*u$oTAZ6T0Pf|xs2bX5pK4zYl1*U!`#;}$rhkK@OpDZ|Po&*@2 zPvD#VY&~$tqDxV8pj+^ z)xvzSnT{5*ZYM6|4SL*XUnwkKhrymwP#>4HYz{eo)Yc}Z?isg4<@2FpwJfar`dd-F z_YJcS9k3-7-p{FgC|cf8t)1pdk+Z6HxKrXpu2w$I{90Q$T2{QxOBVv9AIz>FV9KZC zzrF|c$;M!}tpP@q$phslCZ(9t_}0CBktb)L^&`cCm4Q;?vE*)i`=vWdodlmOn$wk0 z@|lrOT`M^PGQ8Uqlk`Ua!ZXUZ-mnP%Yx; z>h#ubzq#EhLHXc>=UH<^H~n)V$~AM!wf9wS$LdIH&NGWR8J}J-R;;ZoT{+|y_9k(N z&ZRhJsyy|IL^UvCEj? zN5NR1iWltuZ5)YrqrW2BxfK|XUms)Y)LO4tzOp_%Cq{#aa+&v8mOjY~hNBzqD8|dL zK~L`9|4?T^Vtzw>DsgOJ^ot@Q!f4!V-j`>+YxK_YUc`C$X#6sin7~Fn?Gwm?ZXB5R z%WkGQ-f(E%p!_vd!E3jb7c!{?y5aQ(zOyiC)h6n^&hqO{*j2a+LSB^FYw7kAcX$F;}{uPdThTm(av1#Voe#ETJxQ8xK%>`7_RR%6vY zN#NzmQ)c6)3br9Ifen*!|Fl}x!6mkpJm^M}@CNmQhSqvs|H%ustJdvGC?f5L#+cCJ z2Kbrr;+W99{`LHZ^JhHZI&T`7Vn&==jq)x#Fv1 zYOgjNX*dE@d`p4OxxU%Ab5daO#@){&Ze1HWM?B{rXP9RLc`wxT-@%bho99MZfBF6X zv=B%?%T|W}>sv5($d2#3AkN1ZgY)&-Dvi@@ZFAw22R7@Np70%3(-pGLW;ao?I6T(q;}JcViR<`Q!VY-u7ELCh{LYw5bNY>h- z{V7}gmoxl7RDpjMzP*}d{e$qsq^k7)v+%#Nrk|&=y%==xhiMov{AL>eSV;Ss|Mr@R z>JR)=Nf)}mT;@OW|NTm~pQDw3kkmgjj-(r@U#{#Ine)%<*p5m+S{}mjzvue%tp7rT?Ks8$!>nod{KbD8(rr!fF9zG8 zhWQ5u=HD6o0dB@9T3X3JhVW8JeINWM*`J?rWY8qFUJ?UxBhisByCBE;$nOvc>8L__ RMnj5z2!xsRB9Gr1`# Date: Thu, 17 Nov 2022 16:56:42 +0000 Subject: [PATCH 004/247] Add transforms and fitted transforms --- alibi_detect/od/backend/__init__.py | 1 - alibi_detect/od/backend/keops/knn.py | 25 --- .../od/backend/tests/test_ensemble.py | 98 +++++++++ alibi_detect/od/backend/tests/test_knn.py | 41 ++++ alibi_detect/od/backend/torch/ensemble.py | 186 ++++++++++++++++++ alibi_detect/od/backend/torch/knn.py | 27 +-- alibi_detect/od/transforms.py | 135 ------------- 7 files changed, 340 insertions(+), 173 deletions(-) delete mode 100644 alibi_detect/od/backend/keops/knn.py create mode 100644 alibi_detect/od/backend/tests/test_ensemble.py create mode 100644 alibi_detect/od/backend/tests/test_knn.py create mode 100644 alibi_detect/od/backend/torch/ensemble.py delete mode 100644 alibi_detect/od/transforms.py diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index 18ac838bc..d14b24a46 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -1,4 +1,3 @@ from alibi_detect.utils.missing_optional_dependency import import_optional KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) -KNNKeops = import_optional('alibi_detect.od.backend.keops.knn', ['KNNKeops']) diff --git a/alibi_detect/od/backend/keops/knn.py b/alibi_detect/od/backend/keops/knn.py deleted file mode 100644 index 78506f663..000000000 --- a/alibi_detect/od/backend/keops/knn.py +++ /dev/null @@ -1,25 +0,0 @@ -import numpy as np -import torch -from pykeops.torch import LazyTensor - - -def cdist(X, Y): - return ((X - Y)**2).sum(-1).sqrt() - - -class KNNKeops: - def score(X, x_ref, k, kernel=None): - ensemble = isinstance(k, (np.ndarray, list, tuple)) - X = torch.as_tensor(X) - X_keops = LazyTensor(X[:, None, :]) - x_ref_keops = LazyTensor(x_ref[None, :, :]) - K = -kernel(X_keops, x_ref_keops) if kernel else cdist(X_keops, x_ref_keops) - ks = np.array(k) if ensemble else np.array([k]) - bot_k_inds = K.argKmin(np.max(ks), dim=1) - all_knn_dists = (X[:, None, :] - x_ref[bot_k_inds][:, ks-1, :]).norm(dim=2) - all_knn_dists = all_knn_dists if ensemble else all_knn_dists[:,0] - return all_knn_dists.cpu().numpy() - - - def fit(X): - return torch.as_tensor(X) \ No newline at end of file diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/backend/tests/test_ensemble.py new file mode 100644 index 000000000..5d72e8f99 --- /dev/null +++ b/alibi_detect/od/backend/tests/test_ensemble.py @@ -0,0 +1,98 @@ +import pytest +import torch +from alibi_detect.od.backend.torch import ensemble + + +def test_pval_normaliser(): + normaliser = ensemble.PValNormaliser() + x = torch.randn(3, 10) + x_ref = torch.randn(64, 10) + with pytest.raises(ValueError): + normaliser(x) + + normaliser.fit(x_ref) + x_norm = normaliser(x) + normaliser = torch.jit.script(normaliser) + x_norm_2 = normaliser(x) + assert torch.all(x_norm_2 == x_norm) + + +def test_shift_and_scale_normaliser(): + normaliser = ensemble.ShiftAndScaleNormaliser() + x = torch.randn(3, 10) + x_ref = torch.randn(64, 10) + with pytest.raises(ValueError): + normaliser(x) + + normaliser.fit(x_ref) + x_norm = normaliser(x) + normaliser = torch.jit.script(normaliser) + x_norm_2 = normaliser(x) + assert torch.all(x_norm_2 == x_norm) + + +def test_average_aggregator(): + aggregator = ensemble.AverageAggregator() + scores = torch.randn((3, 10)) + aggregated_scores = aggregator(scores) + assert aggregated_scores.shape == (3, ) + aggregator = torch.jit.script(aggregator) + aggregated_scores_2 = aggregator(scores) + assert torch.all(aggregated_scores_2 == aggregated_scores) + + +def test_weighted_average_aggregator(): + aggregator = ensemble.AverageAggregator(weights=torch.randn((10))) + scores = torch.randn((3, 10)) + aggregated_scores = aggregator(scores) + assert aggregated_scores.shape == (3, ) + aggregator = torch.jit.script(aggregator) + aggregated_scores_2 = aggregator(scores) + assert torch.all(aggregated_scores_2 == aggregated_scores) + + +def test_topk_aggregator(): + aggregator = ensemble.TopKAggregator(k=4) + scores = torch.randn((3, 10)) + aggregated_scores = aggregator(scores) + assert aggregated_scores.shape == (3, ) + aggregator = torch.jit.script(aggregator) + aggregated_scores_2 = aggregator(scores) + assert torch.all(aggregated_scores_2 == aggregated_scores) + + +def test_max_aggregator(): + aggregator = ensemble.MaxAggregator() + scores = torch.randn((3, 10)) + aggregated_scores = aggregator(scores) + assert aggregated_scores.shape == (3, ) + aggregator = torch.jit.script(aggregator) + aggregated_scores_2 = aggregator(scores) + assert torch.all(aggregated_scores_2 == aggregated_scores) + + +def test_min_aggregator(): + aggregator = ensemble.MinAggregator() + scores = torch.randn((3, 10)) + aggregated_scores = aggregator(scores) + assert aggregated_scores.shape == (3, ) + aggregator = torch.jit.script(aggregator) + aggregated_scores_2 = aggregator(scores) + assert torch.all(aggregated_scores_2 == aggregated_scores) + + +@pytest.mark.parametrize('aggregator', ['AverageAggregator', 'MaxAggregator', 'MinAggregator', 'TopKAggregator']) +@pytest.mark.parametrize('normaliser', ['PValNormaliser', 'ShiftAndScaleNormaliser']) +def test_accumulator(aggregator, normaliser): + aggregator = getattr(ensemble, aggregator)() + normaliser = getattr(ensemble, normaliser)() + accumulator = ensemble.Accumulator(aggregator=aggregator, normaliser=normaliser) + + x = torch.randn(3, 10) + x_ref = torch.randn(64, 10) + + accumulator.fit(x_ref) + x_norm = accumulator(x) + accumulator = torch.jit.script(accumulator) + x_norm_2 = accumulator(x) + assert torch.all(x_norm_2 == x_norm) \ No newline at end of file diff --git a/alibi_detect/od/backend/tests/test_knn.py b/alibi_detect/od/backend/tests/test_knn.py new file mode 100644 index 000000000..34bf0ff60 --- /dev/null +++ b/alibi_detect/od/backend/tests/test_knn.py @@ -0,0 +1,41 @@ +import torch + +from alibi_detect.od.backend.torch.knn import KNNTorch +from alibi_detect.utils.pytorch.kernels import GaussianRBF + + +def test_knn_torch_backend(): + knn_torch = KNNTorch(k=5) + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + x = torch.randn((3, 10)) + scores = knn_torch(x) + assert scores.shape == (3, ) + knn_torch = torch.jit.script(knn_torch) + scores_2 = knn_torch(x) + assert torch.all(scores == scores_2) + + +def test_knn_torch_backend_ensemble(): + knn_torch = KNNTorch(k=[4, 5]) + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + x = torch.randn((3, 10)) + scores = knn_torch(x) + assert scores.shape == (3, 2) + knn_torch = torch.jit.script(knn_torch) + scores_2 = knn_torch(x) + assert torch.all(scores == scores_2) + + +def test_knn_kernel(): + kernel = GaussianRBF(sigma=torch.tensor((0.1))) + knn_torch = KNNTorch(k=[4, 5], kernel=kernel) + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + x = torch.randn((3, 10)) + scores = knn_torch(x) + assert scores.shape == (3, 2) + # knn_torch = torch.jit.script(knn_torch) + # scores_2 = knn_torch(x) + # assert torch.all(scores == scores_2) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py new file mode 100644 index 000000000..9fed5388a --- /dev/null +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import logging + +from typing import Optional +from abc import ABC, abstractmethod + +import torch +from torch.nn import Module +import numpy as np + + +logger = logging.getLogger(__name__) + + +class BaseTransform(Module, ABC): + """Base Transform class. + + provides abstract methods for transform objects that map a numpy + array. + """ + def __init__(self): + super().__init__() + + def transform(self, X: torch.Tensor): + return self._transform(X) + + @abstractmethod + def _transform(self, X: torch.Tensor): + """Applies class transform to numpy array + + Parameters + ---------- + X + numpy array to be transformed + + Raises + ------ + NotImplementedError + if _transform is not implimented on child class raise + NotImplementedError + """ + raise NotImplementedError() + + def forward(self, X: torch.Tensor): + return self.transform(X=X) + + +class BaseFittedTransform(BaseTransform): + """Base Fitted Transform class. + + Provides abstract methods for transforms that have an aditional + fit step. + """ + fitted = False + + def __init__(self): + super().__init__() + + def fit(self, X: torch.Tensor) -> BaseTransform: + if not self.fitted and hasattr(self, '_fit'): + self._fit(X) + self.fitted = True + return self + + def _fit(self, X: torch.Tensor): + raise NotImplementedError() + + def transform(self, X: torch.Tensor): + if not self.fitted: + raise ValueError('Transform not fitted, call fit before calling transform!') + return self._transform(X) + + +class PValNormaliser(BaseFittedTransform): + """Maps scores to there p values. + + Needs to be fit on a reference dataset using fit. Transform counts the number of scores + in the reference dataset that are greter than the score of interest and divides by the + size of the reference dataset. Output is between 1 and 0. Small values are likely to be + outliers. + """ + def __init__(self): + super().__init__() + self.val_scores = None + + def _fit(self, val_scores: torch.Tensor): + self.val_scores = val_scores + + def _transform(self, scores: torch.Tensor) -> torch.Tensor: + p_vals = ( + 1 + (scores[:, None, :] < self.val_scores[None, :, :]).sum(1) + )/(len(self.val_scores)+1) + return 1 - p_vals + + +class ShiftAndScaleNormaliser(BaseFittedTransform): + """Maps scores to their normalised values. + + Needs to be fit on a reference dataset using fit. Subtracts the dataset mean and + scales by the standard deviation. + """ + def __init__(self): + super().__init__() + self.val_means = None + self.val_scales = None + + def _fit(self, val_scores: torch.Tensor) -> BaseTransform: + self.val_means = val_scores.mean(0)[None, :] + self.val_scales = val_scores.std(0)[None, :] + + def _transform(self, scores: torch.Tensor) -> torch.Tensor: + return (scores - self.val_means)/self.val_scales + + +class TopKAggregator(BaseTransform): + def __init__(self, k: Optional[int] = None): + """Takes the mean of the top k scores. + + Parameters + ---------- + k + number of scores to take the mean of. If `k` is left `None` then will be set to + half the number of scores passed in the forward call. + """ + super().__init__() + self.k = k + + def _transform(self, scores: torch.Tensor) -> torch.Tensor: + if self.k is None: + self.k = int(np.ceil(scores.shape[1]/2)) + sorted_scores, _ = torch.sort(scores, 1) + return sorted_scores[:, -self.k:].mean(-1) + + +class AverageAggregator(BaseTransform): + """Averages the scores of the detectors in an ensemble. + + Parameters + ---------- + weights + Optional parameter to weight the scores. + """ + def __init__(self, weights: Optional[torch.Tensor] = None): + super().__init__() + self.weights = weights + + def _transform(self, scores: torch.Tensor) -> torch.Tensor: + if self.weights is None: + m = scores.shape[-1] + self.weights = torch.ones(m)/m + return scores @ self.weights + + +class MaxAggregator(BaseTransform): + """Takes the max score of a set of detectors in an ensemble.""" + def __init__(self): + super().__init__() + + def _transform(self, scores: torch.Tensor) -> torch.Tensor: + vals, _ = torch.max(scores, dim=-1) + return vals + + +class MinAggregator(BaseTransform): + """Takes the min score of a set of detectors in an ensemble.""" + def __init__(self): + super().__init__() + + def _transform(self, scores: torch.Tensor) -> torch.Tensor: + vals, _ = torch.min(scores, dim=-1) + return vals + + +class Accumulator(BaseFittedTransform): + def __init__(self, normaliser: BaseFittedTransform, aggregator: BaseTransform): + super().__init__() + self.normaliser = normaliser + self.aggregator = aggregator + + def _transform(self, X: torch.Tensor): + X = self.normaliser(X) + return self.aggregator(X) + + def _fit(self, X: torch.Tensor): + return self.normaliser.fit(X) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index db846ac4c..1c1a19205 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -2,16 +2,19 @@ import torch -class KNNTorch: - def score(X, x_ref, k, kernel=None): - ensemble = isinstance(k, (np.ndarray, list, tuple)) - X = torch.as_tensor(X, dtype=torch.float32) - K = -kernel(X, x_ref) if kernel else torch.cdist(X, x_ref) - ks = np.array(k) if ensemble else np.array([k]) - bot_k_dists = torch.topk(K, np.max(ks), dim=1, largest=False) - all_knn_dists = bot_k_dists.values[:,ks-1] - all_knn_dists = all_knn_dists if ensemble else all_knn_dists[:,0] - return all_knn_dists.cpu().numpy() +class KNNTorch(torch.nn.Module): + def __init__(self, k, kernel=None): + super().__init__() + self.kernel = kernel + self.ensemble = isinstance(k, (np.ndarray, list, tuple)) + self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k]) - def fit(X): - return torch.as_tensor(X, dtype=torch.float32) \ No newline at end of file + def forward(self, X): + K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) + bot_k_dists = torch.topk(K, torch.max(self.ks), dim=1, largest=False) + all_knn_dists = bot_k_dists.values[:, self.ks-1] + all_knn_dists = all_knn_dists if self.ensemble else all_knn_dists[:, 0] + return all_knn_dists.cpu() + + def fit(self, X: torch.tensor): + self.x_ref = torch.as_tensor(X, dtype=torch.float32) diff --git a/alibi_detect/od/transforms.py b/alibi_detect/od/transforms.py deleted file mode 100644 index 46937f4c0..000000000 --- a/alibi_detect/od/transforms.py +++ /dev/null @@ -1,135 +0,0 @@ -from __future__ import annotations - -import logging -import numpy as np - -from typing import Optional -from abc import ABC, abstractmethod - -logger = logging.getLogger(__name__) - - -class BaseTransform(ABC): - """Base Transform class. - - provides abstract methods for transform objects that map a numpy - array. - """ - def transform(self, X: np.ndarray): - return self._transform(X) - - @abstractmethod - def _transform(self, X: np.ndarray): - """Applies class transform to numpy array - - Parameters - ---------- - X - numpy array to be transformed - - Raises - ------ - NotImplementedError - if _transform is not implimented on child class raise - NotImplementedError - """ - raise NotImplementedError() - - -class BaseFittedTransform(BaseTransform): - """Base Fitted Transform class. - - Provides abstract methods for transforms that have an aditional - fit step. - """ - fitted = False - - def fit(self, X: np.ndarray) -> BaseTransform: - if not self.fitted and hasattr(self, '_fit'): - self._fit(X) - self.fitted = True - return self - - def _fit(self, X: np.ndarray): - raise NotImplementedError() - - def transform(self, X: np.ndarray): - if not self.fitted: - raise ValueError('Transform not fitted, call fit before calling transform!') - return self._transform(X) - - -class PValNormaliser(BaseTransform): - """Maps scores from an ensemble of detectors to there p values. - - Needs to be fit on a reference dataset using fit. Transform counts the number of scores - in the reference dataset that are greter than the score of interest and divides by the - size of the reference dataset. Output is between 1 and 0. Small values are likely to be - outliers. - """ - def _fit(self, val_scores: np.ndarray): - self.val_scores = val_scores - - def _transform(self, scores: np.ndarray) -> np.ndarray: - p_vals = ( - 1 + (scores[:, None, :] < self.val_scores[None, :, :]).sum(1) - )/(len(self.val_scores)+1) - return 1 - p_vals - - -class ShiftAndScaleNormaliser(BaseTransform): - """Maps scores from an ensemble of to a set of . - - Needs to be fit on a reference dataset using fit. - """ - def _fit(self, val_scores: np.ndarray) -> BaseTransform: - self.val_means = val_scores.mean(0)[None, :] - self.val_scales = val_scores.std(0)[None, :] - - def _transform(self, scores: np.ndarray) -> np.ndarray: - return (scores - self.val_means)/self.val_scales - - -class TopKAggregator(BaseTransform): - def __init__(self, k: Optional[int]): - self.k = k - self.fitted = True - - def _transform(self, scores: np.ndarray) -> np.ndarray: - if self.k is None: - self.k = int(np.ceil(scores.shape[1]/2)) - return np.sort(scores, 1)[:, -self.k:].mean(-1) - - -class AverageAggregator(BaseTransform): - def __init__(self, weights: Optional[np.ndarray] = None): - self.weights = weights - self.fitted = True - - def _transform(self, scores: np.ndarray) -> np.ndarray: - if self.weights is None: - m = scores.shape[-1] - self.weights = np.ones(m)/m - return scores @ self.weights - - -class MaxAggregator(BaseTransform): - def __init__(self): - self.fitted = True - - def _transform(self, scores: np.ndarray) -> np.ndarray: - return np.max(scores, axis=-1) - - -class MinAggregator(BaseTransform): - def __init__(self): - self.fitted = True - - def _transform(self, scores: np.ndarray) -> np.ndarray: - return np.min(scores, axis=-1) - - -class Accumulator(BaseFittedTransform): - def __init__(self, normaliser: BaseFittedTransform, aggregator: BaseTransform): - self.normaliser = normaliser - self.aggregator = aggregator From 1c0c7c01e13085a593a9a1410fce505bf60ef91d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 17 Nov 2022 17:03:12 +0000 Subject: [PATCH 005/247] Fix flake8 errors --- .../od/backend/tests/test_ensemble.py | 2 +- .../{test_knn.py => test_knn_backend.py} | 0 alibi_detect/od/knn.py | 4 +- alibi_detect/od/tests/test_transforms.py | 52 ------------------- 4 files changed, 3 insertions(+), 55 deletions(-) rename alibi_detect/od/backend/tests/{test_knn.py => test_knn_backend.py} (100%) delete mode 100644 alibi_detect/od/tests/test_transforms.py diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/backend/tests/test_ensemble.py index 5d72e8f99..a0b0756f1 100644 --- a/alibi_detect/od/backend/tests/test_ensemble.py +++ b/alibi_detect/od/backend/tests/test_ensemble.py @@ -95,4 +95,4 @@ def test_accumulator(aggregator, normaliser): x_norm = accumulator(x) accumulator = torch.jit.script(accumulator) x_norm_2 = accumulator(x) - assert torch.all(x_norm_2 == x_norm) \ No newline at end of file + assert torch.all(x_norm_2 == x_norm) diff --git a/alibi_detect/od/backend/tests/test_knn.py b/alibi_detect/od/backend/tests/test_knn_backend.py similarity index 100% rename from alibi_detect/od/backend/tests/test_knn.py rename to alibi_detect/od/backend/tests/test_knn_backend.py diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 4f1f40cd1..e1a5fc32e 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -2,7 +2,7 @@ import numpy as np from alibi_detect.od.base import OutlierDetector -from alibi_detect.od.transforms import BaseTransform +from alibi_detect.od.backend.torch.ensemble import Accumulator from alibi_detect.od.backend import KNNTorch from alibi_detect.od.backend import KNNKeops @@ -21,7 +21,7 @@ def __init__( self, k: Union[int, np.ndarray], kernel: Optional[Callable] = None, - accumulator: Optional[]=None, + accumulator: Optional[Accumulator] = None, backend: Literal['pytorch', 'keops'] = 'pytorch' ) -> None: super().__init__() diff --git a/alibi_detect/od/tests/test_transforms.py b/alibi_detect/od/tests/test_transforms.py deleted file mode 100644 index 356c22930..000000000 --- a/alibi_detect/od/tests/test_transforms.py +++ /dev/null @@ -1,52 +0,0 @@ -import numpy as np\ - -from alibi_detect.od.transforms import PValNormaliser, ShiftAndScaleNormaliser, TopKAggregator, MaxAggregator, \ - MinAggregator, AverageAggregator -import pytest - - -def test_p_val_normaliser(): - p_val_norm = PValNormaliser() - scores = np.random.normal(0, 1, (1000, 3)) * np.array([0.5, 2, 1]) + np.array([0, 1, -1]) - - with pytest.raises(ValueError): - p_val_norm.transform(np.array([[0, 0, 0]])) - - p_val_norm.fit(X=scores) - - s = np.array([[0, 0, 0]]) + np.array([0, 1, -1]) - s_transformed = p_val_norm.transform(s) - np.testing.assert_array_almost_equal( - np.array([[0.5, 0.5, 0.5]]), - s_transformed, - decimal=0.1 - ) - - s = np.array([[0, 0, 0]]) - s_transformed = p_val_norm.transform(s) - np.testing.assert_array_almost_equal( - np.array([[0.5, 0.32, 0.85]]), - s_transformed, - decimal=0.1 - ) - - -def test_shift_and_scale_normaliser(): - shift_and_scale_norm = ShiftAndScaleNormaliser() - scores = np.random.normal(0, 1, (1000, 3)) * np.array([0.5, 2, 1]) + np.array([0, 1, -1]) - shift_and_scale_norm.fit(X=scores) - s = np.array([[0, 0, 0]]) + np.array([0, 1, -1]) - s_transformed = shift_and_scale_norm.transform(s) - np.testing.assert_array_almost_equal( - np.array([[0., 0., 0.]]), - s_transformed, - decimal=0.1 - ) - - s = np.array([[0, 0, 0]]) - s_transformed = shift_and_scale_norm.transform(s) - np.testing.assert_array_almost_equal( - np.array([[0., -0.5, 1.]]), - s_transformed, - decimal=0.1 - ) From 2bf0056d05b7e9b5edd693bf661c007d4c298551 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 21 Nov 2022 14:14:14 +0000 Subject: [PATCH 006/247] Add accumulator into KNNTorch backend --- alibi_detect/od/backend/tests/conftest.py | 10 +++ .../od/backend/tests/test_knn_backend.py | 16 ++--- alibi_detect/od/backend/torch/base.py | 62 +++++++++++++++++++ alibi_detect/od/backend/torch/ensemble.py | 28 +++++++-- alibi_detect/od/backend/torch/knn.py | 25 ++++++-- alibi_detect/od/base.py | 4 +- alibi_detect/od/knn.py | 19 +++--- alibi_detect/od/tests/test_knn.py | 4 +- 8 files changed, 141 insertions(+), 27 deletions(-) create mode 100644 alibi_detect/od/backend/tests/conftest.py create mode 100644 alibi_detect/od/backend/torch/base.py diff --git a/alibi_detect/od/backend/tests/conftest.py b/alibi_detect/od/backend/tests/conftest.py new file mode 100644 index 000000000..caacb8f28 --- /dev/null +++ b/alibi_detect/od/backend/tests/conftest.py @@ -0,0 +1,10 @@ +import pytest +from alibi_detect.od.backend.torch.ensemble import Accumulator, PValNormaliser, AverageAggregator + + +@pytest.fixture(scope='session') +def accumulator(request): + return Accumulator( + normaliser=PValNormaliser(), + aggregator=AverageAggregator() + ) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 34bf0ff60..2382eb587 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -4,8 +4,8 @@ from alibi_detect.utils.pytorch.kernels import GaussianRBF -def test_knn_torch_backend(): - knn_torch = KNNTorch(k=5) +def test_knn_torch_backend(accumulator): + knn_torch = KNNTorch(k=5, accumulator=None) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) @@ -16,26 +16,26 @@ def test_knn_torch_backend(): assert torch.all(scores == scores_2) -def test_knn_torch_backend_ensemble(): - knn_torch = KNNTorch(k=[4, 5]) +def test_knn_torch_backend_ensemble(accumulator): + knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) scores = knn_torch(x) - assert scores.shape == (3, 2) + assert scores.shape == (3,) knn_torch = torch.jit.script(knn_torch) scores_2 = knn_torch(x) assert torch.all(scores == scores_2) -def test_knn_kernel(): +def test_knn_kernel(accumulator): kernel = GaussianRBF(sigma=torch.tensor((0.1))) - knn_torch = KNNTorch(k=[4, 5], kernel=kernel) + knn_torch = KNNTorch(k=[4, 5], kernel=kernel, accumulator=accumulator) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) scores = knn_torch(x) - assert scores.shape == (3, 2) + assert scores.shape == (3,) # knn_torch = torch.jit.script(knn_torch) # scores_2 = knn_torch(x) # assert torch.all(scores == scores_2) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py new file mode 100644 index 000000000..6287f536c --- /dev/null +++ b/alibi_detect/od/backend/torch/base.py @@ -0,0 +1,62 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +import torch + +import logging + +logger = logging.getLogger(__name__) + + +class TorchOutlierDetector(torch.nn.Module, ABC): + """ Base class for outlier detection algorithms. """ + threshold_inferred = False + + def __init__(self): + super().__init__() + + @abstractmethod + def fit(self, X: torch.Tensor) -> None: + pass + + @abstractmethod + def score(self, X: torch.Tensor) -> torch.Tensor: + pass + + def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: + """ + Infers the threshold above which only fpr% of inlying data scores. + Also saves down the scores to be later used for computing p-values + of new data points (by comparison to the empirical cdf). + For ensemble models the scores are normalised and aggregated before + saving scores and inferring threshold. + """ + self.val_scores = self.score(X) + self.val_scores = self.accumulator(self.val_scores) if self.accumulator is not None \ + else self.val_scores + self.threshold = torch.quantile(self.val_scores, 1-fpr) + self.threshold_inferred = True + + def predict(self, X: torch.Tensor) -> torch.Tensor: + """ + Scores the instances and then compares to pre-inferred threshold. + For ensemble models the scores from each constituent is added to the output. + p-values are also returned by comparison to validation scores (of inliers) + """ + output = {} + scores = self.score(X) + output['raw_scores'] = scores + + if getattr(self, 'normaliser'): + scores = self.normaliser.transform(scores) + output['normalised_scores'] = scores + + if getattr(self, 'aggregator'): + scores = self.aggregator.transform(scores) + output['aggregate_scores'] = scores + + if self.threshold_inferred: + p_vals = (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) + preds = scores > self.threshold + output.update(scores=scores, preds=preds, p_vals=p_vals) + + return output diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 9fed5388a..ac2419fb8 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -173,14 +173,34 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class Accumulator(BaseFittedTransform): - def __init__(self, normaliser: BaseFittedTransform, aggregator: BaseTransform): + def __init__(self, + normaliser: BaseFittedTransform = None, + aggregator: BaseTransform = AverageAggregator()): + """Wraps a normaliser and aggregator into a single object. + + The accumulator wraps normalisers and aggregators into a single object. + + Parameters + ---------- + normaliser + normaliser that's an instance of BaseFittedTransform. Maps the outputs of + a set of detectors to a common range. + aggregator + aggregator extendng BaseTransform. Maps outputs of the normaliser to + single score. + """ super().__init__() self.normaliser = normaliser + if self.normaliser is None: + self.fitted = True self.aggregator = aggregator def _transform(self, X: torch.Tensor): - X = self.normaliser(X) - return self.aggregator(X) + if self.normaliser is not None: + X = self.normaliser(X) + X = self.aggregator(X) + return X def _fit(self, X: torch.Tensor): - return self.normaliser.fit(X) + if self.normaliser is not None: + X = self.normaliser.fit(X) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 1c1a19205..5b6026d56 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -1,20 +1,37 @@ +from typing import Optional import numpy as np import torch +from alibi_detect.od.backend.torch.ensemble import Accumulator +from alibi_detect.od.backend.torch.base import TorchOutlierDetector -class KNNTorch(torch.nn.Module): - def __init__(self, k, kernel=None): +class KNNTorch(TorchOutlierDetector): + def __init__( + self, + k, + kernel=None, + accumulator: Optional[Accumulator] = None + ): super().__init__() self.kernel = kernel self.ensemble = isinstance(k, (np.ndarray, list, tuple)) self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k]) + self.accumulator = accumulator def forward(self, X): + scores = self.score(X) + predictions = self.accumulator(scores) \ + if self.accumulator is not None else scores + return predictions.cpu() + + def score(self, X): K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) bot_k_dists = torch.topk(K, torch.max(self.ks), dim=1, largest=False) all_knn_dists = bot_k_dists.values[:, self.ks-1] - all_knn_dists = all_knn_dists if self.ensemble else all_knn_dists[:, 0] - return all_knn_dists.cpu() + return all_knn_dists if self.ensemble else all_knn_dists[:, 0] def fit(self, X: torch.tensor): self.x_ref = torch.as_tensor(X, dtype=torch.float32) + if self.accumulator is not None: + scores = self.score(X) + self.accumulator.fit(scores) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 311c93e00..58ee12c58 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -52,11 +52,11 @@ def predict(self, X: np.ndarray) -> np.ndarray: scores = self.score(X) output['raw_scores'] = scores - if getattr(self, 'normaliser') and self.normaliser.fitted: + if getattr(self, 'normaliser'): scores = self.normaliser.transform(scores) output['normalised_scores'] = scores - if getattr(self, 'aggregator') and self.aggregator.fitted: + if getattr(self, 'aggregator'): scores = self.aggregator.transform(scores) output['aggregate_scores'] = scores diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index e1a5fc32e..1537341b7 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -5,14 +5,12 @@ from alibi_detect.od.backend.torch.ensemble import Accumulator from alibi_detect.od.backend import KNNTorch -from alibi_detect.od.backend import KNNKeops from alibi_detect.utils.frameworks import BackendValidator X_REF_FILENAME = 'x_ref.npy' backends = { 'pytorch': KNNTorch, - 'keops': KNNKeops } @@ -21,21 +19,28 @@ def __init__( self, k: Union[int, np.ndarray], kernel: Optional[Callable] = None, - accumulator: Optional[Accumulator] = None, - backend: Literal['pytorch', 'keops'] = 'pytorch' + normaliser=None, + aggregator=None, + backend: Literal['pytorch'] = 'pytorch' ) -> None: super().__init__() backend = backend.lower() BackendValidator( - backend_options={'pytorch': ['pytorch'], - 'keops': ['keops']}, + backend_options={'pytorch': ['pytorch']}, construct_name=self.__class__.__name__ ).verify_backend(backend) + # TODO: Abstract to pydantic model. + if isinstance(k, (list, np.ndarray)) and aggregator is None: + raise ValueError((f'k={k} is type {type(k)} but aggregator is {aggregator}, you must ' + 'specify at least an aggregator if you want to use the knn detector' + 'an ensemble like this.')) + self.k = k self.kernel = kernel self.ensemble = isinstance(self.k, (np.ndarray, list)) - self.accumulator = accumulator + self.normaliser = normaliser + self.aggregator = aggregator self.fitted = False self.backend = backends[backend] diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index 08117e217..9b12db2a4 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -2,8 +2,8 @@ import numpy as np from alibi_detect.od.knn import KNN -from alibi_detect.od.transforms import AverageAggregator, TopKAggregator, MaxAggregator, MinAggregator, \ - ShiftAndScaleNormaliser, PValNormaliser +from alibi_detect.od.backend.torch.ensemble import AverageAggregator, TopKAggregator, MaxAggregator, \ + MinAggregator, ShiftAndScaleNormaliser, PValNormaliser def test_knn_single(): From c7f4825e42a08e339866e6930aa159d8026b24ed Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 21 Nov 2022 17:21:52 +0000 Subject: [PATCH 007/247] Add BaseTorchDetector functionality --- .../od/backend/tests/test_knn_backend.py | 37 ++++++++++---- alibi_detect/od/backend/torch/base.py | 51 +++++++------------ alibi_detect/od/backend/torch/knn.py | 8 +-- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 2382eb587..ffdf66ee9 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -9,11 +9,19 @@ def test_knn_torch_backend(accumulator): x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) + outputs = knn_torch.predict(x) + assert outputs['scores'].shape == (3, ) + assert outputs['preds'] is None + assert outputs['p_vals'] is None scores = knn_torch(x) - assert scores.shape == (3, ) - knn_torch = torch.jit.script(knn_torch) - scores_2 = knn_torch(x) - assert torch.all(scores == scores_2) + assert torch.all(scores == outputs['scores']) + + knn_torch.infer_threshold(x_ref, 0.1) + outputs = knn_torch.predict(x) + assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) + + x = torch.randn((1, 10)) * 100 + assert knn_torch(x) def test_knn_torch_backend_ensemble(accumulator): @@ -23,9 +31,13 @@ def test_knn_torch_backend_ensemble(accumulator): x = torch.randn((3, 10)) scores = knn_torch(x) assert scores.shape == (3,) - knn_torch = torch.jit.script(knn_torch) - scores_2 = knn_torch(x) - assert torch.all(scores == scores_2) + + knn_torch.infer_threshold(x_ref, 0.1) + outputs = knn_torch.predict(x) + assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) + + x = torch.randn((1, 10)) * 100 + assert knn_torch(x) def test_knn_kernel(accumulator): @@ -36,6 +48,11 @@ def test_knn_kernel(accumulator): x = torch.randn((3, 10)) scores = knn_torch(x) assert scores.shape == (3,) - # knn_torch = torch.jit.script(knn_torch) - # scores_2 = knn_torch(x) - # assert torch.all(scores == scores_2) + + knn_torch.infer_threshold(x_ref, 0.1) + outputs = knn_torch.predict(x) + assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) + + x = torch.randn((1, 10)) * 100 + print(knn_torch(x)) + assert knn_torch(x) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 6287f536c..90d36342e 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -8,12 +8,12 @@ class TorchOutlierDetector(torch.nn.Module, ABC): - """ Base class for outlier detection algorithms. """ + """ Base class for torch backend outlier detection algorithms.""" threshold_inferred = False def __init__(self): super().__init__() - + @abstractmethod def fit(self, X: torch.Tensor) -> None: pass @@ -22,41 +22,28 @@ def fit(self, X: torch.Tensor) -> None: def score(self, X: torch.Tensor) -> torch.Tensor: pass + def _accumulator(self, X: torch.Tensor) -> torch.Tensor: + return self.accumulator(X) if self.accumulator is not None else X + + def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: + # check threshold has has been inferred. + return scores > self.threshold if self.threshold_inferred else None + + def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: + return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ + if self.threshold_inferred else None + def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: - """ - Infers the threshold above which only fpr% of inlying data scores. - Also saves down the scores to be later used for computing p-values - of new data points (by comparison to the empirical cdf). - For ensemble models the scores are normalised and aggregated before - saving scores and inferring threshold. - """ + # check detector has been fit. self.val_scores = self.score(X) - self.val_scores = self.accumulator(self.val_scores) if self.accumulator is not None \ - else self.val_scores + self.val_scores = self._accumulator(self.val_scores) self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True def predict(self, X: torch.Tensor) -> torch.Tensor: - """ - Scores the instances and then compares to pre-inferred threshold. - For ensemble models the scores from each constituent is added to the output. - p-values are also returned by comparison to validation scores (of inliers) - """ output = {} - scores = self.score(X) - output['raw_scores'] = scores - - if getattr(self, 'normaliser'): - scores = self.normaliser.transform(scores) - output['normalised_scores'] = scores - - if getattr(self, 'aggregator'): - scores = self.aggregator.transform(scores) - output['aggregate_scores'] = scores - - if self.threshold_inferred: - p_vals = (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) - preds = scores > self.threshold - output.update(scores=scores, preds=preds, p_vals=p_vals) - + raw_scores = self.score(X) + output['scores'] = self._accumulator(raw_scores) + output['preds'] = self._classify_outlier(output['scores']) + output['p_vals'] = self._p_vals(output['scores']) return output diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 5b6026d56..73a0e7d42 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -19,10 +19,10 @@ def __init__( self.accumulator = accumulator def forward(self, X): - scores = self.score(X) - predictions = self.accumulator(scores) \ - if self.accumulator is not None else scores - return predictions.cpu() + raw_scores = self.score(X) + scores = self._accumulator(raw_scores) + preds = self._classify_outlier(scores) + return preds.cpu() if preds is not None else scores def score(self, X): K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) From 02ac5bc147d926ffb27abd89d7fba79efc518d5a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 22 Nov 2022 16:14:49 +0000 Subject: [PATCH 008/247] Add torchscript tests for knn backend module --- .../od/backend/tests/test_knn_backend.py | 66 ++++++++++++++----- alibi_detect/od/backend/torch/base.py | 2 +- alibi_detect/od/backend/torch/ensemble.py | 38 ++++++++--- alibi_detect/od/backend/torch/knn.py | 4 +- 4 files changed, 79 insertions(+), 31 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index ffdf66ee9..722abf9e5 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -1,19 +1,25 @@ +import pytest import torch from alibi_detect.od.backend.torch.knn import KNNTorch -from alibi_detect.utils.pytorch.kernels import GaussianRBF +# from alibi_detect.utils.pytorch.kernels import GaussianRBF -def test_knn_torch_backend(accumulator): - knn_torch = KNNTorch(k=5, accumulator=None) +def test_knn_torch_backend(): + knn_torch = KNNTorch(k=5) + x = torch.randn((3, 10)) + + with pytest.raises(AttributeError): + # TODO: should be a different error! + knn_torch(x) + x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) - x = torch.randn((3, 10)) outputs = knn_torch.predict(x) assert outputs['scores'].shape == (3, ) assert outputs['preds'] is None assert outputs['p_vals'] is None - scores = knn_torch(x) + scores = knn_torch.score(x) assert torch.all(scores == outputs['scores']) knn_torch.infer_threshold(x_ref, 0.1) @@ -29,8 +35,8 @@ def test_knn_torch_backend_ensemble(accumulator): x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) - scores = knn_torch(x) - assert scores.shape == (3,) + result = knn_torch.predict(x) + assert result['scores'].shape == (3, ) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) @@ -40,19 +46,43 @@ def test_knn_torch_backend_ensemble(accumulator): assert knn_torch(x) -def test_knn_kernel(accumulator): - kernel = GaussianRBF(sigma=torch.tensor((0.1))) - knn_torch = KNNTorch(k=[4, 5], kernel=kernel, accumulator=accumulator) +def test_knn_torch_backend_ensemble_ts(accumulator): + knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) - x = torch.randn((3, 10)) - scores = knn_torch(x) - assert scores.shape == (3,) + knn_torch.infer_threshold(x_ref, 0.1) + pred_1 = knn_torch(x) + knn_torch = torch.jit.script(knn_torch) + pred_2 = knn_torch(x) + assert torch.all(pred_1 == pred_2) + +def test_knn_torch_backend_ts(): + knn_torch = KNNTorch(k=7) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) knn_torch.infer_threshold(x_ref, 0.1) - outputs = knn_torch.predict(x) - assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) + pred_1 = knn_torch(x) + knn_torch = torch.jit.script(knn_torch) + pred_2 = knn_torch(x) + assert torch.all(pred_1 == pred_2) - x = torch.randn((1, 10)) * 100 - print(knn_torch(x)) - assert knn_torch(x) + +# def test_knn_kernel(accumulator): +# kernel = GaussianRBF(sigma=torch.tensor((1))) +# knn_torch = KNNTorch(k=[4, 5], kernel=kernel, accumulator=accumulator) +# x_ref = torch.randn((1024, 10)) +# knn_torch.fit(x_ref) +# x = torch.randn((3, 10)) +# scores = knn_torch(x) +# assert scores.shape == (3,) + +# knn_torch.infer_threshold(x_ref, 0.1) +# outputs = knn_torch.predict(x) +# assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) + +# x = torch.randn((1, 10)) * 100 +# print(knn_torch.predict(x)) +# # assert knn_torch(x).item() diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 90d36342e..0a3334abd 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -41,7 +41,7 @@ def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: self.threshold_inferred = True def predict(self, X: torch.Tensor) -> torch.Tensor: - output = {} + output = {'threshold_inferred': self.threshold_inferred} raw_scores = self.score(X) output['scores'] = self._accumulator(raw_scores) output['preds'] = self._classify_outlier(output['scores']) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index ac2419fb8..9213c09f6 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -46,29 +46,47 @@ def forward(self, X: torch.Tensor): return self.transform(X=X) -class BaseFittedTransform(BaseTransform): - """Base Fitted Transform class. +class FitMixin: + """Fit mixin - Provides abstract methods for transforms that have an aditional - fit step. + Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. + + TODO: this should be encorporated into alibi_detect/base.py FitMixin once we can be sure that the + behavour is compatible. """ - fitted = False + _fitted = False def __init__(self): super().__init__() def fit(self, X: torch.Tensor) -> BaseTransform: - if not self.fitted and hasattr(self, '_fit'): - self._fit(X) - self.fitted = True + self._fit(X) + self._fitted = True return self def _fit(self, X: torch.Tensor): raise NotImplementedError() + @torch.jit.ignore + def check_fitted(self): + if not self._fitted: + # TODO: make our own NotFitted Error here! + raise ValueError(f'{self.__class__.__name__} has not been fit!') + + +class BaseFittedTransform(BaseTransform, FitMixin): + """Base Fitted Transform class. + + Provides abstract methods for transforms that have an aditional + fit step. + """ + + def __init__(self): + BaseTransform.__init__(self) + FitMixin.__init__(self) + def transform(self, X: torch.Tensor): - if not self.fitted: - raise ValueError('Transform not fitted, call fit before calling transform!') + self.check_fitted() return self._transform(X) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 73a0e7d42..b0b79a3cd 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -21,8 +21,8 @@ def __init__( def forward(self, X): raw_scores = self.score(X) scores = self._accumulator(raw_scores) - preds = self._classify_outlier(scores) - return preds.cpu() if preds is not None else scores + preds = scores > self.threshold + return preds.cpu() def score(self, X): K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) From 9068eccde464f7348c50467cd36ea539a1761271 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 23 Nov 2022 10:35:49 +0000 Subject: [PATCH 009/247] Fix GaussianRBF knn kernel test --- .../od/backend/tests/test_knn_backend.py | 36 +++++++++++-------- alibi_detect/od/backend/torch/base.py | 6 +++- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 722abf9e5..0025851ce 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -2,7 +2,7 @@ import torch from alibi_detect.od.backend.torch.knn import KNNTorch -# from alibi_detect.utils.pytorch.kernels import GaussianRBF +from alibi_detect.utils.pytorch.kernels import GaussianRBF def test_knn_torch_backend(): @@ -70,19 +70,25 @@ def test_knn_torch_backend_ts(): assert torch.all(pred_1 == pred_2) -# def test_knn_kernel(accumulator): -# kernel = GaussianRBF(sigma=torch.tensor((1))) -# knn_torch = KNNTorch(k=[4, 5], kernel=kernel, accumulator=accumulator) -# x_ref = torch.randn((1024, 10)) -# knn_torch.fit(x_ref) -# x = torch.randn((3, 10)) -# scores = knn_torch(x) -# assert scores.shape == (3,) +def test_knn_kernel(accumulator): + kernel = GaussianRBF(sigma=torch.tensor((0.25))) + knn_torch = KNNTorch(k=[4, 5], kernel=kernel, accumulator=accumulator) + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + result = knn_torch.predict(x) + assert result['scores'].shape == (3,) -# knn_torch.infer_threshold(x_ref, 0.1) -# outputs = knn_torch.predict(x) -# assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) + knn_torch.infer_threshold(x_ref, 0.1) + outputs = knn_torch.predict(x) + assert torch.all(outputs['preds'] == torch.tensor([False, False, True])) -# x = torch.randn((1, 10)) * 100 -# print(knn_torch.predict(x)) -# # assert knn_torch(x).item() + x = torch.randn((1, 10)) * 100 + assert knn_torch(x).item() + + """Can't convert GaussianRBF to torchscript due to torchscript type + constraints""" + # pred_1 = knn_torch(x) + # knn_torch = torch.jit.script(knn_torch) + # pred_2 = knn_torch(x) + # assert torch.all(pred_1 == pred_2) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 0a3334abd..7835b1e02 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -10,6 +10,7 @@ class TorchOutlierDetector(torch.nn.Module, ABC): """ Base class for torch backend outlier detection algorithms.""" threshold_inferred = False + threshold = None def __init__(self): super().__init__() @@ -41,7 +42,10 @@ def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: self.threshold_inferred = True def predict(self, X: torch.Tensor) -> torch.Tensor: - output = {'threshold_inferred': self.threshold_inferred} + output = { + 'threshold_inferred': self.threshold_inferred, + 'threshold': self.threshold + } raw_scores = self.score(X) output['scores'] = self._accumulator(raw_scores) output['preds'] = self._classify_outlier(output['scores']) From e95c884921feeeb91f8032a5dd808980c4077a4b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 23 Nov 2022 10:39:00 +0000 Subject: [PATCH 010/247] Minor correction --- .../od/backend/tests/test_knn_backend.py | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 0025851ce..7e22b5352 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -7,7 +7,7 @@ def test_knn_torch_backend(): knn_torch = KNNTorch(k=5) - x = torch.randn((3, 10)) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) with pytest.raises(AttributeError): # TODO: should be a different error! @@ -24,26 +24,22 @@ def test_knn_torch_backend(): knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) - - x = torch.randn((1, 10)) * 100 - assert knn_torch(x) + assert torch.all(outputs['preds'] == torch.tensor([False, False, True])) + assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) def test_knn_torch_backend_ensemble(accumulator): knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) - x = torch.randn((3, 10)) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) result = knn_torch.predict(x) assert result['scores'].shape == (3, ) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs['preds'] == torch.tensor([False, False, False])) - - x = torch.randn((1, 10)) * 100 - assert knn_torch(x) + assert torch.all(outputs['preds'] == torch.tensor([False, False, True])) + assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) def test_knn_torch_backend_ensemble_ts(accumulator): @@ -82,9 +78,7 @@ def test_knn_kernel(accumulator): knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) assert torch.all(outputs['preds'] == torch.tensor([False, False, True])) - - x = torch.randn((1, 10)) * 100 - assert knn_torch(x).item() + assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) """Can't convert GaussianRBF to torchscript due to torchscript type constraints""" From 47eb6c75c7a200d86039f77f0acbb9dfd708b815 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 23 Nov 2022 16:14:18 +0000 Subject: [PATCH 011/247] Rewrite knn outlier detector --- alibi_detect/od/backend/__init__.py | 6 +++ alibi_detect/od/backend/torch/base.py | 25 +++++++--- alibi_detect/od/backend/torch/ensemble.py | 38 +++++++-------- alibi_detect/od/backend/torch/knn.py | 12 ++--- alibi_detect/od/base.py | 56 +++++++++-------------- alibi_detect/od/knn.py | 54 ++++++++++++---------- alibi_detect/od/tests/test_knn.py | 49 ++++---------------- 7 files changed, 110 insertions(+), 130 deletions(-) diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index d14b24a46..91a969cd9 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -1,3 +1,9 @@ from alibi_detect.utils.missing_optional_dependency import import_optional KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) +PValNormaliserTorch, ShiftAndScaleNormaliserTorch, TopKAggregatorTorch, AverageAggregatorTorch, \ + MaxAggregatorTorch, MinAggregatorTorch, AccumulatorTorch = import_optional( + 'alibi_detect.od.backend.torch.ensemble', + ['PValNormaliser', 'ShiftAndScaleNormaliser', 'TopKAggregator', + 'AverageAggregator', 'MaxAggregator', 'MinAggregator', 'Accumulator'] + ) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 7835b1e02..dcb281384 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -1,8 +1,10 @@ from __future__ import annotations -from abc import ABC, abstractmethod +import numpy as np +from typing import List, Dict, Union import torch import logging +from abc import ABC, abstractmethod logger = logging.getLogger(__name__) @@ -16,18 +18,30 @@ def __init__(self): super().__init__() @abstractmethod - def fit(self, X: torch.Tensor) -> None: - pass + def fit(self, x_ref: torch.Tensor) -> None: + raise NotImplementedError() @abstractmethod def score(self, X: torch.Tensor) -> torch.Tensor: - pass + raise NotImplementedError() + + def _to_tensor(self, X: Union[List, np.ndarray]): + return torch.as_tensor(X, dtype=torch.float32) + + def _to_numpy(self, X: Union[Dict[str, torch.tensor], torch.tensor]): + if isinstance(X, dict): + output = X + for k, v in X.items(): + if isinstance(v, torch.Tensor): + output[k] = v.cpu().detach().numpy() + else: + output = X.cpu().detach().numpy() + return output def _accumulator(self, X: torch.Tensor) -> torch.Tensor: return self.accumulator(X) if self.accumulator is not None else X def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: - # check threshold has has been inferred. return scores > self.threshold if self.threshold_inferred else None def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: @@ -35,7 +49,6 @@ def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: if self.threshold_inferred else None def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: - # check detector has been fit. self.val_scores = self.score(X) self.val_scores = self._accumulator(self.val_scores) self.threshold = torch.quantile(self.val_scores, 1-fpr) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 9213c09f6..aa5fbe9ff 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -class BaseTransform(Module, ABC): +class BaseTransformTorch(Module, ABC): """Base Transform class. provides abstract methods for transform objects that map a numpy @@ -46,12 +46,12 @@ def forward(self, X: torch.Tensor): return self.transform(X=X) -class FitMixin: +class FitMixinTorch: """Fit mixin Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. - TODO: this should be encorporated into alibi_detect/base.py FitMixin once we can be sure that the + TODO: this should be encorporated into alibi_detect/base.py FitMixinTorch once we can be sure that the behavour is compatible. """ _fitted = False @@ -59,7 +59,7 @@ class FitMixin: def __init__(self): super().__init__() - def fit(self, X: torch.Tensor) -> BaseTransform: + def fit(self, X: torch.Tensor) -> BaseTransformTorch: self._fit(X) self._fitted = True return self @@ -74,7 +74,7 @@ def check_fitted(self): raise ValueError(f'{self.__class__.__name__} has not been fit!') -class BaseFittedTransform(BaseTransform, FitMixin): +class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): """Base Fitted Transform class. Provides abstract methods for transforms that have an aditional @@ -82,15 +82,15 @@ class BaseFittedTransform(BaseTransform, FitMixin): """ def __init__(self): - BaseTransform.__init__(self) - FitMixin.__init__(self) + BaseTransformTorch.__init__(self) + FitMixinTorch.__init__(self) def transform(self, X: torch.Tensor): self.check_fitted() return self._transform(X) -class PValNormaliser(BaseFittedTransform): +class PValNormaliser(BaseFittedTransformTorch): """Maps scores to there p values. Needs to be fit on a reference dataset using fit. Transform counts the number of scores @@ -112,7 +112,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return 1 - p_vals -class ShiftAndScaleNormaliser(BaseFittedTransform): +class ShiftAndScaleNormaliser(BaseFittedTransformTorch): """Maps scores to their normalised values. Needs to be fit on a reference dataset using fit. Subtracts the dataset mean and @@ -123,7 +123,7 @@ def __init__(self): self.val_means = None self.val_scales = None - def _fit(self, val_scores: torch.Tensor) -> BaseTransform: + def _fit(self, val_scores: torch.Tensor) -> BaseTransformTorch: self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] @@ -131,7 +131,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return (scores - self.val_means)/self.val_scales -class TopKAggregator(BaseTransform): +class TopKAggregator(BaseTransformTorch): def __init__(self, k: Optional[int] = None): """Takes the mean of the top k scores. @@ -151,7 +151,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return sorted_scores[:, -self.k:].mean(-1) -class AverageAggregator(BaseTransform): +class AverageAggregator(BaseTransformTorch): """Averages the scores of the detectors in an ensemble. Parameters @@ -170,7 +170,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return scores @ self.weights -class MaxAggregator(BaseTransform): +class MaxAggregator(BaseTransformTorch): """Takes the max score of a set of detectors in an ensemble.""" def __init__(self): super().__init__() @@ -180,7 +180,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return vals -class MinAggregator(BaseTransform): +class MinAggregator(BaseTransformTorch): """Takes the min score of a set of detectors in an ensemble.""" def __init__(self): super().__init__() @@ -190,10 +190,10 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return vals -class Accumulator(BaseFittedTransform): +class Accumulator(BaseFittedTransformTorch): def __init__(self, - normaliser: BaseFittedTransform = None, - aggregator: BaseTransform = AverageAggregator()): + normaliser: BaseFittedTransformTorch = None, + aggregator: BaseTransformTorch = AverageAggregator()): """Wraps a normaliser and aggregator into a single object. The accumulator wraps normalisers and aggregators into a single object. @@ -201,10 +201,10 @@ def __init__(self, Parameters ---------- normaliser - normaliser that's an instance of BaseFittedTransform. Maps the outputs of + normaliser that's an instance of BaseFittedTransformTorch. Maps the outputs of a set of detectors to a common range. aggregator - aggregator extendng BaseTransform. Maps outputs of the normaliser to + aggregator extendng BaseTransformTorch. Maps outputs of the normaliser to single score. """ super().__init__() diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index b0b79a3cd..9b6a49f30 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Union, List import numpy as np import torch from alibi_detect.od.backend.torch.ensemble import Accumulator @@ -8,8 +8,8 @@ class KNNTorch(TorchOutlierDetector): def __init__( self, - k, - kernel=None, + k: Union[np.ndarray, List], + kernel: Optional[torch.nn.Module] = None, accumulator: Optional[Accumulator] = None ): super().__init__() @@ -30,8 +30,8 @@ def score(self, X): all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] - def fit(self, X: torch.tensor): - self.x_ref = torch.as_tensor(X, dtype=torch.float32) + def fit(self, x_ref: torch.tensor): + self.x_ref = x_ref if self.accumulator is not None: - scores = self.score(X) + scores = self.score(x_ref) self.accumulator.fit(scores) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 58ee12c58..f5cc38e5f 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod import numpy as np import logging +from typing_extensions import Protocol, runtime_checkable from alibi_detect.base import BaseDetector logger = logging.getLogger(__name__) @@ -25,44 +26,31 @@ def fit(self, X: np.ndarray) -> None: def score(self, X: np.ndarray) -> np.ndarray: pass + @abstractmethod def infer_threshold(self, X: np.ndarray, fpr: float) -> None: - """ - Infers the threshold above which only fpr% of inlying data scores. - Also saves down the scores to be later used for computing p-values - of new data points (by comparison to the empirical cdf). - For ensemble models the scores are normalised and aggregated before - saving scores and inferring threshold. - """ - self.val_scores = self.score(X) - if self.ensemble: - self.val_scores = self.normaliser.fit(self.val_scores).transform(self.val_scores) \ - if getattr(self, 'normaliser') else self.val_scores - self.val_scores = self.aggregator.fit(self.val_scores).transform(self.val_scores) \ - if getattr(self, 'aggregator') else self.val_scores - self.threshold = np.quantile(self.val_scores, 1-fpr) - self.threshold_inferred = True + pass + @abstractmethod def predict(self, X: np.ndarray) -> np.ndarray: - """ - Scores the instances and then compares to pre-inferred threshold. - For ensemble models the scores from each constituent is added to the output. - p-values are also returned by comparison to validation scores (of inliers) - """ - output = {} - scores = self.score(X) - output['raw_scores'] = scores + pass + + +@runtime_checkable +class TransformProtocol(Protocol): + def transform(self, X): + pass + + def _transform(self, X): + pass - if getattr(self, 'normaliser'): - scores = self.normaliser.transform(scores) - output['normalised_scores'] = scores - if getattr(self, 'aggregator'): - scores = self.aggregator.transform(scores) - output['aggregate_scores'] = scores +@runtime_checkable +class FittedTransformProtocol(TransformProtocol, Protocol): + def fit(self, x_ref): + pass - if self.threshold_inferred: - p_vals = (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) - preds = scores > self.threshold - output.update(scores=scores, preds=preds, p_vals=p_vals) + def _fit(self, x_ref): + pass - return output + def check_fitted(self): + pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 1537341b7..1e98ddcfa 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,16 +1,13 @@ -from typing import Callable, Literal, Union, Optional +from typing import Callable, Literal, Union, Optional, List, Dict import numpy as np -from alibi_detect.od.base import OutlierDetector -from alibi_detect.od.backend.torch.ensemble import Accumulator - -from alibi_detect.od.backend import KNNTorch +from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol +from alibi_detect.od.backend import KNNTorch, AccumulatorTorch from alibi_detect.utils.frameworks import BackendValidator -X_REF_FILENAME = 'x_ref.npy' backends = { - 'pytorch': KNNTorch, + 'pytorch': (KNNTorch, AccumulatorTorch) } @@ -19,11 +16,12 @@ def __init__( self, k: Union[int, np.ndarray], kernel: Optional[Callable] = None, - normaliser=None, - aggregator=None, + normaliser: Optional[Union[TransformProtocol, FittedTransformProtocol]] = None, + aggregator: Optional[TransformProtocol] = None, backend: Literal['pytorch'] = 'pytorch' ) -> None: super().__init__() + backend = backend.lower() BackendValidator( backend_options={'pytorch': ['pytorch']}, @@ -36,19 +34,25 @@ def __init__( 'specify at least an aggregator if you want to use the knn detector' 'an ensemble like this.')) - self.k = k - self.kernel = kernel - self.ensemble = isinstance(self.k, (np.ndarray, list)) - self.normaliser = normaliser - self.aggregator = aggregator - self.fitted = False - self.backend = backends[backend] - - def fit(self, X: np.ndarray) -> None: - self.x_ref = self.backend.fit(X) - val_scores = self.score(X) - if getattr(self, 'normaliser'): - self.normaliser.fit(val_scores) - - def score(self, X: np.ndarray) -> np.ndarray: - return self.backend.score(X, self.x_ref, self.k, kernel=self.kernel) + backend, accumulator_cls = backends[backend] + accumulator = None + if normaliser is not None or aggregator is not None: + accumulator = accumulator_cls( + normaliser=normaliser, + aggregator=aggregator + ) + self.backend = backend(k, kernel=kernel, accumulator=accumulator) + + def fit(self, x_ref: Union[np.ndarray, List]) -> None: + self.backend.fit(self.backend._to_tensor(x_ref)) + + def score(self, X: Union[np.ndarray, List]) -> np.ndarray: + score = self.backend.score(self.backend._to_tensor(X)) + return self.backend._to_numpy(score) + + def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: + self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + + def predict(self, X: Union[np.ndarray, List]) -> Dict[str, np.ndarray]: + outputs = self.backend.predict(self.backend._to_tensor(X)) + return self.backend._to_numpy(outputs) diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index 9b12db2a4..40c460283 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -2,8 +2,8 @@ import numpy as np from alibi_detect.od.knn import KNN -from alibi_detect.od.backend.torch.ensemble import AverageAggregator, TopKAggregator, MaxAggregator, \ - MinAggregator, ShiftAndScaleNormaliser, PValNormaliser +from alibi_detect.od.backend import AverageAggregatorTorch, TopKAggregatorTorch, MaxAggregatorTorch, \ + MinAggregatorTorch, ShiftAndScaleNormaliserTorch, PValNormaliserTorch def test_knn_single(): @@ -11,60 +11,35 @@ def test_knn_single(): x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10]]) - assert knn_detector.predict(x)['raw_scores'] > 5 + assert knn_detector.predict(x)['scores'] > 5 x = np.array([[0, 0.1]]) - assert knn_detector.predict(x)['raw_scores'] < 1 + assert knn_detector.predict(x)['scores'] < 1 knn_detector.infer_threshold(x_ref, 0.1) x = np.array([[0, 10]]) pred = knn_detector.predict(x) - assert pred['raw_scores'] > 5 + assert pred['scores'] > 5 assert pred['preds'] assert pred['p_vals'] < 0.05 x = np.array([[0, 0.1]]) pred = knn_detector.predict(x) - assert pred['raw_scores'] < 1 + assert pred['scores'] < 1 assert not pred['preds'] assert pred['p_vals'] > 0.7 -@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), MaxAggregator, MinAggregator]) -@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliser, PValNormaliser, lambda: None]) +@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), + MaxAggregatorTorch, MinAggregatorTorch]) +@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) def test_knn_ensemble(aggregator, normaliser): knn_detector = KNN( k=[8, 9, 10], aggregator=aggregator(), normaliser=normaliser() ) - - x_ref = np.random.randn(100, 2) - knn_detector.fit(x_ref) - x = np.array([[0, 10], [0, 0.1]]) - knn_detector.infer_threshold(x_ref, 0.1) - pred = knn_detector.predict(x) - - assert np.all(pred['preds'] == [True, False]) - if isinstance(knn_detector.normaliser, ShiftAndScaleNormaliser): - assert np.all(pred['normalised_scores'][0] > 1) - assert np.all(pred['normalised_scores'][1] < 0) - elif isinstance(knn_detector.normaliser, PValNormaliser): - assert np.all(pred['normalised_scores'][0] > 0.8) - assert np.all(pred['normalised_scores'][1] < 0.3) - - -@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), MaxAggregator, MinAggregator]) -@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliser, PValNormaliser, lambda: None]) -def test_knn_keops(aggregator, normaliser): - knn_detector = KNN( - k=[8, 9, 10], - aggregator=aggregator(), - normaliser=normaliser(), - backend='keops' - ) - x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0, 0.1]]) @@ -72,9 +47,3 @@ def test_knn_keops(aggregator, normaliser): pred = knn_detector.predict(x) assert np.all(pred['preds'] == [True, False]) - if isinstance(knn_detector.normaliser, ShiftAndScaleNormaliser): - assert np.all(pred['normalised_scores'][0] > 1) - assert np.all(pred['normalised_scores'][1] < 0) - elif isinstance(knn_detector.normaliser, PValNormaliser): - assert np.all(pred['normalised_scores'][0] > 0.8) - assert np.all(pred['normalised_scores'][1] < 0.3) From 4b201dfa865cba26d1a4299f1f396dc8167db595 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 24 Nov 2022 17:43:42 +0000 Subject: [PATCH 012/247] Surface errors if for unfit detectors --- .../od/backend/tests/test_knn_backend.py | 63 +++++++++++++++++-- alibi_detect/od/backend/torch/base.py | 12 +++- alibi_detect/od/backend/torch/ensemble.py | 2 +- alibi_detect/od/backend/torch/knn.py | 4 +- alibi_detect/od/knn.py | 5 +- alibi_detect/od/tests/test_knn.py | 7 +++ 6 files changed, 81 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 7e22b5352..01ffec5f8 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -8,11 +8,6 @@ def test_knn_torch_backend(): knn_torch = KNNTorch(k=5) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - - with pytest.raises(AttributeError): - # TODO: should be a different error! - knn_torch(x) - x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) outputs = knn_torch.predict(x) @@ -45,6 +40,15 @@ def test_knn_torch_backend_ensemble(accumulator): def test_knn_torch_backend_ensemble_ts(accumulator): knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + + with pytest.raises(ValueError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + with pytest.raises(ValueError) as err: + knn_torch.predict(x) + assert str(err.value) == 'KNNTorch has not been fit!' + x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) knn_torch.infer_threshold(x_ref, 0.1) @@ -86,3 +90,52 @@ def test_knn_kernel(accumulator): # knn_torch = torch.jit.script(knn_torch) # pred_2 = knn_torch(x) # assert torch.all(pred_1 == pred_2) + + +def test_knn_torch_backend_ensemble_fit_errors(accumulator): + knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) + assert not knn_torch._fitted + + x = torch.randn((1, 10)) + with pytest.raises(ValueError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + with pytest.raises(ValueError) as err: + knn_torch.predict(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + assert knn_torch._fitted + + with pytest.raises(ValueError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + + assert knn_torch.predict(x) + + +def test_knn_torch_backend_fit_errors(): + knn_torch = KNNTorch(k=4) + assert not knn_torch._fitted + + x = torch.randn((1, 10)) + with pytest.raises(ValueError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + with pytest.raises(ValueError) as err: + knn_torch.predict(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + + assert knn_torch._fitted + + with pytest.raises(ValueError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + + assert knn_torch.predict(x) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index dcb281384..7938907a1 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -2,6 +2,7 @@ import numpy as np from typing import List, Dict, Union import torch +from alibi_detect.od.backend.torch.ensemble import FitMixinTorch import logging from abc import ABC, abstractmethod @@ -9,7 +10,7 @@ logger = logging.getLogger(__name__) -class TorchOutlierDetector(torch.nn.Module, ABC): +class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): """ Base class for torch backend outlier detection algorithms.""" threshold_inferred = False threshold = None @@ -18,13 +19,19 @@ def __init__(self): super().__init__() @abstractmethod - def fit(self, x_ref: torch.Tensor) -> None: + def _fit(self, x_ref: torch.Tensor) -> None: raise NotImplementedError() @abstractmethod def score(self, X: torch.Tensor) -> torch.Tensor: raise NotImplementedError() + @torch.jit.ignore + def check_threshould_infered(self): + if not self.threshold_inferred: + raise ValueError((f'{self.__class__.__name__} has no threshold set, ' + 'call `infer_threshold` before predicting.')) + def _to_tensor(self, X: Union[List, np.ndarray]): return torch.as_tensor(X, dtype=torch.float32) @@ -55,6 +62,7 @@ def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: self.threshold_inferred = True def predict(self, X: torch.Tensor) -> torch.Tensor: + self.check_fitted() output = { 'threshold_inferred': self.threshold_inferred, 'threshold': self.threshold diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index aa5fbe9ff..ba5c6b1be 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -60,8 +60,8 @@ def __init__(self): super().__init__() def fit(self, X: torch.Tensor) -> BaseTransformTorch: - self._fit(X) self._fitted = True + self._fit(X) return self def _fit(self, X: torch.Tensor): diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 9b6a49f30..5bc0a5813 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -21,16 +21,18 @@ def __init__( def forward(self, X): raw_scores = self.score(X) scores = self._accumulator(raw_scores) + self.check_threshould_infered() preds = scores > self.threshold return preds.cpu() def score(self, X): + self.check_fitted() K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) bot_k_dists = torch.topk(K, torch.max(self.ks), dim=1, largest=False) all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] - def fit(self, x_ref: torch.tensor): + def _fit(self, x_ref: torch.tensor): self.x_ref = x_ref if self.accumulator is not None: scores = self.score(x_ref) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 1e98ddcfa..26117ec8e 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -28,11 +28,10 @@ def __init__( construct_name=self.__class__.__name__ ).verify_backend(backend) - # TODO: Abstract to pydantic model. if isinstance(k, (list, np.ndarray)) and aggregator is None: raise ValueError((f'k={k} is type {type(k)} but aggregator is {aggregator}, you must ' - 'specify at least an aggregator if you want to use the knn detector' - 'an ensemble like this.')) + 'specify at least an aggregator if you want to use the knn detector ' + 'ensemble like this.')) backend, accumulator_cls = backends[backend] accumulator = None diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index 40c460283..21f777410 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -47,3 +47,10 @@ def test_knn_ensemble(aggregator, normaliser): pred = knn_detector.predict(x) assert np.all(pred['preds'] == [True, False]) + + +def test_incorrect_knn_ensemble_init(): + with pytest.raises(ValueError) as err: + KNN(k=[8, 9, 10]) + assert str(err.value) == ("k=[8, 9, 10] is type but aggregator is None, you must specify at least an" + " aggregator if you want to use the knn detector ensemble like this.") From 5e3bc7b8957c575436f3544ebb443b6d1da347ca Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 25 Nov 2022 09:16:47 +0000 Subject: [PATCH 013/247] Merge backend test features into test_knn_backend --- alibi_detect/od/backend/tests/conftest.py | 10 ---------- alibi_detect/od/backend/tests/test_knn_backend.py | 9 +++++++++ 2 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 alibi_detect/od/backend/tests/conftest.py diff --git a/alibi_detect/od/backend/tests/conftest.py b/alibi_detect/od/backend/tests/conftest.py deleted file mode 100644 index caacb8f28..000000000 --- a/alibi_detect/od/backend/tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -import pytest -from alibi_detect.od.backend.torch.ensemble import Accumulator, PValNormaliser, AverageAggregator - - -@pytest.fixture(scope='session') -def accumulator(request): - return Accumulator( - normaliser=PValNormaliser(), - aggregator=AverageAggregator() - ) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 01ffec5f8..40cd951da 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -3,6 +3,15 @@ from alibi_detect.od.backend.torch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF +from alibi_detect.od.backend.torch.ensemble import Accumulator, PValNormaliser, AverageAggregator + + +@pytest.fixture(scope='session') +def accumulator(request): + return Accumulator( + normaliser=PValNormaliser(), + aggregator=AverageAggregator() + ) def test_knn_torch_backend(): From e272c74f91a4361d78f37e6f36d897b374f5a489 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 25 Nov 2022 10:16:24 +0000 Subject: [PATCH 014/247] Make knn tests better --- alibi_detect/od/tests/test_knn.py | 112 +++++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index 21f777410..d4dd6c5fa 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -1,40 +1,76 @@ import pytest import numpy as np +import torch from alibi_detect.od.knn import KNN from alibi_detect.od.backend import AverageAggregatorTorch, TopKAggregatorTorch, MaxAggregatorTorch, \ MinAggregatorTorch, ShiftAndScaleNormaliserTorch, PValNormaliserTorch -def test_knn_single(): +def make_knn_detector(k=5, aggregator=None, normaliser=None): + knn_detector = KNN(k=k, aggregator=aggregator, normaliser=normaliser) + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + knn_detector.infer_threshold(x_ref, 0.1) + return knn_detector + + +def test_unfitted_knn_single_score(): + knn_detector = KNN(k=10) + x = np.array([[0, 10], [0.1, 0]]) + with pytest.raises(ValueError) as err: + _ = knn_detector.predict(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + +def test_fitted_knn_single_score(): knn_detector = KNN(k=10) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) - x = np.array([[0, 10]]) - assert knn_detector.predict(x)['scores'] > 5 + x = np.array([[0, 10], [0.1, 0]]) + y = knn_detector.predict(x) + assert y['scores'][0] > 5 + assert y['scores'][1] < 1 + + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['preds'] is None + assert y['p_vals'] is None - x = np.array([[0, 0.1]]) - assert knn_detector.predict(x)['scores'] < 1 +def test_fitted_knn_predict(): + knn_detector = make_knn_detector(k=10) + x_ref = np.random.randn(100, 2) knn_detector.infer_threshold(x_ref, 0.1) + x = np.array([[0, 10], [0, 0.1]]) + y = knn_detector.predict(x) + assert y['scores'][0] > 5 + assert y['scores'][1] < 1 + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_vals'].all() + assert (y['preds'] == [True, False]).all() - x = np.array([[0, 10]]) - pred = knn_detector.predict(x) - assert pred['scores'] > 5 - assert pred['preds'] - assert pred['p_vals'] < 0.05 - x = np.array([[0, 0.1]]) - pred = knn_detector.predict(x) - assert pred['scores'] < 1 - assert not pred['preds'] - assert pred['p_vals'] > 0.7 +@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), + MaxAggregatorTorch, MinAggregatorTorch]) +@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) +def test_unfitted_knn_ensemble(aggregator, normaliser): + knn_detector = KNN( + k=[8, 9, 10], + aggregator=aggregator(), + normaliser=normaliser() + ) + x = np.array([[0, 10], [0.1, 0]]) + with pytest.raises(ValueError) as err: + _ = knn_detector.predict(x) + assert str(err.value) == 'KNNTorch has not been fit!' @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), MaxAggregatorTorch, MinAggregatorTorch]) @pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) -def test_knn_ensemble(aggregator, normaliser): +def test_fitted_knn_ensemble(aggregator, normaliser): knn_detector = KNN( k=[8, 9, 10], aggregator=aggregator(), @@ -43,10 +79,29 @@ def test_knn_ensemble(aggregator, normaliser): x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0, 0.1]]) - knn_detector.infer_threshold(x_ref, 0.1) - pred = knn_detector.predict(x) + y = knn_detector.predict(x) + assert y['scores'].all() + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['preds'] is None + assert y['p_vals'] is None - assert np.all(pred['preds'] == [True, False]) + +@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), + MaxAggregatorTorch, MinAggregatorTorch]) +@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) +def test_fitted_knn_ensemble_predict(aggregator, normaliser): + knn_detector = make_knn_detector( + k=[8, 9, 10], + aggregator=aggregator(), + normaliser=normaliser() + ) + x = np.array([[0, 10], [0, 0.1]]) + y = knn_detector.predict(x) + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_vals'].all() + assert (y['preds'] == [True, False]).all() def test_incorrect_knn_ensemble_init(): @@ -54,3 +109,22 @@ def test_incorrect_knn_ensemble_init(): KNN(k=[8, 9, 10]) assert str(err.value) == ("k=[8, 9, 10] is type but aggregator is None, you must specify at least an" " aggregator if you want to use the knn detector ensemble like this.") + + +@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), + MaxAggregatorTorch, MinAggregatorTorch]) +@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) +def test_knn_ensemble_torch_script(aggregator, normaliser): + knn_detector = make_knn_detector(k=[5, 6, 7], aggregator=aggregator(), normaliser=normaliser()) + tsknn = torch.jit.script(knn_detector.backend) + x = torch.tensor([[0, 10], [0, 0.1]]) + y = tsknn(x) + assert torch.all(y == torch.tensor([True, False])) + + +def test_knn_single_torchscript(): + knn_detector = make_knn_detector(k=5) + tsknn = torch.jit.script(knn_detector.backend) + x = torch.tensor([[0, 10], [0, 0.1]]) + y = tsknn(x) + assert torch.all(y == torch.tensor([True, False])) From bc7de796fcc196e8424331fd6173ad21e771b5f0 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 25 Nov 2022 10:17:42 +0000 Subject: [PATCH 015/247] Remove test file --- test/detectors/0/model.pt | Bin 8346 -> 0 bytes test/detectors/1/model.pt | Bin 8382 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/detectors/0/model.pt delete mode 100644 test/detectors/1/model.pt diff --git a/test/detectors/0/model.pt b/test/detectors/0/model.pt deleted file mode 100644 index 6dd47af59f0e47e47ed73d4071f039515e11868a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8346 zcmbt32|Sct_hU`g>>)duEHh-uHuKmO*|(^SeH}Aqv1BPic3GnATMD5KA?A@S%9^55 zLM25=B})8fs<&6Y_4fb1@A}=D@jUmObI*S6x#q?+v=9gb1LPlr8Nvhc!@9Yn;cm`E zXSlq94hO{HAA|85oVoE9hT`_AcN9dFY5p&Gk(==<80e_~0`ETv{pLY%?)*VA6StJUm*7 zTtk2WHMb?n+o%Xhcg_KXyNS0ZSxEtxjSmB{>j5HhCV2J*GOE7+3o8B7$tLR#VzK}v_@1DP+ffvaX+B)^+Eq&1Zj;Li3saGU2t zkoxK>kQPi08W>NIvJV!4d0PzurBnUj>HGGi{RnrEXL}b?q=OoKFLWC89iMKtFdaj} z)Z-A@k8_X2f^MqbI3Y9FYro- zHKL|W9pK}lMNW;T0bQ&GVBpc+gfFXmd zrq}6b0P;N*GI^*E$<6czaEz(~wF^r?7wKSN2ATz)>RtiHJBLY%_5k=nY#BHn*#gLt z^^~~e3G6#^2&4{w2ej;oCw2Ro00PJGz=tLwlBtp)(87%d9yGNgr5lsL>MLiEG&d^% z#Q_DdSB;N^?;R&q*UkaQ1!RGdf4K?1N+s72(JhmJhvYsPpwBIHoC@sLNC34b(!sa8CP+Kh*GV%UmjNUT0bn=1ir^T_ z17)EH0M;!LAlvm$@DTeMz{H>g>Dl57=*yQPwd@Z9`4u}s4wq=~V;Bt3t}+2_PsoEQ z8ZSvi`IDNLk#!``^^2ffhXjb&&)O=(CIYB$>jSZel0lXWHVB9{0*s981Z=~~z-LWy zNV*pyEj-bgVBmW?V6*~GVn3x1$P8Hm0eD6&d(I00#V={>-0VeYAP~y@ySf$=Hl$?>yB|Fz`iVi5u5|P zF`kTiU*2Q!j5x@y$Z+TZO1L26oxL&SmpH1p13cvaQQicUM*td)qM!(J#=D`suvlLl zwdny4%4#pXJHZQ!c0+k%JlyekcQ+hO{DG~T-{5GCAqU8j4T5#?A>imXZSw10Fc=s) zxCNly2{2bIhDfIFj%PIa@?^g^fyjuXr|iv(D$a{clZuQ-9=Bx^l0R_ji@`7&o0?iv zT%y?HWY==B#712oo|_?-&J7v2?coL(%( z-DDbE6dG>E)MP5u6gpmC(Q$th6h)O}KHU9W(C#>%_y89iFL`b!Pd*sM5*f!hsHDk) zpiGJ-a*;C+1?I| z{LF8dw#zfOyyYG-t+p<~S1XzB&_1MUos*iW($AkIX_el#@bs*pz$L4>4{MGE2nRlK zO^XtRRr8i$<%5;iH6R@s%uC(-i}gyhrw+UuGJYORV?k6aj34P-eBg8@5qyCeMxRL-Z=I&pB4@wsSt1NN6QR#x&vb?yUk= zv<6QGWgBY-R!7|gTeo;*m3^pqH8w3~zm%GM_b}TBzn9cyYN{vZNmLZ|iBxLt*hO{* z(|=W;Aw=wdQy=T}-FA+%S4~h&(y=~O4#dSie;yuU31`Bq{28EKHq1qO2ZYeM- zPoO1gd*b<$_et?xfYAoMWjsDG5w#*jIvy(JS6p_ty)~P#zOb=n`tbS0=+ul0IiVA) zdtQuZAA5bTA8r=DFtIpwhBHw*+dsLc(j!o&qdV!!T4O=1ZREaM_dCggQM1HqHfdvR zkCBEw_C8s8)$=jbGQ=Yg*~FadlIS6psFUWt5l$9TIvr{}xU6WTv3M_m*~Tdo!@3nM zec8PD?jwOk{;*Ru%(!RJf~8tlrIqdZ!cOwL%n^G`Pp|K>iZUByD$~F%$C)B-@V;O; zc*d>IaDR-$CHgT|3#UQBwfWQ-(^4T#Cw`uk?j@1HgpkCtpuo480>{l9w2m*2lRVp& zb7h0~4ti-`UiOTO=5Cv?8mW#cp{I>|HvIZr%BORkWz_sL=8^2>ZP26}s;TMu5p+GC ztx*D7bBI7~gXh}F6kkIWYHJhm*vHJu@$6-3Cc z)?;L=ApWaW?1qT`+A6;KE(Qbt7LVcI$>BG0j_k*Qm9c&5IQUPA4!t zxdg}#J14yHbBjrrP}A8GS?vstny`>fiKO0;8a=^Ih zRdYgJVPk{mJGrnmEsoq$8m{^ZhOp{+iWJp`E%$VilD`j)nh^H?LW-LmiseUn zB5rB>wrOpnn|R=LP#|Qs>~z*^xfa?GxiY6>8@|$QH1qoONf+!7mt<50VLx3vj9_3B zYF@|}druA77}lKSRTAdb&Ea14+Few#p|S3|*V@UcxInl1odbvcVNS{F`q`5g&xpV0 z(ldJC6@>O#jKgDkkJ;a`KpxDhOTyQKAtACE+jLa z_&_!4<1b=;W85sQd9@?i+d4I%njaRqpJ)b#FT*Ow9=t6z8=wfzP4YMsi7dFg7qSe^G{Bvqovd)y2d1sOK%YAI)sG>>;A!Yrp?@7^$z8%F>DeQ^P@LHH0o@v zfpbLDArJJJWbLc9N|y^vs_2*Ois}RPbKJ?-QjTtIWTCy8X)z7&rggjI6%l}x)f(plM7in%5suO|gj9<8(x>4{*d%M3Vc zC?1~A@v8q!$ns%!Zr7Xk9zCRuGX-POs?4Q|vUN~jcvbtA*(#1{rI47ad=#i3WXVgxp5I+F)W8Uml!M;&Yla%_nj+{PdU68b-ro;foPKohP6 zhSSJ#R~?*%n)$JcPsdiZJZd-aVAR)(i2-4Q{`BuBq*9yiJjbg|%&a7Itq zKQ0?nPRB;23qK@IG8YmV3e!sl6V+=@o3s1+ja+iEoSL5;aXcKPaMreyqwDPHQs-(H zEh*c5y_JvUudMlB%qc9q7p_4vJDzS7+0U>sh?!g`eu6u4w*fOawJ!;!X#)qorzfvJB;^)cT7`JDi6k+?*qC zi0Ja(T_UX>yr~I^lBaE#r!FlDaLtg-Ii4Y=(lJ=zP;gxHkh6TRxm6r&Rh-wuGSA6> zdy+g+RWgmR+djP~VZYYSAlf6$W_(Gz-|da#UCfH?8syB@9@9^i{@0?wBLMT|2;+?Y5`%x$f67ej!9-~>Yb~rYCUT}h z8I=;FPmM3p*xb9If1^Ch=|sO}S@p*Sk8`peC2n;SU7tF& z_@2eLx~wpGSa5EZ(=;M?c75%XVhu#=gUV}`pqGT6tiVqRAvYa5BgMvRhzhXuo*geN zpX9z6xqRj2to*Bdg@<{!)oqVIXn;2~CehkmQF+#EftUaEMK`y1=yJ)EW82L?6m0Up zj%~lt&E`lZ`yWNJOj~2DfFZ{@ZYg%D%bYhCf^x5z+!10$T@mUvifF-cln##{uHN>+ zfXATEd3f%eDzbOCx> zA2~E9S7kA{7o+Z3fak=qWgau#`0I{l_^Y$^D@G*VHmgze^~j#kEIEgB4lW7xFZ~l^ zMVaVVU3A}i&d4psPvq$0C*IIYR}rL$G=j+seq-*IhQ-VY<+bV~yviGD4~42(@Qsfy z#;%C^?Dh@_X6j)zbnL-J!7T6hURL$NH?}uMsa{p@b`{gYXsDdhfX%hmH%i9LM!yP} zm@o26eJrA+v>}h6b;{Y$yk}5!Ey_@IZi4RP@=Q$A$9r3E&~vcL>+I&qym_fzGll=~ zPL1ShjhSMu!xNvz-q1Xt&f^`P_ZgVO$UVnot-~h!eMQ*(6tCZ8&+FwBmk?#`tq)Pj zu$tuW5=+UM_Iz0vjS5H(*5qP+SU$rr@27U(dWTYov~zARC&LqC>-}o#=JQ1y583OJ zTY0GbMLw2~T`gNgJ1Yc4#~pE;kRI<5^j^RH?)|Q2QF_mQ%r-_L2kkmB4nNg=8|&*a zPEUQ`WK0VekKEX9ws&`EEK8f`i;z>(Y`r#bQN{V*j&h^8vPL$ISnS)02R$uD0#^Rm zMU|*B7GeZVbgin&t|y^>HhTnl=24TNZs3V@19LZgqV`l1y|-V zS7+B!7g@XMNv0TSgR-XPOBJQbmYwBxGRErC_mWmu9m3yXj#iwTt2B5%vNXNZ`i^}F z@gX=}SZc=!my-8lOPMY-%TBo{)#|$*g9s<0bKGp=y$_$onhg2OKUP_(fx+%l(H@eu zy%~DwfTKfP%ROG1y8B~gh+MSK`g18P{-RZ%A=n)TAK_Nt6Qk^g=%qVX>Y=G0;hr*G zpjU*kzHmDNr6_&de=h_$y*saUl(}e&XpMmJdj}IfM-yi$W?$m~sGMp>=X0OlCH{Ot z`!?0$)zJ#l!Sq3F@0qJ={WwA{-SN6;<(#OyN*U)zIS){sZfT7yGrc`+b%7}dRTOkS zu5)s5=)UOM{++T|pH?GAt(LuZL5({DyxgBV30pfI6ITvNx}Up1^4^~us@}Aq-rS&Z zCEie8cadeMyZP}q=Bmwg6{~x^!=I&$ZSgFNo2ls5Ox_!S;*o4xv*t1TtVhlJwDpkB zC;Tom4?|3BWtz>5!yPrwMfRh1$(}(`kR?+7CqnuH%H|+l{ow1O;Y3@PbmN@oBEpsL zs8nG=k6`e2!(!BlX_)M&_vMgmjWN+?E)%M|63hYa!LU%82RK1!E*>fYKhhi0EA-~l zj`My;^6Ce>TOv^zM_VEkM=r%}K+igdw1uDv)ekrWZ*!&8#(YHf@~SW$s+nZ&*ITb% zIk!H(0Hs4ldoB`IL97p+tZy3IjS=^AGNtqm-_@Ii6G@G(oB=WBh zOk7>rfjkMHNL=9{C9#u^6M|V$oui8Zc{k|}b?jL*sd~zx;?HO2Z;)2bvEly=zO5wf zlSA}L!*oM~AcJ?5GMY2ewqV^&j^DvqHru?>r>Uaq@&O-0)ckbKDd69@jog*~pGf^(5&cYb zbE9YX4@Aw#m#M!b`mZgrpYd<*hiLwQKa=Ud!T)6e`SqALEGkIfP1N6kaH{{L8i-u2&}zBvYQ z{;+F$fxq!@ieg_k_&0*hVT9!e1lHdXd<$IW#`N^E-)QVVKE&o* diff --git a/test/detectors/1/model.pt b/test/detectors/1/model.pt deleted file mode 100644 index 2b45292aa63f464e4355baae8780c4165c43c497..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8382 zcmb_h2|U!>+aJ5E8C$7jC(8_3vd&MIA!A>oGRD4)8Dn3IP_`68_AP~Ir-=Eftl24w zvQ;Ruwh-~oaBo-Ly7&FR|M$%2%<`M_JkL4L@;%Rajxmag8UmrGhy1NELAW5Ectaj8Sy-z++uSG9z*~{s+E@?R;gB8p^-m`!_+qc@Wq>2Dpy3 zBT^EBfi`zmbs?rXU>p_Pdg+ou84ORuOgYP?3A z7$`%mseu3`rzz1*FCRg3#Rh~s3f3o>i2}D!CxEl-YGA1*n|g^@4XEp>qQ+MN5ceKz z0G8Nmfw(eWM0D;$#0PB$urBT!7|(y3xVW1SAs(Cqq)ui4cZ`~dp3RxWHH9;v$m2?I zr^_Hnd1nPk3Zw*)s8M3Zv3xLlhYlcju@g*xZcWrza{{?`H6i#KD8Wy>>7d8(M6HS8 z5CWzgqn7b16ESBm04h{h6KzE#K^yM-#AD*d;M319LD?Jf2$iZ1qE%Tp*uF4>sMK-= zZ#S5$T`yJ!c5_f8#s`vsCYD^VdZPnz&j=1)n6U* zp_f#3O@9L*K2adf^>iRO85aQC$a3&-ULj~N9tcc{rGXb)mVx2MKBBBO0Dckp44jUr z10+dyieGdAjszVCDZ@Sjb%$b!EuIFzo>Shyml|H8q1-;8juQ*KsHsPYS0{iKw=W^6 zno9xME*Y?0X*bckeVABrcLq4UM-u4I?M5i>)&^CX7yz~cN$`;!rCL{$0Qf*$3h>Kf z2ekDqh>j1RAPDc`!LwsTaOuk^!mnH$flU$syrXxiyS5U5``6t-$FGUNReTB9V?IMX z{3yPz+i(atT!H|Dlgz;2g4ViwFAjiN@)v-EDskZ9(PVJxz$kI=`Z{sy>t_JL>;teF z-cf@NWrLDpM*$YPaFF#uBY2$c5@3KVM7Y#>0NT<;2zBdYKu)O$2(^y_zlOqq!{r8` z9f>Ju<`-QJ3B!9@pB;aH48O}xf&P|(FjelSn0CHSXSvgwsQ9&T&{`3~N2;=LVQ(P$V3>t^qUc69P}a`1)W9bxtu2M;HlqYrFz z0L%yD?}l@Mq5mZC#Nb@8PH1TctxXENHv<83ARcO}y)kY$(r^Mr%uz1VZ?u~a z+Sw0_MU$BZV7wjCu6Vo$fzt3Olswwi+sVfjk99=5;hdeky`3BhRIx{QZ2yNqje;B{ zVI2f-@9smO**5L1+2L?765RP=oqS*pc$_aux|279!R8ZvHy>XH0xfxNt`sq@B$*T> zKGFobZA|`U?H)KB1Ip0QoUADZ0{!MPDM?H`dCOZ;SCT;2 zKZPK%c#Q9NDbUTe%MjRInNcky860F8jwni!6iTuj*DZFOe<+G82aKR`}2e-0lG(OTYfrgV)2KGDBFqI7(RiId53%XFhSJ3lAJi%oBXADzJBSIjIMi zyvWM2*9w_I!829<%uP4wVoLw>h4qgeIrTk*W^YDdhpiufl3{lKFx7J2lF?kGMj2|A zj)rMGiHdnX+t*BIpD=hnMl!E`mw#uq+H}PZ8D=E~{9<7!lp&d~0%gKrWP?+?Z<(W! zw8zf!=EaPJ`S|KdG~v*hYoqNQ`8zzS{4KRlg;zxc92~BJ^5&*_q-5;<3VC-dZFYBq zvIZ3m?`a9B%|qgF`W=gLhH_LD5+`w~RK-uK^PWDJSdfnFjjs}tPXJ#146ySnqsw6 z_3tAOS;($(+B+hx*N?YpUaJ=xlz;a*;-O!SJauX@pX@atLsA~qY57GXg0luenWo+G z7+N(|^-#1#UgKs7H~;wFctw;(UNPH*(iw5DjBeq!F=$Ua{kG4!<}fS*}Z@=Y5IlzPwu6NcH+27yOannr?UT zs!DnJl;#slrO^8Hs5sv*XH&}%^Ylca51X5dK%hWF7|9oW|Irr?LIi*93tMrD!@>W+ zUif$H-Hg%w|F!{HoL5VvlwpbvF~pc0zSV7AwV4-i+5k~A7umwSke)rlEO+0Br@ z)D@EVE%b`b82a(W6}ZKa>J!gwsZ(OcIp2zU z&lRyp`z3y4ZsN)qW%ERbWK7l*C4czkvjy+EP&Mys<0|v2@4I}I3SCo&W))F!RF%?) zR?L!PN~zEEP$LQT1W3C4JrJrw*!~ZC+#WW}KWbF%$zhL2>N{yhU$`FI6FgmcYt|(QCbmM5m`I9NHlyBJK{6OFiH{go(BPoDLNb~H8TJ{-aJs8K~s z^~*fxh^l|sXIRc3ZdiUtyG|`UVq4?=94zZ#BRA&YiclzEHO%^S8Rrj-)4JMU_UIhJ+A{ z3u?(4K65YDy#gx@YdOJ+z4B>8_yh{3mkYd9%B=PNGropB&e$R0yYJV^?5{B@V&AdJ zDtA@QaGtxDcydQIGj(&S$ppNS$HVWkIq#r}hwJ^Rb%N+v$6(?-plRIi6&4qpl&N?V zFK6o^oOG|hvFuS1?w*jemMBDipv;7?HJrXO)$g*7U|0_Hednd%&nMV89h$A3TZtQ& za)+W6nTlj3E5$tE<&VdY-tsoRK!tw3zh7?V*%aGX-WK?%*xh8lRMvbOX|e8**#yZq+wb-!E0$UyYR4jmzt-j_b4TE$q$z_NY9e0Y%Yu zaXEP@tt~sMbA!6f|KXA{Fpmrgh|oVmb*T^I;}@MdTYA4T&?>9>9t`lH&KipEl(XM8 zccf%(6V^twaQRP3p}3orG5+4Gv>?j=g;#A4ONHO~*LO$z&6lvGHjk&qyBbq65PgRG zPaoyr&$pu;OsK56DXp>Ja(dFDl*+anD%-kG)KDZtDB!*CBoc<>+`q@<`1>c(dp4*V z&vh5x_kUXzd8{tLwl1=JC%q`>)A^0~2E))L7I)h}j7o~L{MdAKTYtx+rmvMKON`sf z`$k1Msl<3(!)OKrl!bsSBsGtcFr=99mD&|!V=|r`4AH4qCV9Q>t*@dn?Qvy*iq|O#Y6JR_{ik_gT7--dA{^{$%WnV z%GcA4**rY^Z`zxV&yMxmo(Pb+Y}p8Hx}08wsjycUwLH>Z_DcHpn%9lYyu4>&DnuP< zCi>2|sCgn=sO|eZVI$*=UwDd7`m}wX7OJj$?va}{^0d$-^mTYB_*w6R*aqKan~$-E z%S_*RO-^MFxGy(sfcLxHlq+iL6aj^X?p&MP)2hcY((T4(F|ZXuZfDbMJFi^{(!PqqNj?XylUw8T)kOlRgP4qV zCmj5!8^b-H7SYtro}tP#YHlbro0*$y&@fMI%uX?Q{Ps{WLk)QXE#XHuVAVTfl>5@V2ykGeR#NM-Ttn)btzG`lmMro|Os z=j1aay>y>?so!}%7c`T}Q{r9)8+M_oy>NR_NcI9g&*)1Sg~2t{y^cOF<9FA;B;M?k z=d{atfgFBP^%%KF2wi#nbvP1dY@*$tHX)Ju_;DZGae6xLo6%E~+|N(K3KNz? zDESRkdoCm$Ib&7+{8rRSvwd_gL+`i0?R7J&yZVvsyczp4I*4)%sn|G{v>Z5*Gk8(g z13Gn9-Cb(m>$HFn)-t|hv@v#;1HL;P`PauU&VOR|s3^`0?Atdp&2AW;HNC!eQT94S z{foi}=74uTt!e(Naly?tjS&LF*L`JR$*p@QO-HgO`)}QTH!b}>M`kd)P1*AFi~I2V z)d|#Aw-w&hns`gEZrY}8uO^2y3EP_gC1sQTHMaf2HrtU*@;^edR7(_oj}G(-rzo4~ zE%xTQfUMgFy}T^w+q~_1;dKOPQQz>1ik)AOT*wYg-^?ekT!Yw`R-YBlp0K-BzaF7B zM;ip8pnQ|bWSq_`a45-6@{OVr2ZHVuEZ(#fMw> zxvjT-{B)p}pVp-lx08X_=5VC|)Kf9X!u&zB-D{79l!^TX{SS8;=^hL@%lyb?GWg;I zYrDk~x**5RR%(DyT+OO-7QZz5qP0$MkC_*KULmrW**Baj>aL=~fsqhTi$nXkX3_jH zN>Ub_AU3<@RO)Er*PgY3sb2c_4@WGJ3;inCthuw*S4@``!|-=Q>jX8P&MFb|)aTp3 z<%TDZCz?|XGDwXvCv-Ga@~vffy$Uel2u2a1kP=JM=+imqI76{ zyi`CV`iKnYJ*JQ9A6yI3hf}45-th8do=VOXJVl3zTHM!9V|jj2+gDRQQB6^k_E_*s zS;-;hATjw3jxa@y*u$oTAZ6T0Pf|xs2bX5pK4zYl1*U!`#;}$rhkK@OpDZ|Po&*@2 zPvD#VY&~$tqDxV8pj+^ z)xvzSnT{5*ZYM6|4SL*XUnwkKhrymwP#>4HYz{eo)Yc}Z?isg4<@2FpwJfar`dd-F z_YJcS9k3-7-p{FgC|cf8t)1pdk+Z6HxKrXpu2w$I{90Q$T2{QxOBVv9AIz>FV9KZC zzrF|c$;M!}tpP@q$phslCZ(9t_}0CBktb)L^&`cCm4Q;?vE*)i`=vWdodlmOn$wk0 z@|lrOT`M^PGQ8Uqlk`Ua!ZXUZ-mnP%Yx; z>h#ubzq#EhLHXc>=UH<^H~n)V$~AM!wf9wS$LdIH&NGWR8J}J-R;;ZoT{+|y_9k(N z&ZRhJsyy|IL^UvCEj? zN5NR1iWltuZ5)YrqrW2BxfK|XUms)Y)LO4tzOp_%Cq{#aa+&v8mOjY~hNBzqD8|dL zK~L`9|4?T^Vtzw>DsgOJ^ot@Q!f4!V-j`>+YxK_YUc`C$X#6sin7~Fn?Gwm?ZXB5R z%WkGQ-f(E%p!_vd!E3jb7c!{?y5aQ(zOyiC)h6n^&hqO{*j2a+LSB^FYw7kAcX$F;}{uPdThTm(av1#Voe#ETJxQ8xK%>`7_RR%6vY zN#NzmQ)c6)3br9Ifen*!|Fl}x!6mkpJm^M}@CNmQhSqvs|H%ustJdvGC?f5L#+cCJ z2Kbrr;+W99{`LHZ^JhHZI&T`7Vn&==jq)x#Fv1 zYOgjNX*dE@d`p4OxxU%Ab5daO#@){&Ze1HWM?B{rXP9RLc`wxT-@%bho99MZfBF6X zv=B%?%T|W}>sv5($d2#3AkN1ZgY)&-Dvi@@ZFAw22R7@Np70%3(-pGLW;ao?I6T(q;}JcViR<`Q!VY-u7ELCh{LYw5bNY>h- z{V7}gmoxl7RDpjMzP*}d{e$qsq^k7)v+%#Nrk|&=y%==xhiMov{AL>eSV;Ss|Mr@R z>JR)=Nf)}mT;@OW|NTm~pQDw3kkmgjj-(r@U#{#Ine)%<*p5m+S{}mjzvue%tp7rT?Ks8$!>nod{KbD8(rr!fF9zG8 zhWQ5u=HD6o0dB@9T3X3JhVW8JeINWM*`J?rWY8qFUJ?UxBhisByCBE;$nOvc>8L__ RMnj5z2!xsRB9Gr1`# Date: Fri, 25 Nov 2022 15:26:54 +0000 Subject: [PATCH 016/247] Fix mypy errors --- .../od/backend/tests/test_knn_backend.py | 18 +++---- alibi_detect/od/backend/torch/base.py | 51 +++++++++++-------- alibi_detect/od/backend/torch/ensemble.py | 13 +++-- alibi_detect/od/backend/torch/knn.py | 4 +- alibi_detect/od/base.py | 3 +- alibi_detect/od/knn.py | 12 ++--- 6 files changed, 54 insertions(+), 47 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 40cd951da..59e3c5e58 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -20,15 +20,15 @@ def test_knn_torch_backend(): x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) outputs = knn_torch.predict(x) - assert outputs['scores'].shape == (3, ) - assert outputs['preds'] is None - assert outputs['p_vals'] is None + assert outputs.scores.shape == (3, ) + assert outputs.preds is None + assert outputs.p_vals is None scores = knn_torch.score(x) - assert torch.all(scores == outputs['scores']) + assert torch.all(scores == outputs.scores) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs['preds'] == torch.tensor([False, False, True])) + assert torch.all(outputs.preds == torch.tensor([False, False, True])) assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) @@ -38,11 +38,11 @@ def test_knn_torch_backend_ensemble(accumulator): knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) result = knn_torch.predict(x) - assert result['scores'].shape == (3, ) + assert result.scores.shape == (3, ) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs['preds'] == torch.tensor([False, False, True])) + assert torch.all(outputs.preds == torch.tensor([False, False, True])) assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) @@ -86,11 +86,11 @@ def test_knn_kernel(accumulator): knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) result = knn_torch.predict(x) - assert result['scores'].shape == (3,) + assert result.scores.shape == (3,) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs['preds'] == torch.tensor([False, False, True])) + assert torch.all(outputs.preds == torch.tensor([False, False, True])) assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) """Can't convert GaussianRBF to torchscript due to torchscript type diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 7938907a1..e47f8a1a9 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -1,8 +1,9 @@ from __future__ import annotations import numpy as np -from typing import List, Dict, Union +from typing import List, Dict, Union, Optional import torch from alibi_detect.od.backend.torch.ensemble import FitMixinTorch +from dataclasses import dataclass, asdict import logging from abc import ABC, abstractmethod @@ -10,6 +11,22 @@ logger = logging.getLogger(__name__) +@dataclass +class TorchOutlierDetectorOutput: + threshold_inferred: bool + scores: torch.Tensor + threshold: Optional[torch.Tensor] + preds: Optional[torch.Tensor] + p_vals: Optional[torch.Tensor] + + def numpy(self) -> Dict[str, Union[bool, Optional[torch.Tensor]]]: + outputs = asdict(self) + for key, value in outputs.items(): + if isinstance(value, torch.Tensor): + outputs[key] = value.cpu().detach().numpy() + return outputs + + class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): """ Base class for torch backend outlier detection algorithms.""" threshold_inferred = False @@ -35,18 +52,8 @@ def check_threshould_infered(self): def _to_tensor(self, X: Union[List, np.ndarray]): return torch.as_tensor(X, dtype=torch.float32) - def _to_numpy(self, X: Union[Dict[str, torch.tensor], torch.tensor]): - if isinstance(X, dict): - output = X - for k, v in X.items(): - if isinstance(v, torch.Tensor): - output[k] = v.cpu().detach().numpy() - else: - output = X.cpu().detach().numpy() - return output - def _accumulator(self, X: torch.Tensor) -> torch.Tensor: - return self.accumulator(X) if self.accumulator is not None else X + return self.accumulator(X) if self.accumulator is not None else X # type: ignore def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: return scores > self.threshold if self.threshold_inferred else None @@ -61,14 +68,14 @@ def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True - def predict(self, X: torch.Tensor) -> torch.Tensor: - self.check_fitted() - output = { - 'threshold_inferred': self.threshold_inferred, - 'threshold': self.threshold - } + def predict(self, X: torch.Tensor) -> TorchOutlierDetectorOutput: + self.check_fitted() # type: ignore raw_scores = self.score(X) - output['scores'] = self._accumulator(raw_scores) - output['preds'] = self._classify_outlier(output['scores']) - output['p_vals'] = self._p_vals(output['scores']) - return output + scores = self._accumulator(raw_scores) + return TorchOutlierDetectorOutput( + scores=scores, + preds=self._classify_outlier(scores), + p_vals=self._p_vals(scores), + threshold_inferred=self.threshold_inferred, + threshold=self.threshold + ) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index ba5c6b1be..795b3314f 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -50,16 +50,13 @@ class FitMixinTorch: """Fit mixin Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. - - TODO: this should be encorporated into alibi_detect/base.py FitMixinTorch once we can be sure that the - behavour is compatible. """ _fitted = False def __init__(self): super().__init__() - def fit(self, X: torch.Tensor) -> BaseTransformTorch: + def fit(self, X: torch.Tensor) -> FitMixinTorch: self._fitted = True self._fit(X) return self @@ -102,8 +99,9 @@ def __init__(self): super().__init__() self.val_scores = None - def _fit(self, val_scores: torch.Tensor): + def _fit(self, val_scores: torch.Tensor) -> PValNormaliser: self.val_scores = val_scores + return self def _transform(self, scores: torch.Tensor) -> torch.Tensor: p_vals = ( @@ -123,9 +121,10 @@ def __init__(self): self.val_means = None self.val_scales = None - def _fit(self, val_scores: torch.Tensor) -> BaseTransformTorch: + def _fit(self, val_scores: torch.Tensor) -> ShiftAndScaleNormaliser: self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] + return self def _transform(self, scores: torch.Tensor) -> torch.Tensor: return (scores - self.val_means)/self.val_scales @@ -221,4 +220,4 @@ def _transform(self, X: torch.Tensor): def _fit(self, X: torch.Tensor): if self.normaliser is not None: - X = self.normaliser.fit(X) + self.normaliser.fit(X) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 5bc0a5813..63e633e1d 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -12,7 +12,7 @@ def __init__( kernel: Optional[torch.nn.Module] = None, accumulator: Optional[Accumulator] = None ): - super().__init__() + TorchOutlierDetector.__init__(self) self.kernel = kernel self.ensemble = isinstance(k, (np.ndarray, list, tuple)) self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k]) @@ -32,7 +32,7 @@ def score(self, X): all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] - def _fit(self, x_ref: torch.tensor): + def _fit(self, x_ref: torch.Tensor): self.x_ref = x_ref if self.accumulator is not None: scores = self.score(x_ref) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index f5cc38e5f..1c531e67f 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod import numpy as np import logging +from typing import Dict from typing_extensions import Protocol, runtime_checkable from alibi_detect.base import BaseDetector @@ -31,7 +32,7 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: pass @abstractmethod - def predict(self, X: np.ndarray) -> np.ndarray: + def predict(self, X: np.ndarray) -> Dict[str, np.ndarray]: pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 26117ec8e..3c6495887 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -22,36 +22,36 @@ def __init__( ) -> None: super().__init__() - backend = backend.lower() + backend_str: str = backend.lower() BackendValidator( backend_options={'pytorch': ['pytorch']}, construct_name=self.__class__.__name__ - ).verify_backend(backend) + ).verify_backend(backend_str) if isinstance(k, (list, np.ndarray)) and aggregator is None: raise ValueError((f'k={k} is type {type(k)} but aggregator is {aggregator}, you must ' 'specify at least an aggregator if you want to use the knn detector ' 'ensemble like this.')) - backend, accumulator_cls = backends[backend] + backend_cls, accumulator_cls = backends[backend] accumulator = None if normaliser is not None or aggregator is not None: accumulator = accumulator_cls( normaliser=normaliser, aggregator=aggregator ) - self.backend = backend(k, kernel=kernel, accumulator=accumulator) + self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator) def fit(self, x_ref: Union[np.ndarray, List]) -> None: self.backend.fit(self.backend._to_tensor(x_ref)) def score(self, X: Union[np.ndarray, List]) -> np.ndarray: score = self.backend.score(self.backend._to_tensor(X)) - return self.backend._to_numpy(score) + return score.numpy() def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) def predict(self, X: Union[np.ndarray, List]) -> Dict[str, np.ndarray]: outputs = self.backend.predict(self.backend._to_tensor(X)) - return self.backend._to_numpy(outputs) + return outputs.numpy() From 945a15af72c5ada8235bedc10ac28ee1d9093b8b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 25 Nov 2022 15:39:18 +0000 Subject: [PATCH 017/247] Import Literal from typing_extensions for python version compatibility --- alibi_detect/od/knn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 3c6495887..a58fe29bd 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,4 +1,5 @@ -from typing import Callable, Literal, Union, Optional, List, Dict +from typing import Callable, Union, Optional, List, Dict +from typing_extensions import Literal import numpy as np from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol From 7363cb531e73c7ff78c8fb015ba1cdd8694029af Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 14:50:52 +0000 Subject: [PATCH 018/247] Add docstrings for backend ensemble and knn objects --- alibi_detect/od/backend/torch/__init__.py | 0 alibi_detect/od/backend/torch/ensemble.py | 225 +++++++++++++++++----- alibi_detect/od/backend/torch/knn.py | 60 +++++- 3 files changed, 239 insertions(+), 46 deletions(-) create mode 100644 alibi_detect/od/backend/torch/__init__.py diff --git a/alibi_detect/od/backend/torch/__init__.py b/alibi_detect/od/backend/torch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 795b3314f..f327cbeb4 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -14,12 +14,12 @@ class BaseTransformTorch(Module, ABC): - """Base Transform class. - - provides abstract methods for transform objects that map a numpy - array. - """ def __init__(self): + """Base Transform class. + + provides abstract methods for transform objects that map a numpy + array. + """ super().__init__() def transform(self, X: torch.Tensor): @@ -47,13 +47,13 @@ def forward(self, X: torch.Tensor): class FitMixinTorch: - """Fit mixin - - Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. - """ _fitted = False def __init__(self): + """Fit mixin + + Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. + """ super().__init__() def fit(self, X: torch.Tensor) -> FitMixinTorch: @@ -62,48 +62,101 @@ def fit(self, X: torch.Tensor) -> FitMixinTorch: return self def _fit(self, X: torch.Tensor): + """Fit on `X` tensor. + + This method should be overidden on child classes. + + Parameters + ---------- + X + Reference `torch.Tensor` for fitting object. + + Raises + ------ + NotImplementedError + Raised if unimplimented. + """ raise NotImplementedError() @torch.jit.ignore def check_fitted(self): + """Raises error if parent object instance has not been fit. + + Raises + ------ + ValueError + Raised if method called and object has not been fit. + """ if not self._fitted: # TODO: make our own NotFitted Error here! raise ValueError(f'{self.__class__.__name__} has not been fit!') class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): - """Base Fitted Transform class. - - Provides abstract methods for transforms that have an aditional - fit step. - """ - def __init__(self): + """Base Fitted Transform class. + + Extends BaseTransfrom with fit functionality. Ensures that transform has been fit prior to + applying transform. + """ BaseTransformTorch.__init__(self) FitMixinTorch.__init__(self) def transform(self, X: torch.Tensor): + """Checks to make sure transform has been fitted and then applies trasform to input tensor. + + Parameters + ---------- + X + `torch.Tensor` being transformed. + + Returns + ------- + transformed `torch.Tensor`. + """ self.check_fitted() return self._transform(X) class PValNormaliser(BaseFittedTransformTorch): - """Maps scores to there p values. - - Needs to be fit on a reference dataset using fit. Transform counts the number of scores - in the reference dataset that are greter than the score of interest and divides by the - size of the reference dataset. Output is between 1 and 0. Small values are likely to be - outliers. - """ def __init__(self): + """Maps scores to there p values. + + Needs to be fit (see py:obj:alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch). + Transform counts the number of scores in the reference dataset that are greter than the score + of interest and divides by the size of the reference dataset. Output is between 1 and 0. Small + values are likely to be outliers. + """ super().__init__() self.val_scores = None def _fit(self, val_scores: torch.Tensor) -> PValNormaliser: + """Fit transform on scores. + + Parameters + ---------- + val_scores + score outputs of ensemble of detectors applied to reference data. + + Returns + ------- + self + """ self.val_scores = val_scores return self def _transform(self, scores: torch.Tensor) -> torch.Tensor: + """Transform scores to 1 - p values. + + Parameters + ---------- + scores + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + `Torch.Tensor` of 1 - p values. + """ p_vals = ( 1 + (scores[:, None, :] < self.val_scores[None, :, :]).sum(1) )/(len(self.val_scores)+1) @@ -111,22 +164,44 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class ShiftAndScaleNormaliser(BaseFittedTransformTorch): - """Maps scores to their normalised values. - - Needs to be fit on a reference dataset using fit. Subtracts the dataset mean and - scales by the standard deviation. - """ def __init__(self): + """Maps scores to their normalised values. + + Needs to be fit (see py:obj:alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch). + Subtracts the dataset mean and scales by the standard deviation. + """ super().__init__() self.val_means = None self.val_scales = None def _fit(self, val_scores: torch.Tensor) -> ShiftAndScaleNormaliser: + """Computes the mean and standard deviation of the scores and stores them. + + Parameters + ---------- + val_scores + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + self + """ self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] return self def _transform(self, scores: torch.Tensor) -> torch.Tensor: + """Transform scores to normalised values. Subtracts the mean and scales by the standard deviation. + + Parameters + ---------- + scores + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + `Torch.Tensor` of normalised scores. + """ return (scores - self.val_means)/self.val_scales @@ -144,6 +219,17 @@ def __init__(self, k: Optional[int] = None): self.k = k def _transform(self, scores: torch.Tensor) -> torch.Tensor: + """Takes the mean of the top k scores. + + Parameters + ---------- + scores + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + `Torch.Tensor` of mean of top k scores. + """ if self.k is None: self.k = int(np.ceil(scores.shape[1]/2)) sorted_scores, _ = torch.sort(scores, 1) @@ -151,18 +237,31 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class AverageAggregator(BaseTransformTorch): - """Averages the scores of the detectors in an ensemble. - - Parameters - ---------- - weights - Optional parameter to weight the scores. - """ def __init__(self, weights: Optional[torch.Tensor] = None): + """Averages the scores of the detectors in an ensemble. + + Parameters + ---------- + weights + Optional parameter to weight the scores. If `weights` is left `None` then will be set to + a vector of ones. + """ super().__init__() self.weights = weights def _transform(self, scores: torch.Tensor) -> torch.Tensor: + """Averages the scores of the detectors in an ensemble. If weights where passed in the init + then these are used to weight the scores. + + Parameters + ---------- + scores + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + `Torch.Tensor` of mean of scores. + """ if self.weights is None: m = scores.shape[-1] self.weights = torch.ones(m)/m @@ -170,41 +269,61 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class MaxAggregator(BaseTransformTorch): - """Takes the max score of a set of detectors in an ensemble.""" def __init__(self): + """Takes the maximum of the scores of the detectors in an ensemble.""" super().__init__() def _transform(self, scores: torch.Tensor) -> torch.Tensor: + """Takes the max score of a set of detectors in an ensemble. + + Parameters + ---------- + scores + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + `Torch.Tensor` of max of scores. + """ vals, _ = torch.max(scores, dim=-1) return vals class MinAggregator(BaseTransformTorch): - """Takes the min score of a set of detectors in an ensemble.""" def __init__(self): + """Takes the min score of a set of detectors in an ensemble.""" super().__init__() def _transform(self, scores: torch.Tensor) -> torch.Tensor: + """Takes the min score of a set of detectors in an ensemble. + + Parameters + ---------- + scores + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + `Torch.Tensor` of min of scores. + """ vals, _ = torch.min(scores, dim=-1) return vals class Accumulator(BaseFittedTransformTorch): def __init__(self, - normaliser: BaseFittedTransformTorch = None, + normaliser: Optional[BaseFittedTransformTorch] = None, aggregator: BaseTransformTorch = AverageAggregator()): - """Wraps a normaliser and aggregator into a single object. - - The accumulator wraps normalisers and aggregators into a single object. + """Accumulates the scores of the detectors in an ensemble. Can be used to normalise and aggregate + the scores from an ensemble of detectors. Parameters ---------- normaliser - normaliser that's an instance of BaseFittedTransformTorch. Maps the outputs of - a set of detectors to a common range. + `BaseFittedTransformTorch` object to normalise the scores. If `None` then no normalisation + is applied. aggregator - aggregator extendng BaseTransformTorch. Maps outputs of the normaliser to - single score. + `BaseTransformTorch` object to aggregate the scores. """ super().__init__() self.normaliser = normaliser @@ -213,11 +332,29 @@ def __init__(self, self.aggregator = aggregator def _transform(self, X: torch.Tensor): + """Apply the normaliser and aggregator to the scores. + + Parameters + ---------- + X + `Torch.Tensor` of scores from ensemble of detectors. + + Returns + ------- + `Torch.Tensor` of aggregated and normalised scores. + """ if self.normaliser is not None: X = self.normaliser(X) X = self.aggregator(X) return X def _fit(self, X: torch.Tensor): + """Fit the normaliser to the scores. + + Parameters + ---------- + X + `Torch.Tensor` of scores from ensemble of detectors. + """ if self.normaliser is not None: self.normaliser.fit(X) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 63e633e1d..7ab155246 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -12,20 +12,69 @@ def __init__( kernel: Optional[torch.nn.Module] = None, accumulator: Optional[Accumulator] = None ): + """PyTorch backend for KNN detector. + + Parameters + ---------- + k + Number of neirest neighbors to compute distance to. `k` can be a single value or + an array of integers. If `k` is a single value the outlier score is the distance/kernel + similarity to the `k`-th nearest neighbor. If `k` is a list then it returns the distance/kernel + similarity to each of the specified `k` neighbors. + kernel + If a kernel is specified then instead of using torch.cdist we compute the kernel similarity + while computing the k nearest neighbor distance. + accumulator + If `k` is an array of integers then the accumulator must not be None. Should be an instance + of :py:obj:`alibi_detect.od.backend.torch.ensemble.Accumulator`. Responsible for combining + multiple scores into a single score. + """ TorchOutlierDetector.__init__(self) self.kernel = kernel self.ensemble = isinstance(k, (np.ndarray, list, tuple)) self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k]) self.accumulator = accumulator - def forward(self, X): + def forward(self, X: torch.Tensor) -> torch.Tensor: + """Detect if X is an outlier. + + Parameters + ---------- + X + `torch.Tensor` with leading batch dimension. + + Returns + ------- + `torch.Tensor` of `bool` values with leading batch dimension. + + Raises + ------ + ValueError + If called before detector has had threshould_infered method called. + """ raw_scores = self.score(X) scores = self._accumulator(raw_scores) self.check_threshould_infered() preds = scores > self.threshold return preds.cpu() - def score(self, X): + def score(self, X: torch.Tensor) -> torch.Tensor: + """Computes the score of `X` + + Parameters + ---------- + X + Score a tensor of instances. First dimesnion corresponds to batch. + + Returns + ------- + Tensor of scores for each element in `X`. + + Raises + ------ + ValueError + If called before detector has been fit. + """ self.check_fitted() K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) bot_k_dists = torch.topk(K, torch.max(self.ks), dim=1, largest=False) @@ -33,6 +82,13 @@ def score(self, X): return all_knn_dists if self.ensemble else all_knn_dists[:, 0] def _fit(self, x_ref: torch.Tensor): + """Fits the detector + + Parameters + ---------- + x_ref + The Dataset tensor. + """ self.x_ref = x_ref if self.accumulator is not None: scores = self.score(x_ref) From 6ca806248e46a7b98666c63c897520a40459480b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 15:27:51 +0000 Subject: [PATCH 019/247] Add docstrings for base torch outlier detector class --- alibi_detect/od/backend/torch/base.py | 119 +++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index e47f8a1a9..e9dfb7d04 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -13,6 +13,7 @@ @dataclass class TorchOutlierDetectorOutput: + """Output of the outlier detector.""" threshold_inferred: bool scores: torch.Tensor threshold: Optional[torch.Tensor] @@ -20,6 +21,7 @@ class TorchOutlierDetectorOutput: p_vals: Optional[torch.Tensor] def numpy(self) -> Dict[str, Union[bool, Optional[torch.Tensor]]]: + """Converts the output to numpy.""" outputs = asdict(self) for key, value in outputs.items(): if isinstance(value, torch.Tensor): @@ -28,7 +30,7 @@ def numpy(self) -> Dict[str, Union[bool, Optional[torch.Tensor]]]: class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): - """ Base class for torch backend outlier detection algorithms.""" + """Base class for torch backend outlier detection algorithms.""" threshold_inferred = False threshold = None @@ -37,38 +39,153 @@ def __init__(self): @abstractmethod def _fit(self, x_ref: torch.Tensor) -> None: + """Fit the outlier detector to the reference data. + + Parameters + ---------- + x_ref + Reference data. + + Raises + ------ + `NotImplementedError` + Raised if not implemented. + """ raise NotImplementedError() @abstractmethod def score(self, X: torch.Tensor) -> torch.Tensor: + """Score the data. + + Parameters + ---------- + X + Data to score. + + Raises + ------ + `NotImplementedError` + Raised if not implemented. + """ raise NotImplementedError() @torch.jit.ignore def check_threshould_infered(self): + """Check if threshold is inferred. + + Raises + ------ + ValueError + Raised if threshold is not inferred. + """ if not self.threshold_inferred: raise ValueError((f'{self.__class__.__name__} has no threshold set, ' 'call `infer_threshold` before predicting.')) def _to_tensor(self, X: Union[List, np.ndarray]): + """Converts the data to a tensor. + + Parameters + ---------- + X + Data to convert. + + Returns + ------- + `torch.Tensor` + """ return torch.as_tensor(X, dtype=torch.float32) def _accumulator(self, X: torch.Tensor) -> torch.Tensor: + """Accumulates the data. + + Parameters + ---------- + X + Data to accumulate. + + Returns + ------- + `torch.Tensor` or just returns original data + """ + # `type: ignore` here becuase self.accumulator here causes an error with mypy when using torch.jit.script. + # For some reason it thinks self.accumulator is a torch.Tensor and therefore is not callable. return self.accumulator(X) if self.accumulator is not None else X # type: ignore def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: + """Classify the data as outlier or not. + + Parameters + ---------- + scores + Scores to classify. Larger scores indicate more likely outliers. + + Returns + ------- + `torch.Tensor` or `None` + """ return scores > self.threshold if self.threshold_inferred else None def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: + """Compute p-values for the scores. + + Parameters + ---------- + scores + Scores to compute p-values for. + + Returns + ------- + `torch.Tensor` or `None` + """ return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ if self.threshold_inferred else None def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: + """Infer the threshold for the data. Prerequisite for outlier predictions. + + Parameters + ---------- + X + Data to infer the threshold for. + fpr + False positive rate to use for threshold inference. + + Raises + ------ + ValueError + Raised if fpr is not in (0, 1). + """ + if not 0 < fpr < 1: + ValueError('fpr must be in (0, 1).') self.val_scores = self.score(X) self.val_scores = self._accumulator(self.val_scores) self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True def predict(self, X: torch.Tensor) -> TorchOutlierDetectorOutput: + """Predict outlier labels for the data. + + Computes the outlier scores. If the detector is not fit on reference data we raise an error. + If the threshold is inferred, the outlier labels and p-values are also computed and returned. + Otherwise, the outlier labels and p-values are set to `None`. + + Parameters + ---------- + X + Data to predict. + + Raises + ------ + ValueError + Raised if the detector is not fit on reference data. + + Returns + ------- + `TorchOutlierDetectorOutput` + Output of the outlier detector. + + """ self.check_fitted() # type: ignore raw_scores = self.score(X) scores = self._accumulator(raw_scores) From 4674f8ead7946576bf9bb69dc786d2d657d618b3 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 15:53:37 +0000 Subject: [PATCH 020/247] Add docstrings for kNN detector --- alibi_detect/od/base.py | 9 +++-- alibi_detect/od/knn.py | 78 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 1c531e67f..a48c259dc 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -2,13 +2,18 @@ from abc import ABC, abstractmethod import numpy as np import logging -from typing import Dict +from typing import Dict, TypedDict, Union from typing_extensions import Protocol, runtime_checkable from alibi_detect.base import BaseDetector logger = logging.getLogger(__name__) +class OutlierDetectorOutput(TypedDict): + meta: Dict[str, str] + data: Dict[str, Union[bool, np.ndarray]] + + class OutlierDetector(BaseDetector, ABC): """ Base class for outlier detection algorithms. """ threshold_inferred = False @@ -32,7 +37,7 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: pass @abstractmethod - def predict(self, X: np.ndarray) -> Dict[str, np.ndarray]: + def predict(self, X: np.ndarray) -> OutlierDetectorOutput: pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index a58fe29bd..2fc3f23c5 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,8 +1,8 @@ -from typing import Callable, Union, Optional, List, Dict +from typing import Callable, Union, Optional, List from typing_extensions import Literal import numpy as np -from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol +from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol, OutlierDetectorOutput from alibi_detect.od.backend import KNNTorch, AccumulatorTorch from alibi_detect.utils.frameworks import BackendValidator @@ -21,6 +21,30 @@ def __init__( aggregator: Optional[TransformProtocol] = None, backend: Literal['pytorch'] = 'pytorch' ) -> None: + """ + k-Nearest Neighbours (kNN) outlier detector. + + Parameters + ---------- + k + Number of nearest neighbours to use for outlier detection. If an array is passed, a aggregator is required to + aggregate the scores. + kernel + Kernel function to use for outlier detection. If None, `torch.cdist` is used. + normaliser + Normaliser to use for outlier detection. If None, no normalisation is applied. + aggregator + Aggregator to use for outlier detection. If None, no aggregation is applied. + backend + Backend used for outlier detection. Defaults to `'pytorch'`. Options are `'pytorch'`. + + Raises + ------ + ValueError + If `k` is an array and `aggregator` is None. + NotImplementedError + If choice of `backend` is not implemented. + """ super().__init__() backend_str: str = backend.lower() @@ -44,15 +68,61 @@ def __init__( self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator) def fit(self, x_ref: Union[np.ndarray, List]) -> None: + """Fit the kNN detector on reference data. + + Parameters + ---------- + x_ref + Reference data used to fit the kNN detector. + """ self.backend.fit(self.backend._to_tensor(x_ref)) def score(self, X: Union[np.ndarray, List]) -> np.ndarray: + """Score X instances using the kNN. + + Parameters + ---------- + X + Data to score. The shape of `X` should be `(n_instances, n_features)`. + + Returns + ------- + Anomaly scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the + instance. + """ score = self.backend.score(self.backend._to_tensor(X)) return score.numpy() def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: + """Infer the threshold for the kNN detector. + + Parameters + ---------- + x_ref + Reference data used to infer the threshold. + fpr + False positive rate used to infer the threshold. + """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) - def predict(self, X: Union[np.ndarray, List]) -> Dict[str, np.ndarray]: + def predict(self, X: Union[np.ndarray, List]) -> OutlierDetectorOutput: + """Predict whether the instances in X are outliers or not. + + Parameters + ---------- + X + Data to predict. The shape of `X` should be `(n_instances, n_features)`. + + Returns + ------- + Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, + 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is + `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about + the detector. + """ outputs = self.backend.predict(self.backend._to_tensor(X)) - return outputs.numpy() + output: OutlierDetectorOutput = { + 'data': outputs.numpy(), + 'meta': self.meta + } + return output From 07f0ffc7c870b9a53cb21db37a2daf23aed5cd90 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 15:55:15 +0000 Subject: [PATCH 021/247] Minor fixes --- alibi_detect/od/knn.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 2fc3f23c5..d9be2ff84 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -27,14 +27,15 @@ def __init__( Parameters ---------- k - Number of nearest neighbours to use for outlier detection. If an array is passed, a aggregator is required to - aggregate the scores. + Number of nearest neighbours to use for outlier detection. If an array is passed, an aggregator is required + to aggregate the scores. kernel Kernel function to use for outlier detection. If None, `torch.cdist` is used. normaliser Normaliser to use for outlier detection. If None, no normalisation is applied. aggregator - Aggregator to use for outlier detection. If None, no aggregation is applied. + Aggregator to use for outlier detection. If None, no aggregation is applied. If an array is passed for `k`, + then an aggregator is required. backend Backend used for outlier detection. Defaults to `'pytorch'`. Options are `'pytorch'`. @@ -87,7 +88,7 @@ def score(self, X: Union[np.ndarray, List]) -> np.ndarray: Returns ------- - Anomaly scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the + Anomaly scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the instance. """ score = self.backend.score(self.backend._to_tensor(X)) From 5e0c3f75840bed583af6c68476eaf7361aecc46a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 15:56:09 +0000 Subject: [PATCH 022/247] Minor fixes --- alibi_detect/od/backend/torch/knn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 7ab155246..80179d637 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -26,7 +26,7 @@ def __init__( while computing the k nearest neighbor distance. accumulator If `k` is an array of integers then the accumulator must not be None. Should be an instance - of :py:obj:`alibi_detect.od.backend.torch.ensemble.Accumulator`. Responsible for combining + of :py:obj:`alibi_detect.od.backend.torch.ensemble.Accumulator`. Responsible for combining multiple scores into a single score. """ TorchOutlierDetector.__init__(self) @@ -64,7 +64,7 @@ def score(self, X: torch.Tensor) -> torch.Tensor: Parameters ---------- X - Score a tensor of instances. First dimesnion corresponds to batch. + Score a tensor of instances. First dimesnion corresponds to batch. Returns ------- From e884e9447dbf164cf1446c5cd12dd551f45cd995 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 16:26:42 +0000 Subject: [PATCH 023/247] Add docstrings for outlier detector base class --- alibi_detect/od/base.py | 50 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index a48c259dc..51e16eb01 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -10,39 +10,86 @@ class OutlierDetectorOutput(TypedDict): + """Outlier detector output type definition.""" meta: Dict[str, str] data: Dict[str, Union[bool, np.ndarray]] class OutlierDetector(BaseDetector, ABC): - """ Base class for outlier detection algorithms. """ threshold_inferred = False ensemble = False def __init__(self): + """ Base class for outlier detection algorithms.""" super().__init__() self.meta['online'] = False self.meta['detector_type'] = 'outlier' @abstractmethod def fit(self, X: np.ndarray) -> None: + """ + Fit outlier detector to data. + + Parameters + ---------- + X + Reference data. + """ pass @abstractmethod def score(self, X: np.ndarray) -> np.ndarray: + """ + Compute anomaly scores of the instances in X. + + Parameters + ---------- + X + Data to score. + + Returns + ------- + Anomaly scores. The higher the score, the more anomalous the instance. + """ pass @abstractmethod def infer_threshold(self, X: np.ndarray, fpr: float) -> None: + """ + Infer the threshold for the outlier detector. + + Parameters + ---------- + X + Reference data. + fpr + False positive rate used to infer the threshold. + """ pass @abstractmethod def predict(self, X: np.ndarray) -> OutlierDetectorOutput: + """ + Predict whether the instances in X are outliers or not. + + Parameters + ---------- + X + Data to predict. + + Returns + ------- + Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, + 'data' also contains the outlier labels. + """ pass +# Using Protocols instead base classes for the backend classes. This is a bit more flexible and allows us to +# avoid the torch/tensorflow imports in the base class. @runtime_checkable class TransformProtocol(Protocol): + """Protocol for transformer objects.""" def transform(self, X): pass @@ -52,6 +99,7 @@ def _transform(self, X): @runtime_checkable class FittedTransformProtocol(TransformProtocol, Protocol): + """Protocol for fitted transformer objects.""" def fit(self, x_ref): pass From 756def5f93e3abf4720df39a7feb40a891e83f71 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 16:58:44 +0000 Subject: [PATCH 024/247] Fix mypy issue and test --- alibi_detect/od/backend/torch/knn.py | 2 +- alibi_detect/od/tests/test_knn.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 80179d637..55006d98b 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -77,7 +77,7 @@ def score(self, X: torch.Tensor) -> torch.Tensor: """ self.check_fitted() K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) - bot_k_dists = torch.topk(K, torch.max(self.ks), dim=1, largest=False) + bot_k_dists = torch.topk(K, int(torch.max(self.ks)), dim=1, largest=False) all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index d4dd6c5fa..4f00c928c 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -29,6 +29,7 @@ def test_fitted_knn_single_score(): knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) y = knn_detector.predict(x) + y = y['data'] assert y['scores'][0] > 5 assert y['scores'][1] < 1 @@ -44,6 +45,7 @@ def test_fitted_knn_predict(): knn_detector.infer_threshold(x_ref, 0.1) x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) + y = y['data'] assert y['scores'][0] > 5 assert y['scores'][1] < 1 assert y['threshold_inferred'] @@ -80,6 +82,7 @@ def test_fitted_knn_ensemble(aggregator, normaliser): knn_detector.fit(x_ref) x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) + y = y['data'] assert y['scores'].all() assert not y['threshold_inferred'] assert y['threshold'] is None @@ -98,6 +101,7 @@ def test_fitted_knn_ensemble_predict(aggregator, normaliser): ) x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) + y = y['data'] assert y['threshold_inferred'] assert y['threshold'] is not None assert y['p_vals'].all() From 96009e7a0d0389d1c1508c1a9b0651c107273dc9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 17:07:35 +0000 Subject: [PATCH 025/247] Reorder imports --- alibi_detect/od/backend/tests/test_ensemble.py | 1 + alibi_detect/od/backend/torch/base.py | 10 ++++------ alibi_detect/od/backend/torch/ensemble.py | 10 ++-------- alibi_detect/od/backend/torch/knn.py | 2 ++ alibi_detect/od/base.py | 7 +++---- alibi_detect/od/knn.py | 1 + 6 files changed, 13 insertions(+), 18 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/backend/tests/test_ensemble.py index a0b0756f1..64c01ede1 100644 --- a/alibi_detect/od/backend/tests/test_ensemble.py +++ b/alibi_detect/od/backend/tests/test_ensemble.py @@ -1,5 +1,6 @@ import pytest import torch + from alibi_detect.od.backend.torch import ensemble diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index e9dfb7d04..974b8a5c5 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -1,14 +1,12 @@ from __future__ import annotations -import numpy as np from typing import List, Dict, Union, Optional -import torch -from alibi_detect.od.backend.torch.ensemble import FitMixinTorch from dataclasses import dataclass, asdict - -import logging from abc import ABC, abstractmethod -logger = logging.getLogger(__name__) +import numpy as np +import torch + +from alibi_detect.od.backend.torch.ensemble import FitMixinTorch @dataclass diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index f327cbeb4..4b147c0c8 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -1,16 +1,10 @@ from __future__ import annotations - -import logging - -from typing import Optional from abc import ABC, abstractmethod +from typing import Optional import torch -from torch.nn import Module import numpy as np - - -logger = logging.getLogger(__name__) +from torch.nn import Module class BaseTransformTorch(Module, ABC): diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 55006d98b..5e9f1510f 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -1,6 +1,8 @@ from typing import Optional, Union, List + import numpy as np import torch + from alibi_detect.od.backend.torch.ensemble import Accumulator from alibi_detect.od.backend.torch.base import TorchOutlierDetector diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 51e16eb01..6fb1e5ac4 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -1,12 +1,11 @@ from __future__ import annotations from abc import ABC, abstractmethod -import numpy as np -import logging from typing import Dict, TypedDict, Union + from typing_extensions import Protocol, runtime_checkable -from alibi_detect.base import BaseDetector +import numpy as np -logger = logging.getLogger(__name__) +from alibi_detect.base import BaseDetector class OutlierDetectorOutput(TypedDict): diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index d9be2ff84..b4163d52a 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,4 +1,5 @@ from typing import Callable, Union, Optional, List + from typing_extensions import Literal import numpy as np From b6ac822d40773fd1abb30c28b346abe9b99e1fb8 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 18:05:29 +0000 Subject: [PATCH 026/247] Replace normaliser with normalizer --- alibi_detect/od/backend/__init__.py | 4 +-- .../od/backend/tests/test_ensemble.py | 36 +++++++++---------- .../od/backend/tests/test_knn_backend.py | 4 +-- alibi_detect/od/backend/torch/ensemble.py | 28 +++++++-------- alibi_detect/od/knn.py | 10 +++--- alibi_detect/od/tests/test_knn.py | 30 ++++++++-------- 6 files changed, 56 insertions(+), 56 deletions(-) diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index 91a969cd9..ab431f67b 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -1,9 +1,9 @@ from alibi_detect.utils.missing_optional_dependency import import_optional KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) -PValNormaliserTorch, ShiftAndScaleNormaliserTorch, TopKAggregatorTorch, AverageAggregatorTorch, \ +PValNormalizerTorch, ShiftAndScaleNormalizerTorch, TopKAggregatorTorch, AverageAggregatorTorch, \ MaxAggregatorTorch, MinAggregatorTorch, AccumulatorTorch = import_optional( 'alibi_detect.od.backend.torch.ensemble', - ['PValNormaliser', 'ShiftAndScaleNormaliser', 'TopKAggregator', + ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', 'AverageAggregator', 'MaxAggregator', 'MinAggregator', 'Accumulator'] ) diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/backend/tests/test_ensemble.py index 64c01ede1..7f4767428 100644 --- a/alibi_detect/od/backend/tests/test_ensemble.py +++ b/alibi_detect/od/backend/tests/test_ensemble.py @@ -4,31 +4,31 @@ from alibi_detect.od.backend.torch import ensemble -def test_pval_normaliser(): - normaliser = ensemble.PValNormaliser() +def test_pval_normalizer(): + normalizer = ensemble.PValNormalizer() x = torch.randn(3, 10) x_ref = torch.randn(64, 10) with pytest.raises(ValueError): - normaliser(x) + normalizer(x) - normaliser.fit(x_ref) - x_norm = normaliser(x) - normaliser = torch.jit.script(normaliser) - x_norm_2 = normaliser(x) + normalizer.fit(x_ref) + x_norm = normalizer(x) + normalizer = torch.jit.script(normalizer) + x_norm_2 = normalizer(x) assert torch.all(x_norm_2 == x_norm) -def test_shift_and_scale_normaliser(): - normaliser = ensemble.ShiftAndScaleNormaliser() +def test_shift_and_scale_normalizer(): + normalizer = ensemble.ShiftAndScaleNormalizer() x = torch.randn(3, 10) x_ref = torch.randn(64, 10) with pytest.raises(ValueError): - normaliser(x) + normalizer(x) - normaliser.fit(x_ref) - x_norm = normaliser(x) - normaliser = torch.jit.script(normaliser) - x_norm_2 = normaliser(x) + normalizer.fit(x_ref) + x_norm = normalizer(x) + normalizer = torch.jit.script(normalizer) + x_norm_2 = normalizer(x) assert torch.all(x_norm_2 == x_norm) @@ -83,11 +83,11 @@ def test_min_aggregator(): @pytest.mark.parametrize('aggregator', ['AverageAggregator', 'MaxAggregator', 'MinAggregator', 'TopKAggregator']) -@pytest.mark.parametrize('normaliser', ['PValNormaliser', 'ShiftAndScaleNormaliser']) -def test_accumulator(aggregator, normaliser): +@pytest.mark.parametrize('normalizer', ['PValNormalizer', 'ShiftAndScaleNormalizer']) +def test_accumulator(aggregator, normalizer): aggregator = getattr(ensemble, aggregator)() - normaliser = getattr(ensemble, normaliser)() - accumulator = ensemble.Accumulator(aggregator=aggregator, normaliser=normaliser) + normalizer = getattr(ensemble, normalizer)() + accumulator = ensemble.Accumulator(aggregator=aggregator, normalizer=normalizer) x = torch.randn(3, 10) x_ref = torch.randn(64, 10) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 59e3c5e58..f77b7fd44 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -3,13 +3,13 @@ from alibi_detect.od.backend.torch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF -from alibi_detect.od.backend.torch.ensemble import Accumulator, PValNormaliser, AverageAggregator +from alibi_detect.od.backend.torch.ensemble import Accumulator, PValNormalizer, AverageAggregator @pytest.fixture(scope='session') def accumulator(request): return Accumulator( - normaliser=PValNormaliser(), + normalizer=PValNormalizer(), aggregator=AverageAggregator() ) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 4b147c0c8..6fd0d3db4 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -112,7 +112,7 @@ def transform(self, X: torch.Tensor): return self._transform(X) -class PValNormaliser(BaseFittedTransformTorch): +class PValNormalizer(BaseFittedTransformTorch): def __init__(self): """Maps scores to there p values. @@ -124,7 +124,7 @@ def __init__(self): super().__init__() self.val_scores = None - def _fit(self, val_scores: torch.Tensor) -> PValNormaliser: + def _fit(self, val_scores: torch.Tensor) -> PValNormalizer: """Fit transform on scores. Parameters @@ -157,7 +157,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return 1 - p_vals -class ShiftAndScaleNormaliser(BaseFittedTransformTorch): +class ShiftAndScaleNormalizer(BaseFittedTransformTorch): def __init__(self): """Maps scores to their normalised values. @@ -168,7 +168,7 @@ def __init__(self): self.val_means = None self.val_scales = None - def _fit(self, val_scores: torch.Tensor) -> ShiftAndScaleNormaliser: + def _fit(self, val_scores: torch.Tensor) -> ShiftAndScaleNormalizer: """Computes the mean and standard deviation of the scores and stores them. Parameters @@ -306,27 +306,27 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class Accumulator(BaseFittedTransformTorch): def __init__(self, - normaliser: Optional[BaseFittedTransformTorch] = None, + normalizer: Optional[BaseFittedTransformTorch] = None, aggregator: BaseTransformTorch = AverageAggregator()): """Accumulates the scores of the detectors in an ensemble. Can be used to normalise and aggregate the scores from an ensemble of detectors. Parameters ---------- - normaliser + normalizer `BaseFittedTransformTorch` object to normalise the scores. If `None` then no normalisation is applied. aggregator `BaseTransformTorch` object to aggregate the scores. """ super().__init__() - self.normaliser = normaliser - if self.normaliser is None: + self.normalizer = normalizer + if self.normalizer is None: self.fitted = True self.aggregator = aggregator def _transform(self, X: torch.Tensor): - """Apply the normaliser and aggregator to the scores. + """Apply the normalizer and aggregator to the scores. Parameters ---------- @@ -337,18 +337,18 @@ def _transform(self, X: torch.Tensor): ------- `Torch.Tensor` of aggregated and normalised scores. """ - if self.normaliser is not None: - X = self.normaliser(X) + if self.normalizer is not None: + X = self.normalizer(X) X = self.aggregator(X) return X def _fit(self, X: torch.Tensor): - """Fit the normaliser to the scores. + """Fit the normalizer to the scores. Parameters ---------- X `Torch.Tensor` of scores from ensemble of detectors. """ - if self.normaliser is not None: - self.normaliser.fit(X) + if self.normalizer is not None: + self.normalizer.fit(X) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index b4163d52a..f2b3df6ca 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -18,7 +18,7 @@ def __init__( self, k: Union[int, np.ndarray], kernel: Optional[Callable] = None, - normaliser: Optional[Union[TransformProtocol, FittedTransformProtocol]] = None, + normalizer: Optional[Union[TransformProtocol, FittedTransformProtocol]] = None, aggregator: Optional[TransformProtocol] = None, backend: Literal['pytorch'] = 'pytorch' ) -> None: @@ -32,8 +32,8 @@ def __init__( to aggregate the scores. kernel Kernel function to use for outlier detection. If None, `torch.cdist` is used. - normaliser - Normaliser to use for outlier detection. If None, no normalisation is applied. + normalizer + Normalizer to use for outlier detection. If None, no normalisation is applied. aggregator Aggregator to use for outlier detection. If None, no aggregation is applied. If an array is passed for `k`, then an aggregator is required. @@ -62,9 +62,9 @@ def __init__( backend_cls, accumulator_cls = backends[backend] accumulator = None - if normaliser is not None or aggregator is not None: + if normalizer is not None or aggregator is not None: accumulator = accumulator_cls( - normaliser=normaliser, + normalizer=normalizer, aggregator=aggregator ) self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator) diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index 4f00c928c..e9c5acb0d 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -4,11 +4,11 @@ from alibi_detect.od.knn import KNN from alibi_detect.od.backend import AverageAggregatorTorch, TopKAggregatorTorch, MaxAggregatorTorch, \ - MinAggregatorTorch, ShiftAndScaleNormaliserTorch, PValNormaliserTorch + MinAggregatorTorch, ShiftAndScaleNormalizerTorch, PValNormalizerTorch -def make_knn_detector(k=5, aggregator=None, normaliser=None): - knn_detector = KNN(k=k, aggregator=aggregator, normaliser=normaliser) +def make_knn_detector(k=5, aggregator=None, normalizer=None): + knn_detector = KNN(k=k, aggregator=aggregator, normalizer=normalizer) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) knn_detector.infer_threshold(x_ref, 0.1) @@ -56,12 +56,12 @@ def test_fitted_knn_predict(): @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) -def test_unfitted_knn_ensemble(aggregator, normaliser): +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +def test_unfitted_knn_ensemble(aggregator, normalizer): knn_detector = KNN( k=[8, 9, 10], aggregator=aggregator(), - normaliser=normaliser() + normalizer=normalizer() ) x = np.array([[0, 10], [0.1, 0]]) with pytest.raises(ValueError) as err: @@ -71,12 +71,12 @@ def test_unfitted_knn_ensemble(aggregator, normaliser): @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) -def test_fitted_knn_ensemble(aggregator, normaliser): +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +def test_fitted_knn_ensemble(aggregator, normalizer): knn_detector = KNN( k=[8, 9, 10], aggregator=aggregator(), - normaliser=normaliser() + normalizer=normalizer() ) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) @@ -92,12 +92,12 @@ def test_fitted_knn_ensemble(aggregator, normaliser): @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) -def test_fitted_knn_ensemble_predict(aggregator, normaliser): +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +def test_fitted_knn_ensemble_predict(aggregator, normalizer): knn_detector = make_knn_detector( k=[8, 9, 10], aggregator=aggregator(), - normaliser=normaliser() + normalizer=normalizer() ) x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) @@ -117,9 +117,9 @@ def test_incorrect_knn_ensemble_init(): @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normaliser", [ShiftAndScaleNormaliserTorch, PValNormaliserTorch, lambda: None]) -def test_knn_ensemble_torch_script(aggregator, normaliser): - knn_detector = make_knn_detector(k=[5, 6, 7], aggregator=aggregator(), normaliser=normaliser()) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +def test_knn_ensemble_torch_script(aggregator, normalizer): + knn_detector = make_knn_detector(k=[5, 6, 7], aggregator=aggregator(), normalizer=normalizer()) tsknn = torch.jit.script(knn_detector.backend) x = torch.tensor([[0, 10], [0, 0.1]]) y = tsknn(x) From 3dc8408e53a970a5b5e2397f2fa7ab2d7c0cf2f5 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 28 Nov 2022 18:17:51 +0000 Subject: [PATCH 027/247] Add optional dependency tests --- alibi_detect/od/__init__.py | 2 ++ alibi_detect/od/base.py | 2 +- alibi_detect/tests/test_dep_management.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index f0f0b9350..a0fb048e1 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -11,6 +11,7 @@ OutlierSeq2Seq = import_optional('alibi_detect.od.seq2seq', names=['OutlierSeq2Seq']) LLR = import_optional('alibi_detect.od.llr', names=['LLR']) OutlierProphet = import_optional('alibi_detect.od.prophet', names=['OutlierProphet']) +KNN = import_optional('alibi_detect.od.knn', names=['KNN']) __all__ = [ "OutlierAEGMM", @@ -23,4 +24,5 @@ "SpectralResidual", "LLR", "OutlierProphet" + "KNN" ] diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 6fb1e5ac4..77377c5b4 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -75,7 +75,7 @@ def predict(self, X: np.ndarray) -> OutlierDetectorOutput: ---------- X Data to predict. - + Returns ------- Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index a2fc38da3..a13669a00 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -124,6 +124,25 @@ def test_od_dependencies(opt_dep): check_correct_dependencies(od, dependency_map, opt_dep) +def test_od_backend_dependencies(opt_dep): + """Tests that the od module correctly protects against uninstalled optional dependencies. + """ + dependency_map = defaultdict(lambda: ['default']) + for dependency, relations in [ + ('PValNormalizerTorch', ['torch', 'keops']), + ('ShiftAndScaleNormalizerTorch', ['torch', 'keops']), + ('TopKAggregatorTorch', ['torch', 'keops']), + ('AverageAggregatorTorch', ['torch', 'keops']), + ('MaxAggregatorTorch', ['torch', 'keops']), + ('MinAggregatorTorch', ['torch', 'keops']), + ('AccumulatorTorch', ['torch', 'keops']), + ('KNNTorch', ['torch', 'keops']), + ]: + dependency_map[dependency] = relations + from alibi_detect.od import backend as od_backend + check_correct_dependencies(od_backend, dependency_map, opt_dep) + + def test_tensorflow_model_dependencies(opt_dep): """Tests that the tensorflow models module correctly protects against uninstalled optional dependencies. """ From 322989ea04f98067aefb52d35a30a2b8e616ba9f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 29 Nov 2022 13:56:38 +0000 Subject: [PATCH 028/247] Add make_moons dataset tests for ensemble and single kNN detectors --- alibi_detect/od/tests/test_knn.py | 50 +++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index e9c5acb0d..debc2c769 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -5,6 +5,7 @@ from alibi_detect.od.knn import KNN from alibi_detect.od.backend import AverageAggregatorTorch, TopKAggregatorTorch, MaxAggregatorTorch, \ MinAggregatorTorch, ShiftAndScaleNormalizerTorch, PValNormalizerTorch +from sklearn.datasets import make_moons def make_knn_detector(k=5, aggregator=None, normalizer=None): @@ -132,3 +133,52 @@ def test_knn_single_torchscript(): x = torch.tensor([[0, 10], [0, 0.1]]) y = tsknn(x) assert torch.all(y == torch.tensor([True, False])) + + +@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), + MaxAggregatorTorch, MinAggregatorTorch]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +def test_knn_ensemble_integration(aggregator, normalizer): + knn_detector = KNN( + k=[10, 14, 18], + aggregator=aggregator(), + normalizer=normalizer() + ) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + knn_detector.fit(X_ref) + knn_detector.infer_threshold(X_ref, 0.1) + result = knn_detector.predict(x_inlier) + result = result['data']['preds'][0] + assert not result + + x_outlier = np.array([[-1, 1.5]]) + result = knn_detector.predict(x_outlier) + result = result['data']['preds'][0] + assert result + + tsknn = torch.jit.script(knn_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = tsknn(x) + assert torch.all(y == torch.tensor([False, True])) + + +def test_knn_integration(): + knn_detector = KNN(k=18) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + knn_detector.fit(X_ref) + knn_detector.infer_threshold(X_ref, 0.1) + result = knn_detector.predict(x_inlier) + result = result['data']['preds'][0] + assert not result + + x_outlier = np.array([[-1, 1.5]]) + result = knn_detector.predict(x_outlier) + result = result['data']['preds'][0] + assert result + + tsknn = torch.jit.script(knn_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = tsknn(x) + assert torch.all(y == torch.tensor([False, True])) From 68e3a452444d0f9a086efdfc43bcd2b36ac9de09 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 29 Nov 2022 14:18:01 +0000 Subject: [PATCH 029/247] Fix minor mypy incompatibiity --- alibi_detect/od/base.py | 10 ++-------- alibi_detect/od/knn.py | 8 ++++---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 77377c5b4..b143cb749 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -1,6 +1,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Dict, TypedDict, Union +from typing import Dict, Any from typing_extensions import Protocol, runtime_checkable import numpy as np @@ -8,12 +8,6 @@ from alibi_detect.base import BaseDetector -class OutlierDetectorOutput(TypedDict): - """Outlier detector output type definition.""" - meta: Dict[str, str] - data: Dict[str, Union[bool, np.ndarray]] - - class OutlierDetector(BaseDetector, ABC): threshold_inferred = False ensemble = False @@ -67,7 +61,7 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: pass @abstractmethod - def predict(self, X: np.ndarray) -> OutlierDetectorOutput: + def predict(self, X: np.ndarray) -> Dict[str, Any]: """ Predict whether the instances in X are outliers or not. diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index f2b3df6ca..9059d9ca7 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,9 +1,9 @@ -from typing import Callable, Union, Optional, List +from typing import Callable, Union, Optional, List, Dict, Any from typing_extensions import Literal import numpy as np -from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol, OutlierDetectorOutput +from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol from alibi_detect.od.backend import KNNTorch, AccumulatorTorch from alibi_detect.utils.frameworks import BackendValidator @@ -107,7 +107,7 @@ def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) - def predict(self, X: Union[np.ndarray, List]) -> OutlierDetectorOutput: + def predict(self, X: Union[np.ndarray, List]) -> Dict[str, Any]: """Predict whether the instances in X are outliers or not. Parameters @@ -123,7 +123,7 @@ def predict(self, X: Union[np.ndarray, List]) -> OutlierDetectorOutput: the detector. """ outputs = self.backend.predict(self.backend._to_tensor(X)) - output: OutlierDetectorOutput = { + output: Dict[str, Any] = { 'data': outputs.numpy(), 'meta': self.meta } From 77ffdabe6ffa5d16039dd196c0f350bd08674531 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 29 Nov 2022 14:42:19 +0000 Subject: [PATCH 030/247] Add line breaks in return docstrings --- alibi_detect/od/base.py | 2 +- alibi_detect/od/knn.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index b143cb749..cd8f5bcbc 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -72,7 +72,7 @@ def predict(self, X: np.ndarray) -> Dict[str, Any]: Returns ------- - Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, + Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, \ 'data' also contains the outlier labels. """ pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 9059d9ca7..76ce88369 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -89,7 +89,7 @@ def score(self, X: Union[np.ndarray, List]) -> np.ndarray: Returns ------- - Anomaly scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the + Anomaly scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. """ score = self.backend.score(self.backend._to_tensor(X)) @@ -117,9 +117,9 @@ def predict(self, X: Union[np.ndarray, List]) -> Dict[str, Any]: Returns ------- - Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, - 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is - `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about + Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, \ + 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. """ outputs = self.backend.predict(self.backend._to_tensor(X)) From 111b40f2e3a9113dca8b3652c3bf0d2ead6c587b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 14 Dec 2022 16:32:23 +0000 Subject: [PATCH 031/247] Add torch.jit.is_scripting checks for unscriptable control flow --- alibi_detect/od/backend/tests/test_knn_backend.py | 14 ++++++++++++-- alibi_detect/od/backend/torch/base.py | 2 +- alibi_detect/od/backend/torch/ensemble.py | 5 +++-- alibi_detect/od/backend/torch/knn.py | 6 ++++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index f77b7fd44..aef0f0e05 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -46,7 +46,7 @@ def test_knn_torch_backend_ensemble(accumulator): assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) -def test_knn_torch_backend_ensemble_ts(accumulator): +def test_knn_torch_backend_ensemble_ts(tmp_path, accumulator): knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) @@ -66,8 +66,13 @@ def test_knn_torch_backend_ensemble_ts(accumulator): pred_2 = knn_torch(x) assert torch.all(pred_1 == pred_2) + knn_torch.save(tmp_path / 'knn_torch.pt') + knn_torch = torch.load(tmp_path / 'knn_torch.pt') + pred_2 = knn_torch(x) + assert torch.all(pred_1 == pred_2) + -def test_knn_torch_backend_ts(): +def test_knn_torch_backend_ts(tmp_path): knn_torch = KNNTorch(k=7) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) x_ref = torch.randn((1024, 10)) @@ -78,6 +83,11 @@ def test_knn_torch_backend_ts(): pred_2 = knn_torch(x) assert torch.all(pred_1 == pred_2) + knn_torch.save(tmp_path / 'knn_torch.pt') + knn_torch = torch.load(tmp_path / 'knn_torch.pt') + pred_2 = knn_torch(x) + assert torch.all(pred_1 == pred_2) + def test_knn_kernel(accumulator): kernel = GaussianRBF(sigma=torch.tensor((0.25))) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 974b8a5c5..2cd914d4e 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -67,7 +67,7 @@ def score(self, X: torch.Tensor) -> torch.Tensor: """ raise NotImplementedError() - @torch.jit.ignore + @torch.jit.unused def check_threshould_infered(self): """Check if threshold is inferred. diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 6fd0d3db4..4be058ddd 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -72,7 +72,7 @@ def _fit(self, X: torch.Tensor): """ raise NotImplementedError() - @torch.jit.ignore + @torch.jit.unused def check_fitted(self): """Raises error if parent object instance has not been fit. @@ -108,7 +108,8 @@ def transform(self, X: torch.Tensor): ------- transformed `torch.Tensor`. """ - self.check_fitted() + if not torch.jit.is_scripting(): + self.check_fitted() return self._transform(X) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 5e9f1510f..62b1d8f35 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -56,7 +56,8 @@ def forward(self, X: torch.Tensor) -> torch.Tensor: """ raw_scores = self.score(X) scores = self._accumulator(raw_scores) - self.check_threshould_infered() + if not torch.jit.is_scripting(): + self.check_threshould_infered() preds = scores > self.threshold return preds.cpu() @@ -77,7 +78,8 @@ def score(self, X: torch.Tensor) -> torch.Tensor: ValueError If called before detector has been fit. """ - self.check_fitted() + if not torch.jit.is_scripting(): + self.check_fitted() K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) bot_k_dists = torch.topk(K, int(torch.max(self.ks)), dim=1, largest=False) all_knn_dists = bot_k_dists.values[:, self.ks-1] From 42893bb0c54be08a5bebd773452a5fb6b64a051f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 3 Jan 2023 12:25:54 +0000 Subject: [PATCH 032/247] Add torch device logic for knn backend --- alibi_detect/od/backend/torch/base.py | 7 ++++--- alibi_detect/od/backend/torch/knn.py | 7 ++++--- alibi_detect/od/knn.py | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 2cd914d4e..242d36493 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -7,7 +7,7 @@ import torch from alibi_detect.od.backend.torch.ensemble import FitMixinTorch - +from alibi_detect.utils.pytorch.misc import get_device @dataclass class TorchOutlierDetectorOutput: @@ -32,7 +32,8 @@ class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): threshold_inferred = False threshold = None - def __init__(self): + def __init__(self, device: Optional[str] = None): + self.device = get_device(device) super().__init__() @abstractmethod @@ -92,7 +93,7 @@ def _to_tensor(self, X: Union[List, np.ndarray]): ------- `torch.Tensor` """ - return torch.as_tensor(X, dtype=torch.float32) + return torch.as_tensor(X, dtype=torch.float32, device=self.device) def _accumulator(self, X: torch.Tensor) -> torch.Tensor: """Accumulates the data. diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 62b1d8f35..d66d38f7d 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -12,7 +12,8 @@ def __init__( self, k: Union[np.ndarray, List], kernel: Optional[torch.nn.Module] = None, - accumulator: Optional[Accumulator] = None + accumulator: Optional[Accumulator] = None, + device: Optional[str] = None ): """PyTorch backend for KNN detector. @@ -31,10 +32,10 @@ def __init__( of :py:obj:`alibi_detect.od.backend.torch.ensemble.Accumulator`. Responsible for combining multiple scores into a single score. """ - TorchOutlierDetector.__init__(self) + TorchOutlierDetector.__init__(self, device=device) self.kernel = kernel self.ensemble = isinstance(k, (np.ndarray, list, tuple)) - self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k]) + self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) self.accumulator = accumulator def forward(self, X: torch.Tensor) -> torch.Tensor: diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 76ce88369..6343cdd2f 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -20,7 +20,8 @@ def __init__( kernel: Optional[Callable] = None, normalizer: Optional[Union[TransformProtocol, FittedTransformProtocol]] = None, aggregator: Optional[TransformProtocol] = None, - backend: Literal['pytorch'] = 'pytorch' + backend: Literal['pytorch'] = 'pytorch', + device: Literal['cuda', 'gpu', 'cpu'] = 'cpu', ) -> None: """ k-Nearest Neighbours (kNN) outlier detector. @@ -67,7 +68,7 @@ def __init__( normalizer=normalizer, aggregator=aggregator ) - self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator) + self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator, device=device) def fit(self, x_ref: Union[np.ndarray, List]) -> None: """Fit the kNN detector on reference data. From f74dfc857cc30e83dce90dd8642709044b8b4fd7 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 3 Jan 2023 13:34:53 +0000 Subject: [PATCH 033/247] Ensure cuda output tensors are converted to cpu --- alibi_detect/od/backend/torch/base.py | 17 ++++++++++++++++- alibi_detect/od/knn.py | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 242d36493..28ce5e09c 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -9,6 +9,7 @@ from alibi_detect.od.backend.torch.ensemble import FitMixinTorch from alibi_detect.utils.pytorch.misc import get_device + @dataclass class TorchOutlierDetectorOutput: """Output of the outlier detector.""" @@ -18,7 +19,7 @@ class TorchOutlierDetectorOutput: preds: Optional[torch.Tensor] p_vals: Optional[torch.Tensor] - def numpy(self) -> Dict[str, Union[bool, Optional[torch.Tensor]]]: + def _to_numpy(self) -> Dict[str, Union[bool, Optional[torch.Tensor]]]: """Converts the output to numpy.""" outputs = asdict(self) for key, value in outputs.items(): @@ -95,6 +96,20 @@ def _to_tensor(self, X: Union[List, np.ndarray]): """ return torch.as_tensor(X, dtype=torch.float32, device=self.device) + def _to_numpy(self, X: torch.Tensor): + """Converts the data to numpy. + + Parameters + ---------- + X + Data to convert. + + Returns + ------- + `np.ndarray` + """ + return X.cpu().detach().numpy() + def _accumulator(self, X: torch.Tensor) -> torch.Tensor: """Accumulates the data. diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 6343cdd2f..4ab351efa 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -94,7 +94,7 @@ def score(self, X: Union[np.ndarray, List]) -> np.ndarray: instance. """ score = self.backend.score(self.backend._to_tensor(X)) - return score.numpy() + return self.backend._to_numpy(score) def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: """Infer the threshold for the kNN detector. @@ -125,7 +125,7 @@ def predict(self, X: Union[np.ndarray, List]) -> Dict[str, Any]: """ outputs = self.backend.predict(self.backend._to_tensor(X)) output: Dict[str, Any] = { - 'data': outputs.numpy(), + 'data': outputs._to_numpy(), 'meta': self.meta } return output From 0754d35763d83ca34a8e2c496c84f0b24f0af0c8 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 3 Jan 2023 14:42:47 +0000 Subject: [PATCH 034/247] Set default device for knn to None --- alibi_detect/od/backend/torch/base.py | 2 +- alibi_detect/od/backend/torch/knn.py | 5 ++++- alibi_detect/od/knn.py | 9 ++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 28ce5e09c..3d2fe2740 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -33,7 +33,7 @@ class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): threshold_inferred = False threshold = None - def __init__(self, device: Optional[str] = None): + def __init__(self, device: Optional[Union[str, torch.device]] = None): self.device = get_device(device) super().__init__() diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index d66d38f7d..1d01589de 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -13,7 +13,7 @@ def __init__( k: Union[np.ndarray, List], kernel: Optional[torch.nn.Module] = None, accumulator: Optional[Accumulator] = None, - device: Optional[str] = None + device: Optional[Union[str, torch.device]] = None ): """PyTorch backend for KNN detector. @@ -31,6 +31,9 @@ def __init__( If `k` is an array of integers then the accumulator must not be None. Should be an instance of :py:obj:`alibi_detect.od.backend.torch.ensemble.Accumulator`. Responsible for combining multiple scores into a single score. + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. """ TorchOutlierDetector.__init__(self, device=device) self.kernel = kernel diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 4ab351efa..11b925578 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -6,6 +6,10 @@ from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol from alibi_detect.od.backend import KNNTorch, AccumulatorTorch from alibi_detect.utils.frameworks import BackendValidator +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import torch backends = { @@ -21,7 +25,7 @@ def __init__( normalizer: Optional[Union[TransformProtocol, FittedTransformProtocol]] = None, aggregator: Optional[TransformProtocol] = None, backend: Literal['pytorch'] = 'pytorch', - device: Literal['cuda', 'gpu', 'cpu'] = 'cpu', + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """ k-Nearest Neighbours (kNN) outlier detector. @@ -40,6 +44,9 @@ def __init__( then an aggregator is required. backend Backend used for outlier detection. Defaults to `'pytorch'`. Options are `'pytorch'`. + device + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. Raises ------ From d0d04d6270a3cef3c9bc8fd17815a5ada88c6863 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 3 Jan 2023 15:36:40 +0000 Subject: [PATCH 035/247] Place tensors on correct device in transform --- alibi_detect/od/backend/torch/ensemble.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 4be058ddd..ea6a6cb14 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -259,7 +259,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: """ if self.weights is None: m = scores.shape[-1] - self.weights = torch.ones(m)/m + self.weights = torch.ones(m, device=scores.device)/m return scores @ self.weights From 09412c4ca8091417be7b1ecdde706981a3e3436e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 3 Jan 2023 18:09:30 +0000 Subject: [PATCH 036/247] Update default knn ensemble aggregator and normalizer values --- alibi_detect/od/backend/__init__.py | 5 +++++ alibi_detect/od/backend/torch/knn.py | 2 +- alibi_detect/od/base.py | 1 - alibi_detect/od/knn.py | 29 ++++++++++++++-------------- alibi_detect/od/tests/test_knn.py | 25 ++++++++++++++++-------- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index ab431f67b..916f750c7 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -1,4 +1,5 @@ from alibi_detect.utils.missing_optional_dependency import import_optional +from typing import Literal KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) PValNormalizerTorch, ShiftAndScaleNormalizerTorch, TopKAggregatorTorch, AverageAggregatorTorch, \ @@ -7,3 +8,7 @@ ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', 'AverageAggregator', 'MaxAggregator', 'MinAggregator', 'Accumulator'] ) + +NormalizerLiterals = Literal['PValNormalizerTorch', 'ShiftAndScaleNormalizerTorch'] +AggregatorLiterals = Literal['TopKAggregatorTorch', 'AverageAggregatorTorch', + 'MaxAggregatorTorch', 'MinAggregatorTorch'] diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 1d01589de..497cc9c9d 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -98,6 +98,6 @@ def _fit(self, x_ref: torch.Tensor): The Dataset tensor. """ self.x_ref = x_ref - if self.accumulator is not None: + if self.ensemble: scores = self.score(x_ref) self.accumulator.fit(scores) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index cd8f5bcbc..cf0898ea9 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -10,7 +10,6 @@ class OutlierDetector(BaseDetector, ABC): threshold_inferred = False - ensemble = False def __init__(self): """ Base class for outlier detection algorithms.""" diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 11b925578..8fe17062b 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -4,7 +4,8 @@ import numpy as np from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol -from alibi_detect.od.backend import KNNTorch, AccumulatorTorch +from alibi_detect.od import backend as backend_objs +from alibi_detect.od.backend import NormalizerLiterals, AggregatorLiterals, KNNTorch, AccumulatorTorch from alibi_detect.utils.frameworks import BackendValidator from typing import TYPE_CHECKING @@ -22,8 +23,9 @@ def __init__( self, k: Union[int, np.ndarray], kernel: Optional[Callable] = None, - normalizer: Optional[Union[TransformProtocol, FittedTransformProtocol]] = None, - aggregator: Optional[TransformProtocol] = None, + normalizer: Optional[Union[TransformProtocol, FittedTransformProtocol, NormalizerLiterals]] + = 'ShiftAndScaleNormalizerTorch', + aggregator: Optional[Union[TransformProtocol, AggregatorLiterals]] = 'AverageAggregatorTorch', backend: Literal['pytorch'] = 'pytorch', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: @@ -37,11 +39,11 @@ def __init__( to aggregate the scores. kernel Kernel function to use for outlier detection. If None, `torch.cdist` is used. - normalizer - Normalizer to use for outlier detection. If None, no normalisation is applied. - aggregator - Aggregator to use for outlier detection. If None, no aggregation is applied. If an array is passed for `k`, - then an aggregator is required. + # normalizer + # Normalizer to use for outlier detection. If None, no normalisation is applied. + # aggregator + # Aggregator to use for outlier detection. If None, no aggregation is applied. If an array is passed for `k`, + # then an aggregator is required. backend Backend used for outlier detection. Defaults to `'pytorch'`. Options are `'pytorch'`. device @@ -63,14 +65,13 @@ def __init__( construct_name=self.__class__.__name__ ).verify_backend(backend_str) - if isinstance(k, (list, np.ndarray)) and aggregator is None: - raise ValueError((f'k={k} is type {type(k)} but aggregator is {aggregator}, you must ' - 'specify at least an aggregator if you want to use the knn detector ' - 'ensemble like this.')) - backend_cls, accumulator_cls = backends[backend] accumulator = None - if normalizer is not None or aggregator is not None: + if isinstance(k, (list, np.ndarray, tuple)): + if isinstance(aggregator, str): + aggregator = getattr(backend_objs, aggregator)() + if isinstance(normalizer, str): + normalizer = getattr(backend_objs, normalizer)() accumulator = accumulator_cls( normalizer=normalizer, aggregator=aggregator diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index debc2c769..bbaf9b689 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -40,6 +40,22 @@ def test_fitted_knn_single_score(): assert y['p_vals'] is None +def test_default_knn_ensemble_init(): + knn_detector = KNN(k=[8, 9, 10]) + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + x = np.array([[0, 10], [0.1, 0]]) + y = knn_detector.predict(x) + y = y['data'] + assert y['scores'][0] > 5 + assert y['scores'][1] < 1 + + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['preds'] is None + assert y['p_vals'] is None + + def test_fitted_knn_predict(): knn_detector = make_knn_detector(k=10) x_ref = np.random.randn(100, 2) @@ -107,14 +123,7 @@ def test_fitted_knn_ensemble_predict(aggregator, normalizer): assert y['threshold'] is not None assert y['p_vals'].all() assert (y['preds'] == [True, False]).all() - - -def test_incorrect_knn_ensemble_init(): - with pytest.raises(ValueError) as err: - KNN(k=[8, 9, 10]) - assert str(err.value) == ("k=[8, 9, 10] is type but aggregator is None, you must specify at least an" - " aggregator if you want to use the knn detector ensemble like this.") - + @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), MaxAggregatorTorch, MinAggregatorTorch]) From 65fbeac7121823cac3b06730367c6c3cedf7991c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 10:54:06 +0000 Subject: [PATCH 037/247] Add tests for aggregator and normalizer default values --- alibi_detect/od/backend/__init__.py | 6 +++--- alibi_detect/od/base.py | 5 ++++- alibi_detect/od/knn.py | 22 +++++++++++----------- alibi_detect/od/tests/test_knn.py | 20 ++++++++++++++++---- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index 916f750c7..1948df1b9 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -9,6 +9,6 @@ 'AverageAggregator', 'MaxAggregator', 'MinAggregator', 'Accumulator'] ) -NormalizerLiterals = Literal['PValNormalizerTorch', 'ShiftAndScaleNormalizerTorch'] -AggregatorLiterals = Literal['TopKAggregatorTorch', 'AverageAggregatorTorch', - 'MaxAggregatorTorch', 'MinAggregatorTorch'] +normalizer_literals = Literal['PValNormalizerTorch', 'ShiftAndScaleNormalizerTorch'] +aggregator_literals = Literal['TopKAggregatorTorch', 'AverageAggregatorTorch', + 'MaxAggregatorTorch', 'MinAggregatorTorch'] diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index cf0898ea9..94c230818 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -1,6 +1,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Dict, Any +from typing import Dict, Any, Union from typing_extensions import Protocol, runtime_checkable import numpy as np @@ -100,3 +100,6 @@ def _fit(self, x_ref): def check_fitted(self): pass + + +transform_protocols = Union[TransformProtocol, FittedTransformProtocol] diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 8fe17062b..3b26d1cd0 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -3,9 +3,9 @@ from typing_extensions import Literal import numpy as np -from alibi_detect.od.base import OutlierDetector, TransformProtocol, FittedTransformProtocol +from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols from alibi_detect.od import backend as backend_objs -from alibi_detect.od.backend import NormalizerLiterals, AggregatorLiterals, KNNTorch, AccumulatorTorch +from alibi_detect.od.backend import normalizer_literals, aggregator_literals, KNNTorch, AccumulatorTorch from alibi_detect.utils.frameworks import BackendValidator from typing import TYPE_CHECKING @@ -23,11 +23,10 @@ def __init__( self, k: Union[int, np.ndarray], kernel: Optional[Callable] = None, - normalizer: Optional[Union[TransformProtocol, FittedTransformProtocol, NormalizerLiterals]] - = 'ShiftAndScaleNormalizerTorch', - aggregator: Optional[Union[TransformProtocol, AggregatorLiterals]] = 'AverageAggregatorTorch', - backend: Literal['pytorch'] = 'pytorch', + normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizerTorch', + aggregator: Optional[Union[TransformProtocol, aggregator_literals]] = 'AverageAggregatorTorch', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, + backend: Literal['pytorch'] = 'pytorch', ) -> None: """ k-Nearest Neighbours (kNN) outlier detector. @@ -39,11 +38,10 @@ def __init__( to aggregate the scores. kernel Kernel function to use for outlier detection. If None, `torch.cdist` is used. - # normalizer - # Normalizer to use for outlier detection. If None, no normalisation is applied. - # aggregator - # Aggregator to use for outlier detection. If None, no aggregation is applied. If an array is passed for `k`, - # then an aggregator is required. + normalizer + Normalizer to use for outlier detection. If None, no normalisation is applied. + aggregator + Aggregator to use for outlier detection. Can be set to None if `k` is a single value. backend Backend used for outlier detection. Defaults to `'pytorch'`. Options are `'pytorch'`. device @@ -70,6 +68,8 @@ def __init__( if isinstance(k, (list, np.ndarray, tuple)): if isinstance(aggregator, str): aggregator = getattr(backend_objs, aggregator)() + if aggregator is None: + raise ValueError("If `k` is an array, an aggregator is required.") if isinstance(normalizer, str): normalizer = getattr(backend_objs, normalizer)() accumulator = accumulator_cls( diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index bbaf9b689..fcb111ee6 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -9,7 +9,10 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): - knn_detector = KNN(k=k, aggregator=aggregator, normalizer=normalizer) + knn_detector = KNN( + k=k, aggregator=aggregator, + normalizer=normalizer + ) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) knn_detector.infer_threshold(x_ref, 0.1) @@ -56,6 +59,12 @@ def test_default_knn_ensemble_init(): assert y['p_vals'] is None +def test_incorrect_knn_ensemble_init(): + with pytest.raises(ValueError) as err: + KNN(k=[8, 9, 10], aggregator=None) + assert str(err.value) == 'If `k` is an array, an aggregator is required.' + + def test_fitted_knn_predict(): knn_detector = make_knn_detector(k=10) x_ref = np.random.randn(100, 2) @@ -123,7 +132,7 @@ def test_fitted_knn_ensemble_predict(aggregator, normalizer): assert y['threshold'] is not None assert y['p_vals'].all() assert (y['preds'] == [True, False]).all() - + @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), MaxAggregatorTorch, MinAggregatorTorch]) @@ -145,8 +154,11 @@ def test_knn_single_torchscript(): @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), - MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) + MaxAggregatorTorch, MinAggregatorTorch, lambda: 'AverageAggregatorTorch', + lambda: 'TopKAggregatorTorch', lambda: 'MaxAggregatorTorch', + lambda: 'MinAggregatorTorch']) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None, + lambda: 'ShiftAndScaleNormalizerTorch', lambda: 'PValNormalizerTorch']) def test_knn_ensemble_integration(aggregator, normalizer): knn_detector = KNN( k=[10, 14, 18], From 4cd2ebaa0c93a782b5e254a3929ff01763efb792 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 11:19:56 +0000 Subject: [PATCH 038/247] Remove Optional type from aggregator --- alibi_detect/od/knn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 3b26d1cd0..ba045faec 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -24,7 +24,7 @@ def __init__( k: Union[int, np.ndarray], kernel: Optional[Callable] = None, normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizerTorch', - aggregator: Optional[Union[TransformProtocol, aggregator_literals]] = 'AverageAggregatorTorch', + aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregatorTorch', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', ) -> None: From 98d2ba5f8724029592ad04c429ce68d0a21b4aa9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 11:29:24 +0000 Subject: [PATCH 039/247] Change X -> x throughout --- alibi_detect/od/backend/torch/base.py | 34 +++++++++--------- alibi_detect/od/backend/torch/ensemble.py | 44 +++++++++++------------ alibi_detect/od/backend/torch/knn.py | 18 +++++----- alibi_detect/od/base.py | 26 +++++++------- alibi_detect/od/knn.py | 20 +++++------ 5 files changed, 71 insertions(+), 71 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 3d2fe2740..364c5c032 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -54,12 +54,12 @@ def _fit(self, x_ref: torch.Tensor) -> None: raise NotImplementedError() @abstractmethod - def score(self, X: torch.Tensor) -> torch.Tensor: + def score(self, x: torch.Tensor) -> torch.Tensor: """Score the data. Parameters ---------- - X + x Data to score. Raises @@ -82,40 +82,40 @@ def check_threshould_infered(self): raise ValueError((f'{self.__class__.__name__} has no threshold set, ' 'call `infer_threshold` before predicting.')) - def _to_tensor(self, X: Union[List, np.ndarray]): + def _to_tensor(self, x: Union[List, np.ndarray]): """Converts the data to a tensor. Parameters ---------- - X + x Data to convert. Returns ------- `torch.Tensor` """ - return torch.as_tensor(X, dtype=torch.float32, device=self.device) + return torch.as_tensor(x, dtype=torch.float32, device=self.device) - def _to_numpy(self, X: torch.Tensor): + def _to_numpy(self, x: torch.Tensor): """Converts the data to numpy. Parameters ---------- - X + x Data to convert. Returns ------- `np.ndarray` """ - return X.cpu().detach().numpy() + return x.cpu().detach().numpy() - def _accumulator(self, X: torch.Tensor) -> torch.Tensor: + def _accumulator(self, x: torch.Tensor) -> torch.Tensor: """Accumulates the data. Parameters ---------- - X + x Data to accumulate. Returns @@ -124,7 +124,7 @@ def _accumulator(self, X: torch.Tensor) -> torch.Tensor: """ # `type: ignore` here becuase self.accumulator here causes an error with mypy when using torch.jit.script. # For some reason it thinks self.accumulator is a torch.Tensor and therefore is not callable. - return self.accumulator(X) if self.accumulator is not None else X # type: ignore + return self.accumulator(x) if self.accumulator is not None else x # type: ignore def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: """Classify the data as outlier or not. @@ -155,12 +155,12 @@ def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ if self.threshold_inferred else None - def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: + def infer_threshold(self, x: torch.Tensor, fpr: float) -> None: """Infer the threshold for the data. Prerequisite for outlier predictions. Parameters ---------- - X + x Data to infer the threshold for. fpr False positive rate to use for threshold inference. @@ -172,12 +172,12 @@ def infer_threshold(self, X: torch.Tensor, fpr: float) -> None: """ if not 0 < fpr < 1: ValueError('fpr must be in (0, 1).') - self.val_scores = self.score(X) + self.val_scores = self.score(x) self.val_scores = self._accumulator(self.val_scores) self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True - def predict(self, X: torch.Tensor) -> TorchOutlierDetectorOutput: + def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: """Predict outlier labels for the data. Computes the outlier scores. If the detector is not fit on reference data we raise an error. @@ -186,7 +186,7 @@ def predict(self, X: torch.Tensor) -> TorchOutlierDetectorOutput: Parameters ---------- - X + x Data to predict. Raises @@ -201,7 +201,7 @@ def predict(self, X: torch.Tensor) -> TorchOutlierDetectorOutput: """ self.check_fitted() # type: ignore - raw_scores = self.score(X) + raw_scores = self.score(x) scores = self._accumulator(raw_scores) return TorchOutlierDetectorOutput( scores=scores, diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index ea6a6cb14..bb3937f3d 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -16,16 +16,16 @@ def __init__(self): """ super().__init__() - def transform(self, X: torch.Tensor): - return self._transform(X) + def transform(self, x: torch.Tensor): + return self._transform(x) @abstractmethod - def _transform(self, X: torch.Tensor): + def _transform(self, x: torch.Tensor): """Applies class transform to numpy array Parameters ---------- - X + x numpy array to be transformed Raises @@ -36,8 +36,8 @@ def _transform(self, X: torch.Tensor): """ raise NotImplementedError() - def forward(self, X: torch.Tensor): - return self.transform(X=X) + def forward(self, x: torch.Tensor): + return self.transform(x=x) class FitMixinTorch: @@ -50,19 +50,19 @@ def __init__(self): """ super().__init__() - def fit(self, X: torch.Tensor) -> FitMixinTorch: + def fit(self, x: torch.Tensor) -> FitMixinTorch: self._fitted = True - self._fit(X) + self._fit(x) return self - def _fit(self, X: torch.Tensor): - """Fit on `X` tensor. + def _fit(self, x: torch.Tensor): + """Fit on `x` tensor. This method should be overidden on child classes. Parameters ---------- - X + x Reference `torch.Tensor` for fitting object. Raises @@ -96,12 +96,12 @@ def __init__(self): BaseTransformTorch.__init__(self) FitMixinTorch.__init__(self) - def transform(self, X: torch.Tensor): + def transform(self, x: torch.Tensor): """Checks to make sure transform has been fitted and then applies trasform to input tensor. Parameters ---------- - X + x `torch.Tensor` being transformed. Returns @@ -110,7 +110,7 @@ def transform(self, X: torch.Tensor): """ if not torch.jit.is_scripting(): self.check_fitted() - return self._transform(X) + return self._transform(x) class PValNormalizer(BaseFittedTransformTorch): @@ -326,12 +326,12 @@ def __init__(self, self.fitted = True self.aggregator = aggregator - def _transform(self, X: torch.Tensor): + def _transform(self, x: torch.Tensor): """Apply the normalizer and aggregator to the scores. Parameters ---------- - X + x `Torch.Tensor` of scores from ensemble of detectors. Returns @@ -339,17 +339,17 @@ def _transform(self, X: torch.Tensor): `Torch.Tensor` of aggregated and normalised scores. """ if self.normalizer is not None: - X = self.normalizer(X) - X = self.aggregator(X) - return X + x = self.normalizer(x) + x = self.aggregator(x) + return x - def _fit(self, X: torch.Tensor): + def _fit(self, x: torch.Tensor): """Fit the normalizer to the scores. Parameters ---------- - X + x `Torch.Tensor` of scores from ensemble of detectors. """ if self.normalizer is not None: - self.normalizer.fit(X) + self.normalizer.fit(x) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 497cc9c9d..e9c62beeb 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -41,12 +41,12 @@ def __init__( self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) self.accumulator = accumulator - def forward(self, X: torch.Tensor) -> torch.Tensor: - """Detect if X is an outlier. + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Detect if x is an outlier. Parameters ---------- - X + x `torch.Tensor` with leading batch dimension. Returns @@ -58,24 +58,24 @@ def forward(self, X: torch.Tensor) -> torch.Tensor: ValueError If called before detector has had threshould_infered method called. """ - raw_scores = self.score(X) + raw_scores = self.score(x) scores = self._accumulator(raw_scores) if not torch.jit.is_scripting(): self.check_threshould_infered() preds = scores > self.threshold return preds.cpu() - def score(self, X: torch.Tensor) -> torch.Tensor: - """Computes the score of `X` + def score(self, x: torch.Tensor) -> torch.Tensor: + """Computes the score of `x` Parameters ---------- - X + x Score a tensor of instances. First dimesnion corresponds to batch. Returns ------- - Tensor of scores for each element in `X`. + Tensor of scores for each element in `x`. Raises ------ @@ -84,7 +84,7 @@ def score(self, X: torch.Tensor) -> torch.Tensor: """ if not torch.jit.is_scripting(): self.check_fitted() - K = -self.kernel(X, self.x_ref) if self.kernel is not None else torch.cdist(X, self.x_ref) + K = -self.kernel(x, self.x_ref) if self.kernel is not None else torch.cdist(x, self.x_ref) bot_k_dists = torch.topk(K, int(torch.max(self.ks)), dim=1, largest=False) all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 94c230818..44e45d04c 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -18,25 +18,25 @@ def __init__(self): self.meta['detector_type'] = 'outlier' @abstractmethod - def fit(self, X: np.ndarray) -> None: + def fit(self, x: np.ndarray) -> None: """ Fit outlier detector to data. Parameters ---------- - X + x Reference data. """ pass @abstractmethod - def score(self, X: np.ndarray) -> np.ndarray: + def score(self, x: np.ndarray) -> np.ndarray: """ - Compute anomaly scores of the instances in X. + Compute anomaly scores of the instances in x. Parameters ---------- - X + x Data to score. Returns @@ -46,13 +46,13 @@ def score(self, X: np.ndarray) -> np.ndarray: pass @abstractmethod - def infer_threshold(self, X: np.ndarray, fpr: float) -> None: + def infer_threshold(self, x: np.ndarray, fpr: float) -> None: """ Infer the threshold for the outlier detector. Parameters ---------- - X + x Reference data. fpr False positive rate used to infer the threshold. @@ -60,13 +60,13 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: pass @abstractmethod - def predict(self, X: np.ndarray) -> Dict[str, Any]: + def predict(self, x: np.ndarray) -> Dict[str, Any]: """ - Predict whether the instances in X are outliers or not. + Predict whether the instances in x are outliers or not. Parameters ---------- - X + x Data to predict. Returns @@ -77,15 +77,15 @@ def predict(self, X: np.ndarray) -> Dict[str, Any]: pass -# Using Protocols instead base classes for the backend classes. This is a bit more flexible and allows us to +# Use Protocols instead of base classes for the backend associated objects. This is a bit more flexible and allows us to # avoid the torch/tensorflow imports in the base class. @runtime_checkable class TransformProtocol(Protocol): """Protocol for transformer objects.""" - def transform(self, X): + def transform(self, x): pass - def _transform(self, X): + def _transform(self, x): pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index ba045faec..ffb5dc55b 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -88,20 +88,20 @@ def fit(self, x_ref: Union[np.ndarray, List]) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) - def score(self, X: Union[np.ndarray, List]) -> np.ndarray: - """Score X instances using the kNN. + def score(self, x: Union[np.ndarray, List]) -> np.ndarray: + """Score x instances using the kNN. Parameters ---------- - X - Data to score. The shape of `X` should be `(n_instances, n_features)`. + x + Data to score. The shape of `x` should be `(n_instances, n_features)`. Returns ------- Anomaly scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. """ - score = self.backend.score(self.backend._to_tensor(X)) + score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: @@ -116,13 +116,13 @@ def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) - def predict(self, X: Union[np.ndarray, List]) -> Dict[str, Any]: - """Predict whether the instances in X are outliers or not. + def predict(self, x: Union[np.ndarray, List]) -> Dict[str, Any]: + """Predict whether the instances in x are outliers or not. Parameters ---------- - X - Data to predict. The shape of `X` should be `(n_instances, n_features)`. + x + Data to predict. The shape of `x` should be `(n_instances, n_features)`. Returns ------- @@ -131,7 +131,7 @@ def predict(self, X: Union[np.ndarray, List]) -> Dict[str, Any]: `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. """ - outputs = self.backend.predict(self.backend._to_tensor(X)) + outputs = self.backend.predict(self.backend._to_tensor(x)) output: Dict[str, Any] = { 'data': outputs._to_numpy(), 'meta': self.meta From 3d57b6d04ea7c9865441ed2f5506f22ad1ff116e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 11:31:47 +0000 Subject: [PATCH 040/247] Change anomaly -> outlier throughout --- alibi_detect/od/base.py | 4 ++-- alibi_detect/od/knn.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 44e45d04c..ae4a409dd 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -32,7 +32,7 @@ def fit(self, x: np.ndarray) -> None: @abstractmethod def score(self, x: np.ndarray) -> np.ndarray: """ - Compute anomaly scores of the instances in x. + Compute outlier scores of the instances in x. Parameters ---------- @@ -41,7 +41,7 @@ def score(self, x: np.ndarray) -> np.ndarray: Returns ------- - Anomaly scores. The higher the score, the more anomalous the instance. + Outlier scores. The higher the score, the more anomalous the instance. """ pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index ffb5dc55b..113e1052b 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -98,7 +98,7 @@ def score(self, x: Union[np.ndarray, List]) -> np.ndarray: Returns ------- - Anomaly scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. """ score = self.backend.score(self.backend._to_tensor(x)) From bc01ddf6f603121852cc4960e1fb4adead78e052 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 11:51:20 +0000 Subject: [PATCH 041/247] Improve fpr description --- alibi_detect/od/knn.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 113e1052b..08072ed82 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -105,14 +105,17 @@ def score(self, x: Union[np.ndarray, List]) -> np.ndarray: return self.backend._to_numpy(score) def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: - """Infer the threshold for the kNN detector. + """Infer the threshold for the kNN detector. The threshold is inferred using the reference data and the false + positive rate. The threshold is used to determine the outlier labels in the predict method. Parameters ---------- x_ref Reference data used to infer the threshold. fpr - False positive rate used to infer the threshold. + False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ + `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ + `(0, 1)`. """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) From 47566b8b6ad5b0929aa32babfe131aef97de0e3b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 11:54:34 +0000 Subject: [PATCH 042/247] Update PValNormalizer docstring --- alibi_detect/od/backend/torch/ensemble.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index bb3937f3d..e7f3237e2 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -118,9 +118,8 @@ def __init__(self): """Maps scores to there p values. Needs to be fit (see py:obj:alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch). - Transform counts the number of scores in the reference dataset that are greter than the score - of interest and divides by the size of the reference dataset. Output is between 1 and 0. Small - values are likely to be outliers. + Returns the proportion of scores in the reference dataset that are greater than the score of + interest. Output is between 1 and 0. Small values are likely to be outliers. """ super().__init__() self.val_scores = None @@ -205,8 +204,7 @@ def __init__(self, k: Optional[int] = None): """Takes the mean of the top k scores. Parameters - ---------- - k + ----------Anomaly number of scores to take the mean of. If `k` is left `None` then will be set to half the number of scores passed in the forward call. """ From bda15fd8793e27ce64bd2a1d9fba9dacdb3f63fc Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 13:53:50 +0000 Subject: [PATCH 043/247] Add custom error types --- alibi_detect/od/backend/tests/test_ensemble.py | 9 +++++++-- .../od/backend/tests/test_knn_backend.py | 17 +++++++++-------- alibi_detect/od/backend/torch/base.py | 9 +++++---- alibi_detect/od/backend/torch/ensemble.py | 18 +++++++++--------- alibi_detect/od/base.py | 10 ++++++++++ alibi_detect/od/tests/test_knn.py | 6 ++++-- 6 files changed, 44 insertions(+), 25 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/backend/tests/test_ensemble.py index 7f4767428..a07f7b6ba 100644 --- a/alibi_detect/od/backend/tests/test_ensemble.py +++ b/alibi_detect/od/backend/tests/test_ensemble.py @@ -2,14 +2,17 @@ import torch from alibi_detect.od.backend.torch import ensemble +from alibi_detect.od.base import NotFitException def test_pval_normalizer(): normalizer = ensemble.PValNormalizer() x = torch.randn(3, 10) x_ref = torch.randn(64, 10) - with pytest.raises(ValueError): + # unfit normalizer raises exception + with pytest.raises(NotFitException) as err: normalizer(x) + assert err.value.args[0] == 'PValNormalizer has not been fit!' normalizer.fit(x_ref) x_norm = normalizer(x) @@ -22,8 +25,10 @@ def test_shift_and_scale_normalizer(): normalizer = ensemble.ShiftAndScaleNormalizer() x = torch.randn(3, 10) x_ref = torch.randn(64, 10) - with pytest.raises(ValueError): + # unfit normalizer raises exception + with pytest.raises(NotFitException) as err: normalizer(x) + assert err.value.args[0] == 'ShiftAndScaleNormalizer has not been fit!' normalizer.fit(x_ref) x_norm = normalizer(x) diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index aef0f0e05..0acbe6d35 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -4,6 +4,7 @@ from alibi_detect.od.backend.torch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.backend.torch.ensemble import Accumulator, PValNormalizer, AverageAggregator +from alibi_detect.od.base import NotFitException, ThresholdNotInferredException @pytest.fixture(scope='session') @@ -50,11 +51,11 @@ def test_knn_torch_backend_ensemble_ts(tmp_path, accumulator): knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has not been fit!' - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: knn_torch.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -116,11 +117,11 @@ def test_knn_torch_backend_ensemble_fit_errors(accumulator): assert not knn_torch._fitted x = torch.randn((1, 10)) - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has not been fit!' - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: knn_torch.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -128,7 +129,7 @@ def test_knn_torch_backend_ensemble_fit_errors(accumulator): knn_torch.fit(x_ref) assert knn_torch._fitted - with pytest.raises(ValueError) as err: + with pytest.raises(ThresholdNotInferredException) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' @@ -140,11 +141,11 @@ def test_knn_torch_backend_fit_errors(): assert not knn_torch._fitted x = torch.randn((1, 10)) - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has not been fit!' - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: knn_torch.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -153,7 +154,7 @@ def test_knn_torch_backend_fit_errors(): assert knn_torch._fitted - with pytest.raises(ValueError) as err: + with pytest.raises(ThresholdNotInferredException) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 364c5c032..d3383d633 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -8,6 +8,7 @@ from alibi_detect.od.backend.torch.ensemble import FitMixinTorch from alibi_detect.utils.pytorch.misc import get_device +from alibi_detect.od.base import ThresholdNotInferredException @dataclass @@ -75,12 +76,12 @@ def check_threshould_infered(self): Raises ------ - ValueError + ThresholdNotInferredException Raised if threshold is not inferred. """ if not self.threshold_inferred: - raise ValueError((f'{self.__class__.__name__} has no threshold set, ' - 'call `infer_threshold` before predicting.')) + raise ThresholdNotInferredException((f'{self.__class__.__name__} has no threshold set, ' + 'call `infer_threshold` before predicting.')) def _to_tensor(self, x: Union[List, np.ndarray]): """Converts the data to a tensor. @@ -88,7 +89,7 @@ def _to_tensor(self, x: Union[List, np.ndarray]): Parameters ---------- x - Data to convert. + Data to convert.ThresholdNotInferredException Returns ------- diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index e7f3237e2..87602b8de 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -6,6 +6,8 @@ import numpy as np from torch.nn import Module +from alibi_detect.od.base import NotFitException + class BaseTransformTorch(Module, ABC): def __init__(self): @@ -78,12 +80,11 @@ def check_fitted(self): Raises ------ - ValueError + NotFitException Raised if method called and object has not been fit. """ if not self._fitted: - # TODO: make our own NotFitted Error here! - raise ValueError(f'{self.__class__.__name__} has not been fit!') + raise NotFitException(f'{self.__class__.__name__} has not been fit!') class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): @@ -117,7 +118,7 @@ class PValNormalizer(BaseFittedTransformTorch): def __init__(self): """Maps scores to there p values. - Needs to be fit (see py:obj:alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch). + Needs to be fit (see :py:obj:`~alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch`). Returns the proportion of scores in the reference dataset that are greater than the score of interest. Output is between 1 and 0. Small values are likely to be outliers. """ @@ -161,7 +162,7 @@ class ShiftAndScaleNormalizer(BaseFittedTransformTorch): def __init__(self): """Maps scores to their normalised values. - Needs to be fit (see py:obj:alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch). + Needs to be fit (see :py:obj:`~alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch`). Subtracts the dataset mean and scales by the standard deviation. """ super().__init__() @@ -204,7 +205,8 @@ def __init__(self, k: Optional[int] = None): """Takes the mean of the top k scores. Parameters - ----------Anomaly + ---------- + k number of scores to take the mean of. If `k` is left `None` then will be set to half the number of scores passed in the forward call. """ @@ -244,9 +246,7 @@ def __init__(self, weights: Optional[torch.Tensor] = None): def _transform(self, scores: torch.Tensor) -> torch.Tensor: """Averages the scores of the detectors in an ensemble. If weights where passed in the init - then these are used to weight the scores. - - Parameters + then these are used to weight the scores.Anomaly ---------- scores `Torch.Tensor` of scores from ensemble of detectors. diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index ae4a409dd..3cb739fc5 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -103,3 +103,13 @@ def check_fitted(self): transform_protocols = Union[TransformProtocol, FittedTransformProtocol] + + +class NotFitException(Exception): + """Exception raised when a transform is not fitted.""" + pass + + +class ThresholdNotInferredException(Exception): + """Exception raised when a transform is not fitted.""" + pass diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index fcb111ee6..e7f9a4150 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -5,6 +5,8 @@ from alibi_detect.od.knn import KNN from alibi_detect.od.backend import AverageAggregatorTorch, TopKAggregatorTorch, MaxAggregatorTorch, \ MinAggregatorTorch, ShiftAndScaleNormalizerTorch, PValNormalizerTorch +from alibi_detect.od.base import NotFitException + from sklearn.datasets import make_moons @@ -22,7 +24,7 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): def test_unfitted_knn_single_score(): knn_detector = KNN(k=10) x = np.array([[0, 10], [0.1, 0]]) - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: _ = knn_detector.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -90,7 +92,7 @@ def test_unfitted_knn_ensemble(aggregator, normalizer): normalizer=normalizer() ) x = np.array([[0, 10], [0.1, 0]]) - with pytest.raises(ValueError) as err: + with pytest.raises(NotFitException) as err: _ = knn_detector.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' From bacbfcc4514854892c5b28d6d412ee72b37533be Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 14:50:02 +0000 Subject: [PATCH 044/247] Test pval and shift and scale normalizer output values --- alibi_detect/od/backend/tests/test_ensemble.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/backend/tests/test_ensemble.py index a07f7b6ba..70b0d382e 100644 --- a/alibi_detect/od/backend/tests/test_ensemble.py +++ b/alibi_detect/od/backend/tests/test_ensemble.py @@ -16,6 +16,17 @@ def test_pval_normalizer(): normalizer.fit(x_ref) x_norm = normalizer(x) + + # check that the p-values are correct + assert torch.all(0 < x_norm) + assert torch.all(x_norm < 1) + for i in range(3): + for j in range(10): + comp_pval = ((x_ref[:, j] > x[i][j]).to(torch.float32)).sum() + 1 + comp_pval /= (x_ref.shape[0] + 1) + normalizer_pval = x_norm[i][j].to(torch.float32) + assert torch.isclose(1 - comp_pval, normalizer_pval, atol=1e-4) + normalizer = torch.jit.script(normalizer) x_norm_2 = normalizer(x) assert torch.all(x_norm_2 == x_norm) @@ -23,8 +34,8 @@ def test_pval_normalizer(): def test_shift_and_scale_normalizer(): normalizer = ensemble.ShiftAndScaleNormalizer() - x = torch.randn(3, 10) - x_ref = torch.randn(64, 10) + x = torch.randn(3, 10) * 3 + 2 + x_ref = torch.randn(5000, 10) * 3 + 2 # unfit normalizer raises exception with pytest.raises(NotFitException) as err: normalizer(x) @@ -32,6 +43,9 @@ def test_shift_and_scale_normalizer(): normalizer.fit(x_ref) x_norm = normalizer(x) + assert torch.isclose(x_norm.mean(), torch.tensor(0.), atol=0.1) + assert torch.isclose(x_norm.std(), torch.tensor(1.), atol=0.1) + normalizer = torch.jit.script(normalizer) x_norm_2 = normalizer(x) assert torch.all(x_norm_2 == x_norm) From ea436b098441017728554d75df6603427e6c047f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 15:19:57 +0000 Subject: [PATCH 045/247] Test aggregator output values --- .../od/backend/tests/test_ensemble.py | 20 ++++++++++++++++++- alibi_detect/od/backend/torch/ensemble.py | 7 +++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/backend/tests/test_ensemble.py index 70b0d382e..1a4146fe9 100644 --- a/alibi_detect/od/backend/tests/test_ensemble.py +++ b/alibi_detect/od/backend/tests/test_ensemble.py @@ -55,17 +55,28 @@ def test_average_aggregator(): aggregator = ensemble.AverageAggregator() scores = torch.randn((3, 10)) aggregated_scores = aggregator(scores) + assert torch.all(torch.isclose(aggregated_scores, scores.mean(dim=1))) assert aggregated_scores.shape == (3, ) + aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) def test_weighted_average_aggregator(): - aggregator = ensemble.AverageAggregator(weights=torch.randn((10))) + weights = abs(torch.randn((10))) + + with pytest.raises(ValueError) as err: + aggregator = ensemble.AverageAggregator(weights=weights) + assert err.value.args[0] == 'Weights must sum to 1.' + + weights /= weights.sum() + aggregator = ensemble.AverageAggregator(weights=weights) scores = torch.randn((3, 10)) aggregated_scores = aggregator(scores) + torch.allclose(aggregated_scores, (weights @ scores.T)) assert aggregated_scores.shape == (3, ) + aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -76,6 +87,9 @@ def test_topk_aggregator(): scores = torch.randn((3, 10)) aggregated_scores = aggregator(scores) assert aggregated_scores.shape == (3, ) + scores_sorted, _ = torch.sort(scores) + torch.allclose(scores_sorted[:, -4:].mean(dim=1), aggregated_scores) + aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -86,6 +100,8 @@ def test_max_aggregator(): scores = torch.randn((3, 10)) aggregated_scores = aggregator(scores) assert aggregated_scores.shape == (3, ) + max_vals, _ = scores.max(dim=1) + torch.all(max_vals == aggregated_scores) aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -96,6 +112,8 @@ def test_min_aggregator(): scores = torch.randn((3, 10)) aggregated_scores = aggregator(scores) assert aggregated_scores.shape == (3, ) + min_vals, _ = scores.min(dim=1) + torch.all(min_vals == aggregated_scores) aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 87602b8de..f7c0f7204 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -240,8 +240,15 @@ def __init__(self, weights: Optional[torch.Tensor] = None): weights Optional parameter to weight the scores. If `weights` is left `None` then will be set to a vector of ones. + + Raises + ------ + ValueError + If `weights` does not sum to 1. """ super().__init__() + if weights is not None and weights.sum() != 1: + raise ValueError("Weights must sum to 1.") self.weights = weights def _transform(self, scores: torch.Tensor) -> torch.Tensor: From c3af856b51a1794fc8415f5e7d2cfe864f52e832 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 15:39:33 +0000 Subject: [PATCH 046/247] Remove unneeded NotImplemnentedErrors from ABC abstract methods --- alibi_detect/od/backend/torch/base.py | 13 ++----------- alibi_detect/od/backend/torch/ensemble.py | 18 ++++-------------- alibi_detect/od/base.py | 2 +- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index d3383d633..ba4543c39 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -46,13 +46,8 @@ def _fit(self, x_ref: torch.Tensor) -> None: ---------- x_ref Reference data. - - Raises - ------ - `NotImplementedError` - Raised if not implemented. """ - raise NotImplementedError() + pass @abstractmethod def score(self, x: torch.Tensor) -> torch.Tensor: @@ -63,12 +58,8 @@ def score(self, x: torch.Tensor) -> torch.Tensor: x Data to score. - Raises - ------ - `NotImplementedError` - Raised if not implemented. """ - raise NotImplementedError() + pass @torch.jit.unused def check_threshould_infered(self): diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index f7c0f7204..45aec5fba 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -29,20 +29,14 @@ def _transform(self, x: torch.Tensor): ---------- x numpy array to be transformed - - Raises - ------ - NotImplementedError - if _transform is not implimented on child class raise - NotImplementedError """ - raise NotImplementedError() + pass def forward(self, x: torch.Tensor): return self.transform(x=x) -class FitMixinTorch: +class FitMixinTorch(ABC): _fitted = False def __init__(self): @@ -57,6 +51,7 @@ def fit(self, x: torch.Tensor) -> FitMixinTorch: self._fit(x) return self + @abstractmethod def _fit(self, x: torch.Tensor): """Fit on `x` tensor. @@ -66,13 +61,8 @@ def _fit(self, x: torch.Tensor): ---------- x Reference `torch.Tensor` for fitting object. - - Raises - ------ - NotImplementedError - Raised if unimplimented. """ - raise NotImplementedError() + pass @torch.jit.unused def check_fitted(self): diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 3cb739fc5..89f4d6e57 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -111,5 +111,5 @@ class NotFitException(Exception): class ThresholdNotInferredException(Exception): - """Exception raised when a transform is not fitted.""" + """Exception raised when a threshold not inferred for an outlier detector.""" pass From 351e5a70c737b6c8455792a36488331b8d93d060 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 15:59:14 +0000 Subject: [PATCH 047/247] Fix typos --- alibi_detect/od/backend/torch/base.py | 2 +- alibi_detect/od/backend/torch/ensemble.py | 22 +++++++++++----------- alibi_detect/od/backend/torch/knn.py | 8 ++++---- alibi_detect/od/base.py | 4 ++-- alibi_detect/od/knn.py | 8 ++++---- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index ba4543c39..c381da369 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -89,7 +89,7 @@ def _to_tensor(self, x: Union[List, np.ndarray]): return torch.as_tensor(x, dtype=torch.float32, device=self.device) def _to_numpy(self, x: torch.Tensor): - """Converts the data to numpy. + """Converts the data to `numpy.ndarray`. Parameters ---------- diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/backend/torch/ensemble.py index 45aec5fba..880893cbd 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/backend/torch/ensemble.py @@ -192,7 +192,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class TopKAggregator(BaseTransformTorch): def __init__(self, k: Optional[int] = None): - """Takes the mean of the top k scores. + """Takes the mean of the top `k` scores. Parameters ---------- @@ -204,7 +204,7 @@ def __init__(self, k: Optional[int] = None): self.k = k def _transform(self, scores: torch.Tensor) -> torch.Tensor: - """Takes the mean of the top k scores. + """Takes the mean of the top `k` scores. Parameters ---------- @@ -213,7 +213,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: Returns ------- - `Torch.Tensor` of mean of top k scores. + `Torch.Tensor` of mean of top `k` scores. """ if self.k is None: self.k = int(np.ceil(scores.shape[1]/2)) @@ -234,7 +234,7 @@ def __init__(self, weights: Optional[torch.Tensor] = None): Raises ------ ValueError - If `weights` does not sum to 1. + If `weights` does not sum to `1`. """ super().__init__() if weights is not None and weights.sum() != 1: @@ -242,8 +242,8 @@ def __init__(self, weights: Optional[torch.Tensor] = None): self.weights = weights def _transform(self, scores: torch.Tensor) -> torch.Tensor: - """Averages the scores of the detectors in an ensemble. If weights where passed in the init - then these are used to weight the scores.Anomaly + """Averages the scores of the detectors in an ensemble. If weights where passed in the `__init__` + then these are used to weight the scores. ---------- scores `Torch.Tensor` of scores from ensemble of detectors. @@ -264,7 +264,7 @@ def __init__(self): super().__init__() def _transform(self, scores: torch.Tensor) -> torch.Tensor: - """Takes the max score of a set of detectors in an ensemble. + """Takes the maximum score of a set of detectors in an ensemble. Parameters ---------- @@ -273,7 +273,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: Returns ------- - `Torch.Tensor` of max of scores. + `Torch.Tensor` of maximum scores. """ vals, _ = torch.max(scores, dim=-1) return vals @@ -281,11 +281,11 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class MinAggregator(BaseTransformTorch): def __init__(self): - """Takes the min score of a set of detectors in an ensemble.""" + """Takes the minimum score of a set of detectors in an ensemble.""" super().__init__() def _transform(self, scores: torch.Tensor) -> torch.Tensor: - """Takes the min score of a set of detectors in an ensemble. + """Takes the minimum score of a set of detectors in an ensemble. Parameters ---------- @@ -294,7 +294,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: Returns ------- - `Torch.Tensor` of min of scores. + `Torch.Tensor` of minimum scores. """ vals, _ = torch.min(scores, dim=-1) return vals diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index e9c62beeb..7fd821b16 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -1,4 +1,4 @@ -from typing import Optional, Union, List +from typing import Optional, Union, List, Tuple import numpy as np import torch @@ -10,7 +10,7 @@ class KNNTorch(TorchOutlierDetector): def __init__( self, - k: Union[np.ndarray, List], + k: Union[np.ndarray, List, Tuple], kernel: Optional[torch.nn.Module] = None, accumulator: Optional[Accumulator] = None, device: Optional[Union[str, torch.device]] = None @@ -42,7 +42,7 @@ def __init__( self.accumulator = accumulator def forward(self, x: torch.Tensor) -> torch.Tensor: - """Detect if x is an outlier. + """Detect if `x` is an outlier. Parameters ---------- @@ -56,7 +56,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Raises ------ ValueError - If called before detector has had threshould_infered method called. + If called before detector has had `infer_threshold` method called. """ raw_scores = self.score(x) scores = self._accumulator(raw_scores) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 89f4d6e57..455f1444c 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -32,7 +32,7 @@ def fit(self, x: np.ndarray) -> None: @abstractmethod def score(self, x: np.ndarray) -> np.ndarray: """ - Compute outlier scores of the instances in x. + Compute outlier scores of the instances in `x`. Parameters ---------- @@ -62,7 +62,7 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: @abstractmethod def predict(self, x: np.ndarray) -> Dict[str, Any]: """ - Predict whether the instances in x are outliers or not. + Predict whether the instances in `x` are outliers or not. Parameters ---------- diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 08072ed82..0d86c360c 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -79,17 +79,17 @@ def __init__( self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator, device=device) def fit(self, x_ref: Union[np.ndarray, List]) -> None: - """Fit the kNN detector on reference data. + """Fit the detector on reference data. Parameters ---------- x_ref - Reference data used to fit the kNN detector. + Reference data used to fit the detector. """ self.backend.fit(self.backend._to_tensor(x_ref)) def score(self, x: Union[np.ndarray, List]) -> np.ndarray: - """Score x instances using the kNN. + """Score `x` instances using the detector. Parameters ---------- @@ -120,7 +120,7 @@ def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) def predict(self, x: Union[np.ndarray, List]) -> Dict[str, Any]: - """Predict whether the instances in x are outliers or not. + """Predict whether the instances in `x` are outliers or not. Parameters ---------- From a190cc3171c6b52d9cbf7b80dadaca05857abe0c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 16:00:59 +0000 Subject: [PATCH 048/247] Fix method typo --- alibi_detect/od/backend/torch/base.py | 2 +- alibi_detect/od/backend/torch/knn.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index c381da369..7896cdf39 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -62,7 +62,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: pass @torch.jit.unused - def check_threshould_infered(self): + def check_threshold_infered(self): """Check if threshold is inferred. Raises diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 7fd821b16..188980640 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -61,7 +61,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: raw_scores = self.score(x) scores = self._accumulator(raw_scores) if not torch.jit.is_scripting(): - self.check_threshould_infered() + self.check_threshold_infered() preds = scores > self.threshold return preds.cpu() From 0daa98fee9cf77fbc9105735699206aac7836bca Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 16:05:40 +0000 Subject: [PATCH 049/247] Fix docstrings for KNNTorch --- alibi_detect/od/backend/torch/knn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/backend/torch/knn.py index 188980640..5610f41ac 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/backend/torch/knn.py @@ -55,7 +55,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Raises ------ - ValueError + ThresholdNotInferredException If called before detector has had `infer_threshold` method called. """ raw_scores = self.score(x) @@ -71,7 +71,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: Parameters ---------- x - Score a tensor of instances. First dimesnion corresponds to batch. + The tensor of instances. First dimension corresponds to batch. Returns ------- @@ -79,7 +79,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: Raises ------ - ValueError + NotFitException If called before detector has been fit. """ if not torch.jit.is_scripting(): From 8d2fa3beeebdf9858e9f360c6a2d67c2395ac13a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 17:04:49 +0000 Subject: [PATCH 050/247] Set api signatures to accept np.ndarray and not List types --- alibi_detect/od/base.py | 4 ++-- alibi_detect/od/knn.py | 15 +++++++-------- alibi_detect/utils/_types.py | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 455f1444c..8130d84b4 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Dict, Any, Union -from typing_extensions import Protocol, runtime_checkable +from alibi_detect.utils._types import Protocol, runtime_checkable import numpy as np from alibi_detect.base import BaseDetector @@ -41,7 +41,7 @@ def score(self, x: np.ndarray) -> np.ndarray: Returns ------- - Outlier scores. The higher the score, the more anomalous the instance. + Outlier scores. The higher the score, the more outlying the instance. """ pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 0d86c360c..3d1daef54 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,6 +1,5 @@ -from typing import Callable, Union, Optional, List, Dict, Any - -from typing_extensions import Literal +from typing import Callable, Union, Optional, Dict, Any, List, Tuple +from alibi_detect.utils._types import Literal import numpy as np from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols @@ -21,7 +20,7 @@ class KNN(OutlierDetector): def __init__( self, - k: Union[int, np.ndarray], + k: Union[int, np.ndarray, List[int], Tuple[int]], kernel: Optional[Callable] = None, normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizerTorch', aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregatorTorch', @@ -78,7 +77,7 @@ def __init__( ) self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator, device=device) - def fit(self, x_ref: Union[np.ndarray, List]) -> None: + def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. Parameters @@ -88,7 +87,7 @@ def fit(self, x_ref: Union[np.ndarray, List]) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) - def score(self, x: Union[np.ndarray, List]) -> np.ndarray: + def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. Parameters @@ -104,7 +103,7 @@ def score(self, x: Union[np.ndarray, List]) -> np.ndarray: score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) - def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: + def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. The threshold is inferred using the reference data and the false positive rate. The threshold is used to determine the outlier labels in the predict method. @@ -119,7 +118,7 @@ def infer_threshold(self, x_ref: Union[np.ndarray, List], fpr: float) -> None: """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) - def predict(self, x: Union[np.ndarray, List]) -> Dict[str, Any]: + def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. Parameters diff --git a/alibi_detect/utils/_types.py b/alibi_detect/utils/_types.py index 0348521f5..d751017ca 100644 --- a/alibi_detect/utils/_types.py +++ b/alibi_detect/utils/_types.py @@ -7,9 +7,9 @@ # Literal for typing if sys.version_info >= (3, 8): - from typing import Literal # noqa + from typing import Literal, Protocol, runtime_checkable # noqa else: - from typing_extensions import Literal # noqa + from typing_extensions import Literal, Protocol, runtime_checkable # noqa # Optional dep dependent tuples of types From af8ab43109c315b60babcbcb43d6acb5294d572c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 4 Jan 2023 17:31:50 +0000 Subject: [PATCH 051/247] Fix mypy error --- alibi_detect/od/backend/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index 1948df1b9..f3bcd46c5 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -1,5 +1,5 @@ from alibi_detect.utils.missing_optional_dependency import import_optional -from typing import Literal +from alibi_detect.utils._types import Literal KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) PValNormalizerTorch, ShiftAndScaleNormalizerTorch, TopKAggregatorTorch, AverageAggregatorTorch, \ From 89c48158e1a6b14f7d31e19a506fffb6b18995b9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 5 Jan 2023 09:15:45 +0000 Subject: [PATCH 052/247] Move to numpy logic from OutlierDetectorOutput dataclass to base class --- alibi_detect/od/backend/torch/base.py | 21 ++++++++++----------- alibi_detect/od/knn.py | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 7896cdf39..8f7910a1f 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import List, Dict, Union, Optional +from typing import List, Union, Optional from dataclasses import dataclass, asdict from abc import ABC, abstractmethod @@ -20,14 +20,6 @@ class TorchOutlierDetectorOutput: preds: Optional[torch.Tensor] p_vals: Optional[torch.Tensor] - def _to_numpy(self) -> Dict[str, Union[bool, Optional[torch.Tensor]]]: - """Converts the output to numpy.""" - outputs = asdict(self) - for key, value in outputs.items(): - if isinstance(value, torch.Tensor): - outputs[key] = value.cpu().detach().numpy() - return outputs - class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): """Base class for torch backend outlier detection algorithms.""" @@ -88,7 +80,7 @@ def _to_tensor(self, x: Union[List, np.ndarray]): """ return torch.as_tensor(x, dtype=torch.float32, device=self.device) - def _to_numpy(self, x: torch.Tensor): + def _to_numpy(self, x: Union[torch.Tensor, TorchOutlierDetectorOutput]): """Converts the data to `numpy.ndarray`. Parameters @@ -100,7 +92,14 @@ def _to_numpy(self, x: torch.Tensor): ------- `np.ndarray` """ - return x.cpu().detach().numpy() + if isinstance(x, torch.Tensor): + return x.cpu().detach().numpy() + elif isinstance(x, TorchOutlierDetectorOutput): + outputs = asdict(x) + for key, value in outputs.items(): + if isinstance(value, torch.Tensor): + outputs[key] = value.cpu().detach().numpy() + return outputs def _accumulator(self, x: torch.Tensor) -> torch.Tensor: """Accumulates the data. diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 3d1daef54..3cdd1bb3b 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -135,7 +135,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: """ outputs = self.backend.predict(self.backend._to_tensor(x)) output: Dict[str, Any] = { - 'data': outputs._to_numpy(), + 'data': self.backend._to_numpy(outputs), 'meta': self.meta } return output From c909f4c9bd79a9adecd876ae0ff3e9b3c55cc412 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 5 Jan 2023 09:16:44 +0000 Subject: [PATCH 053/247] Refactor init knn logic --- alibi_detect/od/knn.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 3cdd1bb3b..f147aaeee 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -64,19 +64,28 @@ def __init__( backend_cls, accumulator_cls = backends[backend] accumulator = None + + if aggregator is None and isinstance(k, (list, np.ndarray, tuple)): + raise ValueError("If `k` is an array, an aggregator is required.") + if isinstance(k, (list, np.ndarray, tuple)): - if isinstance(aggregator, str): - aggregator = getattr(backend_objs, aggregator)() - if aggregator is None: - raise ValueError("If `k` is an array, an aggregator is required.") - if isinstance(normalizer, str): - normalizer = getattr(backend_objs, normalizer)() accumulator = accumulator_cls( - normalizer=normalizer, - aggregator=aggregator + normalizer=self._make_normalizer(normalizer), + aggregator=self._make_aggregator(aggregator) ) + self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator, device=device) + def _make_aggregator(self, aggregator: Union[TransformProtocol, aggregator_literals]) -> TransformProtocol: + if isinstance(aggregator, str): + aggregator = getattr(backend_objs, aggregator)() + return aggregator + + def _make_normalizer(self, normalizer: Union[transform_protocols, normalizer_literals]) -> transform_protocols: + if isinstance(normalizer, str): + normalizer = getattr(backend_objs, normalizer)() + return normalizer + def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. From bc7a771d2b2856aeb1d33f1a96e01e2b0e19d1b9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 5 Jan 2023 09:45:49 +0000 Subject: [PATCH 054/247] Refator str to aggregator and normalizer methods to backend --- alibi_detect/od/backend/__init__.py | 29 +++++++++++++++++++++++++++++ alibi_detect/od/knn.py | 22 ++++++---------------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index f3bcd46c5..1aea33b7f 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -1,5 +1,8 @@ from alibi_detect.utils.missing_optional_dependency import import_optional from alibi_detect.utils._types import Literal +from alibi_detect.od.base import TransformProtocol, transform_protocols +from typing import Union + KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) PValNormalizerTorch, ShiftAndScaleNormalizerTorch, TopKAggregatorTorch, AverageAggregatorTorch, \ @@ -12,3 +15,29 @@ normalizer_literals = Literal['PValNormalizerTorch', 'ShiftAndScaleNormalizerTorch'] aggregator_literals = Literal['TopKAggregatorTorch', 'AverageAggregatorTorch', 'MaxAggregatorTorch', 'MinAggregatorTorch'] + + +def get_normalizer(normalizer: Union[transform_protocols, normalizer_literals]) -> TransformProtocol: + if isinstance(normalizer, str): + try: + return { + 'PValNormalizerTorch': PValNormalizerTorch, + 'ShiftAndScaleNormalizerTorch': ShiftAndScaleNormalizerTorch, + }.get(normalizer, )() + except KeyError: + raise NotImplementedError(f'Normalizer {normalizer} not implemented.') + return normalizer + + +def get_aggregator(aggregator: Union[TransformProtocol, aggregator_literals]) -> TransformProtocol: + if isinstance(aggregator, str): + try: + return { + 'TopKAggregatorTorch': TopKAggregatorTorch, + 'AverageAggregatorTorch': AverageAggregatorTorch, + 'MaxAggregatorTorch': MaxAggregatorTorch, + 'MinAggregatorTorch': MinAggregatorTorch, + }.get(aggregator, )() + except KeyError: + raise NotImplementedError(f'Aggregator {aggregator} not implemented.') + return aggregator diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index f147aaeee..1669362fa 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -3,8 +3,8 @@ import numpy as np from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols -from alibi_detect.od import backend as backend_objs -from alibi_detect.od.backend import normalizer_literals, aggregator_literals, KNNTorch, AccumulatorTorch +from alibi_detect.od.backend import normalizer_literals, aggregator_literals, KNNTorch, AccumulatorTorch, \ + get_aggregator, get_normalizer from alibi_detect.utils.frameworks import BackendValidator from typing import TYPE_CHECKING @@ -33,8 +33,8 @@ def __init__( Parameters ---------- k - Number of nearest neighbours to use for outlier detection. If an array is passed, an aggregator is required - to aggregate the scores. + Number of nearest neighboursreveal_type(normalizer) to use for outlier detection. If an array is passed, an + aggregator is required to aggregate the scores. kernel Kernel function to use for outlier detection. If None, `torch.cdist` is used. normalizer @@ -70,22 +70,12 @@ def __init__( if isinstance(k, (list, np.ndarray, tuple)): accumulator = accumulator_cls( - normalizer=self._make_normalizer(normalizer), - aggregator=self._make_aggregator(aggregator) + normalizer=get_normalizer(normalizer), + aggregator=get_aggregator(aggregator) ) self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator, device=device) - def _make_aggregator(self, aggregator: Union[TransformProtocol, aggregator_literals]) -> TransformProtocol: - if isinstance(aggregator, str): - aggregator = getattr(backend_objs, aggregator)() - return aggregator - - def _make_normalizer(self, normalizer: Union[transform_protocols, normalizer_literals]) -> transform_protocols: - if isinstance(normalizer, str): - normalizer = getattr(backend_objs, normalizer)() - return normalizer - def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. From dfd613d1f93cb38dd939718d6b309feafadeff38 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 6 Jan 2023 14:59:33 +0000 Subject: [PATCH 055/247] Align kNN output with other outlier detectors --- alibi_detect/base.py | 3 +- alibi_detect/od/backend/__init__.py | 4 +- .../od/backend/tests/test_knn_backend.py | 18 ++++---- alibi_detect/od/backend/torch/base.py | 13 +++--- alibi_detect/od/knn.py | 22 +++++++--- alibi_detect/od/tests/test_knn.py | 42 +++++++++---------- 6 files changed, 57 insertions(+), 45 deletions(-) diff --git a/alibi_detect/base.py b/alibi_detect/base.py index adc896dfa..c599a2a8d 100644 --- a/alibi_detect/base.py +++ b/alibi_detect/base.py @@ -154,8 +154,7 @@ def from_config(cls, config: dict): return detector def _set_config(self, inputs): # TODO - move to BaseDetector once config save/load implemented for non-drift - """ - Set a detectors `config` attribute upon detector instantiation. + """ Large artefacts are overwritten with `None` in order to avoid memory duplication. They're added back into the config later on by `get_config()`. diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py index 1aea33b7f..7fb3d5e96 100644 --- a/alibi_detect/od/backend/__init__.py +++ b/alibi_detect/od/backend/__init__.py @@ -23,7 +23,7 @@ def get_normalizer(normalizer: Union[transform_protocols, normalizer_literals]) return { 'PValNormalizerTorch': PValNormalizerTorch, 'ShiftAndScaleNormalizerTorch': ShiftAndScaleNormalizerTorch, - }.get(normalizer, )() + }.get(normalizer)() except KeyError: raise NotImplementedError(f'Normalizer {normalizer} not implemented.') return normalizer @@ -37,7 +37,7 @@ def get_aggregator(aggregator: Union[TransformProtocol, aggregator_literals]) -> 'AverageAggregatorTorch': AverageAggregatorTorch, 'MaxAggregatorTorch': MaxAggregatorTorch, 'MinAggregatorTorch': MinAggregatorTorch, - }.get(aggregator, )() + }.get(aggregator)() except KeyError: raise NotImplementedError(f'Aggregator {aggregator} not implemented.') return aggregator diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/backend/tests/test_knn_backend.py index 0acbe6d35..23048be39 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/backend/tests/test_knn_backend.py @@ -21,15 +21,15 @@ def test_knn_torch_backend(): x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) outputs = knn_torch.predict(x) - assert outputs.scores.shape == (3, ) - assert outputs.preds is None - assert outputs.p_vals is None + assert outputs.instance_score.shape == (3, ) + assert outputs.is_outlier is None + assert outputs.p_value is None scores = knn_torch.score(x) - assert torch.all(scores == outputs.scores) + assert torch.all(scores == outputs.instance_score) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs.preds == torch.tensor([False, False, True])) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) @@ -39,11 +39,11 @@ def test_knn_torch_backend_ensemble(accumulator): knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) result = knn_torch.predict(x) - assert result.scores.shape == (3, ) + assert result.instance_score.shape == (3, ) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs.preds == torch.tensor([False, False, True])) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) @@ -97,11 +97,11 @@ def test_knn_kernel(accumulator): knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) result = knn_torch.predict(x) - assert result.scores.shape == (3,) + assert result.instance_score.shape == (3,) knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) - assert torch.all(outputs.preds == torch.tensor([False, False, True])) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) """Can't convert GaussianRBF to torchscript due to torchscript type diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/backend/torch/base.py index 8f7910a1f..87c9f7de9 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/backend/torch/base.py @@ -15,10 +15,10 @@ class TorchOutlierDetectorOutput: """Output of the outlier detector.""" threshold_inferred: bool - scores: torch.Tensor + instance_score: torch.Tensor threshold: Optional[torch.Tensor] - preds: Optional[torch.Tensor] - p_vals: Optional[torch.Tensor] + is_outlier: Optional[torch.Tensor] + p_value: Optional[torch.Tensor] class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): @@ -194,10 +194,11 @@ def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: self.check_fitted() # type: ignore raw_scores = self.score(x) scores = self._accumulator(raw_scores) + return TorchOutlierDetectorOutput( - scores=scores, - preds=self._classify_outlier(scores), - p_vals=self._p_vals(scores), + instance_score=scores, + is_outlier=self._classify_outlier(scores), + p_value=self._p_vals(scores), threshold_inferred=self.threshold_inferred, threshold=self.threshold ) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 1669362fa..40328af12 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -1,12 +1,16 @@ from typing import Callable, Union, Optional, Dict, Any, List, Tuple -from alibi_detect.utils._types import Literal +from typing import TYPE_CHECKING + import numpy as np +from alibi_detect.utils._types import Literal +from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols from alibi_detect.od.backend import normalizer_literals, aggregator_literals, KNNTorch, AccumulatorTorch, \ get_aggregator, get_normalizer from alibi_detect.utils.frameworks import BackendValidator -from typing import TYPE_CHECKING +from alibi_detect.version import __version__ + if TYPE_CHECKING: import torch @@ -133,8 +137,16 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: the detector. """ outputs = self.backend.predict(self.backend._to_tensor(x)) - output: Dict[str, Any] = { - 'data': self.backend._to_numpy(outputs), - 'meta': self.meta + output = outlier_prediction_dict() + output['data'] = { + **output['data'], + **self.backend._to_numpy(outputs) + } + output['meta'] = { + **output['meta'], + 'name': self.__class__.__name__, + 'detector_type': 'outlier', + 'online': False, + 'version': __version__, } return output diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn.py index e7f9a4150..77a9d144e 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn.py @@ -36,13 +36,13 @@ def test_fitted_knn_single_score(): x = np.array([[0, 10], [0.1, 0]]) y = knn_detector.predict(x) y = y['data'] - assert y['scores'][0] > 5 - assert y['scores'][1] < 1 + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 assert not y['threshold_inferred'] assert y['threshold'] is None - assert y['preds'] is None - assert y['p_vals'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None def test_default_knn_ensemble_init(): @@ -52,13 +52,13 @@ def test_default_knn_ensemble_init(): x = np.array([[0, 10], [0.1, 0]]) y = knn_detector.predict(x) y = y['data'] - assert y['scores'][0] > 5 - assert y['scores'][1] < 1 + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 assert not y['threshold_inferred'] assert y['threshold'] is None - assert y['preds'] is None - assert y['p_vals'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None def test_incorrect_knn_ensemble_init(): @@ -74,12 +74,12 @@ def test_fitted_knn_predict(): x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) y = y['data'] - assert y['scores'][0] > 5 - assert y['scores'][1] < 1 + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 assert y['threshold_inferred'] assert y['threshold'] is not None - assert y['p_vals'].all() - assert (y['preds'] == [True, False]).all() + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), @@ -111,11 +111,11 @@ def test_fitted_knn_ensemble(aggregator, normalizer): x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) y = y['data'] - assert y['scores'].all() + assert y['instance_score'].all() assert not y['threshold_inferred'] assert y['threshold'] is None - assert y['preds'] is None - assert y['p_vals'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), @@ -132,8 +132,8 @@ def test_fitted_knn_ensemble_predict(aggregator, normalizer): y = y['data'] assert y['threshold_inferred'] assert y['threshold'] is not None - assert y['p_vals'].all() - assert (y['preds'] == [True, False]).all() + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() @pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), @@ -172,12 +172,12 @@ def test_knn_ensemble_integration(aggregator, normalizer): knn_detector.fit(X_ref) knn_detector.infer_threshold(X_ref, 0.1) result = knn_detector.predict(x_inlier) - result = result['data']['preds'][0] + result = result['data']['is_outlier'][0] assert not result x_outlier = np.array([[-1, 1.5]]) result = knn_detector.predict(x_outlier) - result = result['data']['preds'][0] + result = result['data']['is_outlier'][0] assert result tsknn = torch.jit.script(knn_detector.backend) @@ -193,12 +193,12 @@ def test_knn_integration(): knn_detector.fit(X_ref) knn_detector.infer_threshold(X_ref, 0.1) result = knn_detector.predict(x_inlier) - result = result['data']['preds'][0] + result = result['data']['is_outlier'][0] assert not result x_outlier = np.array([[-1, 1.5]]) result = knn_detector.predict(x_outlier) - result = result['data']['preds'][0] + result = result['data']['is_outlier'][0] assert result tsknn = torch.jit.script(knn_detector.backend) From 3fb167bf234c3452b9b3243ae71cb2b89eb554ad Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 6 Jan 2023 15:35:11 +0000 Subject: [PATCH 056/247] Refactor backend.pytorch into pytorch module --- alibi_detect/od/__init__.py | 49 +++++++++++++++++++ alibi_detect/od/backend/__init__.py | 43 ---------------- alibi_detect/od/backend/torch/__init__.py | 0 alibi_detect/od/knn.py | 10 ++-- alibi_detect/od/pytorch/__init__.py | 4 ++ .../od/{backend/torch => pytorch}/base.py | 2 +- .../od/{backend/torch => pytorch}/ensemble.py | 4 +- .../od/{backend/torch => pytorch}/knn.py | 6 +-- .../od/{backend => }/tests/test_ensemble.py | 2 +- .../od/tests/{ => test_knn}/test_knn.py | 40 +++++++-------- .../test_knn}/test_knn_backend.py | 4 +- 11 files changed, 87 insertions(+), 77 deletions(-) delete mode 100644 alibi_detect/od/backend/__init__.py delete mode 100644 alibi_detect/od/backend/torch/__init__.py create mode 100644 alibi_detect/od/pytorch/__init__.py rename alibi_detect/od/{backend/torch => pytorch}/base.py (98%) rename alibi_detect/od/{backend/torch => pytorch}/ensemble.py (97%) rename alibi_detect/od/{backend/torch => pytorch}/knn.py (93%) rename alibi_detect/od/{backend => }/tests/test_ensemble.py (98%) rename alibi_detect/od/tests/{ => test_knn}/test_knn.py (75%) rename alibi_detect/od/{backend/tests => tests/test_knn}/test_knn_backend.py (97%) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index a0fb048e1..c4e6edead 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -4,6 +4,49 @@ from .mahalanobis import Mahalanobis from .sr import SpectralResidual +from alibi_detect.od.base import TransformProtocol, transform_protocols +from alibi_detect.utils._types import Literal +from typing import Union + +PValNormalizer, ShiftAndScaleNormalizer, TopKAggregator, AverageAggregator, \ + MaxAggregator, MinAggregator = import_optional( + 'alibi_detect.od.pytorch.ensemble', + ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', + 'AverageAggregator', 'MaxAggregator', 'MinAggregator'] + ) + + +normalizer_literals = Literal['PValNormalizer', 'ShiftAndScaleNormalizer'] +aggregator_literals = Literal['TopKAggregator', 'AverageAggregator', + 'MaxAggregator', 'MinAggregator'] + + +def get_normalizer(normalizer: Union[transform_protocols, normalizer_literals]) -> TransformProtocol: + if isinstance(normalizer, str): + try: + return { + 'PValNormalizer': PValNormalizer, + 'ShiftAndScaleNormalizer': ShiftAndScaleNormalizer, + }.get(normalizer)() + except KeyError: + raise NotImplementedError(f'Normalizer {normalizer} not implemented.') + return normalizer + + +def get_aggregator(aggregator: Union[TransformProtocol, aggregator_literals]) -> TransformProtocol: + if isinstance(aggregator, str): + try: + return { + 'TopKAggregator': TopKAggregator, + 'AverageAggregator': AverageAggregator, + 'MaxAggregator': MaxAggregator, + 'MinAggregator': MinAggregator, + }.get(aggregator)() + except KeyError: + raise NotImplementedError(f'Aggregator {aggregator} not implemented.') + return aggregator + + OutlierAEGMM = import_optional('alibi_detect.od.aegmm', names=['OutlierAEGMM']) OutlierAE = import_optional('alibi_detect.od.ae', names=['OutlierAE']) OutlierVAE = import_optional('alibi_detect.od.vae', names=['OutlierVAE']) @@ -25,4 +68,10 @@ "LLR", "OutlierProphet" "KNN" + "PValNormalizer", + "ShiftAndScaleNormalizer", + "TopKAggregator", + "AverageAggregator", + "MaxAggregator", + "MinAggregator", ] diff --git a/alibi_detect/od/backend/__init__.py b/alibi_detect/od/backend/__init__.py deleted file mode 100644 index 7fb3d5e96..000000000 --- a/alibi_detect/od/backend/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -from alibi_detect.utils.missing_optional_dependency import import_optional -from alibi_detect.utils._types import Literal -from alibi_detect.od.base import TransformProtocol, transform_protocols -from typing import Union - - -KNNTorch = import_optional('alibi_detect.od.backend.torch.knn', ['KNNTorch']) -PValNormalizerTorch, ShiftAndScaleNormalizerTorch, TopKAggregatorTorch, AverageAggregatorTorch, \ - MaxAggregatorTorch, MinAggregatorTorch, AccumulatorTorch = import_optional( - 'alibi_detect.od.backend.torch.ensemble', - ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', - 'AverageAggregator', 'MaxAggregator', 'MinAggregator', 'Accumulator'] - ) - -normalizer_literals = Literal['PValNormalizerTorch', 'ShiftAndScaleNormalizerTorch'] -aggregator_literals = Literal['TopKAggregatorTorch', 'AverageAggregatorTorch', - 'MaxAggregatorTorch', 'MinAggregatorTorch'] - - -def get_normalizer(normalizer: Union[transform_protocols, normalizer_literals]) -> TransformProtocol: - if isinstance(normalizer, str): - try: - return { - 'PValNormalizerTorch': PValNormalizerTorch, - 'ShiftAndScaleNormalizerTorch': ShiftAndScaleNormalizerTorch, - }.get(normalizer)() - except KeyError: - raise NotImplementedError(f'Normalizer {normalizer} not implemented.') - return normalizer - - -def get_aggregator(aggregator: Union[TransformProtocol, aggregator_literals]) -> TransformProtocol: - if isinstance(aggregator, str): - try: - return { - 'TopKAggregatorTorch': TopKAggregatorTorch, - 'AverageAggregatorTorch': AverageAggregatorTorch, - 'MaxAggregatorTorch': MaxAggregatorTorch, - 'MinAggregatorTorch': MinAggregatorTorch, - }.get(aggregator)() - except KeyError: - raise NotImplementedError(f'Aggregator {aggregator} not implemented.') - return aggregator diff --git a/alibi_detect/od/backend/torch/__init__.py b/alibi_detect/od/backend/torch/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 40328af12..dfa6819a9 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -6,8 +6,8 @@ from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols -from alibi_detect.od.backend import normalizer_literals, aggregator_literals, KNNTorch, AccumulatorTorch, \ - get_aggregator, get_normalizer +from alibi_detect.od.pytorch import KNNTorch, Accumulator +from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -17,7 +17,7 @@ backends = { - 'pytorch': (KNNTorch, AccumulatorTorch) + 'pytorch': (KNNTorch, Accumulator) } @@ -26,8 +26,8 @@ def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], kernel: Optional[Callable] = None, - normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizerTorch', - aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregatorTorch', + normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizer', + aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregator', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', ) -> None: diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py new file mode 100644 index 000000000..c90fa0835 --- /dev/null +++ b/alibi_detect/od/pytorch/__init__.py @@ -0,0 +1,4 @@ +from alibi_detect.utils.missing_optional_dependency import import_optional + +KNNTorch = import_optional('alibi_detect.od.pytorch.knn', ['KNNTorch']) +Accumulator = import_optional('alibi_detect.od.pytorch.ensemble', ['Accumulator']) diff --git a/alibi_detect/od/backend/torch/base.py b/alibi_detect/od/pytorch/base.py similarity index 98% rename from alibi_detect/od/backend/torch/base.py rename to alibi_detect/od/pytorch/base.py index 87c9f7de9..4574b2246 100644 --- a/alibi_detect/od/backend/torch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -6,7 +6,7 @@ import numpy as np import torch -from alibi_detect.od.backend.torch.ensemble import FitMixinTorch +from alibi_detect.od.pytorch.ensemble import FitMixinTorch from alibi_detect.utils.pytorch.misc import get_device from alibi_detect.od.base import ThresholdNotInferredException diff --git a/alibi_detect/od/backend/torch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py similarity index 97% rename from alibi_detect/od/backend/torch/ensemble.py rename to alibi_detect/od/pytorch/ensemble.py index 880893cbd..c80fd3527 100644 --- a/alibi_detect/od/backend/torch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -108,7 +108,7 @@ class PValNormalizer(BaseFittedTransformTorch): def __init__(self): """Maps scores to there p values. - Needs to be fit (see :py:obj:`~alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch`). + Needs to be fit (see :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch`). Returns the proportion of scores in the reference dataset that are greater than the score of interest. Output is between 1 and 0. Small values are likely to be outliers. """ @@ -152,7 +152,7 @@ class ShiftAndScaleNormalizer(BaseFittedTransformTorch): def __init__(self): """Maps scores to their normalised values. - Needs to be fit (see :py:obj:`~alibi_detect.od.backend.torch.ensemble.BaseFittedTransformTorch`). + Needs to be fit (see :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch`). Subtracts the dataset mean and scales by the standard deviation. """ super().__init__() diff --git a/alibi_detect/od/backend/torch/knn.py b/alibi_detect/od/pytorch/knn.py similarity index 93% rename from alibi_detect/od/backend/torch/knn.py rename to alibi_detect/od/pytorch/knn.py index 5610f41ac..d26446970 100644 --- a/alibi_detect/od/backend/torch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -3,8 +3,8 @@ import numpy as np import torch -from alibi_detect.od.backend.torch.ensemble import Accumulator -from alibi_detect.od.backend.torch.base import TorchOutlierDetector +from alibi_detect.od.pytorch.ensemble import Accumulator +from alibi_detect.od.pytorch.base import TorchOutlierDetector class KNNTorch(TorchOutlierDetector): @@ -29,7 +29,7 @@ def __init__( while computing the k nearest neighbor distance. accumulator If `k` is an array of integers then the accumulator must not be None. Should be an instance - of :py:obj:`alibi_detect.od.backend.torch.ensemble.Accumulator`. Responsible for combining + of :py:obj:`alibi_detect.od.pytorch.ensemble.Accumulator`. Responsible for combining multiple scores into a single score. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. diff --git a/alibi_detect/od/backend/tests/test_ensemble.py b/alibi_detect/od/tests/test_ensemble.py similarity index 98% rename from alibi_detect/od/backend/tests/test_ensemble.py rename to alibi_detect/od/tests/test_ensemble.py index 1a4146fe9..52cc15cd3 100644 --- a/alibi_detect/od/backend/tests/test_ensemble.py +++ b/alibi_detect/od/tests/test_ensemble.py @@ -1,7 +1,7 @@ import pytest import torch -from alibi_detect.od.backend.torch import ensemble +from alibi_detect.od.pytorch import ensemble from alibi_detect.od.base import NotFitException diff --git a/alibi_detect/od/tests/test_knn.py b/alibi_detect/od/tests/test_knn/test_knn.py similarity index 75% rename from alibi_detect/od/tests/test_knn.py rename to alibi_detect/od/tests/test_knn/test_knn.py index 77a9d144e..dd005c7ed 100644 --- a/alibi_detect/od/tests/test_knn.py +++ b/alibi_detect/od/tests/test_knn/test_knn.py @@ -3,8 +3,8 @@ import torch from alibi_detect.od.knn import KNN -from alibi_detect.od.backend import AverageAggregatorTorch, TopKAggregatorTorch, MaxAggregatorTorch, \ - MinAggregatorTorch, ShiftAndScaleNormalizerTorch, PValNormalizerTorch +from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ + MinAggregator, ShiftAndScaleNormalizer, PValNormalizer from alibi_detect.od.base import NotFitException from sklearn.datasets import make_moons @@ -82,9 +82,9 @@ def test_fitted_knn_predict(): assert (y['is_outlier'] == [True, False]).all() -@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), - MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_unfitted_knn_ensemble(aggregator, normalizer): knn_detector = KNN( k=[8, 9, 10], @@ -97,9 +97,9 @@ def test_unfitted_knn_ensemble(aggregator, normalizer): assert str(err.value) == 'KNNTorch has not been fit!' -@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), - MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_fitted_knn_ensemble(aggregator, normalizer): knn_detector = KNN( k=[8, 9, 10], @@ -118,9 +118,9 @@ def test_fitted_knn_ensemble(aggregator, normalizer): assert y['p_value'] is None -@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), - MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_fitted_knn_ensemble_predict(aggregator, normalizer): knn_detector = make_knn_detector( k=[8, 9, 10], @@ -136,9 +136,9 @@ def test_fitted_knn_ensemble_predict(aggregator, normalizer): assert (y['is_outlier'] == [True, False]).all() -@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), - MaxAggregatorTorch, MinAggregatorTorch]) -@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None]) +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_knn_ensemble_torch_script(aggregator, normalizer): knn_detector = make_knn_detector(k=[5, 6, 7], aggregator=aggregator(), normalizer=normalizer()) tsknn = torch.jit.script(knn_detector.backend) @@ -155,12 +155,12 @@ def test_knn_single_torchscript(): assert torch.all(y == torch.tensor([True, False])) -@pytest.mark.parametrize("aggregator", [AverageAggregatorTorch, lambda: TopKAggregatorTorch(k=7), - MaxAggregatorTorch, MinAggregatorTorch, lambda: 'AverageAggregatorTorch', - lambda: 'TopKAggregatorTorch', lambda: 'MaxAggregatorTorch', - lambda: 'MinAggregatorTorch']) -@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizerTorch, PValNormalizerTorch, lambda: None, - lambda: 'ShiftAndScaleNormalizerTorch', lambda: 'PValNormalizerTorch']) +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator, lambda: 'AverageAggregator', + lambda: 'TopKAggregator', lambda: 'MaxAggregator', + lambda: 'MinAggregator']) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None, + lambda: 'ShiftAndScaleNormalizer', lambda: 'PValNormalizer']) def test_knn_ensemble_integration(aggregator, normalizer): knn_detector = KNN( k=[10, 14, 18], diff --git a/alibi_detect/od/backend/tests/test_knn_backend.py b/alibi_detect/od/tests/test_knn/test_knn_backend.py similarity index 97% rename from alibi_detect/od/backend/tests/test_knn_backend.py rename to alibi_detect/od/tests/test_knn/test_knn_backend.py index 23048be39..5363b9a62 100644 --- a/alibi_detect/od/backend/tests/test_knn_backend.py +++ b/alibi_detect/od/tests/test_knn/test_knn_backend.py @@ -1,9 +1,9 @@ import pytest import torch -from alibi_detect.od.backend.torch.knn import KNNTorch +from alibi_detect.od.pytorch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF -from alibi_detect.od.backend.torch.ensemble import Accumulator, PValNormalizer, AverageAggregator +from alibi_detect.od.pytorch.ensemble import Accumulator, PValNormalizer, AverageAggregator from alibi_detect.od.base import NotFitException, ThresholdNotInferredException From f28cb0f310675129c505a2eb5dfe0a08861dd36a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 6 Jan 2023 16:15:04 +0000 Subject: [PATCH 057/247] Fix optional dependency tests --- alibi_detect/tests/test_dep_management.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index c8b1a3c14..0ab6e30dc 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -118,7 +118,13 @@ def test_od_dependencies(opt_dep): ('OutlierAE', ['tensorflow']), ('OutlierAEGMM', ['tensorflow']), ('OutlierSeq2Seq', ['tensorflow']), - ("OutlierProphet", ['prophet']) + ("OutlierProphet", ['prophet']), + ('PValNormalizer', ['torch', 'keops']), + ('ShiftAndScaleNormalizer', ['torch', 'keops']), + ('TopKAggregator', ['torch', 'keops']), + ('AverageAggregator', ['torch', 'keops']), + ('MaxAggregator', ['torch', 'keops']), + ('MinAggregator', ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect import od @@ -130,18 +136,12 @@ def test_od_backend_dependencies(opt_dep): """ dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ('PValNormalizerTorch', ['torch', 'keops']), - ('ShiftAndScaleNormalizerTorch', ['torch', 'keops']), - ('TopKAggregatorTorch', ['torch', 'keops']), - ('AverageAggregatorTorch', ['torch', 'keops']), - ('MaxAggregatorTorch', ['torch', 'keops']), - ('MinAggregatorTorch', ['torch', 'keops']), - ('AccumulatorTorch', ['torch', 'keops']), - ('KNNTorch', ['torch', 'keops']), + ('Accumulator', ['torch', 'keops']), + ('KNNTorch', ['torch', 'keops']), ]: dependency_map[dependency] = relations - from alibi_detect.od import backend as od_backend - check_correct_dependencies(od_backend, dependency_map, opt_dep) + from alibi_detect.od import pytorch as od_pt_backend + check_correct_dependencies(od_pt_backend, dependency_map, opt_dep) def test_tensorflow_model_dependencies(opt_dep): From 6ef8c62139d79e1b0bc42067de6b57dbfaebf0d5 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 10 Jan 2023 17:19:37 +0000 Subject: [PATCH 058/247] Add backticks do docstrings --- alibi_detect/od/knn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index dfa6819a9..a2e149478 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -40,16 +40,16 @@ def __init__( Number of nearest neighboursreveal_type(normalizer) to use for outlier detection. If an array is passed, an aggregator is required to aggregate the scores. kernel - Kernel function to use for outlier detection. If None, `torch.cdist` is used. + Kernel function to use for outlier detection. If ``None``, `torch.cdist` is used. normalizer - Normalizer to use for outlier detection. If None, no normalisation is applied. + Normalizer to use for outlier detection. If ``None``, no normalisation is applied. aggregator - Aggregator to use for outlier detection. Can be set to None if `k` is a single value. + Aggregator to use for outlier detection. Can be set to ``None`` if `k` is a single value. backend Backend used for outlier detection. Defaults to `'pytorch'`. Options are `'pytorch'`. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. + passing either `'cuda'`, `'gpu'` or `'cpu'`. Only relevant for `pytorch` backend. Raises ------ From 76b31d728f665ebff4621a8aa409f3bbfdec3f17 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 11 Jan 2023 11:37:39 +0000 Subject: [PATCH 059/247] Update docstrings --- alibi_detect/od/base.py | 4 ++-- alibi_detect/od/knn.py | 15 ++++++------ alibi_detect/od/pytorch/base.py | 12 +++++----- alibi_detect/od/pytorch/ensemble.py | 27 +++++++++++----------- alibi_detect/od/pytorch/knn.py | 8 +++---- alibi_detect/od/tests/test_knn/test_knn.py | 3 ++- 6 files changed, 35 insertions(+), 34 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 8130d84b4..832da0e3a 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -71,8 +71,8 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: Returns ------- - Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, \ - 'data' also contains the outlier labels. + Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ + performed, 'data' also contains the outlier labels. """ pass diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index a2e149478..50eecdecf 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -37,7 +37,7 @@ def __init__( Parameters ---------- k - Number of nearest neighboursreveal_type(normalizer) to use for outlier detection. If an array is passed, an + Number of nearest neighbours to use for outlier detection. If an array is passed, an aggregator is required to aggregate the scores. kernel Kernel function to use for outlier detection. If ``None``, `torch.cdist` is used. @@ -46,10 +46,10 @@ def __init__( aggregator Aggregator to use for outlier detection. Can be set to ``None`` if `k` is a single value. backend - Backend used for outlier detection. Defaults to `'pytorch'`. Options are `'pytorch'`. + Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either `'cuda'`, `'gpu'` or `'cpu'`. Only relevant for `pytorch` backend. + passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. Raises ------ @@ -70,7 +70,8 @@ def __init__( accumulator = None if aggregator is None and isinstance(k, (list, np.ndarray, tuple)): - raise ValueError("If `k` is an array, an aggregator is required.") + raise ValueError('If `k` is a `np.ndarray`, `list` or `tuple`, ' + 'the `aggregator` argument cannot be ``None``.') if isinstance(k, (list, np.ndarray, tuple)): accumulator = accumulator_cls( @@ -117,7 +118,7 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: fpr False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ - `(0, 1)`. + ``(0, 1)``. """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) @@ -131,8 +132,8 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: Returns ------- - Dict with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was performed, \ - 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ + performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. """ diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 4574b2246..ef8973cea 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -72,7 +72,7 @@ def _to_tensor(self, x: Union[List, np.ndarray]): Parameters ---------- x - Data to convert.ThresholdNotInferredException + Data to convert. Returns ------- @@ -127,7 +127,7 @@ def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: Returns ------- - `torch.Tensor` or `None` + `torch.Tensor` or ``None`` """ return scores > self.threshold if self.threshold_inferred else None @@ -141,7 +141,7 @@ def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: Returns ------- - `torch.Tensor` or `None` + `torch.Tensor` or ``None`` """ return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ if self.threshold_inferred else None @@ -159,10 +159,10 @@ def infer_threshold(self, x: torch.Tensor, fpr: float) -> None: Raises ------ ValueError - Raised if fpr is not in (0, 1). + Raised if `fpr` is not in ``(0, 1)``. """ if not 0 < fpr < 1: - ValueError('fpr must be in (0, 1).') + ValueError('`fpr` must be in `(0, 1)`.') self.val_scores = self.score(x) self.val_scores = self._accumulator(self.val_scores) self.threshold = torch.quantile(self.val_scores, 1-fpr) @@ -173,7 +173,7 @@ def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: Computes the outlier scores. If the detector is not fit on reference data we raise an error. If the threshold is inferred, the outlier labels and p-values are also computed and returned. - Otherwise, the outlier labels and p-values are set to `None`. + Otherwise, the outlier labels and p-values are set to ``None``. Parameters ---------- diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index c80fd3527..af02fa02f 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -13,8 +13,7 @@ class BaseTransformTorch(Module, ABC): def __init__(self): """Base Transform class. - provides abstract methods for transform objects that map a numpy - array. + provides abstract methods for transform objects that map `numpy` arrays. """ super().__init__() @@ -28,7 +27,7 @@ def _transform(self, x: torch.Tensor): Parameters ---------- x - numpy array to be transformed + `numpy` array to be transformed """ pass @@ -81,7 +80,7 @@ class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): def __init__(self): """Base Fitted Transform class. - Extends BaseTransfrom with fit functionality. Ensures that transform has been fit prior to + Extends `BaseTransfrom` with fit functionality. Ensures that transform has been fit prior to applying transform. """ BaseTransformTorch.__init__(self) @@ -106,11 +105,11 @@ def transform(self, x: torch.Tensor): class PValNormalizer(BaseFittedTransformTorch): def __init__(self): - """Maps scores to there p values. + """Maps scores to there p-values. Needs to be fit (see :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch`). Returns the proportion of scores in the reference dataset that are greater than the score of - interest. Output is between 1 and 0. Small values are likely to be outliers. + interest. Output is between ``1`` and ``0``. Small values are likely to be outliers. """ super().__init__() self.val_scores = None @@ -125,13 +124,13 @@ def _fit(self, val_scores: torch.Tensor) -> PValNormalizer: Returns ------- - self + `self` """ self.val_scores = val_scores return self def _transform(self, scores: torch.Tensor) -> torch.Tensor: - """Transform scores to 1 - p values. + """Transform scores to 1 - p-values. Parameters ---------- @@ -140,7 +139,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: Returns ------- - `Torch.Tensor` of 1 - p values. + `Torch.Tensor` of 1 - p-values. """ p_vals = ( 1 + (scores[:, None, :] < self.val_scores[None, :, :]).sum(1) @@ -169,7 +168,7 @@ def _fit(self, val_scores: torch.Tensor) -> ShiftAndScaleNormalizer: Returns ------- - self + `self` """ self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] @@ -197,7 +196,7 @@ def __init__(self, k: Optional[int] = None): Parameters ---------- k - number of scores to take the mean of. If `k` is left `None` then will be set to + number of scores to take the mean of. If `k` is left ``None`` then will be set to half the number of scores passed in the forward call. """ super().__init__() @@ -228,13 +227,13 @@ def __init__(self, weights: Optional[torch.Tensor] = None): Parameters ---------- weights - Optional parameter to weight the scores. If `weights` is left `None` then will be set to + Optional parameter to weight the scores. If `weights` is left ``None`` then will be set to a vector of ones. Raises ------ ValueError - If `weights` does not sum to `1`. + If `weights` does not sum to ``1``. """ super().__init__() if weights is not None and weights.sum() != 1: @@ -310,7 +309,7 @@ def __init__(self, Parameters ---------- normalizer - `BaseFittedTransformTorch` object to normalise the scores. If `None` then no normalisation + `BaseFittedTransformTorch` object to normalise the scores. If ``None`` then no normalisation is applied. aggregator `BaseTransformTorch` object to aggregate the scores. diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index d26446970..b718b49d2 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -25,15 +25,15 @@ def __init__( similarity to the `k`-th nearest neighbor. If `k` is a list then it returns the distance/kernel similarity to each of the specified `k` neighbors. kernel - If a kernel is specified then instead of using torch.cdist we compute the kernel similarity + If a kernel is specified then instead of using `torch.cdist` we compute the kernel similarity while computing the k nearest neighbor distance. accumulator - If `k` is an array of integers then the accumulator must not be None. Should be an instance + If `k` is an array of integers then the accumulator must not be ``None``. Should be an instance of :py:obj:`alibi_detect.od.pytorch.ensemble.Accumulator`. Responsible for combining multiple scores into a single score. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. + Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. """ TorchOutlierDetector.__init__(self, device=device) self.kernel = kernel @@ -51,7 +51,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - `torch.Tensor` of `bool` values with leading batch dimension. + `torch.Tensor` of ``bool`` values with leading batch dimension. Raises ------ diff --git a/alibi_detect/od/tests/test_knn/test_knn.py b/alibi_detect/od/tests/test_knn/test_knn.py index dd005c7ed..26bb6905d 100644 --- a/alibi_detect/od/tests/test_knn/test_knn.py +++ b/alibi_detect/od/tests/test_knn/test_knn.py @@ -64,7 +64,8 @@ def test_default_knn_ensemble_init(): def test_incorrect_knn_ensemble_init(): with pytest.raises(ValueError) as err: KNN(k=[8, 9, 10], aggregator=None) - assert str(err.value) == 'If `k` is an array, an aggregator is required.' + assert str(err.value) == ('If `k` is a `np.ndarray`, `list` or `tuple`, ' + 'the `aggregator` argument cannot be ``None``.') def test_fitted_knn_predict(): From 6fb600cc50ec93bdc50fa0e781a22f9f67bcf3fd Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 11 Jan 2023 11:43:39 +0000 Subject: [PATCH 060/247] reword numpy to torch tensor in transform object docstrings --- alibi_detect/od/pytorch/ensemble.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index af02fa02f..b3a49d13d 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -13,7 +13,7 @@ class BaseTransformTorch(Module, ABC): def __init__(self): """Base Transform class. - provides abstract methods for transform objects that map `numpy` arrays. + provides abstract methods for transform objects that map `torch` tensors. """ super().__init__() @@ -22,12 +22,12 @@ def transform(self, x: torch.Tensor): @abstractmethod def _transform(self, x: torch.Tensor): - """Applies class transform to numpy array + """Applies class transform to `torch.Tensor` Parameters ---------- x - `numpy` array to be transformed + `torch.Tensor` array to be transformed """ pass From cf6bba1c81f9ab9f4d34e5aeb13d2b21265b1b63 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 11 Jan 2023 11:51:36 +0000 Subject: [PATCH 061/247] Update return type hints --- alibi_detect/od/pytorch/base.py | 12 +++++++----- alibi_detect/od/pytorch/ensemble.py | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index ef8973cea..b7f17fec0 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import List, Union, Optional +from typing import List, Union, Optional, Dict from dataclasses import dataclass, asdict from abc import ABC, abstractmethod @@ -66,7 +66,7 @@ def check_threshold_infered(self): raise ThresholdNotInferredException((f'{self.__class__.__name__} has no threshold set, ' 'call `infer_threshold` before predicting.')) - def _to_tensor(self, x: Union[List, np.ndarray]): + def _to_tensor(self, x: Union[List, np.ndarray]) -> torch.Tensor: """Converts the data to a tensor. Parameters @@ -80,8 +80,10 @@ def _to_tensor(self, x: Union[List, np.ndarray]): """ return torch.as_tensor(x, dtype=torch.float32, device=self.device) - def _to_numpy(self, x: Union[torch.Tensor, TorchOutlierDetectorOutput]): - """Converts the data to `numpy.ndarray`. + def _to_numpy(self, x: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict]: + """Converts any `torch` tensors found in input to `numpy` arrays. + + Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays Parameters ---------- @@ -90,7 +92,7 @@ def _to_numpy(self, x: Union[torch.Tensor, TorchOutlierDetectorOutput]): Returns ------- - `np.ndarray` + `np.ndarray` or dictionary of containing `numpy` arrays """ if isinstance(x, torch.Tensor): return x.cpu().detach().numpy() diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index b3a49d13d..8c1dccab1 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -31,7 +31,7 @@ def _transform(self, x: torch.Tensor): """ pass - def forward(self, x: torch.Tensor): + def forward(self, x: torch.Tensor) -> torch.Tensor: return self.transform(x=x) @@ -86,7 +86,7 @@ def __init__(self): BaseTransformTorch.__init__(self) FitMixinTorch.__init__(self) - def transform(self, x: torch.Tensor): + def transform(self, x: torch.Tensor) -> torch.Tensor: """Checks to make sure transform has been fitted and then applies trasform to input tensor. Parameters @@ -320,7 +320,7 @@ def __init__(self, self.fitted = True self.aggregator = aggregator - def _transform(self, x: torch.Tensor): + def _transform(self, x: torch.Tensor) -> torch.Tensor: """Apply the normalizer and aggregator to the scores. Parameters From f8ae93d2daeace66e9ae68cda8a7e09018ee0437 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 16 Jan 2023 15:38:38 +0000 Subject: [PATCH 062/247] Add Mahalanobis detector --- alibi_detect/od/__init__.py | 2 +- alibi_detect/od/mahalanobis.py | 408 +++++------------- alibi_detect/od/pytorch/base.py | 3 +- alibi_detect/od/pytorch/mahalanobis.py | 109 +++++ alibi_detect/od/stateful_mahalanobis.py | 352 +++++++++++++++ .../test_mahalanobis_backend.py | 86 ++++ alibi_detect/utils/fetching/fetching.py | 2 +- 7 files changed, 648 insertions(+), 314 deletions(-) create mode 100644 alibi_detect/od/pytorch/mahalanobis.py create mode 100644 alibi_detect/od/stateful_mahalanobis.py create mode 100644 alibi_detect/od/tests/test_mahalanobis/test_mahalanobis_backend.py diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index c4e6edead..2c373e80e 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -1,7 +1,7 @@ from alibi_detect.utils.missing_optional_dependency import import_optional from .isolationforest import IForest -from .mahalanobis import Mahalanobis +from .stateful_mahalanobis import Mahalanobis from .sr import SpectralResidual from alibi_detect.od.base import TransformProtocol, transform_protocols diff --git a/alibi_detect/od/mahalanobis.py b/alibi_detect/od/mahalanobis.py index a1ec087fd..5bf4fa8bb 100644 --- a/alibi_detect/od/mahalanobis.py +++ b/alibi_detect/od/mahalanobis.py @@ -1,352 +1,138 @@ -import logging -import numpy as np -from scipy.linalg import eigh -from typing import Dict, Union -from alibi_detect.utils.discretizer import Discretizer -from alibi_detect.utils.distance import abdm, mvdm, multidim_scaling -from alibi_detect.utils.mapping import ohe2ord, ord2num -from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin, outlier_prediction_dict - -logger = logging.getLogger(__name__) - -EPSILON = 1e-8 - - -class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin): +from typing import Union, Optional, Dict, Any +from typing import TYPE_CHECKING - def __init__(self, - threshold: float = None, - n_components: int = 3, - std_clip: int = 3, - start_clip: int = 100, - max_n: int = None, - cat_vars: dict = None, - ohe: bool = False, - data_type: str = 'tabular' - ) -> None: - """ - Outlier detector for tabular data using the Mahalanobis distance. +import numpy as np - Parameters - ---------- - threshold - Mahalanobis distance threshold used to classify outliers. - n_components - Number of principal components used. - std_clip - Feature-wise stdev used to clip the observations before updating the mean and cov. - start_clip - Number of observations before clipping is applied. - max_n - Algorithm behaves as if it has seen at most max_n points. - cat_vars - Dict with as keys the categorical columns and as values - the number of categories per categorical variable. - ohe - Whether the categorical variables are one-hot encoded (OHE) or not. If not OHE, they are - assumed to have ordinal encodings. - data_type - Optionally specifiy the data type (tabular, image or time-series). Added to metadata. - """ - super().__init__() +from alibi_detect.utils._types import Literal +from alibi_detect.base import outlier_prediction_dict +from alibi_detect.od.base import OutlierDetector +from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch +from alibi_detect.utils.frameworks import BackendValidator +from alibi_detect.version import __version__ - if threshold is None: - logger.warning('No threshold level set. Need to infer threshold using `infer_threshold`.') - self.threshold = threshold - self.n_components = n_components - self.std_clip = std_clip - self.start_clip = start_clip - self.max_n = max_n +if TYPE_CHECKING: + import torch - # variables used in mapping from categorical to numerical values - # keys = categorical columns; values = numerical value for each of the categories - self.cat_vars = cat_vars - self.ohe = ohe - self.d_abs = {} # type: Dict - # initial parameter values - self.clip = None # type: Union[None, list] - self.mean = 0 - self.C = 0 - self.n = 0 +backends = { + 'pytorch': MahalanobisTorch +} - # set metadata - self.meta['detector_type'] = 'outlier' - self.meta['data_type'] = data_type - self.meta['online'] = True - def fit(self, - X: np.ndarray, - y: np.ndarray = None, - d_type: str = 'abdm', - w: float = None, - disc_perc: list = [25, 50, 75], - standardize_cat_vars: bool = True, - feature_range: tuple = (-1e10, 1e10), - smooth: float = 1., - center: bool = True - ) -> None: +class Mahalanobis(OutlierDetector): + def __init__( + self, + min_eigenvalue: float = 1e-6, + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, + backend: Literal['pytorch'] = 'pytorch', + ) -> None: """ - If categorical variables are present, then transform those to numerical values. - This step is not necessary in the absence of categorical variables. + Outliers identified via Mahalanobis distance. + + The kernel variant isn't exactly well known. For details/motivation see section + 3.4.3 of Aggarwal's Outlier Analysis. In summary, the linear variant can be + interpreted as projecting onto (orthogonal) eigenvectors of the covariance matrix, scaled + such that projections onto the eigenvectors have mean 0 and std 1. The Mahalanobis distance + is then the l2-norm from the origin. The same thing is done for the kernel case. It is + important to center the kernel matrix, however, Parameters ---------- - X - Batch of instances used to infer distances between categories from. - y - Model class predictions or ground truth labels for X. - Used for 'mvdm' and 'abdm-mvdm' pairwise distance metrics. - Note that this is only compatible with classification problems. For regression problems, - use the 'abdm' distance metric. - d_type - Pairwise distance metric used for categorical variables. Currently, 'abdm', 'mvdm' and 'abdm-mvdm' - are supported. 'abdm' infers context from the other variables while 'mvdm' uses the model predictions. - 'abdm-mvdm' is a weighted combination of the two metrics. - w - Weight on 'abdm' (between 0. and 1.) distance if d_type equals 'abdm-mvdm'. - disc_perc - List with percentiles used in binning of numerical features used for the 'abdm' - and 'abdm-mvdm' pairwise distance measures. - standardize_cat_vars - Standardize numerical values of categorical variables if True. - feature_range - Tuple with min and max ranges to allow for perturbed instances. Min and max ranges can be floats or - numpy arrays with dimension (1x nb of features) for feature-wise ranges. - smooth - Smoothing exponent between 0 and 1 for the distances. Lower values of l will smooth the difference in - distance metric between different features. - center - Whether to center the scaled distance measures. If False, the min distance for each feature - except for the feature with the highest raw max distance will be the lower bound of the - feature range, but the upper bound will be below the max feature range. + min_eigenvalue + Eigenvectors with eigenvalues below this value will be discarded. + backend + Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. + device + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + + Raises + ------ + NotImplementedError + If choice of `backend` is not implemented. """ - if self.cat_vars is None: - raise TypeError('No categorical variables specified in the "cat_vars" argument.') - - if d_type not in ['abdm', 'mvdm', 'abdm-mvdm']: - raise ValueError('d_type needs to be "abdm", "mvdm" or "abdm-mvdm". ' - '{} is not supported.'.format(d_type)) - - if self.ohe: - X_ord, cat_vars_ord = ohe2ord(X, self.cat_vars) - else: - X_ord, cat_vars_ord = X, self.cat_vars - - # bin numerical features to compute the pairwise distance matrices - cat_keys = list(cat_vars_ord.keys()) - n_ord = X_ord.shape[1] - if d_type in ['abdm', 'abdm-mvdm'] and len(cat_keys) != n_ord: - fnames = [str(_) for _ in range(n_ord)] - disc = Discretizer(X_ord, cat_keys, fnames, percentiles=disc_perc) - X_bin = disc.discretize(X_ord) - cat_vars_bin = {k: len(disc.names[k]) for k in range(n_ord) if k not in cat_keys} - else: - X_bin = X_ord - cat_vars_bin = {} - - # pairwise distances for categorical variables - if d_type == 'abdm': - d_pair = abdm(X_bin, cat_vars_ord, cat_vars_bin) - elif d_type == 'mvdm': - d_pair = mvdm(X_ord, y, cat_vars_ord, alpha=1) - - if (type(feature_range[0]) == type(feature_range[1]) and # noqa - type(feature_range[0]) in [int, float]): - feature_range = (np.ones((1, n_ord)) * feature_range[0], - np.ones((1, n_ord)) * feature_range[1]) - - if d_type == 'abdm-mvdm': - # pairwise distances - d_abdm = abdm(X_bin, cat_vars_ord, cat_vars_bin) - d_mvdm = mvdm(X_ord, y, cat_vars_ord, alpha=1) - - # multidim scaled distances - d_abs_abdm = multidim_scaling(d_abdm, n_components=2, use_metric=True, - feature_range=feature_range, - standardize_cat_vars=standardize_cat_vars, - smooth=smooth, center=center, - update_feature_range=False)[0] + super().__init__() - d_abs_mvdm = multidim_scaling(d_mvdm, n_components=2, use_metric=True, - feature_range=feature_range, - standardize_cat_vars=standardize_cat_vars, - smooth=smooth, center=center, - update_feature_range=False)[0] + backend_str: str = backend.lower() + BackendValidator( + backend_options={'pytorch': ['pytorch']}, + construct_name=self.__class__.__name__ + ).verify_backend(backend_str) - # combine abdm and mvdm - for k, v in d_abs_abdm.items(): - self.d_abs[k] = v * w + d_abs_mvdm[k] * (1 - w) - if center: # center the numerical feature values - self.d_abs[k] -= .5 * (self.d_abs[k].max() + self.d_abs[k].min()) - else: - self.d_abs = multidim_scaling(d_pair, n_components=2, use_metric=True, - feature_range=feature_range, - standardize_cat_vars=standardize_cat_vars, - smooth=smooth, center=center, - update_feature_range=False)[0] + backend_cls = backends[backend] + self.backend = backend_cls( + min_eigenvalue, + device=device + ) - def infer_threshold(self, - X: np.ndarray, - threshold_perc: float = 95. - ) -> None: - """ - Update threshold by a value inferred from the percentage of instances considered to be - outliers in a sample of the dataset. + def fit(self, x_ref: np.ndarray) -> None: + """Fit the detector on reference data. Parameters ---------- - X - Batch of instances. - threshold_perc - Percentage of X considered to be normal based on the outlier score. + x_ref + Reference data used to fit the detector. """ - # convert categorical variables to numerical values - X = self.cat2num(X) - - # compute outlier scores - iscore = self.score(X) + self.backend.fit(self.backend._to_tensor(x_ref)) - # update threshold - self.threshold = np.percentile(iscore, threshold_perc) - - def cat2num(self, X: np.ndarray) -> np.ndarray: - """ - Convert categorical variables to numerical values. + def score(self, x: np.ndarray) -> np.ndarray: + """Score `x` instances using the detector. Parameters ---------- - X - Batch of instances to analyze. + x + Data to score. The shape of `x` should be `(n_instances, n_features)`. Returns ------- - Batch of instances where categorical variables are converted to numerical values. + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + instance. """ - if self.cat_vars is not None: # convert categorical variables - if self.ohe: - X = ohe2ord(X, self.cat_vars)[0] - X = ord2num(X, self.d_abs) - return X + score = self.backend.score(self.backend._to_tensor(x)) + return self.backend._to_numpy(score) - def score(self, X: np.ndarray) -> np.ndarray: - """ - Compute outlier scores. + def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + """Infer the threshold for the Mahalanobis detector. The threshold is inferred using the reference data and the false + positive rate. The threshold is used to determine the outlier labels in the predict method. Parameters ---------- - X - Batch of instances to analyze. - - Returns - ------- - Array with outlier scores for each instance in the batch. + x_ref + Reference data used to infer the threshold. + fpr + False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ + `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ + ``(0, 1)``. """ - n_batch, n_params = X.shape # batch size and number of features - n_components = min(self.n_components, n_params) - if self.max_n is not None: - n = min(self.n, self.max_n) # n can never be above max_n - else: - n = self.n - - # clip X - if self.n > self.start_clip: - X_clip = np.clip(X, self.clip[0], self.clip[1]) - else: - X_clip = X - - # track mean and covariance matrix - roll_partial_means = X_clip.cumsum(axis=0) / (np.arange(n_batch) + 1).reshape((n_batch, 1)) - coefs = (np.arange(n_batch) + 1.) / (np.arange(n_batch) + n + 1.) - new_means = self.mean + coefs.reshape((n_batch, 1)) * (roll_partial_means - self.mean) - new_means_offset = np.empty_like(new_means) - new_means_offset[0] = self.mean - new_means_offset[1:] = new_means[:-1] + self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) - coefs = ((n + np.arange(n_batch)) / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) - B = coefs * np.matmul((X_clip - new_means_offset)[:, :, None], (X_clip - new_means_offset)[:, None, :]) - cov_batch = (n - 1.) / (n + max(1, n_batch - 1.)) * self.C + 1. / (n + max(1, n_batch - 1.)) * B.sum(axis=0) - - # PCA - eigvals, eigvects = eigh(cov_batch, eigvals=(n_params - n_components, n_params - 1)) - - # projections - proj_x = np.matmul(X, eigvects) - proj_x_clip = np.matmul(X_clip, eigvects) - proj_means = np.matmul(new_means_offset, eigvects) - if type(self.C) == int and self.C == 0: - proj_cov = np.diag(np.zeros(n_components)) - else: - proj_cov = np.matmul(eigvects.transpose(), np.matmul(self.C, eigvects)) - - # outlier scores are computed in the principal component space - coefs = (1. / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) - B = coefs * np.matmul((proj_x_clip - proj_means)[:, :, None], (proj_x_clip - proj_means)[:, None, :]) - all_C_inv = np.zeros_like(B) - c_inv = None - for i, b in enumerate(B): - if c_inv is None: - if abs(np.linalg.det(proj_cov)) > EPSILON: - c_inv = np.linalg.inv(proj_cov) - all_C_inv[i] = c_inv - continue - else: - if n + i == 0: - continue - proj_cov = (n + i - 1.) / (n + i) * proj_cov + b - continue - else: - c_inv = (n + i - 1.) / float(n + i - 2.) * all_C_inv[i - 1] - BC1 = np.matmul(B[i - 1], c_inv) - all_C_inv[i] = c_inv - 1. / (1. + np.trace(BC1)) * np.matmul(c_inv, BC1) - - # update parameters - self.mean = new_means[-1] - self.C = cov_batch - stdev = np.sqrt(np.diag(cov_batch)) - self.n += n_batch - if self.n > self.start_clip: - self.clip = [self.mean - self.std_clip * stdev, self.mean + self.std_clip * stdev] - - # compute outlier scores - x_diff = proj_x - proj_means - outlier_score = np.matmul(x_diff[:, None, :], np.matmul(all_C_inv, x_diff[:, :, None])).reshape(n_batch) - return outlier_score - - def predict(self, - X: np.ndarray, - return_instance_score: bool = True) \ - -> Dict[Dict[str, str], Dict[np.ndarray, np.ndarray]]: - """ - Compute outlier scores and transform into outlier predictions. + def predict(self, x: np.ndarray) -> Dict[str, Any]: + """Predict whether the instances in `x` are outliers or not. Parameters ---------- - X - Batch of instances. - return_instance_score - Whether to return instance level outlier scores. + x + Data to predict. The shape of `x` should be `(n_instances, n_features)`. Returns ------- - Dictionary containing 'meta' and 'data' dictionaries. - 'meta' has the model's metadata. - 'data' contains the outlier predictions and instance level outlier scores. + Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ + performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ + the detector. """ - # convert categorical variables to numerical values - X = self.cat2num(X) - - # compute outlier scores - iscore = self.score(X) - - # values above threshold are outliers - outlier_pred = (iscore > self.threshold).astype(int) - - # populate output dict - od = outlier_prediction_dict() - od['meta'] = self.meta - od['data']['is_outlier'] = outlier_pred - if return_instance_score: - od['data']['instance_score'] = iscore - return od + outputs = self.backend.predict(self.backend._to_tensor(x)) + output = outlier_prediction_dict() + output['data'] = { + **output['data'], + **self.backend._to_numpy(outputs) + } + output['meta'] = { + **output['meta'], + 'name': self.__class__.__name__, + 'detector_type': 'outlier', + 'online': False, + 'version': __version__, + } + return output diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index b7f17fec0..b045c41d2 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -117,7 +117,8 @@ def _accumulator(self, x: torch.Tensor) -> torch.Tensor: """ # `type: ignore` here becuase self.accumulator here causes an error with mypy when using torch.jit.script. # For some reason it thinks self.accumulator is a torch.Tensor and therefore is not callable. - return self.accumulator(x) if self.accumulator is not None else x # type: ignore + return self.accumulator(x) if hasattr(self, 'accumulator') \ + and self.accumulator is not None else x # type: ignore def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: """Classify the data as outlier or not. diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py new file mode 100644 index 000000000..7d560faad --- /dev/null +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -0,0 +1,109 @@ +from typing import Optional, Union + +import torch + +from alibi_detect.od.pytorch.base import TorchOutlierDetector + + +class MahalanobisTorch(TorchOutlierDetector): + def __init__( + self, + min_eigenvalue: float = 1e-6, + device: Optional[Union[str, torch.device]] = None + ): + """PyTorch backend for KNN detector. + + Parameters + ---------- + min_eigenvalue + Eigenvectors with eigenvalues below this value will be discarded. + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + """ + TorchOutlierDetector.__init__(self, device=device) + self.min_eigenvalue = min_eigenvalue + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Detect if `x` is an outlier. + + Parameters + ---------- + x + `torch.Tensor` with leading batch dimension. + + Returns + ------- + `torch.Tensor` of ``bool`` values with leading batch dimension. + + Raises + ------ + ThresholdNotInferredException + If called before detector has had `infer_threshold` method called. + """ + scores = self.score(x) + if not torch.jit.is_scripting(): + self.check_threshold_infered() + preds = scores > self.threshold + return preds.cpu() + + def score(self, x: torch.Tensor) -> torch.Tensor: + """Computes the score of `x` + + Project onto the PCs. + + Note: that if one computes ``x_ref_proj = self._compute_method_proj(self.x_ref)`` + then one can check that each column has zero mean and unit variance. The idea + is that new data will be similarly distributed if from the same distribution and therefore + its distance from the origin forms a sensible outlier score. + + Parameters + ---------- + x + The tensor of instances. First dimension corresponds to batch. + + Returns + ------- + Tensor of scores for each element in `x`. + + Raises + ------ + NotFitException + If called before detector has been fit. + """ + if not torch.jit.is_scripting(): + self.check_fitted() + x = torch.as_tensor(x) + x_pcs = self._compute_linear_proj(x) + return (x_pcs**2).sum(-1).cpu() + + def _fit(self, x_ref: torch.Tensor): + """Fits the detector + + Parameters + ---------- + x_ref + The Dataset tensor. + """ + self.x_ref = x_ref + self._compute_linear_pcs(self.x_ref) + # As a sanity check one can call x_ref_proj = self._compute_method_proj(self.x_ref) and see that + # we have fully whitened the data: each column has mean 0 and std 1. + + def _compute_linear_pcs(self, X: torch.Tensor): + """ + This saves the *residual* pcs (those whose eigenvalues are not in + the largest n_components). These are all that are needed to compute + the reconstruction error in the linear case. + """ + self.means = X.mean(0) + X = X - self.means + cov_mat = (X.t() @ X)/(len(X)-1) + D, V = torch.linalg.eigh(cov_mat) + non_zero_inds = D > self.min_eigenvalue + self.pcs = V[:, non_zero_inds] / D[None, non_zero_inds].sqrt() + + def _compute_linear_proj(self, X: torch.Tensor) -> torch.Tensor: + X_cen = X - self.means + X_proj = X_cen @ self.pcs + return X_proj diff --git a/alibi_detect/od/stateful_mahalanobis.py b/alibi_detect/od/stateful_mahalanobis.py new file mode 100644 index 000000000..a1ec087fd --- /dev/null +++ b/alibi_detect/od/stateful_mahalanobis.py @@ -0,0 +1,352 @@ +import logging +import numpy as np +from scipy.linalg import eigh +from typing import Dict, Union +from alibi_detect.utils.discretizer import Discretizer +from alibi_detect.utils.distance import abdm, mvdm, multidim_scaling +from alibi_detect.utils.mapping import ohe2ord, ord2num +from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin, outlier_prediction_dict + +logger = logging.getLogger(__name__) + +EPSILON = 1e-8 + + +class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin): + + def __init__(self, + threshold: float = None, + n_components: int = 3, + std_clip: int = 3, + start_clip: int = 100, + max_n: int = None, + cat_vars: dict = None, + ohe: bool = False, + data_type: str = 'tabular' + ) -> None: + """ + Outlier detector for tabular data using the Mahalanobis distance. + + Parameters + ---------- + threshold + Mahalanobis distance threshold used to classify outliers. + n_components + Number of principal components used. + std_clip + Feature-wise stdev used to clip the observations before updating the mean and cov. + start_clip + Number of observations before clipping is applied. + max_n + Algorithm behaves as if it has seen at most max_n points. + cat_vars + Dict with as keys the categorical columns and as values + the number of categories per categorical variable. + ohe + Whether the categorical variables are one-hot encoded (OHE) or not. If not OHE, they are + assumed to have ordinal encodings. + data_type + Optionally specifiy the data type (tabular, image or time-series). Added to metadata. + """ + super().__init__() + + if threshold is None: + logger.warning('No threshold level set. Need to infer threshold using `infer_threshold`.') + + self.threshold = threshold + self.n_components = n_components + self.std_clip = std_clip + self.start_clip = start_clip + self.max_n = max_n + + # variables used in mapping from categorical to numerical values + # keys = categorical columns; values = numerical value for each of the categories + self.cat_vars = cat_vars + self.ohe = ohe + self.d_abs = {} # type: Dict + + # initial parameter values + self.clip = None # type: Union[None, list] + self.mean = 0 + self.C = 0 + self.n = 0 + + # set metadata + self.meta['detector_type'] = 'outlier' + self.meta['data_type'] = data_type + self.meta['online'] = True + + def fit(self, + X: np.ndarray, + y: np.ndarray = None, + d_type: str = 'abdm', + w: float = None, + disc_perc: list = [25, 50, 75], + standardize_cat_vars: bool = True, + feature_range: tuple = (-1e10, 1e10), + smooth: float = 1., + center: bool = True + ) -> None: + """ + If categorical variables are present, then transform those to numerical values. + This step is not necessary in the absence of categorical variables. + + Parameters + ---------- + X + Batch of instances used to infer distances between categories from. + y + Model class predictions or ground truth labels for X. + Used for 'mvdm' and 'abdm-mvdm' pairwise distance metrics. + Note that this is only compatible with classification problems. For regression problems, + use the 'abdm' distance metric. + d_type + Pairwise distance metric used for categorical variables. Currently, 'abdm', 'mvdm' and 'abdm-mvdm' + are supported. 'abdm' infers context from the other variables while 'mvdm' uses the model predictions. + 'abdm-mvdm' is a weighted combination of the two metrics. + w + Weight on 'abdm' (between 0. and 1.) distance if d_type equals 'abdm-mvdm'. + disc_perc + List with percentiles used in binning of numerical features used for the 'abdm' + and 'abdm-mvdm' pairwise distance measures. + standardize_cat_vars + Standardize numerical values of categorical variables if True. + feature_range + Tuple with min and max ranges to allow for perturbed instances. Min and max ranges can be floats or + numpy arrays with dimension (1x nb of features) for feature-wise ranges. + smooth + Smoothing exponent between 0 and 1 for the distances. Lower values of l will smooth the difference in + distance metric between different features. + center + Whether to center the scaled distance measures. If False, the min distance for each feature + except for the feature with the highest raw max distance will be the lower bound of the + feature range, but the upper bound will be below the max feature range. + """ + if self.cat_vars is None: + raise TypeError('No categorical variables specified in the "cat_vars" argument.') + + if d_type not in ['abdm', 'mvdm', 'abdm-mvdm']: + raise ValueError('d_type needs to be "abdm", "mvdm" or "abdm-mvdm". ' + '{} is not supported.'.format(d_type)) + + if self.ohe: + X_ord, cat_vars_ord = ohe2ord(X, self.cat_vars) + else: + X_ord, cat_vars_ord = X, self.cat_vars + + # bin numerical features to compute the pairwise distance matrices + cat_keys = list(cat_vars_ord.keys()) + n_ord = X_ord.shape[1] + if d_type in ['abdm', 'abdm-mvdm'] and len(cat_keys) != n_ord: + fnames = [str(_) for _ in range(n_ord)] + disc = Discretizer(X_ord, cat_keys, fnames, percentiles=disc_perc) + X_bin = disc.discretize(X_ord) + cat_vars_bin = {k: len(disc.names[k]) for k in range(n_ord) if k not in cat_keys} + else: + X_bin = X_ord + cat_vars_bin = {} + + # pairwise distances for categorical variables + if d_type == 'abdm': + d_pair = abdm(X_bin, cat_vars_ord, cat_vars_bin) + elif d_type == 'mvdm': + d_pair = mvdm(X_ord, y, cat_vars_ord, alpha=1) + + if (type(feature_range[0]) == type(feature_range[1]) and # noqa + type(feature_range[0]) in [int, float]): + feature_range = (np.ones((1, n_ord)) * feature_range[0], + np.ones((1, n_ord)) * feature_range[1]) + + if d_type == 'abdm-mvdm': + # pairwise distances + d_abdm = abdm(X_bin, cat_vars_ord, cat_vars_bin) + d_mvdm = mvdm(X_ord, y, cat_vars_ord, alpha=1) + + # multidim scaled distances + d_abs_abdm = multidim_scaling(d_abdm, n_components=2, use_metric=True, + feature_range=feature_range, + standardize_cat_vars=standardize_cat_vars, + smooth=smooth, center=center, + update_feature_range=False)[0] + + d_abs_mvdm = multidim_scaling(d_mvdm, n_components=2, use_metric=True, + feature_range=feature_range, + standardize_cat_vars=standardize_cat_vars, + smooth=smooth, center=center, + update_feature_range=False)[0] + + # combine abdm and mvdm + for k, v in d_abs_abdm.items(): + self.d_abs[k] = v * w + d_abs_mvdm[k] * (1 - w) + if center: # center the numerical feature values + self.d_abs[k] -= .5 * (self.d_abs[k].max() + self.d_abs[k].min()) + else: + self.d_abs = multidim_scaling(d_pair, n_components=2, use_metric=True, + feature_range=feature_range, + standardize_cat_vars=standardize_cat_vars, + smooth=smooth, center=center, + update_feature_range=False)[0] + + def infer_threshold(self, + X: np.ndarray, + threshold_perc: float = 95. + ) -> None: + """ + Update threshold by a value inferred from the percentage of instances considered to be + outliers in a sample of the dataset. + + Parameters + ---------- + X + Batch of instances. + threshold_perc + Percentage of X considered to be normal based on the outlier score. + """ + # convert categorical variables to numerical values + X = self.cat2num(X) + + # compute outlier scores + iscore = self.score(X) + + # update threshold + self.threshold = np.percentile(iscore, threshold_perc) + + def cat2num(self, X: np.ndarray) -> np.ndarray: + """ + Convert categorical variables to numerical values. + + Parameters + ---------- + X + Batch of instances to analyze. + + Returns + ------- + Batch of instances where categorical variables are converted to numerical values. + """ + if self.cat_vars is not None: # convert categorical variables + if self.ohe: + X = ohe2ord(X, self.cat_vars)[0] + X = ord2num(X, self.d_abs) + return X + + def score(self, X: np.ndarray) -> np.ndarray: + """ + Compute outlier scores. + + Parameters + ---------- + X + Batch of instances to analyze. + + Returns + ------- + Array with outlier scores for each instance in the batch. + """ + n_batch, n_params = X.shape # batch size and number of features + n_components = min(self.n_components, n_params) + if self.max_n is not None: + n = min(self.n, self.max_n) # n can never be above max_n + else: + n = self.n + + # clip X + if self.n > self.start_clip: + X_clip = np.clip(X, self.clip[0], self.clip[1]) + else: + X_clip = X + + # track mean and covariance matrix + roll_partial_means = X_clip.cumsum(axis=0) / (np.arange(n_batch) + 1).reshape((n_batch, 1)) + coefs = (np.arange(n_batch) + 1.) / (np.arange(n_batch) + n + 1.) + new_means = self.mean + coefs.reshape((n_batch, 1)) * (roll_partial_means - self.mean) + new_means_offset = np.empty_like(new_means) + new_means_offset[0] = self.mean + new_means_offset[1:] = new_means[:-1] + + coefs = ((n + np.arange(n_batch)) / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) + B = coefs * np.matmul((X_clip - new_means_offset)[:, :, None], (X_clip - new_means_offset)[:, None, :]) + cov_batch = (n - 1.) / (n + max(1, n_batch - 1.)) * self.C + 1. / (n + max(1, n_batch - 1.)) * B.sum(axis=0) + + # PCA + eigvals, eigvects = eigh(cov_batch, eigvals=(n_params - n_components, n_params - 1)) + + # projections + proj_x = np.matmul(X, eigvects) + proj_x_clip = np.matmul(X_clip, eigvects) + proj_means = np.matmul(new_means_offset, eigvects) + if type(self.C) == int and self.C == 0: + proj_cov = np.diag(np.zeros(n_components)) + else: + proj_cov = np.matmul(eigvects.transpose(), np.matmul(self.C, eigvects)) + + # outlier scores are computed in the principal component space + coefs = (1. / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) + B = coefs * np.matmul((proj_x_clip - proj_means)[:, :, None], (proj_x_clip - proj_means)[:, None, :]) + all_C_inv = np.zeros_like(B) + c_inv = None + for i, b in enumerate(B): + if c_inv is None: + if abs(np.linalg.det(proj_cov)) > EPSILON: + c_inv = np.linalg.inv(proj_cov) + all_C_inv[i] = c_inv + continue + else: + if n + i == 0: + continue + proj_cov = (n + i - 1.) / (n + i) * proj_cov + b + continue + else: + c_inv = (n + i - 1.) / float(n + i - 2.) * all_C_inv[i - 1] + BC1 = np.matmul(B[i - 1], c_inv) + all_C_inv[i] = c_inv - 1. / (1. + np.trace(BC1)) * np.matmul(c_inv, BC1) + + # update parameters + self.mean = new_means[-1] + self.C = cov_batch + stdev = np.sqrt(np.diag(cov_batch)) + self.n += n_batch + if self.n > self.start_clip: + self.clip = [self.mean - self.std_clip * stdev, self.mean + self.std_clip * stdev] + + # compute outlier scores + x_diff = proj_x - proj_means + outlier_score = np.matmul(x_diff[:, None, :], np.matmul(all_C_inv, x_diff[:, :, None])).reshape(n_batch) + return outlier_score + + def predict(self, + X: np.ndarray, + return_instance_score: bool = True) \ + -> Dict[Dict[str, str], Dict[np.ndarray, np.ndarray]]: + """ + Compute outlier scores and transform into outlier predictions. + + Parameters + ---------- + X + Batch of instances. + return_instance_score + Whether to return instance level outlier scores. + + Returns + ------- + Dictionary containing 'meta' and 'data' dictionaries. + 'meta' has the model's metadata. + 'data' contains the outlier predictions and instance level outlier scores. + """ + # convert categorical variables to numerical values + X = self.cat2num(X) + + # compute outlier scores + iscore = self.score(X) + + # values above threshold are outliers + outlier_pred = (iscore > self.threshold).astype(int) + + # populate output dict + od = outlier_prediction_dict() + od['meta'] = self.meta + od['data']['is_outlier'] = outlier_pred + if return_instance_score: + od['data']['instance_score'] = iscore + return od diff --git a/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis_backend.py b/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis_backend.py new file mode 100644 index 000000000..063f4b8a7 --- /dev/null +++ b/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis_backend.py @@ -0,0 +1,86 @@ +import pytest +import torch +import numpy as np + +from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch +from alibi_detect.od.base import NotFitException, ThresholdNotInferredException + + +def test_mahalanobis_torch_backend_fit_errors(): + mahalanobis_torch = MahalanobisTorch() + assert not mahalanobis_torch._fitted + + x = torch.randn((1, 10)) + with pytest.raises(NotFitException) as err: + mahalanobis_torch(x) + assert str(err.value) == 'MahalanobisTorch has not been fit!' + + with pytest.raises(NotFitException) as err: + mahalanobis_torch.predict(x) + assert str(err.value) == 'MahalanobisTorch has not been fit!' + + x_ref = torch.randn((1024, 10)) + mahalanobis_torch.fit(x_ref) + + assert mahalanobis_torch._fitted + + with pytest.raises(ThresholdNotInferredException) as err: + mahalanobis_torch(x) + assert str(err.value) == 'MahalanobisTorch has no threshold set, call `infer_threshold` before predicting.' + + assert mahalanobis_torch.predict(x) + + +def test_mahalanobis_linear_scoring(): + mahalanobis_torch = MahalanobisTorch() + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + mahalanobis_torch.fit(x_ref) + p = mahalanobis_torch._compute_linear_proj(mahalanobis_torch.x_ref) + + # test that the x_ref is whitened by the data + assert p.mean() < 0.1 + assert p.std() - 1 < 0.1 + + x_1 = torch.tensor([[8., 8.]]) + scores_1 = mahalanobis_torch.score(x_1) + + x_2 = torch.tensor(np.random.multivariate_normal(mean, cov, 1)) + scores_2 = mahalanobis_torch.score(x_2) + + x_3 = torch.tensor([[-10., 10.]]) + scores_3 = mahalanobis_torch.score(x_3) + + # test correct ordering of scores given outlyingness of data + assert scores_1 < scores_2 < scores_3 + + # test that detector correctly detects true Outlier + mahalanobis_torch.infer_threshold(x_ref, 0.01) + x = np.concatenate((x_1, x_2, x_3)) + outputs = mahalanobis_torch.predict(x) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) + assert torch.all(mahalanobis_torch(x) == torch.tensor([False, False, True])) + + # test that 0.01 of the in distribution data is flagged as outliers + x = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + outputs = mahalanobis_torch.predict(x) + assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.005 + + +def test_mahalanobis_torch_backend_ts(tmp_path): + mahalanobis_torch = MahalanobisTorch() + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + x_ref = torch.randn((1024, 10)) + mahalanobis_torch.fit(x_ref) + mahalanobis_torch.infer_threshold(x_ref, 0.1) + pred_1 = mahalanobis_torch(x) + + mahalanobis_torch = torch.jit.script(mahalanobis_torch) + pred_2 = mahalanobis_torch(x) + assert torch.all(pred_1 == pred_2) + + mahalanobis_torch.save(tmp_path / 'mahalanobis_torch.pt') + mahalanobis_torch = torch.load(tmp_path / 'mahalanobis_torch.pt') + pred_2 = mahalanobis_torch(x) + assert torch.all(pred_1 == pred_2) diff --git a/alibi_detect/utils/fetching/fetching.py b/alibi_detect/utils/fetching/fetching.py index b487e470d..ca8f8e424 100644 --- a/alibi_detect/utils/fetching/fetching.py +++ b/alibi_detect/utils/fetching/fetching.py @@ -21,7 +21,7 @@ from alibi_detect.base import BaseDetector # noqa from alibi_detect.od.llr import LLR # noqa from alibi_detect.od.isolationforest import IForest # noqa - from alibi_detect.od.mahalanobis import Mahalanobis # noqa + from alibi_detect.od.stateful_mahalanobis import Mahalanobis # noqa from alibi_detect.od.aegmm import OutlierAEGMM # noqa from alibi_detect.od.ae import OutlierAE # noqa from alibi_detect.od.prophet import OutlierProphet # noqa From 96f9be06ea17ae2d44183f30f739f4cde71126ed Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 16 Jan 2023 17:11:45 +0000 Subject: [PATCH 063/247] Add hasattr check in _accumulator method --- alibi_detect/od/pytorch/base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index b7f17fec0..2cdcf3071 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -115,9 +115,12 @@ def _accumulator(self, x: torch.Tensor) -> torch.Tensor: ------- `torch.Tensor` or just returns original data """ - # `type: ignore` here becuase self.accumulator here causes an error with mypy when using torch.jit.script. - # For some reason it thinks self.accumulator is a torch.Tensor and therefore is not callable. - return self.accumulator(x) if self.accumulator is not None else x # type: ignore + if hasattr(self, 'accumulator') and self.accumulator is not None: + # `type: ignore` here becuase self.accumulator here causes an error with mypy when using torch.jit.script. + # For some reason it thinks self.accumulator is a torch.Tensor and therefore is not callable. + return self.accumulator(x) # type: ignore + else: + return x def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: """Classify the data as outlier or not. From e936b3673a3754fe5a952dd0b76488dd67fba165 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 16 Jan 2023 17:39:24 +0000 Subject: [PATCH 064/247] Add singlular dispatch pattern for _to_numpy method --- alibi_detect/od/pytorch/base.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 2cdcf3071..b06dc38c3 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -2,6 +2,8 @@ from typing import List, Union, Optional, Dict from dataclasses import dataclass, asdict from abc import ABC, abstractmethod +from functools import singledispatchmethod + import numpy as np import torch @@ -80,7 +82,8 @@ def _to_tensor(self, x: Union[List, np.ndarray]) -> torch.Tensor: """ return torch.as_tensor(x, dtype=torch.float32, device=self.device) - def _to_numpy(self, x: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict]: + @singledispatchmethod + def _to_numpy(self, arg): """Converts any `torch` tensors found in input to `numpy` arrays. Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays @@ -94,14 +97,19 @@ def _to_numpy(self, x: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union ------- `np.ndarray` or dictionary of containing `numpy` arrays """ - if isinstance(x, torch.Tensor): - return x.cpu().detach().numpy() - elif isinstance(x, TorchOutlierDetectorOutput): - outputs = asdict(x) - for key, value in outputs.items(): - if isinstance(value, torch.Tensor): - outputs[key] = value.cpu().detach().numpy() - return outputs + raise NotImplementedError(f"Cannot transform type {type(arg)} to numpy array.") + + @_to_numpy.register + def _(self, x: torch.Tensor) -> np.ndarray: + return x.cpu().detach().numpy() + + @_to_numpy.register + def _(self, x: TorchOutlierDetectorOutput) -> Dict: + outputs = asdict(x) + for key, value in outputs.items(): + if isinstance(value, torch.Tensor): + outputs[key] = value.cpu().detach().numpy() + return outputs def _accumulator(self, x: torch.Tensor) -> torch.Tensor: """Accumulates the data. From 1bf5fdc3c42df3084ba7cbf9a99f9065901fbe7a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 09:33:09 +0000 Subject: [PATCH 065/247] Add Mahalanobis tests --- .../test_mahalanobis/test_mahalanobis.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 alibi_detect/od/tests/test_mahalanobis/test_mahalanobis.py diff --git a/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis.py b/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis.py new file mode 100644 index 000000000..e45061601 --- /dev/null +++ b/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis.py @@ -0,0 +1,74 @@ +import pytest +import numpy as np +import torch + +from alibi_detect.od.mahalanobis import Mahalanobis +from alibi_detect.od.base import NotFitException +from sklearn.datasets import make_moons + + +def make_mahalanobis_detector(): + mahalanobis_detector = Mahalanobis() + x_ref = np.random.randn(100, 2) + mahalanobis_detector.fit(x_ref) + mahalanobis_detector.infer_threshold(x_ref, 0.1) + return mahalanobis_detector + + +def test_unfitted_mahalanobis_single_score(): + mahalanobis_detector = Mahalanobis() + x = np.array([[0, 10], [0.1, 0]]) + with pytest.raises(NotFitException) as err: + _ = mahalanobis_detector.predict(x) + assert str(err.value) == 'MahalanobisTorch has not been fit!' + + +def test_fitted_mahalanobis_single_score(): + mahalanobis_detector = Mahalanobis() + x_ref = np.random.randn(100, 2) + mahalanobis_detector.fit(x_ref) + x = np.array([[0, 10], [0.1, 0]]) + y = mahalanobis_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None + + +def test_fitted_mahalanobis_predict(): + mahalanobis_detector = make_mahalanobis_detector() + x_ref = np.random.randn(100, 2) + mahalanobis_detector.infer_threshold(x_ref, 0.1) + x = np.array([[0, 10], [0, 0.1]]) + y = mahalanobis_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() + + +def test_mahalanobis_integration(): + mahalanobis_detector = Mahalanobis() + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + mahalanobis_detector.fit(X_ref) + mahalanobis_detector.infer_threshold(X_ref, 0.1) + result = mahalanobis_detector.predict(x_inlier) + result = result['data']['is_outlier'][0] + assert not result + + x_outlier = np.array([[-1, 1.5]]) + result = mahalanobis_detector.predict(x_outlier) + result = result['data']['is_outlier'][0] + assert result + + ts_mahalanobis = torch.jit.script(mahalanobis_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = ts_mahalanobis(x) + assert torch.all(y == torch.tensor([False, True])) From a6ce6a28e43a0123a87d729ac624c30fe487a9cf Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 09:54:12 +0000 Subject: [PATCH 066/247] Fix flake8 error --- alibi_detect/od/mahalanobis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/mahalanobis.py b/alibi_detect/od/mahalanobis.py index 5bf4fa8bb..6de1b34e0 100644 --- a/alibi_detect/od/mahalanobis.py +++ b/alibi_detect/od/mahalanobis.py @@ -93,8 +93,8 @@ def score(self, x: np.ndarray) -> np.ndarray: return self.backend._to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: - """Infer the threshold for the Mahalanobis detector. The threshold is inferred using the reference data and the false - positive rate. The threshold is used to determine the outlier labels in the predict method. + """Infer the threshold for the Mahalanobis detector. The threshold is inferred using the reference data + and the false positive rate. The threshold is used to determine the outlier labels in the predict method. Parameters ---------- From babc083317489c0e65b7b182727035fe76865bda Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 13:07:58 +0000 Subject: [PATCH 067/247] Add MahalanobisTorch to test_dep_management tests --- alibi_detect/od/pytorch/__init__.py | 1 + alibi_detect/tests/test_dep_management.py | 1 + 2 files changed, 2 insertions(+) diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py index c90fa0835..cc12af042 100644 --- a/alibi_detect/od/pytorch/__init__.py +++ b/alibi_detect/od/pytorch/__init__.py @@ -1,4 +1,5 @@ from alibi_detect.utils.missing_optional_dependency import import_optional KNNTorch = import_optional('alibi_detect.od.pytorch.knn', ['KNNTorch']) +MahalanobisTorch = import_optional('alibi_detect.od.pytorch.mahalanobis', ['MahalanobisTorch']) Accumulator = import_optional('alibi_detect.od.pytorch.ensemble', ['Accumulator']) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index 0ab6e30dc..3f0fc3780 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -138,6 +138,7 @@ def test_od_backend_dependencies(opt_dep): for dependency, relations in [ ('Accumulator', ['torch', 'keops']), ('KNNTorch', ['torch', 'keops']), + ('MahalanobisTorch', ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.od import pytorch as od_pt_backend From ce28b8ec064de0867425971777dab2e24ad0d025 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 13:16:07 +0000 Subject: [PATCH 068/247] Replace alibi_detect.utils._types imports with typing_extension --- alibi_detect/od/__init__.py | 2 +- alibi_detect/od/base.py | 2 +- alibi_detect/od/knn.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index c4e6edead..45ccfef51 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -5,7 +5,7 @@ from .sr import SpectralResidual from alibi_detect.od.base import TransformProtocol, transform_protocols -from alibi_detect.utils._types import Literal +from typing_extensions import Literal from typing import Union PValNormalizer, ShiftAndScaleNormalizer, TopKAggregator, AverageAggregator, \ diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 832da0e3a..32e990bc5 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Dict, Any, Union -from alibi_detect.utils._types import Protocol, runtime_checkable +from typing_extensions import Protocol, runtime_checkable import numpy as np from alibi_detect.base import BaseDetector diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 50eecdecf..727139c5f 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -3,7 +3,7 @@ import numpy as np -from alibi_detect.utils._types import Literal +from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols from alibi_detect.od.pytorch import KNNTorch, Accumulator From 7fe67f167d1c78264a0e31dd5b7b148810ac199e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 13:22:44 +0000 Subject: [PATCH 069/247] Minor change --- alibi_detect/od/mahalanobis.py | 9 +++------ alibi_detect/od/pytorch/mahalanobis.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/alibi_detect/od/mahalanobis.py b/alibi_detect/od/mahalanobis.py index 6de1b34e0..9bd623e29 100644 --- a/alibi_detect/od/mahalanobis.py +++ b/alibi_detect/od/mahalanobis.py @@ -30,12 +30,9 @@ def __init__( """ Outliers identified via Mahalanobis distance. - The kernel variant isn't exactly well known. For details/motivation see section - 3.4.3 of Aggarwal's Outlier Analysis. In summary, the linear variant can be - interpreted as projecting onto (orthogonal) eigenvectors of the covariance matrix, scaled - such that projections onto the eigenvectors have mean 0 and std 1. The Mahalanobis distance - is then the l2-norm from the origin. The same thing is done for the kernel case. It is - important to center the kernel matrix, however, + The linear variant can be interpreted as projecting onto (orthogonal) eigenvectors of the + covariance matrix, scaled such that projections onto the eigenvectors have mean 0 and std 1. + The Mahalanobis distance is then the l2-norm from the origin. Parameters ---------- diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 7d560faad..05163a45c 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -11,7 +11,7 @@ def __init__( min_eigenvalue: float = 1e-6, device: Optional[Union[str, torch.device]] = None ): - """PyTorch backend for KNN detector. + """PyTorch backend for Mahalanobis detector. Parameters ---------- From 789a1d32b1f89e784d48ded5225a8d9ce0586e36 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 13:49:55 +0000 Subject: [PATCH 070/247] Replace singular_dispatch_method with singular_dispatch --- alibi_detect/od/knn.py | 6 +-- alibi_detect/od/pytorch/__init__.py | 2 + alibi_detect/od/pytorch/base.py | 63 ++++++++++++----------- alibi_detect/tests/test_dep_management.py | 1 + 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/knn.py index 727139c5f..f1ace2361 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/knn.py @@ -6,7 +6,7 @@ from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols -from alibi_detect.od.pytorch import KNNTorch, Accumulator +from alibi_detect.od.pytorch import KNNTorch, Accumulator, to_numpy from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -105,7 +105,7 @@ def score(self, x: np.ndarray) -> np.ndarray: instance. """ score = self.backend.score(self.backend._to_tensor(x)) - return self.backend._to_numpy(score) + return to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. The threshold is inferred using the reference data and the false @@ -141,7 +141,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: output = outlier_prediction_dict() output['data'] = { **output['data'], - **self.backend._to_numpy(outputs) + **to_numpy(outputs) } output['meta'] = { **output['meta'], diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py index c90fa0835..fb64bef77 100644 --- a/alibi_detect/od/pytorch/__init__.py +++ b/alibi_detect/od/pytorch/__init__.py @@ -2,3 +2,5 @@ KNNTorch = import_optional('alibi_detect.od.pytorch.knn', ['KNNTorch']) Accumulator = import_optional('alibi_detect.od.pytorch.ensemble', ['Accumulator']) + +to_numpy = import_optional('alibi_detect.od.pytorch.base', ['to_numpy']) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index b06dc38c3..623a9c678 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -2,7 +2,7 @@ from typing import List, Union, Optional, Dict from dataclasses import dataclass, asdict from abc import ABC, abstractmethod -from functools import singledispatchmethod +from functools import singledispatch import numpy as np @@ -23,6 +23,38 @@ class TorchOutlierDetectorOutput: p_value: Optional[torch.Tensor] +@singledispatch +def to_numpy(arg): + """Converts any `torch` tensors found in input to `numpy` arrays. + + Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays + + Parameters + ---------- + x + Data to convert. + + Returns + ------- + `np.ndarray` or dictionary of containing `numpy` arrays + """ + raise NotImplementedError(f"Cannot transform type {type(arg)} to numpy array.") + + +@to_numpy.register +def _(x: torch.Tensor) -> np.ndarray: + return x.cpu().detach().numpy() + + +@to_numpy.register +def _(x: TorchOutlierDetectorOutput) -> Dict: + outputs = asdict(x) + for key, value in outputs.items(): + if isinstance(value, torch.Tensor): + outputs[key] = value.cpu().detach().numpy() + return outputs + + class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): """Base class for torch backend outlier detection algorithms.""" threshold_inferred = False @@ -82,35 +114,6 @@ def _to_tensor(self, x: Union[List, np.ndarray]) -> torch.Tensor: """ return torch.as_tensor(x, dtype=torch.float32, device=self.device) - @singledispatchmethod - def _to_numpy(self, arg): - """Converts any `torch` tensors found in input to `numpy` arrays. - - Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays - - Parameters - ---------- - x - Data to convert. - - Returns - ------- - `np.ndarray` or dictionary of containing `numpy` arrays - """ - raise NotImplementedError(f"Cannot transform type {type(arg)} to numpy array.") - - @_to_numpy.register - def _(self, x: torch.Tensor) -> np.ndarray: - return x.cpu().detach().numpy() - - @_to_numpy.register - def _(self, x: TorchOutlierDetectorOutput) -> Dict: - outputs = asdict(x) - for key, value in outputs.items(): - if isinstance(value, torch.Tensor): - outputs[key] = value.cpu().detach().numpy() - return outputs - def _accumulator(self, x: torch.Tensor) -> torch.Tensor: """Accumulates the data. diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index 0ab6e30dc..c61f3682c 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -138,6 +138,7 @@ def test_od_backend_dependencies(opt_dep): for dependency, relations in [ ('Accumulator', ['torch', 'keops']), ('KNNTorch', ['torch', 'keops']), + ('to_numpy', ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.od import pytorch as od_pt_backend From fe7c185212f9128113dbae8840236af4fd406703 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 13:55:10 +0000 Subject: [PATCH 071/247] Update changed _to_numpy logic --- alibi_detect/od/mahalanobis.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/mahalanobis.py b/alibi_detect/od/mahalanobis.py index 9bd623e29..5ba6dc66b 100644 --- a/alibi_detect/od/mahalanobis.py +++ b/alibi_detect/od/mahalanobis.py @@ -6,6 +6,7 @@ from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector +from alibi_detect.od.pytorch.base import to_numpy from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -87,7 +88,7 @@ def score(self, x: np.ndarray) -> np.ndarray: instance. """ score = self.backend.score(self.backend._to_tensor(x)) - return self.backend._to_numpy(score) + return to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. The threshold is inferred using the reference data @@ -123,7 +124,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: output = outlier_prediction_dict() output['data'] = { **output['data'], - **self.backend._to_numpy(outputs) + **to_numpy(outputs) } output['meta'] = { **output['meta'], From ada292e9bccde17b30a35bd291d099c05a6071f9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 17 Jan 2023 14:39:06 +0000 Subject: [PATCH 072/247] Rename test files --- alibi_detect/od/mahalanobis.py | 6 +++--- .../{test_mahalanobis.py => test_stateful_mahalanobis.py} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename alibi_detect/od/tests/{test_mahalanobis.py => test_stateful_mahalanobis.py} (100%) diff --git a/alibi_detect/od/mahalanobis.py b/alibi_detect/od/mahalanobis.py index 5ba6dc66b..256ca50c0 100644 --- a/alibi_detect/od/mahalanobis.py +++ b/alibi_detect/od/mahalanobis.py @@ -31,9 +31,9 @@ def __init__( """ Outliers identified via Mahalanobis distance. - The linear variant can be interpreted as projecting onto (orthogonal) eigenvectors of the - covariance matrix, scaled such that projections onto the eigenvectors have mean 0 and std 1. - The Mahalanobis distance is then the l2-norm from the origin. + The Mahalanobis method can be interpreted as projecting data points onto (orthogonal) eigenvectors of the + covariance matrix of the reference dataset. The eigenvectors are scaled such that projections onto them + have mean 0 and std 1. The Mahalanobis distance is then the l2-norm from the origin. Parameters ---------- diff --git a/alibi_detect/od/tests/test_mahalanobis.py b/alibi_detect/od/tests/test_stateful_mahalanobis.py similarity index 100% rename from alibi_detect/od/tests/test_mahalanobis.py rename to alibi_detect/od/tests/test_stateful_mahalanobis.py From 31dbed18d01a141cfcc0416d9735737c832fa70e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 18 Jan 2023 10:06:19 +0000 Subject: [PATCH 073/247] Improove docstrings for mahalanobis detector --- alibi_detect/od/mahalanobis.py | 24 +++++++++++++++++------- alibi_detect/od/pytorch/mahalanobis.py | 26 +++++++++++++------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/alibi_detect/od/mahalanobis.py b/alibi_detect/od/mahalanobis.py index 256ca50c0..7df808960 100644 --- a/alibi_detect/od/mahalanobis.py +++ b/alibi_detect/od/mahalanobis.py @@ -28,12 +28,14 @@ def __init__( device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', ) -> None: - """ - Outliers identified via Mahalanobis distance. + """The Mahalanobis outlier detection method. + + The Mahalanobis method computes the covariance matrix of a reference dataset passed in the `fit` method. It + then saves the eigenvectors of this matrix with eigenvalues greater than `min_eigenvalue`. While doing so + it also scales the eigenvectors such that the reference data projected onto them has mean ``0`` and std ``1``. - The Mahalanobis method can be interpreted as projecting data points onto (orthogonal) eigenvectors of the - covariance matrix of the reference dataset. The eigenvectors are scaled such that projections onto them - have mean 0 and std 1. The Mahalanobis distance is then the l2-norm from the origin. + When we score a test point `x` we project it onto the eigenvectors and compute the l2-norm of the + projected point. The higher the score, the more outlying the instance. Parameters ---------- @@ -67,6 +69,9 @@ def __init__( def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. + Fitting the Mahalanobis method amounts to computing the covariance matrix of the reference data and + saving the eigenvectors with eigenvalues greater than `min_eigenvalue`. + Parameters ---------- x_ref @@ -77,6 +82,9 @@ def fit(self, x_ref: np.ndarray) -> None: def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. + The mahalanobis method projects `x` onto the eigenvectors of the covariance matrix of the reference data. + The score is then the l2-norm of the projected data. The higher the score, the more outlying the instance. + Parameters ---------- x @@ -91,8 +99,10 @@ def score(self, x: np.ndarray) -> np.ndarray: return to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: - """Infer the threshold for the Mahalanobis detector. The threshold is inferred using the reference data - and the false positive rate. The threshold is used to determine the outlier labels in the predict method. + """Infer the threshold for the Mahalanobis detector. + + The threshold is inferred using the reference data and the false positive rate. The threshold is used to + determine the outlier labels in the predict method. Parameters ---------- diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 05163a45c..fec3b569c 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -50,13 +50,6 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` - Project onto the PCs. - - Note: that if one computes ``x_ref_proj = self._compute_method_proj(self.x_ref)`` - then one can check that each column has zero mean and unit variance. The idea - is that new data will be similarly distributed if from the same distribution and therefore - its distance from the origin forms a sensible outlier score. - Parameters ---------- x @@ -87,14 +80,14 @@ def _fit(self, x_ref: torch.Tensor): """ self.x_ref = x_ref self._compute_linear_pcs(self.x_ref) - # As a sanity check one can call x_ref_proj = self._compute_method_proj(self.x_ref) and see that - # we have fully whitened the data: each column has mean 0 and std 1. def _compute_linear_pcs(self, X: torch.Tensor): - """ - This saves the *residual* pcs (those whose eigenvalues are not in - the largest n_components). These are all that are needed to compute - the reconstruction error in the linear case. + """Computes the principle components of the data. + + Parameters + ---------- + X + The reference dataset. """ self.means = X.mean(0) X = X - self.means @@ -104,6 +97,13 @@ def _compute_linear_pcs(self, X: torch.Tensor): self.pcs = V[:, non_zero_inds] / D[None, non_zero_inds].sqrt() def _compute_linear_proj(self, X: torch.Tensor) -> torch.Tensor: + """Projects the data point being tested onto the principle components. + + Parameters + ---------- + X + The data point being tested. + """ X_cen = X - self.means X_proj = X_cen @ self.pcs return X_proj From c865d4cbd5888673600170ea8e522a99fd047820 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 20 Jan 2023 14:06:02 +0000 Subject: [PATCH 074/247] Rename new mahalanobis -> _mahalanobis --- alibi_detect/od/__init__.py | 2 +- alibi_detect/od/_mahalanobis.py | 146 +++++++ alibi_detect/od/mahalanobis.py | 410 +++++++++++++----- alibi_detect/od/stateful_mahalanobis.py | 352 --------------- .../test__mahalanobis.py} | 2 +- .../test__mahalanobis_backend.py} | 0 ...ful_mahalanobis.py => test_mahalanobis.py} | 0 7 files changed, 456 insertions(+), 456 deletions(-) create mode 100644 alibi_detect/od/_mahalanobis.py delete mode 100644 alibi_detect/od/stateful_mahalanobis.py rename alibi_detect/od/tests/{test_mahalanobis/test_mahalanobis.py => test__mahalanobis/test__mahalanobis.py} (97%) rename alibi_detect/od/tests/{test_mahalanobis/test_mahalanobis_backend.py => test__mahalanobis/test__mahalanobis_backend.py} (100%) rename alibi_detect/od/tests/{test_stateful_mahalanobis.py => test_mahalanobis.py} (100%) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index 92e2518de..45ccfef51 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -1,7 +1,7 @@ from alibi_detect.utils.missing_optional_dependency import import_optional from .isolationforest import IForest -from .stateful_mahalanobis import Mahalanobis +from .mahalanobis import Mahalanobis from .sr import SpectralResidual from alibi_detect.od.base import TransformProtocol, transform_protocols diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py new file mode 100644 index 000000000..7df808960 --- /dev/null +++ b/alibi_detect/od/_mahalanobis.py @@ -0,0 +1,146 @@ +from typing import Union, Optional, Dict, Any +from typing import TYPE_CHECKING + +import numpy as np + +from alibi_detect.utils._types import Literal +from alibi_detect.base import outlier_prediction_dict +from alibi_detect.od.base import OutlierDetector +from alibi_detect.od.pytorch.base import to_numpy +from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch +from alibi_detect.utils.frameworks import BackendValidator +from alibi_detect.version import __version__ + + +if TYPE_CHECKING: + import torch + + +backends = { + 'pytorch': MahalanobisTorch +} + + +class Mahalanobis(OutlierDetector): + def __init__( + self, + min_eigenvalue: float = 1e-6, + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, + backend: Literal['pytorch'] = 'pytorch', + ) -> None: + """The Mahalanobis outlier detection method. + + The Mahalanobis method computes the covariance matrix of a reference dataset passed in the `fit` method. It + then saves the eigenvectors of this matrix with eigenvalues greater than `min_eigenvalue`. While doing so + it also scales the eigenvectors such that the reference data projected onto them has mean ``0`` and std ``1``. + + When we score a test point `x` we project it onto the eigenvectors and compute the l2-norm of the + projected point. The higher the score, the more outlying the instance. + + Parameters + ---------- + min_eigenvalue + Eigenvectors with eigenvalues below this value will be discarded. + backend + Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. + device + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + + Raises + ------ + NotImplementedError + If choice of `backend` is not implemented. + """ + super().__init__() + + backend_str: str = backend.lower() + BackendValidator( + backend_options={'pytorch': ['pytorch']}, + construct_name=self.__class__.__name__ + ).verify_backend(backend_str) + + backend_cls = backends[backend] + self.backend = backend_cls( + min_eigenvalue, + device=device + ) + + def fit(self, x_ref: np.ndarray) -> None: + """Fit the detector on reference data. + + Fitting the Mahalanobis method amounts to computing the covariance matrix of the reference data and + saving the eigenvectors with eigenvalues greater than `min_eigenvalue`. + + Parameters + ---------- + x_ref + Reference data used to fit the detector. + """ + self.backend.fit(self.backend._to_tensor(x_ref)) + + def score(self, x: np.ndarray) -> np.ndarray: + """Score `x` instances using the detector. + + The mahalanobis method projects `x` onto the eigenvectors of the covariance matrix of the reference data. + The score is then the l2-norm of the projected data. The higher the score, the more outlying the instance. + + Parameters + ---------- + x + Data to score. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + instance. + """ + score = self.backend.score(self.backend._to_tensor(x)) + return to_numpy(score) + + def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + """Infer the threshold for the Mahalanobis detector. + + The threshold is inferred using the reference data and the false positive rate. The threshold is used to + determine the outlier labels in the predict method. + + Parameters + ---------- + x_ref + Reference data used to infer the threshold. + fpr + False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ + `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ + ``(0, 1)``. + """ + self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + + def predict(self, x: np.ndarray) -> Dict[str, Any]: + """Predict whether the instances in `x` are outliers or not. + + Parameters + ---------- + x + Data to predict. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ + performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ + the detector. + """ + outputs = self.backend.predict(self.backend._to_tensor(x)) + output = outlier_prediction_dict() + output['data'] = { + **output['data'], + **to_numpy(outputs) + } + output['meta'] = { + **output['meta'], + 'name': self.__class__.__name__, + 'detector_type': 'outlier', + 'online': False, + 'version': __version__, + } + return output diff --git a/alibi_detect/od/mahalanobis.py b/alibi_detect/od/mahalanobis.py index 7df808960..a1ec087fd 100644 --- a/alibi_detect/od/mahalanobis.py +++ b/alibi_detect/od/mahalanobis.py @@ -1,146 +1,352 @@ -from typing import Union, Optional, Dict, Any -from typing import TYPE_CHECKING - +import logging import numpy as np +from scipy.linalg import eigh +from typing import Dict, Union +from alibi_detect.utils.discretizer import Discretizer +from alibi_detect.utils.distance import abdm, mvdm, multidim_scaling +from alibi_detect.utils.mapping import ohe2ord, ord2num +from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin, outlier_prediction_dict + +logger = logging.getLogger(__name__) + +EPSILON = 1e-8 -from alibi_detect.utils._types import Literal -from alibi_detect.base import outlier_prediction_dict -from alibi_detect.od.base import OutlierDetector -from alibi_detect.od.pytorch.base import to_numpy -from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch -from alibi_detect.utils.frameworks import BackendValidator -from alibi_detect.version import __version__ +class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin): + + def __init__(self, + threshold: float = None, + n_components: int = 3, + std_clip: int = 3, + start_clip: int = 100, + max_n: int = None, + cat_vars: dict = None, + ohe: bool = False, + data_type: str = 'tabular' + ) -> None: + """ + Outlier detector for tabular data using the Mahalanobis distance. -if TYPE_CHECKING: - import torch + Parameters + ---------- + threshold + Mahalanobis distance threshold used to classify outliers. + n_components + Number of principal components used. + std_clip + Feature-wise stdev used to clip the observations before updating the mean and cov. + start_clip + Number of observations before clipping is applied. + max_n + Algorithm behaves as if it has seen at most max_n points. + cat_vars + Dict with as keys the categorical columns and as values + the number of categories per categorical variable. + ohe + Whether the categorical variables are one-hot encoded (OHE) or not. If not OHE, they are + assumed to have ordinal encodings. + data_type + Optionally specifiy the data type (tabular, image or time-series). Added to metadata. + """ + super().__init__() + if threshold is None: + logger.warning('No threshold level set. Need to infer threshold using `infer_threshold`.') -backends = { - 'pytorch': MahalanobisTorch -} + self.threshold = threshold + self.n_components = n_components + self.std_clip = std_clip + self.start_clip = start_clip + self.max_n = max_n + # variables used in mapping from categorical to numerical values + # keys = categorical columns; values = numerical value for each of the categories + self.cat_vars = cat_vars + self.ohe = ohe + self.d_abs = {} # type: Dict -class Mahalanobis(OutlierDetector): - def __init__( - self, - min_eigenvalue: float = 1e-6, - device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, - backend: Literal['pytorch'] = 'pytorch', - ) -> None: - """The Mahalanobis outlier detection method. + # initial parameter values + self.clip = None # type: Union[None, list] + self.mean = 0 + self.C = 0 + self.n = 0 - The Mahalanobis method computes the covariance matrix of a reference dataset passed in the `fit` method. It - then saves the eigenvectors of this matrix with eigenvalues greater than `min_eigenvalue`. While doing so - it also scales the eigenvectors such that the reference data projected onto them has mean ``0`` and std ``1``. + # set metadata + self.meta['detector_type'] = 'outlier' + self.meta['data_type'] = data_type + self.meta['online'] = True - When we score a test point `x` we project it onto the eigenvectors and compute the l2-norm of the - projected point. The higher the score, the more outlying the instance. + def fit(self, + X: np.ndarray, + y: np.ndarray = None, + d_type: str = 'abdm', + w: float = None, + disc_perc: list = [25, 50, 75], + standardize_cat_vars: bool = True, + feature_range: tuple = (-1e10, 1e10), + smooth: float = 1., + center: bool = True + ) -> None: + """ + If categorical variables are present, then transform those to numerical values. + This step is not necessary in the absence of categorical variables. Parameters ---------- - min_eigenvalue - Eigenvectors with eigenvalues below this value will be discarded. - backend - Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. - device - Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. - - Raises - ------ - NotImplementedError - If choice of `backend` is not implemented. + X + Batch of instances used to infer distances between categories from. + y + Model class predictions or ground truth labels for X. + Used for 'mvdm' and 'abdm-mvdm' pairwise distance metrics. + Note that this is only compatible with classification problems. For regression problems, + use the 'abdm' distance metric. + d_type + Pairwise distance metric used for categorical variables. Currently, 'abdm', 'mvdm' and 'abdm-mvdm' + are supported. 'abdm' infers context from the other variables while 'mvdm' uses the model predictions. + 'abdm-mvdm' is a weighted combination of the two metrics. + w + Weight on 'abdm' (between 0. and 1.) distance if d_type equals 'abdm-mvdm'. + disc_perc + List with percentiles used in binning of numerical features used for the 'abdm' + and 'abdm-mvdm' pairwise distance measures. + standardize_cat_vars + Standardize numerical values of categorical variables if True. + feature_range + Tuple with min and max ranges to allow for perturbed instances. Min and max ranges can be floats or + numpy arrays with dimension (1x nb of features) for feature-wise ranges. + smooth + Smoothing exponent between 0 and 1 for the distances. Lower values of l will smooth the difference in + distance metric between different features. + center + Whether to center the scaled distance measures. If False, the min distance for each feature + except for the feature with the highest raw max distance will be the lower bound of the + feature range, but the upper bound will be below the max feature range. """ - super().__init__() + if self.cat_vars is None: + raise TypeError('No categorical variables specified in the "cat_vars" argument.') + + if d_type not in ['abdm', 'mvdm', 'abdm-mvdm']: + raise ValueError('d_type needs to be "abdm", "mvdm" or "abdm-mvdm". ' + '{} is not supported.'.format(d_type)) + + if self.ohe: + X_ord, cat_vars_ord = ohe2ord(X, self.cat_vars) + else: + X_ord, cat_vars_ord = X, self.cat_vars + + # bin numerical features to compute the pairwise distance matrices + cat_keys = list(cat_vars_ord.keys()) + n_ord = X_ord.shape[1] + if d_type in ['abdm', 'abdm-mvdm'] and len(cat_keys) != n_ord: + fnames = [str(_) for _ in range(n_ord)] + disc = Discretizer(X_ord, cat_keys, fnames, percentiles=disc_perc) + X_bin = disc.discretize(X_ord) + cat_vars_bin = {k: len(disc.names[k]) for k in range(n_ord) if k not in cat_keys} + else: + X_bin = X_ord + cat_vars_bin = {} + + # pairwise distances for categorical variables + if d_type == 'abdm': + d_pair = abdm(X_bin, cat_vars_ord, cat_vars_bin) + elif d_type == 'mvdm': + d_pair = mvdm(X_ord, y, cat_vars_ord, alpha=1) - backend_str: str = backend.lower() - BackendValidator( - backend_options={'pytorch': ['pytorch']}, - construct_name=self.__class__.__name__ - ).verify_backend(backend_str) + if (type(feature_range[0]) == type(feature_range[1]) and # noqa + type(feature_range[0]) in [int, float]): + feature_range = (np.ones((1, n_ord)) * feature_range[0], + np.ones((1, n_ord)) * feature_range[1]) - backend_cls = backends[backend] - self.backend = backend_cls( - min_eigenvalue, - device=device - ) + if d_type == 'abdm-mvdm': + # pairwise distances + d_abdm = abdm(X_bin, cat_vars_ord, cat_vars_bin) + d_mvdm = mvdm(X_ord, y, cat_vars_ord, alpha=1) - def fit(self, x_ref: np.ndarray) -> None: - """Fit the detector on reference data. + # multidim scaled distances + d_abs_abdm = multidim_scaling(d_abdm, n_components=2, use_metric=True, + feature_range=feature_range, + standardize_cat_vars=standardize_cat_vars, + smooth=smooth, center=center, + update_feature_range=False)[0] - Fitting the Mahalanobis method amounts to computing the covariance matrix of the reference data and - saving the eigenvectors with eigenvalues greater than `min_eigenvalue`. + d_abs_mvdm = multidim_scaling(d_mvdm, n_components=2, use_metric=True, + feature_range=feature_range, + standardize_cat_vars=standardize_cat_vars, + smooth=smooth, center=center, + update_feature_range=False)[0] + + # combine abdm and mvdm + for k, v in d_abs_abdm.items(): + self.d_abs[k] = v * w + d_abs_mvdm[k] * (1 - w) + if center: # center the numerical feature values + self.d_abs[k] -= .5 * (self.d_abs[k].max() + self.d_abs[k].min()) + else: + self.d_abs = multidim_scaling(d_pair, n_components=2, use_metric=True, + feature_range=feature_range, + standardize_cat_vars=standardize_cat_vars, + smooth=smooth, center=center, + update_feature_range=False)[0] + + def infer_threshold(self, + X: np.ndarray, + threshold_perc: float = 95. + ) -> None: + """ + Update threshold by a value inferred from the percentage of instances considered to be + outliers in a sample of the dataset. Parameters ---------- - x_ref - Reference data used to fit the detector. + X + Batch of instances. + threshold_perc + Percentage of X considered to be normal based on the outlier score. """ - self.backend.fit(self.backend._to_tensor(x_ref)) + # convert categorical variables to numerical values + X = self.cat2num(X) - def score(self, x: np.ndarray) -> np.ndarray: - """Score `x` instances using the detector. + # compute outlier scores + iscore = self.score(X) - The mahalanobis method projects `x` onto the eigenvectors of the covariance matrix of the reference data. - The score is then the l2-norm of the projected data. The higher the score, the more outlying the instance. + # update threshold + self.threshold = np.percentile(iscore, threshold_perc) + + def cat2num(self, X: np.ndarray) -> np.ndarray: + """ + Convert categorical variables to numerical values. Parameters ---------- - x - Data to score. The shape of `x` should be `(n_instances, n_features)`. + X + Batch of instances to analyze. Returns ------- - Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ - instance. + Batch of instances where categorical variables are converted to numerical values. """ - score = self.backend.score(self.backend._to_tensor(x)) - return to_numpy(score) - - def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: - """Infer the threshold for the Mahalanobis detector. + if self.cat_vars is not None: # convert categorical variables + if self.ohe: + X = ohe2ord(X, self.cat_vars)[0] + X = ord2num(X, self.d_abs) + return X - The threshold is inferred using the reference data and the false positive rate. The threshold is used to - determine the outlier labels in the predict method. + def score(self, X: np.ndarray) -> np.ndarray: + """ + Compute outlier scores. Parameters ---------- - x_ref - Reference data used to infer the threshold. - fpr - False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ - `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ - ``(0, 1)``. + X + Batch of instances to analyze. + + Returns + ------- + Array with outlier scores for each instance in the batch. """ - self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + n_batch, n_params = X.shape # batch size and number of features + n_components = min(self.n_components, n_params) + if self.max_n is not None: + n = min(self.n, self.max_n) # n can never be above max_n + else: + n = self.n + + # clip X + if self.n > self.start_clip: + X_clip = np.clip(X, self.clip[0], self.clip[1]) + else: + X_clip = X + + # track mean and covariance matrix + roll_partial_means = X_clip.cumsum(axis=0) / (np.arange(n_batch) + 1).reshape((n_batch, 1)) + coefs = (np.arange(n_batch) + 1.) / (np.arange(n_batch) + n + 1.) + new_means = self.mean + coefs.reshape((n_batch, 1)) * (roll_partial_means - self.mean) + new_means_offset = np.empty_like(new_means) + new_means_offset[0] = self.mean + new_means_offset[1:] = new_means[:-1] + + coefs = ((n + np.arange(n_batch)) / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) + B = coefs * np.matmul((X_clip - new_means_offset)[:, :, None], (X_clip - new_means_offset)[:, None, :]) + cov_batch = (n - 1.) / (n + max(1, n_batch - 1.)) * self.C + 1. / (n + max(1, n_batch - 1.)) * B.sum(axis=0) - def predict(self, x: np.ndarray) -> Dict[str, Any]: - """Predict whether the instances in `x` are outliers or not. + # PCA + eigvals, eigvects = eigh(cov_batch, eigvals=(n_params - n_components, n_params - 1)) + + # projections + proj_x = np.matmul(X, eigvects) + proj_x_clip = np.matmul(X_clip, eigvects) + proj_means = np.matmul(new_means_offset, eigvects) + if type(self.C) == int and self.C == 0: + proj_cov = np.diag(np.zeros(n_components)) + else: + proj_cov = np.matmul(eigvects.transpose(), np.matmul(self.C, eigvects)) + + # outlier scores are computed in the principal component space + coefs = (1. / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) + B = coefs * np.matmul((proj_x_clip - proj_means)[:, :, None], (proj_x_clip - proj_means)[:, None, :]) + all_C_inv = np.zeros_like(B) + c_inv = None + for i, b in enumerate(B): + if c_inv is None: + if abs(np.linalg.det(proj_cov)) > EPSILON: + c_inv = np.linalg.inv(proj_cov) + all_C_inv[i] = c_inv + continue + else: + if n + i == 0: + continue + proj_cov = (n + i - 1.) / (n + i) * proj_cov + b + continue + else: + c_inv = (n + i - 1.) / float(n + i - 2.) * all_C_inv[i - 1] + BC1 = np.matmul(B[i - 1], c_inv) + all_C_inv[i] = c_inv - 1. / (1. + np.trace(BC1)) * np.matmul(c_inv, BC1) + + # update parameters + self.mean = new_means[-1] + self.C = cov_batch + stdev = np.sqrt(np.diag(cov_batch)) + self.n += n_batch + if self.n > self.start_clip: + self.clip = [self.mean - self.std_clip * stdev, self.mean + self.std_clip * stdev] + + # compute outlier scores + x_diff = proj_x - proj_means + outlier_score = np.matmul(x_diff[:, None, :], np.matmul(all_C_inv, x_diff[:, :, None])).reshape(n_batch) + return outlier_score + + def predict(self, + X: np.ndarray, + return_instance_score: bool = True) \ + -> Dict[Dict[str, str], Dict[np.ndarray, np.ndarray]]: + """ + Compute outlier scores and transform into outlier predictions. Parameters ---------- - x - Data to predict. The shape of `x` should be `(n_instances, n_features)`. + X + Batch of instances. + return_instance_score + Whether to return instance level outlier scores. Returns ------- - Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ - performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ - `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ - the detector. + Dictionary containing 'meta' and 'data' dictionaries. + 'meta' has the model's metadata. + 'data' contains the outlier predictions and instance level outlier scores. """ - outputs = self.backend.predict(self.backend._to_tensor(x)) - output = outlier_prediction_dict() - output['data'] = { - **output['data'], - **to_numpy(outputs) - } - output['meta'] = { - **output['meta'], - 'name': self.__class__.__name__, - 'detector_type': 'outlier', - 'online': False, - 'version': __version__, - } - return output + # convert categorical variables to numerical values + X = self.cat2num(X) + + # compute outlier scores + iscore = self.score(X) + + # values above threshold are outliers + outlier_pred = (iscore > self.threshold).astype(int) + + # populate output dict + od = outlier_prediction_dict() + od['meta'] = self.meta + od['data']['is_outlier'] = outlier_pred + if return_instance_score: + od['data']['instance_score'] = iscore + return od diff --git a/alibi_detect/od/stateful_mahalanobis.py b/alibi_detect/od/stateful_mahalanobis.py deleted file mode 100644 index a1ec087fd..000000000 --- a/alibi_detect/od/stateful_mahalanobis.py +++ /dev/null @@ -1,352 +0,0 @@ -import logging -import numpy as np -from scipy.linalg import eigh -from typing import Dict, Union -from alibi_detect.utils.discretizer import Discretizer -from alibi_detect.utils.distance import abdm, mvdm, multidim_scaling -from alibi_detect.utils.mapping import ohe2ord, ord2num -from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin, outlier_prediction_dict - -logger = logging.getLogger(__name__) - -EPSILON = 1e-8 - - -class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin): - - def __init__(self, - threshold: float = None, - n_components: int = 3, - std_clip: int = 3, - start_clip: int = 100, - max_n: int = None, - cat_vars: dict = None, - ohe: bool = False, - data_type: str = 'tabular' - ) -> None: - """ - Outlier detector for tabular data using the Mahalanobis distance. - - Parameters - ---------- - threshold - Mahalanobis distance threshold used to classify outliers. - n_components - Number of principal components used. - std_clip - Feature-wise stdev used to clip the observations before updating the mean and cov. - start_clip - Number of observations before clipping is applied. - max_n - Algorithm behaves as if it has seen at most max_n points. - cat_vars - Dict with as keys the categorical columns and as values - the number of categories per categorical variable. - ohe - Whether the categorical variables are one-hot encoded (OHE) or not. If not OHE, they are - assumed to have ordinal encodings. - data_type - Optionally specifiy the data type (tabular, image or time-series). Added to metadata. - """ - super().__init__() - - if threshold is None: - logger.warning('No threshold level set. Need to infer threshold using `infer_threshold`.') - - self.threshold = threshold - self.n_components = n_components - self.std_clip = std_clip - self.start_clip = start_clip - self.max_n = max_n - - # variables used in mapping from categorical to numerical values - # keys = categorical columns; values = numerical value for each of the categories - self.cat_vars = cat_vars - self.ohe = ohe - self.d_abs = {} # type: Dict - - # initial parameter values - self.clip = None # type: Union[None, list] - self.mean = 0 - self.C = 0 - self.n = 0 - - # set metadata - self.meta['detector_type'] = 'outlier' - self.meta['data_type'] = data_type - self.meta['online'] = True - - def fit(self, - X: np.ndarray, - y: np.ndarray = None, - d_type: str = 'abdm', - w: float = None, - disc_perc: list = [25, 50, 75], - standardize_cat_vars: bool = True, - feature_range: tuple = (-1e10, 1e10), - smooth: float = 1., - center: bool = True - ) -> None: - """ - If categorical variables are present, then transform those to numerical values. - This step is not necessary in the absence of categorical variables. - - Parameters - ---------- - X - Batch of instances used to infer distances between categories from. - y - Model class predictions or ground truth labels for X. - Used for 'mvdm' and 'abdm-mvdm' pairwise distance metrics. - Note that this is only compatible with classification problems. For regression problems, - use the 'abdm' distance metric. - d_type - Pairwise distance metric used for categorical variables. Currently, 'abdm', 'mvdm' and 'abdm-mvdm' - are supported. 'abdm' infers context from the other variables while 'mvdm' uses the model predictions. - 'abdm-mvdm' is a weighted combination of the two metrics. - w - Weight on 'abdm' (between 0. and 1.) distance if d_type equals 'abdm-mvdm'. - disc_perc - List with percentiles used in binning of numerical features used for the 'abdm' - and 'abdm-mvdm' pairwise distance measures. - standardize_cat_vars - Standardize numerical values of categorical variables if True. - feature_range - Tuple with min and max ranges to allow for perturbed instances. Min and max ranges can be floats or - numpy arrays with dimension (1x nb of features) for feature-wise ranges. - smooth - Smoothing exponent between 0 and 1 for the distances. Lower values of l will smooth the difference in - distance metric between different features. - center - Whether to center the scaled distance measures. If False, the min distance for each feature - except for the feature with the highest raw max distance will be the lower bound of the - feature range, but the upper bound will be below the max feature range. - """ - if self.cat_vars is None: - raise TypeError('No categorical variables specified in the "cat_vars" argument.') - - if d_type not in ['abdm', 'mvdm', 'abdm-mvdm']: - raise ValueError('d_type needs to be "abdm", "mvdm" or "abdm-mvdm". ' - '{} is not supported.'.format(d_type)) - - if self.ohe: - X_ord, cat_vars_ord = ohe2ord(X, self.cat_vars) - else: - X_ord, cat_vars_ord = X, self.cat_vars - - # bin numerical features to compute the pairwise distance matrices - cat_keys = list(cat_vars_ord.keys()) - n_ord = X_ord.shape[1] - if d_type in ['abdm', 'abdm-mvdm'] and len(cat_keys) != n_ord: - fnames = [str(_) for _ in range(n_ord)] - disc = Discretizer(X_ord, cat_keys, fnames, percentiles=disc_perc) - X_bin = disc.discretize(X_ord) - cat_vars_bin = {k: len(disc.names[k]) for k in range(n_ord) if k not in cat_keys} - else: - X_bin = X_ord - cat_vars_bin = {} - - # pairwise distances for categorical variables - if d_type == 'abdm': - d_pair = abdm(X_bin, cat_vars_ord, cat_vars_bin) - elif d_type == 'mvdm': - d_pair = mvdm(X_ord, y, cat_vars_ord, alpha=1) - - if (type(feature_range[0]) == type(feature_range[1]) and # noqa - type(feature_range[0]) in [int, float]): - feature_range = (np.ones((1, n_ord)) * feature_range[0], - np.ones((1, n_ord)) * feature_range[1]) - - if d_type == 'abdm-mvdm': - # pairwise distances - d_abdm = abdm(X_bin, cat_vars_ord, cat_vars_bin) - d_mvdm = mvdm(X_ord, y, cat_vars_ord, alpha=1) - - # multidim scaled distances - d_abs_abdm = multidim_scaling(d_abdm, n_components=2, use_metric=True, - feature_range=feature_range, - standardize_cat_vars=standardize_cat_vars, - smooth=smooth, center=center, - update_feature_range=False)[0] - - d_abs_mvdm = multidim_scaling(d_mvdm, n_components=2, use_metric=True, - feature_range=feature_range, - standardize_cat_vars=standardize_cat_vars, - smooth=smooth, center=center, - update_feature_range=False)[0] - - # combine abdm and mvdm - for k, v in d_abs_abdm.items(): - self.d_abs[k] = v * w + d_abs_mvdm[k] * (1 - w) - if center: # center the numerical feature values - self.d_abs[k] -= .5 * (self.d_abs[k].max() + self.d_abs[k].min()) - else: - self.d_abs = multidim_scaling(d_pair, n_components=2, use_metric=True, - feature_range=feature_range, - standardize_cat_vars=standardize_cat_vars, - smooth=smooth, center=center, - update_feature_range=False)[0] - - def infer_threshold(self, - X: np.ndarray, - threshold_perc: float = 95. - ) -> None: - """ - Update threshold by a value inferred from the percentage of instances considered to be - outliers in a sample of the dataset. - - Parameters - ---------- - X - Batch of instances. - threshold_perc - Percentage of X considered to be normal based on the outlier score. - """ - # convert categorical variables to numerical values - X = self.cat2num(X) - - # compute outlier scores - iscore = self.score(X) - - # update threshold - self.threshold = np.percentile(iscore, threshold_perc) - - def cat2num(self, X: np.ndarray) -> np.ndarray: - """ - Convert categorical variables to numerical values. - - Parameters - ---------- - X - Batch of instances to analyze. - - Returns - ------- - Batch of instances where categorical variables are converted to numerical values. - """ - if self.cat_vars is not None: # convert categorical variables - if self.ohe: - X = ohe2ord(X, self.cat_vars)[0] - X = ord2num(X, self.d_abs) - return X - - def score(self, X: np.ndarray) -> np.ndarray: - """ - Compute outlier scores. - - Parameters - ---------- - X - Batch of instances to analyze. - - Returns - ------- - Array with outlier scores for each instance in the batch. - """ - n_batch, n_params = X.shape # batch size and number of features - n_components = min(self.n_components, n_params) - if self.max_n is not None: - n = min(self.n, self.max_n) # n can never be above max_n - else: - n = self.n - - # clip X - if self.n > self.start_clip: - X_clip = np.clip(X, self.clip[0], self.clip[1]) - else: - X_clip = X - - # track mean and covariance matrix - roll_partial_means = X_clip.cumsum(axis=0) / (np.arange(n_batch) + 1).reshape((n_batch, 1)) - coefs = (np.arange(n_batch) + 1.) / (np.arange(n_batch) + n + 1.) - new_means = self.mean + coefs.reshape((n_batch, 1)) * (roll_partial_means - self.mean) - new_means_offset = np.empty_like(new_means) - new_means_offset[0] = self.mean - new_means_offset[1:] = new_means[:-1] - - coefs = ((n + np.arange(n_batch)) / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) - B = coefs * np.matmul((X_clip - new_means_offset)[:, :, None], (X_clip - new_means_offset)[:, None, :]) - cov_batch = (n - 1.) / (n + max(1, n_batch - 1.)) * self.C + 1. / (n + max(1, n_batch - 1.)) * B.sum(axis=0) - - # PCA - eigvals, eigvects = eigh(cov_batch, eigvals=(n_params - n_components, n_params - 1)) - - # projections - proj_x = np.matmul(X, eigvects) - proj_x_clip = np.matmul(X_clip, eigvects) - proj_means = np.matmul(new_means_offset, eigvects) - if type(self.C) == int and self.C == 0: - proj_cov = np.diag(np.zeros(n_components)) - else: - proj_cov = np.matmul(eigvects.transpose(), np.matmul(self.C, eigvects)) - - # outlier scores are computed in the principal component space - coefs = (1. / (n + np.arange(n_batch) + 1.)).reshape((n_batch, 1, 1)) - B = coefs * np.matmul((proj_x_clip - proj_means)[:, :, None], (proj_x_clip - proj_means)[:, None, :]) - all_C_inv = np.zeros_like(B) - c_inv = None - for i, b in enumerate(B): - if c_inv is None: - if abs(np.linalg.det(proj_cov)) > EPSILON: - c_inv = np.linalg.inv(proj_cov) - all_C_inv[i] = c_inv - continue - else: - if n + i == 0: - continue - proj_cov = (n + i - 1.) / (n + i) * proj_cov + b - continue - else: - c_inv = (n + i - 1.) / float(n + i - 2.) * all_C_inv[i - 1] - BC1 = np.matmul(B[i - 1], c_inv) - all_C_inv[i] = c_inv - 1. / (1. + np.trace(BC1)) * np.matmul(c_inv, BC1) - - # update parameters - self.mean = new_means[-1] - self.C = cov_batch - stdev = np.sqrt(np.diag(cov_batch)) - self.n += n_batch - if self.n > self.start_clip: - self.clip = [self.mean - self.std_clip * stdev, self.mean + self.std_clip * stdev] - - # compute outlier scores - x_diff = proj_x - proj_means - outlier_score = np.matmul(x_diff[:, None, :], np.matmul(all_C_inv, x_diff[:, :, None])).reshape(n_batch) - return outlier_score - - def predict(self, - X: np.ndarray, - return_instance_score: bool = True) \ - -> Dict[Dict[str, str], Dict[np.ndarray, np.ndarray]]: - """ - Compute outlier scores and transform into outlier predictions. - - Parameters - ---------- - X - Batch of instances. - return_instance_score - Whether to return instance level outlier scores. - - Returns - ------- - Dictionary containing 'meta' and 'data' dictionaries. - 'meta' has the model's metadata. - 'data' contains the outlier predictions and instance level outlier scores. - """ - # convert categorical variables to numerical values - X = self.cat2num(X) - - # compute outlier scores - iscore = self.score(X) - - # values above threshold are outliers - outlier_pred = (iscore > self.threshold).astype(int) - - # populate output dict - od = outlier_prediction_dict() - od['meta'] = self.meta - od['data']['is_outlier'] = outlier_pred - if return_instance_score: - od['data']['instance_score'] = iscore - return od diff --git a/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py similarity index 97% rename from alibi_detect/od/tests/test_mahalanobis/test_mahalanobis.py rename to alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index e45061601..e4dc3407b 100644 --- a/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -2,7 +2,7 @@ import numpy as np import torch -from alibi_detect.od.mahalanobis import Mahalanobis +from alibi_detect.od._mahalanobis import Mahalanobis from alibi_detect.od.base import NotFitException from sklearn.datasets import make_moons diff --git a/alibi_detect/od/tests/test_mahalanobis/test_mahalanobis_backend.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py similarity index 100% rename from alibi_detect/od/tests/test_mahalanobis/test_mahalanobis_backend.py rename to alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py diff --git a/alibi_detect/od/tests/test_stateful_mahalanobis.py b/alibi_detect/od/tests/test_mahalanobis.py similarity index 100% rename from alibi_detect/od/tests/test_stateful_mahalanobis.py rename to alibi_detect/od/tests/test_mahalanobis.py From 66a58265d83a1afd7fb09a4408d827cbbb462f5a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 20 Jan 2023 14:13:34 +0000 Subject: [PATCH 075/247] Rename Mahalanobis detector symbol --- alibi_detect/od/_mahalanobis.py | 2 +- .../od/tests/test__mahalanobis/test__mahalanobis.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 7df808960..48bdb7418 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -21,7 +21,7 @@ } -class Mahalanobis(OutlierDetector): +class _Mahalanobis(OutlierDetector): def __init__( self, min_eigenvalue: float = 1e-6, diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index e4dc3407b..4310b7525 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -2,13 +2,13 @@ import numpy as np import torch -from alibi_detect.od._mahalanobis import Mahalanobis +from alibi_detect.od._mahalanobis import _Mahalanobis from alibi_detect.od.base import NotFitException from sklearn.datasets import make_moons def make_mahalanobis_detector(): - mahalanobis_detector = Mahalanobis() + mahalanobis_detector = _Mahalanobis() x_ref = np.random.randn(100, 2) mahalanobis_detector.fit(x_ref) mahalanobis_detector.infer_threshold(x_ref, 0.1) @@ -16,7 +16,7 @@ def make_mahalanobis_detector(): def test_unfitted_mahalanobis_single_score(): - mahalanobis_detector = Mahalanobis() + mahalanobis_detector = _Mahalanobis() x = np.array([[0, 10], [0.1, 0]]) with pytest.raises(NotFitException) as err: _ = mahalanobis_detector.predict(x) @@ -24,7 +24,7 @@ def test_unfitted_mahalanobis_single_score(): def test_fitted_mahalanobis_single_score(): - mahalanobis_detector = Mahalanobis() + mahalanobis_detector = _Mahalanobis() x_ref = np.random.randn(100, 2) mahalanobis_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) @@ -54,7 +54,7 @@ def test_fitted_mahalanobis_predict(): def test_mahalanobis_integration(): - mahalanobis_detector = Mahalanobis() + mahalanobis_detector = _Mahalanobis() X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] mahalanobis_detector.fit(X_ref) From 352e7886c25899fe5a77fa14235e1d9a5da2007e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 20 Jan 2023 17:32:02 +0000 Subject: [PATCH 076/247] Add pca detector --- alibi_detect/od/_pca.py | 151 +++++++++++++++ alibi_detect/od/pytorch/pca.py | 174 ++++++++++++++++++ alibi_detect/od/tests/test__pca/test__pca.py | 112 +++++++++++ .../od/tests/test__pca/test__pca_backend.py | 111 +++++++++++ 4 files changed, 548 insertions(+) create mode 100644 alibi_detect/od/_pca.py create mode 100644 alibi_detect/od/pytorch/pca.py create mode 100644 alibi_detect/od/tests/test__pca/test__pca.py create mode 100644 alibi_detect/od/tests/test__pca/test__pca_backend.py diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py new file mode 100644 index 000000000..8a4fc5e85 --- /dev/null +++ b/alibi_detect/od/_pca.py @@ -0,0 +1,151 @@ +from typing import Union, Optional, Callable, Dict, Any +from typing import TYPE_CHECKING + +import numpy as np + +from alibi_detect.utils._types import Literal +from alibi_detect.base import outlier_prediction_dict +from alibi_detect.od.base import OutlierDetector +from alibi_detect.od.pytorch.base import to_numpy +from alibi_detect.od.pytorch.pca import KernelPCATorch, LinearPCATorch +from alibi_detect.utils.frameworks import BackendValidator +from alibi_detect.version import __version__ + + +if TYPE_CHECKING: + import torch + + +backends = { + 'pytorch': (KernelPCATorch, LinearPCATorch) +} + + +class _PCA(OutlierDetector): + def __init__( + self, + n_components: int, + kernel: Optional[Callable] = None, + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, + backend: Literal['pytorch'] = 'pytorch', + ) -> None: + """ + + Parameters + ---------- + min_eigenvalue + Eigenvectors with eigenvalues below this value will be discarded. + backend + Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. + kernel + Kernel function to use for outlier detection. If ``None``, linear PCA is used instead of the + kernel variant. + device + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + + Raises + ------ + NotImplementedError + If choice of `backend` is not implemented. + """ + super().__init__() + + backend_str: str = backend.lower() + BackendValidator( + backend_options={'pytorch': ['pytorch']}, + construct_name=self.__class__.__name__ + ).verify_backend(backend_str) + + kernel_backend_cls, linear_backend_cls = backends[backend] + + self.backend: Union[KernelPCATorch, LinearPCATorch] + if kernel is not None: + self.backend = kernel_backend_cls( + n_components=n_components, + device=device, + kernel=kernel + ) + else: + self.backend = linear_backend_cls( + n_components=n_components, + device=device, + ) + + def fit(self, x_ref: np.ndarray) -> None: + """Fit the detector on reference data. + + ... + + Parameters + ---------- + x_ref + Reference data used to fit the detector. + """ + self.backend.fit(self.backend._to_tensor(x_ref)) + + def score(self, x: np.ndarray) -> np.ndarray: + """Score `x` instances using the detector. + + ... + + Parameters + ---------- + x + Data to score. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + instance. + """ + score = self.backend.score(self.backend._to_tensor(x)) + return to_numpy(score) + + def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + """Infer the threshold for the Mahalanobis detector. + + ... + + Parameters + ---------- + x_ref + Reference data used to infer the threshold. + fpr + False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ + `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ + ``(0, 1)``. + """ + self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + + def predict(self, x: np.ndarray) -> Dict[str, Any]: + """Predict whether the instances in `x` are outliers or not. + + ... + + Parameters + ---------- + x + Data to predict. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ + performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ + the detector. + """ + outputs = self.backend.predict(self.backend._to_tensor(x)) + output = outlier_prediction_dict() + output['data'] = { + **output['data'], + **to_numpy(outputs) + } + output['meta'] = { + **output['meta'], + 'name': self.__class__.__name__, + 'detector_type': 'outlier', + 'online': False, + 'version': __version__, + } + return output diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py new file mode 100644 index 000000000..66e128cf8 --- /dev/null +++ b/alibi_detect/od/pytorch/pca.py @@ -0,0 +1,174 @@ +from typing import Optional, Union, Callable + +import torch + +from alibi_detect.od.pytorch.base import TorchOutlierDetector + + +class PCATorch(TorchOutlierDetector): + def __init__( + self, + n_components: int, + device: Optional[Union[str, torch.device]] = None + ): + """PyTorch backend for PCA detector. + + Parameters + ---------- + n_components: + The number of dimensions in the principle subspace. For linear pca should have + 1 <= n_components < dim(data). For kernel pca should have 1 <= n_components < len(data). + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + """ + TorchOutlierDetector.__init__(self, device=device) + self.accumulator = None + self.n_components = n_components + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Detect if `x` is an outlier. + + Parameters + ---------- + x + `torch.Tensor` with leading batch dimension. + + Returns + ------- + `torch.Tensor` of ``bool`` values with leading batch dimension. + + Raises + ------ + ThresholdNotInferredException + If called before detector has had `infer_threshold` method called. + """ + raw_scores = self.score(x) + scores = self._accumulator(raw_scores) + if not torch.jit.is_scripting(): + self.check_threshold_infered() + preds = scores > self.threshold + return preds.cpu() + + def score(self, x: torch.Tensor) -> torch.Tensor: + """Score test instance `x`. + + Parameters + ---------- + x + The tensor of instances. First dimension corresponds to batch. + + Returns + ------- + Tensor of scores for each element in `x`. + + Raises + ------ + NotFitException + If called before detector has been fit. + """ + if not torch.jit.is_scripting(): + self.check_fitted() + score = self._compute_score(x) + return score.cpu() + + def _fit(self, x_ref: torch.Tensor) -> None: + """Fits the pca detector. + + Parameters + ---------- + x_ref + The Dataset tensor. + """ + self.x_ref_mean = x_ref.mean(0) + self.pcs = self._compute_pcs(x_ref) + self.x_ref = x_ref + + def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + raise NotImplementedError + + def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + raise NotImplementedError + + +class LinearPCATorch(PCATorch): + def __init__( + self, + n_components: int, + device: Optional[Union[str, torch.device]] = None + ): + """Linear variant of the PyTorch backend for PCA detector. + + Parameters + ---------- + n_components: + The number of dimensions in the principle subspace. For linear pca should have + 1 <= n_components < dim(data). For kernel pca should have 1 <= n_components < len(data). + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + """ + PCATorch.__init__(self, device=device, n_components=n_components) + + def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + x -= self.x_ref_mean + cov_mat = (x.t() @ x)/(len(x)-1) + _, V = torch.linalg.eigh(cov_mat) + return V[:, :-self.n_components] + + def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + x_cen = x - self.x_ref_mean + x_pcs = x_cen @ self.pcs + return (x_pcs**2).sum(1) + + +class KernelPCATorch(PCATorch): + def __init__( + self, + n_components: int, + kernel: Optional[Callable], + device: Optional[Union[str, torch.device]] = None + ): + """Kernel variant of the PyTorch backend for PCA detector. + + Parameters + ---------- + n_components: + The number of dimensions in the principle subspace. For linear pca should have + 1 <= n_components < dim(data). For kernel pca should have 1 <= n_components < len(data). + kernel + Kernel function to use for outlier detection. If ``None``, linear PCA is used instead of the + kernel variant. + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + """ + PCATorch.__init__(self, device=device, n_components=n_components) + self.kernel = kernel + + def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + K = self._compute_kernel_cov_mat(x) + D, V = torch.linalg.eigh(K) + pcs = V / torch.sqrt(D)[None, :] + return pcs[:, -self.n_components:] + + def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + k_xr = self.kernel(x, self.x_ref) + # Now to center + k_xr_row_sums = k_xr.sum(1) + _, n = k_xr.shape + k_xr_cen = k_xr - self.k_col_sums[None, :]/n - k_xr_row_sums[:, None]/n + self.k_sum/(n**2) + x_pcs = k_xr_cen @ self.pcs + scores = -2 * k_xr.mean(-1) - (x_pcs**2).sum(1) + return scores + + def _compute_kernel_cov_mat(self, x: torch.Tensor) -> torch.Tensor: + n = len(x) + # Uncentered kernel matrix + k = self.kernel(x, x) + # Now to center + self.k_col_sums = k.sum(0) + k_row_sums = k.sum(1) + self.k_sum = k_row_sums.sum() + k_cen = k - self.k_col_sums[None, :]/n - k_row_sums[:, None]/n + self.k_sum/(n**2) + return k_cen diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py new file mode 100644 index 000000000..c03d60df8 --- /dev/null +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -0,0 +1,112 @@ +import pytest +import numpy as np +# import torch + +from alibi_detect.utils.pytorch.kernels import GaussianRBF +from alibi_detect.od._pca import _PCA +from alibi_detect.od.base import NotFitException +# from sklearn.datasets import make_moons + + +def make_PCA_detector(kernel=False): + if kernel: + pca_detector = _PCA(n_components=2, kernel=GaussianRBF()) + else: + pca_detector = _PCA(n_components=2) + x_ref = np.random.randn(100, 3) + pca_detector.fit(x_ref) + pca_detector.infer_threshold(x_ref, 0.1) + return pca_detector + + +@pytest.mark.parametrize('detector', [ + lambda: _PCA(n_components=5), + lambda: _PCA(n_components=5, kernel=GaussianRBF()) +]) +def test_unfitted_PCA_single_score(detector): + pca = detector() + x = np.array([[0, 10, 0], [0.1, 0, 0]]) + with pytest.raises(NotFitException) as err: + _ = pca.predict(x) + assert str(err.value) == \ + f'{pca.backend.__class__.__name__} has not been fit!' + + +def test_fitted_PCA_single_score(): + pca_detector = _PCA(n_components=2) + x_ref = np.random.randn(100, 3) + pca_detector.fit(x_ref) + x = np.array([[0, 10, 0], [0.1, 0, 0]]) + y = pca_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None + + +def test_fitted_kernel_PCA_single_score(): + pca_detector = _PCA(n_components=2, kernel=GaussianRBF()) + x_ref = np.random.randn(100, 3) * np.array([1, 10, 0.1]) + pca_detector.fit(x_ref) + x = np.array([[0, 5, 10], [0.1, 5, 0]]) + y = pca_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > y['instance_score'][1] + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None + + +def test_fitted_PCA_predict(): + pca_detector = make_PCA_detector() + x_ref = np.random.randn(100, 3) + pca_detector.infer_threshold(x_ref, 0.1) + x = np.array([[0, 10, 0], [0.1, 0, 0]]) + y = pca_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() + + +def test_fitted_kernel_PCA_predict(): + pca_detector = _PCA(n_components=2, kernel=GaussianRBF()) + x_ref = np.random.randn(100, 3) * np.array([1, 10, 0.1]) + pca_detector.fit(x_ref) + pca_detector.infer_threshold(x_ref, 0.1) + x = np.array([[0, 5, 10], [0.1, 5, 0]]) + y = pca_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > y['instance_score'][1] + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() + + +# def test_PCA_integration(): +# pca_detector = _PCA(n_components=2) +# X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) +# X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] +# pca_detector.fit(X_ref) +# pca_detector.infer_threshold(X_ref, 0.1) +# result = pca_detector.predict(x_inlier) +# result = result['data']['is_outlier'][0] +# assert not result + +# x_outlier = np.array([[-1, 1.5]]) +# result = pca_detector.predict(x_outlier) +# result = result['data']['is_outlier'][0] +# assert result + +# ts_PCA = torch.jit.script(pca_detector.backend) +# x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) +# y = ts_PCA(x) +# assert torch.all(y == torch.tensor([False, True])) diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py new file mode 100644 index 000000000..ccb8c1f56 --- /dev/null +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -0,0 +1,111 @@ +import pytest +import torch +import numpy as np + +from alibi_detect.utils.pytorch.kernels import GaussianRBF +from alibi_detect.od.pytorch.pca import LinearPCATorch, KernelPCATorch +from alibi_detect.od.base import NotFitException, ThresholdNotInferredException + + +@pytest.mark.parametrize('backend_detector', [ + lambda: LinearPCATorch(n_components=5), + lambda: KernelPCATorch(n_components=5, kernel=GaussianRBF()) +]) +def test_pca_torch_backend_fit_errors(backend_detector): + pca_torch = backend_detector() + assert not pca_torch._fitted + + x = torch.randn((1, 10)) + with pytest.raises(NotFitException) as err: + pca_torch(x) + assert str(err.value) == f'{pca_torch.__class__.__name__} has not been fit!' + + with pytest.raises(NotFitException) as err: + pca_torch.predict(x) + assert str(err.value) == f'{pca_torch.__class__.__name__} has not been fit!' + + x_ref = torch.randn((1024, 10)) + pca_torch.fit(x_ref) + + assert pca_torch._fitted + + with pytest.raises(ThresholdNotInferredException) as err: + pca_torch(x) + assert str(err.value) == (f'{pca_torch.__class__.__name__} has no threshold set, call' + ' `infer_threshold` before predicting.') + + assert pca_torch.predict(x) + + +@pytest.mark.parametrize('backend_detector', [ + lambda: LinearPCATorch(n_components=1), + lambda: KernelPCATorch(n_components=1, kernel=GaussianRBF()) +]) +def test_pca_linear_scoring(backend_detector): + pca_torch = backend_detector() + # pca_torch = LinearPCATorch(n_components=1) + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + pca_torch.fit(x_ref) + + x_1 = torch.tensor([[8., 8.]], dtype=torch.float64) + scores_1 = pca_torch.score(x_1) + + x_2 = torch.tensor(np.random.multivariate_normal(mean, cov, 1), dtype=torch.float64) + scores_2 = pca_torch.score(x_2) + + x_3 = torch.tensor([[-20., 20.]], dtype=torch.float64) + scores_3 = pca_torch.score(x_3) + + # test correct ordering of scores given outlyingness of data + assert scores_1 < scores_2 < scores_3 + + # test that detector correctly detects true Outlier + pca_torch.infer_threshold(x_ref, 0.01) + x = torch.cat((x_1, x_2, x_3)) + outputs = pca_torch.predict(x) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) + assert torch.all(pca_torch(x) == torch.tensor([False, False, True])) + + # test that 0.01 of the in distribution data is flagged as outliers + x = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + outputs = pca_torch.predict(x) + assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.005 + + +def test_pca_linear_torch_backend_ts(tmp_path): + pca_torch = LinearPCATorch(n_components=5) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + x_ref = torch.randn((1024, 10)) + pca_torch.fit(x_ref) + pca_torch.infer_threshold(x_ref, 0.1) + pred_1 = pca_torch(x) + + pca_torch = torch.jit.script(pca_torch) + pred_2 = pca_torch(x) + assert torch.all(pred_1 == pred_2) + + pca_torch.save(tmp_path / 'pca_torch.pt') + pca_torch = torch.load(tmp_path / 'pca_torch.pt') + pred_2 = pca_torch(x) + assert torch.all(pred_1 == pred_2) + + +@pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript supporrt yet.') +def test_pca_kernel_torch_backend_ts(tmp_path): + pca_torch = KernelPCATorch(n_components=5, kernel=GaussianRBF()) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + x_ref = torch.randn((1024, 10)) + pca_torch.fit(x_ref) + pca_torch.infer_threshold(x_ref, 0.1) + pred_1 = pca_torch(x) + + pca_torch = torch.jit.script(pca_torch) + pred_2 = pca_torch(x) + assert torch.all(pred_1 == pred_2) + + pca_torch.save(tmp_path / 'pca_torch.pt') + pca_torch = torch.load(tmp_path / 'pca_torch.pt') + pred_2 = pca_torch(x) + assert torch.all(pred_1 == pred_2) From 8c5668f55fda566d44c66ab0409f23510fe1e874 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 23 Jan 2023 11:00:17 +0000 Subject: [PATCH 077/247] Add make_moons integration tests --- alibi_detect/od/tests/test__pca/test__pca.py | 80 +++++++++++++++----- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index c03d60df8..49811ddcf 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -1,11 +1,11 @@ import pytest import numpy as np -# import torch +import torch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od._pca import _PCA from alibi_detect.od.base import NotFitException -# from sklearn.datasets import make_moons +from sklearn.datasets import make_moons def make_PCA_detector(kernel=False): @@ -91,22 +91,60 @@ def test_fitted_kernel_PCA_predict(): assert (y['is_outlier'] == [True, False]).all() -# def test_PCA_integration(): -# pca_detector = _PCA(n_components=2) -# X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) -# X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] -# pca_detector.fit(X_ref) -# pca_detector.infer_threshold(X_ref, 0.1) -# result = pca_detector.predict(x_inlier) -# result = result['data']['is_outlier'][0] -# assert not result - -# x_outlier = np.array([[-1, 1.5]]) -# result = pca_detector.predict(x_outlier) -# result = result['data']['is_outlier'][0] -# assert result - -# ts_PCA = torch.jit.script(pca_detector.backend) -# x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) -# y = ts_PCA(x) -# assert torch.all(y == torch.tensor([False, True])) +def test_PCA_integration(): + pca_detector = _PCA(n_components=1) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + pca_detector.fit(X_ref) + pca_detector.infer_threshold(X_ref, 0.1) + result = pca_detector.predict(x_inlier) + result = result['data']['is_outlier'][0] + assert not result + + x_outlier = np.array([[0, -3]]) + result = pca_detector.predict(x_outlier) + result = result['data']['is_outlier'][0] + assert result + + +def test_PCA_integration_ts(): + pca_detector = _PCA(n_components=1) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + pca_detector.fit(X_ref) + pca_detector.infer_threshold(X_ref, 0.1) + x_outlier = np.array([[0, -3]]) + ts_PCA = torch.jit.script(pca_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = ts_PCA(x) + assert torch.all(y == torch.tensor([False, True])) + + +def test_kernel_PCA_integration(): + pca_detector = _PCA(n_components=10, kernel=GaussianRBF()) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + pca_detector.fit(X_ref) + pca_detector.infer_threshold(X_ref, 0.1) + result = pca_detector.predict(x_inlier) + result = result['data']['is_outlier'][0] + assert not result + + x_outlier = np.array([[1, 1]]) + result = pca_detector.predict(x_outlier) + result = result['data']['is_outlier'][0] + assert result + + +@pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript supporrt yet.') +def test_kernel_PCA_integration_ts(): + pca_detector = _PCA(n_components=10, kernel=GaussianRBF()) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + pca_detector.fit(X_ref) + pca_detector.infer_threshold(X_ref, 0.1) + x_outlier = np.array([[1, 1]]) + ts_PCA = torch.jit.script(pca_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = ts_PCA(x) + assert torch.all(y == torch.tensor([False, True])) From 4161dffea950ea5ae0dd346e2218d2750fdb0dd4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 23 Jan 2023 11:08:18 +0000 Subject: [PATCH 078/247] Add optional dependency functionality --- alibi_detect/od/_pca.py | 2 +- alibi_detect/od/pytorch/__init__.py | 1 + alibi_detect/tests/test_dep_management.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 8a4fc5e85..a0ea213d7 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -7,7 +7,7 @@ from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector from alibi_detect.od.pytorch.base import to_numpy -from alibi_detect.od.pytorch.pca import KernelPCATorch, LinearPCATorch +from alibi_detect.od.pytorch import KernelPCATorch, LinearPCATorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py index 3ce415a4b..5dc803a1a 100644 --- a/alibi_detect/od/pytorch/__init__.py +++ b/alibi_detect/od/pytorch/__init__.py @@ -2,6 +2,7 @@ KNNTorch = import_optional('alibi_detect.od.pytorch.knn', ['KNNTorch']) MahalanobisTorch = import_optional('alibi_detect.od.pytorch.mahalanobis', ['MahalanobisTorch']) +KernelPCATorch, LinearPCATorch = import_optional('alibi_detect.od.pytorch.pca', ['KernelPCATorch', 'LinearPCATorch']) Accumulator = import_optional('alibi_detect.od.pytorch.ensemble', ['Accumulator']) to_numpy = import_optional('alibi_detect.od.pytorch.base', ['to_numpy']) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index 021b2952c..b11baf392 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -139,6 +139,8 @@ def test_od_backend_dependencies(opt_dep): ('Accumulator', ['torch', 'keops']), ('KNNTorch', ['torch', 'keops']), ('MahalanobisTorch', ['torch', 'keops']), + ('KernelPCATorch', ['torch', 'keops']), + ('LinearPCATorch', ['torch', 'keops']), ('to_numpy', ['torch', 'keops']), ]: dependency_map[dependency] = relations From f357bbea0880e8d0d1683f853aeb8f4fbac3e223 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 23 Jan 2023 17:04:40 +0000 Subject: [PATCH 079/247] Add docstrings for pca outlier detector --- alibi_detect/od/_pca.py | 27 ++++-- alibi_detect/od/pytorch/pca.py | 88 ++++++++++++++++--- alibi_detect/od/tests/test__pca/test__pca.py | 2 +- .../od/tests/test__pca/test__pca_backend.py | 3 +- 4 files changed, 98 insertions(+), 22 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index a0ea213d7..e724a4318 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -29,12 +29,23 @@ def __init__( device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', ) -> None: - """ + """Principal Component Analysis (PCA) outlier detector. + + The detector is based on the Principal Component Analysis (PCA) algorithm. There are two variants of PCA: + linear PCA and kernel PCA. Linear PCA computes the eigenvectors of the covariance matrix of the data. Kernel + PCA computes the eigenvectors of the kernel matrix of the data. In each case, we choose the smallest + `n_components` eigenvectors. We do this as they correspond to the invariant directions of the data. i.e the + directions along which the data is least spread out. Thus a point that deviates along these dimensions is more + likely to be an outlier. + + When scoring a test instance we project it onto the eigenvectors and compute its score using the L2 norm. If + a threshold is fitted we use this to determine whether the instance is an outlier or not. Parameters ---------- - min_eigenvalue - Eigenvectors with eigenvalues below this value will be discarded. + n_components: + The number of dimensions in the principle subspace. For linear pca should have + ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. backend Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. kernel @@ -75,7 +86,8 @@ def __init__( def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. - ... + Compute the eigenvectors of the covariance/kernel matrix of `x_ref` and save the smallest `n_components` + eigenvectors. Parameters ---------- @@ -87,7 +99,7 @@ def fit(self, x_ref: np.ndarray) -> None: def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. - ... + Project `x` onto the eigenvectors and compute its score using the L2 norm. Parameters ---------- @@ -105,7 +117,8 @@ def score(self, x: np.ndarray) -> np.ndarray: def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. - ... + The threshold is set such that the false positive rate of the detector on the reference data is `fpr`. + Parameters ---------- @@ -121,8 +134,6 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. - ... - Parameters ---------- x diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 66e128cf8..16f988acf 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -16,8 +16,8 @@ def __init__( Parameters ---------- n_components: - The number of dimensions in the principle subspace. For linear pca should have - 1 <= n_components < dim(data). For kernel pca should have 1 <= n_components < len(data). + The number of dimensions in the principle subspace. For linear PCA should have + ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. @@ -73,7 +73,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: return score.cpu() def _fit(self, x_ref: torch.Tensor) -> None: - """Fits the pca detector. + """Fits the PCA detector. Parameters ---------- @@ -102,8 +102,7 @@ def __init__( Parameters ---------- n_components: - The number of dimensions in the principle subspace. For linear pca should have - 1 <= n_components < dim(data). For kernel pca should have 1 <= n_components < len(data). + The number of dimensions in the principle subspace. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. @@ -111,12 +110,41 @@ def __init__( PCATorch.__init__(self, device=device, n_components=n_components) def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + """Compute the principle components of the reference data. + + We compute the principle components of the reference data using the covariance matrix and then + return the last `n_components` of the eigenvectors. These correspond to the invariant dimensions + of the data. Changes in these dimensions are used to compute the outlier score. + + Parameters + ---------- + x + The reference data. + + Returns + ------- + The principle components of the reference data. + """ x -= self.x_ref_mean cov_mat = (x.t() @ x)/(len(x)-1) _, V = torch.linalg.eigh(cov_mat) return V[:, :-self.n_components] def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + """Compute the outlier score. + + Centers the data and projects it onto the principle components. The score is then the sum of the + squared projections. + + Parameters + ---------- + x + The test data. + + Returns + ------- + The outlier score. + """ x_cen = x - self.x_ref_mean x_pcs = x_cen @ self.pcs return (x_pcs**2).sum(1) @@ -134,11 +162,9 @@ def __init__( Parameters ---------- n_components: - The number of dimensions in the principle subspace. For linear pca should have - 1 <= n_components < dim(data). For kernel pca should have 1 <= n_components < len(data). + The number of dimensions in the principle subspace. kernel - Kernel function to use for outlier detection. If ``None``, linear PCA is used instead of the - kernel variant. + Kernel function to use for outlier detection. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. @@ -147,12 +173,41 @@ def __init__( self.kernel = kernel def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: - K = self._compute_kernel_cov_mat(x) + """Compute the principle components of the reference data. + + We compute the principle components of the reference data using the kernel matrix and then + return the last `n_components` of the eigenvectors. These correspond to the invariant dimensions + of the data. Changes in these dimensions are used to compute the outlier score. + + Parameters + ---------- + x + The reference data. + + Returns + ------- + The principle components of the reference data. + """ + K = self._compute_kernel_mat(x) D, V = torch.linalg.eigh(K) pcs = V / torch.sqrt(D)[None, :] return pcs[:, -self.n_components:] def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + """Compute the outlier score. + + Centers the data and projects it onto the principle components. The score is then the sum of the + squared projections. + + Parameters + ---------- + x + The test data. + + Returns + ------- + The outlier score. + """ k_xr = self.kernel(x, self.x_ref) # Now to center k_xr_row_sums = k_xr.sum(1) @@ -162,7 +217,18 @@ def _compute_score(self, x: torch.Tensor) -> torch.Tensor: scores = -2 * k_xr.mean(-1) - (x_pcs**2).sum(1) return scores - def _compute_kernel_cov_mat(self, x: torch.Tensor) -> torch.Tensor: + def _compute_kernel_mat(self, x: torch.Tensor) -> torch.Tensor: + """Computes the centered kernel matrix. + + Parameters + ---------- + x + The reference data. + + Returns + ------- + The centered kernel matrix. + """ n = len(x) # Uncentered kernel matrix k = self.kernel(x, x) diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index 49811ddcf..fd588dd0b 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -136,7 +136,7 @@ def test_kernel_PCA_integration(): assert result -@pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript supporrt yet.') +@pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript support yet.') def test_kernel_PCA_integration_ts(): pca_detector = _PCA(n_components=10, kernel=GaussianRBF()) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py index ccb8c1f56..953d65aa1 100644 --- a/alibi_detect/od/tests/test__pca/test__pca_backend.py +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -43,7 +43,6 @@ def test_pca_torch_backend_fit_errors(backend_detector): ]) def test_pca_linear_scoring(backend_detector): pca_torch = backend_detector() - # pca_torch = LinearPCATorch(n_components=1) mean = [8, 8] cov = [[2., 0.], [0., 1.]] x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) @@ -92,7 +91,7 @@ def test_pca_linear_torch_backend_ts(tmp_path): assert torch.all(pred_1 == pred_2) -@pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript supporrt yet.') +@pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript support yet.') def test_pca_kernel_torch_backend_ts(tmp_path): pca_torch = KernelPCATorch(n_components=5, kernel=GaussianRBF()) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) From 40a0b2777523000a9a4d3734b1f131594bf6e457 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 24 Jan 2023 10:36:36 +0000 Subject: [PATCH 080/247] Fix minor PR suggested changes --- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/pytorch/knn.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 8c1dccab1..e7e1112c3 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -241,7 +241,7 @@ def __init__(self, weights: Optional[torch.Tensor] = None): self.weights = weights def _transform(self, scores: torch.Tensor) -> torch.Tensor: - """Averages the scores of the detectors in an ensemble. If weights where passed in the `__init__` + """Averages the scores of the detectors in an ensemble. If weights were passed in the `__init__` then these are used to weight the scores. ---------- scores diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index b718b49d2..cfe7e9ac6 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -25,8 +25,8 @@ def __init__( similarity to the `k`-th nearest neighbor. If `k` is a list then it returns the distance/kernel similarity to each of the specified `k` neighbors. kernel - If a kernel is specified then instead of using `torch.cdist` we compute the kernel similarity - while computing the k nearest neighbor distance. + If a kernel is specified then instead of using `torch.cdist` the kernel defines the k nearest + neighbor distance. accumulator If `k` is an array of integers then the accumulator must not be ``None``. Should be an instance of :py:obj:`alibi_detect.od.pytorch.ensemble.Accumulator`. Responsible for combining From a831642adc6626a19c076a1f4b9f2c6b962ce1fa Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 24 Jan 2023 11:31:28 +0000 Subject: [PATCH 081/247] Make knn object private --- alibi_detect/od/__init__.py | 2 -- alibi_detect/od/{knn.py => _knn.py} | 2 +- .../test_knn.py => test__knn/test__knn.py} | 20 +++++++++---------- .../test__knn_backend.py} | 0 4 files changed, 11 insertions(+), 13 deletions(-) rename alibi_detect/od/{knn.py => _knn.py} (99%) rename alibi_detect/od/tests/{test_knn/test_knn.py => test__knn/test__knn.py} (95%) rename alibi_detect/od/tests/{test_knn/test_knn_backend.py => test__knn/test__knn_backend.py} (100%) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index 45ccfef51..367a19c9d 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -54,7 +54,6 @@ def get_aggregator(aggregator: Union[TransformProtocol, aggregator_literals]) -> OutlierSeq2Seq = import_optional('alibi_detect.od.seq2seq', names=['OutlierSeq2Seq']) LLR = import_optional('alibi_detect.od.llr', names=['LLR']) OutlierProphet = import_optional('alibi_detect.od.prophet', names=['OutlierProphet']) -KNN = import_optional('alibi_detect.od.knn', names=['KNN']) __all__ = [ "OutlierAEGMM", @@ -67,7 +66,6 @@ def get_aggregator(aggregator: Union[TransformProtocol, aggregator_literals]) -> "SpectralResidual", "LLR", "OutlierProphet" - "KNN" "PValNormalizer", "ShiftAndScaleNormalizer", "TopKAggregator", diff --git a/alibi_detect/od/knn.py b/alibi_detect/od/_knn.py similarity index 99% rename from alibi_detect/od/knn.py rename to alibi_detect/od/_knn.py index f1ace2361..7fc8b8430 100644 --- a/alibi_detect/od/knn.py +++ b/alibi_detect/od/_knn.py @@ -21,7 +21,7 @@ } -class KNN(OutlierDetector): +class _KNN(OutlierDetector): def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], diff --git a/alibi_detect/od/tests/test_knn/test_knn.py b/alibi_detect/od/tests/test__knn/test__knn.py similarity index 95% rename from alibi_detect/od/tests/test_knn/test_knn.py rename to alibi_detect/od/tests/test__knn/test__knn.py index 26bb6905d..bb165ec7e 100644 --- a/alibi_detect/od/tests/test_knn/test_knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -2,7 +2,7 @@ import numpy as np import torch -from alibi_detect.od.knn import KNN +from alibi_detect.od._knn import _KNN from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer from alibi_detect.od.base import NotFitException @@ -11,7 +11,7 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): - knn_detector = KNN( + knn_detector = _KNN( k=k, aggregator=aggregator, normalizer=normalizer ) @@ -22,7 +22,7 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): def test_unfitted_knn_single_score(): - knn_detector = KNN(k=10) + knn_detector = _KNN(k=10) x = np.array([[0, 10], [0.1, 0]]) with pytest.raises(NotFitException) as err: _ = knn_detector.predict(x) @@ -30,7 +30,7 @@ def test_unfitted_knn_single_score(): def test_fitted_knn_single_score(): - knn_detector = KNN(k=10) + knn_detector = _KNN(k=10) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) @@ -46,7 +46,7 @@ def test_fitted_knn_single_score(): def test_default_knn_ensemble_init(): - knn_detector = KNN(k=[8, 9, 10]) + knn_detector = _KNN(k=[8, 9, 10]) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) @@ -63,7 +63,7 @@ def test_default_knn_ensemble_init(): def test_incorrect_knn_ensemble_init(): with pytest.raises(ValueError) as err: - KNN(k=[8, 9, 10], aggregator=None) + _KNN(k=[8, 9, 10], aggregator=None) assert str(err.value) == ('If `k` is a `np.ndarray`, `list` or `tuple`, ' 'the `aggregator` argument cannot be ``None``.') @@ -87,7 +87,7 @@ def test_fitted_knn_predict(): MaxAggregator, MinAggregator]) @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_unfitted_knn_ensemble(aggregator, normalizer): - knn_detector = KNN( + knn_detector = _KNN( k=[8, 9, 10], aggregator=aggregator(), normalizer=normalizer() @@ -102,7 +102,7 @@ def test_unfitted_knn_ensemble(aggregator, normalizer): MaxAggregator, MinAggregator]) @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_fitted_knn_ensemble(aggregator, normalizer): - knn_detector = KNN( + knn_detector = _KNN( k=[8, 9, 10], aggregator=aggregator(), normalizer=normalizer() @@ -163,7 +163,7 @@ def test_knn_single_torchscript(): @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None, lambda: 'ShiftAndScaleNormalizer', lambda: 'PValNormalizer']) def test_knn_ensemble_integration(aggregator, normalizer): - knn_detector = KNN( + knn_detector = _KNN( k=[10, 14, 18], aggregator=aggregator(), normalizer=normalizer() @@ -188,7 +188,7 @@ def test_knn_ensemble_integration(aggregator, normalizer): def test_knn_integration(): - knn_detector = KNN(k=18) + knn_detector = _KNN(k=18) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] knn_detector.fit(X_ref) diff --git a/alibi_detect/od/tests/test_knn/test_knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py similarity index 100% rename from alibi_detect/od/tests/test_knn/test_knn_backend.py rename to alibi_detect/od/tests/test__knn/test__knn_backend.py From aa334bd186f311132682cfacb0a944015aa39a11 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 24 Jan 2023 14:36:11 +0000 Subject: [PATCH 082/247] Improve the kNN detector docstrings --- alibi_detect/od/_knn.py | 46 ++++++++++++++++++++++++++-------- alibi_detect/od/pytorch/knn.py | 2 +- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 7fc8b8430..de71c75f2 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -34,22 +34,38 @@ def __init__( """ k-Nearest Neighbours (kNN) outlier detector. + The kNN detector is a non-parametric method for outlier detection. The detector computes the distance + between each test point and its `k` nearest neighbors. The distance can be computed using a kernel function + or a distance metric. The distance is then normalized and aggregated to obtain a single outlier score. + + The detector can be initialized with `k` a single value or an array of values. If `k` is a single value then + the outlier score is the distance/kernel similarity to the k-th nearest neighbor. If `k` is an array of + values then the outlier score is the distance/kernel similarity to each of the specified `k` neighbors. + In the latter case, an aggregator must be specified to aggregate the scores. + Parameters ---------- k - Number of nearest neighbours to use for outlier detection. If an array is passed, an - aggregator is required to aggregate the scores. + Number of neirest neighbors to compute distance to. `k` can be a single value or + an array of integers. If an array is passed, an aggregator is required to aggregate + the scores. If `k` is a single value the outlier score is the distance/kernel + similarity to the `k`-th nearest neighbor. If `k` is a list then it returns the + distance/kernel similarity to each of the specified `k` neighbors. kernel Kernel function to use for outlier detection. If ``None``, `torch.cdist` is used. + Otherwise if a kernel is specified then instead of using `torch.cdist` the kernel + defines the k nearest neighbor distance. normalizer Normalizer to use for outlier detection. If ``None``, no normalisation is applied. + For a list of available normalizers, see :mod:`alibi_detect.od.pytorch.ensemble`. aggregator - Aggregator to use for outlier detection. Can be set to ``None`` if `k` is a single value. + Aggregator to use for outlier detection. Can be set to ``None`` if `k` is a single + value. For a list of available aggregators, see :mod:`alibi_detect.od.pytorch.ensemble`. backend Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. device - Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Device type used. The default tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. Raises ------ @@ -94,6 +110,10 @@ def fit(self, x_ref: np.ndarray) -> None: def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. + Computes the k nearest neighbor distance/kernel similarity for each instance in `x`. If `k` is a single + value then this is the score otherwise if `k` is an array of values then the score is aggregated using + the accumulator. + Parameters ---------- x @@ -108,23 +128,27 @@ def score(self, x: np.ndarray) -> np.ndarray: return to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: - """Infer the threshold for the kNN detector. The threshold is inferred using the reference data and the false - positive rate. The threshold is used to determine the outlier labels in the predict method. + """Infer the threshold for the kNN detector. + + The threshold is computed so that the outlier detector would incorectly classify `fpr` proportion of the + reference data as outliers. Parameters ---------- x_ref Reference data used to infer the threshold. fpr - False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ - `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ - ``(0, 1)``. + False positive rate used to infer the threshold. The false positive rate is the proportion of + instances in `x_ref` that are incorrectly classified as outliers. The false positive rate should + be in the range ``(0, 1)``. """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. + Scores the instances in `x` and if the threshold was inferred, returns the outlier labels and p-values as well. + Parameters ---------- x @@ -133,7 +157,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: Returns ------- Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ - performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. """ diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index cfe7e9ac6..f02316ae6 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -25,7 +25,7 @@ def __init__( similarity to the `k`-th nearest neighbor. If `k` is a list then it returns the distance/kernel similarity to each of the specified `k` neighbors. kernel - If a kernel is specified then instead of using `torch.cdist` the kernel defines the k nearest + If a kernel is specified then instead of using `torch.cdist` the kernel defines the `k` nearest neighbor distance. accumulator If `k` is an array of integers then the accumulator must not be ``None``. Should be an instance From d06673f2ad1ef4d76ef14e0ce4a6599ff1d52fb5 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 24 Jan 2023 14:52:59 +0000 Subject: [PATCH 083/247] Rename aggregator to ensembler --- alibi_detect/od/_knn.py | 14 ++++++------ alibi_detect/od/pytorch/__init__.py | 2 +- alibi_detect/od/pytorch/base.py | 20 +++++++++-------- alibi_detect/od/pytorch/ensemble.py | 5 ++--- alibi_detect/od/pytorch/knn.py | 16 +++++++------- .../od/tests/test__knn/test__knn_backend.py | 22 +++++++++---------- alibi_detect/od/tests/test_ensemble.py | 12 +++++----- alibi_detect/tests/test_dep_management.py | 2 +- 8 files changed, 47 insertions(+), 46 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index de71c75f2..7559e254e 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -6,7 +6,7 @@ from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols -from alibi_detect.od.pytorch import KNNTorch, Accumulator, to_numpy +from alibi_detect.od.pytorch import KNNTorch, Ensembler, to_numpy from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -17,7 +17,7 @@ backends = { - 'pytorch': (KNNTorch, Accumulator) + 'pytorch': (KNNTorch, Ensembler) } @@ -82,20 +82,20 @@ def __init__( construct_name=self.__class__.__name__ ).verify_backend(backend_str) - backend_cls, accumulator_cls = backends[backend] - accumulator = None + backend_cls, ensembler_cls = backends[backend] + ensembler = None if aggregator is None and isinstance(k, (list, np.ndarray, tuple)): raise ValueError('If `k` is a `np.ndarray`, `list` or `tuple`, ' 'the `aggregator` argument cannot be ``None``.') if isinstance(k, (list, np.ndarray, tuple)): - accumulator = accumulator_cls( + ensembler = ensembler_cls( normalizer=get_normalizer(normalizer), aggregator=get_aggregator(aggregator) ) - self.backend = backend_cls(k, kernel=kernel, accumulator=accumulator, device=device) + self.backend = backend_cls(k, kernel=kernel, ensembler=ensembler, device=device) def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. @@ -112,7 +112,7 @@ def score(self, x: np.ndarray) -> np.ndarray: Computes the k nearest neighbor distance/kernel similarity for each instance in `x`. If `k` is a single value then this is the score otherwise if `k` is an array of values then the score is aggregated using - the accumulator. + the ensembler. Parameters ---------- diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py index fb64bef77..672409515 100644 --- a/alibi_detect/od/pytorch/__init__.py +++ b/alibi_detect/od/pytorch/__init__.py @@ -1,6 +1,6 @@ from alibi_detect.utils.missing_optional_dependency import import_optional KNNTorch = import_optional('alibi_detect.od.pytorch.knn', ['KNNTorch']) -Accumulator = import_optional('alibi_detect.od.pytorch.ensemble', ['Accumulator']) +Ensembler = import_optional('alibi_detect.od.pytorch.ensemble', ['Ensembler']) to_numpy = import_optional('alibi_detect.od.pytorch.base', ['to_numpy']) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 623a9c678..25d2f0b26 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -114,22 +114,24 @@ def _to_tensor(self, x: Union[List, np.ndarray]) -> torch.Tensor: """ return torch.as_tensor(x, dtype=torch.float32, device=self.device) - def _accumulator(self, x: torch.Tensor) -> torch.Tensor: - """Accumulates the data. + def _ensembler(self, x: torch.Tensor) -> torch.Tensor: + """Aggregates and normalizes the data + + If the detector has an ensembler attribute we use it to aggregate and normalize the data. Parameters ---------- x - Data to accumulate. + Data to aggregate and normalize. Returns ------- `torch.Tensor` or just returns original data """ - if hasattr(self, 'accumulator') and self.accumulator is not None: - # `type: ignore` here becuase self.accumulator here causes an error with mypy when using torch.jit.script. - # For some reason it thinks self.accumulator is a torch.Tensor and therefore is not callable. - return self.accumulator(x) # type: ignore + if hasattr(self, 'ensembler') and self.ensembler is not None: + # `type: ignore` here becuase self.ensembler here causes an error with mypy when using torch.jit.script. + # For some reason it thinks self.ensembler is a torch.Tensor and therefore is not callable. + return self.ensembler(x) # type: ignore else: return x @@ -180,7 +182,7 @@ def infer_threshold(self, x: torch.Tensor, fpr: float) -> None: if not 0 < fpr < 1: ValueError('`fpr` must be in `(0, 1)`.') self.val_scores = self.score(x) - self.val_scores = self._accumulator(self.val_scores) + self.val_scores = self._ensembler(self.val_scores) self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True @@ -209,7 +211,7 @@ def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: """ self.check_fitted() # type: ignore raw_scores = self.score(x) - scores = self._accumulator(raw_scores) + scores = self._ensembler(raw_scores) return TorchOutlierDetectorOutput( instance_score=scores, diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index e7e1112c3..2ca6524da 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -299,12 +299,11 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return vals -class Accumulator(BaseFittedTransformTorch): +class Ensembler(BaseFittedTransformTorch): def __init__(self, normalizer: Optional[BaseFittedTransformTorch] = None, aggregator: BaseTransformTorch = AverageAggregator()): - """Accumulates the scores of the detectors in an ensemble. Can be used to normalise and aggregate - the scores from an ensemble of detectors. + """An Ensembler applies normlization and aggregation operations to the scores of an ensemble of detectors. Parameters ---------- diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index f02316ae6..d120e884c 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -3,7 +3,7 @@ import numpy as np import torch -from alibi_detect.od.pytorch.ensemble import Accumulator +from alibi_detect.od.pytorch.ensemble import Ensembler from alibi_detect.od.pytorch.base import TorchOutlierDetector @@ -12,7 +12,7 @@ def __init__( self, k: Union[np.ndarray, List, Tuple], kernel: Optional[torch.nn.Module] = None, - accumulator: Optional[Accumulator] = None, + ensembler: Optional[Ensembler] = None, device: Optional[Union[str, torch.device]] = None ): """PyTorch backend for KNN detector. @@ -27,9 +27,9 @@ def __init__( kernel If a kernel is specified then instead of using `torch.cdist` the kernel defines the `k` nearest neighbor distance. - accumulator - If `k` is an array of integers then the accumulator must not be ``None``. Should be an instance - of :py:obj:`alibi_detect.od.pytorch.ensemble.Accumulator`. Responsible for combining + ensembler + If `k` is an array of integers then the ensembler must not be ``None``. Should be an instance + of :py:obj:`alibi_detect.od.pytorch.ensemble.ensembler`. Responsible for combining multiple scores into a single score. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. @@ -39,7 +39,7 @@ def __init__( self.kernel = kernel self.ensemble = isinstance(k, (np.ndarray, list, tuple)) self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) - self.accumulator = accumulator + self.ensembler = ensembler def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -59,7 +59,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: If called before detector has had `infer_threshold` method called. """ raw_scores = self.score(x) - scores = self._accumulator(raw_scores) + scores = self._ensembler(raw_scores) if not torch.jit.is_scripting(): self.check_threshold_infered() preds = scores > self.threshold @@ -100,4 +100,4 @@ def _fit(self, x_ref: torch.Tensor): self.x_ref = x_ref if self.ensemble: scores = self.score(x_ref) - self.accumulator.fit(scores) + self.ensembler.fit(scores) diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 5363b9a62..52259fb59 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -3,13 +3,13 @@ from alibi_detect.od.pytorch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF -from alibi_detect.od.pytorch.ensemble import Accumulator, PValNormalizer, AverageAggregator +from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator from alibi_detect.od.base import NotFitException, ThresholdNotInferredException @pytest.fixture(scope='session') -def accumulator(request): - return Accumulator( +def ensembler(request): + return Ensembler( normalizer=PValNormalizer(), aggregator=AverageAggregator() ) @@ -33,8 +33,8 @@ def test_knn_torch_backend(): assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) -def test_knn_torch_backend_ensemble(accumulator): - knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) +def test_knn_torch_backend_ensemble(ensembler): + knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) @@ -47,8 +47,8 @@ def test_knn_torch_backend_ensemble(accumulator): assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) -def test_knn_torch_backend_ensemble_ts(tmp_path, accumulator): - knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) +def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): + knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) with pytest.raises(NotFitException) as err: @@ -90,9 +90,9 @@ def test_knn_torch_backend_ts(tmp_path): assert torch.all(pred_1 == pred_2) -def test_knn_kernel(accumulator): +def test_knn_kernel(ensembler): kernel = GaussianRBF(sigma=torch.tensor((0.25))) - knn_torch = KNNTorch(k=[4, 5], kernel=kernel, accumulator=accumulator) + knn_torch = KNNTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) @@ -112,8 +112,8 @@ def test_knn_kernel(accumulator): # assert torch.all(pred_1 == pred_2) -def test_knn_torch_backend_ensemble_fit_errors(accumulator): - knn_torch = KNNTorch(k=[4, 5], accumulator=accumulator) +def test_knn_torch_backend_ensemble_fit_errors(ensembler): + knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) assert not knn_torch._fitted x = torch.randn((1, 10)) diff --git a/alibi_detect/od/tests/test_ensemble.py b/alibi_detect/od/tests/test_ensemble.py index 52cc15cd3..a0f93b39a 100644 --- a/alibi_detect/od/tests/test_ensemble.py +++ b/alibi_detect/od/tests/test_ensemble.py @@ -121,16 +121,16 @@ def test_min_aggregator(): @pytest.mark.parametrize('aggregator', ['AverageAggregator', 'MaxAggregator', 'MinAggregator', 'TopKAggregator']) @pytest.mark.parametrize('normalizer', ['PValNormalizer', 'ShiftAndScaleNormalizer']) -def test_accumulator(aggregator, normalizer): +def test_ensembler(aggregator, normalizer): aggregator = getattr(ensemble, aggregator)() normalizer = getattr(ensemble, normalizer)() - accumulator = ensemble.Accumulator(aggregator=aggregator, normalizer=normalizer) + ensembler = ensemble.Ensembler(aggregator=aggregator, normalizer=normalizer) x = torch.randn(3, 10) x_ref = torch.randn(64, 10) - accumulator.fit(x_ref) - x_norm = accumulator(x) - accumulator = torch.jit.script(accumulator) - x_norm_2 = accumulator(x) + ensembler.fit(x_ref) + x_norm = ensembler(x) + ensembler = torch.jit.script(ensembler) + x_norm_2 = ensembler(x) assert torch.all(x_norm_2 == x_norm) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index c61f3682c..8883afe84 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -136,7 +136,7 @@ def test_od_backend_dependencies(opt_dep): """ dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ('Accumulator', ['torch', 'keops']), + ('Ensembler', ['torch', 'keops']), ('KNNTorch', ['torch', 'keops']), ('to_numpy', ['torch', 'keops']), ]: From db524ab694e1e69453ee0e24e9eadd97b3ee303d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 24 Jan 2023 16:44:49 +0000 Subject: [PATCH 084/247] Remove unnessesery code --- alibi_detect/od/pytorch/pca.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 16f988acf..f300627b6 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -23,7 +23,6 @@ def __init__( Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. """ TorchOutlierDetector.__init__(self, device=device) - self.accumulator = None self.n_components = n_components def forward(self, x: torch.Tensor) -> torch.Tensor: @@ -43,8 +42,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: ThresholdNotInferredException If called before detector has had `infer_threshold` method called. """ - raw_scores = self.score(x) - scores = self._accumulator(raw_scores) + scores = self.score(x) if not torch.jit.is_scripting(): self.check_threshold_infered() preds = scores > self.threshold From 3d69469d4dc95a6ea616c802ca6988110917b1d9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 31 Jan 2023 13:49:48 +0000 Subject: [PATCH 085/247] Add experimental module --- alibi_detect/experimental/__init__.py | 0 alibi_detect/experimental/od/__init__.py | 1 + alibi_detect/od/_knn.py | 2 +- alibi_detect/od/tests/test__knn/test__knn.py | 20 ++++++++++---------- 4 files changed, 12 insertions(+), 11 deletions(-) create mode 100644 alibi_detect/experimental/__init__.py create mode 100644 alibi_detect/experimental/od/__init__.py diff --git a/alibi_detect/experimental/__init__.py b/alibi_detect/experimental/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/alibi_detect/experimental/od/__init__.py b/alibi_detect/experimental/od/__init__.py new file mode 100644 index 000000000..6cddab30d --- /dev/null +++ b/alibi_detect/experimental/od/__init__.py @@ -0,0 +1 @@ +from alibi_detect.od._knn import KNN # noqa F401 diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 7559e254e..ef813dc2a 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -21,7 +21,7 @@ } -class _KNN(OutlierDetector): +class KNN(OutlierDetector): def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index bb165ec7e..f9f78ba20 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -2,7 +2,7 @@ import numpy as np import torch -from alibi_detect.od._knn import _KNN +from alibi_detect.od._knn import KNN from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer from alibi_detect.od.base import NotFitException @@ -11,7 +11,7 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): - knn_detector = _KNN( + knn_detector = KNN( k=k, aggregator=aggregator, normalizer=normalizer ) @@ -22,7 +22,7 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): def test_unfitted_knn_single_score(): - knn_detector = _KNN(k=10) + knn_detector = KNN(k=10) x = np.array([[0, 10], [0.1, 0]]) with pytest.raises(NotFitException) as err: _ = knn_detector.predict(x) @@ -30,7 +30,7 @@ def test_unfitted_knn_single_score(): def test_fitted_knn_single_score(): - knn_detector = _KNN(k=10) + knn_detector = KNN(k=10) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) @@ -46,7 +46,7 @@ def test_fitted_knn_single_score(): def test_default_knn_ensemble_init(): - knn_detector = _KNN(k=[8, 9, 10]) + knn_detector = KNN(k=[8, 9, 10]) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) @@ -63,7 +63,7 @@ def test_default_knn_ensemble_init(): def test_incorrect_knn_ensemble_init(): with pytest.raises(ValueError) as err: - _KNN(k=[8, 9, 10], aggregator=None) + KNN(k=[8, 9, 10], aggregator=None) assert str(err.value) == ('If `k` is a `np.ndarray`, `list` or `tuple`, ' 'the `aggregator` argument cannot be ``None``.') @@ -87,7 +87,7 @@ def test_fitted_knn_predict(): MaxAggregator, MinAggregator]) @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_unfitted_knn_ensemble(aggregator, normalizer): - knn_detector = _KNN( + knn_detector = KNN( k=[8, 9, 10], aggregator=aggregator(), normalizer=normalizer() @@ -102,7 +102,7 @@ def test_unfitted_knn_ensemble(aggregator, normalizer): MaxAggregator, MinAggregator]) @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_fitted_knn_ensemble(aggregator, normalizer): - knn_detector = _KNN( + knn_detector = KNN( k=[8, 9, 10], aggregator=aggregator(), normalizer=normalizer() @@ -163,7 +163,7 @@ def test_knn_single_torchscript(): @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None, lambda: 'ShiftAndScaleNormalizer', lambda: 'PValNormalizer']) def test_knn_ensemble_integration(aggregator, normalizer): - knn_detector = _KNN( + knn_detector = KNN( k=[10, 14, 18], aggregator=aggregator(), normalizer=normalizer() @@ -188,7 +188,7 @@ def test_knn_ensemble_integration(aggregator, normalizer): def test_knn_integration(): - knn_detector = _KNN(k=18) + knn_detector = KNN(k=18) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] knn_detector.fit(X_ref) From d7a9b82fdc9d660a6e772072b4f9d56fff43a017 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 31 Jan 2023 13:55:35 +0000 Subject: [PATCH 086/247] Add Mahalanobis to experiemental namespace --- alibi_detect/experimental/od/__init__.py | 1 + alibi_detect/od/_mahalanobis.py | 2 +- .../od/tests/test__mahalanobis/test__mahalanobis.py | 10 +++++----- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/alibi_detect/experimental/od/__init__.py b/alibi_detect/experimental/od/__init__.py index 6cddab30d..bbe184b39 100644 --- a/alibi_detect/experimental/od/__init__.py +++ b/alibi_detect/experimental/od/__init__.py @@ -1 +1,2 @@ from alibi_detect.od._knn import KNN # noqa F401 +from alibi_detect.od._mahalanobis import Mahalanobis # noqa F401 \ No newline at end of file diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 48bdb7418..7df808960 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -21,7 +21,7 @@ } -class _Mahalanobis(OutlierDetector): +class Mahalanobis(OutlierDetector): def __init__( self, min_eigenvalue: float = 1e-6, diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index 4310b7525..e4dc3407b 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -2,13 +2,13 @@ import numpy as np import torch -from alibi_detect.od._mahalanobis import _Mahalanobis +from alibi_detect.od._mahalanobis import Mahalanobis from alibi_detect.od.base import NotFitException from sklearn.datasets import make_moons def make_mahalanobis_detector(): - mahalanobis_detector = _Mahalanobis() + mahalanobis_detector = Mahalanobis() x_ref = np.random.randn(100, 2) mahalanobis_detector.fit(x_ref) mahalanobis_detector.infer_threshold(x_ref, 0.1) @@ -16,7 +16,7 @@ def make_mahalanobis_detector(): def test_unfitted_mahalanobis_single_score(): - mahalanobis_detector = _Mahalanobis() + mahalanobis_detector = Mahalanobis() x = np.array([[0, 10], [0.1, 0]]) with pytest.raises(NotFitException) as err: _ = mahalanobis_detector.predict(x) @@ -24,7 +24,7 @@ def test_unfitted_mahalanobis_single_score(): def test_fitted_mahalanobis_single_score(): - mahalanobis_detector = _Mahalanobis() + mahalanobis_detector = Mahalanobis() x_ref = np.random.randn(100, 2) mahalanobis_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) @@ -54,7 +54,7 @@ def test_fitted_mahalanobis_predict(): def test_mahalanobis_integration(): - mahalanobis_detector = _Mahalanobis() + mahalanobis_detector = Mahalanobis() X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] mahalanobis_detector.fit(X_ref) From 35afa55ee1cd338018159ec4064730c9ad8ad2d4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 31 Jan 2023 14:05:37 +0000 Subject: [PATCH 087/247] Add PCA to the experimental namespace --- alibi_detect/experimental/od/__init__.py | 3 ++- alibi_detect/od/_pca.py | 2 +- alibi_detect/od/tests/test__pca/test__pca.py | 24 ++++++++++---------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/alibi_detect/experimental/od/__init__.py b/alibi_detect/experimental/od/__init__.py index bbe184b39..2b15dbac9 100644 --- a/alibi_detect/experimental/od/__init__.py +++ b/alibi_detect/experimental/od/__init__.py @@ -1,2 +1,3 @@ from alibi_detect.od._knn import KNN # noqa F401 -from alibi_detect.od._mahalanobis import Mahalanobis # noqa F401 \ No newline at end of file +from alibi_detect.od._mahalanobis import Mahalanobis # noqa F401 +from alibi_detect.od._pca import PCA # noqa F401 \ No newline at end of file diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index e724a4318..6a4f9d8a8 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -21,7 +21,7 @@ } -class _PCA(OutlierDetector): +class PCA(OutlierDetector): def __init__( self, n_components: int, diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index fd588dd0b..0960ef94d 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -3,16 +3,16 @@ import torch from alibi_detect.utils.pytorch.kernels import GaussianRBF -from alibi_detect.od._pca import _PCA +from alibi_detect.od._pca import PCA from alibi_detect.od.base import NotFitException from sklearn.datasets import make_moons def make_PCA_detector(kernel=False): if kernel: - pca_detector = _PCA(n_components=2, kernel=GaussianRBF()) + pca_detector = PCA(n_components=2, kernel=GaussianRBF()) else: - pca_detector = _PCA(n_components=2) + pca_detector = PCA(n_components=2) x_ref = np.random.randn(100, 3) pca_detector.fit(x_ref) pca_detector.infer_threshold(x_ref, 0.1) @@ -20,8 +20,8 @@ def make_PCA_detector(kernel=False): @pytest.mark.parametrize('detector', [ - lambda: _PCA(n_components=5), - lambda: _PCA(n_components=5, kernel=GaussianRBF()) + lambda: PCA(n_components=5), + lambda: PCA(n_components=5, kernel=GaussianRBF()) ]) def test_unfitted_PCA_single_score(detector): pca = detector() @@ -33,7 +33,7 @@ def test_unfitted_PCA_single_score(detector): def test_fitted_PCA_single_score(): - pca_detector = _PCA(n_components=2) + pca_detector = PCA(n_components=2) x_ref = np.random.randn(100, 3) pca_detector.fit(x_ref) x = np.array([[0, 10, 0], [0.1, 0, 0]]) @@ -48,7 +48,7 @@ def test_fitted_PCA_single_score(): def test_fitted_kernel_PCA_single_score(): - pca_detector = _PCA(n_components=2, kernel=GaussianRBF()) + pca_detector = PCA(n_components=2, kernel=GaussianRBF()) x_ref = np.random.randn(100, 3) * np.array([1, 10, 0.1]) pca_detector.fit(x_ref) x = np.array([[0, 5, 10], [0.1, 5, 0]]) @@ -77,7 +77,7 @@ def test_fitted_PCA_predict(): def test_fitted_kernel_PCA_predict(): - pca_detector = _PCA(n_components=2, kernel=GaussianRBF()) + pca_detector = PCA(n_components=2, kernel=GaussianRBF()) x_ref = np.random.randn(100, 3) * np.array([1, 10, 0.1]) pca_detector.fit(x_ref) pca_detector.infer_threshold(x_ref, 0.1) @@ -92,7 +92,7 @@ def test_fitted_kernel_PCA_predict(): def test_PCA_integration(): - pca_detector = _PCA(n_components=1) + pca_detector = PCA(n_components=1) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] pca_detector.fit(X_ref) @@ -108,7 +108,7 @@ def test_PCA_integration(): def test_PCA_integration_ts(): - pca_detector = _PCA(n_components=1) + pca_detector = PCA(n_components=1) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] pca_detector.fit(X_ref) @@ -121,7 +121,7 @@ def test_PCA_integration_ts(): def test_kernel_PCA_integration(): - pca_detector = _PCA(n_components=10, kernel=GaussianRBF()) + pca_detector = PCA(n_components=10, kernel=GaussianRBF()) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] pca_detector.fit(X_ref) @@ -138,7 +138,7 @@ def test_kernel_PCA_integration(): @pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript support yet.') def test_kernel_PCA_integration_ts(): - pca_detector = _PCA(n_components=10, kernel=GaussianRBF()) + pca_detector = PCA(n_components=10, kernel=GaussianRBF()) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] pca_detector.fit(X_ref) From 2fad9e3be5a4096d0c6a4da428359a1ba2c54cbf Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 31 Jan 2023 17:26:14 +0000 Subject: [PATCH 088/247] Add sklearn gmm od backend --- alibi_detect/od/sklearn/base.py | 257 ++++++++++++++++++ alibi_detect/od/sklearn/gmm.py | 41 +++ .../test__gmm/test__gmm_sklearn_backend.py | 62 +++++ 3 files changed, 360 insertions(+) create mode 100644 alibi_detect/od/sklearn/base.py create mode 100644 alibi_detect/od/sklearn/gmm.py create mode 100644 alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py new file mode 100644 index 000000000..872ee62c4 --- /dev/null +++ b/alibi_detect/od/sklearn/base.py @@ -0,0 +1,257 @@ +from __future__ import annotations +from typing import List, Union, Optional +from dataclasses import dataclass +from abc import ABC, abstractmethod +from alibi_detect.od.base import NotFitException + + +import numpy as np + +from alibi_detect.od.base import ThresholdNotInferredException + + +def to_numpy(arg): + """Map params to numpy arrays. + + This function is for interface compatibility with the other backends. As such it does nothing but + return the input. + + Parameters + ---------- + x + Data to convert. + + Returns + ------- + `np.ndarray` or dictionary of containing `numpy` arrays + """ + return arg + + +@dataclass +class SklearnOutlierDetectorOutput: + """Output of the outlier detector.""" + threshold_inferred: bool + instance_score: np.ndarray + threshold: Optional[np.ndarray] + is_outlier: Optional[np.ndarray] + p_value: Optional[np.ndarray] + + +class FitMixin(ABC): + _fitted = False + + def __init__(self): + """Fit mixin + + Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. + """ + super().__init__() + + def fit(self, x: np.ndarray) -> FitMixin: + self._fitted = True + self._fit(x) + return self + + @abstractmethod + def _fit(self, x: np.ndarray): + """Fit on `x` array. + + This method should be overidden on child classes. + + Parameters + ---------- + x + Reference `np.array` for fitting object. + """ + pass + + def check_fitted(self): + """Raises error if parent object instance has not been fit. + + Raises + ------ + NotFitException + Raised if method called and object has not been fit. + """ + if not self._fitted: + raise NotFitException(f'{self.__class__.__name__} has not been fit!') + + +class SklearnOutlierDetector(FitMixin, ABC): + """Base class for sklearn backend outlier detection algorithms.""" + threshold_inferred = False + threshold = None + + def __init__(self): + super().__init__() + + @abstractmethod + def _fit(self, x_ref: np.ndarray) -> None: + """Fit the outlier detector to the reference data. + + Parameters + ---------- + x_ref + Reference data. + """ + pass + + @abstractmethod + def score(self, x: np.ndarray) -> np.ndarray: + """Score the data. + + Parameters + ---------- + x + Data to score. + + """ + pass + + def check_threshold_infered(self): + """Check if threshold is inferred. + + Raises + ------ + ThresholdNotInferredException + Raised if threshold is not inferred. + """ + if not self.threshold_inferred: + raise ThresholdNotInferredException((f'{self.__class__.__name__} has no threshold set, ' + 'call `infer_threshold` before predicting.')) + + def _to_tensor(self, x: Union[List, np.ndarray]) -> np.ndarray: + """Converts the data to a tensor. + + This function is for interface compatibility with the other backends. As such it does nothing but + return the input. + + Parameters + ---------- + x + Data to convert. + + Returns + ------- + `np.ndarray` + """ + return x + + def _ensembler(self, x: np.ndarray) -> np.ndarray: + """Aggregates and normalizes the data + + If the detector has an ensembler attribute we use it to aggregate and normalize the data. + + Parameters + ---------- + x + Data to aggregate and normalize. + + Returns + ------- + `np.ndarray` or just returns original data + """ + if hasattr(self, 'ensembler') and self.ensembler is not None: + return self.ensembler(x) + else: + return x + + def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: + """Classify the data as outlier or not. + + Parameters + ---------- + scores + Scores to classify. Larger scores indicate more likely outliers. + + Returns + ------- + `np.ndarray` or ``None`` + """ + return scores > self.threshold if self.threshold_inferred else None + + def _p_vals(self, scores: np.ndarray) -> np.ndarray: + """Compute p-values for the scores. + + Parameters + ---------- + scores + Scores to compute p-values for. + + Returns + ------- + `np.ndarray` or ``None`` + """ + return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ + if self.threshold_inferred else None + + def infer_threshold(self, x: np.ndarray, fpr: float) -> None: + """Infer the threshold for the data. Prerequisite for outlier predictions. + + Parameters + ---------- + x + Data to infer the threshold for. + fpr + False positive rate to use for threshold inference. + + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + """ + if not 0 < fpr < 1: + ValueError('`fpr` must be in `(0, 1)`.') + self.val_scores = self.score(x) + self.val_scores = self._ensembler(self.val_scores) + self.threshold = np.quantile(self.val_scores, 1-fpr) + self.threshold_inferred = True + + def predict(self, x: np.ndarray) -> SklearnOutlierDetectorOutput: + """Predict outlier labels for the data. + + Computes the outlier scores. If the detector is not fit on reference data we raise an error. + If the threshold is inferred, the outlier labels and p-values are also computed and returned. + Otherwise, the outlier labels and p-values are set to ``None``. + + Parameters + ---------- + x + Data to predict. + + Raises + ------ + ValueError + Raised if the detector is not fit on reference data. + + Returns + ------- + `SklearnOutlierDetectorOutput` + Output of the outlier detector. + + """ + self.check_fitted() # type: ignore + raw_scores = self.score(x) + scores = self._ensembler(raw_scores) + + return SklearnOutlierDetectorOutput( + instance_score=scores, + is_outlier=self._classify_outlier(scores), + p_value=self._p_vals(scores), + threshold_inferred=self.threshold_inferred, + threshold=self.threshold + ) + + def __call__(self, x: np.ndarray) -> np.ndarray: + """Classify outliers. + + Parameters + ---------- + x + Data to classify. + """ + raw_scores = self.score(x) + scores = self._ensembler(raw_scores) + self.check_threshold_infered() + return self._classify_outlier(scores) diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py new file mode 100644 index 000000000..eaa1d5d64 --- /dev/null +++ b/alibi_detect/od/sklearn/gmm.py @@ -0,0 +1,41 @@ +import numpy as np +from alibi_detect.od.sklearn.base import SklearnOutlierDetector +from sklearn.mixture import GaussianMixture + + +class GMMSklearn(SklearnOutlierDetector): + def __init__( + self, + n_components: int, + ): + """sklearn backend for GMM detector. + + Parameters + ---------- + n_components + Number of components in guassian mixture model. + """ + SklearnOutlierDetector.__init__(self) + self.n_components = n_components + self.gmm = None + + def _fit(self, x_ref: np.ndarray) -> None: + """Fit the outlier detector to the reference data. + + Parameters + ---------- + x_ref + Reference data. + """ + self.gmm = GaussianMixture(n_components=self.n_components).fit(x_ref) + + def score(self, x: np.ndarray) -> np.ndarray: + """Score the data. + + Parameters + ---------- + x + Data to score. + """ + self.check_fitted() + return - self.gmm.score_samples(x) diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py new file mode 100644 index 000000000..cac12eaa5 --- /dev/null +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -0,0 +1,62 @@ +import pytest +import numpy as np + +from alibi_detect.od.sklearn.gmm import GMMSklearn +from alibi_detect.od.base import NotFitException, ThresholdNotInferredException + + +def test_gmm_sklearn_backend_fit_errors(): + gm_sklearn = GMMSklearn(n_components=2) + assert not gm_sklearn._fitted + + x = np.random.randn(1, 10) + with pytest.raises(NotFitException) as err: + gm_sklearn(x) + assert str(err.value) == 'GMMSklearn has not been fit!' + + with pytest.raises(NotFitException) as err: + gm_sklearn.predict(x) + assert str(err.value) == 'GMMSklearn has not been fit!' + + x_ref = np.random.randn(1024, 10) + gm_sklearn.fit(x_ref) + + assert gm_sklearn._fitted + + with pytest.raises(ThresholdNotInferredException) as err: + gm_sklearn(x) + assert str(err.value) == 'GMMSklearn has no threshold set, call `infer_threshold` before predicting.' + + assert gm_sklearn.predict(x) + + +def test_gmm_linear_scoring(): + gm_sklearn = GMMSklearn(n_components=2) + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = np.random.multivariate_normal(mean, cov, 1000) + gm_sklearn.fit(x_ref) + + x_1 = np.array([[8., 8.]]) + scores_1 = gm_sklearn.score(x_1) + + x_2 = np.random.multivariate_normal(mean, cov, 1) + scores_2 = gm_sklearn.score(x_2) + + x_3 = np.array([[-10., 10.]]) + scores_3 = gm_sklearn.score(x_3) + + # test correct ordering of scores given outlyingness of data + assert scores_1 < scores_2 < scores_3 + + # test that detector correctly detects true Outlier + gm_sklearn.infer_threshold(x_ref, 0.01) + x = np.concatenate((x_1, x_2, x_3)) + outputs = gm_sklearn.predict(x) + assert np.all(outputs.is_outlier == np.array([False, False, True])) + assert np.all(gm_sklearn(x) == np.array([False, False, True])) + + # test that 0.01 of the in distribution data is flagged as outliers + x = np.random.multivariate_normal(mean, cov, 1000) + outputs = gm_sklearn.predict(x) + assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.01 From c679ab04f1729e2823f1af40bbef2b8680bd6385 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 11:28:57 +0000 Subject: [PATCH 089/247] Add gmm pytorch backend --- alibi_detect/models/pytorch/gmm.py | 26 +++++ alibi_detect/od/pytorch/gmm.py | 96 +++++++++++++++++++ alibi_detect/od/sklearn/base.py | 2 +- alibi_detect/od/sklearn/gmm.py | 4 +- .../test__gmm/test__gmm_pytorch_backend.py | 63 ++++++++++++ .../test__gmm/test__gmm_sklearn_backend.py | 2 +- alibi_detect/utils/pytorch/data.py | 2 +- 7 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 alibi_detect/models/pytorch/gmm.py create mode 100644 alibi_detect/od/pytorch/gmm.py create mode 100644 alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py diff --git a/alibi_detect/models/pytorch/gmm.py b/alibi_detect/models/pytorch/gmm.py new file mode 100644 index 000000000..2284d25de --- /dev/null +++ b/alibi_detect/models/pytorch/gmm.py @@ -0,0 +1,26 @@ +from torch import nn +import torch + + +class GMMModel(nn.Module): + def __init__(self, n_components: int, dim: int) -> None: + super().__init__() + self.weight_logits = nn.Parameter(torch.zeros(n_components)) + self.means = nn.Parameter(torch.randn(n_components, dim)) + self.inv_cov_factor = nn.Parameter(torch.randn(n_components, dim, dim)/10) + + @property + def _inv_cov(self) -> torch.Tensor: + return torch.bmm(self.inv_cov_factor, self.inv_cov_factor.transpose(1, 2)) + + @property + def _weights(self) -> torch.Tensor: + return nn.functional.softmax(self.weight_logits, dim=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + det = torch.linalg.det(self._inv_cov) # Note det(A^-1)=1/det(A) + to_means = x[:, None, :] - self.means[None, :, :] + likelihood = ((-0.5 * ( + torch.einsum('bke,bke->bk', (torch.einsum('bkd,kde->bke', to_means, self._inv_cov), to_means)) + )).exp()*det[None, :]*self._weights[None, :]).sum(-1) + return -likelihood.log() diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py new file mode 100644 index 000000000..029c06f71 --- /dev/null +++ b/alibi_detect/od/pytorch/gmm.py @@ -0,0 +1,96 @@ +from typing import Callable, Optional +from tqdm import tqdm +import torch +from torch.utils.data import DataLoader + +from alibi_detect.utils.pytorch.data import TorchDataset +from alibi_detect.utils.pytorch.prediction import predict_batch +from alibi_detect.od.pytorch.base import TorchOutlierDetector +from alibi_detect.models.pytorch.gmm import GMMModel + + +class GMMTorch(TorchOutlierDetector): + def __init__( + self, + n_components: int, + device: Optional[str] = None, + ) -> None: + """ + Fits a Gaussian mixture model to the training data and scores new data points + via the negative log-likhood under the corresponding density function. + Parameters + ---------- + n_components: + The number of Gaussian mixture components. + optimizer: + Used to learn the GMM params. + rest should be obvious. + """ + self.n_components = n_components + TorchOutlierDetector.__init__(self, device=device) + + def _fit( + self, + X: torch.Tensor, + optimizer: Callable = torch.optim.Adam, + learning_rate: float = 0.1, + batch_size: int = 32, + epochs: int = 10, + verbose: int = 0, + ) -> None: + self.model = GMMModel(self.n_components, X.shape[-1]) + X = X.to(torch.float32) + + ds = TorchDataset(X) + dl = DataLoader(ds, batch_size=batch_size, shuffle=True) + optimizer = optimizer(self.model.parameters(), lr=learning_rate) + self.model.train() + + for epoch in range(epochs): + dl = tqdm(enumerate(dl), total=len(dl)) if verbose == 1 else enumerate(dl) + loss_ma = 0 + for step, x in dl: + x = x.to(self.device) + nll = self.model(x).mean() + optimizer.zero_grad() # type: ignore + nll.backward() + optimizer.step() # type: ignore + if verbose == 1 and isinstance(dl, tqdm): + loss_ma = loss_ma + (nll.item() - loss_ma) / (step + 1) + dl.set_description(f'Epoch {epoch + 1}/{self.epochs}') + dl.set_postfix(dict(loss_ma=loss_ma)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Detect if `x` is an outlier. + + Parameters + ---------- + x + `torch.Tensor` with leading batch dimension. + + Returns + ------- + `torch.Tensor` of ``bool`` values with leading batch dimension. + + Raises + ------ + ThresholdNotInferredException + If called before detector has had `infer_threshold` method called. + """ + raw_scores = self.score(x) + scores = self._ensembler(raw_scores) + if not torch.jit.is_scripting(): + self.check_threshold_infered() + preds = scores > self.threshold + return preds.cpu() + + def score(self, X: torch.Tensor) -> torch.Tensor: + self.check_fitted() + batch_size, *_ = X.shape + X = X.to(torch.float32) + preds = predict_batch( + X, self.model.eval(), + device=self.device, + batch_size=batch_size + ) + return torch.tensor(preds) diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 872ee62c4..20588bf9c 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -136,7 +136,7 @@ def _to_tensor(self, x: Union[List, np.ndarray]) -> np.ndarray: ------- `np.ndarray` """ - return x + return np.array(x) def _ensembler(self, x: np.ndarray) -> np.ndarray: """Aggregates and normalizes the data diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index eaa1d5d64..abf7ae9ac 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -17,7 +17,7 @@ def __init__( """ SklearnOutlierDetector.__init__(self) self.n_components = n_components - self.gmm = None + self.gmm = GaussianMixture(n_components=self.n_components) def _fit(self, x_ref: np.ndarray) -> None: """Fit the outlier detector to the reference data. @@ -27,7 +27,7 @@ def _fit(self, x_ref: np.ndarray) -> None: x_ref Reference data. """ - self.gmm = GaussianMixture(n_components=self.n_components).fit(x_ref) + self.gmm = self.gmm.fit(x_ref) def score(self, x: np.ndarray) -> np.ndarray: """Score the data. diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py new file mode 100644 index 000000000..4c36d7ddb --- /dev/null +++ b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py @@ -0,0 +1,63 @@ +import pytest +import numpy as np +import torch + +from alibi_detect.od.pytorch.gmm import GMMTorch +from alibi_detect.od.base import NotFitException, ThresholdNotInferredException + + +def test_gmm_pytorch_backend_fit_errors(): + gmm_torch = GMMTorch(n_components=2) + assert not gmm_torch._fitted + + x = torch.tensor(np.random.randn(1, 10)) + with pytest.raises(NotFitException) as err: + gmm_torch(x) + assert str(err.value) == 'GMMTorch has not been fit!' + + with pytest.raises(NotFitException) as err: + gmm_torch.predict(x) + assert str(err.value) == 'GMMTorch has not been fit!' + + x_ref = torch.tensor(np.random.randn(1024, 10)) + gmm_torch.fit(x_ref) + + assert gmm_torch._fitted + + with pytest.raises(ThresholdNotInferredException) as err: + gmm_torch(x) + assert str(err.value) == 'GMMTorch has no threshold set, call `infer_threshold` before predicting.' + + assert gmm_torch.predict(x) + + +def test_gmm_pytorch_scoring(): + gmm_torch = GMMTorch(n_components=1) + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + gmm_torch.fit(x_ref) + + x_1 = torch.tensor(np.array([[8., 8.]])) + scores_1 = gmm_torch.score(x_1) + + x_2 = torch.tensor(np.random.multivariate_normal(mean, cov, 1)) + scores_2 = gmm_torch.score(x_2) + + x_3 = torch.tensor(np.array([[-10., 10.]])) + scores_3 = gmm_torch.score(x_3) + + # test correct ordering of scores given outlyingness of data + assert scores_1 < scores_2 < scores_3 + + # test that detector correctly detects true Outlier + gmm_torch.infer_threshold(x_ref, 0.01) + x = torch.cat((x_1, x_2, x_3)) + outputs = gmm_torch.predict(x) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) + assert torch.all(gmm_torch(x) == torch.tensor([False, False, True])) + + # test that 0.01 of the in distribution data is flagged as outliers + x = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + outputs = gmm_torch.predict(x) + assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.01 diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py index cac12eaa5..69bac21e2 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -30,7 +30,7 @@ def test_gmm_sklearn_backend_fit_errors(): assert gm_sklearn.predict(x) -def test_gmm_linear_scoring(): +def test_gmm_sklearn_scoring(): gm_sklearn = GMMSklearn(n_components=2) mean = [8, 8] cov = [[2., 0.], [0., 1.]] diff --git a/alibi_detect/utils/pytorch/data.py b/alibi_detect/utils/pytorch/data.py index f7e506bba..9d3147f73 100644 --- a/alibi_detect/utils/pytorch/data.py +++ b/alibi_detect/utils/pytorch/data.py @@ -6,7 +6,7 @@ class TorchDataset(torch.utils.data.Dataset): - def __init__(self, *indexables: Tuple[Indexable, ...]) -> None: + def __init__(self, *indexables: Union[Tuple[Indexable, ...], Indexable]) -> None: self.indexables = indexables def __getitem__(self, idx: int) -> Union[Tuple[Indexable, ...], Indexable]: From 7826155915e6bff2659f792a2252947033246168 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 13:39:08 +0000 Subject: [PATCH 090/247] Add _gmm tests --- alibi_detect/od/_gmm.py | 133 +++++++++++++++++++ alibi_detect/od/pytorch/__init__.py | 1 + alibi_detect/od/sklearn/__init__.py | 1 + alibi_detect/od/sklearn/base.py | 46 ++++--- alibi_detect/od/tests/test__gmm/test__gmm.py | 71 ++++++++++ 5 files changed, 230 insertions(+), 22 deletions(-) create mode 100644 alibi_detect/od/_gmm.py create mode 100644 alibi_detect/od/sklearn/__init__.py create mode 100644 alibi_detect/od/tests/test__gmm/test__gmm.py diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py new file mode 100644 index 000000000..297acfcee --- /dev/null +++ b/alibi_detect/od/_gmm.py @@ -0,0 +1,133 @@ +from typing import Union, Optional, Dict, Any +from typing import TYPE_CHECKING + +import numpy as np + +from alibi_detect.utils._types import Literal +from alibi_detect.base import outlier_prediction_dict +from alibi_detect.od.base import OutlierDetector +from alibi_detect.od.pytorch import GMMTorch +from alibi_detect.od.sklearn import GMMSklearn +from alibi_detect.utils.frameworks import BackendValidator +from alibi_detect.version import __version__ + + +if TYPE_CHECKING: + import torch + + +backends = { + 'pytorch': GMMTorch, + 'sklearn': GMMSklearn +} + + +class GMM(OutlierDetector): + def __init__( + self, + n_components: int, + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, + backend: Literal['pytorch', 'sklearn'] = 'pytorch', + ) -> None: + """Gaussian Mixture Model (GMM) outlier detector. + + Parameters + ---------- + n_components: + The number of dimensions in the principle subspace. For linear pca should have + ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. + backend + Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'`` and ``'sklearn'``. + device + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + + Raises + ------ + NotImplementedError + If choice of `backend` is not implemented. + """ + super().__init__() + + backend_str: str = backend.lower() + BackendValidator( + backend_options={'pytorch': ['pytorch'], 'sklearn': ['sklearn']}, + construct_name=self.__class__.__name__ + ).verify_backend(backend_str) + + backend_cls = backends[backend] + args = {'n_components': n_components} + if backend == 'pytorch': + args['device'] = device + self.backend = backend_cls(**args) + + def fit(self, x_ref: np.ndarray, *args) -> None: + """Fit the detector on reference data. + + Parameters + ---------- + x_ref + Reference data used to fit the detector. + """ + self.backend.fit(self.backend._to_tensor(x_ref)) + + def score(self, x: np.ndarray) -> np.ndarray: + """Score `x` instances using the detector. + + Parameters + ---------- + x + Data to score. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + instance. + """ + score = self.backend.score(self.backend._to_tensor(x)) + return self.backend._to_numpy(score) + + def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + """Infer the threshold for the GMM detector. + + + Parameters + ---------- + x_ref + Reference data used to infer the threshold. + fpr + False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ + `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ + ``(0, 1)``. + """ + self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + + def predict(self, x: np.ndarray) -> Dict[str, Any]: + """Predict whether the instances in `x` are outliers or not. + + Parameters + ---------- + x + Data to predict. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ + performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ + the detector. + """ + outputs = self.backend.predict(self.backend._to_tensor(x)) + output = outlier_prediction_dict() + output['data'] = { + **output['data'], + **self.backend._to_numpy(outputs) + } + output['meta'] = { + **output['meta'], + 'name': self.__class__.__name__, + 'detector_type': 'outlier', + 'online': False, + 'version': __version__, + } + return output diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py index 971659255..28a6eef3c 100644 --- a/alibi_detect/od/pytorch/__init__.py +++ b/alibi_detect/od/pytorch/__init__.py @@ -4,5 +4,6 @@ MahalanobisTorch = import_optional('alibi_detect.od.pytorch.mahalanobis', ['MahalanobisTorch']) KernelPCATorch, LinearPCATorch = import_optional('alibi_detect.od.pytorch.pca', ['KernelPCATorch', 'LinearPCATorch']) Ensembler = import_optional('alibi_detect.od.pytorch.ensemble', ['Ensembler']) +GMMTorch = import_optional('alibi_detect.od.pytorch.gmm', ['GMMTorch']) to_numpy = import_optional('alibi_detect.od.pytorch.base', ['to_numpy']) diff --git a/alibi_detect/od/sklearn/__init__.py b/alibi_detect/od/sklearn/__init__.py new file mode 100644 index 000000000..be838d97b --- /dev/null +++ b/alibi_detect/od/sklearn/__init__.py @@ -0,0 +1 @@ +from alibi_detect.od.sklearn.gmm import GMMSklearn # noqa: F401 diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 20588bf9c..675ed0750 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -1,33 +1,14 @@ from __future__ import annotations from typing import List, Union, Optional -from dataclasses import dataclass +from dataclasses import dataclass, asdict from abc import ABC, abstractmethod -from alibi_detect.od.base import NotFitException - import numpy as np +from alibi_detect.od.base import NotFitException from alibi_detect.od.base import ThresholdNotInferredException -def to_numpy(arg): - """Map params to numpy arrays. - - This function is for interface compatibility with the other backends. As such it does nothing but - return the input. - - Parameters - ---------- - x - Data to convert. - - Returns - ------- - `np.ndarray` or dictionary of containing `numpy` arrays - """ - return arg - - @dataclass class SklearnOutlierDetectorOutput: """Output of the outlier detector.""" @@ -121,7 +102,28 @@ def check_threshold_infered(self): raise ThresholdNotInferredException((f'{self.__class__.__name__} has no threshold set, ' 'call `infer_threshold` before predicting.')) - def _to_tensor(self, x: Union[List, np.ndarray]) -> np.ndarray: + @staticmethod + def _to_numpy(arg): + """Map params to numpy arrays. + + This function is for interface compatibility with the other backends. As such it does nothing but + return the input. + + Parameters + ---------- + x + Data to convert. + + Returns + ------- + `np.ndarray` or dictionary of containing `numpy` arrays + """ + if isinstance(arg, SklearnOutlierDetectorOutput): + return asdict(arg) + return arg + + @staticmethod + def _to_tensor(x: Union[List, np.ndarray]) -> np.ndarray: """Converts the data to a tensor. This function is for interface compatibility with the other backends. As such it does nothing but diff --git a/alibi_detect/od/tests/test__gmm/test__gmm.py b/alibi_detect/od/tests/test__gmm/test__gmm.py new file mode 100644 index 000000000..575cc612b --- /dev/null +++ b/alibi_detect/od/tests/test__gmm/test__gmm.py @@ -0,0 +1,71 @@ +import pytest +import numpy as np +import torch + +from alibi_detect.od._gmm import GMM +from alibi_detect.od.base import NotFitException +from sklearn.datasets import make_moons + + +@pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) +def test_unfitted_gmm_single_score(backend): + gmm_detector = GMM(n_components=1, backend=backend) + x = np.array([[0, 10], [0.1, 0]]) + with pytest.raises(NotFitException) as err: + _ = gmm_detector.predict(x) + assert str(err.value) == f'{gmm_detector.backend.__class__.__name__} has not been fit!' + + +@pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) +def test_fitted_gmm_single_score(backend): + gmm_detector = GMM(n_components=1, backend=backend) + x_ref = np.random.randn(100, 2) + gmm_detector.fit(x_ref) + x = np.array([[0, 10], [0.1, 0]]) + y = gmm_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 2 + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None + + +@pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) +def test_fitted_gmm_predict(backend): + gmm_detector = GMM(n_components=1, backend=backend) + x_ref = np.random.randn(100, 2) + gmm_detector.fit(x_ref) + gmm_detector.infer_threshold(x_ref, 0.1) + x = np.array([[0, 10], [0, 0.1]]) + y = gmm_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 2 + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() + + +@pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) +def test_gmm_integration(backend): + gmm_detector = GMM(n_components=8, backend=backend) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + gmm_detector.fit(X_ref) + gmm_detector.infer_threshold(X_ref, 0.1) + result = gmm_detector.predict(x_inlier) + result = result['data']['is_outlier'][0] + assert not result + + x_outlier = np.array([[-1, 1.5]]) + result = gmm_detector.predict(x_outlier) + result = result['data']['is_outlier'][0] + assert result + + # ts_gmm = torch.jit.script(gmm_detector.backend) + # x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + # y = ts_gmm(x) + # assert torch.all(y == torch.tensor([False, True])) From 901ad53d707490b2c602d77f4c9a8c3b04009064 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 14:21:18 +0000 Subject: [PATCH 091/247] Add _to_numpy as a static method on base backend class --- alibi_detect/od/_knn.py | 6 +++--- alibi_detect/od/pytorch/base.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index ef813dc2a..e25f13b20 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -6,7 +6,7 @@ from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols -from alibi_detect.od.pytorch import KNNTorch, Ensembler, to_numpy +from alibi_detect.od.pytorch import KNNTorch, Ensembler from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -125,7 +125,7 @@ def score(self, x: np.ndarray) -> np.ndarray: instance. """ score = self.backend.score(self.backend._to_tensor(x)) - return to_numpy(score) + return self.backend._to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. @@ -165,7 +165,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: output = outlier_prediction_dict() output['data'] = { **output['data'], - **to_numpy(outputs) + **self.backend._to_numpy(outputs) } output['meta'] = { **output['meta'], diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 25d2f0b26..b30e79960 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -100,6 +100,23 @@ def check_threshold_infered(self): raise ThresholdNotInferredException((f'{self.__class__.__name__} has no threshold set, ' 'call `infer_threshold` before predicting.')) + @staticmethod + def _to_numpy(arg: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict]: + """Converts any `torch` tensors found in input to `numpy` arrays. + + Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays + + Parameters + ---------- + x + Data to convert. + + Returns + ------- + `np.ndarray` or dictionary of containing `numpy` arrays + """ + return to_numpy(arg) + def _to_tensor(self, x: Union[List, np.ndarray]) -> torch.Tensor: """Converts the data to a tensor. From a1b0a0d9127abe779634904288472b299d45e1d4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 14:23:10 +0000 Subject: [PATCH 092/247] Refactor mahalanobis to use staticmethod _to_numpy --- alibi_detect/od/_mahalanobis.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 7df808960..3b4b908d7 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -6,7 +6,6 @@ from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector -from alibi_detect.od.pytorch.base import to_numpy from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -96,7 +95,7 @@ def score(self, x: np.ndarray) -> np.ndarray: instance. """ score = self.backend.score(self.backend._to_tensor(x)) - return to_numpy(score) + return self.backend._to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. @@ -134,7 +133,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: output = outlier_prediction_dict() output['data'] = { **output['data'], - **to_numpy(outputs) + **self.backend._to_numpy(outputs) } output['meta'] = { **output['meta'], From 09f8faac26988a79ea98c83f04ea2a513016ffe6 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 14:24:29 +0000 Subject: [PATCH 093/247] Refactor PCA to use staticmethod _to_numpy --- alibi_detect/od/_pca.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 6a4f9d8a8..c8eb00eba 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -6,7 +6,6 @@ from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector -from alibi_detect.od.pytorch.base import to_numpy from alibi_detect.od.pytorch import KernelPCATorch, LinearPCATorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -112,7 +111,7 @@ def score(self, x: np.ndarray) -> np.ndarray: instance. """ score = self.backend.score(self.backend._to_tensor(x)) - return to_numpy(score) + return self.backend._to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. @@ -150,7 +149,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: output = outlier_prediction_dict() output['data'] = { **output['data'], - **to_numpy(outputs) + **self.backend._to_numpy(outputs) } output['meta'] = { **output['meta'], From 27230e36fe6ca74e369cc338ead39ce5a0d200cb Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 14:34:26 +0000 Subject: [PATCH 094/247] Add GMM to experimental namespace --- alibi_detect/experimental/od/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alibi_detect/experimental/od/__init__.py b/alibi_detect/experimental/od/__init__.py index 2b15dbac9..d96bea66e 100644 --- a/alibi_detect/experimental/od/__init__.py +++ b/alibi_detect/experimental/od/__init__.py @@ -1,3 +1,4 @@ from alibi_detect.od._knn import KNN # noqa F401 from alibi_detect.od._mahalanobis import Mahalanobis # noqa F401 -from alibi_detect.od._pca import PCA # noqa F401 \ No newline at end of file +from alibi_detect.od._pca import PCA # noqa F401 +from alibi_detect.od._gmm import GMM # noqa F401 From d64c6b9eba22036fe9e1a48d19a6fdbd1c8e96b5 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 15:45:53 +0000 Subject: [PATCH 095/247] Add args param to fit --- alibi_detect/od/_gmm.py | 2 +- alibi_detect/od/tests/test__gmm/test__gmm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 297acfcee..a49cd1790 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -69,7 +69,7 @@ def fit(self, x_ref: np.ndarray, *args) -> None: x_ref Reference data used to fit the detector. """ - self.backend.fit(self.backend._to_tensor(x_ref)) + self.backend.fit(self.backend._to_tensor(x_ref), *args) def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. diff --git a/alibi_detect/od/tests/test__gmm/test__gmm.py b/alibi_detect/od/tests/test__gmm/test__gmm.py index 575cc612b..d93697278 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm.py @@ -1,6 +1,6 @@ import pytest import numpy as np -import torch +# import torch from alibi_detect.od._gmm import GMM from alibi_detect.od.base import NotFitException From 5c56dc2057af0b230b166025946cb9d69c3dbc70 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 15:49:01 +0000 Subject: [PATCH 096/247] Make args kwargs --- alibi_detect/od/_gmm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index a49cd1790..24b7e4604 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -61,7 +61,7 @@ def __init__( args['device'] = device self.backend = backend_cls(**args) - def fit(self, x_ref: np.ndarray, *args) -> None: + def fit(self, x_ref: np.ndarray, *kwargs) -> None: """Fit the detector on reference data. Parameters @@ -69,7 +69,7 @@ def fit(self, x_ref: np.ndarray, *args) -> None: x_ref Reference data used to fit the detector. """ - self.backend.fit(self.backend._to_tensor(x_ref), *args) + self.backend.fit(self.backend._to_tensor(x_ref), *kwargs) def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. From 170a55f51127a764d3d515fb5c06837f3e07669c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 1 Feb 2023 16:10:45 +0000 Subject: [PATCH 097/247] Fix minor issue --- alibi_detect/od/_gmm.py | 4 ++-- alibi_detect/od/pytorch/ensemble.py | 4 ++-- alibi_detect/od/pytorch/gmm.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 24b7e4604..05024b5f6 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -61,7 +61,7 @@ def __init__( args['device'] = device self.backend = backend_cls(**args) - def fit(self, x_ref: np.ndarray, *kwargs) -> None: + def fit(self, x_ref: np.ndarray, **kwargs) -> None: """Fit the detector on reference data. Parameters @@ -69,7 +69,7 @@ def fit(self, x_ref: np.ndarray, *kwargs) -> None: x_ref Reference data used to fit the detector. """ - self.backend.fit(self.backend._to_tensor(x_ref), *kwargs) + self.backend.fit(self.backend._to_tensor(x_ref), **kwargs) def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 2ca6524da..15a757efa 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -45,9 +45,9 @@ def __init__(self): """ super().__init__() - def fit(self, x: torch.Tensor) -> FitMixinTorch: + def fit(self, x: torch.Tensor, **kwargs: dict) -> FitMixinTorch: self._fitted = True - self._fit(x) + self._fit(x, **kwargs) return self @abstractmethod diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 029c06f71..7b49e338c 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -57,7 +57,7 @@ def _fit( optimizer.step() # type: ignore if verbose == 1 and isinstance(dl, tqdm): loss_ma = loss_ma + (nll.item() - loss_ma) / (step + 1) - dl.set_description(f'Epoch {epoch + 1}/{self.epochs}') + dl.set_description(f'Epoch {epoch + 1}/{epochs}') dl.set_postfix(dict(loss_ma=loss_ma)) def forward(self, x: torch.Tensor) -> torch.Tensor: From 5a92bb0e16c353d1177cd0fd9c604cd9fc286eb5 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 2 Feb 2023 11:55:44 +0000 Subject: [PATCH 098/247] Fix typing issues --- alibi_detect/od/_gmm.py | 4 ++-- alibi_detect/od/base.py | 2 +- alibi_detect/od/pytorch/base.py | 2 +- alibi_detect/od/pytorch/gmm.py | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 05024b5f6..b0285fc7a 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -56,12 +56,12 @@ def __init__( ).verify_backend(backend_str) backend_cls = backends[backend] - args = {'n_components': n_components} + args: Dict[str, Any] = {'n_components': n_components} if backend == 'pytorch': args['device'] = device self.backend = backend_cls(**args) - def fit(self, x_ref: np.ndarray, **kwargs) -> None: + def fit(self, x_ref: np.ndarray, **kwargs: Dict) -> None: """Fit the detector on reference data. Parameters diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 32e990bc5..b01e0759b 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -18,7 +18,7 @@ def __init__(self): self.meta['detector_type'] = 'outlier' @abstractmethod - def fit(self, x: np.ndarray) -> None: + def fit(self, x_ref: np.ndarray) -> None: """ Fit outlier detector to data. diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index b30e79960..f85568670 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -101,7 +101,7 @@ def check_threshold_infered(self): 'call `infer_threshold` before predicting.')) @staticmethod - def _to_numpy(arg: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict]: + def _to_numpy(arg: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict[str, np.ndarray]]: """Converts any `torch` tensors found in input to `numpy` arrays. Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 7b49e338c..b394ef3de 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional +from typing import Callable, Optional, Union from tqdm import tqdm import torch from torch.utils.data import DataLoader @@ -13,7 +13,7 @@ class GMMTorch(TorchOutlierDetector): def __init__( self, n_components: int, - device: Optional[str] = None, + device: Optional[Union[str, torch.device]] = None ) -> None: """ Fits a Gaussian mixture model to the training data and scores new data points @@ -41,13 +41,13 @@ def _fit( self.model = GMMModel(self.n_components, X.shape[-1]) X = X.to(torch.float32) - ds = TorchDataset(X) - dl = DataLoader(ds, batch_size=batch_size, shuffle=True) + dataset = TorchDataset(X) + dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True) optimizer = optimizer(self.model.parameters(), lr=learning_rate) self.model.train() for epoch in range(epochs): - dl = tqdm(enumerate(dl), total=len(dl)) if verbose == 1 else enumerate(dl) + dl = tqdm(enumerate(dataloader), total=len(dataloader), disable=not verbose) loss_ma = 0 for step, x in dl: x = x.to(self.device) From 90318e2de8025aac101ac9fc4c4bf1d909f7e6cf Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 2 Feb 2023 12:06:31 +0000 Subject: [PATCH 099/247] Import mahalanobis from module __init__ --- alibi_detect/od/_mahalanobis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 3b4b908d7..894c5a4a5 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -6,7 +6,7 @@ from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import OutlierDetector -from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch +from alibi_detect.od.pytorch import MahalanobisTorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ From 7b939fa837b860f92990a0dbab8dc29bf01a5619 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 2 Feb 2023 13:29:53 +0000 Subject: [PATCH 100/247] Add device logic to gmm torch --- alibi_detect/od/pytorch/gmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index b394ef3de..4c67c155d 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -38,7 +38,7 @@ def _fit( epochs: int = 10, verbose: int = 0, ) -> None: - self.model = GMMModel(self.n_components, X.shape[-1]) + self.model = GMMModel(self.n_components, X.shape[-1]).to(self.device) X = X.to(torch.float32) dataset = TorchDataset(X) From 0f415b63b71fc36b177bdd738a3aab9bc90cb334 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 2 Feb 2023 15:54:19 +0000 Subject: [PATCH 101/247] Minor change to docstring --- alibi_detect/od/pytorch/gmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 4c67c155d..6470feb72 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -17,7 +17,7 @@ def __init__( ) -> None: """ Fits a Gaussian mixture model to the training data and scores new data points - via the negative log-likhood under the corresponding density function. + via the negative log-liklihood under the corresponding density function. Parameters ---------- n_components: From 049f93fb2eef9797fa9659ab5079b255444f50f8 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 3 Feb 2023 14:19:36 +0000 Subject: [PATCH 102/247] Remove OutlierDetector --- alibi_detect/base.py | 18 +++- alibi_detect/od/_knn.py | 18 ++-- alibi_detect/od/base.py | 85 +------------------ alibi_detect/od/pytorch/base.py | 2 +- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/tests/test__knn/test__knn.py | 2 +- .../od/tests/test__knn/test__knn_backend.py | 2 +- alibi_detect/od/tests/test_ensemble.py | 2 +- 8 files changed, 32 insertions(+), 99 deletions(-) diff --git a/alibi_detect/base.py b/alibi_detect/base.py index a061efa6b..068059fb1 100644 --- a/alibi_detect/base.py +++ b/alibi_detect/base.py @@ -87,14 +87,14 @@ def predict(self, X: np.ndarray): class FitMixin(ABC): @abstractmethod - def fit(self, X: np.ndarray) -> None: - pass + def fit(self, *args, **kwargs) -> None: + ... class ThresholdMixin(ABC): @abstractmethod - def infer_threshold(self, X: np.ndarray) -> None: - pass + def infer_threshold(self, *args, **kwargs) -> None: + ... # "Large artefacts" - to save memory these are skipped in _set_config(), but added back in get_config() @@ -264,3 +264,13 @@ def default(self, obj): elif isinstance(obj, (np.ndarray,)): return obj.tolist() return json.JSONEncoder.default(self, obj) + + +class NotFitException(Exception): + """Exception raised when a transform is not fitted.""" + pass + + +class ThresholdNotInferredException(Exception): + """Exception raised when a threshold not inferred for an outlier detector.""" + pass diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index e25f13b20..4eb384b46 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -5,7 +5,8 @@ from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict -from alibi_detect.od.base import OutlierDetector, TransformProtocol, transform_protocols +from alibi_detect.od.base import TransformProtocol, transform_protocols +from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin from alibi_detect.od.pytorch import KNNTorch, Ensembler from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer from alibi_detect.utils.frameworks import BackendValidator @@ -21,7 +22,7 @@ } -class KNN(OutlierDetector): +class KNN(BaseDetector, FitMixin, ThresholdMixin): def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], @@ -97,6 +98,11 @@ def __init__( self.backend = backend_cls(k, kernel=kernel, ensembler=ensembler, device=device) + # set metadata + self.meta['detector_type'] = 'outlier' + self.meta['data_type'] = 'numeric' + self.meta['online'] = False + def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. @@ -107,7 +113,7 @@ def fit(self, x_ref: np.ndarray) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) - def score(self, x: np.ndarray) -> np.ndarray: + def score(self, X: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. Computes the k nearest neighbor distance/kernel similarity for each instance in `x`. If `k` is a single @@ -124,10 +130,10 @@ def score(self, x: np.ndarray) -> np.ndarray: Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. """ - score = self.backend.score(self.backend._to_tensor(x)) + score = self.backend.score(self.backend._to_tensor(X)) return self.backend._to_numpy(score) - def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + def infer_threshold(self, X: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. The threshold is computed so that the outlier detector would incorectly classify `fpr` proportion of the @@ -142,7 +148,7 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: instances in `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range ``(0, 1)``. """ - self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + self.backend.infer_threshold(self.backend._to_tensor(X), fpr) def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 32e990bc5..5edb1eba6 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -1,80 +1,7 @@ from __future__ import annotations -from abc import ABC, abstractmethod -from typing import Dict, Any, Union +from typing import Union from typing_extensions import Protocol, runtime_checkable -import numpy as np - -from alibi_detect.base import BaseDetector - - -class OutlierDetector(BaseDetector, ABC): - threshold_inferred = False - - def __init__(self): - """ Base class for outlier detection algorithms.""" - super().__init__() - self.meta['online'] = False - self.meta['detector_type'] = 'outlier' - - @abstractmethod - def fit(self, x: np.ndarray) -> None: - """ - Fit outlier detector to data. - - Parameters - ---------- - x - Reference data. - """ - pass - - @abstractmethod - def score(self, x: np.ndarray) -> np.ndarray: - """ - Compute outlier scores of the instances in `x`. - - Parameters - ---------- - x - Data to score. - - Returns - ------- - Outlier scores. The higher the score, the more outlying the instance. - """ - pass - - @abstractmethod - def infer_threshold(self, x: np.ndarray, fpr: float) -> None: - """ - Infer the threshold for the outlier detector. - - Parameters - ---------- - x - Reference data. - fpr - False positive rate used to infer the threshold. - """ - pass - - @abstractmethod - def predict(self, x: np.ndarray) -> Dict[str, Any]: - """ - Predict whether the instances in `x` are outliers or not. - - Parameters - ---------- - x - Data to predict. - - Returns - ------- - Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ - performed, 'data' also contains the outlier labels. - """ - pass # Use Protocols instead of base classes for the backend associated objects. This is a bit more flexible and allows us to @@ -103,13 +30,3 @@ def check_fitted(self): transform_protocols = Union[TransformProtocol, FittedTransformProtocol] - - -class NotFitException(Exception): - """Exception raised when a transform is not fitted.""" - pass - - -class ThresholdNotInferredException(Exception): - """Exception raised when a threshold not inferred for an outlier detector.""" - pass diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index b30e79960..e2b58947b 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -10,7 +10,7 @@ from alibi_detect.od.pytorch.ensemble import FitMixinTorch from alibi_detect.utils.pytorch.misc import get_device -from alibi_detect.od.base import ThresholdNotInferredException +from alibi_detect.base import ThresholdNotInferredException @dataclass diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 2ca6524da..38886f67a 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -6,7 +6,7 @@ import numpy as np from torch.nn import Module -from alibi_detect.od.base import NotFitException +from alibi_detect.base import NotFitException class BaseTransformTorch(Module, ABC): diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index f9f78ba20..d9db4581e 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -5,7 +5,7 @@ from alibi_detect.od._knn import KNN from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer -from alibi_detect.od.base import NotFitException +from alibi_detect.base import NotFitException from sklearn.datasets import make_moons diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 52259fb59..8cfd35e26 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -4,7 +4,7 @@ from alibi_detect.od.pytorch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator -from alibi_detect.od.base import NotFitException, ThresholdNotInferredException +from alibi_detect.base import NotFitException, ThresholdNotInferredException @pytest.fixture(scope='session') diff --git a/alibi_detect/od/tests/test_ensemble.py b/alibi_detect/od/tests/test_ensemble.py index a0f93b39a..f53f32569 100644 --- a/alibi_detect/od/tests/test_ensemble.py +++ b/alibi_detect/od/tests/test_ensemble.py @@ -2,7 +2,7 @@ import torch from alibi_detect.od.pytorch import ensemble -from alibi_detect.od.base import NotFitException +from alibi_detect.base import NotFitException def test_pval_normalizer(): From c1a2ea8a8ebf8e0b99452be2839f2e947698b764 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 3 Feb 2023 15:19:00 +0000 Subject: [PATCH 103/247] Add comments to test and remove duplicate test --- .../od/tests/test__knn/test__knn_backend.py | 67 +++++++++++-------- alibi_detect/od/tests/test_ensemble.py | 27 +++++++- 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 8cfd35e26..2ebf8d114 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -16,6 +16,9 @@ def ensembler(request): def test_knn_torch_backend(): + # Test the knn torch backend can be correctly initialized, fit and used to + # predict outliers. + knn_torch = KNNTorch(k=5) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) x_ref = torch.randn((1024, 10)) @@ -34,6 +37,9 @@ def test_knn_torch_backend(): def test_knn_torch_backend_ensemble(ensembler): + # Test the knn torch backend can be correctly initialized as an ensemble, fit + # on data and used to predict outliers. + knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) @@ -48,6 +54,9 @@ def test_knn_torch_backend_ensemble(ensembler): def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): + # Test the knn torch backend can be initalized as an ensemble and + # torchscripted, as well as saved and loaded to and from disk. + knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) @@ -74,6 +83,9 @@ def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): def test_knn_torch_backend_ts(tmp_path): + # Test the knn torch backend can be initalized and torchscripted, as well as + # saved and loaded to and from disk. + knn_torch = KNNTorch(k=7) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) x_ref = torch.randn((1024, 10)) @@ -91,6 +103,9 @@ def test_knn_torch_backend_ts(tmp_path): def test_knn_kernel(ensembler): + # Test the knn torch backend can be correctly initialized with a kernel, fit + # on data and used to predict outliers. + kernel = GaussianRBF(sigma=torch.tensor((0.25))) knn_torch = KNNTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) x_ref = torch.randn((1024, 10)) @@ -104,58 +119,52 @@ def test_knn_kernel(ensembler): assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) assert torch.all(knn_torch(x) == torch.tensor([False, False, True])) - """Can't convert GaussianRBF to torchscript due to torchscript type - constraints""" - # pred_1 = knn_torch(x) - # knn_torch = torch.jit.script(knn_torch) - # pred_2 = knn_torch(x) - # assert torch.all(pred_1 == pred_2) - - -def test_knn_torch_backend_ensemble_fit_errors(ensembler): - knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) - assert not knn_torch._fitted - x = torch.randn((1, 10)) - with pytest.raises(NotFitException) as err: - knn_torch(x) - assert str(err.value) == 'KNNTorch has not been fit!' - - with pytest.raises(NotFitException) as err: - knn_torch.predict(x) - assert str(err.value) == 'KNNTorch has not been fit!' +@pytest.mark.skip(reason="Can't convert GaussianRBF to torchscript due to torchscript type constraints") +def test_knn_kernel_ts(ensembler): + # Test the knn torch backend can be correctly initialized with a kernel, + # and torchscripted, as well as saved and loaded to and from disk. + kernel = GaussianRBF(sigma=torch.tensor((0.25))) + knn_torch = KNNTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) - assert knn_torch._fitted - - with pytest.raises(ThresholdNotInferredException) as err: - knn_torch(x) - assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' - - assert knn_torch.predict(x) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + knn_torch.infer_threshold(x_ref, 0.1) + pred_1 = knn_torch(x) + knn_torch = torch.jit.script(knn_torch) + pred_2 = knn_torch(x) + assert torch.all(pred_1 == pred_2) -def test_knn_torch_backend_fit_errors(): - knn_torch = KNNTorch(k=4) +@pytest.mark.parametrize('k', [[4, 5], 4]) +def test_knn_torch_backend_ensemble_fit_errors(k, ensembler): + knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) assert not knn_torch._fitted + # Test that the backend raises an error if it is not fitted before + # calling forward method. x = torch.randn((1, 10)) with pytest.raises(NotFitException) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has not been fit!' + # Test that the backend raises an error if it is not fitted before + # predicting. with pytest.raises(NotFitException) as err: knn_torch.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' + # Test the backend updates _fitted flag on fit. x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) - assert knn_torch._fitted + # Test that the backend raises an if the forward method is called without the + # threshold being inferred. with pytest.raises(ThresholdNotInferredException) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + # Test that the backend can call predict without the threshold being inferred. assert knn_torch.predict(x) diff --git a/alibi_detect/od/tests/test_ensemble.py b/alibi_detect/od/tests/test_ensemble.py index f53f32569..e5f350267 100644 --- a/alibi_detect/od/tests/test_ensemble.py +++ b/alibi_detect/od/tests/test_ensemble.py @@ -17,7 +17,8 @@ def test_pval_normalizer(): normalizer.fit(x_ref) x_norm = normalizer(x) - # check that the p-values are correct + # compute the p-values explicitly and compare to the normalizer + # output. assert torch.all(0 < x_norm) assert torch.all(x_norm < 1) for i in range(3): @@ -27,6 +28,7 @@ def test_pval_normalizer(): normalizer_pval = x_norm[i][j].to(torch.float32) assert torch.isclose(1 - comp_pval, normalizer_pval, atol=1e-4) + # Test the scriptability of the normalizer normalizer = torch.jit.script(normalizer) x_norm_2 = normalizer(x) assert torch.all(x_norm_2 == x_norm) @@ -36,16 +38,19 @@ def test_shift_and_scale_normalizer(): normalizer = ensemble.ShiftAndScaleNormalizer() x = torch.randn(3, 10) * 3 + 2 x_ref = torch.randn(5000, 10) * 3 + 2 + # unfit normalizer raises exception with pytest.raises(NotFitException) as err: normalizer(x) assert err.value.args[0] == 'ShiftAndScaleNormalizer has not been fit!' + # test the normalizer correctly shifts and scales the data normalizer.fit(x_ref) x_norm = normalizer(x) assert torch.isclose(x_norm.mean(), torch.tensor(0.), atol=0.1) assert torch.isclose(x_norm.std(), torch.tensor(1.), atol=0.1) + # Test the scriptability of the normalizer normalizer = torch.jit.script(normalizer) x_norm_2 = normalizer(x) assert torch.all(x_norm_2 == x_norm) @@ -54,10 +59,13 @@ def test_shift_and_scale_normalizer(): def test_average_aggregator(): aggregator = ensemble.AverageAggregator() scores = torch.randn((3, 10)) + + # test the aggregator correctly averages the scores aggregated_scores = aggregator(scores) assert torch.all(torch.isclose(aggregated_scores, scores.mean(dim=1))) assert aggregated_scores.shape == (3, ) + # test the scriptability of the aggregator aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -70,6 +78,8 @@ def test_weighted_average_aggregator(): aggregator = ensemble.AverageAggregator(weights=weights) assert err.value.args[0] == 'Weights must sum to 1.' + # test the aggregator correctly weights the scores when computing the + # average weights /= weights.sum() aggregator = ensemble.AverageAggregator(weights=weights) scores = torch.randn((3, 10)) @@ -77,6 +87,7 @@ def test_weighted_average_aggregator(): torch.allclose(aggregated_scores, (weights @ scores.T)) assert aggregated_scores.shape == (3, ) + # test the scriptability of the aggregator aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -85,11 +96,14 @@ def test_weighted_average_aggregator(): def test_topk_aggregator(): aggregator = ensemble.TopKAggregator(k=4) scores = torch.randn((3, 10)) + + # test the aggregator correctly computes the top k scores aggregated_scores = aggregator(scores) assert aggregated_scores.shape == (3, ) scores_sorted, _ = torch.sort(scores) torch.allclose(scores_sorted[:, -4:].mean(dim=1), aggregated_scores) + # test the scriptability of the aggregator aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -98,10 +112,14 @@ def test_topk_aggregator(): def test_max_aggregator(): aggregator = ensemble.MaxAggregator() scores = torch.randn((3, 10)) + + # test the aggregator correctly computes the max scores aggregated_scores = aggregator(scores) assert aggregated_scores.shape == (3, ) max_vals, _ = scores.max(dim=1) torch.all(max_vals == aggregated_scores) + + # test the scriptability of the aggregator aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -110,10 +128,14 @@ def test_max_aggregator(): def test_min_aggregator(): aggregator = ensemble.MinAggregator() scores = torch.randn((3, 10)) + + # test the aggregator correctly computes the min scores aggregated_scores = aggregator(scores) assert aggregated_scores.shape == (3, ) min_vals, _ = scores.min(dim=1) torch.all(min_vals == aggregated_scores) + + # test the scriptability of the aggregator aggregator = torch.jit.script(aggregator) aggregated_scores_2 = aggregator(scores) assert torch.all(aggregated_scores_2 == aggregated_scores) @@ -129,8 +151,11 @@ def test_ensembler(aggregator, normalizer): x = torch.randn(3, 10) x_ref = torch.randn(64, 10) + # test the ensembler correctly aggregates and normalizes the scores ensembler.fit(x_ref) x_norm = ensembler(x) + + # test the scriptability of the ensembler ensembler = torch.jit.script(ensembler) x_norm_2 = ensembler(x) assert torch.all(x_norm_2 == x_norm) From 885201f8aeafd5a7894fce534d27b4335b2556e0 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 3 Feb 2023 15:38:25 +0000 Subject: [PATCH 104/247] Add further comments to _knn tests --- alibi_detect/od/tests/test__knn/test__knn.py | 51 ++++++++++++------- .../od/tests/test__knn/test__knn_backend.py | 2 +- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index d9db4581e..e8ca682ed 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -24,32 +24,22 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): def test_unfitted_knn_single_score(): knn_detector = KNN(k=10) x = np.array([[0, 10], [0.1, 0]]) + + # test predict raises exception when not fitted with pytest.raises(NotFitException) as err: _ = knn_detector.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' -def test_fitted_knn_single_score(): - knn_detector = KNN(k=10) +@pytest.mark.parametrize('k', [10, [8, 9, 10]]) +def test_fitted_knn_single_score(k): + knn_detector = KNN(k=k) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) - y = knn_detector.predict(x) - y = y['data'] - assert y['instance_score'][0] > 5 - assert y['instance_score'][1] < 1 - assert not y['threshold_inferred'] - assert y['threshold'] is None - assert y['is_outlier'] is None - assert y['p_value'] is None - - -def test_default_knn_ensemble_init(): - knn_detector = KNN(k=[8, 9, 10]) - x_ref = np.random.randn(100, 2) - knn_detector.fit(x_ref) - x = np.array([[0, 10], [0.1, 0]]) + # test fitted but not threshold inferred detectors + # can still score data using the predict method. y = knn_detector.predict(x) y = y['data'] assert y['instance_score'][0] > 5 @@ -62,6 +52,8 @@ def test_default_knn_ensemble_init(): def test_incorrect_knn_ensemble_init(): + # test knn ensemble with aggregator passed as None raises exception + with pytest.raises(ValueError) as err: KNN(k=[8, 9, 10], aggregator=None) assert str(err.value) == ('If `k` is a `np.ndarray`, `list` or `tuple`, ' @@ -71,6 +63,9 @@ def test_incorrect_knn_ensemble_init(): def test_fitted_knn_predict(): knn_detector = make_knn_detector(k=10) x_ref = np.random.randn(100, 2) + + # test detector fitted on data and with threshold inferred correctly scores and + # labels outliers, as well as return the p-values using the predict method. knn_detector.infer_threshold(x_ref, 0.1) x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) @@ -93,6 +88,8 @@ def test_unfitted_knn_ensemble(aggregator, normalizer): normalizer=normalizer() ) x = np.array([[0, 10], [0.1, 0]]) + + # Test unfit knn ensemble raises exception when calling predict method. with pytest.raises(NotFitException) as err: _ = knn_detector.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -110,6 +107,8 @@ def test_fitted_knn_ensemble(aggregator, normalizer): x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0, 0.1]]) + + # test fitted but not threshold inferred detectors can still score data using the predict method. y = knn_detector.predict(x) y = y['data'] assert y['instance_score'].all() @@ -129,6 +128,8 @@ def test_fitted_knn_ensemble_predict(aggregator, normalizer): normalizer=normalizer() ) x = np.array([[0, 10], [0, 0.1]]) + + # test fitted detectors with inferred thresholds can score data using the predict method. y = knn_detector.predict(x) y = y['data'] assert y['threshold_inferred'] @@ -144,6 +145,8 @@ def test_knn_ensemble_torch_script(aggregator, normalizer): knn_detector = make_knn_detector(k=[5, 6, 7], aggregator=aggregator(), normalizer=normalizer()) tsknn = torch.jit.script(knn_detector.backend) x = torch.tensor([[0, 10], [0, 0.1]]) + + # test torchscripted ensemble knn detector can be saved and loaded correctly. y = tsknn(x) assert torch.all(y == torch.tensor([True, False])) @@ -152,6 +155,8 @@ def test_knn_single_torchscript(): knn_detector = make_knn_detector(k=5) tsknn = torch.jit.script(knn_detector.backend) x = torch.tensor([[0, 10], [0, 0.1]]) + + # test torchscripted single knn detector can be saved and loaded correctly. y = tsknn(x) assert torch.all(y == torch.tensor([True, False])) @@ -163,6 +168,13 @@ def test_knn_single_torchscript(): @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None, lambda: 'ShiftAndScaleNormalizer', lambda: 'PValNormalizer']) def test_knn_ensemble_integration(aggregator, normalizer): + """Test knn ensemble detector on moons dataset. + + Tests ensemble knn detector with every combination of aggregator and normalizer on the moons dataset. + Fits and infers thresholds in each case. Verifies that the detector can correctly detect inliers + and outliers and that it can be serialized using the torchscript. + """ + knn_detector = KNN( k=[10, 14, 18], aggregator=aggregator(), @@ -188,6 +200,11 @@ def test_knn_ensemble_integration(aggregator, normalizer): def test_knn_integration(): + """Test knn detector on moons dataset. + + Tests knn detector on the moons dataset. Fits and infers thresholds and verifies that the detector can + correctly detect inliers and outliers. Checks that it can be serialized using the torchscript. + """ knn_detector = KNN(k=18) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 2ebf8d114..14a107d07 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -16,7 +16,7 @@ def ensembler(request): def test_knn_torch_backend(): - # Test the knn torch backend can be correctly initialized, fit and used to + # Test the knn torch backend can be correctly initialized, fit and used to # predict outliers. knn_torch = KNNTorch(k=5) From 8667af5f8fccc4def24652f284211e22732d6445 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 3 Feb 2023 15:40:00 +0000 Subject: [PATCH 105/247] Correct method name spelling --- alibi_detect/od/pytorch/base.py | 2 +- alibi_detect/od/pytorch/knn.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index e2b58947b..d0bcdc559 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -88,7 +88,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: pass @torch.jit.unused - def check_threshold_infered(self): + def check_threshold_inferred(self): """Check if threshold is inferred. Raises diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index d120e884c..ed5ae13c3 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -61,7 +61,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: raw_scores = self.score(x) scores = self._ensembler(raw_scores) if not torch.jit.is_scripting(): - self.check_threshold_infered() + self.check_threshold_inferred() preds = scores > self.threshold return preds.cpu() From fb7584a6a28215cd58e7b7c7d947d571f05a027d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 3 Feb 2023 15:45:31 +0000 Subject: [PATCH 106/247] Fix return in docstring --- alibi_detect/od/pytorch/knn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index ed5ae13c3..76714fee0 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -75,7 +75,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - Tensor of scores for each element in `x`. + Tensor of scores for each element in `x`. Raises ------ From cddb4706af6075e60507566af4a4e795ea0b8df6 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 3 Feb 2023 15:49:13 +0000 Subject: [PATCH 107/247] Add no grad decorator to backend methods --- alibi_detect/od/pytorch/ensemble.py | 1 + alibi_detect/od/pytorch/knn.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 38886f67a..b4896135b 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -31,6 +31,7 @@ def _transform(self, x: torch.Tensor): """ pass + @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: return self.transform(x=x) diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index 76714fee0..3f412f58c 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -41,6 +41,7 @@ def __init__( self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) self.ensembler = ensembler + @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -63,8 +64,9 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: if not torch.jit.is_scripting(): self.check_threshold_inferred() preds = scores > self.threshold - return preds.cpu() + return preds + @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` From 910584332ea4fbc99bf61d9808b3eb502d2a0d2f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 6 Feb 2023 09:12:54 +0000 Subject: [PATCH 108/247] Add minor fixes from merging knn branch --- alibi_detect/od/_mahalanobis.py | 5 ++--- alibi_detect/od/pytorch/mahalanobis.py | 2 +- alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py | 2 +- .../od/tests/test__mahalanobis/test__mahalanobis_backend.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 894c5a4a5..597dd6a04 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -4,8 +4,7 @@ import numpy as np from alibi_detect.utils._types import Literal -from alibi_detect.base import outlier_prediction_dict -from alibi_detect.od.base import OutlierDetector +from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin, outlier_prediction_dict from alibi_detect.od.pytorch import MahalanobisTorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -20,7 +19,7 @@ } -class Mahalanobis(OutlierDetector): +class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin,): def __init__( self, min_eigenvalue: float = 1e-6, diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index fec3b569c..837b076f0 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -43,7 +43,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: """ scores = self.score(x) if not torch.jit.is_scripting(): - self.check_threshold_infered() + self.check_threshold_inferred() preds = scores > self.threshold return preds.cpu() diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index e4dc3407b..228ec3509 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -3,7 +3,7 @@ import torch from alibi_detect.od._mahalanobis import Mahalanobis -from alibi_detect.od.base import NotFitException +from alibi_detect.base import NotFitException from sklearn.datasets import make_moons diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py index 063f4b8a7..ec678e739 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py @@ -3,7 +3,7 @@ import numpy as np from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch -from alibi_detect.od.base import NotFitException, ThresholdNotInferredException +from alibi_detect.base import NotFitException, ThresholdNotInferredException def test_mahalanobis_torch_backend_fit_errors(): From 5ade8e7c0801ea1c8ac4b76cca5e1a179f619007 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 6 Feb 2023 09:14:53 +0000 Subject: [PATCH 109/247] Minor fix --- alibi_detect/od/_mahalanobis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 597dd6a04..e7b24909d 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -19,7 +19,7 @@ } -class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin,): +class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin): def __init__( self, min_eigenvalue: float = 1e-6, From cae15ac1a4b5af32dca5b1a015848bbd722b1eb5 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 6 Feb 2023 09:18:29 +0000 Subject: [PATCH 110/247] Add fixes for merged pca branch --- alibi_detect/od/_pca.py | 4 ++-- alibi_detect/od/pytorch/pca.py | 2 +- alibi_detect/od/tests/test__pca/test__pca.py | 2 +- alibi_detect/od/tests/test__pca/test__pca_backend.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index c8eb00eba..21e497ffd 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -5,7 +5,7 @@ from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict -from alibi_detect.od.base import OutlierDetector +from alibi_detect.base import BaseDetector, ThresholdMixin, FitMixin from alibi_detect.od.pytorch import KernelPCATorch, LinearPCATorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -20,7 +20,7 @@ } -class PCA(OutlierDetector): +class PCA(BaseDetector, ThresholdMixin, FitMixin): def __init__( self, n_components: int, diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index f300627b6..226194522 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -44,7 +44,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: """ scores = self.score(x) if not torch.jit.is_scripting(): - self.check_threshold_infered() + self.check_threshold_inferred() preds = scores > self.threshold return preds.cpu() diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index 0960ef94d..92270bf8c 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -4,7 +4,7 @@ from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od._pca import PCA -from alibi_detect.od.base import NotFitException +from alibi_detect.base import NotFitException from sklearn.datasets import make_moons diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py index 953d65aa1..2a0613e95 100644 --- a/alibi_detect/od/tests/test__pca/test__pca_backend.py +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -4,7 +4,7 @@ from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.pca import LinearPCATorch, KernelPCATorch -from alibi_detect.od.base import NotFitException, ThresholdNotInferredException +from alibi_detect.base import NotFitException, ThresholdNotInferredException @pytest.mark.parametrize('backend_detector', [ From 2899c90749cc9ca24abb91db7306e3474e6287b7 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 6 Feb 2023 09:25:09 +0000 Subject: [PATCH 111/247] Fix merge issues from pca --- alibi_detect/od/_gmm.py | 4 ++-- alibi_detect/od/pytorch/gmm.py | 2 +- alibi_detect/od/sklearn/base.py | 3 +-- alibi_detect/od/tests/test__gmm/test__gmm.py | 2 +- alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py | 2 +- alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index b0285fc7a..9a7bffa74 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -5,7 +5,7 @@ from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict -from alibi_detect.od.base import OutlierDetector +from alibi_detect.base import BaseDetector, ThresholdMixin, FitMixin from alibi_detect.od.pytorch import GMMTorch from alibi_detect.od.sklearn import GMMSklearn from alibi_detect.utils.frameworks import BackendValidator @@ -22,7 +22,7 @@ } -class GMM(OutlierDetector): +class GMM(BaseDetector, ThresholdMixin, FitMixin): def __init__( self, n_components: int, diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 6470feb72..e4fec439e 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -80,7 +80,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: raw_scores = self.score(x) scores = self._ensembler(raw_scores) if not torch.jit.is_scripting(): - self.check_threshold_infered() + self.check_threshold_inferred() preds = scores > self.threshold return preds.cpu() diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 675ed0750..bb95cbadb 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -5,8 +5,7 @@ import numpy as np -from alibi_detect.od.base import NotFitException -from alibi_detect.od.base import ThresholdNotInferredException +from alibi_detect.base import NotFitException, ThresholdNotInferredException @dataclass diff --git a/alibi_detect/od/tests/test__gmm/test__gmm.py b/alibi_detect/od/tests/test__gmm/test__gmm.py index d93697278..2eae5d7c5 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm.py @@ -3,7 +3,7 @@ # import torch from alibi_detect.od._gmm import GMM -from alibi_detect.od.base import NotFitException +from alibi_detect.base import NotFitException from sklearn.datasets import make_moons diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py index 4c36d7ddb..d2009119e 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py @@ -3,7 +3,7 @@ import torch from alibi_detect.od.pytorch.gmm import GMMTorch -from alibi_detect.od.base import NotFitException, ThresholdNotInferredException +from alibi_detect.base import NotFitException, ThresholdNotInferredException def test_gmm_pytorch_backend_fit_errors(): diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py index 69bac21e2..b8456ecc4 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -2,7 +2,7 @@ import numpy as np from alibi_detect.od.sklearn.gmm import GMMSklearn -from alibi_detect.od.base import NotFitException, ThresholdNotInferredException +from alibi_detect.base import NotFitException, ThresholdNotInferredException def test_gmm_sklearn_backend_fit_errors(): From 6af7d37cb0890d3acc482a34794e94865e9ec474 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 6 Feb 2023 09:27:10 +0000 Subject: [PATCH 112/247] Fix minor linting issue --- alibi_detect/od/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index eaafee3c7..5edb1eba6 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -3,6 +3,7 @@ from typing_extensions import Protocol, runtime_checkable + # Use Protocols instead of base classes for the backend associated objects. This is a bit more flexible and allows us to # avoid the torch/tensorflow imports in the base class. @runtime_checkable From 04da0a2b3289b5589221e2aea5d7a954957da588 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 13 Feb 2023 09:40:49 +0000 Subject: [PATCH 113/247] Use docstring in test instead of comments --- alibi_detect/od/tests/test__knn/test__knn.py | 8 +++-- .../od/tests/test__knn/test__knn_backend.py | 36 ++++++++++++------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index e8ca682ed..8a3ddc377 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -61,11 +61,13 @@ def test_incorrect_knn_ensemble_init(): def test_fitted_knn_predict(): + """ + test that a detector fitted on data and with threshold inferred correctly, will score + and label outliers, as well as return the p-values using the predict method. + """ + knn_detector = make_knn_detector(k=10) x_ref = np.random.randn(100, 2) - - # test detector fitted on data and with threshold inferred correctly scores and - # labels outliers, as well as return the p-values using the predict method. knn_detector.infer_threshold(x_ref, 0.1) x = np.array([[0, 10], [0, 0.1]]) y = knn_detector.predict(x) diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 14a107d07..9138059b4 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -16,8 +16,10 @@ def ensembler(request): def test_knn_torch_backend(): - # Test the knn torch backend can be correctly initialized, fit and used to - # predict outliers. + """ + Test the knn torch backend can be correctly initialized, fit and used to + predict outliers. + """ knn_torch = KNNTorch(k=5) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) @@ -37,8 +39,10 @@ def test_knn_torch_backend(): def test_knn_torch_backend_ensemble(ensembler): - # Test the knn torch backend can be correctly initialized as an ensemble, fit - # on data and used to predict outliers. + """ + Test the knn torch backend can be correctly initialized as an ensemble, fit + on data and used to predict outliers. + """ knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x_ref = torch.randn((1024, 10)) @@ -54,8 +58,10 @@ def test_knn_torch_backend_ensemble(ensembler): def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): - # Test the knn torch backend can be initalized as an ensemble and - # torchscripted, as well as saved and loaded to and from disk. + """ + Test the knn torch backend can be initalized as an ensemble and + torchscripted, as well as saved and loaded to and from disk. + """ knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) @@ -83,8 +89,10 @@ def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): def test_knn_torch_backend_ts(tmp_path): - # Test the knn torch backend can be initalized and torchscripted, as well as - # saved and loaded to and from disk. + """ + Test the knn torch backend can be initalized and torchscripted, as well as + saved and loaded to and from disk. + """ knn_torch = KNNTorch(k=7) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) @@ -103,8 +111,10 @@ def test_knn_torch_backend_ts(tmp_path): def test_knn_kernel(ensembler): - # Test the knn torch backend can be correctly initialized with a kernel, fit - # on data and used to predict outliers. + """ + Test the knn torch backend can be correctly initialized with a kernel, fit + on data and used to predict outliers. + """ kernel = GaussianRBF(sigma=torch.tensor((0.25))) knn_torch = KNNTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) @@ -122,8 +132,10 @@ def test_knn_kernel(ensembler): @pytest.mark.skip(reason="Can't convert GaussianRBF to torchscript due to torchscript type constraints") def test_knn_kernel_ts(ensembler): - # Test the knn torch backend can be correctly initialized with a kernel, - # and torchscripted, as well as saved and loaded to and from disk. + """ + Test the knn torch backend can be correctly initialized with a kernel, + and torchscripted, as well as saved and loaded to and from disk. + """ kernel = GaussianRBF(sigma=torch.tensor((0.25))) knn_torch = KNNTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) From a677e30269780495bd1a1480d5fe1ee6fcb2f0cc Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 13 Feb 2023 17:40:40 +0000 Subject: [PATCH 114/247] Address minor pr comments --- alibi_detect/od/pytorch/base.py | 3 +-- alibi_detect/od/pytorch/ensemble.py | 8 -------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index d0bcdc559..8ee234e8f 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -222,9 +222,8 @@ def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: Returns ------- - `TorchOutlierDetectorOutput` + :py:obj:`alibi_detect.od.pytorch.base.TorchOutlierDetectorOutput` Output of the outlier detector. - """ self.check_fitted() # type: ignore raw_scores = self.score(x) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index b4896135b..ad97a1ee7 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -122,10 +122,6 @@ def _fit(self, val_scores: torch.Tensor) -> PValNormalizer: ---------- val_scores score outputs of ensemble of detectors applied to reference data. - - Returns - ------- - `self` """ self.val_scores = val_scores return self @@ -166,10 +162,6 @@ def _fit(self, val_scores: torch.Tensor) -> ShiftAndScaleNormalizer: ---------- val_scores `Torch.Tensor` of scores from ensemble of detectors. - - Returns - ------- - `self` """ self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] From f54586af32f7a7706cd89aa451caf53391c819d2 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Feb 2023 15:11:41 +0000 Subject: [PATCH 115/247] Add kwarg formatting for fit method parameters on gmm --- alibi_detect/od/_gmm.py | 36 ++++++++++--- alibi_detect/od/pytorch/gmm.py | 83 +++++++++++++++++++++++------- alibi_detect/od/sklearn/base.py | 4 +- alibi_detect/od/sklearn/gmm.py | 62 +++++++++++++++++++--- alibi_detect/utils/pytorch/misc.py | 21 ++++++++ 5 files changed, 172 insertions(+), 34 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 9a7bffa74..7c361ab8a 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -1,5 +1,4 @@ -from typing import Union, Optional, Dict, Any -from typing import TYPE_CHECKING +from typing import Union, Optional, Dict, Any, TYPE_CHECKING import numpy as np @@ -25,17 +24,24 @@ class GMM(BaseDetector, ThresholdMixin, FitMixin): def __init__( self, - n_components: int, + n_components: int = 1, device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch', 'sklearn'] = 'pytorch', ) -> None: """Gaussian Mixture Model (GMM) outlier detector. + The guassian mixture model outlier detector fits a mixture of gaussian distributions to the reference data. + Test points are scored via the negative log-likhood under the corresponding density function. + + We support two backends: ``'pytorch'`` and ``'sklearn'``. The ``'pytorch'`` backend allows for GPU acceleration + and uses gradient descent to fit the mixture of gaussians. We recommend using the ``'pytorch'`` backend for + for large datasets. The ``'sklearn'`` backend is a pure python implementation and is recommended for smaller + datasets. + Parameters ---------- n_components: - The number of dimensions in the principle subspace. For linear pca should have - ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. + The number of mixture components. Defaults to ``1``. backend Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'`` and ``'sklearn'``. device @@ -61,15 +67,31 @@ def __init__( args['device'] = device self.backend = backend_cls(**args) - def fit(self, x_ref: np.ndarray, **kwargs: Dict) -> None: + def fit( + self, + x_ref: np.ndarray, + optimizer: Optional[str] = 'Adam', + learning_rate: float = 0.1, + batch_size: int = 32, + epochs: Optional[int] = None, + tol: float = 1e-3, + n_init: int = 1, + init_params: str = 'kmeans', + verbose: int = 0, + ) -> None: """Fit the detector on reference data. + If the ``'pytorch'`` backend is used, the detector is fitted using gradient descent. + Parameters ---------- x_ref Reference data used to fit the detector. """ - self.backend.fit(self.backend._to_tensor(x_ref), **kwargs) + self.backend.fit( + self.backend._to_tensor(x_ref), + **self.backend.format_fit_kwargs(locals()) + ) def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index e4fec439e..c0b37568d 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional, Union +from typing import Callable, Optional, Union, Dict from tqdm import tqdm import torch from torch.utils.data import DataLoader @@ -7,6 +7,7 @@ from alibi_detect.utils.pytorch.prediction import predict_batch from alibi_detect.od.pytorch.base import TorchOutlierDetector from alibi_detect.models.pytorch.gmm import GMMModel +from alibi_detect.utils.pytorch.misc import get_optimizer class GMMTorch(TorchOutlierDetector): @@ -16,32 +17,49 @@ def __init__( device: Optional[Union[str, torch.device]] = None ) -> None: """ - Fits a Gaussian mixture model to the training data and scores new data points - via the negative log-liklihood under the corresponding density function. + Pytorch Backend for the Gaussian Mixture Model (GMM) outlier detector. + Parameters ---------- - n_components: - The number of Gaussian mixture components. - optimizer: - Used to learn the GMM params. - rest should be obvious. + n_components + Number of components in guassian mixture model. + device + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. """ self.n_components = n_components TorchOutlierDetector.__init__(self, device=device) def _fit( - self, - X: torch.Tensor, - optimizer: Callable = torch.optim.Adam, - learning_rate: float = 0.1, - batch_size: int = 32, - epochs: int = 10, - verbose: int = 0, - ) -> None: - self.model = GMMModel(self.n_components, X.shape[-1]).to(self.device) - X = X.to(torch.float32) + self, + x_ref: torch.Tensor, + optimizer: Callable = torch.optim.Adam, + learning_rate: float = 0.1, + batch_size: int = 32, + epochs: int = 10, + verbose: int = 0, + ) -> None: + """Fit the GMM model. - dataset = TorchDataset(X) + Parameters + ---------- + X + Training data. + optimizer + Optimizer used to train the model. + learning_rate + Learning rate used to train the model. + batch_size + Batch size used to train the model. + epochs + Number of training epochs. + verbose + Verbosity level during training. 0 is silent, 1 a progress bar. + """ + self.model = GMMModel(self.n_components, x_ref.shape[-1]).to(self.device) + x_ref = x_ref.to(torch.float32) + + dataset = TorchDataset(x_ref) dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True) optimizer = optimizer(self.model.parameters(), lr=learning_rate) self.model.train() @@ -60,6 +78,26 @@ def _fit( dl.set_description(f'Epoch {epoch + 1}/{epochs}') dl.set_postfix(dict(loss_ma=loss_ma)) + def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: + """Format kwargs for `fit` method. + + Parameters + ---------- + kwargs + Kwargs to format. + + Returns + ------- + Formatted kwargs. + """ + return dict( + optimizer=get_optimizer(fit_kwargs.get('optimizer')), + learning_rate=fit_kwargs.get('learning_rate', 0.1), + batch_size=fit_kwargs.get('batch_size', 32), + epochs=(lambda v: 10 if v is None else v)(fit_kwargs.get('epochs', None)), + verbose=fit_kwargs.get('verbose', 0) + ) + def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -85,6 +123,13 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return preds.cpu() def score(self, X: torch.Tensor) -> torch.Tensor: + """Score `X` using the GMM model. + + Parameters + ---------- + X + `torch.Tensor` with leading batch dimension. + """ self.check_fitted() batch_size, *_ = X.shape X = X.to(torch.float32) diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index bb95cbadb..61df2d43a 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -28,9 +28,9 @@ def __init__(self): """ super().__init__() - def fit(self, x: np.ndarray) -> FitMixin: + def fit(self, x: np.ndarray, **kwargs: dict) -> FitMixin: self._fitted = True - self._fit(x) + self._fit(x, **kwargs) return self @abstractmethod diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index abf7ae9ac..ef6c41a09 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -1,13 +1,14 @@ import numpy as np +from typing import Dict from alibi_detect.od.sklearn.base import SklearnOutlierDetector from sklearn.mixture import GaussianMixture class GMMSklearn(SklearnOutlierDetector): def __init__( - self, - n_components: int, - ): + self, + n_components: int, + ): """sklearn backend for GMM detector. Parameters @@ -17,17 +18,66 @@ def __init__( """ SklearnOutlierDetector.__init__(self) self.n_components = n_components - self.gmm = GaussianMixture(n_components=self.n_components) - def _fit(self, x_ref: np.ndarray) -> None: + def _fit( + self, + x_ref: np.ndarray, + tol: float = 1e-3, + max_iter: int = 100, + n_init: int = 1, + init_params: str = 'kmeans', + verbose: int = 0, + ) -> None: """Fit the outlier detector to the reference data. Parameters ---------- x_ref Reference data. + tol + Convergence threshold. EM iterations will stop when the lower bound average gain is below this threshold. + max_iter + Maximum number of EM iterations to perform. + n_init + Number of initializations to perform. + init_params + Method used to initialize the weights, the means and the precisions. Must be one of: + 'kmeans' : responsibilities are initialized using kmeans. + 'kmeans++' : responsibilities are initialized using kmeans++. + 'random' : responsibilities are initialized randomly. + 'random_from_data' : responsibilities are initialized randomly from the data. + verbose + Enable verbose output. If 1 then it prints the current initialization and each iteration step. If greater + than 1 then it prints also the log probability and the time needed for each step. + + """ + self.gmm = GaussianMixture( + n_components=self.n_components, + tol=tol, + max_iter=max_iter, + n_init=n_init, + init_params=init_params, + verbose=verbose, + ) + self.gmm = self.gmm.fit( + x_ref, + ) + + def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: + """Format the kwargs for the fit method. + + Parameters + ---------- + kwargs + Keyword arguments for the fit method. """ - self.gmm = self.gmm.fit(x_ref) + return dict( + tol=fit_kwargs.get('tol', 1e-3), + max_iter=(lambda v: 100 if v is None else v)(fit_kwargs.get('epochs', None)), + n_init=fit_kwargs.get('n_init', 1), + init_params=fit_kwargs.get('init_params', 'kmeans'), + verbose=fit_kwargs.get('verbose', 0), + ) def score(self, x: np.ndarray) -> np.ndarray: """Score the data. diff --git a/alibi_detect/utils/pytorch/misc.py b/alibi_detect/utils/pytorch/misc.py index bb65df59a..cfcab439b 100644 --- a/alibi_detect/utils/pytorch/misc.py +++ b/alibi_detect/utils/pytorch/misc.py @@ -92,3 +92,24 @@ def get_device(device: Optional[Union[str, torch.device]] = None) -> torch.devic if device.lower() != 'cpu': logger.warning('Requested device not recognised, fall back on CPU.') return torch_device + + +def get_optimizer(name: str = 'Adam'): + """ + Get an optimizer class from its name. + + Parameters + ---------- + name + Name of the optimizer. + + Returns + ------- + The optimizer class. + """ + optimizer = getattr(torch.optim, name, None) + + if optimizer is None: + raise NotImplementedError(f"Optimizer {name} not implemented.") + + return optimizer From 5bede8a1b8c9c5f5fa6edb3b659516ab7c6efe8f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Feb 2023 16:05:41 +0000 Subject: [PATCH 116/247] Add method docstrings for GMM detector --- alibi_detect/od/_gmm.py | 43 ++++++++++++++++++++++++++++++++-- alibi_detect/od/pytorch/gmm.py | 14 +++++------ alibi_detect/od/sklearn/gmm.py | 10 +++++--- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 7c361ab8a..e013ee06b 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -72,7 +72,7 @@ def fit( x_ref: np.ndarray, optimizer: Optional[str] = 'Adam', learning_rate: float = 0.1, - batch_size: int = 32, + batch_size: Optional[int] = None, epochs: Optional[int] = None, tol: float = 1e-3, n_init: int = 1, @@ -81,12 +81,44 @@ def fit( ) -> None: """Fit the detector on reference data. - If the ``'pytorch'`` backend is used, the detector is fitted using gradient descent. + If the ``'pytorch'`` backend is used, the detector is fitted using gradient descent. This is the recommended + backend for larger datasets. + + If the ``'sklearn'`` backend is used, the detector is fitted using the EM algorithm. The ``'sklearn'`` + backend is recommended for smaller datasets. For more information on the EM algorithm and the sklearn Gaussian + Mixture Model, see `here `_. # noqa: E501 Parameters ---------- x_ref Reference data used to fit the detector. + optimizer + Optimizer used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``'Adam'``. + learning_rate + Learning rate used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``0.1``. + batch_size + Batch size used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``None``. + If ``None``, the entire dataset is used in each epoch. + epochs + Number of epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` backends. + If the backend is ``'sklearn'``, the detector is fitted using the EM algorithm and the number of epochs + defualts to ``10``. If the backend is ``'pytorch'``, the detector is fitted using gradient descent and + the number of epochs defaults to ``100``. + tol + Tolerance used to fit the detector. Only used if the ``'sklearn'`` backend is used. Defaults to ``1e-3``. + n_init + Number of initializations used to fit the detector. Only used if the ``'sklearn'`` backend is used. + Defaults to ``1``. + init_params + Initialization method used to fit the detector. Only used if the ``'sklearn'`` backend is used. Must be + one of: + 'kmeans' : responsibilities are initialized using kmeans. + 'kmeans++' : responsibilities are initialized using kmeans++. + 'random' : responsibilities are initialized randomly. + 'random_from_data' : responsibilities are initialized randomly from the data. + Defaults to ``'kmeans'``. + verbose + Verbosity level used to fit the detector. Only used if the ``'sklearn'`` backend is used. Defaults to ``0``. """ self.backend.fit( self.backend._to_tensor(x_ref), @@ -96,6 +128,9 @@ def fit( def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. + To score an instance, we compute the negative log-likelihood under the corresponding density function of + the fitted gaussian mixture model. + Parameters ---------- x @@ -112,6 +147,8 @@ def score(self, x: np.ndarray) -> np.ndarray: def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the GMM detector. + The threshold is computed so that the outlier detector would incorectly classify `fpr` proportion of the + reference data as outliers. Parameters ---------- @@ -127,6 +164,8 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. + Scores the instances in `x` and if the threshold was inferred, returns the outlier labels and p-values as well. + Parameters ---------- x diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index c0b37568d..986b78de3 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -16,8 +16,7 @@ def __init__( n_components: int, device: Optional[Union[str, torch.device]] = None ) -> None: - """ - Pytorch Backend for the Gaussian Mixture Model (GMM) outlier detector. + """Pytorch backend for the Gaussian Mixture Model (GMM) outlier detector. Parameters ---------- @@ -59,9 +58,10 @@ def _fit( self.model = GMMModel(self.n_components, x_ref.shape[-1]).to(self.device) x_ref = x_ref.to(torch.float32) + batch_size = len(x_ref) if batch_size is None else batch_size dataset = TorchDataset(x_ref) dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True) - optimizer = optimizer(self.model.parameters(), lr=learning_rate) + optimizer_instance: torch.optim.Optimizer = optimizer(self.model.parameters(), lr=learning_rate) self.model.train() for epoch in range(epochs): @@ -70,9 +70,9 @@ def _fit( for step, x in dl: x = x.to(self.device) nll = self.model(x).mean() - optimizer.zero_grad() # type: ignore + optimizer_instance.zero_grad() nll.backward() - optimizer.step() # type: ignore + optimizer_instance.step() if verbose == 1 and isinstance(dl, tqdm): loss_ma = loss_ma + (nll.item() - loss_ma) / (step + 1) dl.set_description(f'Epoch {epoch + 1}/{epochs}') @@ -84,7 +84,7 @@ def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: Parameters ---------- kwargs - Kwargs to format. + dictionary of Kwargs to format. See `fit` method for details. Returns ------- @@ -93,7 +93,7 @@ def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: return dict( optimizer=get_optimizer(fit_kwargs.get('optimizer')), learning_rate=fit_kwargs.get('learning_rate', 0.1), - batch_size=fit_kwargs.get('batch_size', 32), + batch_size=fit_kwargs.get('batch_size', None), epochs=(lambda v: 10 if v is None else v)(fit_kwargs.get('epochs', None)), verbose=fit_kwargs.get('verbose', 0) ) diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index ef6c41a09..0240cc33a 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -9,7 +9,7 @@ def __init__( self, n_components: int, ): - """sklearn backend for GMM detector. + """sklearn backend for the Gaussian Mixture Model (GMM) outlier detector. Parameters ---------- @@ -64,12 +64,16 @@ def _fit( ) def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: - """Format the kwargs for the fit method. + """Format kwargs for `fit` method. Parameters ---------- kwargs - Keyword arguments for the fit method. + dictionary of Kwargs to format. See `fit` method for details. + + Returns + ------- + Formatted kwargs. """ return dict( tol=fit_kwargs.get('tol', 1e-3), From 57dfd9d63038398aa7994f5617c1c661302aca9f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Feb 2023 17:05:41 +0000 Subject: [PATCH 117/247] Add torchscript test and comments --- alibi_detect/od/pytorch/gmm.py | 15 +++++++-------- alibi_detect/od/tests/test__gmm/test__gmm.py | 18 +++++++++++++----- .../test__gmm/test__gmm_pytorch_backend.py | 9 ++++++++- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 986b78de3..85e98af8b 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -11,6 +11,7 @@ class GMMTorch(TorchOutlierDetector): + def __init__( self, n_components: int, @@ -26,6 +27,7 @@ def __init__( Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. """ + self.ensembler = None self.n_components = n_components TorchOutlierDetector.__init__(self, device=device) @@ -122,6 +124,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: preds = scores > self.threshold return preds.cpu() + @torch.no_grad() def score(self, X: torch.Tensor) -> torch.Tensor: """Score `X` using the GMM model. @@ -130,12 +133,8 @@ def score(self, X: torch.Tensor) -> torch.Tensor: X `torch.Tensor` with leading batch dimension. """ - self.check_fitted() - batch_size, *_ = X.shape + if not torch.jit.is_scripting(): + self.check_fitted() X = X.to(torch.float32) - preds = predict_batch( - X, self.model.eval(), - device=self.device, - batch_size=batch_size - ) - return torch.tensor(preds) + preds = self.model(X.to(self.device)).cpu() + return preds diff --git a/alibi_detect/od/tests/test__gmm/test__gmm.py b/alibi_detect/od/tests/test__gmm/test__gmm.py index 2eae5d7c5..93eb4c20e 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm.py @@ -1,6 +1,6 @@ import pytest import numpy as np -# import torch +import torch from alibi_detect.od._gmm import GMM from alibi_detect.base import NotFitException @@ -65,7 +65,15 @@ def test_gmm_integration(backend): result = result['data']['is_outlier'][0] assert result - # ts_gmm = torch.jit.script(gmm_detector.backend) - # x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) - # y = ts_gmm(x) - # assert torch.all(y == torch.tensor([False, True])) + +def test_gmm_torchscript(): + gmm_detector = GMM(n_components=8, backend='pytorch') + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + gmm_detector.fit(X_ref) + gmm_detector.infer_threshold(X_ref, 0.1) + x_outlier = np.array([[-1, 1.5]]) + ts_gmm = torch.jit.script(gmm_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = ts_gmm(x) + assert torch.all(y == torch.tensor([False, True])) diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py index d2009119e..247d41ff7 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py @@ -10,24 +10,31 @@ def test_gmm_pytorch_backend_fit_errors(): gmm_torch = GMMTorch(n_components=2) assert not gmm_torch._fitted + # Test that the backend raises an error if it is not fitted before + # calling forward method. x = torch.tensor(np.random.randn(1, 10)) with pytest.raises(NotFitException) as err: gmm_torch(x) assert str(err.value) == 'GMMTorch has not been fit!' + # Test that the backend raises an error if it is not fitted before + # predicting. with pytest.raises(NotFitException) as err: gmm_torch.predict(x) assert str(err.value) == 'GMMTorch has not been fit!' + # Test the backend updates _fitted flag on fit. x_ref = torch.tensor(np.random.randn(1024, 10)) gmm_torch.fit(x_ref) - assert gmm_torch._fitted + # Test that the backend raises an if the forward method is called without the + # threshold being inferred. with pytest.raises(ThresholdNotInferredException) as err: gmm_torch(x) assert str(err.value) == 'GMMTorch has no threshold set, call `infer_threshold` before predicting.' + # Test that the backend can call predict without the threshold being inferred. assert gmm_torch.predict(x) From b8c08eb9cb70674e0ad15414f1808414d4a06db6 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Feb 2023 17:23:40 +0000 Subject: [PATCH 118/247] Add further documentation for tests --- alibi_detect/od/pytorch/gmm.py | 1 - alibi_detect/od/tests/test__gmm/test__gmm.py | 20 +++++++++ .../test__gmm/test__gmm_sklearn_backend.py | 43 +++++++++++-------- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 85e98af8b..4b22a595f 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -4,7 +4,6 @@ from torch.utils.data import DataLoader from alibi_detect.utils.pytorch.data import TorchDataset -from alibi_detect.utils.pytorch.prediction import predict_batch from alibi_detect.od.pytorch.base import TorchOutlierDetector from alibi_detect.models.pytorch.gmm import GMMModel from alibi_detect.utils.pytorch.misc import get_optimizer diff --git a/alibi_detect/od/tests/test__gmm/test__gmm.py b/alibi_detect/od/tests/test__gmm/test__gmm.py index 93eb4c20e..34ff0a70c 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm.py @@ -9,8 +9,12 @@ @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) def test_unfitted_gmm_single_score(backend): + """ + test predict raises exception when not fitted + """ gmm_detector = GMM(n_components=1, backend=backend) x = np.array([[0, 10], [0.1, 0]]) + with pytest.raises(NotFitException) as err: _ = gmm_detector.predict(x) assert str(err.value) == f'{gmm_detector.backend.__class__.__name__} has not been fit!' @@ -18,6 +22,10 @@ def test_unfitted_gmm_single_score(backend): @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) def test_fitted_gmm_single_score(backend): + """ + Test that a detector that has been fitted on data but that has not got an inferred + threshold, will correctly score outliers using the predict method. + """ gmm_detector = GMM(n_components=1, backend=backend) x_ref = np.random.randn(100, 2) gmm_detector.fit(x_ref) @@ -34,6 +42,11 @@ def test_fitted_gmm_single_score(backend): @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) def test_fitted_gmm_predict(backend): + """ + Test that a detector that has been fitted on data and with an inferred threshold, + will correctly score and label outliers, as well as return the p-values using the + predict method. + """ gmm_detector = GMM(n_components=1, backend=backend) x_ref = np.random.randn(100, 2) gmm_detector.fit(x_ref) @@ -51,6 +64,10 @@ def test_fitted_gmm_predict(backend): @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) def test_gmm_integration(backend): + """ + Tests gmm detector on the moons dataset. Fits and infers thresholds and + verifies that the detector can correctly detect inliers and outliers. + """ gmm_detector = GMM(n_components=8, backend=backend) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] @@ -67,6 +84,9 @@ def test_gmm_integration(backend): def test_gmm_torchscript(): + """ + Tests gmm detector fitted on the moons dataset can be torchscripted correctly. + """ gmm_detector = GMM(n_components=8, backend='pytorch') X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py index b8456ecc4..f122a3cee 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -6,57 +6,64 @@ def test_gmm_sklearn_backend_fit_errors(): - gm_sklearn = GMMSklearn(n_components=2) - assert not gm_sklearn._fitted + gmm_sklearn = GMMSklearn(n_components=2) + assert not gmm_sklearn._fitted + # Test that the backend raises an error if it is not fitted before + # calling forward method. x = np.random.randn(1, 10) with pytest.raises(NotFitException) as err: - gm_sklearn(x) + gmm_sklearn(x) assert str(err.value) == 'GMMSklearn has not been fit!' + # Test that the backend raises an error if it is not fitted before + # predicting. with pytest.raises(NotFitException) as err: - gm_sklearn.predict(x) + gmm_sklearn.predict(x) assert str(err.value) == 'GMMSklearn has not been fit!' + # Test the backend updates _fitted flag on fit. x_ref = np.random.randn(1024, 10) - gm_sklearn.fit(x_ref) - - assert gm_sklearn._fitted + gmm_sklearn.fit(x_ref) + assert gmm_sklearn._fitted + # Test that the backend raises an if the forward method is called without the + # threshold being inferred. with pytest.raises(ThresholdNotInferredException) as err: - gm_sklearn(x) + gmm_sklearn(x) assert str(err.value) == 'GMMSklearn has no threshold set, call `infer_threshold` before predicting.' - assert gm_sklearn.predict(x) + # Test that the backend can call predict without the threshold being inferred. + assert gmm_sklearn.predict(x) def test_gmm_sklearn_scoring(): - gm_sklearn = GMMSklearn(n_components=2) + gmm_sklearn = GMMSklearn(n_components=2) mean = [8, 8] cov = [[2., 0.], [0., 1.]] x_ref = np.random.multivariate_normal(mean, cov, 1000) - gm_sklearn.fit(x_ref) + gmm_sklearn.fit(x_ref) x_1 = np.array([[8., 8.]]) - scores_1 = gm_sklearn.score(x_1) + scores_1 = gmm_sklearn.score(x_1) x_2 = np.random.multivariate_normal(mean, cov, 1) - scores_2 = gm_sklearn.score(x_2) + scores_2 = gmm_sklearn.score(x_2) x_3 = np.array([[-10., 10.]]) - scores_3 = gm_sklearn.score(x_3) + scores_3 = gmm_sklearn.score(x_3) # test correct ordering of scores given outlyingness of data assert scores_1 < scores_2 < scores_3 # test that detector correctly detects true Outlier - gm_sklearn.infer_threshold(x_ref, 0.01) + gmm_sklearn.infer_threshold(x_ref, 0.01) x = np.concatenate((x_1, x_2, x_3)) - outputs = gm_sklearn.predict(x) + outputs = gmm_sklearn.predict(x) assert np.all(outputs.is_outlier == np.array([False, False, True])) - assert np.all(gm_sklearn(x) == np.array([False, False, True])) + assert np.all(gmm_sklearn(x) == np.array([False, False, True])) # test that 0.01 of the in distribution data is flagged as outliers x = np.random.multivariate_normal(mean, cov, 1000) - outputs = gm_sklearn.predict(x) + outputs = gmm_sklearn.predict(x) assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.01 From b227db4cf77a848b60d4950f3fe373e3d0f27445 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 15 Feb 2023 11:59:40 +0000 Subject: [PATCH 119/247] Remove singular dispatch pattern --- alibi_detect/od/pytorch/base.py | 47 +++++++++----------- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/tests/test__knn/test__knn.py | 9 ++-- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 8ee234e8f..c3d26a65f 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -2,8 +2,6 @@ from typing import List, Union, Optional, Dict from dataclasses import dataclass, asdict from abc import ABC, abstractmethod -from functools import singledispatch - import numpy as np import torch @@ -22,9 +20,19 @@ class TorchOutlierDetectorOutput: is_outlier: Optional[torch.Tensor] p_value: Optional[torch.Tensor] + def to_numpy(self): + outputs = asdict(self) + for key, value in outputs.items(): + if isinstance(value, torch.Tensor): + outputs[key] = value.cpu().detach().numpy() + return outputs + + +def _raise_type_error(x): + raise TypeError(f'x is type={type(x)} but must be one of TorchOutlierDetectorOutput or a torch Tensor') -@singledispatch -def to_numpy(arg): + +def to_numpy(x: Union[torch.Tensor, TorchOutlierDetectorOutput]): """Converts any `torch` tensors found in input to `numpy` arrays. Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays @@ -38,21 +46,14 @@ def to_numpy(arg): ------- `np.ndarray` or dictionary of containing `numpy` arrays """ - raise NotImplementedError(f"Cannot transform type {type(arg)} to numpy array.") - -@to_numpy.register -def _(x: torch.Tensor) -> np.ndarray: - return x.cpu().detach().numpy() - - -@to_numpy.register -def _(x: TorchOutlierDetectorOutput) -> Dict: - outputs = asdict(x) - for key, value in outputs.items(): - if isinstance(value, torch.Tensor): - outputs[key] = value.cpu().detach().numpy() - return outputs + return { + 'TorchOutlierDetectorOutput': lambda x: x.to_numpy(), + 'Tensor': lambda x: x.cpu().detach().numpy() + }.get( + x.__class__.__name__, + _raise_type_error + )(x) class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): @@ -124,10 +125,6 @@ def _to_tensor(self, x: Union[List, np.ndarray]) -> torch.Tensor: ---------- x Data to convert. - - Returns - ------- - `torch.Tensor` """ return torch.as_tensor(x, dtype=torch.float32, device=self.device) @@ -143,7 +140,7 @@ def _ensembler(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - `torch.Tensor` or just returns original data + `torch.Tensor` or original data without alteration """ if hasattr(self, 'ensembler') and self.ensembler is not None: # `type: ignore` here becuase self.ensembler here causes an error with mypy when using torch.jit.script. @@ -222,8 +219,8 @@ def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: Returns ------- - :py:obj:`alibi_detect.od.pytorch.base.TorchOutlierDetectorOutput` - Output of the outlier detector. + Output of the outlier detector. Includes the p-values, outlier labels, instance scores and \ + threshold. """ self.check_fitted() # type: ignore raw_scores = self.score(x) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index ad97a1ee7..79cdd24de 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -97,7 +97,7 @@ def transform(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - transformed `torch.Tensor`. + the transformed `torch.Tensor`. """ if not torch.jit.is_scripting(): self.check_fitted() diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index 8a3ddc377..53e8ca237 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -37,7 +37,6 @@ def test_fitted_knn_single_score(k): x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) - # test fitted but not threshold inferred detectors # can still score data using the predict method. y = knn_detector.predict(x) @@ -62,16 +61,20 @@ def test_incorrect_knn_ensemble_init(): def test_fitted_knn_predict(): """ - test that a detector fitted on data and with threshold inferred correctly, will score - and label outliers, as well as return the p-values using the predict method. + Test that a detector fitted on data and with threshold inferred correctly, will score + and label outliers, as well as return the p-values using the predict method. Also Check + that the score method gives the same results. """ knn_detector = make_knn_detector(k=10) x_ref = np.random.randn(100, 2) knn_detector.infer_threshold(x_ref, 0.1) x = np.array([[0, 10], [0, 0.1]]) + y = knn_detector.predict(x) y = y['data'] + scores = knn_detector.score(x) + assert np.all(y['instance_score'] == scores) assert y['instance_score'][0] > 5 assert y['instance_score'][1] < 1 assert y['threshold_inferred'] From 94dd9c8d53c3f9f05d8f17b4bbf0bfd04a6da716 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 21 Feb 2023 14:40:41 +0000 Subject: [PATCH 120/247] Add pytorch backend for LOF detector --- alibi_detect/od/pytorch/lof.py | 166 ++++++++++++++++ .../od/tests/test__lof/test__lof_backend.py | 182 ++++++++++++++++++ 2 files changed, 348 insertions(+) create mode 100644 alibi_detect/od/pytorch/lof.py create mode 100644 alibi_detect/od/tests/test__lof/test__lof_backend.py diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py new file mode 100644 index 000000000..885deb32d --- /dev/null +++ b/alibi_detect/od/pytorch/lof.py @@ -0,0 +1,166 @@ +from typing import Optional, Union, List, Tuple + +import numpy as np +import torch + +from alibi_detect.od.pytorch.ensemble import Ensembler +from alibi_detect.od.pytorch.base import TorchOutlierDetector + + +class LOFTorch(TorchOutlierDetector): + def __init__( + self, + k: Union[np.ndarray, List, Tuple], + kernel: Optional[torch.nn.Module] = None, + ensembler: Optional[Ensembler] = None, + device: Optional[Union[str, torch.device]] = None + ): + """PyTorch backend for LOF detector. + + Computes the Local Outlier Factor (LOF) of each instance in `x` with respect to a reference set `x_ref`. + + Parameters + ---------- + k + Number of nearest neighbors used to compute LOF. If `k` is a list or array, then an ensemble of LOF detectors + is created with one detector for each value of `k`. + kernel + If a kernel is specified then instead of using `torch.cdist` the kernel defines the `k` nearest + neighbor distance. + ensembler + If `k` is an array of integers then the ensembler must not be ``None``. Should be an instance + of :py:obj:`alibi_detect.od.pytorch.ensemble.ensembler`. Responsible for combining + multiple scores into a single score. + device + Device on which to run the detector. + """ + TorchOutlierDetector.__init__(self, device=device) + self.kernel = kernel + self.ensemble = isinstance(k, (np.ndarray, list, tuple)) + self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) + self.ensembler = ensembler + + @torch.no_grad() + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Detect if `x` is an outlier. + + Parameters + ---------- + x + `torch.Tensor` with leading batch dimension. + + Returns + ------- + `torch.Tensor` of ``bool`` values with leading batch dimension. + + Raises + ------ + ThresholdNotInferredException + If called before detector has had `infer_threshold` method called. + """ + raw_scores = self.score(x) + scores = self._ensembler(raw_scores) + if not torch.jit.is_scripting(): + self.check_threshold_inferred() + preds = scores > self.threshold + return preds + + def _make_mask(self, reachabilities: torch.Tensor): + mask = torch.zeros_like(reachabilities[0]) + for i, k in enumerate(self.ks): + mask[:k, i] = torch.ones(k)/k + return mask + + @torch.no_grad() + def score(self, x: torch.Tensor) -> torch.Tensor: + """Computes the score of `x` + + The score step proceeds as follows: + 1. Compute the distance between each instance in `x` and the reference set. + 2. Compute the k-nearest neighbors of each instance in `x` in the reference set. + 3. Compute the reachability distance of each instance in `x` to its k-nearest neighbors. + 4. For each instance sum the inv_avg_reachabilities of its neighbours. + 5. LOF is average reachability of instance over average reachability of neighbours. + + + Parameters + ---------- + x + The tensor of instances. First dimension corresponds to batch. + + Returns + ------- + Tensor of scores for each element in `x`. + + Raises + ------ + NotFitException + If called before detector has been fit. + """ + if not torch.jit.is_scripting(): + self.check_fitted() + + X = torch.as_tensor(x) + D = torch.cdist(X, self.x_ref) + + # K = -self.kernel(x, self.x_ref) if self.kernel is not None else torch.cdist(x, self.x_ref) + + max_k = torch.max(self.ks) + bot_k_items = torch.topk(D, max_k, dim=1, largest=False) + bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values + lower_bounds = self.knn_dists_ref[bot_k_inds] + reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) + mask = self._make_mask(reachabilities) + avg_reachabilities = (reachabilities*mask[None, :, :]).sum(1) + factors = (self.ref_inv_avg_reachabilities[bot_k_inds]*mask[None, :, :]).sum(1) + lofs = (avg_reachabilities * factors) + return lofs if self.ensemble else lofs[:, 0] + + @torch.no_grad() + def _fit(self, x_ref: torch.Tensor): + """Fits the detector + + The LOF algorithm fit step prodeeds as follows: + 1. Compute the distance matrix, D, between all instances in `x_ref`. + 2. For each instance, compute the k nearest neighbours. (Note we prevent an instance from + considering itself a neighbour by setting the diagonal of D to be the maximum value of D.) + 3. For each instance we store the distance to its kth nearest neighbour for each k in `ks`. + 4. For each instance and k in `ks` we obtain a tensor of the k neighbours k nearest neighbour + distances. + 5. The reachability of an instance is the maximum of its k nearest neighbours distances and + the distance to its kth nearest neighbour. + 6. The reachabilites tensor is of shape `(n_instances, max(ks), len(ks))`. Where the second + dimension is the each of the k neighbours nearest distances and the third dimension is + the specific k. + 7. The local reachability density is then given by 1 over the average reachability + over the second dimension of this tensor. However we only want to consider the k nearest + neighbours for each k in `ks`, so we use a mask that prevents k from the second dimension + greater than k from the third dimension from being considered. This value is stored as + we use it in the score step. + 8. If multiple k are passed in ks then the detector also needs to fit the ensembler. To do so + we need to score the x_ref as well. The local outlier factor (LOF) is then given by the + average reachability of an instance over the average reachability of its k neighbours. + + Parameters + ---------- + x_ref + The Dataset tensor. + """ + X = torch.as_tensor(x_ref) + D = torch.cdist(X, X) + D += torch.eye(len(D)) * torch.max(D) + max_k = torch.max(self.ks) + bot_k_items = torch.topk(D, max_k, dim=1, largest=False) + bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values + self.knn_dists_ref = bot_k_dists[:, self.ks-1] + lower_bounds = self.knn_dists_ref[bot_k_inds] + reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) + mask = self._make_mask(reachabilities) + avg_reachabilities = (reachabilities*mask[None, :, :]).sum(1) + self.ref_inv_avg_reachabilities = 1/avg_reachabilities + self.x_ref = X + + if self.ensemble: + factors = (self.ref_inv_avg_reachabilities[bot_k_inds]*mask[None, :, :]).sum(1) + scores = (avg_reachabilities * factors) + self.ensembler.fit(scores) diff --git a/alibi_detect/od/tests/test__lof/test__lof_backend.py b/alibi_detect/od/tests/test__lof/test__lof_backend.py new file mode 100644 index 000000000..28f1c78b4 --- /dev/null +++ b/alibi_detect/od/tests/test__lof/test__lof_backend.py @@ -0,0 +1,182 @@ +import pytest +import torch + +from alibi_detect.od.pytorch.lof import LOFTorch +from alibi_detect.utils.pytorch.kernels import GaussianRBF +from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator +from alibi_detect.base import NotFitException, ThresholdNotInferredException + + +@pytest.fixture(scope='session') +def ensembler(request): + return Ensembler( + normalizer=PValNormalizer(), + aggregator=AverageAggregator() + ) + + +def test_lof_torch_backend(): + """ + Test the lof torch backend can be correctly initialized, fit and used to + predict outliers. + """ + + lof_torch = LOFTorch(k=5) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + outputs = lof_torch.predict(x) + assert outputs.instance_score.shape == (3, ) + assert outputs.is_outlier is None + assert outputs.p_value is None + scores = lof_torch.score(x) + assert torch.all(scores == outputs.instance_score) + + lof_torch.infer_threshold(x_ref, 0.1) + outputs = lof_torch.predict(x) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) + assert torch.all(lof_torch(x) == torch.tensor([False, False, True])) + + +def test_lof_torch_backend_ensemble(ensembler): + """ + Test the lof torch backend can be correctly initialized as an ensemble, fit + on data and used to predict outliers. + """ + + lof_torch = LOFTorch(k=[4, 5], ensembler=ensembler) + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + result = lof_torch.predict(x) + assert result.instance_score.shape == (3, ) + + lof_torch.infer_threshold(x_ref, 0.1) + outputs = lof_torch.predict(x) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) + assert torch.all(lof_torch(x) == torch.tensor([False, False, True])) + + +def test_lof_torch_backend_ensemble_ts(tmp_path, ensembler): + """ + Test the lof torch backend can be initalized as an ensemble and + torchscripted, as well as saved and loaded to and from disk. + """ + + lof_torch = LOFTorch(k=[4, 5], ensembler=ensembler) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + + with pytest.raises(NotFitException) as err: + lof_torch(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + with pytest.raises(NotFitException) as err: + lof_torch.predict(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + lof_torch.infer_threshold(x_ref, 0.1) + pred_1 = lof_torch(x) + lof_torch = torch.jit.script(lof_torch) + pred_2 = lof_torch(x) + assert torch.all(pred_1 == pred_2) + + lof_torch.save(tmp_path / 'lof_torch.pt') + lof_torch = torch.load(tmp_path / 'lof_torch.pt') + pred_2 = lof_torch(x) + assert torch.all(pred_1 == pred_2) + + +def test_lof_torch_backend_ts(tmp_path): + """ + Test the lof torch backend can be initalized and torchscripted, as well as + saved and loaded to and from disk. + """ + + lof_torch = LOFTorch(k=7) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + lof_torch.infer_threshold(x_ref, 0.1) + pred_1 = lof_torch(x) + lof_torch = torch.jit.script(lof_torch) + pred_2 = lof_torch(x) + assert torch.all(pred_1 == pred_2) + + lof_torch.save(tmp_path / 'lof_torch.pt') + lof_torch = torch.load(tmp_path / 'lof_torch.pt') + pred_2 = lof_torch(x) + assert torch.all(pred_1 == pred_2) + + +def test_lof_kernel(ensembler): + """ + Test the lof torch backend can be correctly initialized with a kernel, fit + on data and used to predict outliers. + """ + + kernel = GaussianRBF(sigma=torch.tensor((0.25))) + lof_torch = LOFTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + result = lof_torch.predict(x) + assert result.instance_score.shape == (3,) + + lof_torch.infer_threshold(x_ref, 0.1) + outputs = lof_torch.predict(x) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) + assert torch.all(lof_torch(x) == torch.tensor([False, False, True])) + + +@pytest.mark.skip(reason="Can't convert GaussianRBF to torchscript due to torchscript type constraints") +def test_lof_kernel_ts(ensembler): + """ + Test the lof torch backend can be correctly initialized with a kernel, + and torchscripted, as well as saved and loaded to and from disk. + """ + + kernel = GaussianRBF(sigma=torch.tensor((0.25))) + lof_torch = LOFTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + lof_torch.infer_threshold(x_ref, 0.1) + pred_1 = lof_torch(x) + lof_torch = torch.jit.script(lof_torch) + pred_2 = lof_torch(x) + assert torch.all(pred_1 == pred_2) + + +@pytest.mark.parametrize('k', [[4, 5], 4]) +def test_lof_torch_backend_ensemble_fit_errors(k, ensembler): + lof_torch = LOFTorch(k=[4, 5], ensembler=ensembler) + assert not lof_torch._fitted + + # Test that the backend raises an error if it is not fitted before + # calling forward method. + x = torch.randn((1, 10)) + with pytest.raises(NotFitException) as err: + lof_torch(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + # Test that the backend raises an error if it is not fitted before + # predicting. + with pytest.raises(NotFitException) as err: + lof_torch.predict(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + # Test the backend updates _fitted flag on fit. + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + assert lof_torch._fitted + + # Test that the backend raises an if the forward method is called without the + # threshold being inferred. + with pytest.raises(ThresholdNotInferredException) as err: + lof_torch(x) + assert str(err.value) == 'LOFTorch has no threshold set, call `infer_threshold` before predicting.' + + # Test that the backend can call predict without the threshold being inferred. + assert lof_torch.predict(x) From 7382a01d7677c626e3dfbe9a32bcd35dff11d1fc Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 21 Feb 2023 15:09:13 +0000 Subject: [PATCH 121/247] Add kernel option to lof detector pytorch backend --- alibi_detect/od/pytorch/lof.py | 14 +++++++------- .../od/tests/test__lof/test__lof_backend.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 885deb32d..822c1cd67 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -22,8 +22,8 @@ def __init__( Parameters ---------- k - Number of nearest neighbors used to compute LOF. If `k` is a list or array, then an ensemble of LOF detectors - is created with one detector for each value of `k`. + Number of nearest neighbors used to compute LOF. If `k` is a list or array, then an ensemble of LOF + detectors is created with one detector for each value of `k`. kernel If a kernel is specified then instead of using `torch.cdist` the kernel defines the `k` nearest neighbor distance. @@ -71,6 +71,9 @@ def _make_mask(self, reachabilities: torch.Tensor): mask[:k, i] = torch.ones(k)/k return mask + def _compute_K(self, x, y): + return torch.exp(-self.kernel(x, y)) if self.kernel is not None else torch.cdist(x, y) + @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` @@ -101,10 +104,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: self.check_fitted() X = torch.as_tensor(x) - D = torch.cdist(X, self.x_ref) - - # K = -self.kernel(x, self.x_ref) if self.kernel is not None else torch.cdist(x, self.x_ref) - + D = self._compute_K(X, self.x_ref) max_k = torch.max(self.ks) bot_k_items = torch.topk(D, max_k, dim=1, largest=False) bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values @@ -147,7 +147,7 @@ def _fit(self, x_ref: torch.Tensor): The Dataset tensor. """ X = torch.as_tensor(x_ref) - D = torch.cdist(X, X) + D = self._compute_K(X, X) D += torch.eye(len(D)) * torch.max(D) max_k = torch.max(self.ks) bot_k_items = torch.topk(D, max_k, dim=1, largest=False) diff --git a/alibi_detect/od/tests/test__lof/test__lof_backend.py b/alibi_detect/od/tests/test__lof/test__lof_backend.py index 28f1c78b4..96b7d6b6b 100644 --- a/alibi_detect/od/tests/test__lof/test__lof_backend.py +++ b/alibi_detect/od/tests/test__lof/test__lof_backend.py @@ -116,7 +116,7 @@ def test_lof_kernel(ensembler): on data and used to predict outliers. """ - kernel = GaussianRBF(sigma=torch.tensor((0.25))) + kernel = GaussianRBF(sigma=torch.tensor((1))) lof_torch = LOFTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) x_ref = torch.randn((1024, 10)) lof_torch.fit(x_ref) From c50cb47b8672d64638ff07e926807daa75923bb3 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 21 Feb 2023 15:48:49 +0000 Subject: [PATCH 122/247] Add _lof outlier detector frontend --- alibi_detect/od/_lof.py | 177 ++++++++++++++ alibi_detect/od/pytorch/__init__.py | 1 + alibi_detect/od/tests/test__lof/test__lof.py | 230 +++++++++++++++++++ 3 files changed, 408 insertions(+) create mode 100644 alibi_detect/od/_lof.py create mode 100644 alibi_detect/od/tests/test__lof/test__lof.py diff --git a/alibi_detect/od/_lof.py b/alibi_detect/od/_lof.py new file mode 100644 index 000000000..fa23a2e0c --- /dev/null +++ b/alibi_detect/od/_lof.py @@ -0,0 +1,177 @@ +from typing import Callable, Union, Optional, Dict, Any, List, Tuple +from typing import TYPE_CHECKING + +import numpy as np + +from typing_extensions import Literal +from alibi_detect.base import outlier_prediction_dict +from alibi_detect.od.base import TransformProtocol, transform_protocols +from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin +from alibi_detect.od.pytorch import LOFTorch, Ensembler +from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer +from alibi_detect.utils.frameworks import BackendValidator +from alibi_detect.version import __version__ + + +if TYPE_CHECKING: + import torch + + +backends = { + 'pytorch': (LOFTorch, Ensembler) +} + + +class LOF(BaseDetector, FitMixin, ThresholdMixin): + def __init__( + self, + k: Union[int, np.ndarray, List[int], Tuple[int]], + kernel: Optional[Callable] = None, + normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizer', + aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregator', + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, + backend: Literal['pytorch'] = 'pytorch', + ) -> None: + """ + Local Outlier Factor (LOF) outlier detector. + + The LOF detector is a non-parametric method for outlier detection. It computes the local density + deviation of a given data point with respect to its neighbors. It considers as outliers the + samples that have a substantially lower density than their neighbors. + + Parameters + ---------- + k + Number of neirest neighbors to compute distance to. `k` can be a single value or + an array of integers. If an array is passed, an aggregator is required to aggregate + the scores. If `k` is a single value we comput the local outlier factor for that `k`. + Otherwise if `k` is a list then we compute and aggregate the local outlier factor for each + value in `k`. + kernel + Kernel function to use for outlier detection. If ``None``, `torch.cdist` is used. + Otherwise if a kernel is specified then instead of using `torch.cdist` the kernel + defines the k nearest neighbor distance. + normalizer + Normalizer to use for outlier detection. If ``None``, no normalisation is applied. + For a list of available normalizers, see :mod:`alibi_detect.od.pytorch.ensemble`. + aggregator + Aggregator to use for outlier detection. Can be set to ``None`` if `k` is a single + value. For a list of available aggregators, see :mod:`alibi_detect.od.pytorch.ensemble`. + backend + Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. + device + Device type used. The default tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + + Raises + ------ + ValueError + If `k` is an array and `aggregator` is None. + NotImplementedError + If choice of `backend` is not implemented. + """ + super().__init__() + + backend_str: str = backend.lower() + BackendValidator( + backend_options={'pytorch': ['pytorch']}, + construct_name=self.__class__.__name__ + ).verify_backend(backend_str) + + backend_cls, ensembler_cls = backends[backend] + ensembler = None + + if aggregator is None and isinstance(k, (list, np.ndarray, tuple)): + raise ValueError('If `k` is a `np.ndarray`, `list` or `tuple`, ' + 'the `aggregator` argument cannot be ``None``.') + + if isinstance(k, (list, np.ndarray, tuple)): + ensembler = ensembler_cls( + normalizer=get_normalizer(normalizer), + aggregator=get_aggregator(aggregator) + ) + + self.backend = backend_cls(k, kernel=kernel, ensembler=ensembler, device=device) + + # set metadata + self.meta['detector_type'] = 'outlier' + self.meta['data_type'] = 'numeric' + self.meta['online'] = False + + def fit(self, x_ref: np.ndarray) -> None: + """Fit the detector on reference data. + + Parameters + ---------- + x_ref + Reference data used to fit the detector. + """ + self.backend.fit(self.backend._to_tensor(x_ref)) + + def score(self, X: np.ndarray) -> np.ndarray: + """Score `x` instances using the detector. + + The LOF detector scores the instances in `x` by computing the local outlier factor for each instance. The + higher the score, the more anomalous the instance. + + Parameters + ---------- + x + Data to score. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + instance. + """ + score = self.backend.score(self.backend._to_tensor(X)) + return self.backend._to_numpy(score) + + def infer_threshold(self, X: np.ndarray, fpr: float) -> None: + """Infer the threshold for the LOF detector. + + The threshold is computed so that the outlier detector would incorectly classify `fpr` proportion of the + reference data as outliers. + + Parameters + ---------- + x_ref + Reference data used to infer the threshold. + fpr + False positive rate used to infer the threshold. The false positive rate is the proportion of + instances in `x_ref` that are incorrectly classified as outliers. The false positive rate should + be in the range ``(0, 1)``. + """ + self.backend.infer_threshold(self.backend._to_tensor(X), fpr) + + def predict(self, x: np.ndarray) -> Dict[str, Any]: + """Predict whether the instances in `x` are outliers or not. + + Scores the instances in `x` and if the threshold was inferred, returns the outlier labels and p-values as well. + + Parameters + ---------- + x + Data to predict. The shape of `x` should be `(n_instances, n_features)`. + + Returns + ------- + Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ + performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ + `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ + the detector. + """ + outputs = self.backend.predict(self.backend._to_tensor(x)) + output = outlier_prediction_dict() + output['data'] = { + **output['data'], + **self.backend._to_numpy(outputs) + } + output['meta'] = { + **output['meta'], + 'name': self.__class__.__name__, + 'detector_type': 'outlier', + 'online': False, + 'version': __version__, + } + return output diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py index 28a6eef3c..7d3a90181 100644 --- a/alibi_detect/od/pytorch/__init__.py +++ b/alibi_detect/od/pytorch/__init__.py @@ -1,6 +1,7 @@ from alibi_detect.utils.missing_optional_dependency import import_optional KNNTorch = import_optional('alibi_detect.od.pytorch.knn', ['KNNTorch']) +LOFTorch = import_optional('alibi_detect.od.pytorch.lof', ['LOFTorch']) MahalanobisTorch = import_optional('alibi_detect.od.pytorch.mahalanobis', ['MahalanobisTorch']) KernelPCATorch, LinearPCATorch = import_optional('alibi_detect.od.pytorch.pca', ['KernelPCATorch', 'LinearPCATorch']) Ensembler = import_optional('alibi_detect.od.pytorch.ensemble', ['Ensembler']) diff --git a/alibi_detect/od/tests/test__lof/test__lof.py b/alibi_detect/od/tests/test__lof/test__lof.py new file mode 100644 index 000000000..0074ceae2 --- /dev/null +++ b/alibi_detect/od/tests/test__lof/test__lof.py @@ -0,0 +1,230 @@ +import pytest +import numpy as np +import torch + +from alibi_detect.od._lof import LOF +from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ + MinAggregator, ShiftAndScaleNormalizer, PValNormalizer +from alibi_detect.base import NotFitException + +from sklearn.datasets import make_moons + + +def make_lof_detector(k=5, aggregator=None, normalizer=None): + lof_detector = LOF( + k=k, aggregator=aggregator, + normalizer=normalizer + ) + x_ref = np.random.randn(100, 2) + lof_detector.fit(x_ref) + lof_detector.infer_threshold(x_ref, 0.1) + return lof_detector + + +def test_unfitted_lof_single_score(): + lof_detector = LOF(k=10) + x = np.array([[0, 10], [0.1, 0]]) + + # test predict raises exception when not fitted + with pytest.raises(NotFitException) as err: + _ = lof_detector.predict(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + +@pytest.mark.parametrize('k', [10, [8, 9, 10]]) +def test_fitted_lof_single_score(k): + lof_detector = LOF(k=k) + x_ref = np.random.randn(100, 2) + lof_detector.fit(x_ref) + x = np.array([[0, 10], [0.1, 0]]) + # test fitted but not threshold inferred detectors + # can still score data using the predict method. + y = lof_detector.predict(x) + y = y['data'] + assert y['instance_score'][0] > 7 + assert y['instance_score'][1] < 2 + + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None + + +def test_incorrect_lof_ensemble_init(): + # test lof ensemble with aggregator passed as None raises exception + + with pytest.raises(ValueError) as err: + LOF(k=[8, 9, 10], aggregator=None) + assert str(err.value) == ('If `k` is a `np.ndarray`, `list` or `tuple`, ' + 'the `aggregator` argument cannot be ``None``.') + + +def test_fitted_lof_predict(): + """ + Test that a detector fitted on data and with threshold inferred correctly, will score + and label outliers, as well as return the p-values using the predict method. Also Check + that the score method gives the same results. + """ + + lof_detector = make_lof_detector(k=10) + x_ref = np.random.randn(100, 2) + lof_detector.infer_threshold(x_ref, 0.1) + x = np.array([[0, 10], [0, 0.1]]) + + y = lof_detector.predict(x) + y = y['data'] + scores = lof_detector.score(x) + assert np.all(y['instance_score'] == scores) + assert y['instance_score'][0] > 7 + assert y['instance_score'][1] < 2 + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() + + +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) +def test_unfitted_lof_ensemble(aggregator, normalizer): + lof_detector = LOF( + k=[8, 9, 10], + aggregator=aggregator(), + normalizer=normalizer() + ) + x = np.array([[0, 10], [0.1, 0]]) + + # Test unfit lof ensemble raises exception when calling predict method. + with pytest.raises(NotFitException) as err: + _ = lof_detector.predict(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) +def test_fitted_lof_ensemble(aggregator, normalizer): + lof_detector = LOF( + k=[8, 9, 10], + aggregator=aggregator(), + normalizer=normalizer() + ) + x_ref = np.random.randn(100, 2) + lof_detector.fit(x_ref) + x = np.array([[0, 10], [0, 0.1]]) + + # test fitted but not threshold inferred detectors can still score data using the predict method. + y = lof_detector.predict(x) + y = y['data'] + assert y['instance_score'].all() + assert not y['threshold_inferred'] + assert y['threshold'] is None + assert y['is_outlier'] is None + assert y['p_value'] is None + + +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) +def test_fitted_lof_ensemble_predict(aggregator, normalizer): + lof_detector = make_lof_detector( + k=[8, 9, 10], + aggregator=aggregator(), + normalizer=normalizer() + ) + x = np.array([[0, 10], [0, 0.1]]) + + # test fitted detectors with inferred thresholds can score data using the predict method. + y = lof_detector.predict(x) + y = y['data'] + assert y['threshold_inferred'] + assert y['threshold'] is not None + assert y['p_value'].all() + assert (y['is_outlier'] == [True, False]).all() + + +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator]) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) +def test_lof_ensemble_torch_script(aggregator, normalizer): + lof_detector = make_lof_detector(k=[5, 6, 7], aggregator=aggregator(), normalizer=normalizer()) + tslof = torch.jit.script(lof_detector.backend) + x = torch.tensor([[0, 10], [0, 0.1]]) + + # test torchscripted ensemble lof detector can be saved and loaded correctly. + y = tslof(x) + assert torch.all(y == torch.tensor([True, False])) + + +def test_lof_single_torchscript(): + lof_detector = make_lof_detector(k=5) + tslof = torch.jit.script(lof_detector.backend) + x = torch.tensor([[0, 10], [0, 0.1]]) + + # test torchscripted single lof detector can be saved and loaded correctly. + y = tslof(x) + assert torch.all(y == torch.tensor([True, False])) + + +@pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), + MaxAggregator, MinAggregator, lambda: 'AverageAggregator', + lambda: 'TopKAggregator', lambda: 'MaxAggregator', + lambda: 'MinAggregator']) +@pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None, + lambda: 'ShiftAndScaleNormalizer', lambda: 'PValNormalizer']) +def test_lof_ensemble_integration(aggregator, normalizer): + """Test lof ensemble detector on moons dataset. + + Tests ensemble lof detector with every combination of aggregator and normalizer on the moons dataset. + Fits and infers thresholds in each case. Verifies that the detector can correctly detect inliers + and outliers and that it can be serialized using the torchscript. + """ + + lof_detector = LOF( + k=[10, 14, 18], + aggregator=aggregator(), + normalizer=normalizer() + ) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + lof_detector.fit(X_ref) + lof_detector.infer_threshold(X_ref, 0.1) + result = lof_detector.predict(x_inlier) + result = result['data']['is_outlier'][0] + assert not result + + x_outlier = np.array([[-1, 1.5]]) + result = lof_detector.predict(x_outlier) + result = result['data']['is_outlier'][0] + assert result + + tslof = torch.jit.script(lof_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = tslof(x) + assert torch.all(y == torch.tensor([False, True])) + + +def test_lof_integration(): + """Test lof detector on moons dataset. + + Tests lof detector on the moons dataset. Fits and infers thresholds and verifies that the detector can + correctly detect inliers and outliers. Checks that it can be serialized using the torchscript. + """ + lof_detector = LOF(k=18) + X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) + X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] + lof_detector.fit(X_ref) + lof_detector.infer_threshold(X_ref, 0.1) + result = lof_detector.predict(x_inlier) + result = result['data']['is_outlier'][0] + assert not result + + x_outlier = np.array([[-1, 1.5]]) + result = lof_detector.predict(x_outlier) + result = result['data']['is_outlier'][0] + assert result + + tslof = torch.jit.script(lof_detector.backend) + x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + y = tslof(x) + assert torch.all(y == torch.tensor([False, True])) From e53d4d134feb51c03f60f23dc86f1e8e2b74cb07 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 21 Feb 2023 15:57:53 +0000 Subject: [PATCH 123/247] Add device logic --- alibi_detect/od/pytorch/lof.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 822c1cd67..98c5da3ff 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -66,9 +66,9 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return preds def _make_mask(self, reachabilities: torch.Tensor): - mask = torch.zeros_like(reachabilities[0]) + mask = torch.zeros_like(reachabilities[0], device=self.device) for i, k in enumerate(self.ks): - mask[:k, i] = torch.ones(k)/k + mask[:k, i] = torch.ones(k, device=self.device)/k return mask def _compute_K(self, x, y): @@ -148,7 +148,7 @@ def _fit(self, x_ref: torch.Tensor): """ X = torch.as_tensor(x_ref) D = self._compute_K(X, X) - D += torch.eye(len(D)) * torch.max(D) + D += torch.eye(len(D), device=self.device) * torch.max(D) max_k = torch.max(self.ks) bot_k_items = torch.topk(D, max_k, dim=1, largest=False) bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values From c86d0ada32a001b01164e6a83cfe23ba628b7d71 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 21 Feb 2023 16:11:11 +0000 Subject: [PATCH 124/247] Fix typing errors --- alibi_detect/od/pytorch/lof.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 98c5da3ff..677ffa190 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -106,7 +106,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: X = torch.as_tensor(x) D = self._compute_K(X, self.x_ref) max_k = torch.max(self.ks) - bot_k_items = torch.topk(D, max_k, dim=1, largest=False) + bot_k_items = torch.topk(D, int(max_k), dim=1, largest=False) bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values lower_bounds = self.knn_dists_ref[bot_k_inds] reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) @@ -150,7 +150,7 @@ def _fit(self, x_ref: torch.Tensor): D = self._compute_K(X, X) D += torch.eye(len(D), device=self.device) * torch.max(D) max_k = torch.max(self.ks) - bot_k_items = torch.topk(D, max_k, dim=1, largest=False) + bot_k_items = torch.topk(D, int(max_k), dim=1, largest=False) bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values self.knn_dists_ref = bot_k_dists[:, self.ks-1] lower_bounds = self.knn_dists_ref[bot_k_inds] From 46bb7475d1ae0314f8224c1c8b2843d06ae72c1e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 22 Feb 2023 11:12:44 +0000 Subject: [PATCH 125/247] Fit ensembler in infer_threshold step not fit step --- alibi_detect/od/pytorch/base.py | 15 +++- alibi_detect/od/pytorch/ensemble.py | 6 +- alibi_detect/od/pytorch/knn.py | 3 - alibi_detect/od/tests/test__knn/test__knn.py | 43 ++++++++---- .../od/tests/test__knn/test__knn_backend.py | 70 +++++++++---------- 5 files changed, 77 insertions(+), 60 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index c3d26a65f..a0a9e3ad5 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -141,10 +141,22 @@ def _ensembler(self, x: torch.Tensor) -> torch.Tensor: Returns ------- `torch.Tensor` or original data without alteration + + Raises + ------ + ThresholdNotInferredException + If the detector is an ensemble, and the ensembler used to aggregate the outlier scores has a fitable + component, then the detector threshold must be inferred before predictions can be made. This is because + while the scoring functionality of the detector is fit within the `.fit` method on the training data + the ensembler has to be fit on the validation data along with the threshold and this is done in the + `.infer_threshold` method. """ if hasattr(self, 'ensembler') and self.ensembler is not None: # `type: ignore` here becuase self.ensembler here causes an error with mypy when using torch.jit.script. # For some reason it thinks self.ensembler is a torch.Tensor and therefore is not callable. + if not torch.jit.is_scripting(): + if not self.ensembler.fitted: # type: ignore + self.check_threshold_inferred() return self.ensembler(x) # type: ignore else: return x @@ -196,7 +208,8 @@ def infer_threshold(self, x: torch.Tensor, fpr: float) -> None: if not 0 < fpr < 1: ValueError('`fpr` must be in `(0, 1)`.') self.val_scores = self.score(x) - self.val_scores = self._ensembler(self.val_scores) + if self.ensemble: + self.val_scores = self.ensembler.fit(self.val_scores).transform(self.val_scores) # type: ignore self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 79cdd24de..ec165b1c7 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -37,7 +37,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class FitMixinTorch(ABC): - _fitted = False + fitted = False def __init__(self): """Fit mixin @@ -47,7 +47,7 @@ def __init__(self): super().__init__() def fit(self, x: torch.Tensor) -> FitMixinTorch: - self._fitted = True + self.fitted = True self._fit(x) return self @@ -73,7 +73,7 @@ def check_fitted(self): NotFitException Raised if method called and object has not been fit. """ - if not self._fitted: + if not self.fitted: raise NotFitException(f'{self.__class__.__name__} has not been fit!') diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index 3f412f58c..ce6f2bd96 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -100,6 +100,3 @@ def _fit(self, x_ref: torch.Tensor): The Dataset tensor. """ self.x_ref = x_ref - if self.ensemble: - scores = self.score(x_ref) - self.ensembler.fit(scores) diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index 53e8ca237..6ff803c1e 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -5,7 +5,7 @@ from alibi_detect.od._knn import KNN from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer -from alibi_detect.base import NotFitException +from alibi_detect.base import NotFitException, ThresholdNotInferredException from sklearn.datasets import make_moons @@ -31,25 +31,39 @@ def test_unfitted_knn_single_score(): assert str(err.value) == 'KNNTorch has not been fit!' -@pytest.mark.parametrize('k', [10, [8, 9, 10]]) -def test_fitted_knn_single_score(k): - knn_detector = KNN(k=k) +def test_fitted_knn_score(): + """ + Test fitted but not threshold inferred non-ensemble detectors can still score data using the predict method. + Unlike the ensemble detectors, the non-ensemble detectors do not require the ensembler to be fit in the + infer_threshold method. See the test_fitted_knn_ensemble_score test for the ensemble case. + """ + knn_detector = KNN(k=10) x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) - # test fitted but not threshold inferred detectors - # can still score data using the predict method. y = knn_detector.predict(x) y = y['data'] assert y['instance_score'][0] > 5 assert y['instance_score'][1] < 1 - assert not y['threshold_inferred'] assert y['threshold'] is None assert y['is_outlier'] is None assert y['p_value'] is None +def test_fitted_knn_ensemble_score(): + """ + Test fitted but not threshold inferred ensemble detectors correctly raise an error when calling + the predict method. This is becuase the ensembler is fit in the infer_threshold method. + """ + knn_detector = KNN(k=[10, 14, 18]) + x_ref = np.random.randn(100, 2) + knn_detector.fit(x_ref) + x = np.array([[0, 10], [0.1, 0]]) + with pytest.raises(ThresholdNotInferredException): + knn_detector.predict(x) + + def test_incorrect_knn_ensemble_init(): # test knn ensemble with aggregator passed as None raises exception @@ -113,14 +127,13 @@ def test_fitted_knn_ensemble(aggregator, normalizer): knn_detector.fit(x_ref) x = np.array([[0, 10], [0, 0.1]]) - # test fitted but not threshold inferred detectors can still score data using the predict method. - y = knn_detector.predict(x) - y = y['data'] - assert y['instance_score'].all() - assert not y['threshold_inferred'] - assert y['threshold'] is None - assert y['is_outlier'] is None - assert y['p_value'] is None + # test ensemble raises ThresholdNotInferredException if only fit and not threshold inferred and + # the normalizer is not None. + if normalizer() is not None: + with pytest.raises(ThresholdNotInferredException): + knn_detector.predict(x) + else: + knn_detector.predict(x) @pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 9138059b4..68fdd7992 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -4,7 +4,7 @@ from alibi_detect.od.pytorch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.base import NotFitException @pytest.fixture(scope='session') @@ -48,9 +48,6 @@ def test_knn_torch_backend_ensemble(ensembler): x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - result = knn_torch.predict(x) - assert result.instance_score.shape == (3, ) - knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) @@ -121,9 +118,6 @@ def test_knn_kernel(ensembler): x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - result = knn_torch.predict(x) - assert result.instance_score.shape == (3,) - knn_torch.infer_threshold(x_ref, 0.1) outputs = knn_torch.predict(x) assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) @@ -149,34 +143,34 @@ def test_knn_kernel_ts(ensembler): assert torch.all(pred_1 == pred_2) -@pytest.mark.parametrize('k', [[4, 5], 4]) -def test_knn_torch_backend_ensemble_fit_errors(k, ensembler): - knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) - assert not knn_torch._fitted - - # Test that the backend raises an error if it is not fitted before - # calling forward method. - x = torch.randn((1, 10)) - with pytest.raises(NotFitException) as err: - knn_torch(x) - assert str(err.value) == 'KNNTorch has not been fit!' - - # Test that the backend raises an error if it is not fitted before - # predicting. - with pytest.raises(NotFitException) as err: - knn_torch.predict(x) - assert str(err.value) == 'KNNTorch has not been fit!' - - # Test the backend updates _fitted flag on fit. - x_ref = torch.randn((1024, 10)) - knn_torch.fit(x_ref) - assert knn_torch._fitted - - # Test that the backend raises an if the forward method is called without the - # threshold being inferred. - with pytest.raises(ThresholdNotInferredException) as err: - knn_torch(x) - assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' - - # Test that the backend can call predict without the threshold being inferred. - assert knn_torch.predict(x) +# @pytest.mark.parametrize('k', [[4, 5], 4]) +# def test_knn_torch_backend_ensemble_fit_errors(k, ensembler): +# knn_torch = KNNTorch(k=k, ensembler=ensembler) +# assert not knn_torch.fitted + +# # Test that the backend raises an error if it is not fitted before +# # calling forward method. +# x = torch.randn((1, 10)) +# with pytest.raises(NotFitException) as err: +# knn_torch(x) +# assert str(err.value) == 'KNNTorch has not been fit!' + +# # Test that the backend raises an error if it is not fitted before +# # predicting. +# with pytest.raises(NotFitException) as err: +# knn_torch.predict(x) +# assert str(err.value) == 'KNNTorch has not been fit!' + +# # Test the backend updates fitted flag on fit. +# x_ref = torch.randn((1024, 10)) +# knn_torch.fit(x_ref) +# assert knn_torch.fitted + +# # Test that the backend raises an if the forward method is called without the +# # threshold being inferred. +# with pytest.raises(ThresholdNotInferredException) as err: +# knn_torch(x) +# assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + +# # Test that the backend can call predict without the threshold being inferred. +# assert knn_torch.predict(x) From a3e89e35b3da867e2d0c8c42ed7f97f7449cb78f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 22 Feb 2023 11:31:07 +0000 Subject: [PATCH 126/247] Add further documentation to knn detector --- alibi_detect/od/_knn.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 4eb384b46..b39067010 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -42,7 +42,13 @@ def __init__( The detector can be initialized with `k` a single value or an array of values. If `k` is a single value then the outlier score is the distance/kernel similarity to the k-th nearest neighbor. If `k` is an array of values then the outlier score is the distance/kernel similarity to each of the specified `k` neighbors. - In the latter case, an aggregator must be specified to aggregate the scores. + In the latter case, an `ensembler` must be specified to aggregate the scores. + + Note that the `ensembler` is fit in the `infer_threshold` method and so if using an array of `k` values, the + `infer_threshold` method must be called before the `predict` method othewrise an exception is raised. If `k` + is a single value then the predict method can be called without first calling `infer_threshold` but only + scores will be returned and not outlier predictions. + Parameters ---------- From b08572de610dca4a2f84b5d0d275037841292206 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 28 Feb 2023 16:38:34 +0000 Subject: [PATCH 127/247] Refactor exceptions into seperate file and add base class --- alibi_detect/base.py | 10 ------- alibi_detect/exceptions.py | 27 +++++++++++++++++++ alibi_detect/od/pytorch/base.py | 2 +- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/tests/test__knn/test__knn.py | 2 +- .../od/tests/test__knn/test__knn_backend.py | 2 +- alibi_detect/od/tests/test_ensemble.py | 2 +- 7 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 alibi_detect/exceptions.py diff --git a/alibi_detect/base.py b/alibi_detect/base.py index 068059fb1..155f462a7 100644 --- a/alibi_detect/base.py +++ b/alibi_detect/base.py @@ -264,13 +264,3 @@ def default(self, obj): elif isinstance(obj, (np.ndarray,)): return obj.tolist() return json.JSONEncoder.default(self, obj) - - -class NotFitException(Exception): - """Exception raised when a transform is not fitted.""" - pass - - -class ThresholdNotInferredException(Exception): - """Exception raised when a threshold not inferred for an outlier detector.""" - pass diff --git a/alibi_detect/exceptions.py b/alibi_detect/exceptions.py new file mode 100644 index 000000000..73aa9ea71 --- /dev/null +++ b/alibi_detect/exceptions.py @@ -0,0 +1,27 @@ +"""This module defines the Alibi exception hierarchy and common exceptions used across the library.""" + +from abc import ABC + + +class AlibiDetectException(Exception, ABC): + def __init__(self, message: str) -> None: + """Abstract base class of all alibi detect exceptions. + + Parameters + ---------- + message + The error message. + """ + super().__init__(message) + + +class NotFitException(AlibiDetectException): + """Exception raised when a transform is not fitted.""" + + pass + + +class ThresholdNotInferredException(AlibiDetectException): + """Exception raised when a threshold not inferred for an outlier detector.""" + + pass diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index a0a9e3ad5..ef3cb1f96 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -8,7 +8,7 @@ from alibi_detect.od.pytorch.ensemble import FitMixinTorch from alibi_detect.utils.pytorch.misc import get_device -from alibi_detect.base import ThresholdNotInferredException +from alibi_detect.exceptions import ThresholdNotInferredException @dataclass diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index ec165b1c7..6ed0c21d0 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -6,7 +6,7 @@ import numpy as np from torch.nn import Module -from alibi_detect.base import NotFitException +from alibi_detect.exceptions import NotFitException class BaseTransformTorch(Module, ABC): diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index 6ff803c1e..ed8976985 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -5,7 +5,7 @@ from alibi_detect.od._knn import KNN from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFitException, ThresholdNotInferredException from sklearn.datasets import make_moons diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 68fdd7992..cdf71d533 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -4,7 +4,7 @@ from alibi_detect.od.pytorch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator -from alibi_detect.base import NotFitException +from alibi_detect.exceptions import NotFitException @pytest.fixture(scope='session') diff --git a/alibi_detect/od/tests/test_ensemble.py b/alibi_detect/od/tests/test_ensemble.py index e5f350267..dc5e21ce6 100644 --- a/alibi_detect/od/tests/test_ensemble.py +++ b/alibi_detect/od/tests/test_ensemble.py @@ -2,7 +2,7 @@ import torch from alibi_detect.od.pytorch import ensemble -from alibi_detect.base import NotFitException +from alibi_detect.exceptions import NotFitException def test_pval_normalizer(): From 70921b7d379197bbd23ad2fb4190af1383e54e8c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 28 Feb 2023 16:49:22 +0000 Subject: [PATCH 128/247] Rename exceptions to be consistent with alibi --- alibi_detect/exceptions.py | 8 ++++---- alibi_detect/od/pytorch/base.py | 8 ++++---- alibi_detect/od/pytorch/ensemble.py | 6 +++--- alibi_detect/od/pytorch/knn.py | 4 ++-- alibi_detect/od/tests/test__knn/test__knn.py | 12 ++++++------ alibi_detect/od/tests/test__knn/test__knn_backend.py | 12 ++++++------ alibi_detect/od/tests/test_ensemble.py | 6 +++--- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/alibi_detect/exceptions.py b/alibi_detect/exceptions.py index 73aa9ea71..37a6085c5 100644 --- a/alibi_detect/exceptions.py +++ b/alibi_detect/exceptions.py @@ -3,9 +3,9 @@ from abc import ABC -class AlibiDetectException(Exception, ABC): +class AlibiDetectError(Exception, ABC): def __init__(self, message: str) -> None: - """Abstract base class of all alibi detect exceptions. + """Abstract base class of all alibi detect errors. Parameters ---------- @@ -15,13 +15,13 @@ def __init__(self, message: str) -> None: super().__init__(message) -class NotFitException(AlibiDetectException): +class NotFittedError(AlibiDetectError): """Exception raised when a transform is not fitted.""" pass -class ThresholdNotInferredException(AlibiDetectException): +class ThresholdNotInferredError(AlibiDetectError): """Exception raised when a threshold not inferred for an outlier detector.""" pass diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index ef3cb1f96..12127e310 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -8,7 +8,7 @@ from alibi_detect.od.pytorch.ensemble import FitMixinTorch from alibi_detect.utils.pytorch.misc import get_device -from alibi_detect.exceptions import ThresholdNotInferredException +from alibi_detect.exceptions import ThresholdNotInferredError @dataclass @@ -94,11 +94,11 @@ def check_threshold_inferred(self): Raises ------ - ThresholdNotInferredException + ThresholdNotInferredError Raised if threshold is not inferred. """ if not self.threshold_inferred: - raise ThresholdNotInferredException((f'{self.__class__.__name__} has no threshold set, ' + raise ThresholdNotInferredError((f'{self.__class__.__name__} has no threshold set, ' 'call `infer_threshold` before predicting.')) @staticmethod @@ -144,7 +144,7 @@ def _ensembler(self, x: torch.Tensor) -> torch.Tensor: Raises ------ - ThresholdNotInferredException + ThresholdNotInferredError If the detector is an ensemble, and the ensembler used to aggregate the outlier scores has a fitable component, then the detector threshold must be inferred before predictions can be made. This is because while the scoring functionality of the detector is fit within the `.fit` method on the training data diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 6ed0c21d0..049795e89 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -6,7 +6,7 @@ import numpy as np from torch.nn import Module -from alibi_detect.exceptions import NotFitException +from alibi_detect.exceptions import NotFittedError class BaseTransformTorch(Module, ABC): @@ -70,11 +70,11 @@ def check_fitted(self): Raises ------ - NotFitException + NotFittedError Raised if method called and object has not been fit. """ if not self.fitted: - raise NotFitException(f'{self.__class__.__name__} has not been fit!') + raise NotFittedError(f'{self.__class__.__name__} has not been fit!') class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index ce6f2bd96..e0a58681d 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -56,7 +56,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Raises ------ - ThresholdNotInferredException + ThresholdNotInferredError If called before detector has had `infer_threshold` method called. """ raw_scores = self.score(x) @@ -81,7 +81,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: Raises ------ - NotFitException + NotFittedError If called before detector has been fit. """ if not torch.jit.is_scripting(): diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index ed8976985..0f8da69d2 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -5,7 +5,7 @@ from alibi_detect.od._knn import KNN from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer -from alibi_detect.exceptions import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError from sklearn.datasets import make_moons @@ -26,7 +26,7 @@ def test_unfitted_knn_single_score(): x = np.array([[0, 10], [0.1, 0]]) # test predict raises exception when not fitted - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: _ = knn_detector.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -60,7 +60,7 @@ def test_fitted_knn_ensemble_score(): x_ref = np.random.randn(100, 2) knn_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) - with pytest.raises(ThresholdNotInferredException): + with pytest.raises(ThresholdNotInferredError): knn_detector.predict(x) @@ -109,7 +109,7 @@ def test_unfitted_knn_ensemble(aggregator, normalizer): x = np.array([[0, 10], [0.1, 0]]) # Test unfit knn ensemble raises exception when calling predict method. - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: _ = knn_detector.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -127,10 +127,10 @@ def test_fitted_knn_ensemble(aggregator, normalizer): knn_detector.fit(x_ref) x = np.array([[0, 10], [0, 0.1]]) - # test ensemble raises ThresholdNotInferredException if only fit and not threshold inferred and + # test ensemble raises ThresholdNotInferredError if only fit and not threshold inferred and # the normalizer is not None. if normalizer() is not None: - with pytest.raises(ThresholdNotInferredException): + with pytest.raises(ThresholdNotInferredError): knn_detector.predict(x) else: knn_detector.predict(x) diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index cdf71d533..49ffe8838 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -4,7 +4,7 @@ from alibi_detect.od.pytorch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator -from alibi_detect.exceptions import NotFitException +from alibi_detect.exceptions import NotFittedError @pytest.fixture(scope='session') @@ -63,11 +63,11 @@ def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: knn_torch(x) assert str(err.value) == 'KNNTorch has not been fit!' - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: knn_torch.predict(x) assert str(err.value) == 'KNNTorch has not been fit!' @@ -151,13 +151,13 @@ def test_knn_kernel_ts(ensembler): # # Test that the backend raises an error if it is not fitted before # # calling forward method. # x = torch.randn((1, 10)) -# with pytest.raises(NotFitException) as err: +# with pytest.raises(NotFittedError) as err: # knn_torch(x) # assert str(err.value) == 'KNNTorch has not been fit!' # # Test that the backend raises an error if it is not fitted before # # predicting. -# with pytest.raises(NotFitException) as err: +# with pytest.raises(NotFittedError) as err: # knn_torch.predict(x) # assert str(err.value) == 'KNNTorch has not been fit!' @@ -168,7 +168,7 @@ def test_knn_kernel_ts(ensembler): # # Test that the backend raises an if the forward method is called without the # # threshold being inferred. -# with pytest.raises(ThresholdNotInferredException) as err: +# with pytest.raises(ThresholdNotInferredError) as err: # knn_torch(x) # assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' diff --git a/alibi_detect/od/tests/test_ensemble.py b/alibi_detect/od/tests/test_ensemble.py index dc5e21ce6..1fa0cd241 100644 --- a/alibi_detect/od/tests/test_ensemble.py +++ b/alibi_detect/od/tests/test_ensemble.py @@ -2,7 +2,7 @@ import torch from alibi_detect.od.pytorch import ensemble -from alibi_detect.exceptions import NotFitException +from alibi_detect.exceptions import NotFittedError def test_pval_normalizer(): @@ -10,7 +10,7 @@ def test_pval_normalizer(): x = torch.randn(3, 10) x_ref = torch.randn(64, 10) # unfit normalizer raises exception - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: normalizer(x) assert err.value.args[0] == 'PValNormalizer has not been fit!' @@ -40,7 +40,7 @@ def test_shift_and_scale_normalizer(): x_ref = torch.randn(5000, 10) * 3 + 2 # unfit normalizer raises exception - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: normalizer(x) assert err.value.args[0] == 'ShiftAndScaleNormalizer has not been fit!' From 87c8ab7118b950d7755d3088c516a24da5bf1f30 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 14:38:48 +0000 Subject: [PATCH 129/247] Remove __future__ annotations imports and unused _types imports --- alibi_detect/od/base.py | 1 - alibi_detect/od/pytorch/base.py | 3 +-- alibi_detect/od/pytorch/ensemble.py | 7 +++---- alibi_detect/utils/_types.py | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 5edb1eba6..46f504228 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -1,4 +1,3 @@ -from __future__ import annotations from typing import Union from typing_extensions import Protocol, runtime_checkable diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 12127e310..b130f44ce 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -1,4 +1,3 @@ -from __future__ import annotations from typing import List, Union, Optional, Dict from dataclasses import dataclass, asdict from abc import ABC, abstractmethod @@ -99,7 +98,7 @@ def check_threshold_inferred(self): """ if not self.threshold_inferred: raise ThresholdNotInferredError((f'{self.__class__.__name__} has no threshold set, ' - 'call `infer_threshold` before predicting.')) + 'call `infer_threshold` before predicting.')) @staticmethod def _to_numpy(arg: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict]: diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 049795e89..9756e4784 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -1,4 +1,3 @@ -from __future__ import annotations from abc import ABC, abstractmethod from typing import Optional @@ -46,7 +45,7 @@ def __init__(self): """ super().__init__() - def fit(self, x: torch.Tensor) -> FitMixinTorch: + def fit(self, x: torch.Tensor) -> 'FitMixinTorch': self.fitted = True self._fit(x) return self @@ -115,7 +114,7 @@ def __init__(self): super().__init__() self.val_scores = None - def _fit(self, val_scores: torch.Tensor) -> PValNormalizer: + def _fit(self, val_scores: torch.Tensor) -> 'PValNormalizer': """Fit transform on scores. Parameters @@ -155,7 +154,7 @@ def __init__(self): self.val_means = None self.val_scales = None - def _fit(self, val_scores: torch.Tensor) -> ShiftAndScaleNormalizer: + def _fit(self, val_scores: torch.Tensor) -> 'ShiftAndScaleNormalizer': """Computes the mean and standard deviation of the scores and stores them. Parameters diff --git a/alibi_detect/utils/_types.py b/alibi_detect/utils/_types.py index 97796eecb..a639bf08d 100644 --- a/alibi_detect/utils/_types.py +++ b/alibi_detect/utils/_types.py @@ -7,9 +7,9 @@ # Literal for typing if sys.version_info >= (3, 8): - from typing import Literal, Protocol, runtime_checkable # noqa + from typing import Literal # noqa else: - from typing_extensions import Literal, Protocol, runtime_checkable # noqa + from typing_extensions import Literal # noqa # Optional dep dependent tuples of types From 21a5fc0afbe5a43a1d5aa00b2f4361e86c4b148a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 15:45:16 +0000 Subject: [PATCH 130/247] Add link from base protocols to transform docstrings --- alibi_detect/od/base.py | 14 ++++++++-- alibi_detect/od/pytorch/ensemble.py | 41 ++++++++++++++++++----------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 46f504228..46cea1cb4 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -7,7 +7,12 @@ # avoid the torch/tensorflow imports in the base class. @runtime_checkable class TransformProtocol(Protocol): - """Protocol for transformer objects.""" + """Protocol for transformer objects. + + The :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseTransformTorch` object provides abstract methods for + objects that map between `torch` tensors. This protocol models the interface of the `BaseTransformTorch` + class. + """ def transform(self, x): pass @@ -17,7 +22,12 @@ def _transform(self, x): @runtime_checkable class FittedTransformProtocol(TransformProtocol, Protocol): - """Protocol for fitted transformer objects.""" + """Protocol for fitted transformer objects. + + The :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch` object provides abstract methods for + objects that map between `torch` tensors and also require to be fit. This protocol models the interface of + the `BaseFittedTransformTorch` + class.""" def fit(self, x_ref): pass diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 9756e4784..4c5890443 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -17,16 +17,20 @@ def __init__(self): super().__init__() def transform(self, x: torch.Tensor): + """Public transform method. See `_transform` for implementation details. + + Parameters + ---------- + x + `torch.Tensor` array to be transformed + """ return self._transform(x) @abstractmethod def _transform(self, x: torch.Tensor): """Applies class transform to `torch.Tensor` - Parameters - ---------- - x - `torch.Tensor` array to be transformed + This method should be overridden on child classes. """ pass @@ -46,6 +50,16 @@ def __init__(self): super().__init__() def fit(self, x: torch.Tensor) -> 'FitMixinTorch': + """Public fit method. + + The `_fit` method contains the implementation details and should be overridden on child classes. Once the + `_fit` method has fit the object the `fitted` attribute should be set to `True`. + + Parameters + ---------- + x + `torch.Tensor` to fit object on. + """ self.fitted = True self._fit(x) return self @@ -54,12 +68,7 @@ def fit(self, x: torch.Tensor) -> 'FitMixinTorch': def _fit(self, x: torch.Tensor): """Fit on `x` tensor. - This method should be overidden on child classes. - - Parameters - ---------- - x - Reference `torch.Tensor` for fitting object. + This method should be overridden on child classes and should set the `fitted` attribute to `True`. """ pass @@ -80,14 +89,14 @@ class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): def __init__(self): """Base Fitted Transform class. - Extends `BaseTransfrom` with fit functionality. Ensures that transform has been fit prior to + Extends `BaseTransform` with fit functionality. Ensures that transform has been fit prior to applying transform. """ BaseTransformTorch.__init__(self) FitMixinTorch.__init__(self) def transform(self, x: torch.Tensor) -> torch.Tensor: - """Checks to make sure transform has been fitted and then applies trasform to input tensor. + """Checks to make sure transform has been fitted and then applies transform to input tensor. Parameters ---------- @@ -145,7 +154,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class ShiftAndScaleNormalizer(BaseFittedTransformTorch): def __init__(self): - """Maps scores to their normalised values. + """Maps scores to their normalized values. Needs to be fit (see :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch`). Subtracts the dataset mean and scales by the standard deviation. @@ -167,7 +176,7 @@ def _fit(self, val_scores: torch.Tensor) -> 'ShiftAndScaleNormalizer': return self def _transform(self, scores: torch.Tensor) -> torch.Tensor: - """Transform scores to normalised values. Subtracts the mean and scales by the standard deviation. + """Transform scores to normalized values. Subtracts the mean and scales by the standard deviation. Parameters ---------- @@ -176,7 +185,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: Returns ------- - `Torch.Tensor` of normalised scores. + `Torch.Tensor` of normalized scores. """ return (scores - self.val_means)/self.val_scales @@ -321,7 +330,7 @@ def _transform(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - `Torch.Tensor` of aggregated and normalised scores. + `Torch.Tensor` of aggregated and normalized scores. """ if self.normalizer is not None: x = self.normalizer(x) From 75a7fb691772dd83afb3280883cd6fe181b41705 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 15:59:42 +0000 Subject: [PATCH 131/247] Rename transform_protocols and others to PascalCase --- alibi_detect/od/__init__.py | 12 ++++++------ alibi_detect/od/_knn.py | 4 ++-- alibi_detect/od/base.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index 367a19c9d..4eb8ef104 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -4,7 +4,7 @@ from .mahalanobis import Mahalanobis from .sr import SpectralResidual -from alibi_detect.od.base import TransformProtocol, transform_protocols +from alibi_detect.od.base import TransformProtocol, TransformProtocolType from typing_extensions import Literal from typing import Union @@ -16,12 +16,12 @@ ) -normalizer_literals = Literal['PValNormalizer', 'ShiftAndScaleNormalizer'] -aggregator_literals = Literal['TopKAggregator', 'AverageAggregator', - 'MaxAggregator', 'MinAggregator'] +NormalizerLiterals = Literal['PValNormalizer', 'ShiftAndScaleNormalizer'] +AggregatorLiterals = Literal['TopKAggregator', 'AverageAggregator', + 'MaxAggregator', 'MinAggregator'] -def get_normalizer(normalizer: Union[transform_protocols, normalizer_literals]) -> TransformProtocol: +def get_normalizer(normalizer: Union[TransformProtocolType, NormalizerLiterals]) -> TransformProtocol: if isinstance(normalizer, str): try: return { @@ -33,7 +33,7 @@ def get_normalizer(normalizer: Union[transform_protocols, normalizer_literals]) return normalizer -def get_aggregator(aggregator: Union[TransformProtocol, aggregator_literals]) -> TransformProtocol: +def get_aggregator(aggregator: Union[TransformProtocol, AggregatorLiterals]) -> TransformProtocol: if isinstance(aggregator, str): try: return { diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index b39067010..13c6d22c0 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -5,7 +5,7 @@ from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict -from alibi_detect.od.base import TransformProtocol, transform_protocols +from alibi_detect.od.base import TransformProtocol, TransformProtocolType from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin from alibi_detect.od.pytorch import KNNTorch, Ensembler from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer @@ -27,7 +27,7 @@ def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], kernel: Optional[Callable] = None, - normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizer', + normalizer: Optional[Union[TransformProtocolType, normalizer_literals]] = 'ShiftAndScaleNormalizer', aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregator', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 46cea1cb4..c7f04d52f 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -38,4 +38,4 @@ def check_fitted(self): pass -transform_protocols = Union[TransformProtocol, FittedTransformProtocol] +TransformProtocolType = Union[TransformProtocol, FittedTransformProtocol] From f4a13d38b03cc6eb533b50cdaac3b42b9a8d147f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 16:10:21 +0000 Subject: [PATCH 132/247] Remove private methods from public __init__ --- alibi_detect/od/__init__.py | 34 ------------------------------ alibi_detect/od/_knn.py | 6 +++--- alibi_detect/od/base.py | 42 +++++++++++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index 4eb8ef104..63733615f 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -4,9 +4,6 @@ from .mahalanobis import Mahalanobis from .sr import SpectralResidual -from alibi_detect.od.base import TransformProtocol, TransformProtocolType -from typing_extensions import Literal -from typing import Union PValNormalizer, ShiftAndScaleNormalizer, TopKAggregator, AverageAggregator, \ MaxAggregator, MinAggregator = import_optional( @@ -16,37 +13,6 @@ ) -NormalizerLiterals = Literal['PValNormalizer', 'ShiftAndScaleNormalizer'] -AggregatorLiterals = Literal['TopKAggregator', 'AverageAggregator', - 'MaxAggregator', 'MinAggregator'] - - -def get_normalizer(normalizer: Union[TransformProtocolType, NormalizerLiterals]) -> TransformProtocol: - if isinstance(normalizer, str): - try: - return { - 'PValNormalizer': PValNormalizer, - 'ShiftAndScaleNormalizer': ShiftAndScaleNormalizer, - }.get(normalizer)() - except KeyError: - raise NotImplementedError(f'Normalizer {normalizer} not implemented.') - return normalizer - - -def get_aggregator(aggregator: Union[TransformProtocol, AggregatorLiterals]) -> TransformProtocol: - if isinstance(aggregator, str): - try: - return { - 'TopKAggregator': TopKAggregator, - 'AverageAggregator': AverageAggregator, - 'MaxAggregator': MaxAggregator, - 'MinAggregator': MinAggregator, - }.get(aggregator)() - except KeyError: - raise NotImplementedError(f'Aggregator {aggregator} not implemented.') - return aggregator - - OutlierAEGMM = import_optional('alibi_detect.od.aegmm', names=['OutlierAEGMM']) OutlierAE = import_optional('alibi_detect.od.ae', names=['OutlierAE']) OutlierVAE = import_optional('alibi_detect.od.vae', names=['OutlierVAE']) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 13c6d22c0..7ff5b1809 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -8,7 +8,7 @@ from alibi_detect.od.base import TransformProtocol, TransformProtocolType from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin from alibi_detect.od.pytorch import KNNTorch, Ensembler -from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer +from alibi_detect.od.base import get_aggregator, get_normalizer, NormalizerLiterals, AggregatorLiterals from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -27,8 +27,8 @@ def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], kernel: Optional[Callable] = None, - normalizer: Optional[Union[TransformProtocolType, normalizer_literals]] = 'ShiftAndScaleNormalizer', - aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregator', + normalizer: Optional[Union[TransformProtocolType, NormalizerLiterals]] = 'ShiftAndScaleNormalizer', + aggregator: Union[TransformProtocol, AggregatorLiterals] = 'AverageAggregator', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', ) -> None: diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index c7f04d52f..4c3979640 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -1,6 +1,7 @@ -from typing import Union +from alibi_detect.utils.missing_optional_dependency import import_optional -from typing_extensions import Protocol, runtime_checkable +from typing import Union +from typing_extensions import Literal, Protocol, runtime_checkable # Use Protocols instead of base classes for the backend associated objects. This is a bit more flexible and allows us to @@ -39,3 +40,40 @@ def check_fitted(self): TransformProtocolType = Union[TransformProtocol, FittedTransformProtocol] +NormalizerLiterals = Literal['PValNormalizer', 'ShiftAndScaleNormalizer'] +AggregatorLiterals = Literal['TopKAggregator', 'AverageAggregator', + 'MaxAggregator', 'MinAggregator'] + + +PValNormalizer, ShiftAndScaleNormalizer, TopKAggregator, AverageAggregator, \ + MaxAggregator, MinAggregator = import_optional( + 'alibi_detect.od.pytorch.ensemble', + ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', + 'AverageAggregator', 'MaxAggregator', 'MinAggregator'] + ) + + +def get_normalizer(normalizer: Union[TransformProtocolType, NormalizerLiterals]) -> TransformProtocol: + if isinstance(normalizer, str): + try: + return { + 'PValNormalizer': PValNormalizer, + 'ShiftAndScaleNormalizer': ShiftAndScaleNormalizer, + }.get(normalizer)() + except KeyError: + raise NotImplementedError(f'Normalizer {normalizer} not implemented.') + return normalizer + + +def get_aggregator(aggregator: Union[TransformProtocol, AggregatorLiterals]) -> TransformProtocol: + if isinstance(aggregator, str): + try: + return { + 'TopKAggregator': TopKAggregator, + 'AverageAggregator': AverageAggregator, + 'MaxAggregator': MaxAggregator, + 'MinAggregator': MinAggregator, + }.get(aggregator)() + except KeyError: + raise NotImplementedError(f'Aggregator {aggregator} not implemented.') + return aggregator From 186cde2516b4654eb69ba5e7795e1de52d540220 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 16:26:56 +0000 Subject: [PATCH 133/247] Fix missing comma bug --- alibi_detect/od/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index 63733615f..2d098c4fc 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -31,7 +31,7 @@ "OutlierSeq2Seq", "SpectralResidual", "LLR", - "OutlierProphet" + "OutlierProphet", "PValNormalizer", "ShiftAndScaleNormalizer", "TopKAggregator", From adb1f51fe0a24635f56de57524858a6758354ca8 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 16:44:00 +0000 Subject: [PATCH 134/247] Add metion about torch device in KNNTorch and KNN docstrings --- alibi_detect/od/_knn.py | 15 ++++++++------- alibi_detect/od/pytorch/knn.py | 7 ++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 7ff5b1809..cb614c5f3 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -29,11 +29,11 @@ def __init__( kernel: Optional[Callable] = None, normalizer: Optional[Union[TransformProtocolType, NormalizerLiterals]] = 'ShiftAndScaleNormalizer', aggregator: Union[TransformProtocol, AggregatorLiterals] = 'AverageAggregator', - device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """ - k-Nearest Neighbours (kNN) outlier detector. + k-Nearest Neighbors (kNN) outlier detector. The kNN detector is a non-parametric method for outlier detection. The detector computes the distance between each test point and its `k` nearest neighbors. The distance can be computed using a kernel function @@ -45,7 +45,7 @@ def __init__( In the latter case, an `ensembler` must be specified to aggregate the scores. Note that the `ensembler` is fit in the `infer_threshold` method and so if using an array of `k` values, the - `infer_threshold` method must be called before the `predict` method othewrise an exception is raised. If `k` + `infer_threshold` method must be called before the `predict` method otherwise an exception is raised. If `k` is a single value then the predict method can be called without first calling `infer_threshold` but only scores will be returned and not outlier predictions. @@ -53,7 +53,7 @@ def __init__( Parameters ---------- k - Number of neirest neighbors to compute distance to. `k` can be a single value or + Number of nearest neighbors to compute distance to. `k` can be a single value or an array of integers. If an array is passed, an aggregator is required to aggregate the scores. If `k` is a single value the outlier score is the distance/kernel similarity to the `k`-th nearest neighbor. If `k` is a list then it returns the @@ -63,7 +63,7 @@ def __init__( Otherwise if a kernel is specified then instead of using `torch.cdist` the kernel defines the k nearest neighbor distance. normalizer - Normalizer to use for outlier detection. If ``None``, no normalisation is applied. + Normalizer to use for outlier detection. If ``None``, no normalization is applied. For a list of available normalizers, see :mod:`alibi_detect.od.pytorch.ensemble`. aggregator Aggregator to use for outlier detection. Can be set to ``None`` if `k` is a single @@ -72,7 +72,8 @@ def __init__( Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of + ``torch.device``. Raises ------ @@ -142,7 +143,7 @@ def score(self, X: np.ndarray) -> np.ndarray: def infer_threshold(self, X: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. - The threshold is computed so that the outlier detector would incorectly classify `fpr` proportion of the + The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the reference data as outliers. Parameters diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index e0a58681d..c1213c9a8 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -20,7 +20,7 @@ def __init__( Parameters ---------- k - Number of neirest neighbors to compute distance to. `k` can be a single value or + Number of nearest neighbors to compute distance to. `k` can be a single value or an array of integers. If `k` is a single value the outlier score is the distance/kernel similarity to the `k`-th nearest neighbor. If `k` is a list then it returns the distance/kernel similarity to each of the specified `k` neighbors. @@ -32,8 +32,9 @@ def __init__( of :py:obj:`alibi_detect.od.pytorch.ensemble.ensembler`. Responsible for combining multiple scores into a single score. device - Device type used. The default None tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Device type used. The default tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of + ``torch.device``. """ TorchOutlierDetector.__init__(self, device=device) self.kernel = kernel From 789b9225664c768091fbba0db96c7ae8996a1bac Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 16:52:09 +0000 Subject: [PATCH 135/247] Make argument captialization consitent --- alibi_detect/od/_knn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index cb614c5f3..e59c1f971 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -72,7 +72,7 @@ def __init__( Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of + Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. Raises @@ -120,7 +120,7 @@ def fit(self, x_ref: np.ndarray) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) - def score(self, X: np.ndarray) -> np.ndarray: + def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. Computes the k nearest neighbor distance/kernel similarity for each instance in `x`. If `k` is a single @@ -137,10 +137,10 @@ def score(self, X: np.ndarray) -> np.ndarray: Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. """ - score = self.backend.score(self.backend._to_tensor(X)) + score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) - def infer_threshold(self, X: np.ndarray, fpr: float) -> None: + def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the @@ -155,7 +155,7 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: instances in `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range ``(0, 1)``. """ - self.backend.infer_threshold(self.backend._to_tensor(X), fpr) + self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. From bdaf4d4726117b06c3328701c4cb0884b05faf86 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 7 Mar 2023 17:02:19 +0000 Subject: [PATCH 136/247] Add missing raise statment --- alibi_detect/od/pytorch/base.py | 2 +- alibi_detect/od/pytorch/knn.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index b130f44ce..b6ac54c77 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -205,7 +205,7 @@ def infer_threshold(self, x: torch.Tensor, fpr: float) -> None: Raised if `fpr` is not in ``(0, 1)``. """ if not 0 < fpr < 1: - ValueError('`fpr` must be in `(0, 1)`.') + raise ValueError('`fpr` must be in `(0, 1)`.') self.val_scores = self.score(x) if self.ensemble: self.val_scores = self.ensembler.fit(self.val_scores).transform(self.val_scores) # type: ignore diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index c1213c9a8..a8c4241f4 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -10,7 +10,7 @@ class KNNTorch(TorchOutlierDetector): def __init__( self, - k: Union[np.ndarray, List, Tuple], + k: Union[np.ndarray, List, Tuple, int], kernel: Optional[torch.nn.Module] = None, ensembler: Optional[Ensembler] = None, device: Optional[Union[str, torch.device]] = None @@ -36,7 +36,7 @@ def __init__( Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. """ - TorchOutlierDetector.__init__(self, device=device) + super().__init__(device=device) self.kernel = kernel self.ensemble = isinstance(k, (np.ndarray, list, tuple)) self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) From d53906395681ed86f7e30d259aa3c42b2657bb50 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 10:04:25 +0000 Subject: [PATCH 137/247] Remove constructors from mixins --- alibi_detect/od/pytorch/ensemble.py | 58 +++++++++++------------------ 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 4c5890443..e404ba40b 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -9,14 +9,12 @@ class BaseTransformTorch(Module, ABC): - def __init__(self): - """Base Transform class. + """Base Transform class. - provides abstract methods for transform objects that map `torch` tensors. - """ - super().__init__() + Provides abstract methods for transform objects that map `torch` tensors. + """ - def transform(self, x: torch.Tensor): + def transform(self, x: torch.Tensor) -> torch.Tensor: """Public transform method. See `_transform` for implementation details. Parameters @@ -27,7 +25,7 @@ def transform(self, x: torch.Tensor): return self._transform(x) @abstractmethod - def _transform(self, x: torch.Tensor): + def _transform(self, x: torch.Tensor) -> torch.Tensor: """Applies class transform to `torch.Tensor` This method should be overridden on child classes. @@ -40,20 +38,18 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class FitMixinTorch(ABC): - fitted = False - - def __init__(self): - """Fit mixin + """Fit mixin - Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. - """ - super().__init__() + Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. + """ + fitted = False def fit(self, x: torch.Tensor) -> 'FitMixinTorch': """Public fit method. The `_fit` method contains the implementation details and should be overridden on child classes. Once the - `_fit` method has fit the object the `fitted` attribute should be set to `True`. + `_fit` method has fit the object the `fitted` attribute should be set to `True`. The `_fit` method should + return `self` to allow for chaining. Parameters ---------- @@ -65,12 +61,12 @@ def fit(self, x: torch.Tensor) -> 'FitMixinTorch': return self @abstractmethod - def _fit(self, x: torch.Tensor): + def _fit(self, x: torch.Tensor) -> 'FitMixinTorch': """Fit on `x` tensor. This method should be overridden on child classes and should set the `fitted` attribute to `True`. """ - pass + return self @torch.jit.unused def check_fitted(self): @@ -86,14 +82,11 @@ def check_fitted(self): class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): - def __init__(self): - """Base Fitted Transform class. + """Base Fitted Transform class. - Extends `BaseTransform` with fit functionality. Ensures that transform has been fit prior to - applying transform. - """ - BaseTransformTorch.__init__(self) - FitMixinTorch.__init__(self) + Extends `BaseTransform` with fit functionality. Ensures that transform has been fit prior to + applying transform. + """ def transform(self, x: torch.Tensor) -> torch.Tensor: """Checks to make sure transform has been fitted and then applies transform to input tensor. @@ -120,7 +113,6 @@ def __init__(self): Returns the proportion of scores in the reference dataset that are greater than the score of interest. Output is between ``1`` and ``0``. Small values are likely to be outliers. """ - super().__init__() self.val_scores = None def _fit(self, val_scores: torch.Tensor) -> 'PValNormalizer': @@ -159,7 +151,6 @@ def __init__(self): Needs to be fit (see :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch`). Subtracts the dataset mean and scales by the standard deviation. """ - super().__init__() self.val_means = None self.val_scales = None @@ -200,7 +191,6 @@ def __init__(self, k: Optional[int] = None): number of scores to take the mean of. If `k` is left ``None`` then will be set to half the number of scores passed in the forward call. """ - super().__init__() self.k = k def _transform(self, scores: torch.Tensor) -> torch.Tensor: @@ -236,7 +226,6 @@ def __init__(self, weights: Optional[torch.Tensor] = None): ValueError If `weights` does not sum to ``1``. """ - super().__init__() if weights is not None and weights.sum() != 1: raise ValueError("Weights must sum to 1.") self.weights = weights @@ -259,9 +248,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class MaxAggregator(BaseTransformTorch): - def __init__(self): - """Takes the maximum of the scores of the detectors in an ensemble.""" - super().__init__() + """Takes the maximum of the scores of the detectors in an ensemble.""" def _transform(self, scores: torch.Tensor) -> torch.Tensor: """Takes the maximum score of a set of detectors in an ensemble. @@ -280,9 +267,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class MinAggregator(BaseTransformTorch): - def __init__(self): - """Takes the minimum score of a set of detectors in an ensemble.""" - super().__init__() + """Takes the minimum score of a set of detectors in an ensemble.""" def _transform(self, scores: torch.Tensor) -> torch.Tensor: """Takes the minimum score of a set of detectors in an ensemble. @@ -304,17 +289,16 @@ class Ensembler(BaseFittedTransformTorch): def __init__(self, normalizer: Optional[BaseFittedTransformTorch] = None, aggregator: BaseTransformTorch = AverageAggregator()): - """An Ensembler applies normlization and aggregation operations to the scores of an ensemble of detectors. + """An Ensembler applies normalization and aggregation operations to the scores of an ensemble of detectors. Parameters ---------- normalizer - `BaseFittedTransformTorch` object to normalise the scores. If ``None`` then no normalisation + `BaseFittedTransformTorch` object to normalize the scores. If ``None`` then no normalization is applied. aggregator `BaseTransformTorch` object to aggregate the scores. """ - super().__init__() self.normalizer = normalizer if self.normalizer is None: self.fitted = True From 7633d8b330b0e4138051132c4a734483e365eb1b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 10:09:05 +0000 Subject: [PATCH 138/247] Change code formatting to be more readable --- alibi_detect/od/pytorch/ensemble.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index e404ba40b..b7f0e477d 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -138,9 +138,8 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: ------- `Torch.Tensor` of 1 - p-values. """ - p_vals = ( - 1 + (scores[:, None, :] < self.val_scores[None, :, :]).sum(1) - )/(len(self.val_scores)+1) + less_than_val_scores = scores[:, None, :] < self.val_scores[None, :, :] + p_vals = (1 + less_than_val_scores.sum(1))/(len(self.val_scores) + 1) return 1 - p_vals From fc713c38c638b47ba2cefca80a636b89a2baceff Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 13:11:28 +0000 Subject: [PATCH 139/247] Revert "Remove constructors from mixins" This reverts commit d53906395681ed86f7e30d259aa3c42b2657bb50. --- alibi_detect/od/pytorch/ensemble.py | 58 ++++++++++++++++++----------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index b7f0e477d..4dbb16c4d 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -9,12 +9,14 @@ class BaseTransformTorch(Module, ABC): - """Base Transform class. + def __init__(self): + """Base Transform class. - Provides abstract methods for transform objects that map `torch` tensors. - """ + provides abstract methods for transform objects that map `torch` tensors. + """ + super().__init__() - def transform(self, x: torch.Tensor) -> torch.Tensor: + def transform(self, x: torch.Tensor): """Public transform method. See `_transform` for implementation details. Parameters @@ -25,7 +27,7 @@ def transform(self, x: torch.Tensor) -> torch.Tensor: return self._transform(x) @abstractmethod - def _transform(self, x: torch.Tensor) -> torch.Tensor: + def _transform(self, x: torch.Tensor): """Applies class transform to `torch.Tensor` This method should be overridden on child classes. @@ -38,18 +40,20 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class FitMixinTorch(ABC): - """Fit mixin - - Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. - """ fitted = False + def __init__(self): + """Fit mixin + + Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. + """ + super().__init__() + def fit(self, x: torch.Tensor) -> 'FitMixinTorch': """Public fit method. The `_fit` method contains the implementation details and should be overridden on child classes. Once the - `_fit` method has fit the object the `fitted` attribute should be set to `True`. The `_fit` method should - return `self` to allow for chaining. + `_fit` method has fit the object the `fitted` attribute should be set to `True`. Parameters ---------- @@ -61,12 +65,12 @@ def fit(self, x: torch.Tensor) -> 'FitMixinTorch': return self @abstractmethod - def _fit(self, x: torch.Tensor) -> 'FitMixinTorch': + def _fit(self, x: torch.Tensor): """Fit on `x` tensor. This method should be overridden on child classes and should set the `fitted` attribute to `True`. """ - return self + pass @torch.jit.unused def check_fitted(self): @@ -82,11 +86,14 @@ def check_fitted(self): class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): - """Base Fitted Transform class. + def __init__(self): + """Base Fitted Transform class. - Extends `BaseTransform` with fit functionality. Ensures that transform has been fit prior to - applying transform. - """ + Extends `BaseTransform` with fit functionality. Ensures that transform has been fit prior to + applying transform. + """ + BaseTransformTorch.__init__(self) + FitMixinTorch.__init__(self) def transform(self, x: torch.Tensor) -> torch.Tensor: """Checks to make sure transform has been fitted and then applies transform to input tensor. @@ -113,6 +120,7 @@ def __init__(self): Returns the proportion of scores in the reference dataset that are greater than the score of interest. Output is between ``1`` and ``0``. Small values are likely to be outliers. """ + super().__init__() self.val_scores = None def _fit(self, val_scores: torch.Tensor) -> 'PValNormalizer': @@ -150,6 +158,7 @@ def __init__(self): Needs to be fit (see :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch`). Subtracts the dataset mean and scales by the standard deviation. """ + super().__init__() self.val_means = None self.val_scales = None @@ -190,6 +199,7 @@ def __init__(self, k: Optional[int] = None): number of scores to take the mean of. If `k` is left ``None`` then will be set to half the number of scores passed in the forward call. """ + super().__init__() self.k = k def _transform(self, scores: torch.Tensor) -> torch.Tensor: @@ -225,6 +235,7 @@ def __init__(self, weights: Optional[torch.Tensor] = None): ValueError If `weights` does not sum to ``1``. """ + super().__init__() if weights is not None and weights.sum() != 1: raise ValueError("Weights must sum to 1.") self.weights = weights @@ -247,7 +258,9 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class MaxAggregator(BaseTransformTorch): - """Takes the maximum of the scores of the detectors in an ensemble.""" + def __init__(self): + """Takes the maximum of the scores of the detectors in an ensemble.""" + super().__init__() def _transform(self, scores: torch.Tensor) -> torch.Tensor: """Takes the maximum score of a set of detectors in an ensemble. @@ -266,7 +279,9 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: class MinAggregator(BaseTransformTorch): - """Takes the minimum score of a set of detectors in an ensemble.""" + def __init__(self): + """Takes the minimum score of a set of detectors in an ensemble.""" + super().__init__() def _transform(self, scores: torch.Tensor) -> torch.Tensor: """Takes the minimum score of a set of detectors in an ensemble. @@ -288,16 +303,17 @@ class Ensembler(BaseFittedTransformTorch): def __init__(self, normalizer: Optional[BaseFittedTransformTorch] = None, aggregator: BaseTransformTorch = AverageAggregator()): - """An Ensembler applies normalization and aggregation operations to the scores of an ensemble of detectors. + """An Ensembler applies normlization and aggregation operations to the scores of an ensemble of detectors. Parameters ---------- normalizer - `BaseFittedTransformTorch` object to normalize the scores. If ``None`` then no normalization + `BaseFittedTransformTorch` object to normalise the scores. If ``None`` then no normalisation is applied. aggregator `BaseTransformTorch` object to aggregate the scores. """ + super().__init__() self.normalizer = normalizer if self.normalizer is None: self.fitted = True From 1e3d4d8c337564c03f56abfc538666f488f29eab Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 15:36:20 +0000 Subject: [PATCH 140/247] Remove constructors from FitMixinTorch --- alibi_detect/od/pytorch/ensemble.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 4dbb16c4d..b832b9982 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -42,13 +42,6 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class FitMixinTorch(ABC): fitted = False - def __init__(self): - """Fit mixin - - Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. - """ - super().__init__() - def fit(self, x: torch.Tensor) -> 'FitMixinTorch': """Public fit method. @@ -92,8 +85,7 @@ def __init__(self): Extends `BaseTransform` with fit functionality. Ensures that transform has been fit prior to applying transform. """ - BaseTransformTorch.__init__(self) - FitMixinTorch.__init__(self) + super().__init__() def transform(self, x: torch.Tensor) -> torch.Tensor: """Checks to make sure transform has been fitted and then applies transform to input tensor. From ea20b21583a8dd01cdf78390117e4e057995f4a9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 17:59:14 +0000 Subject: [PATCH 141/247] Remove private method pattern in ensemble mixins --- alibi_detect/od/pytorch/base.py | 11 --- alibi_detect/od/pytorch/ensemble.py | 106 +++++++++++----------------- alibi_detect/od/pytorch/knn.py | 6 +- 3 files changed, 43 insertions(+), 80 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index b6ac54c77..658a75498 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -64,17 +64,6 @@ def __init__(self, device: Optional[Union[str, torch.device]] = None): self.device = get_device(device) super().__init__() - @abstractmethod - def _fit(self, x_ref: torch.Tensor) -> None: - """Fit the outlier detector to the reference data. - - Parameters - ---------- - x_ref - Reference data. - """ - pass - @abstractmethod def score(self, x: torch.Tensor) -> torch.Tensor: """Score the data. diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index b832b9982..d18e236ef 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +from abc import ABC from typing import Optional import torch @@ -8,7 +8,7 @@ from alibi_detect.exceptions import NotFittedError -class BaseTransformTorch(Module, ABC): +class BaseTransformTorch(Module): def __init__(self): """Base Transform class. @@ -17,26 +17,18 @@ def __init__(self): super().__init__() def transform(self, x: torch.Tensor): - """Public transform method. See `_transform` for implementation details. + """Public transform method. Parameters ---------- x `torch.Tensor` array to be transformed """ - return self._transform(x) - - @abstractmethod - def _transform(self, x: torch.Tensor): - """Applies class transform to `torch.Tensor` - - This method should be overridden on child classes. - """ - pass + raise NotImplementedError() @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.transform(x=x) + return self.transform(x) class FitMixinTorch(ABC): @@ -53,58 +45,35 @@ def fit(self, x: torch.Tensor) -> 'FitMixinTorch': x `torch.Tensor` to fit object on. """ - self.fitted = True - self._fit(x) - return self + raise NotImplementedError() - @abstractmethod - def _fit(self, x: torch.Tensor): - """Fit on `x` tensor. + def set_fitted(self): + """Sets the fitted attribute to True. - This method should be overridden on child classes and should set the `fitted` attribute to `True`. + Should be called within each transform method. """ - pass + self.fitted = True + return self - @torch.jit.unused def check_fitted(self): - """Raises error if parent object instance has not been fit. + """Checks to make sure object has been fitted. Raises ------ NotFittedError Raised if method called and object has not been fit. """ + if not torch.jit.is_scripting(): + self._check_fitted() + + @torch.jit.unused + def _check_fitted(self): + """Raises error if parent object instance has not been fit.""" if not self.fitted: raise NotFittedError(f'{self.__class__.__name__} has not been fit!') -class BaseFittedTransformTorch(BaseTransformTorch, FitMixinTorch): - def __init__(self): - """Base Fitted Transform class. - - Extends `BaseTransform` with fit functionality. Ensures that transform has been fit prior to - applying transform. - """ - super().__init__() - - def transform(self, x: torch.Tensor) -> torch.Tensor: - """Checks to make sure transform has been fitted and then applies transform to input tensor. - - Parameters - ---------- - x - `torch.Tensor` being transformed. - - Returns - ------- - the transformed `torch.Tensor`. - """ - if not torch.jit.is_scripting(): - self.check_fitted() - return self._transform(x) - - -class PValNormalizer(BaseFittedTransformTorch): +class PValNormalizer(BaseTransformTorch, FitMixinTorch): def __init__(self): """Maps scores to there p-values. @@ -115,7 +84,7 @@ def __init__(self): super().__init__() self.val_scores = None - def _fit(self, val_scores: torch.Tensor) -> 'PValNormalizer': + def fit(self, val_scores: torch.Tensor) -> 'PValNormalizer': """Fit transform on scores. Parameters @@ -124,9 +93,9 @@ def _fit(self, val_scores: torch.Tensor) -> 'PValNormalizer': score outputs of ensemble of detectors applied to reference data. """ self.val_scores = val_scores - return self + return self.set_fitted() - def _transform(self, scores: torch.Tensor) -> torch.Tensor: + def transform(self, scores: torch.Tensor) -> torch.Tensor: """Transform scores to 1 - p-values. Parameters @@ -138,12 +107,13 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: ------- `Torch.Tensor` of 1 - p-values. """ + self.check_fitted() less_than_val_scores = scores[:, None, :] < self.val_scores[None, :, :] p_vals = (1 + less_than_val_scores.sum(1))/(len(self.val_scores) + 1) return 1 - p_vals -class ShiftAndScaleNormalizer(BaseFittedTransformTorch): +class ShiftAndScaleNormalizer(BaseTransformTorch, FitMixinTorch): def __init__(self): """Maps scores to their normalized values. @@ -154,7 +124,7 @@ def __init__(self): self.val_means = None self.val_scales = None - def _fit(self, val_scores: torch.Tensor) -> 'ShiftAndScaleNormalizer': + def fit(self, val_scores: torch.Tensor) -> 'ShiftAndScaleNormalizer': """Computes the mean and standard deviation of the scores and stores them. Parameters @@ -164,9 +134,9 @@ def _fit(self, val_scores: torch.Tensor) -> 'ShiftAndScaleNormalizer': """ self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] - return self + return self.set_fitted() - def _transform(self, scores: torch.Tensor) -> torch.Tensor: + def transform(self, scores: torch.Tensor) -> torch.Tensor: """Transform scores to normalized values. Subtracts the mean and scales by the standard deviation. Parameters @@ -178,6 +148,7 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: ------- `Torch.Tensor` of normalized scores. """ + self.check_fitted() return (scores - self.val_means)/self.val_scales @@ -194,7 +165,7 @@ def __init__(self, k: Optional[int] = None): super().__init__() self.k = k - def _transform(self, scores: torch.Tensor) -> torch.Tensor: + def transform(self, scores: torch.Tensor) -> torch.Tensor: """Takes the mean of the top `k` scores. Parameters @@ -232,9 +203,11 @@ def __init__(self, weights: Optional[torch.Tensor] = None): raise ValueError("Weights must sum to 1.") self.weights = weights - def _transform(self, scores: torch.Tensor) -> torch.Tensor: + def transform(self, scores: torch.Tensor) -> torch.Tensor: """Averages the scores of the detectors in an ensemble. If weights were passed in the `__init__` then these are used to weight the scores. + + Parameters ---------- scores `Torch.Tensor` of scores from ensemble of detectors. @@ -254,7 +227,7 @@ def __init__(self): """Takes the maximum of the scores of the detectors in an ensemble.""" super().__init__() - def _transform(self, scores: torch.Tensor) -> torch.Tensor: + def transform(self, scores: torch.Tensor) -> torch.Tensor: """Takes the maximum score of a set of detectors in an ensemble. Parameters @@ -275,7 +248,7 @@ def __init__(self): """Takes the minimum score of a set of detectors in an ensemble.""" super().__init__() - def _transform(self, scores: torch.Tensor) -> torch.Tensor: + def transform(self, scores: torch.Tensor) -> torch.Tensor: """Takes the minimum score of a set of detectors in an ensemble. Parameters @@ -291,9 +264,9 @@ def _transform(self, scores: torch.Tensor) -> torch.Tensor: return vals -class Ensembler(BaseFittedTransformTorch): +class Ensembler(BaseTransformTorch, FitMixinTorch): def __init__(self, - normalizer: Optional[BaseFittedTransformTorch] = None, + normalizer: Optional[BaseTransformTorch] = None, aggregator: BaseTransformTorch = AverageAggregator()): """An Ensembler applies normlization and aggregation operations to the scores of an ensemble of detectors. @@ -311,7 +284,7 @@ def __init__(self, self.fitted = True self.aggregator = aggregator - def _transform(self, x: torch.Tensor) -> torch.Tensor: + def transform(self, x: torch.Tensor) -> torch.Tensor: """Apply the normalizer and aggregator to the scores. Parameters @@ -328,7 +301,7 @@ def _transform(self, x: torch.Tensor) -> torch.Tensor: x = self.aggregator(x) return x - def _fit(self, x: torch.Tensor): + def fit(self, x: torch.Tensor): """Fit the normalizer to the scores. Parameters @@ -337,4 +310,5 @@ def _fit(self, x: torch.Tensor): `Torch.Tensor` of scores from ensemble of detectors. """ if self.normalizer is not None: - self.normalizer.fit(x) + self.normalizer.fit(x) # type: ignore + return self.set_fitted() diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index a8c4241f4..e90437ff0 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -85,14 +85,13 @@ def score(self, x: torch.Tensor) -> torch.Tensor: NotFittedError If called before detector has been fit. """ - if not torch.jit.is_scripting(): - self.check_fitted() + self.check_fitted() K = -self.kernel(x, self.x_ref) if self.kernel is not None else torch.cdist(x, self.x_ref) bot_k_dists = torch.topk(K, int(torch.max(self.ks)), dim=1, largest=False) all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] - def _fit(self, x_ref: torch.Tensor): + def fit(self, x_ref: torch.Tensor): """Fits the detector Parameters @@ -101,3 +100,4 @@ def _fit(self, x_ref: torch.Tensor): The Dataset tensor. """ self.x_ref = x_ref + return self.set_fitted() From b1302dd3349860c19b6b2ea917747bf6afdbf52d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 18:14:11 +0000 Subject: [PATCH 142/247] Fix spelling mistakes --- alibi_detect/od/base.py | 5 +---- alibi_detect/od/pytorch/ensemble.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 4c3979640..79f55b64b 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -17,9 +17,6 @@ class TransformProtocol(Protocol): def transform(self, x): pass - def _transform(self, x): - pass - @runtime_checkable class FittedTransformProtocol(TransformProtocol, Protocol): @@ -32,7 +29,7 @@ class FittedTransformProtocol(TransformProtocol, Protocol): def fit(self, x_ref): pass - def _fit(self, x_ref): + def set_fitted(self): pass def check_fitted(self): diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index d18e236ef..9c347f285 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -268,12 +268,12 @@ class Ensembler(BaseTransformTorch, FitMixinTorch): def __init__(self, normalizer: Optional[BaseTransformTorch] = None, aggregator: BaseTransformTorch = AverageAggregator()): - """An Ensembler applies normlization and aggregation operations to the scores of an ensemble of detectors. + """An Ensembler applies normalization and aggregation operations to the scores of an ensemble of detectors. Parameters ---------- normalizer - `BaseFittedTransformTorch` object to normalise the scores. If ``None`` then no normalisation + `BaseFittedTransformTorch` object to normalize the scores. If ``None`` then no normalization is applied. aggregator `BaseTransformTorch` object to aggregate the scores. From 09cf9f760b6703f1bd850dca0d9e9aeedd2143d4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 18:24:36 +0000 Subject: [PATCH 143/247] Add np.isclose for sum to one check --- alibi_detect/od/pytorch/ensemble.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 9c347f285..70ada9152 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -199,7 +199,7 @@ def __init__(self, weights: Optional[torch.Tensor] = None): If `weights` does not sum to ``1``. """ super().__init__() - if weights is not None and weights.sum() != 1: + if weights is not None and not np.isclose(weights.sum(), 1): raise ValueError("Weights must sum to 1.") self.weights = weights From 37025edbca8128b68a1c792b3ca8dbcb87d74748 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 8 Mar 2023 18:31:59 +0000 Subject: [PATCH 144/247] Fix mutable default issue --- alibi_detect/od/pytorch/ensemble.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 70ada9152..d4b4cb251 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -267,7 +267,7 @@ def transform(self, scores: torch.Tensor) -> torch.Tensor: class Ensembler(BaseTransformTorch, FitMixinTorch): def __init__(self, normalizer: Optional[BaseTransformTorch] = None, - aggregator: BaseTransformTorch = AverageAggregator()): + aggregator: BaseTransformTorch = None): """An Ensembler applies normalization and aggregation operations to the scores of an ensemble of detectors. Parameters @@ -276,12 +276,14 @@ def __init__(self, `BaseFittedTransformTorch` object to normalize the scores. If ``None`` then no normalization is applied. aggregator - `BaseTransformTorch` object to aggregate the scores. + `BaseTransformTorch` object to aggregate the scores. If ``None`` defaults to `AverageAggregator`. """ super().__init__() self.normalizer = normalizer if self.normalizer is None: self.fitted = True + if aggregator is None: + aggregator = AverageAggregator() self.aggregator = aggregator def transform(self, x: torch.Tensor) -> torch.Tensor: From b48b0a0973df4d0a5197017223233583b941e6d9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 9 Mar 2023 10:26:44 +0000 Subject: [PATCH 145/247] Expose _knn docstrings in the experimental namespace --- alibi_detect/experimental/od/__init__.py | 18 ++++++++++ alibi_detect/od/__init__.py | 14 -------- alibi_detect/od/_knn.py | 35 +++++++++++++++++--- alibi_detect/od/pytorch/base.py | 2 +- alibi_detect/od/tests/test__knn/test__knn.py | 11 ++++-- 5 files changed, 58 insertions(+), 22 deletions(-) diff --git a/alibi_detect/experimental/od/__init__.py b/alibi_detect/experimental/od/__init__.py index 6cddab30d..c8e035f42 100644 --- a/alibi_detect/experimental/od/__init__.py +++ b/alibi_detect/experimental/od/__init__.py @@ -1 +1,19 @@ +from alibi_detect.utils.missing_optional_dependency import import_optional from alibi_detect.od._knn import KNN # noqa F401 + +PValNormalizer, ShiftAndScaleNormalizer, TopKAggregator, AverageAggregator, \ + MaxAggregator, MinAggregator = import_optional( + 'alibi_detect.od.pytorch.ensemble', + ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', + 'AverageAggregator', 'MaxAggregator', 'MinAggregator'] + ) + +__all__ = [ + 'KNN', + 'PValNormalizer', + 'ShiftAndScaleNormalizer', + 'TopKAggregator', + 'AverageAggregator', + 'MaxAggregator', + 'MinAggregator' +] diff --git a/alibi_detect/od/__init__.py b/alibi_detect/od/__init__.py index 2d098c4fc..e89e70b47 100644 --- a/alibi_detect/od/__init__.py +++ b/alibi_detect/od/__init__.py @@ -5,14 +5,6 @@ from .sr import SpectralResidual -PValNormalizer, ShiftAndScaleNormalizer, TopKAggregator, AverageAggregator, \ - MaxAggregator, MinAggregator = import_optional( - 'alibi_detect.od.pytorch.ensemble', - ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', - 'AverageAggregator', 'MaxAggregator', 'MinAggregator'] - ) - - OutlierAEGMM = import_optional('alibi_detect.od.aegmm', names=['OutlierAEGMM']) OutlierAE = import_optional('alibi_detect.od.ae', names=['OutlierAE']) OutlierVAE = import_optional('alibi_detect.od.vae', names=['OutlierVAE']) @@ -32,10 +24,4 @@ "SpectralResidual", "LLR", "OutlierProphet", - "PValNormalizer", - "ShiftAndScaleNormalizer", - "TopKAggregator", - "AverageAggregator", - "MaxAggregator", - "MinAggregator", ] diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index e59c1f971..ef53bc287 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -42,12 +42,12 @@ def __init__( The detector can be initialized with `k` a single value or an array of values. If `k` is a single value then the outlier score is the distance/kernel similarity to the k-th nearest neighbor. If `k` is an array of values then the outlier score is the distance/kernel similarity to each of the specified `k` neighbors. - In the latter case, an `ensembler` must be specified to aggregate the scores. + In the latter case, an `aggregator` must be specified to aggregate the scores. - Note that the `ensembler` is fit in the `infer_threshold` method and so if using an array of `k` values, the - `infer_threshold` method must be called before the `predict` method otherwise an exception is raised. If `k` - is a single value then the predict method can be called without first calling `infer_threshold` but only - scores will be returned and not outlier predictions. + Note that, in the multiple k case, a normalizer can be provided. If a normalizer is passed then it is fit in + the `infer_threshold` method and so this method must be called before the `predict` method. If this is not + done an exception is raised. If `k` is a single value then the predict method can be called without first + calling `infer_threshold` but only scores will be returned and not outlier predictions. Parameters @@ -132,12 +132,20 @@ def score(self, x: np.ndarray) -> np.ndarray: x Data to score. The shape of `x` should be `(n_instances, n_features)`. + Raises + ------ + NotFittedError + If called before detector has been fit. + ThresholdNotInferredError + If k is a list and a threshold was not inferred. + Returns ------- Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. """ score = self.backend.score(self.backend._to_tensor(x)) + score = self.backend._ensembler(score) return self.backend._to_numpy(score) def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: @@ -146,6 +154,16 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the reference data as outliers. + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + + Raises + ------ + NotFittedError + If called before detector has been fit. + Parameters ---------- x_ref @@ -167,6 +185,13 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: x Data to predict. The shape of `x` should be `(n_instances, n_features)`. + Raises + ------ + NotFittedError + If called before detector has been fit. + ThresholdNotInferredError + If k is a list and a threshold was not inferred. + Returns ------- Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 658a75498..fdd954ea2 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -133,7 +133,7 @@ def _ensembler(self, x: torch.Tensor) -> torch.Tensor: Raises ------ ThresholdNotInferredError - If the detector is an ensemble, and the ensembler used to aggregate the outlier scores has a fitable + If the detector is an ensemble, and the ensembler used to aggregate the outlier scores has a fittable component, then the detector threshold must be inferred before predictions can be made. This is because while the scoring functionality of the detector is fit within the `.fit` method on the training data the ensembler has to be fit on the validation data along with the threshold and this is done in the diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index 0f8da69d2..a825d841c 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -3,7 +3,7 @@ import torch from alibi_detect.od._knn import KNN -from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ +from alibi_detect.od.pytorch.ensemble import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError @@ -54,7 +54,7 @@ def test_fitted_knn_score(): def test_fitted_knn_ensemble_score(): """ Test fitted but not threshold inferred ensemble detectors correctly raise an error when calling - the predict method. This is becuase the ensembler is fit in the infer_threshold method. + the predict method. This is because the ensembler is fit in the infer_threshold method. """ knn_detector = KNN(k=[10, 14, 18]) x_ref = np.random.randn(100, 2) @@ -63,6 +63,9 @@ def test_fitted_knn_ensemble_score(): with pytest.raises(ThresholdNotInferredError): knn_detector.predict(x) + with pytest.raises(ThresholdNotInferredError): + knn_detector.score(x) + def test_incorrect_knn_ensemble_init(): # test knn ensemble with aggregator passed as None raises exception @@ -155,6 +158,10 @@ def test_fitted_knn_ensemble_predict(aggregator, normalizer): assert y['p_value'].all() assert (y['is_outlier'] == [True, False]).all() + # test fitted detectors with inferred thresholds can score data using the score method. + scores = knn_detector.score(x) + assert np.all(y['instance_score'] == scores) + @pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), MaxAggregator, MinAggregator]) From 8e423a0ef3383b06f6249bd11fa39ad7f1d290d0 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 9 Mar 2023 10:36:12 +0000 Subject: [PATCH 146/247] Fix minor spelling mistake --- alibi_detect/od/pytorch/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index fdd954ea2..43e0f23f2 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -140,7 +140,7 @@ def _ensembler(self, x: torch.Tensor) -> torch.Tensor: `.infer_threshold` method. """ if hasattr(self, 'ensembler') and self.ensembler is not None: - # `type: ignore` here becuase self.ensembler here causes an error with mypy when using torch.jit.script. + # `type: ignore` here because self.ensembler here causes an error with mypy when using torch.jit.script. # For some reason it thinks self.ensembler is a torch.Tensor and therefore is not callable. if not torch.jit.is_scripting(): if not self.ensembler.fitted: # type: ignore From e0deeaa2b96e44fa14a96be10428b6a65fc72d56 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Mar 2023 11:38:22 +0000 Subject: [PATCH 147/247] Add self return types --- alibi_detect/exceptions.py | 8 ++++---- alibi_detect/od/base.py | 7 +++---- alibi_detect/od/pytorch/base.py | 8 +++++--- alibi_detect/od/pytorch/ensemble.py | 9 +++++---- alibi_detect/od/pytorch/knn.py | 3 ++- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/alibi_detect/exceptions.py b/alibi_detect/exceptions.py index 37a6085c5..8144b750a 100644 --- a/alibi_detect/exceptions.py +++ b/alibi_detect/exceptions.py @@ -1,9 +1,9 @@ -"""This module defines the Alibi exception hierarchy and common exceptions used across the library.""" +"""This module defines the Alibi Detect exception hierarchy and common exceptions used across the library.""" from abc import ABC -class AlibiDetectError(Exception, ABC): +class AlibiDetectException(Exception, ABC): def __init__(self, message: str) -> None: """Abstract base class of all alibi detect errors. @@ -15,13 +15,13 @@ def __init__(self, message: str) -> None: super().__init__(message) -class NotFittedError(AlibiDetectError): +class NotFittedError(AlibiDetectException): """Exception raised when a transform is not fitted.""" pass -class ThresholdNotInferredError(AlibiDetectError): +class ThresholdNotInferredError(AlibiDetectException): """Exception raised when a threshold not inferred for an outlier detector.""" pass diff --git a/alibi_detect/od/base.py b/alibi_detect/od/base.py index 79f55b64b..50c95e0fe 100644 --- a/alibi_detect/od/base.py +++ b/alibi_detect/od/base.py @@ -22,10 +22,9 @@ def transform(self, x): class FittedTransformProtocol(TransformProtocol, Protocol): """Protocol for fitted transformer objects. - The :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseFittedTransformTorch` object provides abstract methods for - objects that map between `torch` tensors and also require to be fit. This protocol models the interface of - the `BaseFittedTransformTorch` - class.""" + This protocol models the joint interface of the :py:obj:`~alibi_detect.od.pytorch.ensemble.BaseTransformTorch` + class and the :py:obj:`~alibi_detect.od.pytorch.ensemble.FitMixinTorch` class. These objects are transforms that + require to be fit.""" def fit(self, x_ref): pass diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 43e0f23f2..24c4d18da 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -1,4 +1,5 @@ from typing import List, Union, Optional, Dict +from typing_extensions import Self from dataclasses import dataclass, asdict from abc import ABC, abstractmethod @@ -178,12 +179,12 @@ def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ if self.threshold_inferred else None - def infer_threshold(self, x: torch.Tensor, fpr: float) -> None: + def infer_threshold(self, x_ref: torch.Tensor, fpr: float) -> Self: """Infer the threshold for the data. Prerequisite for outlier predictions. Parameters ---------- - x + x_ref Data to infer the threshold for. fpr False positive rate to use for threshold inference. @@ -195,11 +196,12 @@ def infer_threshold(self, x: torch.Tensor, fpr: float) -> None: """ if not 0 < fpr < 1: raise ValueError('`fpr` must be in `(0, 1)`.') - self.val_scores = self.score(x) + self.val_scores = self.score(x_ref) if self.ensemble: self.val_scores = self.ensembler.fit(self.val_scores).transform(self.val_scores) # type: ignore self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True + return self def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: """Predict outlier labels for the data. diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index d4b4cb251..8f9747273 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -1,5 +1,6 @@ from abc import ABC from typing import Optional +from typing_extensions import Self import torch import numpy as np @@ -47,7 +48,7 @@ def fit(self, x: torch.Tensor) -> 'FitMixinTorch': """ raise NotImplementedError() - def set_fitted(self): + def set_fitted(self) -> Self: """Sets the fitted attribute to True. Should be called within each transform method. @@ -84,7 +85,7 @@ def __init__(self): super().__init__() self.val_scores = None - def fit(self, val_scores: torch.Tensor) -> 'PValNormalizer': + def fit(self, val_scores: torch.Tensor) -> Self: """Fit transform on scores. Parameters @@ -124,7 +125,7 @@ def __init__(self): self.val_means = None self.val_scales = None - def fit(self, val_scores: torch.Tensor) -> 'ShiftAndScaleNormalizer': + def fit(self, val_scores: torch.Tensor) -> Self: """Computes the mean and standard deviation of the scores and stores them. Parameters @@ -303,7 +304,7 @@ def transform(self, x: torch.Tensor) -> torch.Tensor: x = self.aggregator(x) return x - def fit(self, x: torch.Tensor): + def fit(self, x: torch.Tensor) -> Self: """Fit the normalizer to the scores. Parameters diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index e90437ff0..2753c37d3 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -1,4 +1,5 @@ from typing import Optional, Union, List, Tuple +from typing_extensions import Self import numpy as np import torch @@ -91,7 +92,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] - def fit(self, x_ref: torch.Tensor): + def fit(self, x_ref: torch.Tensor) -> Self: """Fits the detector Parameters From 0a4046a788085df2df5da901c39a7e681dc46f86 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Mar 2023 13:44:11 +0000 Subject: [PATCH 148/247] Make fit an abstract method on FitMixin --- alibi_detect/od/pytorch/ensemble.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 8f9747273..ef23fa7a5 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -1,4 +1,4 @@ -from abc import ABC +from abc import ABC, abstractmethod from typing import Optional from typing_extensions import Self @@ -35,18 +35,16 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class FitMixinTorch(ABC): fitted = False - def fit(self, x: torch.Tensor) -> 'FitMixinTorch': - """Public fit method. - - The `_fit` method contains the implementation details and should be overridden on child classes. Once the - `_fit` method has fit the object the `fitted` attribute should be set to `True`. + @abstractmethod + def fit(self, x: torch.Tensor) -> Self: + """Abstract fit method. Parameters ---------- x `torch.Tensor` to fit object on. """ - raise NotImplementedError() + pass def set_fitted(self) -> Self: """Sets the fitted attribute to True. From c4c70045038fce651d8ccd8aa0ec205d72e0ab0a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Mar 2023 15:13:21 +0000 Subject: [PATCH 149/247] Add tests to check correct errors raised in KNNTorch backend --- alibi_detect/experimental/__init__.py | 0 alibi_detect/experimental/od/__init__.py | 19 ---- .../od/tests/test__knn/test__knn_backend.py | 97 +++++++++++++------ 3 files changed, 65 insertions(+), 51 deletions(-) delete mode 100644 alibi_detect/experimental/__init__.py delete mode 100644 alibi_detect/experimental/od/__init__.py diff --git a/alibi_detect/experimental/__init__.py b/alibi_detect/experimental/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/alibi_detect/experimental/od/__init__.py b/alibi_detect/experimental/od/__init__.py deleted file mode 100644 index c8e035f42..000000000 --- a/alibi_detect/experimental/od/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from alibi_detect.utils.missing_optional_dependency import import_optional -from alibi_detect.od._knn import KNN # noqa F401 - -PValNormalizer, ShiftAndScaleNormalizer, TopKAggregator, AverageAggregator, \ - MaxAggregator, MinAggregator = import_optional( - 'alibi_detect.od.pytorch.ensemble', - ['PValNormalizer', 'ShiftAndScaleNormalizer', 'TopKAggregator', - 'AverageAggregator', 'MaxAggregator', 'MinAggregator'] - ) - -__all__ = [ - 'KNN', - 'PValNormalizer', - 'ShiftAndScaleNormalizer', - 'TopKAggregator', - 'AverageAggregator', - 'MaxAggregator', - 'MinAggregator' -] diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 49ffe8838..6f82bcaed 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -4,7 +4,7 @@ from alibi_detect.od.pytorch.knn import KNNTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator -from alibi_detect.exceptions import NotFittedError +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError @pytest.fixture(scope='session') @@ -143,34 +143,67 @@ def test_knn_kernel_ts(ensembler): assert torch.all(pred_1 == pred_2) -# @pytest.mark.parametrize('k', [[4, 5], 4]) -# def test_knn_torch_backend_ensemble_fit_errors(k, ensembler): -# knn_torch = KNNTorch(k=k, ensembler=ensembler) -# assert not knn_torch.fitted - -# # Test that the backend raises an error if it is not fitted before -# # calling forward method. -# x = torch.randn((1, 10)) -# with pytest.raises(NotFittedError) as err: -# knn_torch(x) -# assert str(err.value) == 'KNNTorch has not been fit!' - -# # Test that the backend raises an error if it is not fitted before -# # predicting. -# with pytest.raises(NotFittedError) as err: -# knn_torch.predict(x) -# assert str(err.value) == 'KNNTorch has not been fit!' - -# # Test the backend updates fitted flag on fit. -# x_ref = torch.randn((1024, 10)) -# knn_torch.fit(x_ref) -# assert knn_torch.fitted - -# # Test that the backend raises an if the forward method is called without the -# # threshold being inferred. -# with pytest.raises(ThresholdNotInferredError) as err: -# knn_torch(x) -# assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' - -# # Test that the backend can call predict without the threshold being inferred. -# assert knn_torch.predict(x) +def test_knn_torch_backend_ensemble_fit_errors(ensembler): + """Tests the correct errors are raised when using the KNNTorch backend as an ensemble.""" + knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) + + # Test that the backend raises an error if it is not fitted before + # calling forward method. + x = torch.randn((1, 10)) + with pytest.raises(NotFittedError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + # Test that the backend raises an error if it is not fitted before + # predicting. + with pytest.raises(NotFittedError) as err: + knn_torch.predict(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + # Test the backend updates fitted flag on fit. + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + assert knn_torch.fitted + + # Test that the backend raises an if the forward method is called without the + # threshold being inferred. + with pytest.raises(ThresholdNotInferredError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + + # Test that the backend can call predict without the threshold being inferred. + with pytest.raises(ThresholdNotInferredError) as err: + knn_torch.predict(x) + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + + +def test_knn_torch_backend_fit_errors(): + """Tests the correct errors are raised when using the KNNTorch backend as a single detector.""" + knn_torch = KNNTorch(k=4) + + # Test that the backend raises an error if it is not fitted before + # calling forward method. + x = torch.randn((1, 10)) + with pytest.raises(NotFittedError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + # Test that the backend raises an error if it is not fitted before + # predicting. + with pytest.raises(NotFittedError) as err: + knn_torch.predict(x) + assert str(err.value) == 'KNNTorch has not been fit!' + + # Test the backend updates fitted flag on fit. + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + assert knn_torch.fitted + + # Test that the backend raises an if the forward method is called without the + # threshold being inferred. + with pytest.raises(ThresholdNotInferredError) as err: + knn_torch(x) + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + + # Test that the backend can call predict without the threshold being inferred. + knn_torch.predict(x) From ffffbe8e9b312362982ba77a374f48a0f888bdad Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 14 Mar 2023 15:23:04 +0000 Subject: [PATCH 150/247] Fix fxiture scope error in tests --- alibi_detect/od/tests/test__knn/test__knn_backend.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 6f82bcaed..75800d462 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -7,7 +7,7 @@ from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def ensembler(request): return Ensembler( normalizer=PValNormalizer(), @@ -62,15 +62,6 @@ def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): knn_torch = KNNTorch(k=[4, 5], ensembler=ensembler) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - - with pytest.raises(NotFittedError) as err: - knn_torch(x) - assert str(err.value) == 'KNNTorch has not been fit!' - - with pytest.raises(NotFittedError) as err: - knn_torch.predict(x) - assert str(err.value) == 'KNNTorch has not been fit!' - x_ref = torch.randn((1024, 10)) knn_torch.fit(x_ref) knn_torch.infer_threshold(x_ref, 0.1) From 02657ec09306884cd58e785462a79564f81625b3 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 15 Mar 2023 14:33:24 +0000 Subject: [PATCH 151/247] Catch and throw less confusing errors from backend components --- alibi_detect/exceptions.py | 46 +++++++++++++++++-- alibi_detect/od/_knn.py | 12 +++++ alibi_detect/od/pytorch/base.py | 3 +- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/tests/test__knn/test__knn.py | 4 +- .../od/tests/test__knn/test__knn_backend.py | 6 +-- 6 files changed, 60 insertions(+), 13 deletions(-) diff --git a/alibi_detect/exceptions.py b/alibi_detect/exceptions.py index 8144b750a..ce5bfc976 100644 --- a/alibi_detect/exceptions.py +++ b/alibi_detect/exceptions.py @@ -1,6 +1,8 @@ """This module defines the Alibi Detect exception hierarchy and common exceptions used across the library.""" - +from typing_extensions import Literal +from typing import Callable from abc import ABC +from functools import wraps class AlibiDetectException(Exception, ABC): @@ -16,12 +18,46 @@ def __init__(self, message: str) -> None: class NotFittedError(AlibiDetectException): - """Exception raised when a transform is not fitted.""" + def __init__(self, object_name: str) -> None: + """Exception raised when a transform is not fitted. - pass + Parameters + ---------- + message + The name of the unfit object. + """ + message = f'{object_name} has not been fit!' + super().__init__(message) class ThresholdNotInferredError(AlibiDetectException): - """Exception raised when a threshold not inferred for an outlier detector.""" + def __init__(self, object_name: str) -> None: + """Exception raised when a threshold not inferred for an outlier detector. + + Parameters + ---------- + message + The name of the object that does not have a threshold fit. + """ + message = f'{object_name} has no threshold set, call `infer_threshold` to fit one!' + super().__init__(message) + - pass +def _catch_error(err_name: Literal['NotFittedError', 'ThresholdNotInferredError']) -> Callable: + """Decorator to catch errors and raise a more informative error message. + + Note: This decorator is used to catch errors raised by specific backend components and + raise specific errors for the corresponding detector. This is done to avoid exposing + the backend components to the user. + """ + error_type = globals()[err_name] + + def decorate(f): + @wraps(f) + def applicator(self, *args, **kwargs): + try: + return f(self, *args, **kwargs) + except error_type as err: + raise error_type(self.__class__.__name__) from err + return applicator + return decorate diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index ef53bc287..b980aee36 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -1,3 +1,9 @@ +""" +.. automodule:: noodle + :members: + :private-members: +""" + from typing import Callable, Union, Optional, Dict, Any, List, Tuple from typing import TYPE_CHECKING @@ -5,6 +11,7 @@ from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict +from alibi_detect.exceptions import _catch_error as catch_error from alibi_detect.od.base import TransformProtocol, TransformProtocolType from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin from alibi_detect.od.pytorch import KNNTorch, Ensembler @@ -120,6 +127,8 @@ def fit(self, x_ref: np.ndarray) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) + @catch_error('NotFittedError') + @catch_error('ThresholdNotInferredError') def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. @@ -148,6 +157,7 @@ def score(self, x: np.ndarray) -> np.ndarray: score = self.backend._ensembler(score) return self.backend._to_numpy(score) + @catch_error('NotFittedError') def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. @@ -175,6 +185,8 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + @catch_error('NotFittedError') + @catch_error('ThresholdNotInferredError') def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 24c4d18da..702381ab6 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -87,8 +87,7 @@ def check_threshold_inferred(self): Raised if threshold is not inferred. """ if not self.threshold_inferred: - raise ThresholdNotInferredError((f'{self.__class__.__name__} has no threshold set, ' - 'call `infer_threshold` before predicting.')) + raise ThresholdNotInferredError(self.__class__.__name__) @staticmethod def _to_numpy(arg: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict]: diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index ef23fa7a5..392facdcc 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -69,7 +69,7 @@ def check_fitted(self): def _check_fitted(self): """Raises error if parent object instance has not been fit.""" if not self.fitted: - raise NotFittedError(f'{self.__class__.__name__} has not been fit!') + raise NotFittedError(self.__class__.__name__) class PValNormalizer(BaseTransformTorch, FitMixinTorch): diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index a825d841c..c5c9a5096 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -28,7 +28,7 @@ def test_unfitted_knn_single_score(): # test predict raises exception when not fitted with pytest.raises(NotFittedError) as err: _ = knn_detector.predict(x) - assert str(err.value) == 'KNNTorch has not been fit!' + assert str(err.value) == 'KNN has not been fit!' def test_fitted_knn_score(): @@ -114,7 +114,7 @@ def test_unfitted_knn_ensemble(aggregator, normalizer): # Test unfit knn ensemble raises exception when calling predict method. with pytest.raises(NotFittedError) as err: _ = knn_detector.predict(x) - assert str(err.value) == 'KNNTorch has not been fit!' + assert str(err.value) == 'KNN has not been fit!' @pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 75800d462..46bdf1b33 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -160,12 +160,12 @@ def test_knn_torch_backend_ensemble_fit_errors(ensembler): # threshold being inferred. with pytest.raises(ThresholdNotInferredError) as err: knn_torch(x) - assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` to fit one!' # Test that the backend can call predict without the threshold being inferred. with pytest.raises(ThresholdNotInferredError) as err: knn_torch.predict(x) - assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` to fit one!' def test_knn_torch_backend_fit_errors(): @@ -194,7 +194,7 @@ def test_knn_torch_backend_fit_errors(): # threshold being inferred. with pytest.raises(ThresholdNotInferredError) as err: knn_torch(x) - assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` before predicting.' + assert str(err.value) == 'KNNTorch has no threshold set, call `infer_threshold` to fit one!' # Test that the backend can call predict without the threshold being inferred. knn_torch.predict(x) From 5b754cd483e8ebf3bf63e4844666fa568c90157e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 15 Mar 2023 14:40:02 +0000 Subject: [PATCH 152/247] Remove return self statments for consistency with old detectors --- alibi_detect/od/pytorch/base.py | 4 +--- alibi_detect/od/pytorch/knn.py | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 702381ab6..f4ae43d1e 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -1,5 +1,4 @@ from typing import List, Union, Optional, Dict -from typing_extensions import Self from dataclasses import dataclass, asdict from abc import ABC, abstractmethod @@ -178,7 +177,7 @@ def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ if self.threshold_inferred else None - def infer_threshold(self, x_ref: torch.Tensor, fpr: float) -> Self: + def infer_threshold(self, x_ref: torch.Tensor, fpr: float): """Infer the threshold for the data. Prerequisite for outlier predictions. Parameters @@ -200,7 +199,6 @@ def infer_threshold(self, x_ref: torch.Tensor, fpr: float) -> Self: self.val_scores = self.ensembler.fit(self.val_scores).transform(self.val_scores) # type: ignore self.threshold = torch.quantile(self.val_scores, 1-fpr) self.threshold_inferred = True - return self def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: """Predict outlier labels for the data. diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index 2753c37d3..b01ad5304 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -1,5 +1,4 @@ from typing import Optional, Union, List, Tuple -from typing_extensions import Self import numpy as np import torch @@ -92,7 +91,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: all_knn_dists = bot_k_dists.values[:, self.ks-1] return all_knn_dists if self.ensemble else all_knn_dists[:, 0] - def fit(self, x_ref: torch.Tensor) -> Self: + def fit(self, x_ref: torch.Tensor): """Fits the detector Parameters @@ -101,4 +100,4 @@ def fit(self, x_ref: torch.Tensor) -> Self: The Dataset tensor. """ self.x_ref = x_ref - return self.set_fitted() + self.set_fitted() From 912afbdd40a6c4c20c02eff5f14628f275bd8796 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 15 Mar 2023 14:46:03 +0000 Subject: [PATCH 153/247] Reword docstring for _catch_error dectorator --- alibi_detect/exceptions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/exceptions.py b/alibi_detect/exceptions.py index ce5bfc976..1d7095c95 100644 --- a/alibi_detect/exceptions.py +++ b/alibi_detect/exceptions.py @@ -46,9 +46,9 @@ def __init__(self, object_name: str) -> None: def _catch_error(err_name: Literal['NotFittedError', 'ThresholdNotInferredError']) -> Callable: """Decorator to catch errors and raise a more informative error message. - Note: This decorator is used to catch errors raised by specific backend components and - raise specific errors for the corresponding detector. This is done to avoid exposing - the backend components to the user. + Note: This decorator should only be used on detector frontend methods. It catches errors raised by + backend components and re-raises them with error messages corresponding to the specific detector frontend. + This is done to avoid exposing the backend components to the user. """ error_type = globals()[err_name] From d880f9264edd04e5d8575c538cee0b27b19e4f9b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 15 Mar 2023 15:25:31 +0000 Subject: [PATCH 154/247] Remove autodoc comment --- alibi_detect/od/_knn.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index b980aee36..fca79e790 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -1,9 +1,3 @@ -""" -.. automodule:: noodle - :members: - :private-members: -""" - from typing import Callable, Union, Optional, Dict, Any, List, Tuple from typing import TYPE_CHECKING From 8d98ee2f3a0d42023d502e96098d2e0a96b9da3e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 16 Mar 2023 17:32:10 +0000 Subject: [PATCH 155/247] Add value error for invalid choice of fpr --- alibi_detect/od/pytorch/base.py | 6 +++++- .../od/tests/test__knn/test__knn_backend.py | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index f4ae43d1e..03c716e0e 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -191,13 +191,17 @@ def infer_threshold(self, x_ref: torch.Tensor, fpr: float): ------ ValueError Raised if `fpr` is not in ``(0, 1)``. + ValueError + Raised if `fpr` is less than ``1/len(x_ref)``. """ if not 0 < fpr < 1: raise ValueError('`fpr` must be in `(0, 1)`.') + if fpr < 1/len(x_ref): + raise ValueError(f'`fpr` must be greater than `1/len(x_ref)={1/len(x_ref)}`.') self.val_scores = self.score(x_ref) if self.ensemble: self.val_scores = self.ensembler.fit(self.val_scores).transform(self.val_scores) # type: ignore - self.threshold = torch.quantile(self.val_scores, 1-fpr) + self.threshold = torch.quantile(self.val_scores, 1-fpr, interpolation='higher') self.threshold_inferred = True def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 46bdf1b33..96a176691 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -198,3 +198,23 @@ def test_knn_torch_backend_fit_errors(): # Test that the backend can call predict without the threshold being inferred. knn_torch.predict(x) + + +def test_knn_infer_threshold_value_errors(): + """Tests the correct errors are raised when using incorrect choice of fpr for the KNNTorch backend detector.""" + knn_torch = KNNTorch(k=4) + x_ref = torch.randn((1024, 10)) + knn_torch.fit(x_ref) + + # fpr must be greater than 1/len(x_ref) otherwise it excludes all points in the reference dataset + with pytest.raises(ValueError) as err: + knn_torch.infer_threshold(x_ref, 1/1025) + assert str(err.value) == '`fpr` must be greater than `1/len(x_ref)=0.0009765625`.' + + # fpr must be between 0 and 1 + with pytest.raises(ValueError) as err: + knn_torch.infer_threshold(x_ref, 1.1) + assert str(err.value) == '`fpr` must be in `(0, 1)`.' + + knn_torch.infer_threshold(x_ref, 0.99) + knn_torch.infer_threshold(x_ref, 1/1023) From 95d0e3cced32fc14d91e2a62c70f9661bbb7e8ca Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 16 Mar 2023 17:38:09 +0000 Subject: [PATCH 156/247] Set default PValNormalizer --- alibi_detect/od/_knn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index fca79e790..62d644810 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -28,7 +28,7 @@ def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], kernel: Optional[Callable] = None, - normalizer: Optional[Union[TransformProtocolType, NormalizerLiterals]] = 'ShiftAndScaleNormalizer', + normalizer: Optional[Union[TransformProtocolType, NormalizerLiterals]] = 'PValNormalizer', aggregator: Union[TransformProtocol, AggregatorLiterals] = 'AverageAggregator', backend: Literal['pytorch'] = 'pytorch', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, From 12d22f70a4622e284eab22548a9be484fea74d67 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 16 Mar 2023 17:47:44 +0000 Subject: [PATCH 157/247] Cast outlier booleans to ints --- alibi_detect/od/pytorch/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 03c716e0e..fbb9629c4 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -160,7 +160,7 @@ def _classify_outlier(self, scores: torch.Tensor) -> torch.Tensor: ------- `torch.Tensor` or ``None`` """ - return scores > self.threshold if self.threshold_inferred else None + return (scores > self.threshold).to(torch.int8) if self.threshold_inferred else None def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: """Compute p-values for the scores. From 767aff93202e19b360881dca61b3342a9cb2d8f9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 16 Mar 2023 17:52:17 +0000 Subject: [PATCH 158/247] Rewrite the 1st paragraph of the knn detector docstring --- alibi_detect/od/_knn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 62d644810..3156f1edf 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -36,9 +36,9 @@ def __init__( """ k-Nearest Neighbors (kNN) outlier detector. - The kNN detector is a non-parametric method for outlier detection. The detector computes the distance - between each test point and its `k` nearest neighbors. The distance can be computed using a kernel function - or a distance metric. The distance is then normalized and aggregated to obtain a single outlier score. + The kNN detector is a non-parametric method for outlier detection. The detector scores each instance + based on the distance to its neighbors. Instances with a large distance to their neighbors are more + likely to be outliers. The detector can be initialized with `k` a single value or an array of values. If `k` is a single value then the outlier score is the distance/kernel similarity to the k-th nearest neighbor. If `k` is an array of From 467aa6a06980714d808ff6a58ddd3356cabc5470 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 16 Mar 2023 18:01:22 +0000 Subject: [PATCH 159/247] Minor change --- alibi_detect/od/pytorch/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index fbb9629c4..6057a6c09 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -223,8 +223,7 @@ def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: Returns ------- - Output of the outlier detector. Includes the p-values, outlier labels, instance scores and \ - threshold. + Output of the outlier detector. Includes the p-values, outlier labels, instance scores and threshold. """ self.check_fitted() # type: ignore raw_scores = self.score(x) From d57d1bdf6055e0edd3096848941617d1d8a4c423 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 16 Mar 2023 18:11:11 +0000 Subject: [PATCH 160/247] Add docstrings for the ensemble tests --- alibi_detect/od/tests/test_ensemble.py | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/alibi_detect/od/tests/test_ensemble.py b/alibi_detect/od/tests/test_ensemble.py index 1fa0cd241..5df13fc59 100644 --- a/alibi_detect/od/tests/test_ensemble.py +++ b/alibi_detect/od/tests/test_ensemble.py @@ -6,6 +6,11 @@ def test_pval_normalizer(): + """Test the PValNormalizer + + - Test the PValNormalizer correctly normalizes data passed to it + - Test the PValNormalizer throws the correct errors if not fit + """ normalizer = ensemble.PValNormalizer() x = torch.randn(3, 10) x_ref = torch.randn(64, 10) @@ -35,6 +40,11 @@ def test_pval_normalizer(): def test_shift_and_scale_normalizer(): + """Test the ShiftAndScaleNormalizer + + - Test the ShiftAndScaleNormalizer correctly normalizes data passed to it + - Test the ShiftAndScaleNormalizer throws the correct errors if not fit. + """ normalizer = ensemble.ShiftAndScaleNormalizer() x = torch.randn(3, 10) * 3 + 2 x_ref = torch.randn(5000, 10) * 3 + 2 @@ -57,6 +67,11 @@ def test_shift_and_scale_normalizer(): def test_average_aggregator(): + """Test the AverageAggregator + + - Test the AverageAggregator correctly aggregates data passed to it. + - Test the AverageAggregator can be torch scripted + """ aggregator = ensemble.AverageAggregator() scores = torch.randn((3, 10)) @@ -72,6 +87,12 @@ def test_average_aggregator(): def test_weighted_average_aggregator(): + """Test the AverageAggregator + + - Test the AverageAggregator correctly aggregates data passed to it + - Test the AverageAggregator throws an error if the weights are not valid + - Test the AverageAggregator can be torch scripted + """ weights = abs(torch.randn((10))) with pytest.raises(ValueError) as err: @@ -94,6 +115,11 @@ def test_weighted_average_aggregator(): def test_topk_aggregator(): + """Test the TopKAggregator + + - Test the TopKAggregator correctly aggregates data passed to it + - Test the TopKAggregator can be torch scripted + """ aggregator = ensemble.TopKAggregator(k=4) scores = torch.randn((3, 10)) @@ -110,6 +136,11 @@ def test_topk_aggregator(): def test_max_aggregator(): + """Test the MaxAggregator + + - Test the MaxAggregator correctly aggregates data passed to it + - Test the MaxAggregator can be torch scripted + """ aggregator = ensemble.MaxAggregator() scores = torch.randn((3, 10)) @@ -126,6 +157,11 @@ def test_max_aggregator(): def test_min_aggregator(): + """Test the MinAggregator + + - Test the MinAggregator correctly aggregates data passed to it + - Test the MinAggregator can be torch scripted + """ aggregator = ensemble.MinAggregator() scores = torch.randn((3, 10)) @@ -144,6 +180,11 @@ def test_min_aggregator(): @pytest.mark.parametrize('aggregator', ['AverageAggregator', 'MaxAggregator', 'MinAggregator', 'TopKAggregator']) @pytest.mark.parametrize('normalizer', ['PValNormalizer', 'ShiftAndScaleNormalizer']) def test_ensembler(aggregator, normalizer): + """Test the Ensembler for each combination of aggregator and normalizer + + - Test the ensembler correctly aggregates and normalizes the scores + - Test the ensembler can be torch scripted + """ aggregator = getattr(ensemble, aggregator)() normalizer = getattr(ensemble, normalizer)() ensembler = ensemble.Ensembler(aggregator=aggregator, normalizer=normalizer) From a535e8d87bd812c2b120770b582591a842f4bfbe Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 17 Mar 2023 09:33:57 +0000 Subject: [PATCH 161/247] Rename x_ref to x in infer_threshold --- alibi_detect/od/_knn.py | 8 ++++---- alibi_detect/od/pytorch/base.py | 12 ++++++------ .../od/tests/test__knn/test__knn_backend.py | 16 ++++++++-------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 3156f1edf..7fe9eefcf 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -152,7 +152,7 @@ def score(self, x: np.ndarray) -> np.ndarray: return self.backend._to_numpy(score) @catch_error('NotFittedError') - def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + def infer_threshold(self, x: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the @@ -170,14 +170,14 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: Parameters ---------- - x_ref + x Reference data used to infer the threshold. fpr False positive rate used to infer the threshold. The false positive rate is the proportion of - instances in `x_ref` that are incorrectly classified as outliers. The false positive rate should + instances in `x` that are incorrectly classified as outliers. The false positive rate should be in the range ``(0, 1)``. """ - self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @catch_error('NotFittedError') @catch_error('ThresholdNotInferredError') diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 6057a6c09..f0ddcbda7 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -177,12 +177,12 @@ def _p_vals(self, scores: torch.Tensor) -> torch.Tensor: return (1 + (scores[:, None] < self.val_scores).sum(-1))/len(self.val_scores) \ if self.threshold_inferred else None - def infer_threshold(self, x_ref: torch.Tensor, fpr: float): + def infer_threshold(self, x: torch.Tensor, fpr: float): """Infer the threshold for the data. Prerequisite for outlier predictions. Parameters ---------- - x_ref + x Data to infer the threshold for. fpr False positive rate to use for threshold inference. @@ -192,13 +192,13 @@ def infer_threshold(self, x_ref: torch.Tensor, fpr: float): ValueError Raised if `fpr` is not in ``(0, 1)``. ValueError - Raised if `fpr` is less than ``1/len(x_ref)``. + Raised if `fpr` is less than ``1/len(x)``. """ if not 0 < fpr < 1: raise ValueError('`fpr` must be in `(0, 1)`.') - if fpr < 1/len(x_ref): - raise ValueError(f'`fpr` must be greater than `1/len(x_ref)={1/len(x_ref)}`.') - self.val_scores = self.score(x_ref) + if fpr < 1/len(x): + raise ValueError(f'`fpr` must be greater than `1/len(x)={1/len(x)}`.') + self.val_scores = self.score(x) if self.ensemble: self.val_scores = self.ensembler.fit(self.val_scores).transform(self.val_scores) # type: ignore self.threshold = torch.quantile(self.val_scores, 1-fpr, interpolation='higher') diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 96a176691..1c9727116 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -203,18 +203,18 @@ def test_knn_torch_backend_fit_errors(): def test_knn_infer_threshold_value_errors(): """Tests the correct errors are raised when using incorrect choice of fpr for the KNNTorch backend detector.""" knn_torch = KNNTorch(k=4) - x_ref = torch.randn((1024, 10)) - knn_torch.fit(x_ref) + x = torch.randn((1024, 10)) + knn_torch.fit(x) - # fpr must be greater than 1/len(x_ref) otherwise it excludes all points in the reference dataset + # fpr must be greater than 1/len(x) otherwise it excludes all points in the reference dataset with pytest.raises(ValueError) as err: - knn_torch.infer_threshold(x_ref, 1/1025) - assert str(err.value) == '`fpr` must be greater than `1/len(x_ref)=0.0009765625`.' + knn_torch.infer_threshold(x, 1/1025) + assert str(err.value) == '`fpr` must be greater than `1/len(x)=0.0009765625`.' # fpr must be between 0 and 1 with pytest.raises(ValueError) as err: - knn_torch.infer_threshold(x_ref, 1.1) + knn_torch.infer_threshold(x, 1.1) assert str(err.value) == '`fpr` must be in `(0, 1)`.' - knn_torch.infer_threshold(x_ref, 0.99) - knn_torch.infer_threshold(x_ref, 1/1023) + knn_torch.infer_threshold(x, 0.99) + knn_torch.infer_threshold(x, 1/1023) From 58d4f21cdf3bd7d13fd52d2b7cdfecda75f8d0fd Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Fri, 17 Mar 2023 12:28:06 +0000 Subject: [PATCH 162/247] Fix failing tests --- alibi_detect/od/_mahalanobis.py | 7 +++++++ alibi_detect/od/pytorch/mahalanobis.py | 8 +++++--- .../tests/test__mahalanobis/test__mahalanobis.py | 6 +++--- .../test__mahalanobis/test__mahalanobis_backend.py | 14 +++++++------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index e7b24909d..b97b5b74d 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -1,5 +1,7 @@ from typing import Union, Optional, Dict, Any from typing import TYPE_CHECKING +from alibi_detect.exceptions import _catch_error as catch_error + import numpy as np @@ -77,6 +79,8 @@ def fit(self, x_ref: np.ndarray) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) + @catch_error('NotFittedError') + @catch_error('ThresholdNotInferredError') def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. @@ -96,6 +100,7 @@ def score(self, x: np.ndarray) -> np.ndarray: score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) + @catch_error('NotFittedError') def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. @@ -113,6 +118,8 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + @catch_error('NotFittedError') + @catch_error('ThresholdNotInferredError') def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 837b076f0..fe5094fef 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -6,6 +6,8 @@ class MahalanobisTorch(TorchOutlierDetector): + ensemble = None + def __init__( self, min_eigenvalue: float = 1e-6, @@ -64,13 +66,12 @@ def score(self, x: torch.Tensor) -> torch.Tensor: NotFitException If called before detector has been fit. """ - if not torch.jit.is_scripting(): - self.check_fitted() + self.check_fitted() x = torch.as_tensor(x) x_pcs = self._compute_linear_proj(x) return (x_pcs**2).sum(-1).cpu() - def _fit(self, x_ref: torch.Tensor): + def fit(self, x_ref: torch.Tensor): """Fits the detector Parameters @@ -80,6 +81,7 @@ def _fit(self, x_ref: torch.Tensor): """ self.x_ref = x_ref self._compute_linear_pcs(self.x_ref) + self.set_fitted() def _compute_linear_pcs(self, X: torch.Tensor): """Computes the principle components of the data. diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index 228ec3509..0702c7159 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -3,7 +3,7 @@ import torch from alibi_detect.od._mahalanobis import Mahalanobis -from alibi_detect.base import NotFitException +from alibi_detect.exceptions import NotFittedError from sklearn.datasets import make_moons @@ -18,9 +18,9 @@ def make_mahalanobis_detector(): def test_unfitted_mahalanobis_single_score(): mahalanobis_detector = Mahalanobis() x = np.array([[0, 10], [0.1, 0]]) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: _ = mahalanobis_detector.predict(x) - assert str(err.value) == 'MahalanobisTorch has not been fit!' + assert str(err.value) == 'Mahalanobis has not been fit!' def test_fitted_mahalanobis_single_score(): diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py index ec678e739..6ddb66bad 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py @@ -3,30 +3,30 @@ import numpy as np from alibi_detect.od.pytorch.mahalanobis import MahalanobisTorch -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError def test_mahalanobis_torch_backend_fit_errors(): mahalanobis_torch = MahalanobisTorch() - assert not mahalanobis_torch._fitted + assert not mahalanobis_torch.fitted x = torch.randn((1, 10)) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: mahalanobis_torch(x) assert str(err.value) == 'MahalanobisTorch has not been fit!' - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: mahalanobis_torch.predict(x) assert str(err.value) == 'MahalanobisTorch has not been fit!' x_ref = torch.randn((1024, 10)) mahalanobis_torch.fit(x_ref) - assert mahalanobis_torch._fitted + assert mahalanobis_torch.fitted - with pytest.raises(ThresholdNotInferredException) as err: + with pytest.raises(ThresholdNotInferredError) as err: mahalanobis_torch(x) - assert str(err.value) == 'MahalanobisTorch has no threshold set, call `infer_threshold` before predicting.' + assert str(err.value) == 'MahalanobisTorch has no threshold set, call `infer_threshold` to fit one!' assert mahalanobis_torch.predict(x) From 1f5be2f298cf0b2b122ee25153ec884db42537a5 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 10:54:37 +0000 Subject: [PATCH 163/247] Minor changes to align mahalanobis and kNN work --- alibi_detect/od/_mahalanobis.py | 24 +++++++++++++----------- alibi_detect/od/pytorch/mahalanobis.py | 2 ++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index b97b5b74d..43d057747 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -25,8 +25,8 @@ class Mahalanobis(BaseDetector, FitMixin, ThresholdMixin): def __init__( self, min_eigenvalue: float = 1e-6, - device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """The Mahalanobis outlier detection method. @@ -61,10 +61,12 @@ def __init__( ).verify_backend(backend_str) backend_cls = backends[backend] - self.backend = backend_cls( - min_eigenvalue, - device=device - ) + self.backend = backend_cls(min_eigenvalue, device=device) + + # set metadata + self.meta['detector_type'] = 'outlier' + self.meta['data_type'] = 'numeric' + self.meta['online'] = False def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. @@ -101,7 +103,7 @@ def score(self, x: np.ndarray) -> np.ndarray: return self.backend._to_numpy(score) @catch_error('NotFittedError') - def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + def infer_threshold(self, x: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. The threshold is inferred using the reference data and the false positive rate. The threshold is used to @@ -109,14 +111,14 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: Parameters ---------- - x_ref + x Reference data used to infer the threshold. fpr - False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ - `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ - ``(0, 1)``. + False positive rate used to infer the threshold. The false positive rate is the proportion of + instances in `x` that are incorrectly classified as outliers. The false positive rate should + be in the range ``(0, 1)``. """ - self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @catch_error('NotFittedError') @catch_error('ThresholdNotInferredError') diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index fe5094fef..0d6f47a16 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -26,6 +26,7 @@ def __init__( TorchOutlierDetector.__init__(self, device=device) self.min_eigenvalue = min_eigenvalue + @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -49,6 +50,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: preds = scores > self.threshold return preds.cpu() + @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` From b573bea0a7bd81c27451609d2f457c85f0787e54 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 10:59:43 +0000 Subject: [PATCH 164/247] Revert renaming change --- alibi_detect/od/_mahalanobis.py | 3 ++- alibi_detect/utils/fetching/fetching.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 43d057747..24ae8493f 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -28,7 +28,8 @@ def __init__( backend: Literal['pytorch'] = 'pytorch', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: - """The Mahalanobis outlier detection method. + """ + The Mahalanobis outlier detection method. The Mahalanobis method computes the covariance matrix of a reference dataset passed in the `fit` method. It then saves the eigenvectors of this matrix with eigenvalues greater than `min_eigenvalue`. While doing so diff --git a/alibi_detect/utils/fetching/fetching.py b/alibi_detect/utils/fetching/fetching.py index 7816cb8be..f61faa288 100644 --- a/alibi_detect/utils/fetching/fetching.py +++ b/alibi_detect/utils/fetching/fetching.py @@ -21,7 +21,7 @@ from alibi_detect.base import BaseDetector # noqa from alibi_detect.od.llr import LLR # noqa from alibi_detect.od.isolationforest import IForest # noqa - from alibi_detect.od.stateful_mahalanobis import Mahalanobis # noqa + from alibi_detect.od.mahalanobis import Mahalanobis # noqa from alibi_detect.od.aegmm import OutlierAEGMM # noqa from alibi_detect.od.ae import OutlierAE # noqa from alibi_detect.od.prophet import OutlierProphet # noqa From dc0f2a8f4e0cb4de40cff523cc24d752deec42e4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 11:19:13 +0000 Subject: [PATCH 165/247] Further alignment changes --- alibi_detect/od/pytorch/__init__.py | 1 - alibi_detect/tests/test_dep_management.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/__init__.py b/alibi_detect/od/pytorch/__init__.py index c6e9cf16c..5f22a39dd 100644 --- a/alibi_detect/od/pytorch/__init__.py +++ b/alibi_detect/od/pytorch/__init__.py @@ -3,4 +3,3 @@ KNNTorch = import_optional('alibi_detect.od.pytorch.knn', ['KNNTorch']) MahalanobisTorch = import_optional('alibi_detect.od.pytorch.mahalanobis', ['MahalanobisTorch']) Ensembler = import_optional('alibi_detect.od.pytorch.ensemble', ['Ensembler']) -to_numpy = import_optional('alibi_detect.od.pytorch.base', ['to_numpy']) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index bc4d5e656..22313de61 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -138,8 +138,7 @@ def test_od_backend_dependencies(opt_dep): for dependency, relations in [ ('Ensembler', ['torch', 'keops']), ('KNNTorch', ['torch', 'keops']), - ('MahalanobisTorch', ['torch', 'keops']), - ('to_numpy', ['torch', 'keops']), + ('MahalanobisTorch', ['torch', 'keops']) ]: dependency_map[dependency] = relations from alibi_detect.od import pytorch as od_pt_backend From 004a19c15f0eac0ee5a692609df6b3612333e7ac Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 11:22:01 +0000 Subject: [PATCH 166/247] Minor change to Mahanobis backend docstrings --- alibi_detect/od/pytorch/mahalanobis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 0d6f47a16..85054411f 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -61,7 +61,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - Tensor of scores for each element in `x`. + Tensor of scores for each element in `x`. Raises ------ From a567b935dfb1498b1bbf46cb5a0ef13f1a0c1695 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 13:31:26 +0000 Subject: [PATCH 167/247] Rewrite docstrings for mahalanobis detector --- alibi_detect/od/_knn.py | 3 -- alibi_detect/od/_mahalanobis.py | 58 +++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index 7fe9eefcf..c9176b19d 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -162,9 +162,6 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: ------ ValueError Raised if `fpr` is not in ``(0, 1)``. - - Raises - ------ NotFittedError If called before detector has been fit. diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 24ae8493f..1a8d42112 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -31,22 +31,28 @@ def __init__( """ The Mahalanobis outlier detection method. - The Mahalanobis method computes the covariance matrix of a reference dataset passed in the `fit` method. It - then saves the eigenvectors of this matrix with eigenvalues greater than `min_eigenvalue`. While doing so - it also scales the eigenvectors such that the reference data projected onto them has mean ``0`` and std ``1``. + The Mahalanobis computes the directions of variation of a dataset and uses them to detect when points are + outliers by checking to see if the point vary from dataset points in unexpected ways. - When we score a test point `x` we project it onto the eigenvectors and compute the l2-norm of the - projected point. The higher the score, the more outlying the instance. + When we fit the Mahalanobis method we compute the covariance matrix of the reference data and its eigenvectors + and eigenvalues. We filter small eigenvalues for numerical stability using the `min_eigenvalue` parameter. We + then inversely weight each eigenvector by its eigenvalue. + + When we score test points we project them onto the eigenvectors and compute the l2-norm of the projected point. + Because the eigenvectors are inversely weighted by the eigenvalues, the score will take into account the + difference in variance along each direction of variation. If a test point lies along a direction of high + variation then it must lie very far out to obtain a high score. If a test point lies along a direction of low + variation then it doesn't need to lie very far out to obtain a high score. Parameters ---------- min_eigenvalue - Eigenvectors with eigenvalues below this value will be discarded. + Eigenvectors with eigenvalues below this value will be discarded. This is to ensure numerical stability. backend Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. Raises ------ @@ -72,8 +78,9 @@ def __init__( def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. - Fitting the Mahalanobis method amounts to computing the covariance matrix of the reference data and - saving the eigenvectors with eigenvalues greater than `min_eigenvalue`. + Fitting the Mahalanobis method amounts to computing the covariance matrix and its eigenvectors. We filter out + very small eigenvalues using the `min_eigenvalue` parameter. We then scale the eigenvectors such that the data + projected onto them has mean ``0`` and std ``1``. Parameters ---------- @@ -83,21 +90,25 @@ def fit(self, x_ref: np.ndarray) -> None: self.backend.fit(self.backend._to_tensor(x_ref)) @catch_error('NotFittedError') - @catch_error('ThresholdNotInferredError') def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. - The mahalanobis method projects `x` onto the eigenvectors of the covariance matrix of the reference data. - The score is then the l2-norm of the projected data. The higher the score, the more outlying the instance. + The mahalanobis method projects `x` onto the scaled eigenvectors computed during the fit step. The score is then + the l2-norm of the projected data. The higher the score, the more outlying the instance. Parameters ---------- x Data to score. The shape of `x` should be `(n_instances, n_features)`. + Raises + ------ + NotFittedError + If called before detector has been fit. + Returns ------- - Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more outlying the \ instance. """ score = self.backend.score(self.backend._to_tensor(x)) @@ -107,8 +118,15 @@ def score(self, x: np.ndarray) -> np.ndarray: def infer_threshold(self, x: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. - The threshold is inferred using the reference data and the false positive rate. The threshold is used to - determine the outlier labels in the predict method. + The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the + reference data as outliers. + + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + NotFittedError + If called before detector has been fit. Parameters ---------- @@ -122,19 +140,25 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @catch_error('NotFittedError') - @catch_error('ThresholdNotInferredError') def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. + Scores the instances in `x` and if the threshold was inferred, returns the outlier labels and p-values as well. + Parameters ---------- x Data to predict. The shape of `x` should be `(n_instances, n_features)`. + Raises + ------ + NotFittedError + If called before detector has been fit. + Returns ------- Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ - performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. """ From f9b023ffa354ad30a6ee6b60409ab75111002eb1 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 14:12:13 +0000 Subject: [PATCH 168/247] Add docstrings for tests --- .../test__mahalanobis/test__mahalanobis.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index 0702c7159..cd11d10d6 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -16,22 +16,38 @@ def make_mahalanobis_detector(): def test_unfitted_mahalanobis_single_score(): + """Test Mahalanobis detector throws errors when not fitted.""" mahalanobis_detector = Mahalanobis() x = np.array([[0, 10], [0.1, 0]]) + + with pytest.raises(NotFittedError) as err: + mahalanobis_detector.score(x) + assert str(err.value) == 'Mahalanobis has not been fit!' + + # test predict raises exception when not fitted with pytest.raises(NotFittedError) as err: - _ = mahalanobis_detector.predict(x) + mahalanobis_detector.predict(x) assert str(err.value) == 'Mahalanobis has not been fit!' -def test_fitted_mahalanobis_single_score(): +def test_fitted_mahalanobis_score(): + """Test Mahalanobis detector score method. + + Test Mahalanobis detector that has been fitted on reference data but has not had a threshold + inferred can still score data using the predict method. Test that it does not raise an error + but does not return `threshold`, `p_value` and `is_outlier` values. + """ mahalanobis_detector = Mahalanobis() x_ref = np.random.randn(100, 2) mahalanobis_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) + scores = mahalanobis_detector.score(x) + y = mahalanobis_detector.predict(x) y = y['data'] assert y['instance_score'][0] > 5 assert y['instance_score'][1] < 1 + assert all(y['instance_score'] == scores) assert not y['threshold_inferred'] assert y['threshold'] is None assert y['is_outlier'] is None @@ -39,6 +55,12 @@ def test_fitted_mahalanobis_single_score(): def test_fitted_mahalanobis_predict(): + """Test Mahalanobis detector predict method. + + Test Mahalanobis detector that has been fitted on reference data and has had a threshold + inferred can score data using the predict method as well as predict outliers. Test that it + returns `threshold`, `p_value` and `is_outlier` values. + """ mahalanobis_detector = make_mahalanobis_detector() x_ref = np.random.randn(100, 2) mahalanobis_detector.infer_threshold(x_ref, 0.1) @@ -54,6 +76,12 @@ def test_fitted_mahalanobis_predict(): def test_mahalanobis_integration(): + """Test Mahalanobis detector on moons dataset. + + Test Mahalanobis detector on a more complex 2d example. Test that the detector can be fitted + on reference data and infer a threshold. Test that it differentiates between inliers and outliers. + Test that the detector can be scripted. + """ mahalanobis_detector = Mahalanobis() X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] From 0cbe044998e7b9597fda0cb445e058bde44ac33d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 14:33:30 +0000 Subject: [PATCH 169/247] Add docstrings for mahalanobis backend tests --- .../test__mahalanobis_backend.py | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py index 6ddb66bad..7c8139e3f 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py @@ -6,32 +6,14 @@ from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError -def test_mahalanobis_torch_backend_fit_errors(): - mahalanobis_torch = MahalanobisTorch() - assert not mahalanobis_torch.fitted - - x = torch.randn((1, 10)) - with pytest.raises(NotFittedError) as err: - mahalanobis_torch(x) - assert str(err.value) == 'MahalanobisTorch has not been fit!' - - with pytest.raises(NotFittedError) as err: - mahalanobis_torch.predict(x) - assert str(err.value) == 'MahalanobisTorch has not been fit!' - - x_ref = torch.randn((1024, 10)) - mahalanobis_torch.fit(x_ref) - - assert mahalanobis_torch.fitted - - with pytest.raises(ThresholdNotInferredError) as err: - mahalanobis_torch(x) - assert str(err.value) == 'MahalanobisTorch has no threshold set, call `infer_threshold` to fit one!' - - assert mahalanobis_torch.predict(x) - - def test_mahalanobis_linear_scoring(): + """Test Mahalanobis detector linear scoring method. + + Test that the Mahalanobis detector _compute_linear_proj method correctly whitens the x_ref data + and that the score method correctly orders different test points. Test that the detector correctly + detects true outliers and that the correct proportion of in distribution data is flagged as + outliers. + """ mahalanobis_torch = MahalanobisTorch() mean = [8, 8] cov = [[2., 0.], [0., 1.]] @@ -69,6 +51,7 @@ def test_mahalanobis_linear_scoring(): def test_mahalanobis_torch_backend_ts(tmp_path): + """Test Mahalanobis detector backend is torch-scriptable and savable.""" mahalanobis_torch = MahalanobisTorch() x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) x_ref = torch.randn((1024, 10)) @@ -84,3 +67,34 @@ def test_mahalanobis_torch_backend_ts(tmp_path): mahalanobis_torch = torch.load(tmp_path / 'mahalanobis_torch.pt') pred_2 = mahalanobis_torch(x) assert torch.all(pred_1 == pred_2) + + +def test_mahalanobis_torch_backend_fit_errors(): + """Test Mahalanobis detector backend fit errors. + + Test that an unfit detector backend raises an error when calling predict or score. Test that the + detector backend raises an error when calling the forward method while the threshold has not been + inferred. + """ + mahalanobis_torch = MahalanobisTorch() + assert not mahalanobis_torch.fitted + + x = torch.randn((1, 10)) + with pytest.raises(NotFittedError) as err: + mahalanobis_torch(x) + assert str(err.value) == 'MahalanobisTorch has not been fit!' + + with pytest.raises(NotFittedError) as err: + mahalanobis_torch.predict(x) + assert str(err.value) == 'MahalanobisTorch has not been fit!' + + x_ref = torch.randn((1024, 10)) + mahalanobis_torch.fit(x_ref) + + assert mahalanobis_torch.fitted + + with pytest.raises(ThresholdNotInferredError) as err: + mahalanobis_torch(x) + assert str(err.value) == 'MahalanobisTorch has no threshold set, call `infer_threshold` to fit one!' + + assert mahalanobis_torch.predict(x) From eecaa4b5c26c84cc299ae1cf97703a5a4cdc4605 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 14:41:35 +0000 Subject: [PATCH 170/247] Minor change --- .../od/tests/test__mahalanobis/test__mahalanobis_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py index 7c8139e3f..a994e9950 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py @@ -9,7 +9,7 @@ def test_mahalanobis_linear_scoring(): """Test Mahalanobis detector linear scoring method. - Test that the Mahalanobis detector _compute_linear_proj method correctly whitens the x_ref data + Test that the Mahalanobis detector `_compute_linear_proj` method correctly whitens the x_ref data and that the score method correctly orders different test points. Test that the detector correctly detects true outliers and that the correct proportion of in distribution data is flagged as outliers. From 858d8ec4a47d6e3ba2da4aff718ca62d3caf9704 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 20 Mar 2023 14:45:16 +0000 Subject: [PATCH 171/247] Make X in detector private methods lowercase --- alibi_detect/od/pytorch/mahalanobis.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 85054411f..f3fb69094 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -85,29 +85,29 @@ def fit(self, x_ref: torch.Tensor): self._compute_linear_pcs(self.x_ref) self.set_fitted() - def _compute_linear_pcs(self, X: torch.Tensor): + def _compute_linear_pcs(self, x: torch.Tensor): """Computes the principle components of the data. Parameters ---------- - X + x The reference dataset. """ - self.means = X.mean(0) - X = X - self.means - cov_mat = (X.t() @ X)/(len(X)-1) + self.means = x.mean(0) + x = x - self.means + cov_mat = (x.t() @ x)/(len(x)-1) D, V = torch.linalg.eigh(cov_mat) non_zero_inds = D > self.min_eigenvalue self.pcs = V[:, non_zero_inds] / D[None, non_zero_inds].sqrt() - def _compute_linear_proj(self, X: torch.Tensor) -> torch.Tensor: + def _compute_linear_proj(self, x: torch.Tensor) -> torch.Tensor: """Projects the data point being tested onto the principle components. Parameters ---------- - X + x The data point being tested. """ - X_cen = X - self.means - X_proj = X_cen @ self.pcs - return X_proj + x_cen = x - self.means + x_proj = x_cen @ self.pcs + return x_proj From 8d105451e4d3785ad33c378b18b2e75cf02e396e Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 09:56:16 +0100 Subject: [PATCH 172/247] Change ensemble=None to ensemble=False in mahalanobis detector torch backend --- alibi_detect/od/pytorch/mahalanobis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index f3fb69094..59c7a10ea 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -6,7 +6,7 @@ class MahalanobisTorch(TorchOutlierDetector): - ensemble = None + ensemble = False def __init__( self, From 70dc0c666f82edaabbe2e493b79a1f236f463abc Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 10:11:56 +0100 Subject: [PATCH 173/247] Make set_fitted private --- alibi_detect/od/pytorch/ensemble.py | 8 ++++---- alibi_detect/od/pytorch/knn.py | 2 +- alibi_detect/od/pytorch/mahalanobis.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 392facdcc..9216365eb 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -46,7 +46,7 @@ def fit(self, x: torch.Tensor) -> Self: """ pass - def set_fitted(self) -> Self: + def _set_fitted(self) -> Self: """Sets the fitted attribute to True. Should be called within each transform method. @@ -92,7 +92,7 @@ def fit(self, val_scores: torch.Tensor) -> Self: score outputs of ensemble of detectors applied to reference data. """ self.val_scores = val_scores - return self.set_fitted() + return self._set_fitted() def transform(self, scores: torch.Tensor) -> torch.Tensor: """Transform scores to 1 - p-values. @@ -133,7 +133,7 @@ def fit(self, val_scores: torch.Tensor) -> Self: """ self.val_means = val_scores.mean(0)[None, :] self.val_scales = val_scores.std(0)[None, :] - return self.set_fitted() + return self._set_fitted() def transform(self, scores: torch.Tensor) -> torch.Tensor: """Transform scores to normalized values. Subtracts the mean and scales by the standard deviation. @@ -312,4 +312,4 @@ def fit(self, x: torch.Tensor) -> Self: """ if self.normalizer is not None: self.normalizer.fit(x) # type: ignore - return self.set_fitted() + return self._set_fitted() diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index b01ad5304..cce26a749 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -100,4 +100,4 @@ def fit(self, x_ref: torch.Tensor): The Dataset tensor. """ self.x_ref = x_ref - self.set_fitted() + self._set_fitted() diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 59c7a10ea..4ecfb0d1f 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -83,7 +83,7 @@ def fit(self, x_ref: torch.Tensor): """ self.x_ref = x_ref self._compute_linear_pcs(self.x_ref) - self.set_fitted() + self._set_fitted() def _compute_linear_pcs(self, x: torch.Tensor): """Computes the principle components of the data. From 76778c9ed98055430c9af92faabfe8273bbe3573 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 10:16:32 +0100 Subject: [PATCH 174/247] Use super in MahalanobisDetector --- alibi_detect/od/pytorch/mahalanobis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 4ecfb0d1f..062e973dc 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -23,7 +23,7 @@ def __init__( Device type used. The default None tries to use the GPU and falls back on CPU if needed. Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. """ - TorchOutlierDetector.__init__(self, device=device) + super().__init__(device=device) self.min_eigenvalue = min_eigenvalue @torch.no_grad() From 6fad0144cb44721f52b6b2da924f3b2e380d309c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 10:48:22 +0100 Subject: [PATCH 175/247] Reorder docstring sections to be compatible with numpy conventions --- alibi_detect/od/_knn.py | 38 ++++++++++++++++----------------- alibi_detect/od/_mahalanobis.py | 34 ++++++++++++++--------------- alibi_detect/od/pytorch/base.py | 8 +++---- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index c9176b19d..dfa9edf74 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -135,17 +135,17 @@ def score(self, x: np.ndarray) -> np.ndarray: x Data to score. The shape of `x` should be `(n_instances, n_features)`. + Returns + ------- + Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ + instance. + Raises ------ NotFittedError If called before detector has been fit. ThresholdNotInferredError If k is a list and a threshold was not inferred. - - Returns - ------- - Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ - instance. """ score = self.backend.score(self.backend._to_tensor(x)) score = self.backend._ensembler(score) @@ -158,13 +158,6 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the reference data as outliers. - Raises - ------ - ValueError - Raised if `fpr` is not in ``(0, 1)``. - NotFittedError - If called before detector has been fit. - Parameters ---------- x @@ -173,6 +166,13 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: False positive rate used to infer the threshold. The false positive rate is the proportion of instances in `x` that are incorrectly classified as outliers. The false positive rate should be in the range ``(0, 1)``. + + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + NotFittedError + If called before detector has been fit. """ self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @@ -188,19 +188,19 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: x Data to predict. The shape of `x` should be `(n_instances, n_features)`. - Raises - ------ - NotFittedError - If called before detector has been fit. - ThresholdNotInferredError - If k is a list and a threshold was not inferred. - Returns ------- Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. + + Raises + ------ + NotFittedError + If called before detector has been fit. + ThresholdNotInferredError + If k is a list and a threshold was not inferred. """ outputs = self.backend.predict(self.backend._to_tensor(x)) output = outlier_prediction_dict() diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 1a8d42112..90752dfbd 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -101,15 +101,15 @@ def score(self, x: np.ndarray) -> np.ndarray: x Data to score. The shape of `x` should be `(n_instances, n_features)`. - Raises - ------ - NotFittedError - If called before detector has been fit. - Returns ------- Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more outlying the \ instance. + + Raises + ------ + NotFittedError + If called before detector has been fit. """ score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) @@ -121,13 +121,6 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the reference data as outliers. - Raises - ------ - ValueError - Raised if `fpr` is not in ``(0, 1)``. - NotFittedError - If called before detector has been fit. - Parameters ---------- x @@ -136,6 +129,13 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: False positive rate used to infer the threshold. The false positive rate is the proportion of instances in `x` that are incorrectly classified as outliers. The false positive rate should be in the range ``(0, 1)``. + + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + NotFittedError + If called before detector has been fit. """ self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @@ -150,17 +150,17 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: x Data to predict. The shape of `x` should be `(n_instances, n_features)`. - Raises - ------ - NotFittedError - If called before detector has been fit. - Returns ------- Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. + + Raises + ------ + NotFittedError + If called before detector has been fit. """ outputs = self.backend.predict(self.backend._to_tensor(x)) output = outlier_prediction_dict() diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index f0ddcbda7..15ba1ae88 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -216,14 +216,14 @@ def predict(self, x: torch.Tensor) -> TorchOutlierDetectorOutput: x Data to predict. + Returns + ------- + Output of the outlier detector. Includes the p-values, outlier labels, instance scores and threshold. + Raises ------ ValueError Raised if the detector is not fit on reference data. - - Returns - ------- - Output of the outlier detector. Includes the p-values, outlier labels, instance scores and threshold. """ self.check_fitted() # type: ignore raw_scores = self.score(x) From 8245edbfd4e7788aa395eeb91ab9480afd6ba01f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 10:51:57 +0100 Subject: [PATCH 176/247] Minor docstring changes --- alibi_detect/od/_mahalanobis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 90752dfbd..37d8359c5 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -31,10 +31,10 @@ def __init__( """ The Mahalanobis outlier detection method. - The Mahalanobis computes the directions of variation of a dataset and uses them to detect when points are - outliers by checking to see if the point vary from dataset points in unexpected ways. + The Mahalanobis detector computes the directions of variation of a dataset and uses them to detect when points + are outliers by checking to see if the points vary from dataset points in unexpected ways. - When we fit the Mahalanobis method we compute the covariance matrix of the reference data and its eigenvectors + When we fit the Mahalanobis detector we compute the covariance matrix of the reference data and its eigenvectors and eigenvalues. We filter small eigenvalues for numerical stability using the `min_eigenvalue` parameter. We then inversely weight each eigenvector by its eigenvalue. @@ -78,7 +78,7 @@ def __init__( def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. - Fitting the Mahalanobis method amounts to computing the covariance matrix and its eigenvectors. We filter out + Fitting the Mahalanobis detector amounts to computing the covariance matrix and its eigenvectors. We filter out very small eigenvalues using the `min_eigenvalue` parameter. We then scale the eigenvectors such that the data projected onto them has mean ``0`` and std ``1``. From 57abc3dc0d3fbfa550c68d46983d2dba87462f94 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 11:08:45 +0100 Subject: [PATCH 177/247] Remove cpu method call on tensor --- alibi_detect/od/pytorch/mahalanobis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 062e973dc..261424c52 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -48,7 +48,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: if not torch.jit.is_scripting(): self.check_threshold_inferred() preds = scores > self.threshold - return preds.cpu() + return preds @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: From f09979d8b6736cf8e5210123b109ce9a24aaaec6 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 11:09:17 +0100 Subject: [PATCH 178/247] Remove cpu method call on tensor 2 --- alibi_detect/od/pytorch/mahalanobis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 261424c52..ed1c8f507 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -71,7 +71,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: self.check_fitted() x = torch.as_tensor(x) x_pcs = self._compute_linear_proj(x) - return (x_pcs**2).sum(-1).cpu() + return (x_pcs**2).sum(-1) def fit(self, x_ref: torch.Tensor): """Fits the detector From c1690894111a7124acec4e1278204a9f7a36776c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 11:11:10 +0100 Subject: [PATCH 179/247] Remove as_tensor method on input to score --- alibi_detect/od/pytorch/mahalanobis.py | 1 - .../od/tests/test__mahalanobis/test__mahalanobis_backend.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index ed1c8f507..64ba0bd2a 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -69,7 +69,6 @@ def score(self, x: torch.Tensor) -> torch.Tensor: If called before detector has been fit. """ self.check_fitted() - x = torch.as_tensor(x) x_pcs = self._compute_linear_proj(x) return (x_pcs**2).sum(-1) diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py index a994e9950..66293f444 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis_backend.py @@ -39,7 +39,7 @@ def test_mahalanobis_linear_scoring(): # test that detector correctly detects true Outlier mahalanobis_torch.infer_threshold(x_ref, 0.01) - x = np.concatenate((x_1, x_2, x_3)) + x = torch.cat((x_1, x_2, x_3)) outputs = mahalanobis_torch.predict(x) assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) assert torch.all(mahalanobis_torch(x) == torch.tensor([False, False, True])) From e11bf5b81152896d437d8746b415b3b700ffdbe9 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 6 Apr 2023 11:15:17 +0100 Subject: [PATCH 180/247] Change principle -> principal --- alibi_detect/od/pytorch/mahalanobis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 64ba0bd2a..bcd17af26 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -85,7 +85,7 @@ def fit(self, x_ref: torch.Tensor): self._set_fitted() def _compute_linear_pcs(self, x: torch.Tensor): - """Computes the principle components of the data. + """Computes the principal components of the data. Parameters ---------- @@ -100,7 +100,7 @@ def _compute_linear_pcs(self, x: torch.Tensor): self.pcs = V[:, non_zero_inds] / D[None, non_zero_inds].sqrt() def _compute_linear_proj(self, x: torch.Tensor) -> torch.Tensor: - """Projects the data point being tested onto the principle components. + """Projects the data point being tested onto the principal components. Parameters ---------- From 3fe923c8811d7f0aae79b1f4e78417fe20376424 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 11:14:39 +0100 Subject: [PATCH 181/247] Fix broken tests --- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/pytorch/knn.py | 4 ++-- alibi_detect/od/pytorch/mahalanobis.py | 4 ++-- alibi_detect/od/pytorch/pca.py | 6 +++--- alibi_detect/od/tests/test__pca/test__pca.py | 4 ++-- alibi_detect/od/tests/test__pca/test__pca_backend.py | 8 ++++---- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 9216365eb..9226b99eb 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -27,7 +27,7 @@ def transform(self, x: torch.Tensor): """ raise NotImplementedError() - @torch.no_grad() + # @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: return self.transform(x) diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index cce26a749..6d54c474d 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -42,7 +42,7 @@ def __init__( self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) self.ensembler = ensembler - @torch.no_grad() + # @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -67,7 +67,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: preds = scores > self.threshold return preds - @torch.no_grad() + # @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index bcd17af26..cf56f7daa 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -26,7 +26,7 @@ def __init__( super().__init__(device=device) self.min_eigenvalue = min_eigenvalue - @torch.no_grad() + # @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -50,7 +50,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: preds = scores > self.threshold return preds - @torch.no_grad() + # @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 226194522..22b8a288a 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -65,12 +65,11 @@ def score(self, x: torch.Tensor) -> torch.Tensor: NotFitException If called before detector has been fit. """ - if not torch.jit.is_scripting(): - self.check_fitted() + self.check_fitted() score = self._compute_score(x) return score.cpu() - def _fit(self, x_ref: torch.Tensor) -> None: + def fit(self, x_ref: torch.Tensor) -> None: """Fits the PCA detector. Parameters @@ -81,6 +80,7 @@ def _fit(self, x_ref: torch.Tensor) -> None: self.x_ref_mean = x_ref.mean(0) self.pcs = self._compute_pcs(x_ref) self.x_ref = x_ref + self._set_fitted() def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: raise NotImplementedError diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index 92270bf8c..2f1c4d5a7 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -4,7 +4,7 @@ from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od._pca import PCA -from alibi_detect.base import NotFitException +from alibi_detect.exceptions import NotFittedError from sklearn.datasets import make_moons @@ -26,7 +26,7 @@ def make_PCA_detector(kernel=False): def test_unfitted_PCA_single_score(detector): pca = detector() x = np.array([[0, 10, 0], [0.1, 0, 0]]) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: _ = pca.predict(x) assert str(err.value) == \ f'{pca.backend.__class__.__name__} has not been fit!' diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py index 2a0613e95..9a9117cfb 100644 --- a/alibi_detect/od/tests/test__pca/test__pca_backend.py +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -4,7 +4,7 @@ from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.pca import LinearPCATorch, KernelPCATorch -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError @pytest.mark.parametrize('backend_detector', [ @@ -16,11 +16,11 @@ def test_pca_torch_backend_fit_errors(backend_detector): assert not pca_torch._fitted x = torch.randn((1, 10)) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: pca_torch(x) assert str(err.value) == f'{pca_torch.__class__.__name__} has not been fit!' - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: pca_torch.predict(x) assert str(err.value) == f'{pca_torch.__class__.__name__} has not been fit!' @@ -29,7 +29,7 @@ def test_pca_torch_backend_fit_errors(backend_detector): assert pca_torch._fitted - with pytest.raises(ThresholdNotInferredException) as err: + with pytest.raises(ThresholdNotInferredError) as err: pca_torch(x) assert str(err.value) == (f'{pca_torch.__class__.__name__} has no threshold set, call' ' `infer_threshold` before predicting.') From 40613656670f7af1e3318d99c03cda36d4612c25 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 11:21:25 +0100 Subject: [PATCH 182/247] Fix remaining tests --- alibi_detect/od/pytorch/pca.py | 2 ++ alibi_detect/od/tests/test__pca/test__pca_backend.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 22b8a288a..8550bbcb8 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -6,6 +6,8 @@ class PCATorch(TorchOutlierDetector): + ensemble = False + def __init__( self, n_components: int, diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py index 9a9117cfb..1360c13e8 100644 --- a/alibi_detect/od/tests/test__pca/test__pca_backend.py +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -13,7 +13,7 @@ ]) def test_pca_torch_backend_fit_errors(backend_detector): pca_torch = backend_detector() - assert not pca_torch._fitted + assert not pca_torch.fitted x = torch.randn((1, 10)) with pytest.raises(NotFittedError) as err: @@ -27,12 +27,12 @@ def test_pca_torch_backend_fit_errors(backend_detector): x_ref = torch.randn((1024, 10)) pca_torch.fit(x_ref) - assert pca_torch._fitted + assert pca_torch.fitted with pytest.raises(ThresholdNotInferredError) as err: pca_torch(x) - assert str(err.value) == (f'{pca_torch.__class__.__name__} has no threshold set, call' - ' `infer_threshold` before predicting.') + + assert str(err.value) == (f'{pca_torch.__class__.__name__} has no threshold set, call `infer_threshold` to fit one!') assert pca_torch.predict(x) From 54aecfd409133d2959e00ef8c41c3fdef3fa2e88 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 11:22:24 +0100 Subject: [PATCH 183/247] Fix linting error --- alibi_detect/od/tests/test__pca/test__pca_backend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py index 1360c13e8..0596d61a1 100644 --- a/alibi_detect/od/tests/test__pca/test__pca_backend.py +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -32,7 +32,8 @@ def test_pca_torch_backend_fit_errors(backend_detector): with pytest.raises(ThresholdNotInferredError) as err: pca_torch(x) - assert str(err.value) == (f'{pca_torch.__class__.__name__} has no threshold set, call `infer_threshold` to fit one!') + assert str(err.value) == (f'{pca_torch.__class__.__name__} has no threshold set, ' + 'call `infer_threshold` to fit one!') assert pca_torch.predict(x) From e3de2a49ab4b4a7d338ab5b0fb49934df9c6f857 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 15:07:42 +0100 Subject: [PATCH 184/247] Minor fix in _pca --- alibi_detect/od/_pca.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 21e497ffd..3103acd4f 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -25,8 +25,8 @@ def __init__( self, n_components: int, kernel: Optional[Callable] = None, - device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """Principal Component Analysis (PCA) outlier detector. @@ -45,14 +45,14 @@ def __init__( n_components: The number of dimensions in the principle subspace. For linear pca should have ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. - backend - Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. kernel Kernel function to use for outlier detection. If ``None``, linear PCA is used instead of the kernel variant. + backend + Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. Raises ------ @@ -118,7 +118,6 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: The threshold is set such that the false positive rate of the detector on the reference data is `fpr`. - Parameters ---------- x_ref From b415372ddb9949eb7747ba92534d1cfd518f995c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 15:53:29 +0100 Subject: [PATCH 185/247] MInor changes --- alibi_detect/od/pytorch/mahalanobis.py | 4 +-- alibi_detect/od/pytorch/pca.py | 42 +++++++++++++------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index cf56f7daa..87f38c0ab 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -20,8 +20,8 @@ def __init__( min_eigenvalue Eigenvectors with eigenvalues below this value will be discarded. device - Device type used. The default None tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. """ super().__init__(device=device) self.min_eigenvalue = min_eigenvalue diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 8550bbcb8..bb072523c 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -21,10 +21,10 @@ def __init__( The number of dimensions in the principle subspace. For linear PCA should have ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. device - Device type used. The default None tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. """ - TorchOutlierDetector.__init__(self, device=device) + super().__init__(device=device) self.n_components = n_components def forward(self, x: torch.Tensor) -> torch.Tensor: @@ -51,7 +51,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return preds.cpu() def score(self, x: torch.Tensor) -> torch.Tensor: - """Score test instance `x`. + """Computes the score of `x` Parameters ---------- @@ -60,7 +60,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - Tensor of scores for each element in `x`. + Tensor of scores for each element in `x`. Raises ------ @@ -68,7 +68,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: If called before detector has been fit. """ self.check_fitted() - score = self._compute_score(x) + score = self.compute_score(x) return score.cpu() def fit(self, x_ref: torch.Tensor) -> None: @@ -80,14 +80,14 @@ def fit(self, x_ref: torch.Tensor) -> None: The Dataset tensor. """ self.x_ref_mean = x_ref.mean(0) - self.pcs = self._compute_pcs(x_ref) + self.pcs = self.compute_pcs(x_ref) self.x_ref = x_ref self._set_fitted() - def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: raise NotImplementedError - def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + def compute_score(self, x: torch.Tensor) -> torch.Tensor: raise NotImplementedError @@ -104,12 +104,12 @@ def __init__( n_components: The number of dimensions in the principle subspace. device - Device type used. The default None tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. """ - PCATorch.__init__(self, device=device, n_components=n_components) + super().__init__(device=device, n_components=n_components) - def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: """Compute the principle components of the reference data. We compute the principle components of the reference data using the covariance matrix and then @@ -130,7 +130,7 @@ def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: _, V = torch.linalg.eigh(cov_mat) return V[:, :-self.n_components] - def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + def compute_score(self, x: torch.Tensor) -> torch.Tensor: """Compute the outlier score. Centers the data and projects it onto the principle components. The score is then the sum of the @@ -166,13 +166,13 @@ def __init__( kernel Kernel function to use for outlier detection. device - Device type used. The default None tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by + passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. """ - PCATorch.__init__(self, device=device, n_components=n_components) + super().__init__(device=device, n_components=n_components) self.kernel = kernel - def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: """Compute the principle components of the reference data. We compute the principle components of the reference data using the kernel matrix and then @@ -188,12 +188,12 @@ def _compute_pcs(self, x: torch.Tensor) -> torch.Tensor: ------- The principle components of the reference data. """ - K = self._compute_kernel_mat(x) + K = self.compute_kernel_mat(x) D, V = torch.linalg.eigh(K) pcs = V / torch.sqrt(D)[None, :] return pcs[:, -self.n_components:] - def _compute_score(self, x: torch.Tensor) -> torch.Tensor: + def compute_score(self, x: torch.Tensor) -> torch.Tensor: """Compute the outlier score. Centers the data and projects it onto the principle components. The score is then the sum of the @@ -217,7 +217,7 @@ def _compute_score(self, x: torch.Tensor) -> torch.Tensor: scores = -2 * k_xr.mean(-1) - (x_pcs**2).sum(1) return scores - def _compute_kernel_mat(self, x: torch.Tensor) -> torch.Tensor: + def compute_kernel_mat(self, x: torch.Tensor) -> torch.Tensor: """Computes the centered kernel matrix. Parameters From e73cbdcd0c50e931db6b43e52b0d8700dcf4a911 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 16:18:22 +0100 Subject: [PATCH 186/247] Surface correct errors --- alibi_detect/od/_pca.py | 15 +++++++++++++++ alibi_detect/od/tests/test__pca/test__pca.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 3103acd4f..fef75a876 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -9,6 +9,7 @@ from alibi_detect.od.pytorch import KernelPCATorch, LinearPCATorch from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ +from alibi_detect.exceptions import _catch_error as catch_error if TYPE_CHECKING: @@ -113,6 +114,7 @@ def score(self, x: np.ndarray) -> np.ndarray: score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) + @catch_error('NotFittedError') def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the Mahalanobis detector. @@ -126,9 +128,17 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ ``(0, 1)``. + + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + NotFittedError + If called before detector has been fit. """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + @catch_error('NotFittedError') def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. @@ -143,6 +153,11 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. + + Raises + ------ + NotFittedError + If called before detector has been fit. """ outputs = self.backend.predict(self.backend._to_tensor(x)) output = outlier_prediction_dict() diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index 2f1c4d5a7..f8fc308fa 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -29,7 +29,7 @@ def test_unfitted_PCA_single_score(detector): with pytest.raises(NotFittedError) as err: _ = pca.predict(x) assert str(err.value) == \ - f'{pca.backend.__class__.__name__} has not been fit!' + f'{pca.__class__.__name__} has not been fit!' def test_fitted_PCA_single_score(): From 94e95e4959c6f04572b39b83bb6164ea77009167 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 16:53:01 +0100 Subject: [PATCH 187/247] Fix tests for error surfacing --- alibi_detect/od/_pca.py | 1 + alibi_detect/od/tests/test__knn/test__knn.py | 11 ++++++++++ .../test__mahalanobis/test__mahalanobis.py | 5 +++++ alibi_detect/od/tests/test__pca/test__pca.py | 20 +++++++++++++++++-- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index fef75a876..558a41a29 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -96,6 +96,7 @@ def fit(self, x_ref: np.ndarray) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) + @catch_error('NotFittedError') def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index c5c9a5096..5f46ec213 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -24,6 +24,17 @@ def make_knn_detector(k=5, aggregator=None, normalizer=None): def test_unfitted_knn_single_score(): knn_detector = KNN(k=10) x = np.array([[0, 10], [0.1, 0]]) + x_ref = np.random.randn(100, 2) + + # test predict raises exception when not fitted + with pytest.raises(NotFittedError) as err: + _ = knn_detector.infer_threshold(x_ref, 0.1) + assert str(err.value) == 'KNN has not been fit!' + + # test predict raises exception when not fitted + with pytest.raises(NotFittedError) as err: + _ = knn_detector.score(x) + assert str(err.value) == 'KNN has not been fit!' # test predict raises exception when not fitted with pytest.raises(NotFittedError) as err: diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index cd11d10d6..6b3b1f97b 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -19,6 +19,11 @@ def test_unfitted_mahalanobis_single_score(): """Test Mahalanobis detector throws errors when not fitted.""" mahalanobis_detector = Mahalanobis() x = np.array([[0, 10], [0.1, 0]]) + x_ref = np.random.randn(100, 2) + + with pytest.raises(NotFittedError) as err: + mahalanobis_detector.infer_threshold(x_ref, 0.1) + assert str(err.value) == 'Mahalanobis has not been fit!' with pytest.raises(NotFittedError) as err: mahalanobis_detector.score(x) diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index f8fc308fa..b4f88d680 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -24,10 +24,26 @@ def make_PCA_detector(kernel=False): lambda: PCA(n_components=5, kernel=GaussianRBF()) ]) def test_unfitted_PCA_single_score(detector): + """Test pca detector throws errors when not fitted.""" pca = detector() - x = np.array([[0, 10, 0], [0.1, 0, 0]]) + x = np.array([[0, 10], [0.1, 0]]) + x_ref = np.random.randn(100, 2) + + # test infer_threshold raises exception when not fitted + with pytest.raises(NotFittedError) as err: + pca.infer_threshold(x_ref, 0.1) + assert str(err.value) == \ + f'{pca.__class__.__name__} has not been fit!' + + # test score raises exception when not fitted + with pytest.raises(NotFittedError) as err: + pca.score(x) + assert str(err.value) == \ + f'{pca.__class__.__name__} has not been fit!' + + # test predict raises exception when not fitted with pytest.raises(NotFittedError) as err: - _ = pca.predict(x) + pca.predict(x) assert str(err.value) == \ f'{pca.__class__.__name__} has not been fit!' From f5c67cc601a9f3afd5a109bd27460bfa14d2b2c8 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 16:53:27 +0100 Subject: [PATCH 188/247] Minor error fix --- alibi_detect/od/tests/test__knn/test__knn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index 5f46ec213..3d4061c1c 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -26,12 +26,12 @@ def test_unfitted_knn_single_score(): x = np.array([[0, 10], [0.1, 0]]) x_ref = np.random.randn(100, 2) - # test predict raises exception when not fitted + # test infer_threshold raises exception when not fitted with pytest.raises(NotFittedError) as err: _ = knn_detector.infer_threshold(x_ref, 0.1) assert str(err.value) == 'KNN has not been fit!' - # test predict raises exception when not fitted + # test score raises exception when not fitted with pytest.raises(NotFittedError) as err: _ = knn_detector.score(x) assert str(err.value) == 'KNN has not been fit!' From 67cb6706b31f21ad106a750107a22514a36c048b Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 17:17:46 +0100 Subject: [PATCH 189/247] Update tests and add docstrings --- alibi_detect/od/tests/test__pca/test__pca.py | 94 ++++++++++---------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index b4f88d680..c0f5d841d 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -8,11 +8,8 @@ from sklearn.datasets import make_moons -def make_PCA_detector(kernel=False): - if kernel: - pca_detector = PCA(n_components=2, kernel=GaussianRBF()) - else: - pca_detector = PCA(n_components=2) +def fit_PCA_detector(detector): + pca_detector = detector() x_ref = np.random.randn(100, 3) pca_detector.fit(x_ref) pca_detector.infer_threshold(x_ref, 0.1) @@ -48,28 +45,23 @@ def test_unfitted_PCA_single_score(detector): f'{pca.__class__.__name__} has not been fit!' -def test_fitted_PCA_single_score(): - pca_detector = PCA(n_components=2) +@pytest.mark.parametrize('detector', [ + lambda: PCA(n_components=2), + lambda: PCA(n_components=2, kernel=GaussianRBF()) +]) +def test_fitted_PCA_score(detector): + """Test Linear and Kernel PCA detector score method. + + Test Linear and Kernel PCA detector that has been fitted on reference data but has not had a threshold + inferred can still score data using the predict method. Test that it does not raise an error + and does not return `threshold`, `p_value` and `is_outlier` values. + """ + pca_detector = detector() x_ref = np.random.randn(100, 3) pca_detector.fit(x_ref) x = np.array([[0, 10, 0], [0.1, 0, 0]]) y = pca_detector.predict(x) y = y['data'] - assert y['instance_score'][0] > 5 - assert y['instance_score'][1] < 1 - assert not y['threshold_inferred'] - assert y['threshold'] is None - assert y['is_outlier'] is None - assert y['p_value'] is None - - -def test_fitted_kernel_PCA_single_score(): - pca_detector = PCA(n_components=2, kernel=GaussianRBF()) - x_ref = np.random.randn(100, 3) * np.array([1, 10, 0.1]) - pca_detector.fit(x_ref) - x = np.array([[0, 5, 10], [0.1, 5, 0]]) - y = pca_detector.predict(x) - y = y['data'] assert y['instance_score'][0] > y['instance_score'][1] assert not y['threshold_inferred'] assert y['threshold'] is None @@ -77,29 +69,23 @@ def test_fitted_kernel_PCA_single_score(): assert y['p_value'] is None -def test_fitted_PCA_predict(): - pca_detector = make_PCA_detector() +@pytest.mark.parametrize('detector', [ + lambda: PCA(n_components=2), + lambda: PCA(n_components=2, kernel=GaussianRBF()) +]) +def test_fitted_PCA_predict(detector): + """Test Linear and Kernel PCA detector predict method. + + Test Linear and Kernel PCA detector that has been fitted on reference data and has had a threshold + inferred can score data using the predict method. Test that it does not raise an error and does + return `threshold`, `p_value` and `is_outlier` values. + """ + pca_detector = fit_PCA_detector(detector) x_ref = np.random.randn(100, 3) pca_detector.infer_threshold(x_ref, 0.1) x = np.array([[0, 10, 0], [0.1, 0, 0]]) y = pca_detector.predict(x) y = y['data'] - assert y['instance_score'][0] > 5 - assert y['instance_score'][1] < 1 - assert y['threshold_inferred'] - assert y['threshold'] is not None - assert y['p_value'].all() - assert (y['is_outlier'] == [True, False]).all() - - -def test_fitted_kernel_PCA_predict(): - pca_detector = PCA(n_components=2, kernel=GaussianRBF()) - x_ref = np.random.randn(100, 3) * np.array([1, 10, 0.1]) - pca_detector.fit(x_ref) - pca_detector.infer_threshold(x_ref, 0.1) - x = np.array([[0, 5, 10], [0.1, 5, 0]]) - y = pca_detector.predict(x) - y = y['data'] assert y['instance_score'][0] > y['instance_score'][1] assert y['threshold_inferred'] assert y['threshold'] is not None @@ -107,7 +93,13 @@ def test_fitted_kernel_PCA_predict(): assert (y['is_outlier'] == [True, False]).all() -def test_PCA_integration(): +def test_PCA_integration(tmp_path): + """Test Linear PCA detector on moons dataset. + + Test the Linear PCA detector on a more complex 2d example. Test that the detector can be fitted + on reference data and infer a threshold. Test that it differentiates between inliers and outliers. + Test that the detector can be scripted. + """ pca_detector = PCA(n_components=1) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] @@ -122,21 +114,24 @@ def test_PCA_integration(): result = result['data']['is_outlier'][0] assert result - -def test_PCA_integration_ts(): - pca_detector = PCA(n_components=1) - X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) - X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] - pca_detector.fit(X_ref) - pca_detector.infer_threshold(X_ref, 0.1) - x_outlier = np.array([[0, -3]]) ts_PCA = torch.jit.script(pca_detector.backend) x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) y = ts_PCA(x) assert torch.all(y == torch.tensor([False, True])) + ts_PCA.save(tmp_path / 'pca.pt') + pca_detector = PCA(n_components=1) + pca_detector = torch.load(tmp_path / 'pca.pt') + y = ts_PCA(x) + assert torch.all(y == torch.tensor([False, True])) + def test_kernel_PCA_integration(): + """Test kernel PCA detector on moons dataset. + + Test the kernel PCA detector on a more complex 2d example. Test that the detector can be fitted + on reference data and infer a threshold. Test that it differentiates between inliers and outliers. + """ pca_detector = PCA(n_components=10, kernel=GaussianRBF()) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] @@ -154,6 +149,7 @@ def test_kernel_PCA_integration(): @pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript support yet.') def test_kernel_PCA_integration_ts(): + """Test the kernel PCA detector can be scripted.""" pca_detector = PCA(n_components=10, kernel=GaussianRBF()) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] From c5631ebf1a5263e5640e730dabc9ce088bb06954 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 17:21:16 +0100 Subject: [PATCH 190/247] Add save scrited detectors to temp_file in integration tests --- alibi_detect/od/tests/test__knn/test__knn.py | 22 ++++++++++++++----- .../test__mahalanobis/test__mahalanobis.py | 8 ++++++- alibi_detect/od/tests/test__pca/test__pca.py | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index 3d4061c1c..7e04f5e5f 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -203,7 +203,7 @@ def test_knn_single_torchscript(): lambda: 'MinAggregator']) @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None, lambda: 'ShiftAndScaleNormalizer', lambda: 'PValNormalizer']) -def test_knn_ensemble_integration(aggregator, normalizer): +def test_knn_ensemble_integration(tmp_path, aggregator, normalizer): """Test knn ensemble detector on moons dataset. Tests ensemble knn detector with every combination of aggregator and normalizer on the moons dataset. @@ -229,13 +229,18 @@ def test_knn_ensemble_integration(aggregator, normalizer): result = result['data']['is_outlier'][0] assert result - tsknn = torch.jit.script(knn_detector.backend) + ts_knn = torch.jit.script(knn_detector.backend) x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) - y = tsknn(x) + y = ts_knn(x) + assert torch.all(y == torch.tensor([False, True])) + + ts_knn.save(tmp_path / 'pca.pt') + knn_detector = torch.load(tmp_path / 'pca.pt') + y = knn_detector(x) assert torch.all(y == torch.tensor([False, True])) -def test_knn_integration(): +def test_knn_integration(tmp_path): """Test knn detector on moons dataset. Tests knn detector on the moons dataset. Fits and infers thresholds and verifies that the detector can @@ -255,7 +260,12 @@ def test_knn_integration(): result = result['data']['is_outlier'][0] assert result - tsknn = torch.jit.script(knn_detector.backend) + ts_knn = torch.jit.script(knn_detector.backend) x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) - y = tsknn(x) + y = ts_knn(x) assert torch.all(y == torch.tensor([False, True])) + + ts_knn.save(tmp_path / 'pca.pt') + knn_detector = torch.load(tmp_path / 'pca.pt') + y = knn_detector(x) + assert torch.all(y == torch.tensor([False, True])) \ No newline at end of file diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index 6b3b1f97b..4309c8a72 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -80,7 +80,7 @@ def test_fitted_mahalanobis_predict(): assert (y['is_outlier'] == [True, False]).all() -def test_mahalanobis_integration(): +def test_mahalanobis_integration(tmp_path): """Test Mahalanobis detector on moons dataset. Test Mahalanobis detector on a more complex 2d example. Test that the detector can be fitted @@ -105,3 +105,9 @@ def test_mahalanobis_integration(): x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) y = ts_mahalanobis(x) assert torch.all(y == torch.tensor([False, True])) + + ts_mahalanobis.save(tmp_path / 'pca.pt') + mahalanobis_detector = Mahalanobis() + mahalanobis_detector = torch.load(tmp_path / 'pca.pt') + y = mahalanobis_detector(x) + assert torch.all(y == torch.tensor([False, True])) diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index c0f5d841d..e590604b3 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -122,7 +122,7 @@ def test_PCA_integration(tmp_path): ts_PCA.save(tmp_path / 'pca.pt') pca_detector = PCA(n_components=1) pca_detector = torch.load(tmp_path / 'pca.pt') - y = ts_PCA(x) + y = pca_detector(x) assert torch.all(y == torch.tensor([False, True])) From e4a1798cf4cb81e160318c7b70ea46b6a2395a65 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 19 Apr 2023 17:28:28 +0100 Subject: [PATCH 191/247] Add docstrings fro backend pca tests --- .../od/tests/test__pca/test__pca_backend.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py index 0596d61a1..1cf6de217 100644 --- a/alibi_detect/od/tests/test__pca/test__pca_backend.py +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -12,6 +12,12 @@ lambda: KernelPCATorch(n_components=5, kernel=GaussianRBF()) ]) def test_pca_torch_backend_fit_errors(backend_detector): + """Test Linear and Kernel PCA detector backend fit errors. + + Test that an unfit detector backend raises an error when calling predict or score. Test that the + detector backend raises an error when calling the forward method while the threshold has not been + inferred. + """ pca_torch = backend_detector() assert not pca_torch.fitted @@ -43,6 +49,11 @@ def test_pca_torch_backend_fit_errors(backend_detector): lambda: KernelPCATorch(n_components=1, kernel=GaussianRBF()) ]) def test_pca_linear_scoring(backend_detector): + """Test Linear and Kernel PCATorch detector backend scoring methods. + + Test that the detector correctly detects true outliers and that the correct proportion of in + distribution data is flagged as outliers. + """ pca_torch = backend_detector() mean = [8, 8] cov = [[2., 0.], [0., 1.]] @@ -75,6 +86,7 @@ def test_pca_linear_scoring(backend_detector): def test_pca_linear_torch_backend_ts(tmp_path): + """Test Linear PCA detector backend is torch-scriptable and savable.""" pca_torch = LinearPCATorch(n_components=5) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) x_ref = torch.randn((1024, 10)) @@ -94,6 +106,7 @@ def test_pca_linear_torch_backend_ts(tmp_path): @pytest.mark.skip(reason='GaussianRBF kernel does not have torchscript support yet.') def test_pca_kernel_torch_backend_ts(tmp_path): + """Test Kernel PCA detector backend is torch-scriptable and savable.""" pca_torch = KernelPCATorch(n_components=5, kernel=GaussianRBF()) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) x_ref = torch.randn((1024, 10)) From 7378f6987aae21276d2b5c1681b3e6b251c9054f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 20 Apr 2023 10:00:17 +0100 Subject: [PATCH 192/247] Minor fix --- alibi_detect/od/tests/test__knn/test__knn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index 7e04f5e5f..c2144da48 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -268,4 +268,4 @@ def test_knn_integration(tmp_path): ts_knn.save(tmp_path / 'pca.pt') knn_detector = torch.load(tmp_path / 'pca.pt') y = knn_detector(x) - assert torch.all(y == torch.tensor([False, True])) \ No newline at end of file + assert torch.all(y == torch.tensor([False, True])) From 944d5efcf8840f1041a04ff1731d4c320be32c8a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 20 Apr 2023 11:11:54 +0100 Subject: [PATCH 193/247] Update docstrings on PCA detector --- alibi_detect/od/_pca.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 558a41a29..86473c24e 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -100,7 +100,7 @@ def fit(self, x_ref: np.ndarray) -> None: def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. - Project `x` onto the eigenvectors and compute its score using the L2 norm. + Project `x` onto the eigenvectors and compute the score using the L2 norm. Parameters ---------- @@ -111,24 +111,30 @@ def score(self, x: np.ndarray) -> np.ndarray: ------- Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. + + Raises + ------ + NotFittedError + If called before detector has been fit. """ score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) @catch_error('NotFittedError') def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: - """Infer the threshold for the Mahalanobis detector. + """Infer the threshold for the kNN detector. - The threshold is set such that the false positive rate of the detector on the reference data is `fpr`. + The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the + reference data as outliers. Parameters ---------- - x_ref + x Reference data used to infer the threshold. fpr - False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ - `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ - ``(0, 1)``. + False positive rate used to infer the threshold. The false positive rate is the proportion of + instances in `x` that are incorrectly classified as outliers. The false positive rate should + be in the range ``(0, 1)``. Raises ------ @@ -143,6 +149,8 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. + Scores the instances in `x` and if the threshold was inferred, returns the outlier labels and p-values as well. + Parameters ---------- x @@ -151,7 +159,7 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: Returns ------- Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ - performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. From 278b6196805111fa0b47ec5d138d267fafd8a46f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 20 Apr 2023 11:17:32 +0100 Subject: [PATCH 194/247] Revert minor change --- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/pytorch/knn.py | 4 ++-- alibi_detect/od/pytorch/mahalanobis.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 9226b99eb..9216365eb 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -27,7 +27,7 @@ def transform(self, x: torch.Tensor): """ raise NotImplementedError() - # @torch.no_grad() + @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: return self.transform(x) diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index 6d54c474d..cce26a749 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -42,7 +42,7 @@ def __init__( self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) self.ensembler = ensembler - # @torch.no_grad() + @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -67,7 +67,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: preds = scores > self.threshold return preds - # @torch.no_grad() + @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index 87f38c0ab..71fca5ab5 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -26,7 +26,7 @@ def __init__( super().__init__(device=device) self.min_eigenvalue = min_eigenvalue - # @torch.no_grad() + @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -50,7 +50,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: preds = scores > self.threshold return preds - # @torch.no_grad() + @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` From 83d834640e7ea50f492b7da71bda900e998aacec Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 20 Apr 2023 11:29:58 +0100 Subject: [PATCH 195/247] Update pca backend docstrings --- alibi_detect/od/pytorch/pca.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index bb072523c..9bd2dc6ba 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -123,7 +123,7 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - The principle components of the reference data. + The principle components of the reference data. """ x -= self.x_ref_mean cov_mat = (x.t() @ x)/(len(x)-1) @@ -143,7 +143,7 @@ def compute_score(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - The outlier score. + The outlier score. """ x_cen = x - self.x_ref_mean x_pcs = x_cen @ self.pcs @@ -186,7 +186,7 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - The principle components of the reference data. + The principle components of the reference data. """ K = self.compute_kernel_mat(x) D, V = torch.linalg.eigh(K) @@ -206,10 +206,9 @@ def compute_score(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - The outlier score. + The outlier score. """ k_xr = self.kernel(x, self.x_ref) - # Now to center k_xr_row_sums = k_xr.sum(1) _, n = k_xr.shape k_xr_cen = k_xr - self.k_col_sums[None, :]/n - k_xr_row_sums[:, None]/n + self.k_sum/(n**2) @@ -227,12 +226,10 @@ def compute_kernel_mat(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - The centered kernel matrix. + The centered kernel matrix. """ n = len(x) - # Uncentered kernel matrix k = self.kernel(x, x) - # Now to center self.k_col_sums = k.sum(0) k_row_sums = k.sum(1) self.k_sum = k_row_sums.sum() From fb6bb1819c04c4b862ceb364b3d0a9d167d1c6a3 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 12:01:49 +0100 Subject: [PATCH 196/247] Add and raise errors for incorrect n_components values --- alibi_detect/od/_pca.py | 8 +++++++ alibi_detect/od/pytorch/pca.py | 24 ++++++++++++++++++++ alibi_detect/od/tests/test__pca/test__pca.py | 24 ++++++++++++++++---- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 86473c24e..bb4d68c31 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -59,6 +59,8 @@ def __init__( ------ NotImplementedError If choice of `backend` is not implemented. + ValueError + If `n_components` is less than 1. """ super().__init__() @@ -93,6 +95,12 @@ def fit(self, x_ref: np.ndarray) -> None: ---------- x_ref Reference data used to fit the detector. + + Raises + ------ + ValueError + If using linear pca variant and `n_components` is greater than or equal to number of features or if + using kernel pca variant and `n_components` is greater than or equal to number of instances. """ self.backend.fit(self.backend._to_tensor(x_ref)) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 9bd2dc6ba..f35dfeeb1 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -23,10 +23,18 @@ def __init__( device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. + + Raises + ------ + ValueError + If `n_components` is less than 1. """ super().__init__(device=device) self.n_components = n_components + if n_components < 1: + raise ValueError('n_components must be at least 1') + def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -124,7 +132,15 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: Returns ------- The principle components of the reference data. + + Raises + ------ + ValueError + If `n_components` is greater than or equal to number of features """ + if self.n_components >= x.shape[1]: + raise ValueError("n_components must be less than the number of features.") + x -= self.x_ref_mean cov_mat = (x.t() @ x)/(len(x)-1) _, V = torch.linalg.eigh(cov_mat) @@ -187,7 +203,15 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: Returns ------- The principle components of the reference data. + + Raises + ------ + ValueError + If `n_components` is greater than or equal to the number of reference samples. """ + if self.n_components >= x.shape[0]: + raise ValueError("n_components must be less than the number of reference instances.") + K = self.compute_kernel_mat(x) D, V = torch.linalg.eigh(K) pcs = V / torch.sqrt(D)[None, :] diff --git a/alibi_detect/od/tests/test__pca/test__pca.py b/alibi_detect/od/tests/test__pca/test__pca.py index e590604b3..0381a93eb 100644 --- a/alibi_detect/od/tests/test__pca/test__pca.py +++ b/alibi_detect/od/tests/test__pca/test__pca.py @@ -17,14 +17,14 @@ def fit_PCA_detector(detector): @pytest.mark.parametrize('detector', [ - lambda: PCA(n_components=5), - lambda: PCA(n_components=5, kernel=GaussianRBF()) + lambda: PCA(n_components=3), + lambda: PCA(n_components=3, kernel=GaussianRBF()) ]) def test_unfitted_PCA_single_score(detector): """Test pca detector throws errors when not fitted.""" pca = detector() - x = np.array([[0, 10], [0.1, 0]]) - x_ref = np.random.randn(100, 2) + x = np.array([[0, 10, 11], [0.1, 0, 11]]) + x_ref = np.random.randn(100, 3) # test infer_threshold raises exception when not fitted with pytest.raises(NotFittedError) as err: @@ -45,6 +45,22 @@ def test_unfitted_PCA_single_score(detector): f'{pca.__class__.__name__} has not been fit!' +def test_pca_value_errors(): + with pytest.raises(ValueError) as err: + PCA(n_components=0) + assert str(err.value) == 'n_components must be at least 1' + + with pytest.raises(ValueError) as err: + pca = PCA(n_components=4) + pca.fit(np.random.randn(100, 3)) + assert str(err.value) == 'n_components must be less than the number of features.' + + with pytest.raises(ValueError) as err: + pca = PCA(n_components=10, kernel=GaussianRBF()) + pca.fit(np.random.randn(9, 3)) + assert str(err.value) == 'n_components must be less than the number of reference instances.' + + @pytest.mark.parametrize('detector', [ lambda: PCA(n_components=2), lambda: PCA(n_components=2, kernel=GaussianRBF()) From e2d62245076304289685aee5cc4e916b8cecadd7 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 12:12:05 +0100 Subject: [PATCH 197/247] Fix inccorect docstrings for pca --- alibi_detect/od/_pca.py | 20 ++++++++++++++++---- alibi_detect/od/pytorch/pca.py | 11 +++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index bb4d68c31..a3cb2bb72 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -88,8 +88,19 @@ def __init__( def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. - Compute the eigenvectors of the covariance/kernel matrix of `x_ref` and save the smallest `n_components` - eigenvectors. + In the linear case we compute the principle components of the reference data using the + covariance matrix and then remove the largest `n_components` eigenvectors. The remaining + eigenvectors correspond to the invariant dimensions of the data. Changes in these + dimensions are used to compute the outlier score which is the distance to the principle + subspace spanned by the first `n_components` eigenvectors. + + In the kernel Case we compute the principle components of the reference data using the + kernel matrix and then return the largest `n_components` eigenvectors. These are then + normalized to have length equal to `1/eigenvalue`. Note that this differs from the + linear case where we remove the largest eigenvectors. + + In both cases we then store the computed principle components and use later when we score + test instances. Parameters ---------- @@ -99,8 +110,9 @@ def fit(self, x_ref: np.ndarray) -> None: Raises ------ ValueError - If using linear pca variant and `n_components` is greater than or equal to number of features or if - using kernel pca variant and `n_components` is greater than or equal to number of instances. + If using linear pca variant and `n_components` is greater than or equal to number of + features or if using kernel pca variant and `n_components` is greater than or equal + to number of instances. """ self.backend.fit(self.backend._to_tensor(x_ref)) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index f35dfeeb1..02da9846d 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -121,8 +121,10 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: """Compute the principle components of the reference data. We compute the principle components of the reference data using the covariance matrix and then - return the last `n_components` of the eigenvectors. These correspond to the invariant dimensions - of the data. Changes in these dimensions are used to compute the outlier score. + remove the largest `n_components` eigenvectors. The remaining eigenvectors correspond to the + invariant dimensions of the data. Changes in these dimensions are used to compute the outlier + score which is the distance to the principle subspace spanned by the first `n_components` + eigenvectors. Parameters ---------- @@ -192,8 +194,9 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: """Compute the principle components of the reference data. We compute the principle components of the reference data using the kernel matrix and then - return the last `n_components` of the eigenvectors. These correspond to the invariant dimensions - of the data. Changes in these dimensions are used to compute the outlier score. + return the largest `n_components` eigenvectors. These are then normalized to have length + equal to `1/eigenvalue`. Note that this differs from the linear case where we remove the + largest eigenvectors. Parameters ---------- From cd99ec299c06c54f8757c981cf9b09fdab750fc4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 12:16:06 +0100 Subject: [PATCH 198/247] Fix minor issue --- alibi_detect/od/_pca.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index a3cb2bb72..ef4b7fcca 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -141,7 +141,7 @@ def score(self, x: np.ndarray) -> np.ndarray: return self.backend._to_numpy(score) @catch_error('NotFittedError') - def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + def infer_threshold(self, x: np.ndarray, fpr: float) -> None: """Infer the threshold for the kNN detector. The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the @@ -163,7 +163,7 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: NotFittedError If called before detector has been fit. """ - self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @catch_error('NotFittedError') def predict(self, x: np.ndarray) -> Dict[str, Any]: From 2cb77964ecb380ed51bb102e55074f7d6e8dbef8 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 12:17:08 +0100 Subject: [PATCH 199/247] Fix incorrect detector name in docstring --- alibi_detect/od/_pca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index ef4b7fcca..0eacc99ff 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -142,7 +142,7 @@ def score(self, x: np.ndarray) -> np.ndarray: @catch_error('NotFittedError') def infer_threshold(self, x: np.ndarray, fpr: float) -> None: - """Infer the threshold for the kNN detector. + """Infer the threshold for the PCA detector. The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the reference data as outliers. From 3ccca074e1f0c03a274ff165e07f3ad236bb72f0 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 13:36:23 +0100 Subject: [PATCH 200/247] Remove redundant storage and computation in fit step --- alibi_detect/od/_pca.py | 14 ++++++++------ alibi_detect/od/pytorch/pca.py | 12 ++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 0eacc99ff..c7924b28c 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -33,13 +33,15 @@ def __init__( The detector is based on the Principal Component Analysis (PCA) algorithm. There are two variants of PCA: linear PCA and kernel PCA. Linear PCA computes the eigenvectors of the covariance matrix of the data. Kernel - PCA computes the eigenvectors of the kernel matrix of the data. In each case, we choose the smallest - `n_components` eigenvectors. We do this as they correspond to the invariant directions of the data. i.e the - directions along which the data is least spread out. Thus a point that deviates along these dimensions is more - likely to be an outlier. + PCA computes the eigenvectors of the kernel matrix of the data. - When scoring a test instance we project it onto the eigenvectors and compute its score using the L2 norm. If - a threshold is fitted we use this to determine whether the instance is an outlier or not. + When scoring a test instance using the linear variant compute the distance to the principle subspace spanned + by the first `n_components` eigenvectors. + + When scoring a test instance using the kernel variant we project it onto the largest eigenvectors and + compute its score using the L2 norm. + + If a threshold is fitted we use this to determine whether the instance is an outlier or not. Parameters ---------- diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 02da9846d..d34896050 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -87,12 +87,10 @@ def fit(self, x_ref: torch.Tensor) -> None: x_ref The Dataset tensor. """ - self.x_ref_mean = x_ref.mean(0) - self.pcs = self.compute_pcs(x_ref) - self.x_ref = x_ref + self.pcs = self._fit(x_ref) self._set_fitted() - def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + def _fit(self, x: torch.Tensor) -> torch.Tensor: raise NotImplementedError def compute_score(self, x: torch.Tensor) -> torch.Tensor: @@ -117,7 +115,7 @@ def __init__( """ super().__init__(device=device, n_components=n_components) - def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + def _fit(self, x: torch.Tensor) -> torch.Tensor: """Compute the principle components of the reference data. We compute the principle components of the reference data using the covariance matrix and then @@ -143,6 +141,7 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: if self.n_components >= x.shape[1]: raise ValueError("n_components must be less than the number of features.") + self.x_ref_mean = x.mean(0) x -= self.x_ref_mean cov_mat = (x.t() @ x)/(len(x)-1) _, V = torch.linalg.eigh(cov_mat) @@ -190,7 +189,7 @@ def __init__( super().__init__(device=device, n_components=n_components) self.kernel = kernel - def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: + def _fit(self, x: torch.Tensor) -> torch.Tensor: """Compute the principle components of the reference data. We compute the principle components of the reference data using the kernel matrix and then @@ -215,6 +214,7 @@ def compute_pcs(self, x: torch.Tensor) -> torch.Tensor: if self.n_components >= x.shape[0]: raise ValueError("n_components must be less than the number of reference instances.") + self.x_ref = x K = self.compute_kernel_mat(x) D, V = torch.linalg.eigh(K) pcs = V / torch.sqrt(D)[None, :] From d2175dda275a4c359f88642d3d3e201053872b6d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 13:40:32 +0100 Subject: [PATCH 201/247] Rename incorrectly named save files in tests --- alibi_detect/od/tests/test__knn/test__knn.py | 8 ++++---- .../od/tests/test__mahalanobis/test__mahalanobis.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn.py b/alibi_detect/od/tests/test__knn/test__knn.py index c2144da48..7bf3f324f 100644 --- a/alibi_detect/od/tests/test__knn/test__knn.py +++ b/alibi_detect/od/tests/test__knn/test__knn.py @@ -234,8 +234,8 @@ def test_knn_ensemble_integration(tmp_path, aggregator, normalizer): y = ts_knn(x) assert torch.all(y == torch.tensor([False, True])) - ts_knn.save(tmp_path / 'pca.pt') - knn_detector = torch.load(tmp_path / 'pca.pt') + ts_knn.save(tmp_path / 'knn.pt') + knn_detector = torch.load(tmp_path / 'knn.pt') y = knn_detector(x) assert torch.all(y == torch.tensor([False, True])) @@ -265,7 +265,7 @@ def test_knn_integration(tmp_path): y = ts_knn(x) assert torch.all(y == torch.tensor([False, True])) - ts_knn.save(tmp_path / 'pca.pt') - knn_detector = torch.load(tmp_path / 'pca.pt') + ts_knn.save(tmp_path / 'knn.pt') + knn_detector = torch.load(tmp_path / 'knn.pt') y = knn_detector(x) assert torch.all(y == torch.tensor([False, True])) diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index 4309c8a72..255dc53fb 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -106,8 +106,8 @@ def test_mahalanobis_integration(tmp_path): y = ts_mahalanobis(x) assert torch.all(y == torch.tensor([False, True])) - ts_mahalanobis.save(tmp_path / 'pca.pt') + ts_mahalanobis.save(tmp_path / 'mahalanobis.pt') mahalanobis_detector = Mahalanobis() - mahalanobis_detector = torch.load(tmp_path / 'pca.pt') + mahalanobis_detector = torch.load(tmp_path / 'mahalanobis.pt') y = mahalanobis_detector(x) assert torch.all(y == torch.tensor([False, True])) From abd270d42a88febe3ca1343219800e489564544d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 14:40:42 +0100 Subject: [PATCH 202/247] Fix issue in centering simialrity matrix --- alibi_detect/od/pytorch/pca.py | 4 ++-- alibi_detect/od/tests/test__pca/test__pca_backend.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index d34896050..bf513abd9 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -237,8 +237,8 @@ def compute_score(self, x: torch.Tensor) -> torch.Tensor: """ k_xr = self.kernel(x, self.x_ref) k_xr_row_sums = k_xr.sum(1) - _, n = k_xr.shape - k_xr_cen = k_xr - self.k_col_sums[None, :]/n - k_xr_row_sums[:, None]/n + self.k_sum/(n**2) + n, m = k_xr.shape + k_xr_cen = k_xr - self.k_col_sums[None, :]/m - k_xr_row_sums[:, None]/n + self.k_sum/(m*n) x_pcs = k_xr_cen @ self.pcs scores = -2 * k_xr.mean(-1) - (x_pcs**2).sum(1) return scores diff --git a/alibi_detect/od/tests/test__pca/test__pca_backend.py b/alibi_detect/od/tests/test__pca/test__pca_backend.py index 1cf6de217..ac937eb63 100644 --- a/alibi_detect/od/tests/test__pca/test__pca_backend.py +++ b/alibi_detect/od/tests/test__pca/test__pca_backend.py @@ -48,7 +48,7 @@ def test_pca_torch_backend_fit_errors(backend_detector): lambda: LinearPCATorch(n_components=1), lambda: KernelPCATorch(n_components=1, kernel=GaussianRBF()) ]) -def test_pca_linear_scoring(backend_detector): +def test_pca_scoring(backend_detector): """Test Linear and Kernel PCATorch detector backend scoring methods. Test that the detector correctly detects true outliers and that the correct proportion of in @@ -63,10 +63,10 @@ def test_pca_linear_scoring(backend_detector): x_1 = torch.tensor([[8., 8.]], dtype=torch.float64) scores_1 = pca_torch.score(x_1) - x_2 = torch.tensor(np.random.multivariate_normal(mean, cov, 1), dtype=torch.float64) + x_2 = torch.tensor([[10., 8.]], dtype=torch.float64) scores_2 = pca_torch.score(x_2) - x_3 = torch.tensor([[-20., 20.]], dtype=torch.float64) + x_3 = torch.tensor([[8., 20.]], dtype=torch.float64) scores_3 = pca_torch.score(x_3) # test correct ordering of scores given outlyingness of data From 90bc9fc906f6cabf374309f6c587c5fa0ea55d82 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 26 Apr 2023 15:01:35 +0100 Subject: [PATCH 203/247] Minor fix --- alibi_detect/od/_pca.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index c7924b28c..6fe6c409d 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -101,8 +101,8 @@ def fit(self, x_ref: np.ndarray) -> None: normalized to have length equal to `1/eigenvalue`. Note that this differs from the linear case where we remove the largest eigenvectors. - In both cases we then store the computed principle components and use later when we score - test instances. + In both cases we then store the computed components and use later when we score test + instances. Parameters ---------- From 665c8ae59bc38c2f4471e9cc2c07e3bb650e43f1 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 27 Apr 2023 14:16:30 +0100 Subject: [PATCH 204/247] Update device types --- alibi_detect/od/_pca.py | 6 +++--- alibi_detect/od/pytorch/base.py | 6 +++++- alibi_detect/od/pytorch/knn.py | 4 ++-- alibi_detect/od/pytorch/mahalanobis.py | 4 ++-- alibi_detect/od/pytorch/pca.py | 8 ++++---- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 6fe6c409d..3599c7da3 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -1,9 +1,9 @@ from typing import Union, Optional, Callable, Dict, Any from typing import TYPE_CHECKING +from typing_extensions import Literal import numpy as np -from alibi_detect.utils._types import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.base import BaseDetector, ThresholdMixin, FitMixin from alibi_detect.od.pytorch import KernelPCATorch, LinearPCATorch @@ -96,12 +96,12 @@ def fit(self, x_ref: np.ndarray) -> None: dimensions are used to compute the outlier score which is the distance to the principle subspace spanned by the first `n_components` eigenvectors. - In the kernel Case we compute the principle components of the reference data using the + In the kernel case we compute the principle components of the reference data using the kernel matrix and then return the largest `n_components` eigenvectors. These are then normalized to have length equal to `1/eigenvalue`. Note that this differs from the linear case where we remove the largest eigenvectors. - In both cases we then store the computed components and use later when we score test + In both cases we then store the computed components to use later when we score test instances. Parameters diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index 15ba1ae88..37f0098a2 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -1,4 +1,5 @@ from typing import List, Union, Optional, Dict +from typing_extensions import Literal from dataclasses import dataclass, asdict from abc import ABC, abstractmethod @@ -60,7 +61,10 @@ class TorchOutlierDetector(torch.nn.Module, FitMixinTorch, ABC): threshold_inferred = False threshold = None - def __init__(self, device: Optional[Union[str, torch.device]] = None): + def __init__( + self, + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, + ): self.device = get_device(device) super().__init__() diff --git a/alibi_detect/od/pytorch/knn.py b/alibi_detect/od/pytorch/knn.py index 6171a594e..556930b3f 100644 --- a/alibi_detect/od/pytorch/knn.py +++ b/alibi_detect/od/pytorch/knn.py @@ -1,5 +1,5 @@ from typing import Optional, Union, List, Tuple - +from typing_extensions import Literal import numpy as np import torch @@ -13,7 +13,7 @@ def __init__( k: Union[np.ndarray, List, Tuple, int], kernel: Optional[torch.nn.Module] = None, ensembler: Optional[Ensembler] = None, - device: Optional[Union[str, torch.device]] = None + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ): """PyTorch backend for KNN detector. diff --git a/alibi_detect/od/pytorch/mahalanobis.py b/alibi_detect/od/pytorch/mahalanobis.py index f6ca10625..a51e4a9b6 100644 --- a/alibi_detect/od/pytorch/mahalanobis.py +++ b/alibi_detect/od/pytorch/mahalanobis.py @@ -1,5 +1,5 @@ from typing import Optional, Union - +from typing_extensions import Literal import torch from alibi_detect.od.pytorch.base import TorchOutlierDetector @@ -11,7 +11,7 @@ class MahalanobisTorch(TorchOutlierDetector): def __init__( self, min_eigenvalue: float = 1e-6, - device: Optional[Union[str, torch.device]] = None + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ): """PyTorch backend for Mahalanobis detector. diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index bf513abd9..c34409b23 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -1,4 +1,4 @@ -from typing import Optional, Union, Callable +from typing import Optional, Union, Callable, Literal import torch @@ -11,7 +11,7 @@ class PCATorch(TorchOutlierDetector): def __init__( self, n_components: int, - device: Optional[Union[str, torch.device]] = None + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ): """PyTorch backend for PCA detector. @@ -101,7 +101,7 @@ class LinearPCATorch(PCATorch): def __init__( self, n_components: int, - device: Optional[Union[str, torch.device]] = None + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ): """Linear variant of the PyTorch backend for PCA detector. @@ -172,7 +172,7 @@ def __init__( self, n_components: int, kernel: Optional[Callable], - device: Optional[Union[str, torch.device]] = None + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ): """Kernel variant of the PyTorch backend for PCA detector. From 7eefb93e1de020486113ad270ab6bc4288f1e415 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 27 Apr 2023 14:18:22 +0100 Subject: [PATCH 205/247] Import Literals from typing_extensions instead of internal types --- alibi_detect/od/_knn.py | 2 +- alibi_detect/od/_mahalanobis.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/_knn.py b/alibi_detect/od/_knn.py index dfa9edf74..b893b032d 100644 --- a/alibi_detect/od/_knn.py +++ b/alibi_detect/od/_knn.py @@ -1,9 +1,9 @@ from typing import Callable, Union, Optional, Dict, Any, List, Tuple from typing import TYPE_CHECKING +from typing_extensions import Literal import numpy as np -from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict from alibi_detect.exceptions import _catch_error as catch_error from alibi_detect.od.base import TransformProtocol, TransformProtocolType diff --git a/alibi_detect/od/_mahalanobis.py b/alibi_detect/od/_mahalanobis.py index 37d8359c5..763f59898 100644 --- a/alibi_detect/od/_mahalanobis.py +++ b/alibi_detect/od/_mahalanobis.py @@ -1,11 +1,10 @@ from typing import Union, Optional, Dict, Any from typing import TYPE_CHECKING from alibi_detect.exceptions import _catch_error as catch_error - +from typing_extensions import Literal import numpy as np -from alibi_detect.utils._types import Literal from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin, outlier_prediction_dict from alibi_detect.od.pytorch import MahalanobisTorch from alibi_detect.utils.frameworks import BackendValidator From c4b3719775130ea29932139caa4dc821476f4b98 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 27 Apr 2023 14:23:44 +0100 Subject: [PATCH 206/247] Remove cpu calls from backends --- alibi_detect/od/pytorch/pca.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index c34409b23..99725c255 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -56,7 +56,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: if not torch.jit.is_scripting(): self.check_threshold_inferred() preds = scores > self.threshold - return preds.cpu() + return preds def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` @@ -77,7 +77,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: """ self.check_fitted() score = self.compute_score(x) - return score.cpu() + return score def fit(self, x_ref: torch.Tensor) -> None: """Fits the PCA detector. From 3ea402dddc5dbff82f8c3895cac5a9dd4d2b7d98 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 27 Apr 2023 14:25:12 +0100 Subject: [PATCH 207/247] Rename compute_score to _score --- alibi_detect/od/pytorch/pca.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 99725c255..7808b0e6b 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -76,7 +76,7 @@ def score(self, x: torch.Tensor) -> torch.Tensor: If called before detector has been fit. """ self.check_fitted() - score = self.compute_score(x) + score = self._score(x) return score def fit(self, x_ref: torch.Tensor) -> None: @@ -93,7 +93,7 @@ def fit(self, x_ref: torch.Tensor) -> None: def _fit(self, x: torch.Tensor) -> torch.Tensor: raise NotImplementedError - def compute_score(self, x: torch.Tensor) -> torch.Tensor: + def _score(self, x: torch.Tensor) -> torch.Tensor: raise NotImplementedError @@ -147,7 +147,7 @@ def _fit(self, x: torch.Tensor) -> torch.Tensor: _, V = torch.linalg.eigh(cov_mat) return V[:, :-self.n_components] - def compute_score(self, x: torch.Tensor) -> torch.Tensor: + def _score(self, x: torch.Tensor) -> torch.Tensor: """Compute the outlier score. Centers the data and projects it onto the principle components. The score is then the sum of the @@ -220,7 +220,7 @@ def _fit(self, x: torch.Tensor) -> torch.Tensor: pcs = V / torch.sqrt(D)[None, :] return pcs[:, -self.n_components:] - def compute_score(self, x: torch.Tensor) -> torch.Tensor: + def _score(self, x: torch.Tensor) -> torch.Tensor: """Compute the outlier score. Centers the data and projects it onto the principle components. The score is then the sum of the From 3ceffbcca1939252842571fa18bac017f3da57e3 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 27 Apr 2023 14:28:05 +0100 Subject: [PATCH 208/247] Change principle -> principal --- alibi_detect/od/_pca.py | 10 +++++----- alibi_detect/od/pytorch/pca.py | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/alibi_detect/od/_pca.py b/alibi_detect/od/_pca.py index 3599c7da3..e14edcac6 100644 --- a/alibi_detect/od/_pca.py +++ b/alibi_detect/od/_pca.py @@ -35,7 +35,7 @@ def __init__( linear PCA and kernel PCA. Linear PCA computes the eigenvectors of the covariance matrix of the data. Kernel PCA computes the eigenvectors of the kernel matrix of the data. - When scoring a test instance using the linear variant compute the distance to the principle subspace spanned + When scoring a test instance using the linear variant compute the distance to the principal subspace spanned by the first `n_components` eigenvectors. When scoring a test instance using the kernel variant we project it onto the largest eigenvectors and @@ -46,7 +46,7 @@ def __init__( Parameters ---------- n_components: - The number of dimensions in the principle subspace. For linear pca should have + The number of dimensions in the principal subspace. For linear pca should have ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. kernel Kernel function to use for outlier detection. If ``None``, linear PCA is used instead of the @@ -90,13 +90,13 @@ def __init__( def fit(self, x_ref: np.ndarray) -> None: """Fit the detector on reference data. - In the linear case we compute the principle components of the reference data using the + In the linear case we compute the principal components of the reference data using the covariance matrix and then remove the largest `n_components` eigenvectors. The remaining eigenvectors correspond to the invariant dimensions of the data. Changes in these - dimensions are used to compute the outlier score which is the distance to the principle + dimensions are used to compute the outlier score which is the distance to the principal subspace spanned by the first `n_components` eigenvectors. - In the kernel case we compute the principle components of the reference data using the + In the kernel case we compute the principal components of the reference data using the kernel matrix and then return the largest `n_components` eigenvectors. These are then normalized to have length equal to `1/eigenvalue`. Note that this differs from the linear case where we remove the largest eigenvectors. diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 7808b0e6b..71987740f 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -18,7 +18,7 @@ def __init__( Parameters ---------- n_components: - The number of dimensions in the principle subspace. For linear PCA should have + The number of dimensions in the principal subspace. For linear PCA should have ``1 <= n_components < dim(data)``. For kernel pca should have ``1 <= n_components < len(data)``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by @@ -108,7 +108,7 @@ def __init__( Parameters ---------- n_components: - The number of dimensions in the principle subspace. + The number of dimensions in the principal subspace. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. @@ -116,12 +116,12 @@ def __init__( super().__init__(device=device, n_components=n_components) def _fit(self, x: torch.Tensor) -> torch.Tensor: - """Compute the principle components of the reference data. + """Compute the principal components of the reference data. - We compute the principle components of the reference data using the covariance matrix and then + We compute the principal components of the reference data using the covariance matrix and then remove the largest `n_components` eigenvectors. The remaining eigenvectors correspond to the invariant dimensions of the data. Changes in these dimensions are used to compute the outlier - score which is the distance to the principle subspace spanned by the first `n_components` + score which is the distance to the principal subspace spanned by the first `n_components` eigenvectors. Parameters @@ -131,7 +131,7 @@ def _fit(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - The principle components of the reference data. + The principal components of the reference data. Raises ------ @@ -150,7 +150,7 @@ def _fit(self, x: torch.Tensor) -> torch.Tensor: def _score(self, x: torch.Tensor) -> torch.Tensor: """Compute the outlier score. - Centers the data and projects it onto the principle components. The score is then the sum of the + Centers the data and projects it onto the principal components. The score is then the sum of the squared projections. Parameters @@ -179,7 +179,7 @@ def __init__( Parameters ---------- n_components: - The number of dimensions in the principle subspace. + The number of dimensions in the principal subspace. kernel Kernel function to use for outlier detection. device @@ -190,9 +190,9 @@ def __init__( self.kernel = kernel def _fit(self, x: torch.Tensor) -> torch.Tensor: - """Compute the principle components of the reference data. + """Compute the principal components of the reference data. - We compute the principle components of the reference data using the kernel matrix and then + We compute the principal components of the reference data using the kernel matrix and then return the largest `n_components` eigenvectors. These are then normalized to have length equal to `1/eigenvalue`. Note that this differs from the linear case where we remove the largest eigenvectors. @@ -204,7 +204,7 @@ def _fit(self, x: torch.Tensor) -> torch.Tensor: Returns ------- - The principle components of the reference data. + The principal components of the reference data. Raises ------ @@ -223,7 +223,7 @@ def _fit(self, x: torch.Tensor) -> torch.Tensor: def _score(self, x: torch.Tensor) -> torch.Tensor: """Compute the outlier score. - Centers the data and projects it onto the principle components. The score is then the sum of the + Centers the data and projects it onto the principal components. The score is then the sum of the squared projections. Parameters From ef5a09414c31821dd6e00be156694d0c3b6ad90d Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 27 Apr 2023 14:37:49 +0100 Subject: [PATCH 209/247] Fix minor typing issue --- alibi_detect/od/pytorch/pca.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/pca.py b/alibi_detect/od/pytorch/pca.py index 71987740f..23f46a200 100644 --- a/alibi_detect/od/pytorch/pca.py +++ b/alibi_detect/od/pytorch/pca.py @@ -1,4 +1,5 @@ -from typing import Optional, Union, Callable, Literal +from typing import Optional, Union, Callable +from typing_extensions import Literal import torch From 30eff38e3997f2ee7ed36a3241000418707f9050 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 2 May 2023 14:53:44 +0100 Subject: [PATCH 210/247] Fix tests --- alibi_detect/od/_gmm.py | 6 ++- alibi_detect/od/pytorch/base.py | 15 +++++--- alibi_detect/od/pytorch/gmm.py | 37 +++++++++++-------- alibi_detect/od/sklearn/base.py | 11 +++--- alibi_detect/od/sklearn/gmm.py | 2 +- alibi_detect/od/tests/test__gmm/test__gmm.py | 7 ++-- .../test__gmm/test__gmm_pytorch_backend.py | 14 +++---- .../test__gmm/test__gmm_sklearn_backend.py | 10 ++--- 8 files changed, 58 insertions(+), 44 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index e013ee06b..9f5d4f8d6 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -9,6 +9,7 @@ from alibi_detect.od.sklearn import GMMSklearn from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ +from alibi_detect.exceptions import _catch_error as catch_error if TYPE_CHECKING: @@ -25,8 +26,8 @@ class GMM(BaseDetector, ThresholdMixin, FitMixin): def __init__( self, n_components: int = 1, - device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch', 'sklearn'] = 'pytorch', + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """Gaussian Mixture Model (GMM) outlier detector. @@ -125,6 +126,7 @@ def fit( **self.backend.format_fit_kwargs(locals()) ) + @catch_error('NotFittedError') def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. @@ -144,6 +146,7 @@ def score(self, x: np.ndarray) -> np.ndarray: score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) + @catch_error('NotFittedError') def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """Infer the threshold for the GMM detector. @@ -161,6 +164,7 @@ def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: """ self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + @catch_error('NotFittedError') def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index ed6ec4686..fb9a64601 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -1,6 +1,6 @@ from typing import List, Union, Optional, Dict from typing_extensions import Literal -from dataclasses import dataclass, asdict +from dataclasses import dataclass, asdict, fields from abc import ABC, abstractmethod import numpy as np @@ -21,18 +21,21 @@ class TorchOutlierDetectorOutput: p_value: Optional[torch.Tensor] def to_numpy(self): - outputs = asdict(self) - for key, value in outputs.items(): + result = {} + for f in fields(self): + value = getattr(self, f.name) if isinstance(value, torch.Tensor): - outputs[key] = value.cpu().detach().numpy() - return outputs + result[f.name] = value.cpu().detach().numpy() + else: + result[f.name] = value + return result def _raise_type_error(x): raise TypeError(f'x is type={type(x)} but must be one of TorchOutlierDetectorOutput or a torch Tensor') -def to_numpy(x: Union[torch.Tensor, TorchOutlierDetectorOutput]): +def to_numpy(x: Union[torch.Tensor, TorchOutlierDetectorOutput]) -> Union[np.ndarray, Dict[str, np.ndarray]]: """Converts any `torch` tensors found in input to `numpy` arrays. Takes a `torch` tensor or `TorchOutlierDetectorOutput` and converts any `torch` tensors found to `numpy` arrays diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 4b22a595f..e9f67e97f 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -1,4 +1,5 @@ from typing import Callable, Optional, Union, Dict +from typing_extensions import Literal from tqdm import tqdm import torch from torch.utils.data import DataLoader @@ -10,27 +11,34 @@ class GMMTorch(TorchOutlierDetector): + ensemble = False def __init__( self, n_components: int, - device: Optional[Union[str, torch.device]] = None + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """Pytorch backend for the Gaussian Mixture Model (GMM) outlier detector. Parameters ---------- n_components - Number of components in guassian mixture model. + Number of components in gaussian mixture model. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of ``torch.device``. + + Raises + ------ + ValueError + If `n_components` is less than 1. """ - self.ensembler = None + super().__init__(device=device) + if n_components < 1: + raise ValueError('n_components must be at least 1') self.n_components = n_components - TorchOutlierDetector.__init__(self, device=device) - def _fit( + def fit( self, x_ref: torch.Tensor, optimizer: Callable = torch.optim.Adam, @@ -43,7 +51,7 @@ def _fit( Parameters ---------- - X + x_ref Training data. optimizer Optimizer used to train the model. @@ -78,6 +86,7 @@ def _fit( loss_ma = loss_ma + (nll.item() - loss_ma) / (step + 1) dl.set_description(f'Epoch {epoch + 1}/{epochs}') dl.set_postfix(dict(loss_ma=loss_ma)) + self._set_fitted() def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: """Format kwargs for `fit` method. @@ -116,24 +125,22 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: ThresholdNotInferredException If called before detector has had `infer_threshold` method called. """ - raw_scores = self.score(x) - scores = self._ensembler(raw_scores) + scores = self.score(x) if not torch.jit.is_scripting(): self.check_threshold_inferred() preds = scores > self.threshold return preds.cpu() - @torch.no_grad() - def score(self, X: torch.Tensor) -> torch.Tensor: - """Score `X` using the GMM model. + def score(self, x: torch.Tensor) -> torch.Tensor: + """Computes the score of `x` Parameters ---------- - X + x `torch.Tensor` with leading batch dimension. """ if not torch.jit.is_scripting(): self.check_fitted() - X = X.to(torch.float32) - preds = self.model(X.to(self.device)).cpu() + x = x.to(torch.float32) + preds = self.model(x.to(self.device)) return preds diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 61df2d43a..3d2a281cd 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -5,7 +5,7 @@ import numpy as np -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError @dataclass @@ -51,11 +51,11 @@ def check_fitted(self): Raises ------ - NotFitException + NotFittedError Raised if method called and object has not been fit. """ if not self._fitted: - raise NotFitException(f'{self.__class__.__name__} has not been fit!') + raise NotFittedError(self.__class__.__name__) class SklearnOutlierDetector(FitMixin, ABC): @@ -94,12 +94,11 @@ def check_threshold_infered(self): Raises ------ - ThresholdNotInferredException + ThresholdNotInferredError Raised if threshold is not inferred. """ if not self.threshold_inferred: - raise ThresholdNotInferredException((f'{self.__class__.__name__} has no threshold set, ' - 'call `infer_threshold` before predicting.')) + raise ThresholdNotInferredError(self.__class__.__name__) @staticmethod def _to_numpy(arg): diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index 0240cc33a..5a916c664 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -16,7 +16,7 @@ def __init__( n_components Number of components in guassian mixture model. """ - SklearnOutlierDetector.__init__(self) + super().__init__() self.n_components = n_components def _fit( diff --git a/alibi_detect/od/tests/test__gmm/test__gmm.py b/alibi_detect/od/tests/test__gmm/test__gmm.py index 34ff0a70c..3245b04de 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm.py @@ -3,7 +3,8 @@ import torch from alibi_detect.od._gmm import GMM -from alibi_detect.base import NotFitException +from alibi_detect.exceptions import NotFittedError + from sklearn.datasets import make_moons @@ -15,9 +16,9 @@ def test_unfitted_gmm_single_score(backend): gmm_detector = GMM(n_components=1, backend=backend) x = np.array([[0, 10], [0.1, 0]]) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: _ = gmm_detector.predict(x) - assert str(err.value) == f'{gmm_detector.backend.__class__.__name__} has not been fit!' + assert str(err.value) == f'{gmm_detector.__class__.__name__} has not been fit!' @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py index 247d41ff7..15c3cefeb 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py @@ -3,36 +3,36 @@ import torch from alibi_detect.od.pytorch.gmm import GMMTorch -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError def test_gmm_pytorch_backend_fit_errors(): gmm_torch = GMMTorch(n_components=2) - assert not gmm_torch._fitted + assert not gmm_torch.fitted # Test that the backend raises an error if it is not fitted before # calling forward method. x = torch.tensor(np.random.randn(1, 10)) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: gmm_torch(x) assert str(err.value) == 'GMMTorch has not been fit!' # Test that the backend raises an error if it is not fitted before # predicting. - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: gmm_torch.predict(x) assert str(err.value) == 'GMMTorch has not been fit!' # Test the backend updates _fitted flag on fit. x_ref = torch.tensor(np.random.randn(1024, 10)) gmm_torch.fit(x_ref) - assert gmm_torch._fitted + assert gmm_torch.fitted # Test that the backend raises an if the forward method is called without the # threshold being inferred. - with pytest.raises(ThresholdNotInferredException) as err: + with pytest.raises(ThresholdNotInferredError) as err: gmm_torch(x) - assert str(err.value) == 'GMMTorch has no threshold set, call `infer_threshold` before predicting.' + assert str(err.value) == 'GMMTorch has no threshold set, call `infer_threshold` to fit one!' # Test that the backend can call predict without the threshold being inferred. assert gmm_torch.predict(x) diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py index f122a3cee..18425d922 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -2,7 +2,7 @@ import numpy as np from alibi_detect.od.sklearn.gmm import GMMSklearn -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError def test_gmm_sklearn_backend_fit_errors(): @@ -12,13 +12,13 @@ def test_gmm_sklearn_backend_fit_errors(): # Test that the backend raises an error if it is not fitted before # calling forward method. x = np.random.randn(1, 10) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: gmm_sklearn(x) assert str(err.value) == 'GMMSklearn has not been fit!' # Test that the backend raises an error if it is not fitted before # predicting. - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: gmm_sklearn.predict(x) assert str(err.value) == 'GMMSklearn has not been fit!' @@ -29,9 +29,9 @@ def test_gmm_sklearn_backend_fit_errors(): # Test that the backend raises an if the forward method is called without the # threshold being inferred. - with pytest.raises(ThresholdNotInferredException) as err: + with pytest.raises(ThresholdNotInferredError) as err: gmm_sklearn(x) - assert str(err.value) == 'GMMSklearn has no threshold set, call `infer_threshold` before predicting.' + assert str(err.value) == 'GMMSklearn has no threshold set, call `infer_threshold` to fit one!' # Test that the backend can call predict without the threshold being inferred. assert gmm_sklearn.predict(x) From 9994169ee44614c9cef3c865ba3067c1e0b5a4d1 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 2 May 2023 15:58:41 +0100 Subject: [PATCH 211/247] Update sklearn backend base class to match torch base class --- alibi_detect/od/pytorch/base.py | 2 +- alibi_detect/od/sklearn/base.py | 109 ++++++------------ alibi_detect/od/sklearn/gmm.py | 3 +- .../test__gmm/test__gmm_sklearn_backend.py | 4 +- 4 files changed, 40 insertions(+), 78 deletions(-) diff --git a/alibi_detect/od/pytorch/base.py b/alibi_detect/od/pytorch/base.py index fb9a64601..0afbd8bf4 100644 --- a/alibi_detect/od/pytorch/base.py +++ b/alibi_detect/od/pytorch/base.py @@ -1,6 +1,6 @@ from typing import List, Union, Optional, Dict from typing_extensions import Literal -from dataclasses import dataclass, asdict, fields +from dataclasses import dataclass, fields from abc import ABC, abstractmethod import numpy as np diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 3d2a281cd..4e4637db4 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import List, Union, Optional +from typing import List, Union, Optional, Dict from dataclasses import dataclass, asdict from abc import ABC, abstractmethod +from typing_extensions import Self import numpy as np @@ -18,47 +19,41 @@ class SklearnOutlierDetectorOutput: p_value: Optional[np.ndarray] -class FitMixin(ABC): - _fitted = False - - def __init__(self): - """Fit mixin - - Utility class that provides fitted checks for alibi-detect objects that require to be fit before use. - """ - super().__init__() - - def fit(self, x: np.ndarray, **kwargs: dict) -> FitMixin: - self._fitted = True - self._fit(x, **kwargs) - return self +class FitMixinSklearn(ABC): + fitted = False @abstractmethod - def _fit(self, x: np.ndarray): - """Fit on `x` array. - - This method should be overidden on child classes. + def fit(self, x: np.ndarray, **kwargs: dict) -> Self: + """Abstract fit method. Parameters ---------- x - Reference `np.array` for fitting object. + `torch.Tensor` to fit object on. """ - pass + return self + + def _set_fitted(self) -> Self: + """Sets the fitted attribute to True. + + Should be called within the object fit method. + """ + self.fitted = True + return self def check_fitted(self): - """Raises error if parent object instance has not been fit. + """Checks to make sure object has been fitted. Raises ------ NotFittedError Raised if method called and object has not been fit. """ - if not self._fitted: + if not self.fitted: raise NotFittedError(self.__class__.__name__) -class SklearnOutlierDetector(FitMixin, ABC): +class SklearnOutlierDetector(FitMixinSklearn, ABC): """Base class for sklearn backend outlier detection algorithms.""" threshold_inferred = False threshold = None @@ -66,17 +61,6 @@ class SklearnOutlierDetector(FitMixin, ABC): def __init__(self): super().__init__() - @abstractmethod - def _fit(self, x_ref: np.ndarray) -> None: - """Fit the outlier detector to the reference data. - - Parameters - ---------- - x_ref - Reference data. - """ - pass - @abstractmethod def score(self, x: np.ndarray) -> np.ndarray: """Score the data. @@ -89,7 +73,7 @@ def score(self, x: np.ndarray) -> np.ndarray: """ pass - def check_threshold_infered(self): + def check_threshold_inferred(self): """Check if threshold is inferred. Raises @@ -101,7 +85,7 @@ def check_threshold_infered(self): raise ThresholdNotInferredError(self.__class__.__name__) @staticmethod - def _to_numpy(arg): + def _to_numpy(arg: Union[np.ndarray, SklearnOutlierDetectorOutput]) -> Union[np.ndarray, Dict[str, np.ndarray]]: """Map params to numpy arrays. This function is for interface compatibility with the other backends. As such it does nothing but @@ -131,32 +115,9 @@ def _to_tensor(x: Union[List, np.ndarray]) -> np.ndarray: ---------- x Data to convert. - - Returns - ------- - `np.ndarray` """ return np.array(x) - def _ensembler(self, x: np.ndarray) -> np.ndarray: - """Aggregates and normalizes the data - - If the detector has an ensembler attribute we use it to aggregate and normalize the data. - - Parameters - ---------- - x - Data to aggregate and normalize. - - Returns - ------- - `np.ndarray` or just returns original data - """ - if hasattr(self, 'ensembler') and self.ensembler is not None: - return self.ensembler(x) - else: - return x - def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: """Classify the data as outlier or not. @@ -200,12 +161,15 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: ------ ValueError Raised if `fpr` is not in ``(0, 1)``. + ValueError + Raised if `fpr` is less than ``1/len(x)``. """ if not 0 < fpr < 1: - ValueError('`fpr` must be in `(0, 1)`.') + raise ValueError('`fpr` must be in `(0, 1)`.') + if fpr < 1/len(x): + raise ValueError(f'`fpr` must be greater than `1/len(x)={1/len(x)}`.') self.val_scores = self.score(x) - self.val_scores = self._ensembler(self.val_scores) - self.threshold = np.quantile(self.val_scores, 1-fpr) + self.threshold = np.quantile(self.val_scores, 1-fpr, interpolation='higher') self.threshold_inferred = True def predict(self, x: np.ndarray) -> SklearnOutlierDetectorOutput: @@ -220,20 +184,18 @@ def predict(self, x: np.ndarray) -> SklearnOutlierDetectorOutput: x Data to predict. - Raises - ------ - ValueError - Raised if the detector is not fit on reference data. - Returns ------- `SklearnOutlierDetectorOutput` Output of the outlier detector. + Raises + ------ + ValueError + Raised if the detector is not fit on reference data. """ - self.check_fitted() # type: ignore - raw_scores = self.score(x) - scores = self._ensembler(raw_scores) + self.check_fitted() + scores = self.score(x) return SklearnOutlierDetectorOutput( instance_score=scores, @@ -251,7 +213,6 @@ def __call__(self, x: np.ndarray) -> np.ndarray: x Data to classify. """ - raw_scores = self.score(x) - scores = self._ensembler(raw_scores) - self.check_threshold_infered() + scores = self.score(x) + self.check_threshold_inferred() return self._classify_outlier(scores) diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index 5a916c664..9b1ad82c1 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -19,7 +19,7 @@ def __init__( super().__init__() self.n_components = n_components - def _fit( + def fit( self, x_ref: np.ndarray, tol: float = 1e-3, @@ -62,6 +62,7 @@ def _fit( self.gmm = self.gmm.fit( x_ref, ) + self._set_fitted() def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: """Format kwargs for `fit` method. diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py index 18425d922..f4bcb83ed 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -7,7 +7,7 @@ def test_gmm_sklearn_backend_fit_errors(): gmm_sklearn = GMMSklearn(n_components=2) - assert not gmm_sklearn._fitted + assert not gmm_sklearn.fitted # Test that the backend raises an error if it is not fitted before # calling forward method. @@ -25,7 +25,7 @@ def test_gmm_sklearn_backend_fit_errors(): # Test the backend updates _fitted flag on fit. x_ref = np.random.randn(1024, 10) gmm_sklearn.fit(x_ref) - assert gmm_sklearn._fitted + assert gmm_sklearn.fitted # Test that the backend raises an if the forward method is called without the # threshold being inferred. From 49670aaa14b462f1cac6416eb5d0cbe95b7060c1 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 2 May 2023 16:38:34 +0100 Subject: [PATCH 212/247] Update sklearn gmm implementation --- alibi_detect/od/sklearn/base.py | 6 ++++-- alibi_detect/od/sklearn/gmm.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 4e4637db4..16ec905f8 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -23,7 +23,7 @@ class FitMixinSklearn(ABC): fitted = False @abstractmethod - def fit(self, x: np.ndarray, **kwargs: dict) -> Self: + def fit(self, x_ref: np.ndarray, **kwargs: dict) -> Self: """Abstract fit method. Parameters @@ -169,7 +169,9 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: if fpr < 1/len(x): raise ValueError(f'`fpr` must be greater than `1/len(x)={1/len(x)}`.') self.val_scores = self.score(x) - self.threshold = np.quantile(self.val_scores, 1-fpr, interpolation='higher') + self.threshold = np.quantile( + self.val_scores, 1-fpr, method='higher' + ) self.threshold_inferred = True def predict(self, x: np.ndarray) -> SklearnOutlierDetectorOutput: diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index 9b1ad82c1..f1d8db678 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -49,7 +49,6 @@ def fit( verbose Enable verbose output. If 1 then it prints the current initialization and each iteration step. If greater than 1 then it prints also the log probability and the time needed for each step. - """ self.gmm = GaussianMixture( n_components=self.n_components, From b93dcb7f6a0be742431f6ef5b3577a8e37c762d6 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 2 May 2023 16:46:14 +0100 Subject: [PATCH 213/247] Fix mypy errors --- alibi_detect/od/pytorch/ensemble.py | 2 +- alibi_detect/od/sklearn/base.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/pytorch/ensemble.py b/alibi_detect/od/pytorch/ensemble.py index 28f7bd390..f5103a721 100644 --- a/alibi_detect/od/pytorch/ensemble.py +++ b/alibi_detect/od/pytorch/ensemble.py @@ -35,7 +35,7 @@ class FitMixinTorch(ABC): fitted = False @abstractmethod - def fit(self, x: torch.Tensor) -> Self: + def fit(self, x_ref: torch.Tensor) -> Self: """Abstract fit method. Parameters diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 16ec905f8..02e43b407 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -23,7 +23,7 @@ class FitMixinSklearn(ABC): fitted = False @abstractmethod - def fit(self, x_ref: np.ndarray, **kwargs: dict) -> Self: + def fit(self, x_ref: np.ndarray) -> Self: """Abstract fit method. Parameters @@ -169,9 +169,7 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: if fpr < 1/len(x): raise ValueError(f'`fpr` must be greater than `1/len(x)={1/len(x)}`.') self.val_scores = self.score(x) - self.threshold = np.quantile( - self.val_scores, 1-fpr, method='higher' - ) + self.threshold = np.quantile(self.val_scores, 1-fpr, method='higher') self.threshold_inferred = True def predict(self, x: np.ndarray) -> SklearnOutlierDetectorOutput: From 4e5aa4b83f69220852ec31245dfb8604cb20f317 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 2 May 2023 17:29:19 +0100 Subject: [PATCH 214/247] Catch n_components < 1 error --- alibi_detect/od/sklearn/gmm.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index f1d8db678..c85f2a1f9 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -14,9 +14,16 @@ def __init__( Parameters ---------- n_components - Number of components in guassian mixture model. + Number of components in gaussian mixture model. + + Raises + ------ + ValueError + If `n_components` is less than 1. """ super().__init__() + if n_components < 1: + raise ValueError('n_components must be at least 1') self.n_components = n_components def fit( From 3481e0c8cdfb5489ed50be70155b41aa8253a6eb Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 10:03:27 +0100 Subject: [PATCH 215/247] Fix tests --- alibi_detect/od/pytorch/gmm.py | 2 +- alibi_detect/od/tests/test__gmm/test__gmm.py | 65 ++++++++----- .../test__gmm/test__gmm_pytorch_backend.py | 91 ++++++++++++------- .../test__gmm/test__gmm_sklearn_backend.py | 72 ++++++++------- .../test__mahalanobis/test__mahalanobis.py | 1 - 5 files changed, 142 insertions(+), 89 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index e9f67e97f..26866623f 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -129,7 +129,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: if not torch.jit.is_scripting(): self.check_threshold_inferred() preds = scores > self.threshold - return preds.cpu() + return preds def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` diff --git a/alibi_detect/od/tests/test__gmm/test__gmm.py b/alibi_detect/od/tests/test__gmm/test__gmm.py index 3245b04de..c210a134b 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm.py @@ -9,32 +9,45 @@ @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) -def test_unfitted_gmm_single_score(backend): - """ - test predict raises exception when not fitted - """ - gmm_detector = GMM(n_components=1, backend=backend) +def test_unfitted_gmm_score(backend): + """Test GMM detector raises exceptions when not fitted.""" + gmm_detector = GMM(n_components=2, backend=backend) x = np.array([[0, 10], [0.1, 0]]) + x_ref = np.random.randn(100, 2) + + with pytest.raises(NotFittedError) as err: + gmm_detector.infer_threshold(x_ref, 0.1) + assert str(err.value) == 'GMM has not been fit!' with pytest.raises(NotFittedError) as err: - _ = gmm_detector.predict(x) - assert str(err.value) == f'{gmm_detector.__class__.__name__} has not been fit!' + gmm_detector.score(x) + assert str(err.value) == 'GMM has not been fit!' + + # test predict raises exception when not fitted + with pytest.raises(NotFittedError) as err: + gmm_detector.predict(x) + assert str(err.value) == 'GMM has not been fit!' @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) -def test_fitted_gmm_single_score(backend): - """ - Test that a detector that has been fitted on data but that has not got an inferred - threshold, will correctly score outliers using the predict method. +def test_fitted_gmm_score(backend): + """Test GMM detector score method. + + Test GMM detector that has been fitted on reference data but has not had a threshold + inferred can still score data using the predict method. Test that it does not raise an error + but does not return `threshold`, `p_value` and `is_outlier` values. """ gmm_detector = GMM(n_components=1, backend=backend) x_ref = np.random.randn(100, 2) gmm_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) + scores = gmm_detector.score(x) + y = gmm_detector.predict(x) y = y['data'] assert y['instance_score'][0] > 5 assert y['instance_score'][1] < 2 + assert all(y['instance_score'] == scores) assert not y['threshold_inferred'] assert y['threshold'] is None assert y['is_outlier'] is None @@ -43,10 +56,11 @@ def test_fitted_gmm_single_score(backend): @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) def test_fitted_gmm_predict(backend): - """ - Test that a detector that has been fitted on data and with an inferred threshold, - will correctly score and label outliers, as well as return the p-values using the - predict method. + """Test GMM detector predict method. + + Test GMM detector that has been fitted on reference data and has had a threshold + inferred can score data using the predict method as well as predict outliers. Test that it + returns `threshold`, `p_value` and `is_outlier` values. """ gmm_detector = GMM(n_components=1, backend=backend) x_ref = np.random.randn(100, 2) @@ -65,9 +79,10 @@ def test_fitted_gmm_predict(backend): @pytest.mark.parametrize('backend', ['pytorch', 'sklearn']) def test_gmm_integration(backend): - """ - Tests gmm detector on the moons dataset. Fits and infers thresholds and - verifies that the detector can correctly detect inliers and outliers. + """Test GMM detector on moons dataset. + + Test GMM detector on a more complex 2d example. Test that the detector can be fitted + on reference data and infer a threshold. Test that it differentiates between inliers and outliers. """ gmm_detector = GMM(n_components=8, backend=backend) X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) @@ -84,17 +99,21 @@ def test_gmm_integration(backend): assert result -def test_gmm_torchscript(): - """ - Tests gmm detector fitted on the moons dataset can be torchscripted correctly. - """ +def test_gmm_torchscript(tmp_path): + """Tests user can torch-script gmm detector.""" gmm_detector = GMM(n_components=8, backend='pytorch') X_ref, _ = make_moons(1001, shuffle=True, noise=0.05, random_state=None) X_ref, x_inlier = X_ref[0:1000], X_ref[1000][None] gmm_detector.fit(X_ref) gmm_detector.infer_threshold(X_ref, 0.1) x_outlier = np.array([[-1, 1.5]]) - ts_gmm = torch.jit.script(gmm_detector.backend) x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) + + ts_gmm = torch.jit.script(gmm_detector.backend) + y = ts_gmm(x) + assert torch.all(y == torch.tensor([False, True])) + + ts_gmm.save(tmp_path / 'gmm.pt') + ts_gmm = torch.load(tmp_path / 'gmm.pt') y = ts_gmm(x) assert torch.all(y == torch.tensor([False, True])) diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py index 15c3cefeb..ee41c16ad 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py @@ -6,7 +6,66 @@ from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError +def test_gmm_pytorch_scoring(): + """Test GMM detector pytorch scoring method. + + Tests the scoring method of the GMMTorch pytorch backend detector. + """ + gmm_torch = GMMTorch(n_components=1) + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + gmm_torch.fit(x_ref) + + x_1 = torch.tensor(np.array([[8., 8.]])) + scores_1 = gmm_torch.score(x_1) + + x_2 = torch.tensor(np.random.multivariate_normal(mean, cov, 1)) + scores_2 = gmm_torch.score(x_2) + + x_3 = torch.tensor(np.array([[-10., 10.]])) + scores_3 = gmm_torch.score(x_3) + + # test correct ordering of scores given outlyingness of data + assert scores_1 < scores_2 < scores_3 + + # test that detector correctly detects true outlier + gmm_torch.infer_threshold(x_ref, 0.01) + x = torch.cat((x_1, x_2, x_3)) + outputs = gmm_torch.predict(x) + assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) + assert torch.all(gmm_torch(x) == torch.tensor([False, False, True])) + + # test that 0.01 of the in distribution data is flagged as outliers + x = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + outputs = gmm_torch.predict(x) + assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.01 + + +def test_gmm_torch_backend_ts(tmp_path): + """Test GMM detector backend is torch-scriptable and savable.""" + gmm_torch = GMMTorch(n_components=2) + x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) + x_ref = torch.randn((1024, 10)) + gmm_torch.fit(x_ref) + gmm_torch.infer_threshold(x_ref, 0.1) + pred_1 = gmm_torch(x) + + gmm_torch = torch.jit.script(gmm_torch) + pred_2 = gmm_torch(x) + assert torch.all(pred_1 == pred_2) + + gmm_torch.save(tmp_path / 'gmm_torch.pt') + gmm_torch = torch.load(tmp_path / 'gmm_torch.pt') + pred_2 = gmm_torch(x) + assert torch.all(pred_1 == pred_2) + + def test_gmm_pytorch_backend_fit_errors(): + """Test gmm detector pytorch backend fit errors. + + Tests the correct errors are raised when using the GMMTorch pytorch backend detector. + """ gmm_torch = GMMTorch(n_components=2) assert not gmm_torch.fitted @@ -36,35 +95,3 @@ def test_gmm_pytorch_backend_fit_errors(): # Test that the backend can call predict without the threshold being inferred. assert gmm_torch.predict(x) - - -def test_gmm_pytorch_scoring(): - gmm_torch = GMMTorch(n_components=1) - mean = [8, 8] - cov = [[2., 0.], [0., 1.]] - x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) - gmm_torch.fit(x_ref) - - x_1 = torch.tensor(np.array([[8., 8.]])) - scores_1 = gmm_torch.score(x_1) - - x_2 = torch.tensor(np.random.multivariate_normal(mean, cov, 1)) - scores_2 = gmm_torch.score(x_2) - - x_3 = torch.tensor(np.array([[-10., 10.]])) - scores_3 = gmm_torch.score(x_3) - - # test correct ordering of scores given outlyingness of data - assert scores_1 < scores_2 < scores_3 - - # test that detector correctly detects true Outlier - gmm_torch.infer_threshold(x_ref, 0.01) - x = torch.cat((x_1, x_2, x_3)) - outputs = gmm_torch.predict(x) - assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) - assert torch.all(gmm_torch(x) == torch.tensor([False, False, True])) - - # test that 0.01 of the in distribution data is flagged as outliers - x = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) - outputs = gmm_torch.predict(x) - assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.01 diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py index f4bcb83ed..5831e0a78 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -5,7 +5,47 @@ from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError +def test_gmm_sklearn_scoring(): + """Test GMM detector sklearn scoring method. + + Tests the scoring method of the GMMTorch sklearn backend detector. + """ + gmm_sklearn = GMMSklearn(n_components=2) + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = np.random.multivariate_normal(mean, cov, 1000) + gmm_sklearn.fit(x_ref) + + x_1 = np.array([[8., 8.]]) + scores_1 = gmm_sklearn.score(x_1) + + x_2 = np.random.multivariate_normal(mean, cov, 1) + scores_2 = gmm_sklearn.score(x_2) + + x_3 = np.array([[-10., 10.]]) + scores_3 = gmm_sklearn.score(x_3) + + # test correct ordering of scores given outlyingness of data + assert scores_1 < scores_2 < scores_3 + + # test that detector correctly detects true outlier + gmm_sklearn.infer_threshold(x_ref, 0.01) + x = np.concatenate((x_1, x_2, x_3)) + outputs = gmm_sklearn.predict(x) + assert np.all(outputs.is_outlier == np.array([False, False, True])) + assert np.all(gmm_sklearn(x) == np.array([False, False, True])) + + # test that 0.01 of the in distribution data is flagged as outliers + x = np.random.multivariate_normal(mean, cov, 1000) + outputs = gmm_sklearn.predict(x) + assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.01 + + def test_gmm_sklearn_backend_fit_errors(): + """Test gmm detector sklearn backend fit errors. + + Tests the correct errors are raised when using the GMMTorch sklearn backend detector. + """ gmm_sklearn = GMMSklearn(n_components=2) assert not gmm_sklearn.fitted @@ -35,35 +75,3 @@ def test_gmm_sklearn_backend_fit_errors(): # Test that the backend can call predict without the threshold being inferred. assert gmm_sklearn.predict(x) - - -def test_gmm_sklearn_scoring(): - gmm_sklearn = GMMSklearn(n_components=2) - mean = [8, 8] - cov = [[2., 0.], [0., 1.]] - x_ref = np.random.multivariate_normal(mean, cov, 1000) - gmm_sklearn.fit(x_ref) - - x_1 = np.array([[8., 8.]]) - scores_1 = gmm_sklearn.score(x_1) - - x_2 = np.random.multivariate_normal(mean, cov, 1) - scores_2 = gmm_sklearn.score(x_2) - - x_3 = np.array([[-10., 10.]]) - scores_3 = gmm_sklearn.score(x_3) - - # test correct ordering of scores given outlyingness of data - assert scores_1 < scores_2 < scores_3 - - # test that detector correctly detects true Outlier - gmm_sklearn.infer_threshold(x_ref, 0.01) - x = np.concatenate((x_1, x_2, x_3)) - outputs = gmm_sklearn.predict(x) - assert np.all(outputs.is_outlier == np.array([False, False, True])) - assert np.all(gmm_sklearn(x) == np.array([False, False, True])) - - # test that 0.01 of the in distribution data is flagged as outliers - x = np.random.multivariate_normal(mean, cov, 1000) - outputs = gmm_sklearn.predict(x) - assert (outputs.is_outlier.sum()/1000) - 0.01 < 0.01 diff --git a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py index 255dc53fb..a5a20be12 100644 --- a/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py +++ b/alibi_detect/od/tests/test__mahalanobis/test__mahalanobis.py @@ -107,7 +107,6 @@ def test_mahalanobis_integration(tmp_path): assert torch.all(y == torch.tensor([False, True])) ts_mahalanobis.save(tmp_path / 'mahalanobis.pt') - mahalanobis_detector = Mahalanobis() mahalanobis_detector = torch.load(tmp_path / 'mahalanobis.pt') y = mahalanobis_detector(x) assert torch.all(y == torch.tensor([False, True])) From e43dc344c83097d384bc75aea64d0366224d6295 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 10:44:22 +0100 Subject: [PATCH 216/247] Fix mypy issue in np.quantile --- alibi_detect/od/pytorch/gmm.py | 4 ++-- alibi_detect/od/sklearn/base.py | 2 +- alibi_detect/od/sklearn/gmm.py | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 26866623f..4340f4251 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -17,7 +17,7 @@ def __init__( self, n_components: int, device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, - ) -> None: + ): """Pytorch backend for the Gaussian Mixture Model (GMM) outlier detector. Parameters @@ -46,7 +46,7 @@ def fit( batch_size: int = 32, epochs: int = 10, verbose: int = 0, - ) -> None: + ): """Fit the GMM model. Parameters diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 02e43b407..9111f7f8e 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -169,7 +169,7 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: if fpr < 1/len(x): raise ValueError(f'`fpr` must be greater than `1/len(x)={1/len(x)}`.') self.val_scores = self.score(x) - self.threshold = np.quantile(self.val_scores, 1-fpr, method='higher') + self.threshold = np.quantile(self.val_scores, 1-fpr, interpolation='higher') # type: ignore self.threshold_inferred = True def predict(self, x: np.ndarray) -> SklearnOutlierDetectorOutput: diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index c85f2a1f9..d19f8dcdf 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -1,8 +1,9 @@ import numpy as np from typing import Dict -from alibi_detect.od.sklearn.base import SklearnOutlierDetector from sklearn.mixture import GaussianMixture +from alibi_detect.od.sklearn.base import SklearnOutlierDetector + class GMMSklearn(SklearnOutlierDetector): def __init__( @@ -35,7 +36,7 @@ def fit( init_params: str = 'kmeans', verbose: int = 0, ) -> None: - """Fit the outlier detector to the reference data. + """Fit the SKLearn GMM model`. Parameters ---------- From 2cc13b1fca33c3e56a39633f64ca74a94ee286d2 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 11:30:28 +0100 Subject: [PATCH 217/247] Fix optional dependencies issue --- alibi_detect/tests/test_dep_management.py | 1 + 1 file changed, 1 insertion(+) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index d5d5b40ce..b51c7c6fd 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -141,6 +141,7 @@ def test_od_backend_dependencies(opt_dep): ('MahalanobisTorch', ['torch', 'keops']), ('KernelPCATorch', ['torch', 'keops']), ('LinearPCATorch', ['torch', 'keops']), + ('GMMTorch', ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.od import pytorch as od_pt_backend From 81b7c77845e60d8c9f9f35fd01b55ddbf24a8c35 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 11:39:48 +0100 Subject: [PATCH 218/247] Update docstrings on gmm backends --- alibi_detect/od/pytorch/gmm.py | 26 +++++++++++++++++++++++--- alibi_detect/od/sklearn/gmm.py | 13 +++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 4340f4251..96b26589a 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -69,12 +69,23 @@ def fit( batch_size = len(x_ref) if batch_size is None else batch_size dataset = TorchDataset(x_ref) - dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True) - optimizer_instance: torch.optim.Optimizer = optimizer(self.model.parameters(), lr=learning_rate) + dataloader = DataLoader( + dataset, + batch_size=batch_size, + shuffle=True + ) + optimizer_instance: torch.optim.Optimizer = optimizer( + self.model.parameters(), + lr=learning_rate + ) self.model.train() for epoch in range(epochs): - dl = tqdm(enumerate(dataloader), total=len(dataloader), disable=not verbose) + dl = tqdm( + enumerate(dataloader), + total=len(dataloader), + disable=not verbose + ) loss_ma = 0 for step, x in dl: x = x.to(self.device) @@ -138,6 +149,15 @@ def score(self, x: torch.Tensor) -> torch.Tensor: ---------- x `torch.Tensor` with leading batch dimension. + + Returns + ------- + `torch.Tensor` of scores with leading batch dimension. + + Raises + ------ + NotFittedError + Raised if method called and detector has not been fit. """ if not torch.jit.is_scripting(): self.check_fitted() diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index d19f8dcdf..cad187f8c 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -92,12 +92,21 @@ def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: ) def score(self, x: np.ndarray) -> np.ndarray: - """Score the data. + """Computes the score of `x` Parameters ---------- x - Data to score. + `mp.ndarray` with leading batch dimension. + + Returns + ------- + `np.ndarray` of scores with leading batch dimension. + + Raises + ------ + NotFittedError + Raised if method called and detector has not been fit. """ self.check_fitted() return - self.gmm.score_samples(x) From ea5e117ffa736d80af756748bbd8f2eac0a5330f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 11:50:35 +0100 Subject: [PATCH 219/247] Update docstrings for sklearn base OD class --- alibi_detect/od/sklearn/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 9111f7f8e..e9d2ec321 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -88,8 +88,10 @@ def check_threshold_inferred(self): def _to_numpy(arg: Union[np.ndarray, SklearnOutlierDetectorOutput]) -> Union[np.ndarray, Dict[str, np.ndarray]]: """Map params to numpy arrays. + Note + ---- This function is for interface compatibility with the other backends. As such it does nothing but - return the input. + return the input and unpack `SklearnOutlierDetectorOutput` into a `dict`. Parameters ---------- @@ -116,7 +118,7 @@ def _to_tensor(x: Union[List, np.ndarray]) -> np.ndarray: x Data to convert. """ - return np.array(x) + return x def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: """Classify the data as outlier or not. From 4fb213181529f00353bf8a6ca241b22939ce1c25 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 14:46:58 +0100 Subject: [PATCH 220/247] Update _gmm docstrings --- alibi_detect/od/_gmm.py | 42 ++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 9f5d4f8d6..d4b1d8b59 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -31,13 +31,12 @@ def __init__( ) -> None: """Gaussian Mixture Model (GMM) outlier detector. - The guassian mixture model outlier detector fits a mixture of gaussian distributions to the reference data. - Test points are scored via the negative log-likhood under the corresponding density function. + The gaussian mixture model outlier detector fits a mixture of gaussian distributions to the reference data. + Test points are scored via the negative log-likelihood under the corresponding density function. We support two backends: ``'pytorch'`` and ``'sklearn'``. The ``'pytorch'`` backend allows for GPU acceleration - and uses gradient descent to fit the mixture of gaussians. We recommend using the ``'pytorch'`` backend for - for large datasets. The ``'sklearn'`` backend is a pure python implementation and is recommended for smaller - datasets. + and uses gradient descent to fit the GMM. We recommend using the ``'pytorch'`` backend for for large datasets. + The ``'sklearn'`` backend is a pure python implementation and is recommended for smaller datasets. Parameters ---------- @@ -142,27 +141,39 @@ def score(self, x: np.ndarray) -> np.ndarray: ------- Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. + + Raises + ------ + NotFittedError + If called before detector has been fit. """ score = self.backend.score(self.backend._to_tensor(x)) return self.backend._to_numpy(score) @catch_error('NotFittedError') - def infer_threshold(self, x_ref: np.ndarray, fpr: float) -> None: + def infer_threshold(self, x: np.ndarray, fpr: float) -> None: """Infer the threshold for the GMM detector. - The threshold is computed so that the outlier detector would incorectly classify `fpr` proportion of the + The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the reference data as outliers. Parameters ---------- - x_ref + x Reference data used to infer the threshold. fpr - False positive rate used to infer the threshold. The false positive rate is the proportion of instances in \ - `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range \ - ``(0, 1)``. + False positive rate used to infer the threshold. The false positive rate is the proportion of + instances in `x` that are incorrectly classified as outliers. The false positive rate should + be in the range ``(0, 1)``. + + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + NotFittedError + If called before detector has been fit. """ - self.backend.infer_threshold(self.backend._to_tensor(x_ref), fpr) + self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @catch_error('NotFittedError') def predict(self, x: np.ndarray) -> Dict[str, Any]: @@ -178,9 +189,14 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: Returns ------- Dictionary with keys 'data' and 'meta'. 'data' contains the outlier scores. If threshold inference was \ - performed, 'data' also contains the threshold value, outlier labels and p_vals . The shape of the scores is \ + performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. + + Raises + ------ + NotFittedError + If called before detector has been fit. """ outputs = self.backend.predict(self.backend._to_tensor(x)) output = outlier_prediction_dict() From 482e7387a0b5567d15419a8a9aba6bf49cb50a84 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 14:48:47 +0100 Subject: [PATCH 221/247] Minor fix --- alibi_detect/od/pytorch/gmm.py | 4 ++-- alibi_detect/od/sklearn/base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 96b26589a..7c448c029 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -82,8 +82,8 @@ def fit( for epoch in range(epochs): dl = tqdm( - enumerate(dataloader), - total=len(dataloader), + enumerate(dataloader), + total=len(dataloader), disable=not verbose ) loss_ma = 0 diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index e9d2ec321..7f5ee9fff 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -118,7 +118,7 @@ def _to_tensor(x: Union[List, np.ndarray]) -> np.ndarray: x Data to convert. """ - return x + return np.array(x) def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: """Classify the data as outlier or not. From 6dde34c9f773f21f3bdd1209cbb00c4de50987bc Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 3 May 2023 15:01:40 +0100 Subject: [PATCH 222/247] Add docstrings for GMModel --- alibi_detect/models/pytorch/gmm.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/alibi_detect/models/pytorch/gmm.py b/alibi_detect/models/pytorch/gmm.py index 2284d25de..7890017e7 100644 --- a/alibi_detect/models/pytorch/gmm.py +++ b/alibi_detect/models/pytorch/gmm.py @@ -4,6 +4,15 @@ class GMMModel(nn.Module): def __init__(self, n_components: int, dim: int) -> None: + """Gaussian Mixture Model (GMM). + + Parameters + ---------- + n_components: + The number of mixture components. + dim: + The dimensionality of the data. + """ super().__init__() self.weight_logits = nn.Parameter(torch.zeros(n_components)) self.means = nn.Parameter(torch.randn(n_components, dim)) @@ -18,6 +27,13 @@ def _weights(self) -> torch.Tensor: return nn.functional.softmax(self.weight_logits, dim=0) def forward(self, x: torch.Tensor) -> torch.Tensor: + """Compute the log-likelihood of the data. + + Parameters + ---------- + x: + Data to score. + """ det = torch.linalg.det(self._inv_cov) # Note det(A^-1)=1/det(A) to_means = x[:, None, :] - self.means[None, :, :] likelihood = ((-0.5 * ( From 14b717510ca4b791fda3ea0563718712c40fb132 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 11 May 2023 11:44:05 +0100 Subject: [PATCH 223/247] Make requested changes --- alibi_detect/models/pytorch/gmm.py | 6 +++--- alibi_detect/od/_gmm.py | 6 +++--- alibi_detect/od/pytorch/gmm.py | 8 ++++---- alibi_detect/od/sklearn/base.py | 13 +++---------- alibi_detect/utils/pytorch/misc.py | 2 +- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/alibi_detect/models/pytorch/gmm.py b/alibi_detect/models/pytorch/gmm.py index 7890017e7..6a6a8d7e0 100644 --- a/alibi_detect/models/pytorch/gmm.py +++ b/alibi_detect/models/pytorch/gmm.py @@ -8,9 +8,9 @@ def __init__(self, n_components: int, dim: int) -> None: Parameters ---------- - n_components: + n_components The number of mixture components. - dim: + dim The dimensionality of the data. """ super().__init__() @@ -31,7 +31,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Parameters ---------- - x: + x Data to score. """ det = torch.linalg.det(self._inv_cov) # Note det(A^-1)=1/det(A) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index d4b1d8b59..2896dc19a 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -35,8 +35,8 @@ def __init__( Test points are scored via the negative log-likelihood under the corresponding density function. We support two backends: ``'pytorch'`` and ``'sklearn'``. The ``'pytorch'`` backend allows for GPU acceleration - and uses gradient descent to fit the GMM. We recommend using the ``'pytorch'`` backend for for large datasets. - The ``'sklearn'`` backend is a pure python implementation and is recommended for smaller datasets. + and uses gradient descent to fit the GMM. We recommend using the ``'pytorch'`` backend for large datasets. The + ``'sklearn'`` backend is a pure python implementation and is recommended for smaller datasets. Parameters ---------- @@ -118,7 +118,7 @@ def fit( 'random_from_data' : responsibilities are initialized randomly from the data. Defaults to ``'kmeans'``. verbose - Verbosity level used to fit the detector. Only used if the ``'sklearn'`` backend is used. Defaults to ``0``. + Verbosity level used to fit the detector. Used for both ``'sklearn'`` and ``'pytorch'`` backends. Defaults to ``0``. """ self.backend.fit( self.backend._to_tensor(x_ref), diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 7c448c029..66dca49e3 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional, Union, Dict +from typing import Optional, Union, Dict, Type from typing_extensions import Literal from tqdm import tqdm import torch @@ -41,7 +41,7 @@ def __init__( def fit( self, x_ref: torch.Tensor, - optimizer: Callable = torch.optim.Adam, + optimizer: Type[torch.optim.Optimizer] = torch.optim.Adam, learning_rate: float = 0.1, batch_size: int = 32, epochs: int = 10, @@ -77,7 +77,7 @@ def fit( optimizer_instance: torch.optim.Optimizer = optimizer( self.model.parameters(), lr=learning_rate - ) + ) # type: ignore[call-arg] self.model.train() for epoch in range(epochs): @@ -93,7 +93,7 @@ def fit( optimizer_instance.zero_grad() nll.backward() optimizer_instance.step() - if verbose == 1 and isinstance(dl, tqdm): + if verbose and isinstance(dl, tqdm): loss_ma = loss_ma + (nll.item() - loss_ma) / (step + 1) dl.set_description(f'Epoch {epoch + 1}/{epochs}') dl.set_postfix(dict(loss_ma=loss_ma)) diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 7f5ee9fff..e98e2d600 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -1,4 +1,3 @@ -from __future__ import annotations from typing import List, Union, Optional, Dict from dataclasses import dataclass, asdict from abc import ABC, abstractmethod @@ -58,9 +57,6 @@ class SklearnOutlierDetector(FitMixinSklearn, ABC): threshold_inferred = False threshold = None - def __init__(self): - super().__init__() - @abstractmethod def score(self, x: np.ndarray) -> np.ndarray: """Score the data. @@ -86,12 +82,9 @@ def check_threshold_inferred(self): @staticmethod def _to_numpy(arg: Union[np.ndarray, SklearnOutlierDetectorOutput]) -> Union[np.ndarray, Dict[str, np.ndarray]]: - """Map params to numpy arrays. + """Map arg to the frontend format. - Note - ---- - This function is for interface compatibility with the other backends. As such it does nothing but - return the input and unpack `SklearnOutlierDetectorOutput` into a `dict`. + If `arg` is a `SklearnOutlierDetectorOutput` object, we unpack it into a `dict` and return it. Parameters ---------- @@ -100,7 +93,7 @@ def _to_numpy(arg: Union[np.ndarray, SklearnOutlierDetectorOutput]) -> Union[np. Returns ------- - `np.ndarray` or dictionary of containing `numpy` arrays + `np.ndarray` or dictionary containing frontend compatible data. """ if isinstance(arg, SklearnOutlierDetectorOutput): return asdict(arg) diff --git a/alibi_detect/utils/pytorch/misc.py b/alibi_detect/utils/pytorch/misc.py index cfcab439b..cc9585fa4 100644 --- a/alibi_detect/utils/pytorch/misc.py +++ b/alibi_detect/utils/pytorch/misc.py @@ -94,7 +94,7 @@ def get_device(device: Optional[Union[str, torch.device]] = None) -> torch.devic return torch_device -def get_optimizer(name: str = 'Adam'): +def get_optimizer(name: str = 'Adam') -> torch.optim.Optimizer: """ Get an optimizer class from its name. From f9326ed161137610043bd12e81ff85aca19fc65a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 11 May 2023 11:46:36 +0100 Subject: [PATCH 224/247] Fix typo --- alibi_detect/od/_gmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 2896dc19a..d7cf75a8d 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -102,7 +102,7 @@ def fit( epochs Number of epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` backends. If the backend is ``'sklearn'``, the detector is fitted using the EM algorithm and the number of epochs - defualts to ``10``. If the backend is ``'pytorch'``, the detector is fitted using gradient descent and + defaults to ``10``. If the backend is ``'pytorch'``, the detector is fitted using gradient descent and the number of epochs defaults to ``100``. tol Tolerance used to fit the detector. Only used if the ``'sklearn'`` backend is used. Defaults to ``1e-3``. From 95645c61995bb9117757572a9ca53123278ca5ba Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 11 May 2023 14:55:47 +0100 Subject: [PATCH 225/247] Make pr requested changes --- alibi_detect/od/_gmm.py | 19 ++++++++++--------- alibi_detect/od/sklearn/base.py | 7 ++++--- alibi_detect/od/sklearn/gmm.py | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index d7cf75a8d..70ed3a1f9 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -26,7 +26,7 @@ class GMM(BaseDetector, ThresholdMixin, FitMixin): def __init__( self, n_components: int = 1, - backend: Literal['pytorch', 'sklearn'] = 'pytorch', + backend: Literal['pytorch', 'sklearn'] = 'sklearn', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """Gaussian Mixture Model (GMM) outlier detector. @@ -43,10 +43,11 @@ def __init__( n_components: The number of mixture components. Defaults to ``1``. backend - Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'`` and ``'sklearn'``. + Backend used for outlier detection. Defaults to ``'sklearn'``. Options are ``'pytorch'`` and ``'sklearn'``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. Can be specified by - passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. The device is only used if the ``'pytorch'`` backend is + used. Defaults to ``None``. Raises ------ @@ -73,7 +74,7 @@ def fit( optimizer: Optional[str] = 'Adam', learning_rate: float = 0.1, batch_size: Optional[int] = None, - epochs: Optional[int] = None, + epochs: Optional[int] = 10, tol: float = 1e-3, n_init: int = 1, init_params: str = 'kmeans', @@ -98,14 +99,14 @@ def fit( Learning rate used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``0.1``. batch_size Batch size used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``None``. - If ``None``, the entire dataset is used in each epoch. + If ``None``, the entire dataset is used for each gradient update. epochs Number of epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` backends. - If the backend is ``'sklearn'``, the detector is fitted using the EM algorithm and the number of epochs - defaults to ``10``. If the backend is ``'pytorch'``, the detector is fitted using gradient descent and - the number of epochs defaults to ``100``. + If the backend is ``'sklearn'``, the detector is fit using the EM algorithm. If the backend is ``'pytorch'``, + the detector is fitted using gradient descent. In both cases the number of epochs defaults to ``10``. tol - Tolerance used to fit the detector. Only used if the ``'sklearn'`` backend is used. Defaults to ``1e-3``. + Convergence threshold used to fit the detector. Will cut the training short when this loss value is reached. + Only used if the ``'sklearn'`` backend is used. Defaults to ``1e-3``. n_init Number of initializations used to fit the detector. Only used if the ``'sklearn'`` backend is used. Defaults to ``1``. diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index e98e2d600..87fb4a725 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -111,7 +111,7 @@ def _to_tensor(x: Union[List, np.ndarray]) -> np.ndarray: x Data to convert. """ - return np.array(x) + return np.asarray(x) def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: """Classify the data as outlier or not. @@ -125,7 +125,8 @@ def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: ------- `np.ndarray` or ``None`` """ - return scores > self.threshold if self.threshold_inferred else None + return (scores > self.threshold).astype(np.int8) \ + if self.threshold_inferred else None def _p_vals(self, scores: np.ndarray) -> np.ndarray: """Compute p-values for the scores. @@ -164,7 +165,7 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: if fpr < 1/len(x): raise ValueError(f'`fpr` must be greater than `1/len(x)={1/len(x)}`.') self.val_scores = self.score(x) - self.threshold = np.quantile(self.val_scores, 1-fpr, interpolation='higher') # type: ignore + self.threshold = np.quantile(self.val_scores, 1-fpr, interpolation='higher') # type: ignore[call-overload] self.threshold_inferred = True def predict(self, x: np.ndarray) -> SklearnOutlierDetectorOutput: diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index cad187f8c..b69c4fbcd 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -97,7 +97,7 @@ def score(self, x: np.ndarray) -> np.ndarray: Parameters ---------- x - `mp.ndarray` with leading batch dimension. + `np.ndarray` with leading batch dimension. Returns ------- From 85fa95221ac0e98bcdec9b7395dfc9b33f47bf22 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 11 May 2023 16:46:23 +0100 Subject: [PATCH 226/247] Add convergence checks in gmm pytorch backend fit method --- alibi_detect/od/_gmm.py | 8 +++- alibi_detect/od/pytorch/gmm.py | 47 +++++++++++++++++-- alibi_detect/od/sklearn/gmm.py | 16 ++++++- .../test__gmm/test__gmm_pytorch_backend.py | 15 ++++++ .../test__gmm/test__gmm_sklearn_backend.py | 19 +++++++- 5 files changed, 95 insertions(+), 10 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 70ed3a1f9..c8b905d57 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -76,6 +76,7 @@ def fit( batch_size: Optional[int] = None, epochs: Optional[int] = 10, tol: float = 1e-3, + n_iter_no_change: int = 25, n_init: int = 1, init_params: str = 'kmeans', verbose: int = 0, @@ -105,8 +106,11 @@ def fit( If the backend is ``'sklearn'``, the detector is fit using the EM algorithm. If the backend is ``'pytorch'``, the detector is fitted using gradient descent. In both cases the number of epochs defaults to ``10``. tol - Convergence threshold used to fit the detector. Will cut the training short when this loss value is reached. - Only used if the ``'sklearn'`` backend is used. Defaults to ``1e-3``. + Convergence threshold used to fit the detector. Used for both ``'sklearn'`` and ``'pytorch'`` backends. + Defaults to ``1e-3``. + n_iter_no_change: + The number of iterations over which the loss must decrease by `tol` in order for optimization to continue. + Only used if the ``'pytorch'`` backend is used. n_init Number of initializations used to fit the detector. Only used if the ``'sklearn'`` backend is used. Defaults to ``1``. diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 66dca49e3..cbf749dfc 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -38,21 +38,29 @@ def __init__( raise ValueError('n_components must be at least 1') self.n_components = n_components - def fit( + def fit( # type: ignore[override] self, x_ref: torch.Tensor, + tol: float = 1e-3, + n_iter_no_change: int = 25, optimizer: Type[torch.optim.Optimizer] = torch.optim.Adam, learning_rate: float = 0.1, batch_size: int = 32, epochs: int = 10, verbose: int = 0, - ): + ) -> Dict: """Fit the GMM model. Parameters ---------- x_ref Training data. + tol + Convergence threshold. Training iterations will stop when the lower bound average + gain is below this threshold. + n_iter_no_change + The number of iterations over which the loss must decrease by `tol` in order for + optimization to continue. optimizer Optimizer used to train the model. learning_rate @@ -63,6 +71,13 @@ def fit( Number of training epochs. verbose Verbosity level during training. 0 is silent, 1 a progress bar. + + Returns + ------- + Dictionary with fit results. The dictionary contains the following keys: + - converged: bool indicating whether EM algorithm converged. + - n_iter: number of EM iterations performed. + - lower_bound: log-likelihood lower bound. """ self.model = GMMModel(self.n_components, x_ref.shape[-1]).to(self.device) x_ref = x_ref.to(torch.float32) @@ -80,7 +95,12 @@ def fit( ) # type: ignore[call-arg] self.model.train() - for epoch in range(epochs): + min_loss = None + converged = False + epoch = 0 + + while not converged and epoch < epochs: + epoch += 1 dl = tqdm( enumerate(dataloader), total=len(dataloader), @@ -93,11 +113,28 @@ def fit( optimizer_instance.zero_grad() nll.backward() optimizer_instance.step() + if verbose and isinstance(dl, tqdm): loss_ma = loss_ma + (nll.item() - loss_ma) / (step + 1) dl.set_description(f'Epoch {epoch + 1}/{epochs}') dl.set_postfix(dict(loss_ma=loss_ma)) + + if min_loss is None or nll < min_loss - tol: + t_since_improv = 0 + min_loss = nll + else: + t_since_improv += 1 + + if t_since_improv > n_iter_no_change: + converged = True + break + self._set_fitted() + return { + 'converged': converged, + 'lower_bound': min_loss, + 'epochs': epoch + } def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: """Format kwargs for `fit` method. @@ -116,7 +153,9 @@ def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: learning_rate=fit_kwargs.get('learning_rate', 0.1), batch_size=fit_kwargs.get('batch_size', None), epochs=(lambda v: 10 if v is None else v)(fit_kwargs.get('epochs', None)), - verbose=fit_kwargs.get('verbose', 0) + verbose=fit_kwargs.get('verbose', 0), + tol=fit_kwargs.get('tol', 1e-3), + n_iter_no_change=fit_kwargs.get('n_iter_no_change', 25) ) def forward(self, x: torch.Tensor) -> torch.Tensor: diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index b69c4fbcd..4288e0519 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -27,7 +27,7 @@ def __init__( raise ValueError('n_components must be at least 1') self.n_components = n_components - def fit( + def fit( # type: ignore[override] self, x_ref: np.ndarray, tol: float = 1e-3, @@ -35,7 +35,7 @@ def fit( n_init: int = 1, init_params: str = 'kmeans', verbose: int = 0, - ) -> None: + ) -> Dict: """Fit the SKLearn GMM model`. Parameters @@ -57,6 +57,13 @@ def fit( verbose Enable verbose output. If 1 then it prints the current initialization and each iteration step. If greater than 1 then it prints also the log probability and the time needed for each step. + + Returns + ------- + Dictionary with fit results. The dictionary contains the following keys: + - converged: bool indicating whether EM algorithm converged. + - n_iter: number of EM iterations performed. + - lower_bound: log-likelihood lower bound. """ self.gmm = GaussianMixture( n_components=self.n_components, @@ -70,6 +77,11 @@ def fit( x_ref, ) self._set_fitted() + return { + 'converged': self.gmm.converged_, + 'n_iter': self.gmm.n_iter_, + 'lower_bound': self.gmm.lower_bound_ + } def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: """Format kwargs for `fit` method. diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py index ee41c16ad..9622caa99 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py @@ -95,3 +95,18 @@ def test_gmm_pytorch_backend_fit_errors(): # Test that the backend can call predict without the threshold being inferred. assert gmm_torch.predict(x) + + +def test_gmm_pytorch_fit(): + """Test GMM detector pytorch fit method. + + Tests pytorch detector checks for convergence and stops early if it does. + """ + gmm_torch = GMMTorch(n_components=1) + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) + fit_results = gmm_torch.fit(x_ref, tol=0.01) + assert fit_results['converged'] + assert fit_results['epochs'] < 10 + assert fit_results['lower_bound'] < 1 diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py index 5831e0a78..47af5ec0e 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_sklearn_backend.py @@ -8,7 +8,7 @@ def test_gmm_sklearn_scoring(): """Test GMM detector sklearn scoring method. - Tests the scoring method of the GMMTorch sklearn backend detector. + Tests the scoring method of the GMM sklearn backend detector. """ gmm_sklearn = GMMSklearn(n_components=2) mean = [8, 8] @@ -44,7 +44,7 @@ def test_gmm_sklearn_scoring(): def test_gmm_sklearn_backend_fit_errors(): """Test gmm detector sklearn backend fit errors. - Tests the correct errors are raised when using the GMMTorch sklearn backend detector. + Tests the correct errors are raised when using the GMMSklearn backend detector. """ gmm_sklearn = GMMSklearn(n_components=2) assert not gmm_sklearn.fitted @@ -75,3 +75,18 @@ def test_gmm_sklearn_backend_fit_errors(): # Test that the backend can call predict without the threshold being inferred. assert gmm_sklearn.predict(x) + + +def test_gmm_sklearn_fit(): + """Test GMM detector sklearn backend fit method. + + Tests the scoring method of the GMMSklearn backend detector. + """ + gmm_sklearn = GMMSklearn(n_components=1) + mean = [8, 8] + cov = [[2., 0.], [0., 1.]] + x_ref = np.random.multivariate_normal(mean, cov, 1000) + fit_results = gmm_sklearn.fit(x_ref, tol=0.01) + assert fit_results['converged'] + assert fit_results['n_iter'] < 10 + assert fit_results['lower_bound'] < 1 From 0cb20d7d21ee90ab54809679df07a570eaa1f8e0 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 11 May 2023 16:51:29 +0100 Subject: [PATCH 227/247] Rename epochs to max_epochs in pytorch fit method --- alibi_detect/od/_gmm.py | 8 ++++---- alibi_detect/od/pytorch/gmm.py | 14 +++++++------- alibi_detect/od/sklearn/gmm.py | 2 +- .../tests/test__gmm/test__gmm_pytorch_backend.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index c8b905d57..74ba79d4b 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -74,7 +74,7 @@ def fit( optimizer: Optional[str] = 'Adam', learning_rate: float = 0.1, batch_size: Optional[int] = None, - epochs: Optional[int] = 10, + max_epochs: Optional[int] = 10, tol: float = 1e-3, n_iter_no_change: int = 25, n_init: int = 1, @@ -101,10 +101,10 @@ def fit( batch_size Batch size used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``None``. If ``None``, the entire dataset is used for each gradient update. - epochs - Number of epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` backends. + max_epochs + Number of max_epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` backends. If the backend is ``'sklearn'``, the detector is fit using the EM algorithm. If the backend is ``'pytorch'``, - the detector is fitted using gradient descent. In both cases the number of epochs defaults to ``10``. + the detector is fitted using gradient descent. In both cases the number of max_epochs defaults to ``10``. tol Convergence threshold used to fit the detector. Used for both ``'sklearn'`` and ``'pytorch'`` backends. Defaults to ``1e-3``. diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index cbf749dfc..eb77b9081 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -46,7 +46,7 @@ def fit( # type: ignore[override] optimizer: Type[torch.optim.Optimizer] = torch.optim.Adam, learning_rate: float = 0.1, batch_size: int = 32, - epochs: int = 10, + max_epochs: int = 10, verbose: int = 0, ) -> Dict: """Fit the GMM model. @@ -67,8 +67,8 @@ def fit( # type: ignore[override] Learning rate used to train the model. batch_size Batch size used to train the model. - epochs - Number of training epochs. + max_epochs + Number of training max_epochs. verbose Verbosity level during training. 0 is silent, 1 a progress bar. @@ -99,7 +99,7 @@ def fit( # type: ignore[override] converged = False epoch = 0 - while not converged and epoch < epochs: + while not converged and epoch < max_epochs: epoch += 1 dl = tqdm( enumerate(dataloader), @@ -116,7 +116,7 @@ def fit( # type: ignore[override] if verbose and isinstance(dl, tqdm): loss_ma = loss_ma + (nll.item() - loss_ma) / (step + 1) - dl.set_description(f'Epoch {epoch + 1}/{epochs}') + dl.set_description(f'Epoch {epoch + 1}/{max_epochs}') dl.set_postfix(dict(loss_ma=loss_ma)) if min_loss is None or nll < min_loss - tol: @@ -133,7 +133,7 @@ def fit( # type: ignore[override] return { 'converged': converged, 'lower_bound': min_loss, - 'epochs': epoch + 'n_epochs': epoch } def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: @@ -152,7 +152,7 @@ def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: optimizer=get_optimizer(fit_kwargs.get('optimizer')), learning_rate=fit_kwargs.get('learning_rate', 0.1), batch_size=fit_kwargs.get('batch_size', None), - epochs=(lambda v: 10 if v is None else v)(fit_kwargs.get('epochs', None)), + max_epochs=(lambda v: 10 if v is None else v)(fit_kwargs.get('max_epochs', None)), verbose=fit_kwargs.get('verbose', 0), tol=fit_kwargs.get('tol', 1e-3), n_iter_no_change=fit_kwargs.get('n_iter_no_change', 25) diff --git a/alibi_detect/od/sklearn/gmm.py b/alibi_detect/od/sklearn/gmm.py index 4288e0519..5550d6e08 100644 --- a/alibi_detect/od/sklearn/gmm.py +++ b/alibi_detect/od/sklearn/gmm.py @@ -97,7 +97,7 @@ def format_fit_kwargs(self, fit_kwargs: Dict) -> Dict: """ return dict( tol=fit_kwargs.get('tol', 1e-3), - max_iter=(lambda v: 100 if v is None else v)(fit_kwargs.get('epochs', None)), + max_iter=(lambda v: 100 if v is None else v)(fit_kwargs.get('max_epochs', None)), n_init=fit_kwargs.get('n_init', 1), init_params=fit_kwargs.get('init_params', 'kmeans'), verbose=fit_kwargs.get('verbose', 0), diff --git a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py index 9622caa99..ea8d4d020 100644 --- a/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py +++ b/alibi_detect/od/tests/test__gmm/test__gmm_pytorch_backend.py @@ -108,5 +108,5 @@ def test_gmm_pytorch_fit(): x_ref = torch.tensor(np.random.multivariate_normal(mean, cov, 1000)) fit_results = gmm_torch.fit(x_ref, tol=0.01) assert fit_results['converged'] - assert fit_results['epochs'] < 10 + assert fit_results['n_epochs'] < 10 assert fit_results['lower_bound'] < 1 From a38288e9b7ab9815e7ac9ce3bf25a199ba530f63 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 11 May 2023 17:09:21 +0100 Subject: [PATCH 228/247] Fix minor typing error --- alibi_detect/od/pytorch/gmm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index eb77b9081..be5226e17 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -91,8 +91,8 @@ def fit( # type: ignore[override] ) optimizer_instance: torch.optim.Optimizer = optimizer( self.model.parameters(), - lr=learning_rate - ) # type: ignore[call-arg] + lr=learning_rate # type: ignore[call-arg] + ) self.model.train() min_loss = None From e75058bff884a4c5b958084a2811ba77e1a96083 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 15 May 2023 10:43:39 +0100 Subject: [PATCH 229/247] Fix py3.7 typing issue --- alibi_detect/od/sklearn/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/sklearn/base.py b/alibi_detect/od/sklearn/base.py index 87fb4a725..d22d1fab5 100644 --- a/alibi_detect/od/sklearn/base.py +++ b/alibi_detect/od/sklearn/base.py @@ -113,7 +113,7 @@ def _to_tensor(x: Union[List, np.ndarray]) -> np.ndarray: """ return np.asarray(x) - def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: + def _classify_outlier(self, scores: np.ndarray) -> Optional[np.ndarray]: """Classify the data as outlier or not. Parameters @@ -125,8 +125,9 @@ def _classify_outlier(self, scores: np.ndarray) -> np.ndarray: ------- `np.ndarray` or ``None`` """ - return (scores > self.threshold).astype(np.int8) \ - if self.threshold_inferred else None + if (self.threshold_inferred and self.threshold is not None): + return (scores > self.threshold).astype(int) + return None def _p_vals(self, scores: np.ndarray) -> np.ndarray: """Compute p-values for the scores. From 246def00fd26d1b5d1c18c6ec69198c0d34ed208 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 15 May 2023 10:47:20 +0100 Subject: [PATCH 230/247] Change fit arg order --- alibi_detect/od/_gmm.py | 8 ++++---- alibi_detect/od/pytorch/gmm.py | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 74ba79d4b..d7d24b9c8 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -73,8 +73,8 @@ def fit( x_ref: np.ndarray, optimizer: Optional[str] = 'Adam', learning_rate: float = 0.1, - batch_size: Optional[int] = None, max_epochs: Optional[int] = 10, + batch_size: Optional[int] = None, tol: float = 1e-3, n_iter_no_change: int = 25, n_init: int = 1, @@ -98,13 +98,13 @@ def fit( Optimizer used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``'Adam'``. learning_rate Learning rate used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``0.1``. - batch_size - Batch size used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``None``. - If ``None``, the entire dataset is used for each gradient update. max_epochs Number of max_epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` backends. If the backend is ``'sklearn'``, the detector is fit using the EM algorithm. If the backend is ``'pytorch'``, the detector is fitted using gradient descent. In both cases the number of max_epochs defaults to ``10``. + batch_size + Batch size used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``None``. + If ``None``, the entire dataset is used for each gradient update. tol Convergence threshold used to fit the detector. Used for both ``'sklearn'`` and ``'pytorch'`` backends. Defaults to ``1e-3``. diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index be5226e17..1fe570031 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -41,12 +41,12 @@ def __init__( def fit( # type: ignore[override] self, x_ref: torch.Tensor, - tol: float = 1e-3, - n_iter_no_change: int = 25, optimizer: Type[torch.optim.Optimizer] = torch.optim.Adam, learning_rate: float = 0.1, - batch_size: int = 32, max_epochs: int = 10, + batch_size: int = 32, + tol: float = 1e-3, + n_iter_no_change: int = 25, verbose: int = 0, ) -> Dict: """Fit the GMM model. @@ -55,20 +55,20 @@ def fit( # type: ignore[override] ---------- x_ref Training data. - tol - Convergence threshold. Training iterations will stop when the lower bound average - gain is below this threshold. - n_iter_no_change - The number of iterations over which the loss must decrease by `tol` in order for - optimization to continue. optimizer Optimizer used to train the model. learning_rate Learning rate used to train the model. - batch_size - Batch size used to train the model. max_epochs Number of training max_epochs. + batch_size + Batch size used to train the model. + tol + Convergence threshold. Training iterations will stop when the lower bound average + gain is below this threshold. + n_iter_no_change + The number of iterations over which the loss must decrease by `tol` in order for + optimization to continue. verbose Verbosity level during training. 0 is silent, 1 a progress bar. From 1d98e80b3c17dea3c7839a328d49c3aaf221a0ad Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 15 May 2023 10:50:00 +0100 Subject: [PATCH 231/247] Fix type --- alibi_detect/od/_gmm.py | 2 +- alibi_detect/utils/pytorch/misc.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index d7d24b9c8..40fadeff0 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -108,7 +108,7 @@ def fit( tol Convergence threshold used to fit the detector. Used for both ``'sklearn'`` and ``'pytorch'`` backends. Defaults to ``1e-3``. - n_iter_no_change: + n_iter_no_change The number of iterations over which the loss must decrease by `tol` in order for optimization to continue. Only used if the ``'pytorch'`` backend is used. n_init diff --git a/alibi_detect/utils/pytorch/misc.py b/alibi_detect/utils/pytorch/misc.py index cc9585fa4..8f7e6e357 100644 --- a/alibi_detect/utils/pytorch/misc.py +++ b/alibi_detect/utils/pytorch/misc.py @@ -1,5 +1,5 @@ import logging -from typing import Optional, Union +from typing import Optional, Union, Type import torch @@ -94,7 +94,7 @@ def get_device(device: Optional[Union[str, torch.device]] = None) -> torch.devic return torch_device -def get_optimizer(name: str = 'Adam') -> torch.optim.Optimizer: +def get_optimizer(name: str = 'Adam') -> Type[torch.optim.Optimizer]: """ Get an optimizer class from its name. From dd298f8fc01d471e431caa80aecda055309b9197 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 15 May 2023 10:55:54 +0100 Subject: [PATCH 232/247] Minor change --- alibi_detect/od/pytorch/gmm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index 1fe570031..b8a96b419 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -89,9 +89,9 @@ def fit( # type: ignore[override] batch_size=batch_size, shuffle=True ) - optimizer_instance: torch.optim.Optimizer = optimizer( + optimizer_instance: torch.optim.Optimizer = optimizer( # type: ignore[call-arg] self.model.parameters(), - lr=learning_rate # type: ignore[call-arg] + lr=learning_rate ) self.model.train() From 95f100686875fa1fd55aa3bd41a5e762fd970ae1 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 15 May 2023 13:47:57 +0100 Subject: [PATCH 233/247] Change max_epochs docstring --- alibi_detect/od/_gmm.py | 7 ++++--- alibi_detect/od/pytorch/gmm.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 40fadeff0..34b627ab0 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -99,9 +99,10 @@ def fit( learning_rate Learning rate used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``0.1``. max_epochs - Number of max_epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` backends. - If the backend is ``'sklearn'``, the detector is fit using the EM algorithm. If the backend is ``'pytorch'``, - the detector is fitted using gradient descent. In both cases the number of max_epochs defaults to ``10``. + Maximum number of training epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` + backends. If the backend is ``'sklearn'``, the detector is fit using the EM algorithm. If the backend is + ``'pytorch'``, the detector is fitted using gradient descent. In both cases the number of max_epochs + defaults to ``10``. batch_size Batch size used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``None``. If ``None``, the entire dataset is used for each gradient update. diff --git a/alibi_detect/od/pytorch/gmm.py b/alibi_detect/od/pytorch/gmm.py index b8a96b419..ee16c4bc3 100644 --- a/alibi_detect/od/pytorch/gmm.py +++ b/alibi_detect/od/pytorch/gmm.py @@ -60,7 +60,7 @@ def fit( # type: ignore[override] learning_rate Learning rate used to train the model. max_epochs - Number of training max_epochs. + Maximum number of training epochs. batch_size Batch size used to train the model. tol From f3e4d15fa2397cfd8d28441b693c929e7a39b3d7 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 15 May 2023 14:10:52 +0100 Subject: [PATCH 234/247] Update docstring --- alibi_detect/od/_gmm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/_gmm.py b/alibi_detect/od/_gmm.py index 34b627ab0..2222110c8 100644 --- a/alibi_detect/od/_gmm.py +++ b/alibi_detect/od/_gmm.py @@ -73,7 +73,7 @@ def fit( x_ref: np.ndarray, optimizer: Optional[str] = 'Adam', learning_rate: float = 0.1, - max_epochs: Optional[int] = 10, + max_epochs: Optional[int] = None, batch_size: Optional[int] = None, tol: float = 1e-3, n_iter_no_change: int = 25, @@ -100,9 +100,9 @@ def fit( Learning rate used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``0.1``. max_epochs Maximum number of training epochs used to fit the detector. Used for both the ``'pytorch'`` and ``'sklearn'`` - backends. If the backend is ``'sklearn'``, the detector is fit using the EM algorithm. If the backend is - ``'pytorch'``, the detector is fitted using gradient descent. In both cases the number of max_epochs - defaults to ``10``. + backends. If the backend is ``'sklearn'``, the detector is fit using the EM algorithm and `max_epochs` + defaults to ``100``. If the backend is ``'pytorch'``, the detector is fitted using gradient descent and + `max_epochs` defaults to ``10``. batch_size Batch size used to fit the detector. Only used if the ``'pytorch'`` backend is used. Defaults to ``None``. If ``None``, the entire dataset is used for each gradient update. From 37ab4e8b87028f749930819b42671abbaa6cee6c Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 15 May 2023 15:53:01 +0100 Subject: [PATCH 235/247] Fix merge import errors --- alibi_detect/od/_lof.py | 8 ++++---- alibi_detect/od/tests/test__lof/test__lof.py | 9 +++++---- alibi_detect/od/tests/test__lof/test__lof_backend.py | 12 ++++++------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/alibi_detect/od/_lof.py b/alibi_detect/od/_lof.py index fa23a2e0c..d32fd06a9 100644 --- a/alibi_detect/od/_lof.py +++ b/alibi_detect/od/_lof.py @@ -5,10 +5,10 @@ from typing_extensions import Literal from alibi_detect.base import outlier_prediction_dict -from alibi_detect.od.base import TransformProtocol, transform_protocols +from alibi_detect.od.base import TransformProtocol, TransformProtocolType from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin from alibi_detect.od.pytorch import LOFTorch, Ensembler -from alibi_detect.od import normalizer_literals, aggregator_literals, get_aggregator, get_normalizer +from alibi_detect.od.base import get_aggregator, get_normalizer, NormalizerLiterals, AggregatorLiterals from alibi_detect.utils.frameworks import BackendValidator from alibi_detect.version import __version__ @@ -27,8 +27,8 @@ def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], kernel: Optional[Callable] = None, - normalizer: Optional[Union[transform_protocols, normalizer_literals]] = 'ShiftAndScaleNormalizer', - aggregator: Union[TransformProtocol, aggregator_literals] = 'AverageAggregator', + normalizer: Optional[Union[TransformProtocolType, NormalizerLiterals]] = 'ShiftAndScaleNormalizer', + aggregator: Union[TransformProtocol, AggregatorLiterals] = 'AverageAggregator', device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', ) -> None: diff --git a/alibi_detect/od/tests/test__lof/test__lof.py b/alibi_detect/od/tests/test__lof/test__lof.py index 0074ceae2..a7e78f211 100644 --- a/alibi_detect/od/tests/test__lof/test__lof.py +++ b/alibi_detect/od/tests/test__lof/test__lof.py @@ -3,9 +3,10 @@ import torch from alibi_detect.od._lof import LOF -from alibi_detect.od import AverageAggregator, TopKAggregator, MaxAggregator, \ +from alibi_detect.od.pytorch.ensemble import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer -from alibi_detect.base import NotFitException +from alibi_detect.exceptions import NotFittedError + from sklearn.datasets import make_moons @@ -26,7 +27,7 @@ def test_unfitted_lof_single_score(): x = np.array([[0, 10], [0.1, 0]]) # test predict raises exception when not fitted - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: _ = lof_detector.predict(x) assert str(err.value) == 'LOFTorch has not been fit!' @@ -95,7 +96,7 @@ def test_unfitted_lof_ensemble(aggregator, normalizer): x = np.array([[0, 10], [0.1, 0]]) # Test unfit lof ensemble raises exception when calling predict method. - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: _ = lof_detector.predict(x) assert str(err.value) == 'LOFTorch has not been fit!' diff --git a/alibi_detect/od/tests/test__lof/test__lof_backend.py b/alibi_detect/od/tests/test__lof/test__lof_backend.py index 96b7d6b6b..e7fca555c 100644 --- a/alibi_detect/od/tests/test__lof/test__lof_backend.py +++ b/alibi_detect/od/tests/test__lof/test__lof_backend.py @@ -4,7 +4,7 @@ from alibi_detect.od.pytorch.lof import LOFTorch from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.od.pytorch.ensemble import Ensembler, PValNormalizer, AverageAggregator -from alibi_detect.base import NotFitException, ThresholdNotInferredException +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError @pytest.fixture(scope='session') @@ -66,11 +66,11 @@ def test_lof_torch_backend_ensemble_ts(tmp_path, ensembler): lof_torch = LOFTorch(k=[4, 5], ensembler=ensembler) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: lof_torch(x) assert str(err.value) == 'LOFTorch has not been fit!' - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: lof_torch.predict(x) assert str(err.value) == 'LOFTorch has not been fit!' @@ -157,13 +157,13 @@ def test_lof_torch_backend_ensemble_fit_errors(k, ensembler): # Test that the backend raises an error if it is not fitted before # calling forward method. x = torch.randn((1, 10)) - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: lof_torch(x) assert str(err.value) == 'LOFTorch has not been fit!' # Test that the backend raises an error if it is not fitted before # predicting. - with pytest.raises(NotFitException) as err: + with pytest.raises(NotFittedError) as err: lof_torch.predict(x) assert str(err.value) == 'LOFTorch has not been fit!' @@ -174,7 +174,7 @@ def test_lof_torch_backend_ensemble_fit_errors(k, ensembler): # Test that the backend raises an if the forward method is called without the # threshold being inferred. - with pytest.raises(ThresholdNotInferredException) as err: + with pytest.raises(ThresholdNotInferredError) as err: lof_torch(x) assert str(err.value) == 'LOFTorch has no threshold set, call `infer_threshold` before predicting.' From 2af6641959a3736bdef9fad568a019940fe0b2e0 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Tue, 16 May 2023 17:19:30 +0100 Subject: [PATCH 236/247] Fix tests and typing --- alibi_detect/od/pytorch/lof.py | 36 +++++++++---------- .../od/tests/test__lof/test__lof_backend.py | 8 ++--- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 677ffa190..e50ceb70d 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -1,10 +1,11 @@ -from typing import Optional, Union, List, Tuple +from typing import Optional, Union, List, Tuple, Literal import numpy as np import torch from alibi_detect.od.pytorch.ensemble import Ensembler from alibi_detect.od.pytorch.base import TorchOutlierDetector +from torch import device class LOFTorch(TorchOutlierDetector): @@ -13,7 +14,7 @@ def __init__( k: Union[np.ndarray, List, Tuple], kernel: Optional[torch.nn.Module] = None, ensembler: Optional[Ensembler] = None, - device: Optional[Union[str, torch.device]] = None + device: Union[device, None, Literal['cuda', 'gpu', 'cpu']] = None ): """PyTorch backend for LOF detector. @@ -40,7 +41,6 @@ def __init__( self.ks = torch.tensor(k) if self.ensemble else torch.tensor([k], device=self.device) self.ensembler = ensembler - @torch.no_grad() def forward(self, x: torch.Tensor) -> torch.Tensor: """Detect if `x` is an outlier. @@ -74,7 +74,6 @@ def _make_mask(self, reachabilities: torch.Tensor): def _compute_K(self, x, y): return torch.exp(-self.kernel(x, y)) if self.kernel is not None else torch.cdist(x, y) - @torch.no_grad() def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` @@ -82,8 +81,8 @@ def score(self, x: torch.Tensor) -> torch.Tensor: 1. Compute the distance between each instance in `x` and the reference set. 2. Compute the k-nearest neighbors of each instance in `x` in the reference set. 3. Compute the reachability distance of each instance in `x` to its k-nearest neighbors. - 4. For each instance sum the inv_avg_reachabilities of its neighbours. - 5. LOF is average reachability of instance over average reachability of neighbours. + 4. For each instance sum the inv_avg_reachabilities of its neighbors. + 5. LOF is average reachability of instance over average reachability of neighbors. Parameters @@ -116,30 +115,29 @@ def score(self, x: torch.Tensor) -> torch.Tensor: lofs = (avg_reachabilities * factors) return lofs if self.ensemble else lofs[:, 0] - @torch.no_grad() - def _fit(self, x_ref: torch.Tensor): + def fit(self, x_ref: torch.Tensor): """Fits the detector - The LOF algorithm fit step prodeeds as follows: + The LOF algorithm fit step proceeds as follows: 1. Compute the distance matrix, D, between all instances in `x_ref`. - 2. For each instance, compute the k nearest neighbours. (Note we prevent an instance from - considering itself a neighbour by setting the diagonal of D to be the maximum value of D.) - 3. For each instance we store the distance to its kth nearest neighbour for each k in `ks`. - 4. For each instance and k in `ks` we obtain a tensor of the k neighbours k nearest neighbour + 2. For each instance, compute the k nearest neighbors. (Note we prevent an instance from + considering itself a neighbor by setting the diagonal of D to be the maximum value of D.) + 3. For each instance we store the distance to its kth nearest neighbor for each k in `ks`. + 4. For each instance and k in `ks` we obtain a tensor of the k neighbors k nearest neighbor distances. - 5. The reachability of an instance is the maximum of its k nearest neighbours distances and - the distance to its kth nearest neighbour. + 5. The reachability of an instance is the maximum of its k nearest neighbors distances and + the distance to its kth nearest neighbor. 6. The reachabilites tensor is of shape `(n_instances, max(ks), len(ks))`. Where the second - dimension is the each of the k neighbours nearest distances and the third dimension is + dimension is the each of the k neighbors nearest distances and the third dimension is the specific k. 7. The local reachability density is then given by 1 over the average reachability over the second dimension of this tensor. However we only want to consider the k nearest - neighbours for each k in `ks`, so we use a mask that prevents k from the second dimension + neighbors for each k in `ks`, so we use a mask that prevents k from the second dimension greater than k from the third dimension from being considered. This value is stored as we use it in the score step. 8. If multiple k are passed in ks then the detector also needs to fit the ensembler. To do so we need to score the x_ref as well. The local outlier factor (LOF) is then given by the - average reachability of an instance over the average reachability of its k neighbours. + average reachability of an instance over the average reachability of its k neighbors. Parameters ---------- @@ -164,3 +162,5 @@ def _fit(self, x_ref: torch.Tensor): factors = (self.ref_inv_avg_reachabilities[bot_k_inds]*mask[None, :, :]).sum(1) scores = (avg_reachabilities * factors) self.ensembler.fit(scores) + + self._set_fitted() diff --git a/alibi_detect/od/tests/test__lof/test__lof_backend.py b/alibi_detect/od/tests/test__lof/test__lof_backend.py index e7fca555c..368436fda 100644 --- a/alibi_detect/od/tests/test__lof/test__lof_backend.py +++ b/alibi_detect/od/tests/test__lof/test__lof_backend.py @@ -152,7 +152,7 @@ def test_lof_kernel_ts(ensembler): @pytest.mark.parametrize('k', [[4, 5], 4]) def test_lof_torch_backend_ensemble_fit_errors(k, ensembler): lof_torch = LOFTorch(k=[4, 5], ensembler=ensembler) - assert not lof_torch._fitted + assert not lof_torch.fitted # Test that the backend raises an error if it is not fitted before # calling forward method. @@ -167,16 +167,16 @@ def test_lof_torch_backend_ensemble_fit_errors(k, ensembler): lof_torch.predict(x) assert str(err.value) == 'LOFTorch has not been fit!' - # Test the backend updates _fitted flag on fit. + # Test the backend updates fitted flag on fit. x_ref = torch.randn((1024, 10)) lof_torch.fit(x_ref) - assert lof_torch._fitted + assert lof_torch.fitted # Test that the backend raises an if the forward method is called without the # threshold being inferred. with pytest.raises(ThresholdNotInferredError) as err: lof_torch(x) - assert str(err.value) == 'LOFTorch has no threshold set, call `infer_threshold` before predicting.' + assert str(err.value) == 'LOFTorch has no threshold set, call `infer_threshold` to fit one!' # Test that the backend can call predict without the threshold being inferred. assert lof_torch.predict(x) From 6409bb75b4e9d0cfbe9c3725c5fcd5b696ae5df3 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 17 May 2023 16:34:15 +0100 Subject: [PATCH 237/247] Update tests --- alibi_detect/od/_lof.py | 15 ++- alibi_detect/od/pytorch/lof.py | 10 -- alibi_detect/od/tests/test__lof/test__lof.py | 104 ++++++++++++------ .../od/tests/test__lof/test__lof_backend.py | 94 +++++++++++----- 4 files changed, 149 insertions(+), 74 deletions(-) diff --git a/alibi_detect/od/_lof.py b/alibi_detect/od/_lof.py index d32fd06a9..f08e609df 100644 --- a/alibi_detect/od/_lof.py +++ b/alibi_detect/od/_lof.py @@ -4,6 +4,7 @@ import numpy as np from typing_extensions import Literal +from alibi_detect.exceptions import _catch_error as catch_error from alibi_detect.base import outlier_prediction_dict from alibi_detect.od.base import TransformProtocol, TransformProtocolType from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin @@ -42,9 +43,9 @@ def __init__( Parameters ---------- k - Number of neirest neighbors to compute distance to. `k` can be a single value or + Number of nearest neighbors to compute distance to. `k` can be a single value or an array of integers. If an array is passed, an aggregator is required to aggregate - the scores. If `k` is a single value we comput the local outlier factor for that `k`. + the scores. If `k` is a single value we compute the local outlier factor for that `k`. Otherwise if `k` is a list then we compute and aggregate the local outlier factor for each value in `k`. kernel @@ -52,7 +53,7 @@ def __init__( Otherwise if a kernel is specified then instead of using `torch.cdist` the kernel defines the k nearest neighbor distance. normalizer - Normalizer to use for outlier detection. If ``None``, no normalisation is applied. + Normalizer to use for outlier detection. If ``None``, no normalization is applied. For a list of available normalizers, see :mod:`alibi_detect.od.pytorch.ensemble`. aggregator Aggregator to use for outlier detection. Can be set to ``None`` if `k` is a single @@ -108,6 +109,8 @@ def fit(self, x_ref: np.ndarray) -> None: """ self.backend.fit(self.backend._to_tensor(x_ref)) + @catch_error('NotFittedError') + @catch_error('ThresholdNotInferredError') def score(self, X: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. @@ -125,12 +128,14 @@ def score(self, X: np.ndarray) -> np.ndarray: instance. """ score = self.backend.score(self.backend._to_tensor(X)) + score = self.backend._ensembler(score) return self.backend._to_numpy(score) + @catch_error('NotFittedError') def infer_threshold(self, X: np.ndarray, fpr: float) -> None: """Infer the threshold for the LOF detector. - The threshold is computed so that the outlier detector would incorectly classify `fpr` proportion of the + The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the reference data as outliers. Parameters @@ -144,6 +149,8 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: """ self.backend.infer_threshold(self.backend._to_tensor(X), fpr) + @catch_error('NotFittedError') + @catch_error('ThresholdNotInferredError') def predict(self, x: np.ndarray) -> Dict[str, Any]: """Predict whether the instances in `x` are outliers or not. diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index e50ceb70d..22e7e70d6 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -84,7 +84,6 @@ def score(self, x: torch.Tensor) -> torch.Tensor: 4. For each instance sum the inv_avg_reachabilities of its neighbors. 5. LOF is average reachability of instance over average reachability of neighbors. - Parameters ---------- x @@ -135,9 +134,6 @@ def fit(self, x_ref: torch.Tensor): neighbors for each k in `ks`, so we use a mask that prevents k from the second dimension greater than k from the third dimension from being considered. This value is stored as we use it in the score step. - 8. If multiple k are passed in ks then the detector also needs to fit the ensembler. To do so - we need to score the x_ref as well. The local outlier factor (LOF) is then given by the - average reachability of an instance over the average reachability of its k neighbors. Parameters ---------- @@ -157,10 +153,4 @@ def fit(self, x_ref: torch.Tensor): avg_reachabilities = (reachabilities*mask[None, :, :]).sum(1) self.ref_inv_avg_reachabilities = 1/avg_reachabilities self.x_ref = X - - if self.ensemble: - factors = (self.ref_inv_avg_reachabilities[bot_k_inds]*mask[None, :, :]).sum(1) - scores = (avg_reachabilities * factors) - self.ensembler.fit(scores) - self._set_fitted() diff --git a/alibi_detect/od/tests/test__lof/test__lof.py b/alibi_detect/od/tests/test__lof/test__lof.py index a7e78f211..a892d41de 100644 --- a/alibi_detect/od/tests/test__lof/test__lof.py +++ b/alibi_detect/od/tests/test__lof/test__lof.py @@ -5,8 +5,7 @@ from alibi_detect.od._lof import LOF from alibi_detect.od.pytorch.ensemble import AverageAggregator, TopKAggregator, MaxAggregator, \ MinAggregator, ShiftAndScaleNormalizer, PValNormalizer -from alibi_detect.exceptions import NotFittedError - +from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError from sklearn.datasets import make_moons @@ -25,32 +24,60 @@ def make_lof_detector(k=5, aggregator=None, normalizer=None): def test_unfitted_lof_single_score(): lof_detector = LOF(k=10) x = np.array([[0, 10], [0.1, 0]]) + x_ref = np.random.randn(100, 2) + + # test infer_threshold raises exception when not fitted + with pytest.raises(NotFittedError) as err: + _ = lof_detector.infer_threshold(x_ref, 0.1) + assert str(err.value) == 'LOF has not been fit!' + + # test score raises exception when not fitted + with pytest.raises(NotFittedError) as err: + _ = lof_detector.score(x) + assert str(err.value) == 'LOF has not been fit!' # test predict raises exception when not fitted with pytest.raises(NotFittedError) as err: _ = lof_detector.predict(x) - assert str(err.value) == 'LOFTorch has not been fit!' + assert str(err.value) == 'LOF has not been fit!' -@pytest.mark.parametrize('k', [10, [8, 9, 10]]) -def test_fitted_lof_single_score(k): - lof_detector = LOF(k=k) +def test_fitted_lof_score(): + """ + Test fitted but not threshold inferred non-ensemble detectors can still score data using the predict method. + Unlike the ensemble detectors, the non-ensemble detectors do not require the ensembler to be fit in the + infer_threshold method. See the test_fitted_lof_ensemble_score test for the ensemble case. + """ + lof_detector = LOF(k=10) x_ref = np.random.randn(100, 2) lof_detector.fit(x_ref) x = np.array([[0, 10], [0.1, 0]]) - # test fitted but not threshold inferred detectors - # can still score data using the predict method. y = lof_detector.predict(x) y = y['data'] - assert y['instance_score'][0] > 7 - assert y['instance_score'][1] < 2 - + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 assert not y['threshold_inferred'] assert y['threshold'] is None assert y['is_outlier'] is None assert y['p_value'] is None +def test_fitted_lof_ensemble_score(): + """ + Test fitted but not threshold inferred ensemble detectors correctly raise an error when calling + the predict method. This is because the ensembler is fit in the infer_threshold method. + """ + lof_detector = LOF(k=[10, 14, 18]) + x_ref = np.random.randn(100, 2) + lof_detector.fit(x_ref) + x = np.array([[0, 10], [0.1, 0]]) + with pytest.raises(ThresholdNotInferredError): + lof_detector.predict(x) + + with pytest.raises(ThresholdNotInferredError): + lof_detector.score(x) + + def test_incorrect_lof_ensemble_init(): # test lof ensemble with aggregator passed as None raises exception @@ -76,8 +103,8 @@ def test_fitted_lof_predict(): y = y['data'] scores = lof_detector.score(x) assert np.all(y['instance_score'] == scores) - assert y['instance_score'][0] > 7 - assert y['instance_score'][1] < 2 + assert y['instance_score'][0] > 5 + assert y['instance_score'][1] < 1 assert y['threshold_inferred'] assert y['threshold'] is not None assert y['p_value'].all() @@ -98,7 +125,7 @@ def test_unfitted_lof_ensemble(aggregator, normalizer): # Test unfit lof ensemble raises exception when calling predict method. with pytest.raises(NotFittedError) as err: _ = lof_detector.predict(x) - assert str(err.value) == 'LOFTorch has not been fit!' + assert str(err.value) == 'LOF has not been fit!' @pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), @@ -114,14 +141,13 @@ def test_fitted_lof_ensemble(aggregator, normalizer): lof_detector.fit(x_ref) x = np.array([[0, 10], [0, 0.1]]) - # test fitted but not threshold inferred detectors can still score data using the predict method. - y = lof_detector.predict(x) - y = y['data'] - assert y['instance_score'].all() - assert not y['threshold_inferred'] - assert y['threshold'] is None - assert y['is_outlier'] is None - assert y['p_value'] is None + # test ensemble raises ThresholdNotInferredError if only fit and not threshold inferred and + # the normalizer is not None. + if normalizer() is not None: + with pytest.raises(ThresholdNotInferredError): + lof_detector.predict(x) + else: + lof_detector.predict(x) @pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), @@ -143,27 +169,31 @@ def test_fitted_lof_ensemble_predict(aggregator, normalizer): assert y['p_value'].all() assert (y['is_outlier'] == [True, False]).all() + # test fitted detectors with inferred thresholds can score data using the score method. + scores = lof_detector.score(x) + assert np.all(y['instance_score'] == scores) + @pytest.mark.parametrize("aggregator", [AverageAggregator, lambda: TopKAggregator(k=7), MaxAggregator, MinAggregator]) @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None]) def test_lof_ensemble_torch_script(aggregator, normalizer): lof_detector = make_lof_detector(k=[5, 6, 7], aggregator=aggregator(), normalizer=normalizer()) - tslof = torch.jit.script(lof_detector.backend) + ts_lof = torch.jit.script(lof_detector.backend) x = torch.tensor([[0, 10], [0, 0.1]]) # test torchscripted ensemble lof detector can be saved and loaded correctly. - y = tslof(x) + y = ts_lof(x) assert torch.all(y == torch.tensor([True, False])) def test_lof_single_torchscript(): lof_detector = make_lof_detector(k=5) - tslof = torch.jit.script(lof_detector.backend) + ts_lof = torch.jit.script(lof_detector.backend) x = torch.tensor([[0, 10], [0, 0.1]]) # test torchscripted single lof detector can be saved and loaded correctly. - y = tslof(x) + y = ts_lof(x) assert torch.all(y == torch.tensor([True, False])) @@ -173,7 +203,7 @@ def test_lof_single_torchscript(): lambda: 'MinAggregator']) @pytest.mark.parametrize("normalizer", [ShiftAndScaleNormalizer, PValNormalizer, lambda: None, lambda: 'ShiftAndScaleNormalizer', lambda: 'PValNormalizer']) -def test_lof_ensemble_integration(aggregator, normalizer): +def test_lof_ensemble_integration(tmp_path, aggregator, normalizer): """Test lof ensemble detector on moons dataset. Tests ensemble lof detector with every combination of aggregator and normalizer on the moons dataset. @@ -199,13 +229,18 @@ def test_lof_ensemble_integration(aggregator, normalizer): result = result['data']['is_outlier'][0] assert result - tslof = torch.jit.script(lof_detector.backend) + ts_lof = torch.jit.script(lof_detector.backend) x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) - y = tslof(x) + y = ts_lof(x) assert torch.all(y == torch.tensor([False, True])) + ts_lof.save(tmp_path / 'lof.pt') + lof_detector = torch.load(tmp_path / 'lof.pt') + y = lof_detector(x) + assert torch.all(y == torch.tensor([False, True])) -def test_lof_integration(): + +def test_lof_integration(tmp_path): """Test lof detector on moons dataset. Tests lof detector on the moons dataset. Fits and infers thresholds and verifies that the detector can @@ -225,7 +260,12 @@ def test_lof_integration(): result = result['data']['is_outlier'][0] assert result - tslof = torch.jit.script(lof_detector.backend) + ts_lof = torch.jit.script(lof_detector.backend) x = torch.tensor([x_inlier[0], x_outlier[0]], dtype=torch.float32) - y = tslof(x) + y = ts_lof(x) + assert torch.all(y == torch.tensor([False, True])) + + ts_lof.save(tmp_path / 'lof.pt') + lof_detector = torch.load(tmp_path / 'lof.pt') + y = lof_detector(x) assert torch.all(y == torch.tensor([False, True])) diff --git a/alibi_detect/od/tests/test__lof/test__lof_backend.py b/alibi_detect/od/tests/test__lof/test__lof_backend.py index 368436fda..925dd6847 100644 --- a/alibi_detect/od/tests/test__lof/test__lof_backend.py +++ b/alibi_detect/od/tests/test__lof/test__lof_backend.py @@ -7,7 +7,7 @@ from alibi_detect.exceptions import NotFittedError, ThresholdNotInferredError -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def ensembler(request): return Ensembler( normalizer=PValNormalizer(), @@ -48,9 +48,6 @@ def test_lof_torch_backend_ensemble(ensembler): x_ref = torch.randn((1024, 10)) lof_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - result = lof_torch.predict(x) - assert result.instance_score.shape == (3, ) - lof_torch.infer_threshold(x_ref, 0.1) outputs = lof_torch.predict(x) assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) @@ -59,21 +56,12 @@ def test_lof_torch_backend_ensemble(ensembler): def test_lof_torch_backend_ensemble_ts(tmp_path, ensembler): """ - Test the lof torch backend can be initalized as an ensemble and - torchscripted, as well as saved and loaded to and from disk. + Test the lof torch backend can be initialized as an ensemble and + torch scripted, as well as saved and loaded to and from disk. """ lof_torch = LOFTorch(k=[4, 5], ensembler=ensembler) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - - with pytest.raises(NotFittedError) as err: - lof_torch(x) - assert str(err.value) == 'LOFTorch has not been fit!' - - with pytest.raises(NotFittedError) as err: - lof_torch.predict(x) - assert str(err.value) == 'LOFTorch has not been fit!' - x_ref = torch.randn((1024, 10)) lof_torch.fit(x_ref) lof_torch.infer_threshold(x_ref, 0.1) @@ -90,7 +78,7 @@ def test_lof_torch_backend_ensemble_ts(tmp_path, ensembler): def test_lof_torch_backend_ts(tmp_path): """ - Test the lof torch backend can be initalized and torchscripted, as well as + Test the lof torch backend can be initialized and torch scripted, as well as saved and loaded to and from disk. """ @@ -116,25 +104,22 @@ def test_lof_kernel(ensembler): on data and used to predict outliers. """ - kernel = GaussianRBF(sigma=torch.tensor((1))) + kernel = GaussianRBF(sigma=torch.tensor((0.25))) lof_torch = LOFTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) x_ref = torch.randn((1024, 10)) lof_torch.fit(x_ref) x = torch.randn((3, 10)) * torch.tensor([[1], [1], [100]]) - result = lof_torch.predict(x) - assert result.instance_score.shape == (3,) - lof_torch.infer_threshold(x_ref, 0.1) outputs = lof_torch.predict(x) - assert torch.all(outputs.is_outlier == torch.tensor([False, False, True])) - assert torch.all(lof_torch(x) == torch.tensor([False, False, True])) + assert torch.all(outputs.is_outlier == torch.tensor([0, 0, 1])) + assert torch.all(lof_torch(x) == torch.tensor([0, 0, 1])) -@pytest.mark.skip(reason="Can't convert GaussianRBF to torchscript due to torchscript type constraints") +@pytest.mark.skip(reason="Can't convert GaussianRBF to torch script due to torch script type constraints") def test_lof_kernel_ts(ensembler): """ Test the lof torch backend can be correctly initialized with a kernel, - and torchscripted, as well as saved and loaded to and from disk. + and torch scripted, as well as saved and loaded to and from disk. """ kernel = GaussianRBF(sigma=torch.tensor((0.25))) @@ -149,10 +134,9 @@ def test_lof_kernel_ts(ensembler): assert torch.all(pred_1 == pred_2) -@pytest.mark.parametrize('k', [[4, 5], 4]) -def test_lof_torch_backend_ensemble_fit_errors(k, ensembler): +def test_lof_torch_backend_ensemble_fit_errors(ensembler): + """Tests the correct errors are raised when using the LOFTorch backend as an ensemble.""" lof_torch = LOFTorch(k=[4, 5], ensembler=ensembler) - assert not lof_torch.fitted # Test that the backend raises an error if it is not fitted before # calling forward method. @@ -179,4 +163,58 @@ def test_lof_torch_backend_ensemble_fit_errors(k, ensembler): assert str(err.value) == 'LOFTorch has no threshold set, call `infer_threshold` to fit one!' # Test that the backend can call predict without the threshold being inferred. - assert lof_torch.predict(x) + with pytest.raises(ThresholdNotInferredError) as err: + lof_torch.predict(x) + assert str(err.value) == 'LOFTorch has no threshold set, call `infer_threshold` to fit one!' + + +def test_lof_torch_backend_fit_errors(): + """Tests the correct errors are raised when using the LOFTorch backend as a single detector.""" + lof_torch = LOFTorch(k=4) + + # Test that the backend raises an error if it is not fitted before + # calling forward method. + x = torch.randn((1, 10)) + with pytest.raises(NotFittedError) as err: + lof_torch(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + # Test that the backend raises an error if it is not fitted before + # predicting. + with pytest.raises(NotFittedError) as err: + lof_torch.predict(x) + assert str(err.value) == 'LOFTorch has not been fit!' + + # Test the backend updates fitted flag on fit. + x_ref = torch.randn((1024, 10)) + lof_torch.fit(x_ref) + assert lof_torch.fitted + + # Test that the backend raises an if the forward method is called without the + # threshold being inferred. + with pytest.raises(ThresholdNotInferredError) as err: + lof_torch(x) + assert str(err.value) == 'LOFTorch has no threshold set, call `infer_threshold` to fit one!' + + # Test that the backend can call predict without the threshold being inferred. + lof_torch.predict(x) + + +def test_lof_infer_threshold_value_errors(): + """Tests the correct errors are raised when using incorrect choice of fpr for the LOFTorch backend detector.""" + lof_torch = LOFTorch(k=4) + x = torch.randn((1024, 10)) + lof_torch.fit(x) + + # fpr must be greater than 1/len(x) otherwise it excludes all points in the reference dataset + with pytest.raises(ValueError) as err: + lof_torch.infer_threshold(x, 1/1025) + assert str(err.value) == '`fpr` must be greater than `1/len(x)=0.0009765625`.' + + # fpr must be between 0 and 1 + with pytest.raises(ValueError) as err: + lof_torch.infer_threshold(x, 1.1) + assert str(err.value) == '`fpr` must be in `(0, 1)`.' + + lof_torch.infer_threshold(x, 0.99) + lof_torch.infer_threshold(x, 1/1023) From 023b4c37211a156de5c40c2defb3998bc762db28 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 22 May 2023 11:02:19 +0100 Subject: [PATCH 238/247] Update optional dep tests for LOFTorch --- alibi_detect/tests/test_dep_management.py | 1 + 1 file changed, 1 insertion(+) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index b51c7c6fd..c09a4d64a 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -142,6 +142,7 @@ def test_od_backend_dependencies(opt_dep): ('KernelPCATorch', ['torch', 'keops']), ('LinearPCATorch', ['torch', 'keops']), ('GMMTorch', ['torch', 'keops']), + ('LOFTorch', ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.od import pytorch as od_pt_backend From a88630ae32c694fbdc948d027c5446280bbd2ab1 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 22 May 2023 11:12:10 +0100 Subject: [PATCH 239/247] Fix issues in _lof --- alibi_detect/od/_lof.py | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/alibi_detect/od/_lof.py b/alibi_detect/od/_lof.py index f08e609df..ce22ef076 100644 --- a/alibi_detect/od/_lof.py +++ b/alibi_detect/od/_lof.py @@ -1,11 +1,11 @@ from typing import Callable, Union, Optional, Dict, Any, List, Tuple from typing import TYPE_CHECKING +from typing_extensions import Literal import numpy as np -from typing_extensions import Literal -from alibi_detect.exceptions import _catch_error as catch_error from alibi_detect.base import outlier_prediction_dict +from alibi_detect.exceptions import _catch_error as catch_error from alibi_detect.od.base import TransformProtocol, TransformProtocolType from alibi_detect.base import BaseDetector, FitMixin, ThresholdMixin from alibi_detect.od.pytorch import LOFTorch, Ensembler @@ -28,10 +28,10 @@ def __init__( self, k: Union[int, np.ndarray, List[int], Tuple[int]], kernel: Optional[Callable] = None, - normalizer: Optional[Union[TransformProtocolType, NormalizerLiterals]] = 'ShiftAndScaleNormalizer', + normalizer: Optional[Union[TransformProtocolType, NormalizerLiterals]] = 'PValNormalizer', aggregator: Union[TransformProtocol, AggregatorLiterals] = 'AverageAggregator', - device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, backend: Literal['pytorch'] = 'pytorch', + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ) -> None: """ Local Outlier Factor (LOF) outlier detector. @@ -62,7 +62,8 @@ def __init__( Backend used for outlier detection. Defaults to ``'pytorch'``. Options are ``'pytorch'``. device Device type used. The default tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either ``'cuda'``, ``'gpu'`` or ``'cpu'``. + Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of + ``torch.device``. Raises ------ @@ -111,7 +112,7 @@ def fit(self, x_ref: np.ndarray) -> None: @catch_error('NotFittedError') @catch_error('ThresholdNotInferredError') - def score(self, X: np.ndarray) -> np.ndarray: + def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. The LOF detector scores the instances in `x` by computing the local outlier factor for each instance. The @@ -126,13 +127,20 @@ def score(self, X: np.ndarray) -> np.ndarray: ------- Outlier scores. The shape of the scores is `(n_instances,)`. The higher the score, the more anomalous the \ instance. + + Raises + ------ + NotFittedError + If called before detector has been fit. + ThresholdNotInferredError + If k is a list and a threshold was not inferred. """ - score = self.backend.score(self.backend._to_tensor(X)) + score = self.backend.score(self.backend._to_tensor(x)) score = self.backend._ensembler(score) return self.backend._to_numpy(score) @catch_error('NotFittedError') - def infer_threshold(self, X: np.ndarray, fpr: float) -> None: + def infer_threshold(self, x: np.ndarray, fpr: float) -> None: """Infer the threshold for the LOF detector. The threshold is computed so that the outlier detector would incorrectly classify `fpr` proportion of the @@ -146,8 +154,15 @@ def infer_threshold(self, X: np.ndarray, fpr: float) -> None: False positive rate used to infer the threshold. The false positive rate is the proportion of instances in `x_ref` that are incorrectly classified as outliers. The false positive rate should be in the range ``(0, 1)``. + + Raises + ------ + ValueError + Raised if `fpr` is not in ``(0, 1)``. + NotFittedError + If called before detector has been fit. """ - self.backend.infer_threshold(self.backend._to_tensor(X), fpr) + self.backend.infer_threshold(self.backend._to_tensor(x), fpr) @catch_error('NotFittedError') @catch_error('ThresholdNotInferredError') @@ -167,6 +182,13 @@ def predict(self, x: np.ndarray) -> Dict[str, Any]: performed, 'data' also contains the threshold value, outlier labels and p-vals . The shape of the scores is \ `(n_instances,)`. The higher the score, the more anomalous the instance. 'meta' contains information about \ the detector. + + Raises + ------ + NotFittedError + If called before detector has been fit. + ThresholdNotInferredError + If k is a list and a threshold was not inferred. """ outputs = self.backend.predict(self.backend._to_tensor(x)) output = outlier_prediction_dict() From c13572dd3cae4f95f8cf7f7c5a7170fd91a7afda Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 24 May 2023 15:15:17 +0100 Subject: [PATCH 240/247] Add comments to fit and score logic --- alibi_detect/od/pytorch/lof.py | 70 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 22e7e70d6..914d79248 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -77,13 +77,6 @@ def _compute_K(self, x, y): def score(self, x: torch.Tensor) -> torch.Tensor: """Computes the score of `x` - The score step proceeds as follows: - 1. Compute the distance between each instance in `x` and the reference set. - 2. Compute the k-nearest neighbors of each instance in `x` in the reference set. - 3. Compute the reachability distance of each instance in `x` to its k-nearest neighbors. - 4. For each instance sum the inv_avg_reachabilities of its neighbors. - 5. LOF is average reachability of instance over average reachability of neighbors. - Parameters ---------- x @@ -101,56 +94,65 @@ def score(self, x: torch.Tensor) -> torch.Tensor: if not torch.jit.is_scripting(): self.check_fitted() - X = torch.as_tensor(x) - D = self._compute_K(X, self.x_ref) + # compute the distance matrix between x and x_ref + D = self._compute_K(x, self.x_ref) + + # compute k nearest neighbors for maximum k in self.ks max_k = torch.max(self.ks) bot_k_items = torch.topk(D, int(max_k), dim=1, largest=False) bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values + + # To compute the reachabilities we get the k-distances of each object in the instances + # k nearest neighbors. Then we take the maximum of their k-distances and the distance + # to the instance. lower_bounds = self.knn_dists_ref[bot_k_inds] reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) + + # Compute the average reachability for each instance. We use a mask to manage each k in + # self.ks separately. mask = self._make_mask(reachabilities) avg_reachabilities = (reachabilities*mask[None, :, :]).sum(1) - factors = (self.ref_inv_avg_reachabilities[bot_k_inds]*mask[None, :, :]).sum(1) + + # Compute the LOF score for each instance. Note we don't take 1/avg_reachabilities as + # avg_reachabilities is the denominator in the LOF formula. + factors = (self.ref_inv_avg_reachabilities[bot_k_inds] * mask[None, :, :]).sum(1) lofs = (avg_reachabilities * factors) return lofs if self.ensemble else lofs[:, 0] def fit(self, x_ref: torch.Tensor): """Fits the detector - The LOF algorithm fit step proceeds as follows: - 1. Compute the distance matrix, D, between all instances in `x_ref`. - 2. For each instance, compute the k nearest neighbors. (Note we prevent an instance from - considering itself a neighbor by setting the diagonal of D to be the maximum value of D.) - 3. For each instance we store the distance to its kth nearest neighbor for each k in `ks`. - 4. For each instance and k in `ks` we obtain a tensor of the k neighbors k nearest neighbor - distances. - 5. The reachability of an instance is the maximum of its k nearest neighbors distances and - the distance to its kth nearest neighbor. - 6. The reachabilites tensor is of shape `(n_instances, max(ks), len(ks))`. Where the second - dimension is the each of the k neighbors nearest distances and the third dimension is - the specific k. - 7. The local reachability density is then given by 1 over the average reachability - over the second dimension of this tensor. However we only want to consider the k nearest - neighbors for each k in `ks`, so we use a mask that prevents k from the second dimension - greater than k from the third dimension from being considered. This value is stored as - we use it in the score step. - Parameters ---------- x_ref The Dataset tensor. """ - X = torch.as_tensor(x_ref) - D = self._compute_K(X, X) + # compute the distance matrix + D = self._compute_K(x_ref, x_ref) + # set diagonal to max distance to prevent torch.topk from returning the instance itself D += torch.eye(len(D), device=self.device) * torch.max(D) + + # compute k nearest neighbors for maximum k in self.ks max_k = torch.max(self.ks) bot_k_items = torch.topk(D, int(max_k), dim=1, largest=False) - bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values + bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values # shape (n_instances, max(ks)) + + # store the k-distances for each instance for each k. shape (n_instances, len(ks)) self.knn_dists_ref = bot_k_dists[:, self.ks-1] + + # To compute the reachabilities we get the k-distances of each object in the instances + # k nearest neighbors. Then we take the maximum of their k-distances and the distance + # to the instance. lower_bounds = self.knn_dists_ref[bot_k_inds] - reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) + reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) # shape (n_instances, max(ks), len(ks)) + + # Compute the average reachability for each instance. We use a mask to manage each k in + # self.ks separately. mask = self._make_mask(reachabilities) avg_reachabilities = (reachabilities*mask[None, :, :]).sum(1) - self.ref_inv_avg_reachabilities = 1/avg_reachabilities - self.x_ref = X + + # Compute the inverse average reachability for each instance. + self.ref_inv_avg_reachabilities = 1/avg_reachabilities # shape (n_instances, len(ks)) + + self.x_ref = x_ref self._set_fitted() From 60003098565b9adade42125bb996882863da62c6 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 24 May 2023 15:18:45 +0100 Subject: [PATCH 241/247] Remove shape comments --- alibi_detect/od/pytorch/lof.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 914d79248..9526801ab 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -135,16 +135,16 @@ def fit(self, x_ref: torch.Tensor): # compute k nearest neighbors for maximum k in self.ks max_k = torch.max(self.ks) bot_k_items = torch.topk(D, int(max_k), dim=1, largest=False) - bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values # shape (n_instances, max(ks)) + bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values - # store the k-distances for each instance for each k. shape (n_instances, len(ks)) + # store the k-distances for each instance for each k. self.knn_dists_ref = bot_k_dists[:, self.ks-1] # To compute the reachabilities we get the k-distances of each object in the instances # k nearest neighbors. Then we take the maximum of their k-distances and the distance # to the instance. lower_bounds = self.knn_dists_ref[bot_k_inds] - reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) # shape (n_instances, max(ks), len(ks)) + reachabilities = torch.max(bot_k_dists[:, :, None], lower_bounds) # Compute the average reachability for each instance. We use a mask to manage each k in # self.ks separately. @@ -152,7 +152,7 @@ def fit(self, x_ref: torch.Tensor): avg_reachabilities = (reachabilities*mask[None, :, :]).sum(1) # Compute the inverse average reachability for each instance. - self.ref_inv_avg_reachabilities = 1/avg_reachabilities # shape (n_instances, len(ks)) + self.ref_inv_avg_reachabilities = 1/avg_reachabilities self.x_ref = x_ref self._set_fitted() From 3f091ad0cdfa9d3d411f4cb80da43618ff633e5f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 24 May 2023 15:23:09 +0100 Subject: [PATCH 242/247] Fix tests --- alibi_detect/od/tests/test__lof/test__lof.py | 6 ++---- alibi_detect/od/tests/test__lof/test__lof_backend.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/alibi_detect/od/tests/test__lof/test__lof.py b/alibi_detect/od/tests/test__lof/test__lof.py index a892d41de..f46d126d4 100644 --- a/alibi_detect/od/tests/test__lof/test__lof.py +++ b/alibi_detect/od/tests/test__lof/test__lof.py @@ -54,8 +54,7 @@ def test_fitted_lof_score(): x = np.array([[0, 10], [0.1, 0]]) y = lof_detector.predict(x) y = y['data'] - assert y['instance_score'][0] > 5 - assert y['instance_score'][1] < 1 + assert y['instance_score'][0] > y['instance_score'][1] assert not y['threshold_inferred'] assert y['threshold'] is None assert y['is_outlier'] is None @@ -103,8 +102,7 @@ def test_fitted_lof_predict(): y = y['data'] scores = lof_detector.score(x) assert np.all(y['instance_score'] == scores) - assert y['instance_score'][0] > 5 - assert y['instance_score'][1] < 1 + assert y['instance_score'][0] > y['instance_score'][1] assert y['threshold_inferred'] assert y['threshold'] is not None assert y['p_value'].all() diff --git a/alibi_detect/od/tests/test__lof/test__lof_backend.py b/alibi_detect/od/tests/test__lof/test__lof_backend.py index 925dd6847..fd41e7c6d 100644 --- a/alibi_detect/od/tests/test__lof/test__lof_backend.py +++ b/alibi_detect/od/tests/test__lof/test__lof_backend.py @@ -104,7 +104,7 @@ def test_lof_kernel(ensembler): on data and used to predict outliers. """ - kernel = GaussianRBF(sigma=torch.tensor((0.25))) + kernel = GaussianRBF(sigma=torch.tensor((1))) lof_torch = LOFTorch(k=[4, 5], kernel=kernel, ensembler=ensembler) x_ref = torch.randn((1024, 10)) lof_torch.fit(x_ref) From d52c2f6256fecb222280e3839fa0a1833c71c4f4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 24 May 2023 15:47:28 +0100 Subject: [PATCH 243/247] Update lof docstrings --- alibi_detect/od/_lof.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/alibi_detect/od/_lof.py b/alibi_detect/od/_lof.py index ce22ef076..72eb4e25e 100644 --- a/alibi_detect/od/_lof.py +++ b/alibi_detect/od/_lof.py @@ -40,6 +40,16 @@ def __init__( deviation of a given data point with respect to its neighbors. It considers as outliers the samples that have a substantially lower density than their neighbors. + The detector can be initialized with `k` a single value or an array of values. If `k` is a single value then + the score method uses the distance/kernel similarity to the k-th nearest neighbor. If `k` is an array of + values then the score method uses the distance/kernel similarity to each of the specified `k` neighbors. + In the latter case, an `aggregator` must be specified to aggregate the scores. + + Note that, in the multiple k case, a normalizer can be provided. If a normalizer is passed then it is fit in + the `infer_threshold` method and so this method must be called before the `predict` method. If this is not + done an exception is raised. If `k` is a single value then the predict method can be called without first + calling `infer_threshold` but only scores will be returned and not outlier predictions. + Parameters ---------- k @@ -115,8 +125,8 @@ def fit(self, x_ref: np.ndarray) -> None: def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. - The LOF detector scores the instances in `x` by computing the local outlier factor for each instance. The - higher the score, the more anomalous the instance. + Computes the local outlier factor for each instance in `x`. If `k` is an array of values then the score for + each `k` is aggregated using the ensembler. Parameters ---------- @@ -148,11 +158,11 @@ def infer_threshold(self, x: np.ndarray, fpr: float) -> None: Parameters ---------- - x_ref + x Reference data used to infer the threshold. fpr False positive rate used to infer the threshold. The false positive rate is the proportion of - instances in `x_ref` that are incorrectly classified as outliers. The false positive rate should + instances in `x` that are incorrectly classified as outliers. The false positive rate should be in the range ``(0, 1)``. Raises From b7b4a66711a2ed06f03d6d4ab1b75b397b43c856 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 24 May 2023 16:20:53 +0100 Subject: [PATCH 244/247] Update docstrings for lof backend --- alibi_detect/od/_lof.py | 2 +- alibi_detect/od/pytorch/lof.py | 44 +++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/alibi_detect/od/_lof.py b/alibi_detect/od/_lof.py index 72eb4e25e..b9766342b 100644 --- a/alibi_detect/od/_lof.py +++ b/alibi_detect/od/_lof.py @@ -125,7 +125,7 @@ def fit(self, x_ref: np.ndarray) -> None: def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. - Computes the local outlier factor for each instance in `x`. If `k` is an array of values then the score for + Computes the local outlier factor for each instance in `x`. If `k` is an array of values then the score for each `k` is aggregated using the ensembler. Parameters diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 9526801ab..84c6a0ba5 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -1,30 +1,29 @@ -from typing import Optional, Union, List, Tuple, Literal - +from typing import Optional, Union, List, Tuple +from typing_extensions import Literal import numpy as np import torch from alibi_detect.od.pytorch.ensemble import Ensembler from alibi_detect.od.pytorch.base import TorchOutlierDetector -from torch import device class LOFTorch(TorchOutlierDetector): def __init__( self, - k: Union[np.ndarray, List, Tuple], + k: Union[np.ndarray, List, Tuple, int], kernel: Optional[torch.nn.Module] = None, ensembler: Optional[Ensembler] = None, - device: Union[device, None, Literal['cuda', 'gpu', 'cpu']] = None + device: Optional[Union[Literal['cuda', 'gpu', 'cpu'], 'torch.device']] = None, ): """PyTorch backend for LOF detector. - Computes the Local Outlier Factor (LOF) of each instance in `x` with respect to a reference set `x_ref`. - Parameters ---------- k - Number of nearest neighbors used to compute LOF. If `k` is a list or array, then an ensemble of LOF - detectors is created with one detector for each value of `k`. + Number of nearest neighbors used to compute the local outlier factor. `k` can be a single + value or an array of integers. If `k` is a single value the score method uses the + distance/kernel similarity to the `k`-th nearest neighbor. If `k` is a list then it uses + the distance/kernel similarity to each of the specified `k` neighbors. kernel If a kernel is specified then instead of using `torch.cdist` the kernel defines the `k` nearest neighbor distance. @@ -33,7 +32,9 @@ def __init__( of :py:obj:`alibi_detect.od.pytorch.ensemble.ensembler`. Responsible for combining multiple scores into a single score. device - Device on which to run the detector. + Device type used. The default tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either ``'cuda'``, ``'gpu'``, ``'cpu'`` or an instance of + ``torch.device``. """ TorchOutlierDetector.__init__(self, device=device) self.kernel = kernel @@ -55,7 +56,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Raises ------ - ThresholdNotInferredException + ThresholdNotInferredError If called before detector has had `infer_threshold` method called. """ raw_scores = self.score(x) @@ -66,12 +67,18 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return preds def _make_mask(self, reachabilities: torch.Tensor): + """Generate a mask for computing the average reachability. + + If k is an array then we need to compute the average reachability for each k separately. To do + this we use a mask to weight the reachability of each k-close neighbor by 1/k and the rest to 0. + """ mask = torch.zeros_like(reachabilities[0], device=self.device) for i, k in enumerate(self.ks): mask[:k, i] = torch.ones(k, device=self.device)/k return mask def _compute_K(self, x, y): + """Compute the distance/similarity matrix matrix between `x` and `y`.""" return torch.exp(-self.kernel(x, y)) if self.kernel is not None else torch.cdist(x, y) def score(self, x: torch.Tensor) -> torch.Tensor: @@ -88,18 +95,17 @@ def score(self, x: torch.Tensor) -> torch.Tensor: Raises ------ - NotFitException + NotFittedError If called before detector has been fit. """ - if not torch.jit.is_scripting(): - self.check_fitted() + self.check_fitted() # compute the distance matrix between x and x_ref - D = self._compute_K(x, self.x_ref) + K = self._compute_K(x, self.x_ref) # compute k nearest neighbors for maximum k in self.ks max_k = torch.max(self.ks) - bot_k_items = torch.topk(D, int(max_k), dim=1, largest=False) + bot_k_items = torch.topk(K, int(max_k), dim=1, largest=False) bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values # To compute the reachabilities we get the k-distances of each object in the instances @@ -128,13 +134,13 @@ def fit(self, x_ref: torch.Tensor): The Dataset tensor. """ # compute the distance matrix - D = self._compute_K(x_ref, x_ref) + K = self._compute_K(x_ref, x_ref) # set diagonal to max distance to prevent torch.topk from returning the instance itself - D += torch.eye(len(D), device=self.device) * torch.max(D) + K += torch.eye(len(K), device=self.device) * torch.max(K) # compute k nearest neighbors for maximum k in self.ks max_k = torch.max(self.ks) - bot_k_items = torch.topk(D, int(max_k), dim=1, largest=False) + bot_k_items = torch.topk(K, int(max_k), dim=1, largest=False) bot_k_inds, bot_k_dists = bot_k_items.indices, bot_k_items.values # store the k-distances for each instance for each k. From b8842bd017f48044b11cfdf1442443a8b1b414da Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 24 May 2023 16:25:03 +0100 Subject: [PATCH 245/247] Minor change --- alibi_detect/od/tests/test__knn/test__knn_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/od/tests/test__knn/test__knn_backend.py b/alibi_detect/od/tests/test__knn/test__knn_backend.py index 1c9727116..37f8c3216 100644 --- a/alibi_detect/od/tests/test__knn/test__knn_backend.py +++ b/alibi_detect/od/tests/test__knn/test__knn_backend.py @@ -56,7 +56,7 @@ def test_knn_torch_backend_ensemble(ensembler): def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): """ - Test the knn torch backend can be initalized as an ensemble and + Test the knn torch backend can be initialized as an ensemble and torchscripted, as well as saved and loaded to and from disk. """ @@ -78,7 +78,7 @@ def test_knn_torch_backend_ensemble_ts(tmp_path, ensembler): def test_knn_torch_backend_ts(tmp_path): """ - Test the knn torch backend can be initalized and torchscripted, as well as + Test the knn torch backend can be initialized and torchscripted, as well as saved and loaded to and from disk. """ From 5cec76d26a3a9d456219ce65a84ec9bc4b6091e4 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 12 Jun 2023 10:38:37 +0100 Subject: [PATCH 246/247] Update score docstring --- alibi_detect/od/_lof.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alibi_detect/od/_lof.py b/alibi_detect/od/_lof.py index b9766342b..0196dae40 100644 --- a/alibi_detect/od/_lof.py +++ b/alibi_detect/od/_lof.py @@ -125,7 +125,8 @@ def fit(self, x_ref: np.ndarray) -> None: def score(self, x: np.ndarray) -> np.ndarray: """Score `x` instances using the detector. - Computes the local outlier factor for each instance in `x`. If `k` is an array of values then the score for + Computes the local outlier factor for each point in `x`. This is the density of each point `x` + relative to those of its neighbors in `x_ref`. If `k` is an array of values then the score for each `k` is aggregated using the ensembler. Parameters From 32de716eb35cdebcc465c7887ce8039713fdd97a Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Mon, 12 Jun 2023 10:40:15 +0100 Subject: [PATCH 247/247] Update _compute_K docstring --- alibi_detect/od/pytorch/lof.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/od/pytorch/lof.py b/alibi_detect/od/pytorch/lof.py index 84c6a0ba5..055af2d18 100644 --- a/alibi_detect/od/pytorch/lof.py +++ b/alibi_detect/od/pytorch/lof.py @@ -78,7 +78,7 @@ def _make_mask(self, reachabilities: torch.Tensor): return mask def _compute_K(self, x, y): - """Compute the distance/similarity matrix matrix between `x` and `y`.""" + """Compute the distance matrix matrix between `x` and `y`.""" return torch.exp(-self.kernel(x, y)) if self.kernel is not None else torch.cdist(x, y) def score(self, x: torch.Tensor) -> torch.Tensor: