Optional prediction column: the return

docs/tutorials/data_requirements.rst

@@ -170,7 +170,7 @@ The :term:`predicted label<Predicted labels>`, retrieved by interpreting (thresh
 In the sample data this is the **y_pred** column.
 
 Required for running :ref:`performance estimation<performance-estimation>` or :ref:`performance calculation<performance-calculation>` on binary classification, multiclass, and regression models.
-
+On binary classification models, it is not required for calculating the **AUROC** and **average precision** metrics.
 
 NannyML Functionality Requirements
 ----------------------------------
@@ -190,7 +190,8 @@ You can see those requirements in the table below:
 | y_pred_proba | Required (reference and analysis)   |                                     |                                     |                                   |                                   |                                   | Required (reference and analysis) |
 +--------------+-------------------------------------+-------------------------------------+-------------------------------------+-----------------------------------+-----------------------------------+-----------------------------------+-----------------------------------+
 | y_pred       | | Required (reference and analysis) | Required (reference and analysis)   | | Required (reference and analysis) |                                   |                                   |                                   | Required (reference and analysis) |
-|              | | Not needed for ROC_AUC metric     |                                     | | Not needed for ROC_AUC metric     |                                   |                                   |                                   |                                   |
+|              | | Not needed for ROC_AUC or         |                                     | | Not needed for ROC_AUC or         |                                   |                                   |                                   |                                   |
+|              | | average precision metrics         |                                     | | average precision metrics         |                                   |                                   |                                   |                                   |
 +--------------+-------------------------------------+-------------------------------------+-------------------------------------+-----------------------------------+-----------------------------------+-----------------------------------+-----------------------------------+
 | y_true       | Required (reference only)           |  Required (reference only)          | Required (reference and analysis)   |                                   |                                   | Required (reference and analysis) |                                   |
 +--------------+-------------------------------------+-------------------------------------+-------------------------------------+-----------------------------------+-----------------------------------+-----------------------------------+-----------------------------------+

nannyml/performance_calculation/calculator.py

@@ -84,8 +84,8 @@
         self,
         metrics: Union[str, List[str]],
         y_true: str,
-        y_pred: str,
         problem_type: Union[str, ProblemType],
+        y_pred: Optional[str] = None,
         y_pred_proba: Optional[ModelOutputsType] = None,
         timestamp_column_name: Optional[str] = None,
         thresholds: Optional[Dict[str, Threshold]] = None,
@@ -105,8 +105,10 @@
             A metric or list of metrics to calculate.
         y_true: str
             The name of the column containing target values.
-        y_pred: str
+        y_pred: Optional[str], default=None
             The name of the column containing your model predictions.
+            This parameter is optional for binary classification cases.
+            When it is not given, only the ROC AUC and Average Precision metrics are supported.
         problem_type: Union[str, ProblemType]
             Determines which method to use. Allowed values are:
 
@@ -211,7 +213,12 @@
             self.problem_type = problem_type
 
         if self.problem_type is not ProblemType.REGRESSION and y_pred_proba is None:
-            raise InvalidArgumentsException(f"'y_pred_proba' can not be 'None' for problem type {ProblemType.value}")
+            raise InvalidArgumentsException(
+                f"'y_pred_proba' can not be 'None' for problem type {self.problem_type.value}"
+            )
+
+        if self.problem_type is not ProblemType.CLASSIFICATION_BINARY and y_pred is None:
+            raise InvalidArgumentsException(f"'y_pred' can not be 'None' for problem type {self.problem_type.value}")
 
         self.thresholds = DEFAULT_THRESHOLDS
         if thresholds:
@@ -236,6 +243,8 @@
             if metric not in SUPPORTED_METRIC_VALUES:
                 raise InvalidArgumentsException(f"Metric '{metric}' is not supported.")
 
+        raise_if_metrics_require_y_pred(metrics, y_pred)
+
         self.metrics: List[Metric] = [
             MetricFactory.create(
                 m,
@@ -387,3 +396,16 @@
     tuples = chunk_tuples + reconstruction_tuples
 
     return MultiIndex.from_tuples(tuples)
+
+
+def raise_if_metrics_require_y_pred(metrics: List[str], y_pred: Optional[str]):
+    """Raise an exception if metrics require y_pred and y_pred is not set.
+
+    Current metrics that require 'y_pred' are:
+    - roc_auc
+    - average_precision
+    """
+    metrics_that_need_y_pred = [m for m in metrics if m not in ['roc_auc', 'average_precision']]
+
+    if len(metrics_that_need_y_pred) > 0 and y_pred is None:
+        raise InvalidArgumentsException(f"Metrics '{metrics_that_need_y_pred}' require 'y_pred' to be set.")

nannyml/performance_calculation/metrics/base.py

@@ -22,9 +22,9 @@ def __init__(
         self,
         name: str,
         y_true: str,
-        y_pred: str,
         components: List[Tuple[str, str]],
         threshold: Threshold,
+        y_pred: Optional[str] = None,
         y_pred_proba: Optional[Union[str, Dict[str, str]]] = None,
         upper_threshold_limit: Optional[float] = None,
         lower_threshold_limit: Optional[float] = None,

nannyml/performance_calculation/metrics/binary_classification.py

@@ -57,8 +57,8 @@ class BinaryClassificationAUROC(Metric):
     def __init__(
         self,
         y_true: str,
-        y_pred: str,
         threshold: Threshold,
+        y_pred: Optional[str] = None,
         y_pred_proba: Optional[str] = None,
         **kwargs,
     ):
@@ -97,6 +97,8 @@ def __str__(self):
     def _fit(self, reference_data: pd.DataFrame):
         """Metric _fit implementation on reference data."""
         _list_missing([self.y_true, self.y_pred_proba], list(reference_data.columns))
+        # we don't want to count missing rows for sampling error
+        reference_data = _remove_nans(reference_data, (self.y_true,))
         self._sampling_error_components = auroc_sampling_error_components(
             y_true_reference=reference_data[self.y_true],
             y_pred_proba_reference=reference_data[self.y_pred_proba],
@@ -105,10 +107,10 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred_proba], list(data.columns))
-        data = _remove_nans(data, (self.y_true, self.y_pred))
+        data = _remove_nans(data, (self.y_true,))
 
         y_true = data[self.y_true]
-        y_pred = data[self.y_pred_proba]
+        y_pred_proba = data[self.y_pred_proba]
 
         if y_true.nunique() <= 1:
             warnings.warn(
@@ -117,7 +119,7 @@ def _calculate(self, data: pd.DataFrame):
             )
             return np.NaN
         else:
-            return roc_auc_score(y_true, y_pred)
+            return roc_auc_score(y_true, y_pred_proba)
 
     def _sampling_error(self, data: pd.DataFrame) -> float:
         return auroc_sampling_error(self._sampling_error_components, data)
@@ -133,8 +135,8 @@ class BinaryClassificationAP(Metric):
     def __init__(
         self,
         y_true: str,
-        y_pred: str,
         threshold: Threshold,
+        y_pred: Optional[str] = None,
         y_pred_proba: Optional[str] = None,
         **kwargs,
     ):
@@ -174,7 +176,7 @@ def _fit(self, reference_data: pd.DataFrame):
         """Metric _fit implementation on reference data."""
         _list_missing([self.y_true, self.y_pred_proba], list(reference_data.columns))
         # we don't want to count missing rows for sampling error
-        reference_data = _remove_nans(reference_data, (self.y_true, self.y_pred))
+        reference_data = _remove_nans(reference_data, (self.y_true,))
 
         if 1 not in reference_data[self.y_true].unique():
             self._sampling_error_components = np.NaN, 0
@@ -187,7 +189,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred_proba], list(data.columns))
-        data = _remove_nans(data, (self.y_true, self.y_pred))
+        data = _remove_nans(data, (self.y_true,))
 
         y_true = data[self.y_true]
         y_pred_proba = data[self.y_pred_proba]
@@ -259,6 +261,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -335,6 +338,7 @@ def _fit(self, reference_data: pd.DataFrame):
 
     def _calculate(self, data: pd.DataFrame):
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -411,6 +415,7 @@ def _fit(self, reference_data: pd.DataFrame):
 
     def _calculate(self, data: pd.DataFrame):
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -487,6 +492,7 @@ def _fit(self, reference_data: pd.DataFrame):
 
     def _calculate(self, data: pd.DataFrame):
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -564,6 +570,7 @@ def _fit(self, reference_data: pd.DataFrame):
 
     def _calculate(self, data: pd.DataFrame):
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -674,6 +681,7 @@ def _fit(self, reference_data: pd.DataFrame):
 
     def _calculate(self, data: pd.DataFrame):
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -858,6 +866,7 @@ def _fit(self, reference_data: pd.DataFrame):
 
     def _calculate_true_positives(self, data: pd.DataFrame) -> float:
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -882,6 +891,7 @@ def _calculate_true_positives(self, data: pd.DataFrame) -> float:
 
     def _calculate_true_negatives(self, data: pd.DataFrame) -> float:
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -906,6 +916,7 @@ def _calculate_true_negatives(self, data: pd.DataFrame) -> float:
 
     def _calculate_false_positives(self, data: pd.DataFrame) -> float:
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -930,6 +941,7 @@ def _calculate_false_positives(self, data: pd.DataFrame) -> float:
 
     def _calculate_false_negatives(self, data: pd.DataFrame) -> float:
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]

nannyml/performance_calculation/metrics/multiclass_classification.py

@@ -211,6 +211,7 @@ def _calculate(self, data: pd.DataFrame):
             )
 
         _list_missing([self.y_true, self.y_pred], data)
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         labels = sorted(list(self.y_pred_proba.keys()))
@@ -306,6 +307,7 @@ def _calculate(self, data: pd.DataFrame):
             )
 
         _list_missing([self.y_true, self.y_pred], data)
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         labels = sorted(list(self.y_pred_proba.keys()))
@@ -401,6 +403,7 @@ def _calculate(self, data: pd.DataFrame):
             )
 
         _list_missing([self.y_true, self.y_pred], data)
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         labels = sorted(list(self.y_pred_proba.keys()))
@@ -496,6 +499,7 @@ def _calculate(self, data: pd.DataFrame):
             )
 
         _list_missing([self.y_true, self.y_pred], data)
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         labels = sorted(list(self.y_pred_proba.keys()))
@@ -588,6 +592,7 @@ def _fit(self, reference_data: pd.DataFrame):
 
     def _calculate(self, data: pd.DataFrame):
         _list_missing([self.y_true, self.y_pred], data)
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]

nannyml/performance_calculation/metrics/regression.py

@@ -77,6 +77,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -139,6 +140,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -201,6 +203,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -263,6 +266,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -330,6 +334,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]
@@ -392,6 +397,7 @@ def _fit(self, reference_data: pd.DataFrame):
     def _calculate(self, data: pd.DataFrame):
         """Redefine to handle NaNs and edge cases."""
         _list_missing([self.y_true, self.y_pred], list(data.columns))
+        assert self.y_pred
         data = _remove_nans(data, (self.y_true, self.y_pred))
 
         y_true = data[self.y_true]

nannyml/performance_calculation/result.py

@@ -30,7 +30,7 @@ def __init__(
         self,
         results_data: pd.DataFrame,
         problem_type: ProblemType,
-        y_pred: str,
+        y_pred: Optional[str],
         y_pred_proba: Optional[Union[str, Dict[str, str]]],
         y_true: str,
         metrics: List[Metric],