diff --git a/docs/component/data.rst b/docs/component/data.rst index eeddaec60d0..b57c032a567 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -433,7 +433,7 @@ The ``Processor`` module in ``Qlib`` is designed to be learnable and it is respo - ``DropnaProcessor``: `processor` that drops N/A features. - ``DropnaLabel``: `processor` that drops N/A labels. -- ``TanhProcess``: `processor` that uses `tanh` to process noise data. +- ``TanhProcess``: `processor` that applies ``tanh(x - 1)`` to denoise values. By default it transforms the ``feature`` column group only (via ``fields_group``), leaving labels unchanged. - ``ProcessInf``: `processor` that handles infinity values, it will be replaces by the mean of the column. - ``Fillna``: `processor` that handles N/A values, which will fill the N/A value by 0 or other given number. - ``MinMaxNorm``: `processor` that applies min-max normalization. diff --git a/qlib/data/dataset/processor.py b/qlib/data/dataset/processor.py index d05dbe381c5..c900e73bdcb 100644 --- a/qlib/data/dataset/processor.py +++ b/qlib/data/dataset/processor.py @@ -144,18 +144,31 @@ def readonly(self): class TanhProcess(Processor): - """Use tanh to process noise data""" + """Apply ``tanh(x - 1)`` to denoise feature values. - def __call__(self, df): - def tanh_denoise(data): - mask = data.columns.get_level_values(1).str.contains("LABEL") - col = df.columns[~mask] - data[col] = data[col] - 1 - data[col] = np.tanh(data[col]) + Qlib handlers (e.g. Alpha158/Alpha360) use MultiIndex columns whose first + level is the group name (``feature`` / ``label``) and whose second level is + the column name (e.g. ``LABEL0``). Selecting columns by matching ``"LABEL"`` + on level 1 is brittle and inconsistent with other processors. - return data + This processor follows the same ``fields_group`` pattern as + :class:`Fillna`, :class:`CSZScoreNorm`, etc., so labels are left untouched + by default. + + Parameters + ---------- + fields_group : str, optional + Column group to transform. If ``None``, all columns are transformed. + Default is ``"feature"``. + """ - return tanh_denoise(df) + def __init__(self, fields_group: Optional[str] = "feature"): + self.fields_group = fields_group + + def __call__(self, df: pd.DataFrame): + cols = get_group_columns(df, self.fields_group) + df[cols] = np.tanh(df[cols] - 1) + return df class ProcessInf(Processor): diff --git a/tests/data_mid_layer_tests/test_processor.py b/tests/data_mid_layer_tests/test_processor.py index 46453b31628..94ad6ccd93c 100644 --- a/tests/data_mid_layer_tests/test_processor.py +++ b/tests/data_mid_layer_tests/test_processor.py @@ -3,9 +3,70 @@ import unittest import numpy as np +import pandas as pd from qlib.data import D from qlib.tests import TestAutoData -from qlib.data.dataset.processor import MinMaxNorm, ZScoreNorm, CSZScoreNorm, CSZFillna +from qlib.data.dataset.processor import MinMaxNorm, ZScoreNorm, CSZScoreNorm, CSZFillna, TanhProcess + + +class TestTanhProcess(unittest.TestCase): + """Unit tests for TanhProcess (issue #1687). + + These tests do not require downloaded market data. + """ + + @staticmethod + def _make_handler_like_df(): + # Match Alpha158/Alpha360 MultiIndex layout: + # level 0 = group (feature/label), level 1 = column name. + columns = pd.MultiIndex.from_tuples( + [("feature", "RESI5"), ("feature", "WVMA5"), ("label", "LABEL0")], + names=["field", "name"], + ) + index = pd.MultiIndex.from_tuples( + [("2023-01-01", "SH600000"), ("2023-01-02", "SH600000")], + names=["datetime", "instrument"], + ) + data = np.array([[2.0, 3.0, 0.5], [4.0, 5.0, 0.8]], dtype=float) + return pd.DataFrame(data, index=index, columns=columns) + + def test_default_transforms_feature_group_only(self): + df = self._make_handler_like_df() + label_before = df[("label", "LABEL0")].copy() + + result = TanhProcess()(df) + + pd.testing.assert_series_equal(result[("label", "LABEL0")], label_before) + np.testing.assert_allclose( + result[("feature", "RESI5")].to_numpy(), + np.tanh(np.array([2.0, 4.0]) - 1), + atol=1e-7, + ) + np.testing.assert_allclose( + result[("feature", "WVMA5")].to_numpy(), + np.tanh(np.array([3.0, 5.0]) - 1), + atol=1e-7, + ) + + def test_fields_group_none_transforms_all_columns(self): + df = self._make_handler_like_df() + expected = np.tanh(df.to_numpy() - 1) + + result = TanhProcess(fields_group=None)(df) + + np.testing.assert_allclose(result.to_numpy(), expected, atol=1e-7) + + def test_legacy_label_substring_mask_was_incorrect_for_standard_groups(self): + # Regression: matching "LABEL" on level 1 happens to catch LABEL0, but + # matching on level 0 (as suggested in some reports) would miss lowercase + # "label" and transform labels. fields_group avoids both pitfalls. + df = self._make_handler_like_df() + level0_mask = df.columns.get_level_values(0).str.contains("LABEL") + self.assertFalse(level0_mask.any()) + + label_before = df[("label", "LABEL0")].copy() + TanhProcess(fields_group="feature")(df) + pd.testing.assert_series_equal(df[("label", "LABEL0")], label_before) class TestProcessor(TestAutoData):