Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/component/data.rst
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ The ``Processor`` module in ``Qlib`` is designed to be learnable and it is respo

- ``DropnaProcessor``: `processor` that drops N/A features.
- ``DropnaLabel``: `processor` that drops N/A labels.
- ``TanhProcess``: `processor` that uses `tanh` to process noise data.
- ``TanhProcess``: `processor` that applies ``tanh(x - 1)`` to denoise values. By default it transforms the ``feature`` column group only (via ``fields_group``), leaving labels unchanged.
- ``ProcessInf``: `processor` that handles infinity values, it will be replaces by the mean of the column.
- ``Fillna``: `processor` that handles N/A values, which will fill the N/A value by 0 or other given number.
- ``MinMaxNorm``: `processor` that applies min-max normalization.
Expand Down
31 changes: 22 additions & 9 deletions qlib/data/dataset/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,18 +144,31 @@ def readonly(self):


class TanhProcess(Processor):
"""Use tanh to process noise data"""
"""Apply ``tanh(x - 1)`` to denoise feature values.

def __call__(self, df):
def tanh_denoise(data):
mask = data.columns.get_level_values(1).str.contains("LABEL")
col = df.columns[~mask]
data[col] = data[col] - 1
data[col] = np.tanh(data[col])
Qlib handlers (e.g. Alpha158/Alpha360) use MultiIndex columns whose first
level is the group name (``feature`` / ``label``) and whose second level is
the column name (e.g. ``LABEL0``). Selecting columns by matching ``"LABEL"``
on level 1 is brittle and inconsistent with other processors.

return data
This processor follows the same ``fields_group`` pattern as
:class:`Fillna`, :class:`CSZScoreNorm`, etc., so labels are left untouched
by default.

Parameters
----------
fields_group : str, optional
Column group to transform. If ``None``, all columns are transformed.
Default is ``"feature"``.
"""

return tanh_denoise(df)
def __init__(self, fields_group: Optional[str] = "feature"):
self.fields_group = fields_group

def __call__(self, df: pd.DataFrame):
cols = get_group_columns(df, self.fields_group)
df[cols] = np.tanh(df[cols] - 1)
return df


class ProcessInf(Processor):
Expand Down
63 changes: 62 additions & 1 deletion tests/data_mid_layer_tests/test_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,70 @@

import unittest
import numpy as np
import pandas as pd
from qlib.data import D
from qlib.tests import TestAutoData
from qlib.data.dataset.processor import MinMaxNorm, ZScoreNorm, CSZScoreNorm, CSZFillna
from qlib.data.dataset.processor import MinMaxNorm, ZScoreNorm, CSZScoreNorm, CSZFillna, TanhProcess


class TestTanhProcess(unittest.TestCase):
"""Unit tests for TanhProcess (issue #1687).

These tests do not require downloaded market data.
"""

@staticmethod
def _make_handler_like_df():
# Match Alpha158/Alpha360 MultiIndex layout:
# level 0 = group (feature/label), level 1 = column name.
columns = pd.MultiIndex.from_tuples(
[("feature", "RESI5"), ("feature", "WVMA5"), ("label", "LABEL0")],
names=["field", "name"],
)
index = pd.MultiIndex.from_tuples(
[("2023-01-01", "SH600000"), ("2023-01-02", "SH600000")],
names=["datetime", "instrument"],
)
data = np.array([[2.0, 3.0, 0.5], [4.0, 5.0, 0.8]], dtype=float)
return pd.DataFrame(data, index=index, columns=columns)

def test_default_transforms_feature_group_only(self):
df = self._make_handler_like_df()
label_before = df[("label", "LABEL0")].copy()

result = TanhProcess()(df)

pd.testing.assert_series_equal(result[("label", "LABEL0")], label_before)
np.testing.assert_allclose(
result[("feature", "RESI5")].to_numpy(),
np.tanh(np.array([2.0, 4.0]) - 1),
atol=1e-7,
)
np.testing.assert_allclose(
result[("feature", "WVMA5")].to_numpy(),
np.tanh(np.array([3.0, 5.0]) - 1),
atol=1e-7,
)

def test_fields_group_none_transforms_all_columns(self):
df = self._make_handler_like_df()
expected = np.tanh(df.to_numpy() - 1)

result = TanhProcess(fields_group=None)(df)

np.testing.assert_allclose(result.to_numpy(), expected, atol=1e-7)

def test_legacy_label_substring_mask_was_incorrect_for_standard_groups(self):
# Regression: matching "LABEL" on level 1 happens to catch LABEL0, but
# matching on level 0 (as suggested in some reports) would miss lowercase
# "label" and transform labels. fields_group avoids both pitfalls.
df = self._make_handler_like_df()
level0_mask = df.columns.get_level_values(0).str.contains("LABEL")
self.assertFalse(level0_mask.any())

label_before = df[("label", "LABEL0")].copy()
TanhProcess(fields_group="feature")(df)
pd.testing.assert_series_equal(df[("label", "LABEL0")], label_before)


class TestProcessor(TestAutoData):
Expand Down