From cd6ec7f68ff787f7bb52e8ed7ffa8d1760b892b4 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Sat, 23 Jul 2022 18:49:28 +0530 Subject: [PATCH 01/23] Added the ndcg metric [WIP] --- ignite/metrics/ndcg.py | 73 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 ignite/metrics/ndcg.py diff --git a/ignite/metrics/ndcg.py b/ignite/metrics/ndcg.py new file mode 100644 index 00000000000..4e4b276f64e --- /dev/null +++ b/ignite/metrics/ndcg.py @@ -0,0 +1,73 @@ +from typing import Callable, Sequence, Union + +import torch + +from ignite.exceptions import NotComputableError +from ignite.metrics.metric import Metric + +__all__ = ["NGCD"] + + +class NGCD(Metric): + def __init__( + self, + output_transform: Callable = lambda x: x, + device: Union[str, torch.device] = torch.device("cpu"), + k=None, + log_base=2, + ): + super(NGCD, self).__init__(output_transform=output_transform, device=device) + + self.log_base = log_base + self.k = k + self.num_examples = 0 + self.ngcd = torch.tensor(0, device=self._device, dtype=torch.float32) + + def _dcg_sample_scores(self, y_true, y_score, k=None, log_base=2) -> torch.Tensor: + + discount = torch.div(torch.log(torch.arange(y_true.shape[1]) + 2), torch.log(torch.tensor(log_base))).pow(-1) + + if k is not None: + discount[k:] = 0 + + ranking = torch.argsort(y_score, descending=True) + + ranked = torch.zeros(y_score.shape, dtype=y_score.dtype) + ranked = ranked.scatter_(1, ranking, y_true) + + discounted_gains = torch.mm(ranked, discount.reshape(-1, 1)) + + return discounted_gains + + def _ndcg_sample_scores(self, y_true, y_score, k=None, log_base=2) -> torch.Tensor: + + gain = self._dcg_sample_scores(y_true, y_score, k, log_base=log_base) + + normalizing_gain = self._dcg_sample_scores(y_true, y_true, k, log_base=log_base) + + all_irrelevant = normalizing_gain == 0 + + normalized_gain = torch.div(gain[~all_irrelevant], normalizing_gain[~all_irrelevant]) + + return normalized_gain + + def reset(self) -> None: + + self.num_examples = 0 + self.ngcd = torch.tensor(0, device=self._device) + + def update(self, output: Sequence[torch.Tensor]) -> None: + + y_pred, y = output[0], output[1] + + gain = self._ndcg_sample_scores(y, y_pred, k=self.k) + + self.ngcd += torch.sum(gain) + + self.num_examples += y_pred.shape[0] + + def compute(self) -> float: + if self.num_examples == 0: + raise NotComputableError("NGCD must have at least one example before it can be computed.") + + return self.ngcd / self.num_examples From 4535af15069352dce7cc290d60b1f0da2ead48bf Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Sat, 23 Jul 2022 22:46:57 +0530 Subject: [PATCH 02/23] added GPU support, corrected mypy errors, and minor fixes --- ignite/metrics/ndcg.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/ignite/metrics/ndcg.py b/ignite/metrics/ndcg.py index 4e4b276f64e..5c7d556ee59 100644 --- a/ignite/metrics/ndcg.py +++ b/ignite/metrics/ndcg.py @@ -5,41 +5,49 @@ from ignite.exceptions import NotComputableError from ignite.metrics.metric import Metric -__all__ = ["NGCD"] +__all__ = ["NDCG"] -class NGCD(Metric): +class NDCG(Metric): def __init__( self, output_transform: Callable = lambda x: x, device: Union[str, torch.device] = torch.device("cpu"), - k=None, - log_base=2, + k: Union[int, None] = None, + log_base: Union[int, float] = 2, ): - super(NGCD, self).__init__(output_transform=output_transform, device=device) + super(NDCG, self).__init__(output_transform=output_transform, device=device) self.log_base = log_base self.k = k self.num_examples = 0 - self.ngcd = torch.tensor(0, device=self._device, dtype=torch.float32) + self.ngcd = torch.tensor(0, dtype=torch.float32, device=device) - def _dcg_sample_scores(self, y_true, y_score, k=None, log_base=2) -> torch.Tensor: + def _dcg_sample_scores( + self, y_true: torch.Tensor, y_score: torch.Tensor, k: Union[int, None] = None, log_base: Union[int, float] = 2 + ) -> torch.Tensor: - discount = torch.div(torch.log(torch.arange(y_true.shape[1]) + 2), torch.log(torch.tensor(log_base))).pow(-1) + discount = ( + torch.div(torch.log(torch.arange(y_true.shape[1]) + 2), torch.log(torch.tensor(log_base))) + .pow(-1) + .to(self._device) + ) if k is not None: discount[k:] = 0 ranking = torch.argsort(y_score, descending=True) - ranked = torch.zeros(y_score.shape, dtype=y_score.dtype) + ranked = torch.zeros(y_score.shape, dtype=y_score.dtype, device=self._device) ranked = ranked.scatter_(1, ranking, y_true) discounted_gains = torch.mm(ranked, discount.reshape(-1, 1)) return discounted_gains - def _ndcg_sample_scores(self, y_true, y_score, k=None, log_base=2) -> torch.Tensor: + def _ndcg_sample_scores( + self, y_true: torch.Tensor, y_score: torch.Tensor, k: Union[int, None] = None, log_base: Union[int, float] = 2 + ) -> torch.Tensor: gain = self._dcg_sample_scores(y_true, y_score, k, log_base=log_base) @@ -60,13 +68,13 @@ def update(self, output: Sequence[torch.Tensor]) -> None: y_pred, y = output[0], output[1] - gain = self._ndcg_sample_scores(y, y_pred, k=self.k) + gain = self._ndcg_sample_scores(y, y_pred, k=self.k, log_base=self.log_base) self.ngcd += torch.sum(gain) self.num_examples += y_pred.shape[0] - def compute(self) -> float: + def compute(self) -> torch.float32: # type: ignore if self.num_examples == 0: raise NotComputableError("NGCD must have at least one example before it can be computed.") From 70d06e5bfdf38d19b735c583a8cdf1eb27d6662d Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Sun, 24 Jul 2022 07:22:47 +0530 Subject: [PATCH 03/23] Incorporated the suggested changes --- ignite/metrics/ndcg.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ignite/metrics/ndcg.py b/ignite/metrics/ndcg.py index 5c7d556ee59..46be340e83b 100644 --- a/ignite/metrics/ndcg.py +++ b/ignite/metrics/ndcg.py @@ -1,4 +1,4 @@ -from typing import Callable, Sequence, Union +from typing import Callable, Optional, Sequence, Union import torch @@ -13,18 +13,18 @@ def __init__( self, output_transform: Callable = lambda x: x, device: Union[str, torch.device] = torch.device("cpu"), - k: Union[int, None] = None, + k: Optional[int] = None, log_base: Union[int, float] = 2, ): - super(NDCG, self).__init__(output_transform=output_transform, device=device) self.log_base = log_base + self.k = k - self.num_examples = 0 - self.ngcd = torch.tensor(0, dtype=torch.float32, device=device) + + super(NDCG, self).__init__(output_transform=output_transform, device=device) def _dcg_sample_scores( - self, y_true: torch.Tensor, y_score: torch.Tensor, k: Union[int, None] = None, log_base: Union[int, float] = 2 + self, y_true: torch.Tensor, y_score: torch.Tensor, k: Optional[int] = None, log_base: Union[int, float] = 2 ) -> torch.Tensor: discount = ( @@ -46,27 +46,27 @@ def _dcg_sample_scores( return discounted_gains def _ndcg_sample_scores( - self, y_true: torch.Tensor, y_score: torch.Tensor, k: Union[int, None] = None, log_base: Union[int, float] = 2 + self, y_true: torch.Tensor, y_score: torch.Tensor, k: Optional[int] = None, log_base: Union[int, float] = 2 ) -> torch.Tensor: gain = self._dcg_sample_scores(y_true, y_score, k, log_base=log_base) normalizing_gain = self._dcg_sample_scores(y_true, y_true, k, log_base=log_base) - all_irrelevant = normalizing_gain == 0 + all_relevant = normalizing_gain != 0 - normalized_gain = torch.div(gain[~all_irrelevant], normalizing_gain[~all_irrelevant]) + normalized_gain = torch.div(gain[all_relevant], normalizing_gain[all_relevant]) return normalized_gain def reset(self) -> None: self.num_examples = 0 - self.ngcd = torch.tensor(0, device=self._device) + self.ngcd = torch.tensor(0.0, device=self._device) def update(self, output: Sequence[torch.Tensor]) -> None: - y_pred, y = output[0], output[1] + y_pred, y = output[0].detach(), output[1].detach() gain = self._ndcg_sample_scores(y, y_pred, k=self.k, log_base=self.log_base) @@ -74,7 +74,7 @@ def update(self, output: Sequence[torch.Tensor]) -> None: self.num_examples += y_pred.shape[0] - def compute(self) -> torch.float32: # type: ignore + def compute(self) -> float: if self.num_examples == 0: raise NotComputableError("NGCD must have at least one example before it can be computed.") From 6a86f5f611a00bca73ee0ea1b0707252a153d656 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Sun, 24 Jul 2022 07:33:55 +0530 Subject: [PATCH 04/23] Fixed mypy error --- ignite/metrics/ndcg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ignite/metrics/ndcg.py b/ignite/metrics/ndcg.py index 46be340e83b..d8af5a337af 100644 --- a/ignite/metrics/ndcg.py +++ b/ignite/metrics/ndcg.py @@ -78,4 +78,4 @@ def compute(self) -> float: if self.num_examples == 0: raise NotComputableError("NGCD must have at least one example before it can be computed.") - return self.ngcd / self.num_examples + return (self.ngcd / self.num_examples).item() From 7b7ed6f24fcf99d590ac07ed448502bd6a0742ae Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Sun, 24 Jul 2022 21:27:41 +0530 Subject: [PATCH 05/23] Fixed bugs in NDCG and added tests for output and reset --- ignite/metrics/__init__.py | 2 + ignite/metrics/{ => recsys}/ndcg.py | 26 +++-------- tests/ignite/metrics/test_ndcg.py | 67 +++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 20 deletions(-) rename ignite/metrics/{ => recsys}/ndcg.py (83%) create mode 100644 tests/ignite/metrics/test_ndcg.py diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py index d001436a3ad..d8905b71b8d 100644 --- a/ignite/metrics/__init__.py +++ b/ignite/metrics/__init__.py @@ -19,6 +19,7 @@ from ignite.metrics.precision import Precision from ignite.metrics.psnr import PSNR from ignite.metrics.recall import Recall +from ignite.metrics.recsys.ndcg import NDCG from ignite.metrics.root_mean_squared_error import RootMeanSquaredError from ignite.metrics.running_average import RunningAverage from ignite.metrics.ssim import SSIM @@ -58,4 +59,5 @@ "Rouge", "RougeN", "RougeL", + "NDCG", ] diff --git a/ignite/metrics/ndcg.py b/ignite/metrics/recsys/ndcg.py similarity index 83% rename from ignite/metrics/ndcg.py rename to ignite/metrics/recsys/ndcg.py index d8af5a337af..b12ff150022 100644 --- a/ignite/metrics/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -18,9 +18,7 @@ def __init__( ): self.log_base = log_base - self.k = k - super(NDCG, self).__init__(output_transform=output_transform, device=device) def _dcg_sample_scores( @@ -32,17 +30,12 @@ def _dcg_sample_scores( .pow(-1) .to(self._device) ) - if k is not None: - discount[k:] = 0 + discount[k:] = 0.0 ranking = torch.argsort(y_score, descending=True) - - ranked = torch.zeros(y_score.shape, dtype=y_score.dtype, device=self._device) - ranked = ranked.scatter_(1, ranking, y_true) - + ranked = y_true[torch.arange(ranking.shape[0]).reshape(-1, 1), ranking].to(self._device) discounted_gains = torch.mm(ranked, discount.reshape(-1, 1)) - return discounted_gains def _ndcg_sample_scores( @@ -50,32 +43,25 @@ def _ndcg_sample_scores( ) -> torch.Tensor: gain = self._dcg_sample_scores(y_true, y_score, k, log_base=log_base) - normalizing_gain = self._dcg_sample_scores(y_true, y_true, k, log_base=log_base) - all_relevant = normalizing_gain != 0 - normalized_gain = torch.div(gain[all_relevant], normalizing_gain[all_relevant]) - return normalized_gain def reset(self) -> None: self.num_examples = 0 - self.ngcd = torch.tensor(0.0, device=self._device) + self.ndcg = torch.tensor(0.0, device=self._device) def update(self, output: Sequence[torch.Tensor]) -> None: - y_pred, y = output[0].detach(), output[1].detach() - + y, y_pred = output[0].detach(), output[1].detach() gain = self._ndcg_sample_scores(y, y_pred, k=self.k, log_base=self.log_base) - - self.ngcd += torch.sum(gain) - + self.ndcg = torch.add(self.ndcg, torch.sum(gain)) self.num_examples += y_pred.shape[0] def compute(self) -> float: if self.num_examples == 0: raise NotComputableError("NGCD must have at least one example before it can be computed.") - return (self.ngcd / self.num_examples).item() + return (self.ndcg / self.num_examples).item() diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py new file mode 100644 index 00000000000..d95cee22404 --- /dev/null +++ b/tests/ignite/metrics/test_ndcg.py @@ -0,0 +1,67 @@ +import numpy as np +import pytest +import torch +from sklearn.metrics import ndcg_score + +from ignite.exceptions import NotComputableError +from ignite.metrics.recsys.ndcg import NDCG + + +@pytest.mark.parametrize( + "true_score, pred_score", + [ + (torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]])), + (torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]]), torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]])), + ( + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + ), + ], +) +@pytest.mark.parametrize("k", [None, 2, 3]) +def test_output_cpu(true_score, pred_score, k): + + device = "cpu" + + ndcg = NDCG(k=k, device=device) + ndcg.update([true_score, pred_score]) + result_ignite = ndcg.compute() + result_sklearn = ndcg_score(true_score.numpy(), pred_score.numpy(), k=k) + + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + + +@pytest.mark.parametrize( + "true_score, pred_score", + [ + (torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]])), + (torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]]), torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]])), + ( + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + ), + ], +) +@pytest.mark.parametrize("k", [None, 2, 3]) +@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") +def test_output_gpu(true_score, pred_score, k): + + device = "cuda" + ndcg = NDCG(k=k, device=device) + ndcg.update([true_score, pred_score]) + result_ignite = ndcg.compute() + result_sklearn = ndcg_score(true_score.numpy(), pred_score.numpy(), k=k) + + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + + +def test_reset(): + + true_score = torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]) + pred_score = torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]) + ndcg = NDCG() + ndcg.update([true_score, pred_score]) + ndcg.reset() + + with pytest.raises(NotComputableError, match=r"NGCD must have at least one example before it can be computed."): + ndcg.compute() From 2c87ee1f45c66a2e73f84d70e15bced86835855f Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Sun, 24 Jul 2022 21:34:06 +0530 Subject: [PATCH 06/23] Fixed mypy error --- ignite/metrics/__init__.py | 1 - ignite/metrics/recsys/__init__.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 ignite/metrics/recsys/__init__.py diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py index d8905b71b8d..eb54f086c0a 100644 --- a/ignite/metrics/__init__.py +++ b/ignite/metrics/__init__.py @@ -59,5 +59,4 @@ "Rouge", "RougeN", "RougeL", - "NDCG", ] diff --git a/ignite/metrics/recsys/__init__.py b/ignite/metrics/recsys/__init__.py new file mode 100644 index 00000000000..71e737cc0bd --- /dev/null +++ b/ignite/metrics/recsys/__init__.py @@ -0,0 +1,5 @@ +from ignite.metrics.recsys.ndcg import NDCG + +__all__ = [ + "NDCG", +] From f4c628a532606a6d98ac7f2fdb2bba7ca0dde269 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Mon, 25 Jul 2022 08:00:07 +0530 Subject: [PATCH 07/23] Added the exponential form on https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Discounted_Cumulative_Gain --- ignite/metrics/recsys/ndcg.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index b12ff150022..b727b5176cb 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -15,10 +15,12 @@ def __init__( device: Union[str, torch.device] = torch.device("cpu"), k: Optional[int] = None, log_base: Union[int, float] = 2, + exponential: bool = False, ): self.log_base = log_base self.k = k + self.exponential = exponential super(NDCG, self).__init__(output_transform=output_transform, device=device) def _dcg_sample_scores( @@ -35,6 +37,10 @@ def _dcg_sample_scores( ranking = torch.argsort(y_score, descending=True) ranked = y_true[torch.arange(ranking.shape[0]).reshape(-1, 1), ranking].to(self._device) + + if self.exponential: + ranked = ranked.pow(2) - 1 + discounted_gains = torch.mm(ranked, discount.reshape(-1, 1)) return discounted_gains From e72b59e8b2191c8db63dc6194ad0d69183905eb0 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Mon, 25 Jul 2022 14:38:08 +0530 Subject: [PATCH 08/23] Corrected true, pred order and corresponding tests --- ignite/metrics/recsys/ndcg.py | 2 +- tests/ignite/metrics/test_ndcg.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index b727b5176cb..063aacfcece 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -61,7 +61,7 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: - y, y_pred = output[0].detach(), output[1].detach() + y_pred, y = output[0].detach(), output[1].detach() gain = self._ndcg_sample_scores(y, y_pred, k=self.k, log_base=self.log_base) self.ndcg = torch.add(self.ndcg, torch.sum(gain)) self.num_examples += y_pred.shape[0] diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index d95cee22404..5cd04aab695 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -24,7 +24,7 @@ def test_output_cpu(true_score, pred_score, k): device = "cpu" ndcg = NDCG(k=k, device=device) - ndcg.update([true_score, pred_score]) + ndcg.update([pred_score, true_score]) result_ignite = ndcg.compute() result_sklearn = ndcg_score(true_score.numpy(), pred_score.numpy(), k=k) @@ -48,7 +48,7 @@ def test_output_gpu(true_score, pred_score, k): device = "cuda" ndcg = NDCG(k=k, device=device) - ndcg.update([true_score, pred_score]) + ndcg.update([pred_score, true_score]) result_ignite = ndcg.compute() result_sklearn = ndcg_score(true_score.numpy(), pred_score.numpy(), k=k) @@ -60,7 +60,7 @@ def test_reset(): true_score = torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]) pred_score = torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]) ndcg = NDCG() - ndcg.update([true_score, pred_score]) + ndcg.update([pred_score, true_score]) ndcg.reset() with pytest.raises(NotComputableError, match=r"NGCD must have at least one example before it can be computed."): From ef63d85e555be7f032e3bc7222106603aeb36f30 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Tue, 26 Jul 2022 16:01:21 +0530 Subject: [PATCH 09/23] Added ties case, exponential tests, log_base tests, corresponding tests and other suggested changes --- ignite/metrics/recsys/ndcg.py | 110 +++++++++++++------ tests/ignite/metrics/test_ndcg.py | 175 +++++++++++++++++++++++++++--- 2 files changed, 233 insertions(+), 52 deletions(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index 063aacfcece..937ec5a7a06 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -8,6 +8,72 @@ __all__ = ["NDCG"] +def _tie_averaged_dcg( + y_pred: torch.Tensor, + y_true: torch.Tensor, + discount_cumsum: torch.Tensor, + device: Union[str, torch.device] = torch.device("cpu"), +) -> torch.Tensor: + + _, inv, counts = torch.unique(-y_pred, return_inverse=True, return_counts=True) + ranked = torch.zeros(counts.shape[0]).to(device) + ranked.index_put_([inv], y_true, accumulate=True) + ranked /= counts + groups = torch.cumsum(counts, dim=-1) - 1 + discount_sums = torch.empty(counts.shape[0]).to(device) + discount_sums[0] = discount_cumsum[groups[0]] + discount_sums[1:] = torch.diff(discount_cumsum[groups]) + + return torch.sum(torch.mul(ranked, discount_sums)) + + +def _dcg_sample_scores( + y_pred: torch.Tensor, + y_true: torch.Tensor, + k: Optional[int] = None, + log_base: Union[int, float] = 2, + ignore_ties: bool = False, + device: Union[str, torch.device] = torch.device("cpu"), +) -> torch.Tensor: + + discount = torch.log(torch.tensor(log_base)) / torch.log(torch.arange(y_true.shape[1]) + 2) + discount = discount.to(device) + + if k is not None: + discount[k:] = 0.0 + + if ignore_ties: + ranking = torch.argsort(y_pred, descending=True) + ranked = y_true[torch.arange(ranking.shape[0]).reshape(-1, 1), ranking].to(device) + discounted_gains = torch.mm(ranked, discount.reshape(-1, 1)) + + else: + discount_cumsum = torch.cumsum(discount, dim=-1) + discounted_gains = torch.tensor( + [_tie_averaged_dcg(y_p, y_t, discount_cumsum, device) for y_p, y_t in zip(y_pred, y_true)], device=device + ) + + return discounted_gains + + +def _ndcg_sample_scores( + y_pred: torch.Tensor, + y_true: torch.Tensor, + k: Optional[int] = None, + log_base: Union[int, float] = 2, + ignore_ties: bool = False, + device: Union[str, torch.device] = torch.device("cpu"), +) -> torch.Tensor: + + gain = _dcg_sample_scores(y_pred, y_true, k=k, log_base=log_base, ignore_ties=ignore_ties, device=device) + if not ignore_ties: + gain = gain.unsqueeze(dim=-1) + normalizing_gain = _dcg_sample_scores(y_true, y_true, k=k, log_base=log_base, ignore_ties=True, device=device) + all_relevant = normalizing_gain != 0 + normalized_gain = gain[all_relevant] / normalizing_gain[all_relevant] + return normalized_gain + + class NDCG(Metric): def __init__( self, @@ -16,43 +82,15 @@ def __init__( k: Optional[int] = None, log_base: Union[int, float] = 2, exponential: bool = False, + ignore_ties: bool = False, ): + assert log_base != 1 or log_base <= 0, f"Illegal value {log_base} for log_base" self.log_base = log_base self.k = k self.exponential = exponential super(NDCG, self).__init__(output_transform=output_transform, device=device) - - def _dcg_sample_scores( - self, y_true: torch.Tensor, y_score: torch.Tensor, k: Optional[int] = None, log_base: Union[int, float] = 2 - ) -> torch.Tensor: - - discount = ( - torch.div(torch.log(torch.arange(y_true.shape[1]) + 2), torch.log(torch.tensor(log_base))) - .pow(-1) - .to(self._device) - ) - if k is not None: - discount[k:] = 0.0 - - ranking = torch.argsort(y_score, descending=True) - ranked = y_true[torch.arange(ranking.shape[0]).reshape(-1, 1), ranking].to(self._device) - - if self.exponential: - ranked = ranked.pow(2) - 1 - - discounted_gains = torch.mm(ranked, discount.reshape(-1, 1)) - return discounted_gains - - def _ndcg_sample_scores( - self, y_true: torch.Tensor, y_score: torch.Tensor, k: Optional[int] = None, log_base: Union[int, float] = 2 - ) -> torch.Tensor: - - gain = self._dcg_sample_scores(y_true, y_score, k, log_base=log_base) - normalizing_gain = self._dcg_sample_scores(y_true, y_true, k, log_base=log_base) - all_relevant = normalizing_gain != 0 - normalized_gain = torch.div(gain[all_relevant], normalizing_gain[all_relevant]) - return normalized_gain + self.ignore_ties = ignore_ties def reset(self) -> None: @@ -61,9 +99,13 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: - y_pred, y = output[0].detach(), output[1].detach() - gain = self._ndcg_sample_scores(y, y_pred, k=self.k, log_base=self.log_base) - self.ndcg = torch.add(self.ndcg, torch.sum(gain)) + y_pred, y_true = output[0].detach(), output[1].detach() + + if self.exponential: + y_true = 2 ** y_true - 1 + + gain = _ndcg_sample_scores(y_pred, y_true, k=self.k, log_base=self.log_base, device=self._device) + self.ndcg += torch.sum(gain) self.num_examples += y_pred.shape[0] def compute(self) -> float: diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index 5cd04aab695..f3d5e982e06 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -2,66 +2,205 @@ import pytest import torch from sklearn.metrics import ndcg_score +from sklearn.metrics._ranking import _dcg_sample_scores from ignite.exceptions import NotComputableError from ignite.metrics.recsys.ndcg import NDCG @pytest.mark.parametrize( - "true_score, pred_score", + "y_pred, y_true", [ - (torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]])), - (torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]]), torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]])), + (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), ( - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), + ), + (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + ( + torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), ), ], ) @pytest.mark.parametrize("k", [None, 2, 3]) -def test_output_cpu(true_score, pred_score, k): +def test_output_cpu(y_pred, y_true, k): device = "cpu" ndcg = NDCG(k=k, device=device) - ndcg.update([pred_score, true_score]) + ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() - result_sklearn = ndcg_score(true_score.numpy(), pred_score.numpy(), k=k) + result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k) np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) @pytest.mark.parametrize( - "true_score, pred_score", + "y_pred, y_true", [ - (torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]])), - (torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]]), torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]])), ( - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]], device="cuda"), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), + ), + ( + torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda"), + torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda"), + ), + ( + torch.tensor( + [[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda" + ), + torch.tensor( + [[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda" + ), + ), + ( + torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]], device="cuda"), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), + ), + ( + torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]], device="cuda"), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), ), ], ) @pytest.mark.parametrize("k", [None, 2, 3]) @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") -def test_output_gpu(true_score, pred_score, k): +def test_output_gpu(y_pred, y_true, k): device = "cuda" ndcg = NDCG(k=k, device=device) - ndcg.update([pred_score, true_score]) + ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() - result_sklearn = ndcg_score(true_score.numpy(), pred_score.numpy(), k=k) + result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k) np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) def test_reset(): - true_score = torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]) - pred_score = torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]) + y_true = torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]) + y_pred = torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]) ndcg = NDCG() - ndcg.update([pred_score, true_score]) + ndcg.update([y_pred, y_true]) ndcg.reset() with pytest.raises(NotComputableError, match=r"NGCD must have at least one example before it can be computed."): ndcg.compute() + + +@pytest.mark.parametrize( + "y_pred, y_true", + [ + (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), + ( + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), + ), + (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + ( + torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), + ), + ], +) +@pytest.mark.parametrize("k", [None, 2, 3]) +def test_exponential(y_pred, y_true, k): + + device = "cpu" + + ndcg = NDCG(k=k, device=device, exponential=True) + ndcg.update([y_pred, y_true]) + result_ignite = ndcg.compute() + result_sklearn = ndcg_score(2 ** y_true.numpy() - 1, y_pred.numpy(), k=k) + + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + + +@pytest.mark.parametrize( + "y_pred, y_true", + [ + (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), + ( + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), + ), + ], +) +@pytest.mark.parametrize("k", [None, 2, 3]) +def test_output_cpu_ignore_ties(y_pred, y_true, k): + + device = "cpu" + + ndcg = NDCG(k=k, device=device, ignore_ties=True) + ndcg.update([y_pred, y_true]) + result_ignite = ndcg.compute() + result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k) + + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + + +@pytest.mark.parametrize( + "y_pred, y_true", + [ + ( + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]], device="cuda"), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), + ), + ( + torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda"), + torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda"), + ), + ( + torch.tensor( + [[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda" + ), + torch.tensor( + [[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda" + ), + ), + ], +) +@pytest.mark.parametrize("k", [None, 2, 3]) +@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") +def test_output_gpu_ignore_ties(y_pred, y_true, k): + + device = "cuda" + ndcg = NDCG(k=k, device=device, ignore_ties=True) + ndcg.update([y_pred, y_true]) + result_ignite = ndcg.compute() + result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k) + + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + + +@pytest.mark.parametrize("log_base", [2, 3, 10]) +def test_log_base(log_base): + def _ndcg_sample_scores(y_true, y_score, k=None, ignore_ties=False): + + gain = _dcg_sample_scores(y_true, y_score, k, ignore_ties=ignore_ties) + normalizing_gain = _dcg_sample_scores(y_true, y_true, k, ignore_ties=True) + all_irrelevant = normalizing_gain == 0 + gain[all_irrelevant] = 0 + gain[~all_irrelevant] /= normalizing_gain[~all_irrelevant] + return gain + + def ndcg_score_with_log_base(y_true, y_score, *, k=None, sample_weight=None, ignore_ties=False, log_base=2): + + gain = _ndcg_sample_scores(y_true, y_score, k=k, ignore_ties=ignore_ties) + return np.average(gain, weights=sample_weight) + + y_true = torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]) + y_pred = torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]]) + + ndcg = NDCG(log_base=log_base) + ndcg.update([y_pred, y_true]) + + result_ignite = ndcg.compute() + result_sklearn = ndcg_score_with_log_base(y_true.numpy(), y_pred.numpy(), log_base=log_base) + + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) From 115501b862102bc98c5159695c64485cc709ab1d Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Tue, 26 Jul 2022 16:13:14 +0530 Subject: [PATCH 10/23] Added GPU check on top --- tests/ignite/metrics/test_ndcg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index f3d5e982e06..0afdfcc087c 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -37,6 +37,7 @@ def test_output_cpu(y_pred, y_true, k): np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) +@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") @pytest.mark.parametrize( "y_pred, y_true", [ @@ -67,7 +68,6 @@ def test_output_cpu(y_pred, y_true, k): ], ) @pytest.mark.parametrize("k", [None, 2, 3]) -@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") def test_output_gpu(y_pred, y_true, k): device = "cuda" @@ -144,6 +144,7 @@ def test_output_cpu_ignore_ties(y_pred, y_true, k): np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) +@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") @pytest.mark.parametrize( "y_pred, y_true", [ @@ -166,7 +167,6 @@ def test_output_cpu_ignore_ties(y_pred, y_true, k): ], ) @pytest.mark.parametrize("k", [None, 2, 3]) -@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") def test_output_gpu_ignore_ties(y_pred, y_true, k): device = "cuda" From 189b5791e74b14834179628bf3df21412e5426c6 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Tue, 26 Jul 2022 17:54:42 +0530 Subject: [PATCH 11/23] Put tensors on GPU inside the function to pervent error --- tests/ignite/metrics/test_ndcg.py | 49 ++++++++++--------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index 0afdfcc087c..1c0839a2ff3 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -41,29 +41,16 @@ def test_output_cpu(y_pred, y_true, k): @pytest.mark.parametrize( "y_pred, y_true", [ + (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), ( - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]], device="cuda"), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), - ), - ( - torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda"), - torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda"), - ), - ( - torch.tensor( - [[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda" - ), - torch.tensor( - [[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda" - ), - ), - ( - torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]], device="cuda"), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), ), + (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), ( - torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]], device="cuda"), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), + torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), ), ], ) @@ -71,6 +58,8 @@ def test_output_cpu(y_pred, y_true, k): def test_output_gpu(y_pred, y_true, k): device = "cuda" + y_pred = y_pred.to(device) + y_true = y_true.to(device) ndcg = NDCG(k=k, device=device) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() @@ -148,21 +137,11 @@ def test_output_cpu_ignore_ties(y_pred, y_true, k): @pytest.mark.parametrize( "y_pred, y_true", [ + (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), ( - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]], device="cuda"), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]], device="cuda"), - ), - ( - torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda"), - torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda"), - ), - ( - torch.tensor( - [[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]], device="cuda" - ), - torch.tensor( - [[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]], device="cuda" - ), + torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), ), ], ) @@ -170,6 +149,8 @@ def test_output_cpu_ignore_ties(y_pred, y_true, k): def test_output_gpu_ignore_ties(y_pred, y_true, k): device = "cuda" + y_pred = y_pred.to(device) + y_true = y_true.to(device) ndcg = NDCG(k=k, device=device, ignore_ties=True) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() From c5094562b6b4a2bd88da07c1d3074ce431540c87 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Wed, 27 Jul 2022 13:40:38 +0530 Subject: [PATCH 12/23] Improved tests and minor bugfixes --- ignite/metrics/recsys/ndcg.py | 9 +- tests/ignite/metrics/test_ndcg.py | 172 ++++++++++-------------------- 2 files changed, 61 insertions(+), 120 deletions(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index 937ec5a7a06..2a0648c9285 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -85,12 +85,13 @@ def __init__( ignore_ties: bool = False, ): - assert log_base != 1 or log_base <= 0, f"Illegal value {log_base} for log_base" + if log_base == 1 or log_base <= 0: + raise ValueError(f"Illegal value {log_base} for log_base") self.log_base = log_base self.k = k self.exponential = exponential - super(NDCG, self).__init__(output_transform=output_transform, device=device) self.ignore_ties = ignore_ties + super(NDCG, self).__init__(output_transform=output_transform, device=device) def reset(self) -> None: @@ -104,7 +105,9 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self.exponential: y_true = 2 ** y_true - 1 - gain = _ndcg_sample_scores(y_pred, y_true, k=self.k, log_base=self.log_base, device=self._device) + gain = _ndcg_sample_scores( + y_pred, y_true, k=self.k, log_base=self.log_base, device=self._device, ignore_ties=self.ignore_ties + ) self.ndcg += torch.sum(gain) self.num_examples += y_pred.shape[0] diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index 1c0839a2ff3..aef1a9d6da0 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -8,62 +8,78 @@ from ignite.metrics.recsys.ndcg import NDCG -@pytest.mark.parametrize( - "y_pred, y_true", - [ - (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), - (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), +@pytest.fixture(params=[item for item in range(5)]) +def test_case(request): + + return [ + (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), True), + (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]]), True), ( torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), + True, ), - (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), + (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), False), ( torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), + False, ), - ], -) + ][request.param] + + @pytest.mark.parametrize("k", [None, 2, 3]) -def test_output_cpu(y_pred, y_true, k): +@pytest.mark.parametrize("exponential", [True, False]) +def test_output_cpu(test_case, k, exponential): device = "cpu" - - ndcg = NDCG(k=k, device=device) + y_pred, y_true, try_ignore_ties = test_case + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=False) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() - result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k) + + if try_ignore_ties: + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=True) + ndcg.update([y_pred, y_true]) + result_ignite_ignore_ties = ndcg.compute() + + if exponential: + y_true = 2 ** y_true - 1 + + result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k, ignore_ties=False) + + if try_ignore_ties: + result_sklearn_ignore_ties = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k, ignore_ties=True) + np.testing.assert_allclose(np.array(result_ignite_ignore_ties), result_sklearn_ignore_ties, rtol=2e-7) np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) -@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") -@pytest.mark.parametrize( - "y_pred, y_true", - [ - (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), - (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), - ( - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), - ), - (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), - ( - torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), - ), - ], -) @pytest.mark.parametrize("k", [None, 2, 3]) -def test_output_gpu(y_pred, y_true, k): +@pytest.mark.parametrize("exponential", [True, False]) +def test_output_gpu(test_case, k, exponential): device = "cuda" + y_pred, y_true, try_ignore_ties = test_case y_pred = y_pred.to(device) y_true = y_true.to(device) - ndcg = NDCG(k=k, device=device) + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=False) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() - result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k) + + if try_ignore_ties: + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=True) + ndcg.update([y_pred, y_true]) + result_ignite_ignore_ties = ndcg.compute() + + if exponential: + y_true = 2 ** y_true - 1 + + result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k, ignore_ties=False) + + if try_ignore_ties: + result_sklearn_ignore_ties = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k, ignore_ties=True) + np.testing.assert_allclose(np.array(result_ignite_ignore_ties), result_sklearn_ignore_ties, rtol=2e-7) np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) @@ -80,96 +96,18 @@ def test_reset(): ndcg.compute() -@pytest.mark.parametrize( - "y_pred, y_true", - [ - (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), - (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), - ( - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), - ), - (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), - ( - torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), - ), - ], -) -@pytest.mark.parametrize("k", [None, 2, 3]) -def test_exponential(y_pred, y_true, k): - - device = "cpu" - - ndcg = NDCG(k=k, device=device, exponential=True) - ndcg.update([y_pred, y_true]) - result_ignite = ndcg.compute() - result_sklearn = ndcg_score(2 ** y_true.numpy() - 1, y_pred.numpy(), k=k) - - np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) +def _ndcg_sample_scores(y_true, y_score, k=None, ignore_ties=False): - -@pytest.mark.parametrize( - "y_pred, y_true", - [ - (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), - (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), - ( - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), - ), - ], -) -@pytest.mark.parametrize("k", [None, 2, 3]) -def test_output_cpu_ignore_ties(y_pred, y_true, k): - - device = "cpu" - - ndcg = NDCG(k=k, device=device, ignore_ties=True) - ndcg.update([y_pred, y_true]) - result_ignite = ndcg.compute() - result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k) - - np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) - - -@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") -@pytest.mark.parametrize( - "y_pred, y_true", - [ - (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]])), - (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), - ( - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), - ), - ], -) -@pytest.mark.parametrize("k", [None, 2, 3]) -def test_output_gpu_ignore_ties(y_pred, y_true, k): - - device = "cuda" - y_pred = y_pred.to(device) - y_true = y_true.to(device) - ndcg = NDCG(k=k, device=device, ignore_ties=True) - ndcg.update([y_pred, y_true]) - result_ignite = ndcg.compute() - result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k) - - np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + gain = _dcg_sample_scores(y_true, y_score, k, ignore_ties=ignore_ties) + normalizing_gain = _dcg_sample_scores(y_true, y_true, k, ignore_ties=True) + all_irrelevant = normalizing_gain == 0 + gain[all_irrelevant] = 0 + gain[~all_irrelevant] /= normalizing_gain[~all_irrelevant] + return gain @pytest.mark.parametrize("log_base", [2, 3, 10]) def test_log_base(log_base): - def _ndcg_sample_scores(y_true, y_score, k=None, ignore_ties=False): - - gain = _dcg_sample_scores(y_true, y_score, k, ignore_ties=ignore_ties) - normalizing_gain = _dcg_sample_scores(y_true, y_true, k, ignore_ties=True) - all_irrelevant = normalizing_gain == 0 - gain[all_irrelevant] = 0 - gain[~all_irrelevant] /= normalizing_gain[~all_irrelevant] - return gain - def ndcg_score_with_log_base(y_true, y_score, *, k=None, sample_weight=None, ignore_ties=False, log_base=2): gain = _ndcg_sample_scores(y_true, y_score, k=k, ignore_ties=ignore_ties) From 84900f0cc20dc02c2320949f278faab838134be7 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Wed, 27 Jul 2022 13:56:11 +0530 Subject: [PATCH 13/23] Removed device hyperparam from _ndcg_smaple_scores --- ignite/metrics/recsys/ndcg.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index 2a0648c9285..05702b11478 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -62,9 +62,9 @@ def _ndcg_sample_scores( k: Optional[int] = None, log_base: Union[int, float] = 2, ignore_ties: bool = False, - device: Union[str, torch.device] = torch.device("cpu"), ) -> torch.Tensor: + device = y_true.device gain = _dcg_sample_scores(y_pred, y_true, k=k, log_base=log_base, ignore_ties=ignore_ties, device=device) if not ignore_ties: gain = gain.unsqueeze(dim=-1) @@ -105,9 +105,7 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self.exponential: y_true = 2 ** y_true - 1 - gain = _ndcg_sample_scores( - y_pred, y_true, k=self.k, log_base=self.log_base, device=self._device, ignore_ties=self.ignore_ties - ) + gain = _ndcg_sample_scores(y_pred, y_true, k=self.k, log_base=self.log_base, ignore_ties=self.ignore_ties) self.ndcg += torch.sum(gain) self.num_examples += y_pred.shape[0] From 9bfc06e0980e24b43175731cea06cf24213421f1 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Wed, 27 Jul 2022 13:58:56 +0530 Subject: [PATCH 14/23] Skipped GPU tests for CPU only systems --- tests/ignite/metrics/test_ndcg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index aef1a9d6da0..de098c9057d 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -57,6 +57,7 @@ def test_output_cpu(test_case, k, exponential): @pytest.mark.parametrize("k", [None, 2, 3]) @pytest.mark.parametrize("exponential", [True, False]) +@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") def test_output_gpu(test_case, k, exponential): device = "cuda" From 477e0968547c49076cd1247d541c30438a04d293 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Wed, 27 Jul 2022 15:08:29 +0530 Subject: [PATCH 15/23] Changed Error message --- ignite/metrics/recsys/ndcg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index 05702b11478..b8a14bd2ad9 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -86,7 +86,7 @@ def __init__( ): if log_base == 1 or log_base <= 0: - raise ValueError(f"Illegal value {log_base} for log_base") + raise ValueError(f"Argument log_base should positive and not equal one,but got {log_base}") self.log_base = log_base self.k = k self.exponential = exponential From ac800ff49b7e8938886074aa6380788d49e74d63 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Sat, 27 Aug 2022 16:59:03 +0530 Subject: [PATCH 16/23] Made tests randomised from deterministic and introduced ignore_ties_flag to reduce duplicate code --- ignite/metrics/recsys/ndcg.py | 3 + tests/ignite/metrics/test_ndcg.py | 99 ++++++++++++++++++------------- 2 files changed, 62 insertions(+), 40 deletions(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index b8a14bd2ad9..102e736be5c 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -102,6 +102,9 @@ def update(self, output: Sequence[torch.Tensor]) -> None: y_pred, y_true = output[0].detach(), output[1].detach() + y_pred = y_pred.to(torch.float32) + y_true = y_true.to(torch.float32) + if self.exponential: y_true = 2 ** y_true - 1 diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index de098c9057d..cb74d9078a9 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -8,81 +8,72 @@ from ignite.metrics.recsys.ndcg import NDCG -@pytest.fixture(params=[item for item in range(5)]) +@pytest.fixture(params=[item for item in range(6)]) def test_case(request): return [ - (torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), True), - (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]]), True), - ( - torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [3.7, 4.8, 3.9, 4.3, 4.9], [3.7, 4.8, 3.9, 4.3, 4.9]]), - torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.2, 4.5, 8.9, 5.6, 7.2], [2.9, 5.6, 3.8, 7.9, 6.2]]), - True, - ), - (torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0]]), False), + (torch.tensor([[3.7, 4.8, 3.9, 4.3, 4.9]]), torch.tensor([[2.9, 5.6, 3.8, 7.9, 6.2]])), ( torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), - False, ), - ][request.param] + ][request.param % 2] @pytest.mark.parametrize("k", [None, 2, 3]) @pytest.mark.parametrize("exponential", [True, False]) -def test_output_cpu(test_case, k, exponential): +@pytest.mark.parametrize("ignore_ties_flag", [True, False]) +@pytest.mark.parametrize("ignore_ties_input", [True, False]) +def test_output_cpu(test_case, k, exponential, ignore_ties_flag, ignore_ties_input): device = "cpu" - y_pred, y_true, try_ignore_ties = test_case - ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=False) + y_pred_distribution, y_true = test_case + + y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not ignore_ties_input) + + if not ignore_ties_input and ignore_ties_flag: + return + + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties_flag) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() - if try_ignore_ties: - ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=True) - ndcg.update([y_pred, y_true]) - result_ignite_ignore_ties = ndcg.compute() - if exponential: y_true = 2 ** y_true - 1 - result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k, ignore_ties=False) - - if try_ignore_ties: - result_sklearn_ignore_ties = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k, ignore_ties=True) - np.testing.assert_allclose(np.array(result_ignite_ignore_ties), result_sklearn_ignore_ties, rtol=2e-7) + result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k, ignore_ties=ignore_ties_flag) - np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-6) @pytest.mark.parametrize("k", [None, 2, 3]) @pytest.mark.parametrize("exponential", [True, False]) +@pytest.mark.parametrize("ignore_ties_flag", [True, False]) +@pytest.mark.parametrize("ignore_ties_input", [True, False]) @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") -def test_output_gpu(test_case, k, exponential): +def test_output_gpu(test_case, k, exponential, ignore_ties_flag, ignore_ties_input): device = "cuda" - y_pred, y_true, try_ignore_ties = test_case + y_pred_distribution, y_true = test_case + + y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not ignore_ties_input) + + if not ignore_ties_input and ignore_ties_flag: + return + y_pred = y_pred.to(device) y_true = y_true.to(device) - ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=False) + + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties_flag) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() - if try_ignore_ties: - ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=True) - ndcg.update([y_pred, y_true]) - result_ignite_ignore_ties = ndcg.compute() - if exponential: y_true = 2 ** y_true - 1 - result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k, ignore_ties=False) - - if try_ignore_ties: - result_sklearn_ignore_ties = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k, ignore_ties=True) - np.testing.assert_allclose(np.array(result_ignite_ignore_ties), result_sklearn_ignore_ties, rtol=2e-7) + result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k, ignore_ties=ignore_ties_flag) - np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-6) def test_reset(): @@ -123,4 +114,32 @@ def ndcg_score_with_log_base(y_true, y_score, *, k=None, sample_weight=None, ign result_ignite = ndcg.compute() result_sklearn = ndcg_score_with_log_base(y_true.numpy(), y_pred.numpy(), log_base=log_base) - np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-7) + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-6) + + +def test_update(test_case): + + y_pred, y_true = test_case + + y_pred = y_pred + y_true = y_true + + y1_pred = torch.multinomial(y_pred, 5, replacement=True) + y1_true = torch.multinomial(y_true, 5, replacement=True) + + y2_pred = torch.multinomial(y_pred, 5, replacement=True) + y2_true = torch.multinomial(y_true, 5, replacement=True) + + y_pred_combined = torch.cat((y1_pred, y2_pred)) + y_true_combined = torch.cat((y1_true, y2_true)) + + ndcg = NDCG() + + ndcg.update([y1_pred, y1_true]) + ndcg.update([y2_pred, y2_true]) + + result_ignite = ndcg.compute() + + result_sklearn = ndcg_score(y_true_combined.numpy(), y_pred_combined.numpy()) + + np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-6) From 0c1d6fdf06cfefcd646d354a337ea4a9260b798a Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Mon, 29 Aug 2022 16:36:32 +0530 Subject: [PATCH 17/23] Changed test name to test_output_cuda from test_output_gpu --- tests/ignite/metrics/test_ndcg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index cb74d9078a9..f1a9f6e26b1 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -51,7 +51,7 @@ def test_output_cpu(test_case, k, exponential, ignore_ties_flag, ignore_ties_inp @pytest.mark.parametrize("ignore_ties_flag", [True, False]) @pytest.mark.parametrize("ignore_ties_input", [True, False]) @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") -def test_output_gpu(test_case, k, exponential, ignore_ties_flag, ignore_ties_input): +def test_output_cuda(test_case, k, exponential, ignore_ties_flag, ignore_ties_input): device = "cuda" y_pred_distribution, y_true = test_case From 2931d20edd47c100213853f3a05c8dcf5b803168 Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Tue, 30 Aug 2022 06:17:10 +0530 Subject: [PATCH 18/23] Changed variable names to replacement and ignore_ties and removed redundat tests --- tests/ignite/metrics/test_ndcg.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index f1a9f6e26b1..917ea5677a8 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -22,56 +22,48 @@ def test_case(request): @pytest.mark.parametrize("k", [None, 2, 3]) @pytest.mark.parametrize("exponential", [True, False]) -@pytest.mark.parametrize("ignore_ties_flag", [True, False]) -@pytest.mark.parametrize("ignore_ties_input", [True, False]) -def test_output_cpu(test_case, k, exponential, ignore_ties_flag, ignore_ties_input): +@pytest.mark.parametrize("ignore_ties, replacement", [(True, False), (False, True), (False, False)]) +def test_output_cpu(test_case, k, exponential, ignore_ties, replacement): device = "cpu" y_pred_distribution, y_true = test_case - y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not ignore_ties_input) + y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not replacement) - if not ignore_ties_input and ignore_ties_flag: - return - - ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties_flag) + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() if exponential: y_true = 2 ** y_true - 1 - result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k, ignore_ties=ignore_ties_flag) + result_sklearn = ndcg_score(y_true.numpy(), y_pred.numpy(), k=k, ignore_ties=ignore_ties) np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-6) @pytest.mark.parametrize("k", [None, 2, 3]) @pytest.mark.parametrize("exponential", [True, False]) -@pytest.mark.parametrize("ignore_ties_flag", [True, False]) -@pytest.mark.parametrize("ignore_ties_input", [True, False]) +@pytest.mark.parametrize("ignore_ties, replacement", [(True, False), (False, True), (False, False)]) @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") -def test_output_cuda(test_case, k, exponential, ignore_ties_flag, ignore_ties_input): +def test_output_cuda(test_case, k, exponential, ignore_ties, replacement): device = "cuda" y_pred_distribution, y_true = test_case - y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not ignore_ties_input) - - if not ignore_ties_input and ignore_ties_flag: - return + y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not replacement) y_pred = y_pred.to(device) y_true = y_true.to(device) - ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties_flag) + ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() if exponential: y_true = 2 ** y_true - 1 - result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k, ignore_ties=ignore_ties_flag) + result_sklearn = ndcg_score(y_true.cpu().numpy(), y_pred.cpu().numpy(), k=k, ignore_ties=ignore_ties) np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-6) From c308e4128da84f730f8c0d0ed106ada035bbfdbf Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Tue, 30 Aug 2022 06:18:58 +0530 Subject: [PATCH 19/23] Changed variable names to replacement and ignore_ties and removed redundat tests --- tests/ignite/metrics/test_ndcg.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index 917ea5677a8..1b54c0c685e 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -28,7 +28,10 @@ def test_output_cpu(test_case, k, exponential, ignore_ties, replacement): device = "cpu" y_pred_distribution, y_true = test_case - y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not replacement) + y_pred = torch.multinomial(y_pred_distribution, 5, replacement=replacement) + + if not replacement and ignore_ties: + return ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties) ndcg.update([y_pred, y_true]) @@ -51,7 +54,10 @@ def test_output_cuda(test_case, k, exponential, ignore_ties, replacement): device = "cuda" y_pred_distribution, y_true = test_case - y_pred = torch.multinomial(y_pred_distribution, 5, replacement=not replacement) + y_pred = torch.multinomial(y_pred_distribution, 5, replacement=replacement) + + if not replacement and ignore_ties: + return y_pred = y_pred.to(device) y_true = y_true.to(device) From 6e6627337d8302c5a1d19729d87fd2139cd433fb Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Tue, 30 Aug 2022 13:33:20 +0530 Subject: [PATCH 20/23] Removed redundant test cases and removed the redundant if statement --- tests/ignite/metrics/test_ndcg.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index 1b54c0c685e..b641b1a0079 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -8,7 +8,7 @@ from ignite.metrics.recsys.ndcg import NDCG -@pytest.fixture(params=[item for item in range(6)]) +@pytest.fixture(params=[item for item in range(2)]) def test_case(request): return [ @@ -17,7 +17,7 @@ def test_case(request): torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), ), - ][request.param % 2] + ][request.param] @pytest.mark.parametrize("k", [None, 2, 3]) @@ -30,9 +30,6 @@ def test_output_cpu(test_case, k, exponential, ignore_ties, replacement): y_pred = torch.multinomial(y_pred_distribution, 5, replacement=replacement) - if not replacement and ignore_ties: - return - ndcg = NDCG(k=k, device=device, exponential=exponential, ignore_ties=ignore_ties) ndcg.update([y_pred, y_true]) result_ignite = ndcg.compute() @@ -56,9 +53,6 @@ def test_output_cuda(test_case, k, exponential, ignore_ties, replacement): y_pred = torch.multinomial(y_pred_distribution, 5, replacement=replacement) - if not replacement and ignore_ties: - return - y_pred = y_pred.to(device) y_true = y_true.to(device) From eb75afa4a9e390ab800e42d75b2d4eadae5d606a Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Tue, 30 Aug 2022 23:28:21 +0530 Subject: [PATCH 21/23] Added distributed tests, added multiple test cases corresponding to one multiomial distribution --- ignite/metrics/recsys/ndcg.py | 4 +- tests/ignite/metrics/test_ndcg.py | 125 +++++++++++++++++++++++++++++- 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index 102e736be5c..98dafa7f07c 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -102,8 +102,8 @@ def update(self, output: Sequence[torch.Tensor]) -> None: y_pred, y_true = output[0].detach(), output[1].detach() - y_pred = y_pred.to(torch.float32) - y_true = y_true.to(torch.float32) + y_pred = y_pred.to(torch.float32).to(self._device) + y_true = y_true.to(torch.float32).to(self._device) if self.exponential: y_true = 2 ** y_true - 1 diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index b641b1a0079..a46b4ccadea 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -1,14 +1,19 @@ +import os + import numpy as np import pytest import torch from sklearn.metrics import ndcg_score from sklearn.metrics._ranking import _dcg_sample_scores +import ignite.distributed as idist +from ignite.engine import Engine + from ignite.exceptions import NotComputableError from ignite.metrics.recsys.ndcg import NDCG -@pytest.fixture(params=[item for item in range(2)]) +@pytest.fixture(params=[item for item in range(6)]) def test_case(request): return [ @@ -17,7 +22,7 @@ def test_case(request): torch.tensor([[3.7, 3.7, 3.7, 3.7, 3.7], [3.7, 3.7, 3.7, 3.7, 3.9]]), torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]), ), - ][request.param] + ][request.param % 2] @pytest.mark.parametrize("k", [None, 2, 3]) @@ -135,3 +140,119 @@ def test_update(test_case): result_sklearn = ndcg_score(y_true_combined.numpy(), y_pred_combined.numpy()) np.testing.assert_allclose(np.array(result_ignite), result_sklearn, rtol=2e-6) + + +def _test_distrib_output(device): + + rank = idist.get_rank() + + def _test(n_epochs, metric_device): + + metric_device = torch.device(metric_device) + + n_iters = 80 + batch_size = 16 + n_items = 10 + + torch.manual_seed(12 + rank) + + y_true = torch.rand((n_iters * batch_size, n_items)).to(device) + y_preds = torch.rand((n_iters * batch_size, n_items)).to(device) + + def update(_, i): + return ( + [v for v in y_preds[i * batch_size : (i + 1) * batch_size, ...]], + [v for v in y_true[i * batch_size : (i + 1) * batch_size]], + ) + + engine = Engine(update) + + ndcg = NDCG(device=metric_device) + ndcg.attach(engine, "ndcg") + + data = list(range(n_iters)) + engine.run(data=data, max_epochs=n_epochs) + + y_true = idist.all_gather(y_true) + y_preds = idist.all_gather(y_preds) + + assert ( + ndcg._device == metric_device + ), f"{type(ndcg._device)}:{ndcg._device} vs {type(metric_device)}:{metric_device}" + + assert "ndcg" in engine.state.metrics + res = engine.state.metrics["ndcg"] + if isinstance(res, torch.Tensor): + res = res.cpu().numpy() + + true_res = ndcg_score(y_true.cpu().numpy(), y_preds.cpu().numpy()) + assert pytest.approx(res) == true_res + + metric_devices = ["cpu"] + if device.type != "xla": + metric_devices.append(idist.device()) + for metric_device in metric_devices: + for _ in range(2): + _test(n_epochs=1, metric_device=metric_device) + _test(n_epochs=2, metric_device=metric_device) + + +@pytest.mark.multinode_distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +@pytest.mark.skipif("MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed") +def test_multinode_distrib_gloo_cpu_or_gpu(distributed_context_multi_node_gloo): + + device = idist.device() + _test_distrib_output(device) + + +@pytest.mark.multinode_distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +@pytest.mark.skipif("GPU_MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed") +def test_multinode_distrib_nccl_gpu(distributed_context_multi_node_nccl): + + device = idist.device() + _test_distrib_output(device) + + +@pytest.mark.distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") +def test_distrib_nccl_gpu(distributed_context_single_node_nccl): + + device = idist.device() + _test_distrib_output(device) + + +@pytest.mark.distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +def test_distrib_gloo_cpu_or_gpu(distributed_context_single_node_gloo): + + device = idist.device() + _test_distrib_output(device) + + +@pytest.mark.distributed +@pytest.mark.skipif(not idist.has_hvd_support, reason="Skip if no Horovod dist support") +@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc") +def test_distrib_hvd(gloo_hvd_executor): + + device = torch.device("cpu" if not torch.cuda.is_available() else "cuda") + nproc = 4 if not torch.cuda.is_available() else torch.cuda.device_count() + + gloo_hvd_executor(_test_distrib_output, (device,), np=nproc, do_init=True) + + +@pytest.mark.tpu +@pytest.mark.skipif("NUM_TPU_WORKERS" in os.environ, reason="Skip if NUM_TPU_WORKERS is in env vars") +@pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package") +def test_distrib_single_device_xla(): + + device = idist.device() + _test_distrib_output(device) + + +def _test_distrib_xla_nprocs(index): + + device = idist.device() + _test_distrib_output(device) From dcf276d4ae8a1c33e7286c5c6217dbfff42c8bcc Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Wed, 31 Aug 2022 11:26:06 +0530 Subject: [PATCH 22/23] Made the tests wsork on in ddp configuration --- ignite/metrics/recsys/ndcg.py | 5 ++++- tests/ignite/metrics/test_ndcg.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ignite/metrics/recsys/ndcg.py b/ignite/metrics/recsys/ndcg.py index 98dafa7f07c..0545055e5e4 100644 --- a/ignite/metrics/recsys/ndcg.py +++ b/ignite/metrics/recsys/ndcg.py @@ -3,7 +3,7 @@ import torch from ignite.exceptions import NotComputableError -from ignite.metrics.metric import Metric +from ignite.metrics.metric import Metric, reinit__is_reduced, sync_all_reduce __all__ = ["NDCG"] @@ -93,11 +93,13 @@ def __init__( self.ignore_ties = ignore_ties super(NDCG, self).__init__(output_transform=output_transform, device=device) + @reinit__is_reduced def reset(self) -> None: self.num_examples = 0 self.ndcg = torch.tensor(0.0, device=self._device) + @reinit__is_reduced def update(self, output: Sequence[torch.Tensor]) -> None: y_pred, y_true = output[0].detach(), output[1].detach() @@ -112,6 +114,7 @@ def update(self, output: Sequence[torch.Tensor]) -> None: self.ndcg += torch.sum(gain) self.num_examples += y_pred.shape[0] + @sync_all_reduce("ndcg", "num_examples") def compute(self) -> float: if self.num_examples == 0: raise NotComputableError("NGCD must have at least one example before it can be computed.") diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index a46b4ccadea..6a9da342b78 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -150,9 +150,9 @@ def _test(n_epochs, metric_device): metric_device = torch.device(metric_device) - n_iters = 80 - batch_size = 16 - n_items = 10 + n_iters = 5 + batch_size = 8 + n_items = 5 torch.manual_seed(12 + rank) @@ -256,3 +256,11 @@ def _test_distrib_xla_nprocs(index): device = idist.device() _test_distrib_output(device) + + +@pytest.mark.tpu +@pytest.mark.skipif("NUM_TPU_WORKERS" not in os.environ, reason="Skip if no NUM_TPU_WORKERS in env vars") +@pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package") +def test_distrib_xla_nprocs(xmp_executor): + n = int(os.environ["NUM_TPU_WORKERS"]) + xmp_executor(_test_distrib_xla_nprocs, args=(), nprocs=n) From 6dcf3b210fc7197cfdfd725b5e4b9460c02a9d1e Mon Sep 17 00:00:00 2001 From: Ojasv Kamal Date: Thu, 1 Sep 2022 14:43:41 +0530 Subject: [PATCH 23/23] Returning tuple of two tensors instead of tuple of list of tensors --- tests/ignite/metrics/test_ndcg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ignite/metrics/test_ndcg.py b/tests/ignite/metrics/test_ndcg.py index 6a9da342b78..2429b5bc4a6 100644 --- a/tests/ignite/metrics/test_ndcg.py +++ b/tests/ignite/metrics/test_ndcg.py @@ -159,10 +159,10 @@ def _test(n_epochs, metric_device): y_true = torch.rand((n_iters * batch_size, n_items)).to(device) y_preds = torch.rand((n_iters * batch_size, n_items)).to(device) - def update(_, i): + def update(engine, i): return ( - [v for v in y_preds[i * batch_size : (i + 1) * batch_size, ...]], - [v for v in y_true[i * batch_size : (i + 1) * batch_size]], + y_preds[i * batch_size : (i + 1) * batch_size, ...], + y_true[i * batch_size : (i + 1) * batch_size, ...], ) engine = Engine(update)