From 013aae457a9476b215fe2af47e0fc7bb69408cd5 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Wed, 2 Oct 2024 23:05:06 -0400 Subject: [PATCH 01/28] base judge --- tests/test_judges.py | 10 +++++++++- trl/__init__.py | 4 ++++ trl/trainer/__init__.py | 4 ++++ trl/trainer/judges.py | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index 5c3be75533..7458c846a4 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -14,7 +14,7 @@ import unittest -from trl import HfPairwiseJudge, PairRMJudge, RandomPairwiseJudge, RandomRankJudge +from trl import HfPairwiseJudge, PairRMJudge, RandomPairwiseJudge, RandomRankJudge, RandomBinaryJudge class TestJudges(unittest.TestCase): @@ -22,6 +22,14 @@ def _get_prompts_and_completions(self): prompts = ["The capital of France is", "The biggest planet in the solar system is"] completions = [["Paris", "Marseille"], ["Saturn", "Jupiter"]] return prompts, completions + + def test_random_binary_judge(self): + judge = RandomBinaryJudge() + prompts = prompts = ["The capital of France is", "The capital of France is", "The biggest planet in the solar system is", "The biggest planet in the solar system is"] + completions = ["Paris", "Marseille", "Saturn", "Jupiter"] + judgements = judge.judge(prompts=prompts, completions=completions) + self.assertEqual(len(judgements), 4) + self.assertTrue(all(judgement in {0, 1, -1} for judgement in judgements)) def test_random_pairwise_judge(self): judge = RandomPairwiseJudge() diff --git a/trl/__init__.py b/trl/__init__.py index 87ce9bfa63..08051e45c0 100644 --- a/trl/__init__.py +++ b/trl/__init__.py @@ -51,6 +51,7 @@ "AlignPropConfig", "AlignPropTrainer", "BaseJudge", + "BaseBinaryJudge", "BasePairwiseJudge", "BaseRankJudge", "BCOConfig", @@ -82,6 +83,7 @@ "PPOTrainer", "PPOv2Config", "PPOv2Trainer", + "RandomBinaryJudge", "RandomPairwiseJudge", "RandomRankJudge", "RewardConfig", @@ -146,6 +148,7 @@ AlignPropConfig, AlignPropTrainer, BaseJudge, + BaseBinaryJudge, BasePairwiseJudge, BaseRankJudge, BCOConfig, @@ -177,6 +180,7 @@ PPOTrainer, PPOv2Config, PPOv2Trainer, + RandomBinaryJudge, RandomPairwiseJudge, RandomRankJudge, RewardConfig, diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index f0eba412c6..2ec67b1022 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -35,11 +35,13 @@ "iterative_sft_trainer": ["IterativeSFTTrainer"], "judges": [ "BaseJudge", + "BaseBinaryJudge", "BasePairwiseJudge", "BaseRankJudge", "HfPairwiseJudge", "OpenAIPairwiseJudge", "PairRMJudge", + "RandomBinaryJudge", "RandomPairwiseJudge", "RandomRankJudge", ], @@ -99,11 +101,13 @@ from .iterative_sft_trainer import IterativeSFTTrainer from .judges import ( BaseJudge, + BaseBinaryJudge, BasePairwiseJudge, BaseRankJudge, HfPairwiseJudge, OpenAIPairwiseJudge, PairRMJudge, + RandomBinaryJudge, RandomPairwiseJudge, RandomRankJudge, ) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 31f7d41a18..457e8b13c6 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -132,8 +132,47 @@ def judge(self, prompts: List[str], completions: List[List[str]], shuffle_order: In such cases, the caller should handle these invalid indices appropriately, possibly by implementing fallback logic or error handling. """ raise NotImplementedError("Judge subclasses must implement the `judge` method.") + + +class BaseConstraintJudge(BaseJudge): + """ + Base class for pairwise judges. + """ + + @abstractmethod + def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[int]: + """ + Judge the completion for a given prompt. Used to assess if a completion satisfies a constraint. + + This base class should be used to implement constraint-based evaluation as done in section 4.1.4 of the CGPO paper (https://arxiv.org/pdf/2409.20370). + It is relevant for assessing whether or not a prompt completion pair satisfies a specific contraint. + + Args: + prompts (`List[str]`): List of prompts. + completions (`List[str]`): List of completions. + shuffle_order (`bool`): Whether to shuffle the order of the completions to avoid positional bias. + + Returns: + List[int]: A list of binary labels: + - 1 indicates that the completion satisfies the evaluated constraint. + - 0 indicates that the completion does not satisfy the evaluated constraint. + Note: + If the judge returns -1 for any prompt, it indicates that the inner process used to compute the preference has failed. + For instance, this could occur if the underlying language model or rule based contraint returned an invalid answer. + In such cases, the caller should handle these invalid indices appropriately, possibly by implementing fallback logic or error handling. + """ + raise NotImplementedError("Judge subclasses must implement the `judge` method.") +class RandomConstraintJudge(BaseConstraintJudge): + """ + Random binary judge, for testing purposes. + """ + + def judge(self, prompts, completions, shuffle_order=True): + return [random.choice([0,1]) for _ in len(prompts)] + + class RandomRankJudge(BaseRankJudge): """ Random rank, for testing purposes. @@ -303,3 +342,5 @@ def get_rank(prompt, candidates): # Return the ranks return ranks + + \ No newline at end of file From 0ea5a48f994d64ecee71bed37dcdb4b714b9ae24 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Wed, 2 Oct 2024 23:26:27 -0400 Subject: [PATCH 02/28] adding mixture of judges --- tests/test_judges.py | 16 ++++++++++++---- trl/__init__.py | 10 ++++++---- trl/trainer/__init__.py | 10 ++++++---- trl/trainer/judges.py | 24 +++++++++++++++++++++++- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index 7458c846a4..1fafb98e06 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -14,7 +14,7 @@ import unittest -from trl import HfPairwiseJudge, PairRMJudge, RandomPairwiseJudge, RandomRankJudge, RandomBinaryJudge +from trl import HfPairwiseJudge, PairRMJudge, RandomPairwiseJudge, RandomRankJudge, RandomConstraintJudge, MixtureOfConstraintJudges class TestJudges(unittest.TestCase): @@ -23,9 +23,17 @@ def _get_prompts_and_completions(self): completions = [["Paris", "Marseille"], ["Saturn", "Jupiter"]] return prompts, completions - def test_random_binary_judge(self): - judge = RandomBinaryJudge() - prompts = prompts = ["The capital of France is", "The capital of France is", "The biggest planet in the solar system is", "The biggest planet in the solar system is"] + def test_mixture_of_constraint_judge(self): + moj = MixtureOfConstraintJudges(judges=[RandomConstraintJudge(), RandomConstraintJudge()]) + prompts = ["The capital of France is", "The capital of France is", "The biggest planet in the solar system is", "The biggest planet in the solar system is"] + completions = ["Paris", "Marseille", "Saturn", "Jupiter"] + judgements = moj.judge(prompts=prompts, completions=completions) + self.assertEqual(len(judgements), 4) + self.assertTrue(all(judgement in {0, 1, -1} for judgement in judgements)) + + def test_random_constraint_judge(self): + judge = RandomConstraintJudge() + prompts = ["The capital of France is", "The capital of France is", "The biggest planet in the solar system is", "The biggest planet in the solar system is"] completions = ["Paris", "Marseille", "Saturn", "Jupiter"] judgements = judge.judge(prompts=prompts, completions=completions) self.assertEqual(len(judgements), 4) diff --git a/trl/__init__.py b/trl/__init__.py index 08051e45c0..f62be5f479 100644 --- a/trl/__init__.py +++ b/trl/__init__.py @@ -51,7 +51,7 @@ "AlignPropConfig", "AlignPropTrainer", "BaseJudge", - "BaseBinaryJudge", + "BinaryConstraintJudge", "BasePairwiseJudge", "BaseRankJudge", "BCOConfig", @@ -70,6 +70,7 @@ "KTOConfig", "KTOTrainer", "LogCompletionsCallback", + "MixtureOfConstraintJudges", "ModelConfig", "NashMDConfig", "NashMDTrainer", @@ -83,7 +84,7 @@ "PPOTrainer", "PPOv2Config", "PPOv2Trainer", - "RandomBinaryJudge", + "RandomConstraintJudge", "RandomPairwiseJudge", "RandomRankJudge", "RewardConfig", @@ -148,7 +149,7 @@ AlignPropConfig, AlignPropTrainer, BaseJudge, - BaseBinaryJudge, + BinaryConstraintJudge, BasePairwiseJudge, BaseRankJudge, BCOConfig, @@ -167,6 +168,7 @@ KTOConfig, KTOTrainer, LogCompletionsCallback, + MixtureOfConstraintJudges, ModelConfig, NashMDConfig, NashMDTrainer, @@ -180,7 +182,7 @@ PPOTrainer, PPOv2Config, PPOv2Trainer, - RandomBinaryJudge, + RandomConstraintJudge, RandomPairwiseJudge, RandomRankJudge, RewardConfig, diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index 2ec67b1022..2c323a397b 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -35,13 +35,14 @@ "iterative_sft_trainer": ["IterativeSFTTrainer"], "judges": [ "BaseJudge", - "BaseBinaryJudge", + "BinaryConstraintJudge", "BasePairwiseJudge", "BaseRankJudge", "HfPairwiseJudge", + "MixtureOfConstraintJudges" "OpenAIPairwiseJudge", "PairRMJudge", - "RandomBinaryJudge", + "RandomConstraintJudge", "RandomPairwiseJudge", "RandomRankJudge", ], @@ -101,13 +102,14 @@ from .iterative_sft_trainer import IterativeSFTTrainer from .judges import ( BaseJudge, - BaseBinaryJudge, + BinaryConstraintJudge, BasePairwiseJudge, BaseRankJudge, HfPairwiseJudge, + MixtureOfConstraintJudges, OpenAIPairwiseJudge, PairRMJudge, - RandomBinaryJudge, + RandomConstraintJudge, RandomPairwiseJudge, RandomRankJudge, ) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 457e8b13c6..6dc7f464f4 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -343,4 +343,26 @@ def get_rank(prompt, candidates): # Return the ranks return ranks - \ No newline at end of file + +class MixtureOfConstraintJudges(BaseConstraintJudge): + """ + Unify the decision of multiple BaseConstraintJudge. + + This class returns 0 ("violated") if it fails on any of the constraint judges (ie a judge returns 0 or -1) and returns 1 ("satisfied") otherwise. + + It is an implementation of the Mixture of Judges as described in the CGPO paper: https://arxiv.org/pdf/2409.20370 + + Args: + judges (List[BaseConstraintJudge]): A list of BaseConstraintJudge. + """ + + def __init__(self, judges: List[BaseConstraintJudge]): + self.judges = judges + + def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[int]: + all_constraint_judgments = [judge.judge(prompts, completions, shuffle_order) for judge in self.judges] + + return [ + 1 if all(constraint_judgment == 1 for constraint_judgment in constraint_judgments) else 0 + for constraint_judgments in zip(*all_constraint_judgments) + ] \ No newline at end of file From 517cfb0f97be8324aae11245faa40d5871107254 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Wed, 2 Oct 2024 23:33:17 -0400 Subject: [PATCH 03/28] update doc --- docs/source/judges.mdx | 7 +++++++ trl/trainer/judges.py | 25 +++++++++++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/source/judges.mdx b/docs/source/judges.mdx index 3e1cda6ba8..eee51c488f 100644 --- a/docs/source/judges.mdx +++ b/docs/source/judges.mdx @@ -50,6 +50,10 @@ judge.judge( [[autodoc]] BaseJudge +## BaseConstraintJudge + +[[autodoc]] BaseConstraintJudge + ## BaseRankJudge [[autodoc]] BaseRankJudge @@ -58,6 +62,9 @@ judge.judge( [[autodoc]] BasePairwiseJudge +## MixtureOfConstraintJudges +[[autodoc]] MixtureOfConstraintJudges + ## RandomRankJudge [[autodoc]] RandomRankJudge diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 6dc7f464f4..8e1eb6d82d 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -132,8 +132,8 @@ def judge(self, prompts: List[str], completions: List[List[str]], shuffle_order: In such cases, the caller should handle these invalid indices appropriately, possibly by implementing fallback logic or error handling. """ raise NotImplementedError("Judge subclasses must implement the `judge` method.") - - + + class BaseConstraintJudge(BaseJudge): """ Base class for pairwise judges. @@ -143,7 +143,7 @@ class BaseConstraintJudge(BaseJudge): def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[int]: """ Judge the completion for a given prompt. Used to assess if a completion satisfies a constraint. - + This base class should be used to implement constraint-based evaluation as done in section 4.1.4 of the CGPO paper (https://arxiv.org/pdf/2409.20370). It is relevant for assessing whether or not a prompt completion pair satisfies a specific contraint. @@ -164,15 +164,16 @@ def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool """ raise NotImplementedError("Judge subclasses must implement the `judge` method.") + class RandomConstraintJudge(BaseConstraintJudge): """ Random binary judge, for testing purposes. """ def judge(self, prompts, completions, shuffle_order=True): - return [random.choice([0,1]) for _ in len(prompts)] - - + return [random.choice([0, 1]) for _ in len(prompts)] + + class RandomRankJudge(BaseRankJudge): """ Random rank, for testing purposes. @@ -343,26 +344,26 @@ def get_rank(prompt, candidates): # Return the ranks return ranks - + class MixtureOfConstraintJudges(BaseConstraintJudge): """ Unify the decision of multiple BaseConstraintJudge. This class returns 0 ("violated") if it fails on any of the constraint judges (ie a judge returns 0 or -1) and returns 1 ("satisfied") otherwise. - + It is an implementation of the Mixture of Judges as described in the CGPO paper: https://arxiv.org/pdf/2409.20370 - + Args: judges (List[BaseConstraintJudge]): A list of BaseConstraintJudge. """ - + def __init__(self, judges: List[BaseConstraintJudge]): self.judges = judges def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[int]: all_constraint_judgments = [judge.judge(prompts, completions, shuffle_order) for judge in self.judges] - + return [ 1 if all(constraint_judgment == 1 for constraint_judgment in constraint_judgments) else 0 for constraint_judgments in zip(*all_constraint_judgments) - ] \ No newline at end of file + ] From 9e5ed126f509a5ae4d275e6803ce03d7e39eec71 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Wed, 2 Oct 2024 23:33:36 -0400 Subject: [PATCH 04/28] update doc --- docs/source/judges.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/judges.mdx b/docs/source/judges.mdx index eee51c488f..d7fdf194a3 100644 --- a/docs/source/judges.mdx +++ b/docs/source/judges.mdx @@ -63,6 +63,7 @@ judge.judge( [[autodoc]] BasePairwiseJudge ## MixtureOfConstraintJudges + [[autodoc]] MixtureOfConstraintJudges ## RandomRankJudge From 3406e5389010d5acf074f6c5c5182b7cdf66d4c8 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Wed, 2 Oct 2024 23:34:14 -0400 Subject: [PATCH 05/28] formatting --- tests/test_judges.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index 1fafb98e06..cb9a851a3d 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -14,7 +14,14 @@ import unittest -from trl import HfPairwiseJudge, PairRMJudge, RandomPairwiseJudge, RandomRankJudge, RandomConstraintJudge, MixtureOfConstraintJudges +from trl import ( + HfPairwiseJudge, + MixtureOfConstraintJudges, + PairRMJudge, + RandomConstraintJudge, + RandomPairwiseJudge, + RandomRankJudge, +) class TestJudges(unittest.TestCase): @@ -22,18 +29,28 @@ def _get_prompts_and_completions(self): prompts = ["The capital of France is", "The biggest planet in the solar system is"] completions = [["Paris", "Marseille"], ["Saturn", "Jupiter"]] return prompts, completions - + def test_mixture_of_constraint_judge(self): moj = MixtureOfConstraintJudges(judges=[RandomConstraintJudge(), RandomConstraintJudge()]) - prompts = ["The capital of France is", "The capital of France is", "The biggest planet in the solar system is", "The biggest planet in the solar system is"] + prompts = [ + "The capital of France is", + "The capital of France is", + "The biggest planet in the solar system is", + "The biggest planet in the solar system is", + ] completions = ["Paris", "Marseille", "Saturn", "Jupiter"] judgements = moj.judge(prompts=prompts, completions=completions) self.assertEqual(len(judgements), 4) self.assertTrue(all(judgement in {0, 1, -1} for judgement in judgements)) - + def test_random_constraint_judge(self): judge = RandomConstraintJudge() - prompts = ["The capital of France is", "The capital of France is", "The biggest planet in the solar system is", "The biggest planet in the solar system is"] + prompts = [ + "The capital of France is", + "The capital of France is", + "The biggest planet in the solar system is", + "The biggest planet in the solar system is", + ] completions = ["Paris", "Marseille", "Saturn", "Jupiter"] judgements = judge.judge(prompts=prompts, completions=completions) self.assertEqual(len(judgements), 4) From 568d2b93c9fff5e8b93c15e1947338f7050a432e Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Wed, 2 Oct 2024 23:41:57 -0400 Subject: [PATCH 06/28] fix small typo in doc --- trl/trainer/judges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 8e1eb6d82d..bb2283682d 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -136,7 +136,7 @@ def judge(self, prompts: List[str], completions: List[List[str]], shuffle_order: class BaseConstraintJudge(BaseJudge): """ - Base class for pairwise judges. + Base class for constraint judges. """ @abstractmethod @@ -167,7 +167,7 @@ def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool class RandomConstraintJudge(BaseConstraintJudge): """ - Random binary judge, for testing purposes. + Random constraint judge, for testing purposes. """ def judge(self, prompts, completions, shuffle_order=True): From 466292e0a4e91140b3073aca8207ae2d0cc56c11 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Thu, 3 Oct 2024 17:14:58 -0400 Subject: [PATCH 07/28] fix randomcontraintjudge --- trl/trainer/__init__.py | 6 +++--- trl/trainer/judges.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index 2c323a397b..63d882c0d8 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -35,11 +35,11 @@ "iterative_sft_trainer": ["IterativeSFTTrainer"], "judges": [ "BaseJudge", - "BinaryConstraintJudge", + "BaseConstraintJudge", "BasePairwiseJudge", "BaseRankJudge", "HfPairwiseJudge", - "MixtureOfConstraintJudges" + "MixtureOfConstraintJudges", "OpenAIPairwiseJudge", "PairRMJudge", "RandomConstraintJudge", @@ -102,7 +102,7 @@ from .iterative_sft_trainer import IterativeSFTTrainer from .judges import ( BaseJudge, - BinaryConstraintJudge, + BaseConstraintJudge, BasePairwiseJudge, BaseRankJudge, HfPairwiseJudge, diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index bb2283682d..1e8eaa299e 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -171,7 +171,7 @@ class RandomConstraintJudge(BaseConstraintJudge): """ def judge(self, prompts, completions, shuffle_order=True): - return [random.choice([0, 1]) for _ in len(prompts)] + return [random.choice([0, 1]) for _ in range(len(prompts))] class RandomRankJudge(BaseRankJudge): From 3f0b8b0b8ff17b8444e133be0be15d8ef181cccb Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ LATOUCHE <66413927+gaetanlop@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:54:38 -0400 Subject: [PATCH 08/28] replace arxiv by hf papers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com> --- trl/trainer/judges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 1e8eaa299e..d008f64d10 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -351,7 +351,7 @@ class MixtureOfConstraintJudges(BaseConstraintJudge): This class returns 0 ("violated") if it fails on any of the constraint judges (ie a judge returns 0 or -1) and returns 1 ("satisfied") otherwise. - It is an implementation of the Mixture of Judges as described in the CGPO paper: https://arxiv.org/pdf/2409.20370 + It is an implementation of the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370) Args: judges (List[BaseConstraintJudge]): A list of BaseConstraintJudge. From 8995ab4dfa97faf568d2a42b7969d013248941e9 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ LATOUCHE <66413927+gaetanlop@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:55:00 -0400 Subject: [PATCH 09/28] formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com> --- trl/trainer/judges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index d008f64d10..1411ee4fd8 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -354,7 +354,7 @@ class MixtureOfConstraintJudges(BaseConstraintJudge): It is an implementation of the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370) Args: - judges (List[BaseConstraintJudge]): A list of BaseConstraintJudge. + judges (`List[BaseConstraintJudge]`): A list of [`BaseConstraintJudge`]. """ def __init__(self, judges: List[BaseConstraintJudge]): From ef1feb01a6b8ce8afba6beaeb51291140dba1f05 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ Date: Fri, 4 Oct 2024 17:12:17 -0400 Subject: [PATCH 10/28] fix naming in __init__ --- trl/__init__.py | 4 ++-- trl/trainer/judges.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trl/__init__.py b/trl/__init__.py index f62be5f479..0c6d2814ff 100644 --- a/trl/__init__.py +++ b/trl/__init__.py @@ -51,7 +51,7 @@ "AlignPropConfig", "AlignPropTrainer", "BaseJudge", - "BinaryConstraintJudge", + "BaseConstraintJudge", "BasePairwiseJudge", "BaseRankJudge", "BCOConfig", @@ -149,7 +149,7 @@ AlignPropConfig, AlignPropTrainer, BaseJudge, - BinaryConstraintJudge, + BaseConstraintJudge, BasePairwiseJudge, BaseRankJudge, BCOConfig, diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 1411ee4fd8..4f03ed9051 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -360,7 +360,7 @@ class MixtureOfConstraintJudges(BaseConstraintJudge): def __init__(self, judges: List[BaseConstraintJudge]): self.judges = judges - def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[int]: + def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[bool]: all_constraint_judgments = [judge.judge(prompts, completions, shuffle_order) for judge in self.judges] return [ From 3da4a06d00e0e2752be0a98f09d00bd186b54fe5 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Fri, 4 Oct 2024 18:33:38 -0400 Subject: [PATCH 11/28] run precommi --- trl/__init__.py | 2 +- trl/trainer/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/trl/__init__.py b/trl/__init__.py index 0c6d2814ff..e007fac3be 100644 --- a/trl/__init__.py +++ b/trl/__init__.py @@ -148,8 +148,8 @@ from .trainer import ( AlignPropConfig, AlignPropTrainer, - BaseJudge, BaseConstraintJudge, + BaseJudge, BasePairwiseJudge, BaseRankJudge, BCOConfig, diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index 63d882c0d8..07ef179202 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -101,8 +101,8 @@ from .gkd_trainer import GKDTrainer from .iterative_sft_trainer import IterativeSFTTrainer from .judges import ( - BaseJudge, BaseConstraintJudge, + BaseJudge, BasePairwiseJudge, BaseRankJudge, HfPairwiseJudge, From 765768b0ffb65ee3f88f93ad1055edd4771cb0c0 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Sun, 6 Oct 2024 20:51:56 -0400 Subject: [PATCH 12/28] adding gold answers to judges --- trl/trainer/judges.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 4f03ed9051..002b8034ba 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -140,7 +140,9 @@ class BaseConstraintJudge(BaseJudge): """ @abstractmethod - def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[int]: + def judge( + self, prompts: List[str], completions: List[str], gold_answers: List[str] = None, shuffle_order: bool = True + ) -> List[int]: """ Judge the completion for a given prompt. Used to assess if a completion satisfies a constraint. @@ -150,6 +152,7 @@ def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool Args: prompts (`List[str]`): List of prompts. completions (`List[str]`): List of completions. + gold_answers (`List[str]`): List of gold answers if it exists. shuffle_order (`bool`): Whether to shuffle the order of the completions to avoid positional bias. Returns: @@ -170,7 +173,7 @@ class RandomConstraintJudge(BaseConstraintJudge): Random constraint judge, for testing purposes. """ - def judge(self, prompts, completions, shuffle_order=True): + def judge(self, prompts, completions, gold_answers=None, shuffle_order=True): return [random.choice([0, 1]) for _ in range(len(prompts))] @@ -360,8 +363,12 @@ class MixtureOfConstraintJudges(BaseConstraintJudge): def __init__(self, judges: List[BaseConstraintJudge]): self.judges = judges - def judge(self, prompts: List[str], completions: List[str], shuffle_order: bool = True) -> List[bool]: - all_constraint_judgments = [judge.judge(prompts, completions, shuffle_order) for judge in self.judges] + def judge( + self, prompts: List[str], completions: List[str], gold_answers: List[str] = None, shuffle_order: bool = True + ) -> List[bool]: + all_constraint_judgments = [ + judge.judge(prompts, completions, gold_answers, shuffle_order) for judge in self.judges + ] return [ 1 if all(constraint_judgment == 1 for constraint_judgment in constraint_judgments) else 0 From 8aaaaa1a064e73c16b69f2d976caadfa4f605e71 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Sun, 6 Oct 2024 21:50:11 -0400 Subject: [PATCH 13/28] cgpo llm judges --- tests/test_judges.py | 35 +++++++++-- trl/__init__.py | 4 ++ trl/trainer/__init__.py | 2 + trl/trainer/judges.py | 130 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 4 deletions(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index cb9a851a3d..dbec46902c 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -15,21 +15,30 @@ import unittest from trl import ( + FactualityConstraintJudge, HfPairwiseJudge, MixtureOfConstraintJudges, PairRMJudge, RandomConstraintJudge, RandomPairwiseJudge, RandomRankJudge, + SafetyConstraintJudge, ) class TestJudges(unittest.TestCase): - def _get_prompts_and_completions(self): + def _get_prompts_and_pairwise_completions(self): prompts = ["The capital of France is", "The biggest planet in the solar system is"] completions = [["Paris", "Marseille"], ["Saturn", "Jupiter"]] return prompts, completions + def _get_prompts_completion_and_gold_answer(self): + prompts = ["What's the capital of France?", "What's the color of the sky?"] + completions = ["Marseille", "blue"] + gold_answers = ["Paris", "The color of the sky is blue."] + + return prompts, completions, gold_answers + def test_mixture_of_constraint_judge(self): moj = MixtureOfConstraintJudges(judges=[RandomConstraintJudge(), RandomConstraintJudge()]) prompts = [ @@ -58,14 +67,14 @@ def test_random_constraint_judge(self): def test_random_pairwise_judge(self): judge = RandomPairwiseJudge() - prompts, completions = self._get_prompts_and_completions() + prompts, completions = self._get_prompts_and_pairwise_completions() ranks = judge.judge(prompts=prompts, completions=completions) self.assertEqual(len(ranks), 2) self.assertTrue(all(isinstance(rank, int) for rank in ranks)) def test_random_rank_judge(self): judge = RandomRankJudge() - prompts, completions = self._get_prompts_and_completions() + prompts, completions = self._get_prompts_and_pairwise_completions() ranks = judge.judge(prompts=prompts, completions=completions) self.assertEqual(len(ranks), 2) self.assertTrue(all(isinstance(rank, list) for rank in ranks)) @@ -74,12 +83,30 @@ def test_random_rank_judge(self): @unittest.skip("This test needs to be run manually since it requires a valid Hugging Face API key.") def test_hugging_face_judge(self): judge = HfPairwiseJudge() - prompts, completions = self._get_prompts_and_completions() + prompts, completions = self._get_prompts_and_pairwise_completions() ranks = judge.judge(prompts=prompts, completions=completions) self.assertEqual(len(ranks), 2) self.assertTrue(all(isinstance(rank, int) for rank in ranks)) self.assertEqual(ranks, [0, 1]) + @unittest.skip("This test needs to be run manually since it requires a valid Hugging Face API key.") + def test_factuality_judge(self): + judge = FactualityConstraintJudge() + prompts, completions, gold_answers = self._get_prompts_completion_and_gold_answer() + judgements = judge.judge(prompts=prompts, completions=completions, gold_answers=gold_answers) + self.assertEqual(len(judgements), 2) + self.assertTrue(all(isinstance(judgement, int) for judgement in judgements)) + self.assertEqual(judgements, [0, 1]) + + @unittest.skip("This test needs to be run manually since it requires a valid Hugging Face API key.") + def test_safety_judge(self): + judge = SafetyConstraintJudge(safety_guidelines="S7: Intellectual Property") + prompts, completions, _ = self._get_prompts_completion_and_gold_answer() + judgements = judge.judge(prompts=prompts, completions=completions) + self.assertEqual(len(judgements), 2) + self.assertTrue(all(isinstance(judgement, int) for judgement in judgements)) + self.assertIn(judgements, [1, 1]) + def test_pair_rm_judge(self): judge = PairRMJudge() prompts, completions = self._get_prompts_and_completions() diff --git a/trl/__init__.py b/trl/__init__.py index e007fac3be..67276f91de 100644 --- a/trl/__init__.py +++ b/trl/__init__.py @@ -61,6 +61,7 @@ "DataCollatorForCompletionOnlyLM", "DPOConfig", "DPOTrainer", + "FactualityConstraintJudge", "FDivergenceConstants", "FDivergenceType", "GKDConfig", @@ -91,6 +92,7 @@ "RewardTrainer", "RLOOConfig", "RLOOTrainer", + "SafetyConstraintJudge", "SFTConfig", "SFTTrainer", "WinRateCallback", @@ -159,6 +161,7 @@ DataCollatorForCompletionOnlyLM, DPOConfig, DPOTrainer, + FactualityConstraintJudge, FDivergenceConstants, FDivergenceType, GKDConfig, @@ -189,6 +192,7 @@ RewardTrainer, RLOOConfig, RLOOTrainer, + SafetyConstraintJudge, SFTConfig, SFTTrainer, WinRateCallback, diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index 07ef179202..c98d0f47bd 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -105,6 +105,7 @@ BaseJudge, BasePairwiseJudge, BaseRankJudge, + FactualityConstraintJudge, HfPairwiseJudge, MixtureOfConstraintJudges, OpenAIPairwiseJudge, @@ -112,6 +113,7 @@ RandomConstraintJudge, RandomPairwiseJudge, RandomRankJudge, + SafetyConstraintJudge, ) from .kto_config import KTOConfig from .kto_trainer import KTOTrainer diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 002b8034ba..2715858543 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -62,6 +62,22 @@ ''' +DEFAULT_FACTUALITY_SYSTEM_PROMPT = """Here is a question and a ground truth answer: +Question: {prompt} +Ground truth answer: {gold_answer} +Now, you are given an answer from the AI: +AI: {response} +Does the AI’s answer contains the factual information from the above ground truth answer? Answer in TRUE or FALSE.""" + +DEFAULT_SAFETY_SYSTEM_PROMPT = """Task: Check if there is unsafe content in 'Agent' messages in conversations according our safety policy with the below categories. + {safety_guidelines} + {conversation} +Provide your safety assessment for ONLY THE LAST turn message in the above conversation +- First line must read 'safe' or 'unsafe'. +- If unsafe, a second line must include a comma-separated list of violated +categories.""" + + class BaseJudge(ABC): """ Base class for judges. The subclasses of this class should implement the `judge` method. @@ -227,6 +243,120 @@ def judge(self, prompts: List[str], completions: List[List[str]], shuffle_order: return ranks[:, 0].tolist() +class FactualityConstraintJudge(BaseConstraintJudge): + """ + Factuality judge based on the Hugging Face API with chat completion. + + This judge is relevant for assessing the factuality of a completion with respect to a gold answer and a prompt. + + Args: + model (`str`, *optional*): The model to use for the judge. Defaults to "meta-llama/Meta-Llama-Guard-2-8B". + token (`str`, *optional*): The Hugging Face API token to use for the InferenceClient. + system_prompt (`str`, *optional*): The system prompt to be used for the judge. If not provided, a default prompt is used. + Note that the system prompt should contain the following placeholders: `{prompt}`, `{response}` and `{gold_answer}` + Also, the inference is called with `max_tokens=1`, consequently the system prompt should ask for a single token response between `TRUE` and `FALSE`. + """ + + def __init__( + self, + model="meta-llama/Meta-Llama-3-70B-Instruct", + token: Optional[str] = None, + system_prompt: Optional[str] = None, + ): + self.client = InferenceClient(model=model, token=token) + self.system_prompt = system_prompt or DEFAULT_FACTUALITY_SYSTEM_PROMPT + + def judge(self, prompts, completions, gold_answers=None, shuffle_order=True): + if gold_answers is None: + raise ValueError( + "The FactualityConstraintJudge expects `gold_answers` to judge the factuality of the `completions`." + ) + + if not (len(prompts) == len(completions) == len(gold_answers)): + raise ValueError( + "The FactualityConstraintJudge expects the same number of `prompts`, `completions` and `gold_answers`." + ) + + # Define a function to get the rank for a single prompt, will be called concurrently + def is_factual(prompt, completion, gold_answer): + content = self.system_prompt.format(prompt=prompt, response=completion, gold_answer=gold_answer) + completion = self.client.chat_completion(messages=[{"role": "user", "content": content}], max_tokens=1) + response = completion.choices[0].message.content + response = response.lower() + if response == "true": + return 1 + elif response == "false": + return 0 + else: + logging.debug( + f"Invalid response from the factuality judge model: Expected 'TRUE' or 'FALSE' but got '{response}'. Returning -1." + ) + return -1 + + # Call the completions concurrently + with concurrent.futures.ThreadPoolExecutor() as executor: + factuality = list(executor.map(is_factual, prompts, completions, gold_answers)) + + return factuality + + +class SafetyConstraintJudge(BaseConstraintJudge): + """ + Safety judge based on the Hugging Face API with chat completion. + + This judge is relevant for assessing the safety of a completion. + + Args: + model (`str`, *optional*): The model to use for the judge. Defaults to "meta-llama/Meta-Llama-Guard-2-8B". + token (`str`, *optional*): The Hugging Face API token to use for the InferenceClient. + system_prompt (`str`, *optional*): The system prompt to be used for the judge. If not provided, a default prompt is used. + Note that the system prompt should contain the following placeholders: `{prompt}` and `{conversation}` + Also, the inference is called with `max_tokens=1`, consequently the system prompt should ask for a single token response between `safe` and `unsafe`. + safety_guidelines (`str`, *optional*): The safety guidelines as defined in the Llama Guard paper (https://huggingface.co/papers/2312.06674) + """ + + def __init__( + self, + model="meta-llama/Meta-Llama-Guard-2-8B", + token: Optional[str] = None, + system_prompt: Optional[str] = None, + safety_guidelines: Optional[str] = None, + ): + self.client = InferenceClient(model=model, token=token) + self.system_prompt = system_prompt or DEFAULT_SAFETY_SYSTEM_PROMPT + self.safety_guidelines = safety_guidelines + + if self.safety_guidelines is None: + raise ValueError( + "The safety guidelines cannot be None.", + "Please refer to `https://huggingface.co/meta-llama/Meta-Llama-Guard-2-8B` to see examples of safety labels.", + ) + + def judge(self, prompts, completions, gold_answers=None, shuffle_order=True): + if not (len(prompts) == len(completions) == len(gold_answers)): + raise ValueError("The SafetyConstraintJudge expects the same number of `prompts` and `completions`.") + + def is_safe(prompt, completion): + content = self.system_prompt.format( + safety_guidelines=self.safety_guidelines, conversation=prompt + completion + ) + completion = self.client.chat_completion(messages=[{"role": "user", "content": content}], max_tokens=1) + response = completion.choices[0].message.content + if response == "safe": + return 1 + elif response == "unsafe": + return 0 + else: + logging.debug(f"Invalid response from the safety judge model: '{response}'. Returning -1.") + return -1 + + # Call the completions concurrently + with concurrent.futures.ThreadPoolExecutor() as executor: + safety = list(executor.map(is_safe, prompts, completions)) + + return safety + + class HfPairwiseJudge(BasePairwiseJudge): """ Pairwise judge based on the Hugging Face API with chat completion. From cfc84eda1564b7a38c780375a1d9a40a39ddf5b9 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Sun, 6 Oct 2024 22:43:48 -0400 Subject: [PATCH 14/28] fix init --- tests/test_judges.py | 2 +- trl/trainer/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index dbec46902c..ec8ee446d4 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -109,7 +109,7 @@ def test_safety_judge(self): def test_pair_rm_judge(self): judge = PairRMJudge() - prompts, completions = self._get_prompts_and_completions() + prompts, completions = self._get_prompts_and_pairwise_completions() ranks = judge.judge(prompts=prompts, completions=completions) self.assertEqual(len(ranks), 2) self.assertTrue(all(isinstance(rank, int) for rank in ranks)) diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index c98d0f47bd..5457725431 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -38,6 +38,7 @@ "BaseConstraintJudge", "BasePairwiseJudge", "BaseRankJudge", + "FactualityConstraintJudge", "HfPairwiseJudge", "MixtureOfConstraintJudges", "OpenAIPairwiseJudge", @@ -45,6 +46,7 @@ "RandomConstraintJudge", "RandomPairwiseJudge", "RandomRankJudge", + "SafetyConstraintJudge", ], "kto_config": ["KTOConfig"], "kto_trainer": ["KTOTrainer"], From 689828539aaec43a44835cf5530ce5e3b37a9454 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Sun, 6 Oct 2024 23:51:21 -0400 Subject: [PATCH 15/28] output type --- trl/trainer/judges.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 2715858543..39b59c4288 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -266,7 +266,7 @@ def __init__( self.client = InferenceClient(model=model, token=token) self.system_prompt = system_prompt or DEFAULT_FACTUALITY_SYSTEM_PROMPT - def judge(self, prompts, completions, gold_answers=None, shuffle_order=True): + def judge(self, prompts, completions, gold_answers=None, shuffle_order=True) -> List[int]: if gold_answers is None: raise ValueError( "The FactualityConstraintJudge expects `gold_answers` to judge the factuality of the `completions`." @@ -332,7 +332,7 @@ def __init__( "Please refer to `https://huggingface.co/meta-llama/Meta-Llama-Guard-2-8B` to see examples of safety labels.", ) - def judge(self, prompts, completions, gold_answers=None, shuffle_order=True): + def judge(self, prompts, completions, gold_answers=None, shuffle_order=True) -> List[int]: if not (len(prompts) == len(completions) == len(gold_answers)): raise ValueError("The SafetyConstraintJudge expects the same number of `prompts` and `completions`.") @@ -501,6 +501,6 @@ def judge( ] return [ - 1 if all(constraint_judgment == 1 for constraint_judgment in constraint_judgments) else 0 + True if all(constraint_judgment == 1 for constraint_judgment in constraint_judgments) else False for constraint_judgments in zip(*all_constraint_judgments) ] From f5639a1d560d22f6b5c746bfd21cfd022891ef63 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Sun, 6 Oct 2024 23:53:46 -0400 Subject: [PATCH 16/28] adjust booleans in test --- tests/test_judges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index ec8ee446d4..f168d33f46 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -50,7 +50,7 @@ def test_mixture_of_constraint_judge(self): completions = ["Paris", "Marseille", "Saturn", "Jupiter"] judgements = moj.judge(prompts=prompts, completions=completions) self.assertEqual(len(judgements), 4) - self.assertTrue(all(judgement in {0, 1, -1} for judgement in judgements)) + self.assertTrue(all(judgement in {True, False} for judgement in judgements)) def test_random_constraint_judge(self): judge = RandomConstraintJudge() From 289b855afa2545fb0217d5a0b4e88b6871d6f68c Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Sun, 6 Oct 2024 23:56:21 -0400 Subject: [PATCH 17/28] adapt moj doc --- trl/trainer/judges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 39b59c4288..b4fb59c91a 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -482,7 +482,7 @@ class MixtureOfConstraintJudges(BaseConstraintJudge): """ Unify the decision of multiple BaseConstraintJudge. - This class returns 0 ("violated") if it fails on any of the constraint judges (ie a judge returns 0 or -1) and returns 1 ("satisfied") otherwise. + This class returns False if it fails on any of the constraint judges (ie a judge returns 0 or -1) and returns True otherwise. It is an implementation of the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370) From dedc8590b8b9a3ee245c70ca7daf6d5f41ceede2 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Thu, 10 Oct 2024 21:43:31 -0400 Subject: [PATCH 18/28] renaming and removing factuality and safety judges --- docs/source/judges.mdx | 12 ++-- tests/test_judges.py | 28 ++------ trl/__init__.py | 16 ++--- trl/trainer/__init__.py | 14 ++-- trl/trainer/judges.py | 144 ++-------------------------------------- 5 files changed, 30 insertions(+), 184 deletions(-) diff --git a/docs/source/judges.mdx b/docs/source/judges.mdx index d7fdf194a3..aee881804c 100644 --- a/docs/source/judges.mdx +++ b/docs/source/judges.mdx @@ -46,13 +46,17 @@ judge.judge( ) # Outputs: [0, 1] ``` +## AllTrueJudge + +[[autodoc]] AllTrueJudge + ## BaseJudge [[autodoc]] BaseJudge -## BaseConstraintJudge +## BaseBinaryJudge -[[autodoc]] BaseConstraintJudge +[[autodoc]] BaseBinaryJudge ## BaseRankJudge @@ -62,9 +66,9 @@ judge.judge( [[autodoc]] BasePairwiseJudge -## MixtureOfConstraintJudges +## RandomBinaryJudge -[[autodoc]] MixtureOfConstraintJudges +[[autodoc]] RandomBinaryJudge ## RandomRankJudge diff --git a/tests/test_judges.py b/tests/test_judges.py index f168d33f46..ceb206bc56 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -15,14 +15,12 @@ import unittest from trl import ( - FactualityConstraintJudge, + AllTrueJudge, HfPairwiseJudge, - MixtureOfConstraintJudges, PairRMJudge, - RandomConstraintJudge, + RandomBinaryJudge, RandomPairwiseJudge, RandomRankJudge, - SafetyConstraintJudge, ) @@ -40,7 +38,7 @@ def _get_prompts_completion_and_gold_answer(self): return prompts, completions, gold_answers def test_mixture_of_constraint_judge(self): - moj = MixtureOfConstraintJudges(judges=[RandomConstraintJudge(), RandomConstraintJudge()]) + moj = AllTrueJudge(judges=[RandomBinaryJudge(), RandomBinaryJudge()]) prompts = [ "The capital of France is", "The capital of France is", @@ -53,7 +51,7 @@ def test_mixture_of_constraint_judge(self): self.assertTrue(all(judgement in {True, False} for judgement in judgements)) def test_random_constraint_judge(self): - judge = RandomConstraintJudge() + judge = RandomBinaryJudge() prompts = [ "The capital of France is", "The capital of France is", @@ -89,24 +87,6 @@ def test_hugging_face_judge(self): self.assertTrue(all(isinstance(rank, int) for rank in ranks)) self.assertEqual(ranks, [0, 1]) - @unittest.skip("This test needs to be run manually since it requires a valid Hugging Face API key.") - def test_factuality_judge(self): - judge = FactualityConstraintJudge() - prompts, completions, gold_answers = self._get_prompts_completion_and_gold_answer() - judgements = judge.judge(prompts=prompts, completions=completions, gold_answers=gold_answers) - self.assertEqual(len(judgements), 2) - self.assertTrue(all(isinstance(judgement, int) for judgement in judgements)) - self.assertEqual(judgements, [0, 1]) - - @unittest.skip("This test needs to be run manually since it requires a valid Hugging Face API key.") - def test_safety_judge(self): - judge = SafetyConstraintJudge(safety_guidelines="S7: Intellectual Property") - prompts, completions, _ = self._get_prompts_completion_and_gold_answer() - judgements = judge.judge(prompts=prompts, completions=completions) - self.assertEqual(len(judgements), 2) - self.assertTrue(all(isinstance(judgement, int) for judgement in judgements)) - self.assertIn(judgements, [1, 1]) - def test_pair_rm_judge(self): judge = PairRMJudge() prompts, completions = self._get_prompts_and_pairwise_completions() diff --git a/trl/__init__.py b/trl/__init__.py index 67276f91de..4cb8629440 100644 --- a/trl/__init__.py +++ b/trl/__init__.py @@ -50,8 +50,9 @@ "trainer": [ "AlignPropConfig", "AlignPropTrainer", + "AllTrueJudge", "BaseJudge", - "BaseConstraintJudge", + "BaseBinaryJudge", "BasePairwiseJudge", "BaseRankJudge", "BCOConfig", @@ -61,7 +62,6 @@ "DataCollatorForCompletionOnlyLM", "DPOConfig", "DPOTrainer", - "FactualityConstraintJudge", "FDivergenceConstants", "FDivergenceType", "GKDConfig", @@ -71,7 +71,6 @@ "KTOConfig", "KTOTrainer", "LogCompletionsCallback", - "MixtureOfConstraintJudges", "ModelConfig", "NashMDConfig", "NashMDTrainer", @@ -85,14 +84,13 @@ "PPOTrainer", "PPOv2Config", "PPOv2Trainer", - "RandomConstraintJudge", + "RandomBinaryJudge", "RandomPairwiseJudge", "RandomRankJudge", "RewardConfig", "RewardTrainer", "RLOOConfig", "RLOOTrainer", - "SafetyConstraintJudge", "SFTConfig", "SFTTrainer", "WinRateCallback", @@ -150,7 +148,8 @@ from .trainer import ( AlignPropConfig, AlignPropTrainer, - BaseConstraintJudge, + AllTrueJudge, + BaseBinaryJudge, BaseJudge, BasePairwiseJudge, BaseRankJudge, @@ -161,7 +160,6 @@ DataCollatorForCompletionOnlyLM, DPOConfig, DPOTrainer, - FactualityConstraintJudge, FDivergenceConstants, FDivergenceType, GKDConfig, @@ -171,7 +169,6 @@ KTOConfig, KTOTrainer, LogCompletionsCallback, - MixtureOfConstraintJudges, ModelConfig, NashMDConfig, NashMDTrainer, @@ -185,14 +182,13 @@ PPOTrainer, PPOv2Config, PPOv2Trainer, - RandomConstraintJudge, + RandomBinaryJudge, RandomPairwiseJudge, RandomRankJudge, RewardConfig, RewardTrainer, RLOOConfig, RLOOTrainer, - SafetyConstraintJudge, SFTConfig, SFTTrainer, WinRateCallback, diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index 5457725431..825cec7592 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -34,19 +34,17 @@ "gkd_trainer": ["GKDTrainer"], "iterative_sft_trainer": ["IterativeSFTTrainer"], "judges": [ + "AllTrueJudge", "BaseJudge", "BaseConstraintJudge", "BasePairwiseJudge", "BaseRankJudge", - "FactualityConstraintJudge", "HfPairwiseJudge", - "MixtureOfConstraintJudges", "OpenAIPairwiseJudge", "PairRMJudge", - "RandomConstraintJudge", + "RandomBinaryJudge", "RandomPairwiseJudge", "RandomRankJudge", - "SafetyConstraintJudge", ], "kto_config": ["KTOConfig"], "kto_trainer": ["KTOTrainer"], @@ -103,19 +101,17 @@ from .gkd_trainer import GKDTrainer from .iterative_sft_trainer import IterativeSFTTrainer from .judges import ( - BaseConstraintJudge, + AllTrueJudge, + BaseBinaryJudge, BaseJudge, BasePairwiseJudge, BaseRankJudge, - FactualityConstraintJudge, HfPairwiseJudge, - MixtureOfConstraintJudges, OpenAIPairwiseJudge, PairRMJudge, - RandomConstraintJudge, + RandomBinaryJudge, RandomPairwiseJudge, RandomRankJudge, - SafetyConstraintJudge, ) from .kto_config import KTOConfig from .kto_trainer import KTOTrainer diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index b4fb59c91a..414e58f7f6 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -62,22 +62,6 @@ ''' -DEFAULT_FACTUALITY_SYSTEM_PROMPT = """Here is a question and a ground truth answer: -Question: {prompt} -Ground truth answer: {gold_answer} -Now, you are given an answer from the AI: -AI: {response} -Does the AI’s answer contains the factual information from the above ground truth answer? Answer in TRUE or FALSE.""" - -DEFAULT_SAFETY_SYSTEM_PROMPT = """Task: Check if there is unsafe content in 'Agent' messages in conversations according our safety policy with the below categories. - {safety_guidelines} - {conversation} -Provide your safety assessment for ONLY THE LAST turn message in the above conversation -- First line must read 'safe' or 'unsafe'. -- If unsafe, a second line must include a comma-separated list of violated -categories.""" - - class BaseJudge(ABC): """ Base class for judges. The subclasses of this class should implement the `judge` method. @@ -150,7 +134,7 @@ def judge(self, prompts: List[str], completions: List[List[str]], shuffle_order: raise NotImplementedError("Judge subclasses must implement the `judge` method.") -class BaseConstraintJudge(BaseJudge): +class BaseBinaryJudge(BaseJudge): """ Base class for constraint judges. """ @@ -184,9 +168,9 @@ def judge( raise NotImplementedError("Judge subclasses must implement the `judge` method.") -class RandomConstraintJudge(BaseConstraintJudge): +class RandomBinaryJudge(BaseBinaryJudge): """ - Random constraint judge, for testing purposes. + Random binary judge, for testing purposes. """ def judge(self, prompts, completions, gold_answers=None, shuffle_order=True): @@ -243,120 +227,6 @@ def judge(self, prompts: List[str], completions: List[List[str]], shuffle_order: return ranks[:, 0].tolist() -class FactualityConstraintJudge(BaseConstraintJudge): - """ - Factuality judge based on the Hugging Face API with chat completion. - - This judge is relevant for assessing the factuality of a completion with respect to a gold answer and a prompt. - - Args: - model (`str`, *optional*): The model to use for the judge. Defaults to "meta-llama/Meta-Llama-Guard-2-8B". - token (`str`, *optional*): The Hugging Face API token to use for the InferenceClient. - system_prompt (`str`, *optional*): The system prompt to be used for the judge. If not provided, a default prompt is used. - Note that the system prompt should contain the following placeholders: `{prompt}`, `{response}` and `{gold_answer}` - Also, the inference is called with `max_tokens=1`, consequently the system prompt should ask for a single token response between `TRUE` and `FALSE`. - """ - - def __init__( - self, - model="meta-llama/Meta-Llama-3-70B-Instruct", - token: Optional[str] = None, - system_prompt: Optional[str] = None, - ): - self.client = InferenceClient(model=model, token=token) - self.system_prompt = system_prompt or DEFAULT_FACTUALITY_SYSTEM_PROMPT - - def judge(self, prompts, completions, gold_answers=None, shuffle_order=True) -> List[int]: - if gold_answers is None: - raise ValueError( - "The FactualityConstraintJudge expects `gold_answers` to judge the factuality of the `completions`." - ) - - if not (len(prompts) == len(completions) == len(gold_answers)): - raise ValueError( - "The FactualityConstraintJudge expects the same number of `prompts`, `completions` and `gold_answers`." - ) - - # Define a function to get the rank for a single prompt, will be called concurrently - def is_factual(prompt, completion, gold_answer): - content = self.system_prompt.format(prompt=prompt, response=completion, gold_answer=gold_answer) - completion = self.client.chat_completion(messages=[{"role": "user", "content": content}], max_tokens=1) - response = completion.choices[0].message.content - response = response.lower() - if response == "true": - return 1 - elif response == "false": - return 0 - else: - logging.debug( - f"Invalid response from the factuality judge model: Expected 'TRUE' or 'FALSE' but got '{response}'. Returning -1." - ) - return -1 - - # Call the completions concurrently - with concurrent.futures.ThreadPoolExecutor() as executor: - factuality = list(executor.map(is_factual, prompts, completions, gold_answers)) - - return factuality - - -class SafetyConstraintJudge(BaseConstraintJudge): - """ - Safety judge based on the Hugging Face API with chat completion. - - This judge is relevant for assessing the safety of a completion. - - Args: - model (`str`, *optional*): The model to use for the judge. Defaults to "meta-llama/Meta-Llama-Guard-2-8B". - token (`str`, *optional*): The Hugging Face API token to use for the InferenceClient. - system_prompt (`str`, *optional*): The system prompt to be used for the judge. If not provided, a default prompt is used. - Note that the system prompt should contain the following placeholders: `{prompt}` and `{conversation}` - Also, the inference is called with `max_tokens=1`, consequently the system prompt should ask for a single token response between `safe` and `unsafe`. - safety_guidelines (`str`, *optional*): The safety guidelines as defined in the Llama Guard paper (https://huggingface.co/papers/2312.06674) - """ - - def __init__( - self, - model="meta-llama/Meta-Llama-Guard-2-8B", - token: Optional[str] = None, - system_prompt: Optional[str] = None, - safety_guidelines: Optional[str] = None, - ): - self.client = InferenceClient(model=model, token=token) - self.system_prompt = system_prompt or DEFAULT_SAFETY_SYSTEM_PROMPT - self.safety_guidelines = safety_guidelines - - if self.safety_guidelines is None: - raise ValueError( - "The safety guidelines cannot be None.", - "Please refer to `https://huggingface.co/meta-llama/Meta-Llama-Guard-2-8B` to see examples of safety labels.", - ) - - def judge(self, prompts, completions, gold_answers=None, shuffle_order=True) -> List[int]: - if not (len(prompts) == len(completions) == len(gold_answers)): - raise ValueError("The SafetyConstraintJudge expects the same number of `prompts` and `completions`.") - - def is_safe(prompt, completion): - content = self.system_prompt.format( - safety_guidelines=self.safety_guidelines, conversation=prompt + completion - ) - completion = self.client.chat_completion(messages=[{"role": "user", "content": content}], max_tokens=1) - response = completion.choices[0].message.content - if response == "safe": - return 1 - elif response == "unsafe": - return 0 - else: - logging.debug(f"Invalid response from the safety judge model: '{response}'. Returning -1.") - return -1 - - # Call the completions concurrently - with concurrent.futures.ThreadPoolExecutor() as executor: - safety = list(executor.map(is_safe, prompts, completions)) - - return safety - - class HfPairwiseJudge(BasePairwiseJudge): """ Pairwise judge based on the Hugging Face API with chat completion. @@ -478,19 +348,19 @@ def get_rank(prompt, candidates): return ranks -class MixtureOfConstraintJudges(BaseConstraintJudge): +class AllTrueJudge(BaseBinaryJudge): """ - Unify the decision of multiple BaseConstraintJudge. + Unify the decision of multiple BaseBinaryJudge. This class returns False if it fails on any of the constraint judges (ie a judge returns 0 or -1) and returns True otherwise. It is an implementation of the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370) Args: - judges (`List[BaseConstraintJudge]`): A list of [`BaseConstraintJudge`]. + judges (`List[BaseBinaryJudge]`): A list of [`BaseBinaryJudge`]. """ - def __init__(self, judges: List[BaseConstraintJudge]): + def __init__(self, judges: List[BaseBinaryJudge]): self.judges = judges def judge( From ba0fffbfb50d03d8098e09def6d2c31c884e0135 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Thu, 10 Oct 2024 21:51:13 -0400 Subject: [PATCH 19/28] fix typo in import --- trl/trainer/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/__init__.py b/trl/trainer/__init__.py index 825cec7592..b6878ae11e 100644 --- a/trl/trainer/__init__.py +++ b/trl/trainer/__init__.py @@ -36,7 +36,7 @@ "judges": [ "AllTrueJudge", "BaseJudge", - "BaseConstraintJudge", + "BaseBinaryJudge", "BasePairwiseJudge", "BaseRankJudge", "HfPairwiseJudge", From 226de827da96237e4e22683c5d337473da1c5fd3 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Thu, 10 Oct 2024 21:54:05 -0400 Subject: [PATCH 20/28] fix small typo in naming --- trl/trainer/judges.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 414e58f7f6..ec1bed3a21 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -136,7 +136,7 @@ def judge(self, prompts: List[str], completions: List[List[str]], shuffle_order: class BaseBinaryJudge(BaseJudge): """ - Base class for constraint judges. + Base class for binary judges. """ @abstractmethod @@ -146,7 +146,7 @@ def judge( """ Judge the completion for a given prompt. Used to assess if a completion satisfies a constraint. - This base class should be used to implement constraint-based evaluation as done in section 4.1.4 of the CGPO paper (https://arxiv.org/pdf/2409.20370). + This base class should be used to implement binary evaluations as done in section 4.1.4 of the CGPO paper (https://arxiv.org/pdf/2409.20370). It is relevant for assessing whether or not a prompt completion pair satisfies a specific contraint. Args: @@ -352,7 +352,7 @@ class AllTrueJudge(BaseBinaryJudge): """ Unify the decision of multiple BaseBinaryJudge. - This class returns False if it fails on any of the constraint judges (ie a judge returns 0 or -1) and returns True otherwise. + This class returns False if it fails on any of the binary judges (ie a judge returns 0 or -1) and returns True otherwise. It is an implementation of the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370) @@ -366,11 +366,11 @@ def __init__(self, judges: List[BaseBinaryJudge]): def judge( self, prompts: List[str], completions: List[str], gold_answers: List[str] = None, shuffle_order: bool = True ) -> List[bool]: - all_constraint_judgments = [ + all_binary_judgments = [ judge.judge(prompts, completions, gold_answers, shuffle_order) for judge in self.judges ] return [ - True if all(constraint_judgment == 1 for constraint_judgment in constraint_judgments) else False - for constraint_judgments in zip(*all_constraint_judgments) + True if all(all_binary_judgment == 1 for all_binary_judgment in binary_judgments) else False + for binary_judgments in zip(*all_binary_judgments) ] From 567b798c2f01b0f6f6f7cc0b77db598b3c7eb10f Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Mon, 14 Oct 2024 18:17:29 -0400 Subject: [PATCH 21/28] formatting --- tests/test_judges.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index ff771f5512..3ade64cd76 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -14,7 +14,6 @@ import unittest - from trl import ( AllTrueJudge, HfPairwiseJudge, @@ -22,7 +21,7 @@ RandomBinaryJudge, RandomPairwiseJudge, RandomRankJudge, - is_llmblender_available + is_llmblender_available, ) From 559cd1bd6f38ffdbc981f3a52511b13225db2471 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ LATOUCHE <66413927+gaetanlop@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:51:31 -0400 Subject: [PATCH 22/28] Update trl/trainer/judges.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com> --- trl/trainer/judges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index fcef8a7c13..bab2fc7d57 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -142,7 +142,7 @@ class BaseBinaryJudge(BaseJudge): @abstractmethod def judge( - self, prompts: List[str], completions: List[str], gold_answers: List[str] = None, shuffle_order: bool = True + self, prompts: List[str], completions: List[str], gold_answers: Optional[List[str]] = None, shuffle_order: bool = True ) -> List[int]: """ Judge the completion for a given prompt. Used to assess if a completion satisfies a constraint. From 2c29ef5fd0a7e72f4eb2f8384fec8407be73caab Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Thu, 24 Oct 2024 20:34:34 -0400 Subject: [PATCH 23/28] update parameter name --- trl/trainer/judges.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index bab2fc7d57..f2311baf3f 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -142,7 +142,11 @@ class BaseBinaryJudge(BaseJudge): @abstractmethod def judge( - self, prompts: List[str], completions: List[str], gold_answers: Optional[List[str]] = None, shuffle_order: bool = True + self, + prompts: List[str], + completions: List[str], + gold_completions: Optional[List[str]] = None, + shuffle_order: bool = True, ) -> List[int]: """ Judge the completion for a given prompt. Used to assess if a completion satisfies a constraint. @@ -174,7 +178,7 @@ class RandomBinaryJudge(BaseBinaryJudge): Random binary judge, for testing purposes. """ - def judge(self, prompts, completions, gold_answers=None, shuffle_order=True): + def judge(self, prompts, completions, gold_completions=None, shuffle_order=True): return [random.choice([0, 1]) for _ in range(len(prompts))] @@ -422,10 +426,14 @@ def __init__(self, judges: List[BaseBinaryJudge]): self.judges = judges def judge( - self, prompts: List[str], completions: List[str], gold_answers: List[str] = None, shuffle_order: bool = True + self, + prompts: List[str], + completions: List[str], + gold_completions: Optional[List[str]] = None, + shuffle_order: bool = True, ) -> List[bool]: all_binary_judgments = [ - judge.judge(prompts, completions, gold_answers, shuffle_order) for judge in self.judges + judge.judge(prompts, completions, gold_completions, shuffle_order) for judge in self.judges ] return [ From bd1bed84fbc11e998627af2edefca8d0a46e99f0 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Thu, 24 Oct 2024 20:43:57 -0400 Subject: [PATCH 24/28] update tests --- tests/test_judges.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/tests/test_judges.py b/tests/test_judges.py index 3ade64cd76..9917aa1c7d 100644 --- a/tests/test_judges.py +++ b/tests/test_judges.py @@ -31,37 +31,23 @@ def _get_prompts_and_pairwise_completions(self): completions = [["Paris", "Marseille"], ["Saturn", "Jupiter"]] return prompts, completions - def _get_prompts_completion_and_gold_answer(self): + def _get_prompts_and_single_completions(self): prompts = ["What's the capital of France?", "What's the color of the sky?"] completions = ["Marseille", "blue"] - gold_answers = ["Paris", "The color of the sky is blue."] - - return prompts, completions, gold_answers + return prompts, completions - def test_mixture_of_constraint_judge(self): + def test_all_true_judge(self): moj = AllTrueJudge(judges=[RandomBinaryJudge(), RandomBinaryJudge()]) - prompts = [ - "The capital of France is", - "The capital of France is", - "The biggest planet in the solar system is", - "The biggest planet in the solar system is", - ] - completions = ["Paris", "Marseille", "Saturn", "Jupiter"] + prompts, completions = self._get_prompts_and_single_completions() judgements = moj.judge(prompts=prompts, completions=completions) - self.assertEqual(len(judgements), 4) + self.assertEqual(len(judgements), 2) self.assertTrue(all(judgement in {True, False} for judgement in judgements)) def test_random_constraint_judge(self): judge = RandomBinaryJudge() - prompts = [ - "The capital of France is", - "The capital of France is", - "The biggest planet in the solar system is", - "The biggest planet in the solar system is", - ] - completions = ["Paris", "Marseille", "Saturn", "Jupiter"] + prompts, completions = self._get_prompts_and_single_completions() judgements = judge.judge(prompts=prompts, completions=completions) - self.assertEqual(len(judgements), 4) + self.assertEqual(len(judgements), 2) self.assertTrue(all(judgement in {0, 1, -1} for judgement in judgements)) def test_random_pairwise_judge(self): @@ -100,7 +86,7 @@ def test_pair_rm_judge(self): @unittest.skipIf(not is_llmblender_available(), "llm-blender is not available") def test_pair_rm_judge_return_scores(self): judge = PairRMJudge() - prompts, completions = self._get_prompts_and_completions() + prompts, completions = self._get_prompts_and_pairwise_completions() probs = judge.judge(prompts=prompts, completions=completions, return_scores=True) self.assertEqual(len(probs), 2) self.assertTrue(all(isinstance(prob, float) for prob in probs)) From 21e3ccd3e6858f9dc32248daf831d2f55db5b57b Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Thu, 24 Oct 2024 20:47:56 -0400 Subject: [PATCH 25/28] update doc --- trl/trainer/judges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index f2311baf3f..7171154a4c 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -157,7 +157,7 @@ def judge( Args: prompts (`List[str]`): List of prompts. completions (`List[str]`): List of completions. - gold_answers (`List[str]`): List of gold answers if it exists. + gold_completions (`List[str]`, `optional`): List of gold completions if it exists. shuffle_order (`bool`): Whether to shuffle the order of the completions to avoid positional bias. Returns: From 9eca0f818254e84bfd4e4511e999445c252082f6 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ LATOUCHE <66413927+gaetanlop@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:04:07 -0400 Subject: [PATCH 26/28] Update trl/trainer/judges.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com> --- trl/trainer/judges.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 7171154a4c..34bc7dd059 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -151,7 +151,8 @@ def judge( """ Judge the completion for a given prompt. Used to assess if a completion satisfies a constraint. - This base class should be used to implement binary evaluations as done in section 4.1.4 of the CGPO paper (https://arxiv.org/pdf/2409.20370). + This base class should be used to implement binary evaluations as done in section 4.1.4 of the + [CGPO paper](https://huggingface.co/papers/2409.20370). It is relevant for assessing whether or not a prompt completion pair satisfies a specific contraint. Args: From d5b32f0466f5221486e0437ea639a28b5b519b85 Mon Sep 17 00:00:00 2001 From: Gaetan LOPEZ LATOUCHE <66413927+gaetanlop@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:05:21 -0400 Subject: [PATCH 27/28] Update doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com> --- trl/trainer/judges.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 34bc7dd059..0c5c244b07 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -412,16 +412,17 @@ def get_rank(prompt, candidates): class AllTrueJudge(BaseBinaryJudge): - """ - Unify the decision of multiple BaseBinaryJudge. + """ + Unify the decision of multiple [`BaseBinaryJudge`] instances. - This class returns False if it fails on any of the binary judges (ie a judge returns 0 or -1) and returns True otherwise. + Returns `1` only if all inner binary judges return `1`. If any judge returns `0`, it returns `0`. + If any judge returns `-1`, indicating a failure in its process, this judge will also return `-1`. - It is an implementation of the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370) + Implements the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370). - Args: - judges (`List[BaseBinaryJudge]`): A list of [`BaseBinaryJudge`]. - """ + Args: + judges (`List[BaseBinaryJudge]`): A list of [`BaseBinaryJudge`] instances whose decisions will be unified. + """ def __init__(self, judges: List[BaseBinaryJudge]): self.judges = judges From ac88c63fe3e695066dd9e5b9749189083f2fca71 Mon Sep 17 00:00:00 2001 From: gaetanlop Date: Mon, 28 Oct 2024 21:07:36 -0400 Subject: [PATCH 28/28] fix alltruejudge type --- trl/trainer/judges.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/trl/trainer/judges.py b/trl/trainer/judges.py index 0c5c244b07..bc4438576a 100644 --- a/trl/trainer/judges.py +++ b/trl/trainer/judges.py @@ -412,17 +412,17 @@ def get_rank(prompt, candidates): class AllTrueJudge(BaseBinaryJudge): - """ - Unify the decision of multiple [`BaseBinaryJudge`] instances. + """ + Unify the decision of multiple [`BaseBinaryJudge`] instances. - Returns `1` only if all inner binary judges return `1`. If any judge returns `0`, it returns `0`. - If any judge returns `-1`, indicating a failure in its process, this judge will also return `-1`. + Returns `1` only if all inner binary judges return `1`. If any judge returns `0`, it returns `0`. + If any judge returns `-1`, indicating a failure in its process, this judge will also return `-1`. - Implements the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370). + Implements the Mixture of Judges as described in the [CGPO paper](https://huggingface.co/papers/2409.20370). - Args: - judges (`List[BaseBinaryJudge]`): A list of [`BaseBinaryJudge`] instances whose decisions will be unified. - """ + Args: + judges (`List[BaseBinaryJudge]`): A list of [`BaseBinaryJudge`] instances whose decisions will be unified. + """ def __init__(self, judges: List[BaseBinaryJudge]): self.judges = judges @@ -433,7 +433,7 @@ def judge( completions: List[str], gold_completions: Optional[List[str]] = None, shuffle_order: bool = True, - ) -> List[bool]: + ) -> List[int]: all_binary_judgments = [ judge.judge(prompts, completions, gold_completions, shuffle_order) for judge in self.judges ]