From ab0de46cbd8a37527896d8f8e1556b828f1171e9 Mon Sep 17 00:00:00 2001 From: Amna Mubashar Date: Tue, 4 Feb 2025 14:52:03 +0100 Subject: [PATCH 1/5] Add a List joiner --- haystack/components/joiners/__init__.py | 3 +- haystack/components/joiners/list_joiner.py | 101 +++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 haystack/components/joiners/list_joiner.py diff --git a/haystack/components/joiners/__init__.py b/haystack/components/joiners/__init__.py index ea9082ba4d..a7f43b9e93 100644 --- a/haystack/components/joiners/__init__.py +++ b/haystack/components/joiners/__init__.py @@ -5,6 +5,7 @@ from .answer_joiner import AnswerJoiner from .branch import BranchJoiner from .document_joiner import DocumentJoiner +from .list_joiner import ListJoiner from .string_joiner import StringJoiner -__all__ = ["DocumentJoiner", "BranchJoiner", "AnswerJoiner", "StringJoiner"] +__all__ = ["DocumentJoiner", "BranchJoiner", "AnswerJoiner", "StringJoiner", "ListJoiner"] diff --git a/haystack/components/joiners/list_joiner.py b/haystack/components/joiners/list_joiner.py new file mode 100644 index 0000000000..072949ba61 --- /dev/null +++ b/haystack/components/joiners/list_joiner.py @@ -0,0 +1,101 @@ +from itertools import chain +from typing import Any, Dict, List, Type + +from haystack import component, default_from_dict, default_to_dict +from haystack.core.component.types import Variadic +from haystack.utils import deserialize_type, serialize_type + + +@component +class ListJoiner: + """ + A component that joins multiple lists into a single flat list. + + The ListJoiner receives multiple lists of the same type and concatenates them into a single flat list. + The output order respects the pipeline's execution sequence, with earlier inputs being added first. + + Usage example: + ```python + from haystack.components.builders import ChatPromptBuilder + from haystack.components.generators.chat import OpenAIChatGenerator + from haystack.dataclasses import ChatMessage + from haystack import Pipeline + from haystack.components.joiners import ListJoiner + from typing import List + + + user_message = [ChatMessage.from_user("Give a brief answer the following question: {{query}}")] + + feedback_prompt = \""" + You are given a question and an answer. + Your task is to provide a score and a brief feedback on the answer. + Question: {{query}} + Answer: {{response}} + \""" + feedback_message = [ChatMessage.from_system(feedback_prompt)] + + prompt_builder = ChatPromptBuilder(template=user_message) + feedback_prompt_builder = ChatPromptBuilder(template=feedback_message) + llm = OpenAIChatGenerator(model="gpt-4o-mini") + feedback_llm = OpenAIChatGenerator(model="gpt-4o-mini") + + pipe = Pipeline() + pipe.add_component("prompt_builder", prompt_builder) + pipe.add_component("llm", llm) + pipe.add_component("feedback_prompt_builder", feedback_prompt_builder) + pipe.add_component("feedback_llm", feedback_llm) + pipe.add_component("list_joiner", ListJoiner(List[ChatMessage])) + + pipe.connect("prompt_builder.prompt", "llm.messages") + pipe.connect("prompt_builder.prompt", "list_joiner") + pipe.connect("llm.replies", "list_joiner") + pipe.connect("llm.replies", "feedback_prompt_builder.response") + pipe.connect("feedback_prompt_builder.prompt", "feedback_llm.messages") + pipe.connect("feedback_llm.replies", "list_joiner") + + query = "What is nuclear physics?" + ans = pipe.run(data={"prompt_builder": {"template_variables":{"query": query}}, + "feedback_prompt_builder": {"template_variables":{"query": query}}}) + + print(ans["list_joiner"]["values"]) + ``` + """ + + def __init__(self, type_: Type): + """ + Creates a ListJoiner component. + + :param type_: The type of list that this joiner will handle (e.g., List[ChatMessage]). + All input lists must be of this type. + """ + self.type_ = type_ + component.set_output_types(self, values=type_) + + def to_dict(self) -> Dict[str, Any]: + """ + Serializes the component to a dictionary. + + :returns: Dictionary with serialized data. + """ + return default_to_dict(self, type_=serialize_type(self.type_)) + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ListJoiner": + """ + Deserializes the component from a dictionary. + + :param data: Dictionary to deserialize from. + :returns: Deserialized component. + """ + data["init_parameters"]["type_"] = deserialize_type(data["init_parameters"]["type_"]) + return default_from_dict(cls, data) + + def run(self, values: Variadic[Any]) -> Dict[str, Any]: + """ + Joins multiple lists into a single flat list. + + :param values:The list to be joined. + :returns: Dictionary with 'values' key containing the joined list. + """ + result = list(chain(*values)) + return {"values": result} From cef64f22f8ce676a35fe3cd4b58f518cb34077b2 Mon Sep 17 00:00:00 2001 From: Amna Mubashar Date: Tue, 4 Feb 2025 16:18:41 +0100 Subject: [PATCH 2/5] Add tests and release notes --- .../add-list-joiner-4f0ea84e195fa461.yaml | 4 ++ test/components/joiners/test_list_joiner.py | 71 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml create mode 100644 test/components/joiners/test_list_joiner.py diff --git a/releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml b/releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml new file mode 100644 index 0000000000..d270380292 --- /dev/null +++ b/releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Added component ListJoiner to join lists of values from different components to a single list. diff --git a/test/components/joiners/test_list_joiner.py b/test/components/joiners/test_list_joiner.py new file mode 100644 index 0000000000..667411c5ab --- /dev/null +++ b/test/components/joiners/test_list_joiner.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import List + +from haystack import Document +from haystack.dataclasses import ChatMessage +from haystack.dataclasses.answer import GeneratedAnswer +from haystack.components.joiners.list_joiner import ListJoiner + + +class TestListJoiner: + def test_init(self): + joiner = ListJoiner(List[ChatMessage]) + assert isinstance(joiner, ListJoiner) + assert joiner.type_ == List[ChatMessage] + + def test_to_dict(self): + joiner = ListJoiner(List[ChatMessage]) + data = joiner.to_dict() + assert data == { + "type": "haystack.components.joiners.list_joiner.ListJoiner", + "init_parameters": {"type_": "typing.List[haystack.dataclasses.chat_message.ChatMessage]"}, + } + + def test_from_dict(self): + data = { + "type": "haystack.components.joiners.list_joiner.ListJoiner", + "init_parameters": {"type_": "typing.List[haystack.dataclasses.chat_message.ChatMessage]"}, + } + list_joiner = ListJoiner.from_dict(data) + assert isinstance(list_joiner, ListJoiner) + assert list_joiner.type_ == List[ChatMessage] + + def test_empty_list(self): + joiner = ListJoiner(List[ChatMessage]) + result = joiner.run([]) + assert result == {"values": []} + + def test_list_of_empty_lists(self): + joiner = ListJoiner(List[ChatMessage]) + result = joiner.run([[], []]) + assert result == {"values": []} + + def test_single_list_of_chat_messages(self): + joiner = ListJoiner(List[ChatMessage]) + messages = [ChatMessage.from_user("Hello"), ChatMessage.from_assistant("Hi there")] + result = joiner.run([messages]) + assert result == {"values": messages} + + def test_multiple_lists_of_chat_messages(self): + joiner = ListJoiner(List[ChatMessage]) + messages1 = [ChatMessage.from_user("Hello")] + messages2 = [ChatMessage.from_assistant("Hi there")] + messages3 = [ChatMessage.from_system("System message")] + result = joiner.run([messages1, messages2, messages3]) + assert result == {"values": messages1 + messages2 + messages3} + + def test_list_of_generated_answers(self): + joiner = ListJoiner(List[GeneratedAnswer]) + answers1 = [GeneratedAnswer(query="q1", data="a1", meta={}, documents=[Document(content="d1")])] + answers2 = [GeneratedAnswer(query="q2", data="a2", meta={}, documents=[Document(content="d2")])] + result = joiner.run([answers1, answers2]) + assert result == {"values": answers1 + answers2} + + def test_mixed_empty_and_non_empty_lists(self): + joiner = ListJoiner(List[ChatMessage]) + messages = [ChatMessage.from_user("Hello")] + result = joiner.run([messages, [], messages]) + assert result == {"values": messages + messages} From 1b8df448958e97c172b2d7b9376e535468874230 Mon Sep 17 00:00:00 2001 From: Amna Mubashar Date: Tue, 4 Feb 2025 17:23:54 +0100 Subject: [PATCH 3/5] fix license --- haystack/components/joiners/list_joiner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/haystack/components/joiners/list_joiner.py b/haystack/components/joiners/list_joiner.py index 072949ba61..28ef32e5f5 100644 --- a/haystack/components/joiners/list_joiner.py +++ b/haystack/components/joiners/list_joiner.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + from itertools import chain from typing import Any, Dict, List, Type From 7079cd37a531e8f0ed46a81616d0f4863acce0ec Mon Sep 17 00:00:00 2001 From: Amna Mubashar Date: Tue, 4 Feb 2025 17:36:13 +0100 Subject: [PATCH 4/5] Fix linting error --- haystack/components/joiners/list_joiner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haystack/components/joiners/list_joiner.py b/haystack/components/joiners/list_joiner.py index 28ef32e5f5..a2919f1a96 100644 --- a/haystack/components/joiners/list_joiner.py +++ b/haystack/components/joiners/list_joiner.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from itertools import chain -from typing import Any, Dict, List, Type +from typing import Any, Dict, Type from haystack import component, default_from_dict, default_to_dict from haystack.core.component.types import Variadic From 43e313c5a9db56d51d363c6e9ea56633219cf460 Mon Sep 17 00:00:00 2001 From: Amna Mubashar Date: Wed, 5 Feb 2025 16:34:13 +0100 Subject: [PATCH 5/5] Changes based on PR comments --- haystack/components/joiners/list_joiner.py | 12 ++++++------ .../notes/add-list-joiner-4f0ea84e195fa461.yaml | 2 +- test/components/joiners/test_list_joiner.py | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/haystack/components/joiners/list_joiner.py b/haystack/components/joiners/list_joiner.py index a2919f1a96..af5b109e15 100644 --- a/haystack/components/joiners/list_joiner.py +++ b/haystack/components/joiners/list_joiner.py @@ -65,15 +65,15 @@ class ListJoiner: ``` """ - def __init__(self, type_: Type): + def __init__(self, list_type_: Type): """ Creates a ListJoiner component. - :param type_: The type of list that this joiner will handle (e.g., List[ChatMessage]). + :param list_type_: The type of list that this joiner will handle (e.g., List[ChatMessage]). All input lists must be of this type. """ - self.type_ = type_ - component.set_output_types(self, values=type_) + self.list_type_ = list_type_ + component.set_output_types(self, values=list_type_) def to_dict(self) -> Dict[str, Any]: """ @@ -81,7 +81,7 @@ def to_dict(self) -> Dict[str, Any]: :returns: Dictionary with serialized data. """ - return default_to_dict(self, type_=serialize_type(self.type_)) + return default_to_dict(self, list_type_=serialize_type(self.list_type_)) @classmethod def from_dict(cls, data: Dict[str, Any]) -> "ListJoiner": @@ -91,7 +91,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "ListJoiner": :param data: Dictionary to deserialize from. :returns: Deserialized component. """ - data["init_parameters"]["type_"] = deserialize_type(data["init_parameters"]["type_"]) + data["init_parameters"]["list_type_"] = deserialize_type(data["init_parameters"]["list_type_"]) return default_from_dict(cls, data) def run(self, values: Variadic[Any]) -> Dict[str, Any]: diff --git a/releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml b/releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml index d270380292..df960a7da0 100644 --- a/releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml +++ b/releasenotes/notes/add-list-joiner-4f0ea84e195fa461.yaml @@ -1,4 +1,4 @@ --- features: - | - Added component ListJoiner to join lists of values from different components to a single list. + Added a new component `ListJoiner` which joins lists of values from different components to a single list. diff --git a/test/components/joiners/test_list_joiner.py b/test/components/joiners/test_list_joiner.py index 667411c5ab..9ea9fa57e3 100644 --- a/test/components/joiners/test_list_joiner.py +++ b/test/components/joiners/test_list_joiner.py @@ -14,24 +14,24 @@ class TestListJoiner: def test_init(self): joiner = ListJoiner(List[ChatMessage]) assert isinstance(joiner, ListJoiner) - assert joiner.type_ == List[ChatMessage] + assert joiner.list_type_ == List[ChatMessage] def test_to_dict(self): joiner = ListJoiner(List[ChatMessage]) data = joiner.to_dict() assert data == { "type": "haystack.components.joiners.list_joiner.ListJoiner", - "init_parameters": {"type_": "typing.List[haystack.dataclasses.chat_message.ChatMessage]"}, + "init_parameters": {"list_type_": "typing.List[haystack.dataclasses.chat_message.ChatMessage]"}, } def test_from_dict(self): data = { "type": "haystack.components.joiners.list_joiner.ListJoiner", - "init_parameters": {"type_": "typing.List[haystack.dataclasses.chat_message.ChatMessage]"}, + "init_parameters": {"list_type_": "typing.List[haystack.dataclasses.chat_message.ChatMessage]"}, } list_joiner = ListJoiner.from_dict(data) assert isinstance(list_joiner, ListJoiner) - assert list_joiner.type_ == List[ChatMessage] + assert list_joiner.list_type_ == List[ChatMessage] def test_empty_list(self): joiner = ListJoiner(List[ChatMessage])