From c153d0ea5ad758ca9591a7334df192868ba39aee Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Fri, 6 Sep 2024 10:28:58 -0400 Subject: [PATCH 1/3] add standard tests --- .../tests/integration_tests/test_standard.py | 44 +++++++++++++++++++ libs/genai/tests/unit_tests/test_standard.py | 26 +++++++++++ 2 files changed, 70 insertions(+) create mode 100644 libs/genai/tests/integration_tests/test_standard.py create mode 100644 libs/genai/tests/unit_tests/test_standard.py diff --git a/libs/genai/tests/integration_tests/test_standard.py b/libs/genai/tests/integration_tests/test_standard.py new file mode 100644 index 00000000..a0a9c583 --- /dev/null +++ b/libs/genai/tests/integration_tests/test_standard.py @@ -0,0 +1,44 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +from langchain_core.language_models import BaseChatModel +from langchain_core.rate_limiters import InMemoryRateLimiter +from langchain_standard_tests.integration_tests import ChatModelIntegrationTests +import pytest + +from langchain_google_genai import ChatGoogleGenerativeAI + +rate_limiter = InMemoryRateLimiter(requests_per_second=0.5) + + +class TestGeminiAIStandard(ChatModelIntegrationTests): + @property + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatGoogleGenerativeAI + + @property + def chat_model_params(self) -> dict: + return { + "model": "models/gemini-1.0-pro-001", + "rate_limiter": rate_limiter, + } + + @pytest.mark.xfail(reason="Gemini 1.0 doesn't support tool_choice='any'") + def test_structured_few_shot_examples(self, model: BaseChatModel) -> None: + super().test_structured_few_shot_examples(model) + + @pytest.mark.xfail(reason="with_structured_output with JSON schema not supported.") + def test_structured_output(self, model: BaseChatModel) -> None: + super().test_structured_output(model) + + @pytest.mark.xfail(reason="with_structured_output with JSON schema not supported.") + def test_structured_output_pydantic_2_v1(self, model: BaseChatModel) -> None: + super().test_structured_output_pydantic_2_v1(model) + + @pytest.mark.xfail(reason="Not yet supported") + def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None: + super().test_tool_message_histories_list_content(model) + + +# TODO: increase quota on gemini-1.5-pro-001 and test as well diff --git a/libs/genai/tests/unit_tests/test_standard.py b/libs/genai/tests/unit_tests/test_standard.py new file mode 100644 index 00000000..fd8b79cc --- /dev/null +++ b/libs/genai/tests/unit_tests/test_standard.py @@ -0,0 +1,26 @@ +from typing import Type + +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.unit_tests import ChatModelUnitTests + +from langchain_google_genai import ChatGoogleGenerativeAI + + +class TestGeminiAIStandard(ChatModelUnitTests): + @property + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatGoogleGenerativeAI + + @property + def chat_model_params(self) -> dict: + return {"model": "models/gemini-1.0-pro-001"} + + +class TestGemini_15_AIStandard(ChatModelUnitTests): + @property + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatGoogleGenerativeAI + + @property + def chat_model_params(self) -> dict: + return {"model": "models/gemini-1.5-pro-001"} From 6689048ca31d26fe1536f2ec442b40e88dfcd8ba Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Fri, 6 Sep 2024 10:32:45 -0400 Subject: [PATCH 2/3] format --- libs/genai/tests/integration_tests/test_standard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/genai/tests/integration_tests/test_standard.py b/libs/genai/tests/integration_tests/test_standard.py index a0a9c583..4ce86b04 100644 --- a/libs/genai/tests/integration_tests/test_standard.py +++ b/libs/genai/tests/integration_tests/test_standard.py @@ -2,10 +2,10 @@ from typing import Type +import pytest from langchain_core.language_models import BaseChatModel from langchain_core.rate_limiters import InMemoryRateLimiter from langchain_standard_tests.integration_tests import ChatModelIntegrationTests -import pytest from langchain_google_genai import ChatGoogleGenerativeAI @@ -31,7 +31,7 @@ def test_structured_few_shot_examples(self, model: BaseChatModel) -> None: @pytest.mark.xfail(reason="with_structured_output with JSON schema not supported.") def test_structured_output(self, model: BaseChatModel) -> None: super().test_structured_output(model) - + @pytest.mark.xfail(reason="with_structured_output with JSON schema not supported.") def test_structured_output_pydantic_2_v1(self, model: BaseChatModel) -> None: super().test_structured_output_pydantic_2_v1(model) From cb9fb5e548c30840c2438b18609d89874b19735e Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Fri, 6 Sep 2024 10:43:36 -0400 Subject: [PATCH 3/3] rate limit more --- libs/genai/tests/integration_tests/test_standard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/genai/tests/integration_tests/test_standard.py b/libs/genai/tests/integration_tests/test_standard.py index 4ce86b04..4cf277af 100644 --- a/libs/genai/tests/integration_tests/test_standard.py +++ b/libs/genai/tests/integration_tests/test_standard.py @@ -9,7 +9,7 @@ from langchain_google_genai import ChatGoogleGenerativeAI -rate_limiter = InMemoryRateLimiter(requests_per_second=0.5) +rate_limiter = InMemoryRateLimiter(requests_per_second=0.25) class TestGeminiAIStandard(ChatModelIntegrationTests):