Skip to content

Commit

Permalink
genai[patch]: add standard tests (#478)
Browse files Browse the repository at this point in the history
TODO:
- Add quota so we can test vision models;
- Fix xfails on structured output (currently doesn't appear to support
JSON-schema inputs)

Will plan to merge as-is to support testing for pydantic v2 migration.
  • Loading branch information
ccurme authored Sep 6, 2024
2 parents 638b5e0 + cb9fb5e commit 2fe7942
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 0 deletions.
44 changes: 44 additions & 0 deletions libs/genai/tests/integration_tests/test_standard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
"""Standard LangChain interface tests"""

from typing import Type

import pytest
from langchain_core.language_models import BaseChatModel
from langchain_core.rate_limiters import InMemoryRateLimiter
from langchain_standard_tests.integration_tests import ChatModelIntegrationTests

from langchain_google_genai import ChatGoogleGenerativeAI

rate_limiter = InMemoryRateLimiter(requests_per_second=0.25)


class TestGeminiAIStandard(ChatModelIntegrationTests):
@property
def chat_model_class(self) -> Type[BaseChatModel]:
return ChatGoogleGenerativeAI

@property
def chat_model_params(self) -> dict:
return {
"model": "models/gemini-1.0-pro-001",
"rate_limiter": rate_limiter,
}

@pytest.mark.xfail(reason="Gemini 1.0 doesn't support tool_choice='any'")
def test_structured_few_shot_examples(self, model: BaseChatModel) -> None:
super().test_structured_few_shot_examples(model)

@pytest.mark.xfail(reason="with_structured_output with JSON schema not supported.")
def test_structured_output(self, model: BaseChatModel) -> None:
super().test_structured_output(model)

@pytest.mark.xfail(reason="with_structured_output with JSON schema not supported.")
def test_structured_output_pydantic_2_v1(self, model: BaseChatModel) -> None:
super().test_structured_output_pydantic_2_v1(model)

@pytest.mark.xfail(reason="Not yet supported")
def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None:
super().test_tool_message_histories_list_content(model)


# TODO: increase quota on gemini-1.5-pro-001 and test as well
26 changes: 26 additions & 0 deletions libs/genai/tests/unit_tests/test_standard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
from typing import Type

from langchain_core.language_models import BaseChatModel
from langchain_standard_tests.unit_tests import ChatModelUnitTests

from langchain_google_genai import ChatGoogleGenerativeAI


class TestGeminiAIStandard(ChatModelUnitTests):
@property
def chat_model_class(self) -> Type[BaseChatModel]:
return ChatGoogleGenerativeAI

@property
def chat_model_params(self) -> dict:
return {"model": "models/gemini-1.0-pro-001"}


class TestGemini_15_AIStandard(ChatModelUnitTests):
@property
def chat_model_class(self) -> Type[BaseChatModel]:
return ChatGoogleGenerativeAI

@property
def chat_model_params(self) -> dict:
return {"model": "models/gemini-1.5-pro-001"}

0 comments on commit 2fe7942

Please sign in to comment.