From 04be476fca244dadec01c89d93996fe703f5fe27 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 4 Sep 2026 19:19:06 +0000 Subject: [PATCH 1/3] refactor(integrations): share auto_instrument smoke runner across scripts Add `run_auto_smoke()` to `braintrust.integrations.test_utils` and rewrite 23 of the 29 `auto_test_scripts/` to use it. The helper encodes the standard contract for `auto_instrument()` smoke tests: optional pre/post `is_patched` check, first call returns `{name: True, ...}`, second call is idempotent, and an optional memory-logger + VCR context that delegates to a per-script `run()` callback for the API call and span-shape asserts. Net -58 lines across the auto_test_scripts, but the real win is a single authoritative definition of "auto_instrument works" that every simple provider script now shares. Complex scripts that are mostly provider-specific setup rather than boilerplate (agentscope, pipecat, agno) are left alone, as are the `test_patch_litellm_*` scripts which exercise `patch_litellm()` directly rather than `auto_instrument()`. Verified: `nox -s pylint`, `nox -s test_core`, `nox -s test_litellm(latest)` covering both the refactored `test_auto_litellm` and the untouched `test_patch_litellm_*` scripts, plus pre-commit hooks on all changed files. Co-Authored-By: Claude Opus 4.7 --- .../auto_test_scripts/test_auto_adk.py | 46 +++++++--------- .../auto_test_scripts/test_auto_ai_sdk.py | 43 +++++++-------- .../auto_test_scripts/test_auto_anthropic.py | 51 ++++++++---------- .../auto_test_scripts/test_auto_autogen.py | 30 +++++------ .../test_auto_bedrock_runtime.py | 12 ++--- .../test_auto_claude_agent_sdk.py | 20 +++---- .../auto_test_scripts/test_auto_cohere.py | 14 ++--- .../auto_test_scripts/test_auto_crewai.py | 25 ++++----- .../auto_test_scripts/test_auto_cursor_sdk.py | 25 ++++----- .../auto_test_scripts/test_auto_dspy.py | 17 +++--- .../test_auto_google_genai.py | 16 ++---- .../test_auto_huggingface_hub.py | 14 ++--- .../auto_test_scripts/test_auto_instructor.py | 36 +++++-------- .../auto_test_scripts/test_auto_langchain.py | 36 +++++++------ .../auto_test_scripts/test_auto_litellm.py | 32 ++++++----- .../test_auto_livekit_agents.py | 53 ++++++++----------- .../auto_test_scripts/test_auto_mistral.py | 14 ++--- .../auto_test_scripts/test_auto_openai.py | 22 ++------ .../test_auto_openai_agents.py | 17 ++---- .../auto_test_scripts/test_auto_openrouter.py | 13 ++--- .../test_auto_pydantic_ai.py | 19 ++----- .../auto_test_scripts/test_auto_temporal.py | 8 +-- .../test_auto_transformers.py | 19 +++---- py/src/braintrust/integrations/test_utils.py | 53 +++++++++++++++++++ 24 files changed, 282 insertions(+), 353 deletions(-) diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py index a6abc3f3c..e14498847 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py @@ -3,7 +3,6 @@ import importlib from importlib.metadata import version as pkg_version -from braintrust.auto import auto_instrument from braintrust.integrations.adk.patchers import ( AgentRunAsyncPatcher, AgentRunPatcher, @@ -11,6 +10,7 @@ _ThreadBridgePlatformSubPatcher, _ThreadBridgeRunnersSubPatcher, ) +from braintrust.integrations.test_utils import run_auto_smoke from google.adk import runners as adk_runners from google.adk.agents import BaseAgent from google.adk.runners import Runner @@ -22,34 +22,26 @@ assert pkg_version("google-adk") -def is_patched(target, patcher): +agent_run_target = base_node.BaseNode.run if base_node is not None else BaseAgent.run_async +agent_run_patcher = AgentRunPatcher if base_node is not None else AgentRunAsyncPatcher + + +def _marker(target, patcher) -> bool: return bool(getattr(target, patcher.patch_marker_attr(), False)) -# 1. Verify ADK surfaces are not patched initially. -agent_run_target = base_node.BaseNode.run if base_node is not None else BaseAgent.run_async -agent_run_patcher = AgentRunPatcher if base_node is not None else AgentRunAsyncPatcher -assert not is_patched(agent_run_target, agent_run_patcher) -assert not is_patched(Runner.run_async, _RunnerRunAsyncSubPatcher) -assert not is_patched(platform_thread.create_thread, _ThreadBridgePlatformSubPatcher) -assert not is_patched(adk_runners.create_thread, _ThreadBridgeRunnersSubPatcher) - -# 2. Instrument. -results = auto_instrument() -assert results.get("adk") == True, "auto_instrument should return True for adk" - -# 3. Verify the imported google.adk surfaces are patched. -assert is_patched(agent_run_target, agent_run_patcher) -assert is_patched(Runner.run_async, _RunnerRunAsyncSubPatcher) -assert not is_patched(Runner.run, _RunnerRunAsyncSubPatcher) -assert is_patched(platform_thread.create_thread, _ThreadBridgePlatformSubPatcher) -assert is_patched(adk_runners.create_thread, _ThreadBridgeRunnersSubPatcher) - -# 4. Idempotent. -results2 = auto_instrument() -assert results2.get("adk") == True, "auto_instrument should still return True on second call" -assert is_patched(agent_run_target, agent_run_patcher) -assert is_patched(Runner.run_async, _RunnerRunAsyncSubPatcher) -assert not is_patched(Runner.run, _RunnerRunAsyncSubPatcher) +def _is_patched() -> bool: + return ( + _marker(agent_run_target, agent_run_patcher) + and _marker(Runner.run_async, _RunnerRunAsyncSubPatcher) + and _marker(platform_thread.create_thread, _ThreadBridgePlatformSubPatcher) + and _marker(adk_runners.create_thread, _ThreadBridgeRunnersSubPatcher) + ) + + +run_auto_smoke("adk", is_patched=_is_patched) + +# Runner.run must stay unpatched even after auto_instrument — only run_async is instrumented. +assert not _marker(Runner.run, _RunnerRunAsyncSubPatcher) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py index a221e6c61..1b69fab5b 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py @@ -4,16 +4,11 @@ import asyncio import ai -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke -async def main(): - results = auto_instrument() - assert results.get("ai_sdk") is True - assert auto_instrument().get("ai_sdk") is True - - with autoinstrument_test_context("test_auto_ai_sdk", integration="ai_sdk") as memory_logger: +def _call(memory_logger): + async def drive(): async with ai.stream( ai.get_model("openai:gpt-4o-mini"), [ai.user_message("Reply with the single word hello.")], @@ -21,19 +16,21 @@ async def main(): async for _ in stream: pass - spans = memory_logger.pop() - assert len(spans) == 2, f"Expected AI SDK and provider spans, got: {spans!r}" - ai_span = next(span for span in spans if span["span_attributes"]["name"] == "ai.stream") - provider_span = next(span for span in spans if span["span_attributes"].get("type") == "llm") - assert ai_span["span_attributes"]["type"] == "task" - for token_metric in ("tokens", "prompt_tokens", "completion_tokens"): - assert token_metric not in ai_span["metrics"] - assert provider_span["metrics"]["tokens"] > 0 - assert provider_span["metrics"]["completion_reasoning_tokens"] >= 0 - assert provider_span["metrics"]["prompt_cached_tokens"] >= 0 - assert ai_span["context"]["span_origin"]["instrumentation"]["name"] == "ai-sdk-auto" - assert provider_span["context"]["span_origin"]["instrumentation"]["name"] == "openai-auto" - - -asyncio.run(main()) + asyncio.run(drive()) + + spans = memory_logger.pop() + assert len(spans) == 2, f"Expected AI SDK and provider spans, got: {spans!r}" + ai_span = next(span for span in spans if span["span_attributes"]["name"] == "ai.stream") + provider_span = next(span for span in spans if span["span_attributes"].get("type") == "llm") + assert ai_span["span_attributes"]["type"] == "task" + for token_metric in ("tokens", "prompt_tokens", "completion_tokens"): + assert token_metric not in ai_span["metrics"] + assert provider_span["metrics"]["tokens"] > 0 + assert provider_span["metrics"]["completion_reasoning_tokens"] >= 0 + assert provider_span["metrics"]["prompt_cached_tokens"] >= 0 + assert ai_span["context"]["span_origin"]["instrumentation"]["name"] == "ai-sdk-auto" + assert provider_span["context"]["span_origin"]["instrumentation"]["name"] == "openai-auto" + + +run_auto_smoke("ai_sdk", integration="ai_sdk", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py index ec7039c04..3905e9c16 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py @@ -3,36 +3,25 @@ import os import anthropic -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context - - -# 1. Verify not patched initially -original_sync_module = type(anthropic.Anthropic(api_key="test-key").messages).__module__ -original_async_module = type(anthropic.AsyncAnthropic(api_key="test-key").messages).__module__ - -# 2. Instrument -results = auto_instrument() -assert results.get("anthropic") == True - -patched_sync = anthropic.Anthropic(api_key="test-key") -patched_async = anthropic.AsyncAnthropic(api_key="test-key") -assert type(patched_sync.messages).__module__ == "braintrust.integrations.anthropic.tracing" -assert type(patched_async.messages).__module__ == "braintrust.integrations.anthropic.tracing" -assert type(patched_sync.messages).__module__ != original_sync_module -assert type(patched_async.messages).__module__ != original_async_module - -# 3. Idempotent -results2 = auto_instrument() -assert results2.get("anthropic") == True - -# 4. Make API call and verify span -model = ( - "claude-haiku-4-5-20251001" - if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") == "latest" - else "claude-3-haiku-20240307" -) -with autoinstrument_test_context("test_auto_anthropic", integration="anthropic") as memory_logger: +from braintrust.integrations.test_utils import run_auto_smoke + + +_TRACING_MODULE = "braintrust.integrations.anthropic.tracing" + + +def _is_patched() -> bool: + return ( + type(anthropic.Anthropic(api_key="test-key").messages).__module__ == _TRACING_MODULE + and type(anthropic.AsyncAnthropic(api_key="test-key").messages).__module__ == _TRACING_MODULE + ) + + +def _call(memory_logger): + model = ( + "claude-haiku-4-5-20251001" + if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") == "latest" + else "claude-3-haiku-20240307" + ) client = anthropic.Anthropic() response = client.messages.create( model=model, @@ -47,4 +36,6 @@ assert span["metadata"]["provider"] == "anthropic" assert "claude" in span["metadata"]["model"] + +run_auto_smoke("anthropic", is_patched=_is_patched, integration="anthropic", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py index 93a1a52af..9fd6f1f7f 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py @@ -1,32 +1,30 @@ from autogen_agentchat.agents import AssistantAgent, BaseChatAgent from autogen_agentchat.teams import BaseGroupChat from autogen_core.tools import FunctionTool -from braintrust.auto import auto_instrument +from braintrust.integrations.test_utils import run_auto_smoke -assert not getattr(BaseChatAgent.run, "__braintrust_patched_autogen_chat_agent_run__", False) -assert not getattr( - AssistantAgent.on_messages_stream, "__braintrust_patched_autogen_chat_agent_assistant_on_messages_stream__", False -) -assert not getattr(BaseGroupChat.run, "__braintrust_patched_autogen_team_run__", False) -assert not getattr(FunctionTool.run, "__braintrust_patched_autogen_function_tool_run__", False) +_PATCHED_MARKERS = { + BaseChatAgent.run: "__braintrust_patched_autogen_chat_agent_run__", + AssistantAgent.on_messages_stream: "__braintrust_patched_autogen_chat_agent_assistant_on_messages_stream__", + BaseGroupChat.run: "__braintrust_patched_autogen_team_run__", + FunctionTool.run: "__braintrust_patched_autogen_function_tool_run__", +} + + +def _is_patched() -> bool: + return all(getattr(target, marker, False) for target, marker in _PATCHED_MARKERS.items()) -results = auto_instrument() -assert results.get("autogen") == True -assert auto_instrument().get("autogen") == True -assert getattr(BaseChatAgent.run, "__braintrust_patched_autogen_chat_agent_run__", False) +run_auto_smoke("autogen", is_patched=_is_patched) + +# Additional marker checks not covered by the shared runner. assert getattr(BaseChatAgent.run_stream, "__braintrust_patched_autogen_chat_agent_run_stream__", False) assert getattr(BaseChatAgent.on_messages, "__braintrust_patched_autogen_chat_agent_base_on_messages__", False) assert getattr( BaseChatAgent.on_messages_stream, "__braintrust_patched_autogen_chat_agent_base_on_messages_stream__", False ) assert getattr(AssistantAgent.on_messages, "__braintrust_patched_autogen_chat_agent_assistant_on_messages__", False) -assert getattr( - AssistantAgent.on_messages_stream, "__braintrust_patched_autogen_chat_agent_assistant_on_messages_stream__", False -) -assert getattr(BaseGroupChat.run, "__braintrust_patched_autogen_team_run__", False) assert getattr(BaseGroupChat.run_stream, "__braintrust_patched_autogen_team_run_stream__", False) -assert getattr(FunctionTool.run, "__braintrust_patched_autogen_function_tool_run__", False) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py index 76ce42f98..60e0a7b00 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py @@ -10,20 +10,14 @@ os.environ.setdefault("AWS_SESSION_TOKEN", "testing") import boto3 -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke MODEL = os.getenv("BRAINTRUST_BEDROCK_CONVERSE_MODEL", "us.amazon.nova-lite-v1:0") REGION = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-east-1" -results = auto_instrument() -assert results.get("bedrock_runtime") is True -results2 = auto_instrument() -assert results2.get("bedrock_runtime") is True - -with autoinstrument_test_context("test_auto_bedrock_runtime", integration="bedrock_runtime") as memory_logger: +def _call(memory_logger): client = boto3.client("bedrock-runtime", region_name=REGION) response = client.converse( modelId=MODEL, @@ -40,4 +34,6 @@ assert span["metadata"]["endpoint"] == "converse" assert span["span_attributes"]["name"] == "bedrock.converse" + +run_auto_smoke("bedrock_runtime", integration="bedrock_runtime", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py index d7524b393..b8a6e5271 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py @@ -1,20 +1,12 @@ """Test auto_instrument for Claude Agent SDK (no uninstrument available).""" -from braintrust.auto import auto_instrument -from braintrust.integrations.claude_agent_sdk._test_transport import make_cassette_transport -from braintrust.integrations.test_utils import autoinstrument_test_context - +import asyncio -# 1. Instrument -results = auto_instrument() -assert results.get("claude_agent_sdk") == True +from braintrust.integrations.claude_agent_sdk._test_transport import make_cassette_transport +from braintrust.integrations.test_utils import run_auto_smoke -# 2. Idempotent -results2 = auto_instrument() -assert results2.get("claude_agent_sdk") == True -# 3. Make API call and verify span -with autoinstrument_test_context("test_auto_claude_agent_sdk", use_vcr=False) as memory_logger: +def _call(memory_logger): import claude_agent_sdk # pylint: disable=import-error options = claude_agent_sdk.ClaudeAgentOptions( @@ -35,12 +27,12 @@ async def run_agent(): return message return None - import asyncio - result = asyncio.run(run_agent()) assert result is not None spans = memory_logger.pop() assert len(spans) >= 1, f"Expected at least 1 span, got {len(spans)}" + +run_auto_smoke("claude_agent_sdk", use_vcr=False, run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py index 1345bda03..46c2a963b 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py @@ -7,18 +7,10 @@ os.environ.setdefault("COHERE_API_KEY", os.environ["CO_API_KEY"]) import cohere -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke -results = auto_instrument() -assert results.get("cohere") is True - -results2 = auto_instrument() -assert results2.get("cohere") is True - - -with autoinstrument_test_context("test_auto_cohere", integration="cohere") as memory_logger: +def _call(memory_logger): use_v2 = hasattr(cohere, "ClientV2") and hasattr(cohere.ClientV2, "chat") if use_v2: client = cohere.ClientV2(api_key=os.environ["CO_API_KEY"]) @@ -44,4 +36,6 @@ assert span["metadata"]["model"] == "command-a-03-2025" assert span["span_attributes"]["name"] == "cohere.chat" + +run_auto_smoke("cohere", integration="cohere", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py index 1a1af5858..e936c85a0 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py @@ -7,27 +7,24 @@ # pylint: disable=import-error -from braintrust.auto import auto_instrument from braintrust.integrations.crewai import BraintrustCrewAIListener from braintrust.integrations.crewai.patchers import _get_registered_listener +from braintrust.integrations.test_utils import run_auto_smoke -# 1. Not registered initially. -assert _get_registered_listener() is None +def _is_patched() -> bool: + listener = _get_registered_listener() + return isinstance(listener, BraintrustCrewAIListener) -# 2. Instrument once. -results = auto_instrument() -assert results.get("crewai") is True -listener1 = _get_registered_listener() -assert listener1 is not None -assert isinstance(listener1, BraintrustCrewAIListener) -# 3. Idempotent — same listener, still reports True. -results2 = auto_instrument() -assert results2.get("crewai") is True -assert _get_registered_listener() is listener1 +run_auto_smoke("crewai", is_patched=_is_patched) -# 4. Listener is actually subscribed on the CrewAI event bus. +# Listener stays the same across the two auto_instrument calls. +listener = _get_registered_listener() +assert listener is not None +assert isinstance(listener, BraintrustCrewAIListener) + +# Listener is actually subscribed on the CrewAI event bus. from crewai.events import CrewKickoffStartedEvent from crewai.events.event_bus import crewai_event_bus diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py index 0c05cef4f..9fd491d21 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py @@ -7,25 +7,15 @@ from pathlib import Path import cursor_sdk -from braintrust.auto import auto_instrument from braintrust.integrations.cursor_sdk._test_vcr import cursor_vcr_config -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_spans_by_type -results = auto_instrument() -assert results.get("cursor_sdk") is True -assert auto_instrument().get("cursor_sdk") is True - - -with tempfile.TemporaryDirectory() as workspace: - Path(workspace, "README.md").write_text("Cursor auto-instrumentation workspace.\n", encoding="utf-8") - with autoinstrument_test_context( - "test_auto_cursor_sdk", - integration="cursor_sdk", - vcr_config=cursor_vcr_config(), - ) as memory_logger: +def _call(memory_logger): + with tempfile.TemporaryDirectory() as workspace: + Path(workspace, "README.md").write_text("Cursor auto-instrumentation workspace.\n", encoding="utf-8") with cursor_sdk.CursorClient.launch_bridge(workspace=workspace) as client: with client.agents.create( model="composer-2.5", @@ -40,4 +30,11 @@ assert find_spans_by_type(spans, SpanTypeAttribute.LLM) assert all(span["context"]["span_origin"]["instrumentation"]["name"] == "cursor-sdk-auto" for span in spans) + +run_auto_smoke( + "cursor_sdk", + integration="cursor_sdk", + vcr_config=cursor_vcr_config(), + run=_call, +) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py index bb6de8c07..3bc0c0b2a 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py @@ -6,27 +6,22 @@ """ import dspy -from braintrust.auto import auto_instrument from braintrust.integrations.dspy import BraintrustDSpyCallback +from braintrust.integrations.test_utils import run_auto_smoke -# 1. Verify not patched initially -assert not getattr(dspy.configure, "__braintrust_patched_dspy_configure__", False) +def _is_patched() -> bool: + return bool(getattr(dspy.configure, "__braintrust_patched_dspy_configure__", False)) -# 2. Instrument -results = auto_instrument() -assert results.get("dspy") == True -# 3. Idempotent -results2 = auto_instrument() -assert results2.get("dspy") == True +run_auto_smoke("dspy", is_patched=_is_patched) -# 4. Verify callback is added when configure() is called +# Verify callback is added when configure() is called. dspy.configure(lm=None) from dspy.dsp.utils.settings import settings has_bt_callback = any(isinstance(cb, BraintrustDSpyCallback) for cb in settings.callbacks) -assert has_bt_callback, f"Expected BraintrustDSpyCallback in callbacks after configure()" +assert has_bt_callback, "Expected BraintrustDSpyCallback in callbacks after configure()" print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py index 4db3f37f5..66df92577 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py @@ -2,20 +2,10 @@ import os -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke -# 1. Instrument -results = auto_instrument() -assert results.get("google_genai") == True - -# 2. Idempotent -results2 = auto_instrument() -assert results2.get("google_genai") == True - -# 3. Make API call and verify span -with autoinstrument_test_context("test_auto_google_genai", integration="google_genai") as memory_logger: +def _call(memory_logger): from google.genai import types from google.genai.client import Client @@ -36,4 +26,6 @@ span = spans[0] assert "gemini" in span["metadata"]["model"] + +run_auto_smoke("google_genai", integration="google_genai", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py index 949880ccf..277c91401 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py @@ -7,22 +7,14 @@ # ``provider="auto"`` routing (validated locally before any HTTP request). os.environ.setdefault("HF_TOKEN", "hf_test_dummy_api_key_for_vcr_tests") -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke from huggingface_hub import InferenceClient -results = auto_instrument() -assert results.get("huggingface_hub") is True - -results2 = auto_instrument() -assert results2.get("huggingface_hub") is True - - CHAT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" -with autoinstrument_test_context("test_auto_huggingface_hub", integration="huggingface_hub") as memory_logger: +def _call(memory_logger): # ``provider="cerebras"`` hosts ``meta-llama/Llama-3.1-8B-Instruct`` across # the matrix; ``hf-inference`` no longer hosts most conversational checkpoints. client = InferenceClient(model=CHAT_MODEL, provider="cerebras", token=os.environ["HF_TOKEN"]) @@ -40,4 +32,6 @@ assert span["metadata"]["provider"] == "cerebras" assert span["span_attributes"]["name"] == "huggingface.chat_completion" + +run_auto_smoke("huggingface_hub", integration="huggingface_hub", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py index 526d64e4d..4d1bb5ee5 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py @@ -2,8 +2,7 @@ import instructor import openai -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke from pydantic import BaseModel @@ -12,22 +11,11 @@ class Person(BaseModel): age: int -# 1. Instrument -results = auto_instrument() -assert results.get("instructor") is True, results - -# 2. Idempotent -results2 = auto_instrument() -assert results2.get("instructor") is True - -# 3. Drive a real instructor.from_openai call against a recorded cassette and -# verify a parent task-typed Instructor span shows up alongside the OpenAI -# llm child span. Cassette is shared with the in-process test suite under -# integrations/instructor/cassettes//. -with autoinstrument_test_context( - "TestInstructorOpenAISpans.test_instructor_openai_single_success", - integration="instructor", -) as memory_logger: +def _call(memory_logger): + # Drive a real instructor.from_openai call against a recorded cassette and + # verify a parent task-typed Instructor span shows up alongside the OpenAI + # llm child span. Cassette is shared with the in-process test suite under + # integrations/instructor/cassettes//. client = openai.OpenAI(api_key="sk-test-dummy-api-key-for-vcr-tests") patched = instructor.from_openai(client, mode=instructor.Mode.TOOLS) result = patched.chat.completions.create( @@ -49,10 +37,12 @@ class Person(BaseModel): types = [s["span_attributes"].get("type") for s in spans] assert "task" in types, f"missing instructor parent (task) span: {types}" assert "llm" in types, f"missing openai child (llm) span: {types}" - parent = next(s for s in spans if s["span_attributes"].get("type") == "task") - assert parent["span_attributes"]["name"] == "instructor.create" - assert parent["metadata"]["response_model"] == "Person" - assert parent["metadata"]["mode"] == "TOOLS" - assert parent["metadata"]["retry_count"] == 0 + +run_auto_smoke( + "instructor", + cassette="TestInstructorOpenAISpans.test_instructor_openai_single_success", + integration="instructor", + run=_call, +) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py index 87f809b04..3583ec15c 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py @@ -1,36 +1,24 @@ """Test auto_instrument for LangChain.""" -from braintrust.auto import auto_instrument from braintrust.integrations.langchain import BraintrustCallbackHandler from braintrust.integrations.langchain.context import clear_global_handler, get_global_handler -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke from langchain_core.callbacks import CallbackManager from langchain_core.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI -# 1. Verify not patched initially. +# Ensure a clean starting state so the pre-check reflects a fresh process. clear_global_handler() -assert get_global_handler() is None manager = CallbackManager.configure() assert next((h for h in manager.handlers if isinstance(h, BraintrustCallbackHandler)), None) is None -# 2. Instrument. -results = auto_instrument() -assert results.get("langchain") == True -handler = get_global_handler() -assert isinstance(handler, BraintrustCallbackHandler) -manager = CallbackManager.configure() -assert next((h for h in manager.handlers if isinstance(h, BraintrustCallbackHandler)), None) is handler +def _is_patched() -> bool: + return isinstance(get_global_handler(), BraintrustCallbackHandler) -# 3. Idempotent. -results2 = auto_instrument() -assert results2.get("langchain") == True -assert get_global_handler() is handler -# 4. Make an API call and verify spans. -with autoinstrument_test_context("test_global_handler", integration="langchain") as memory_logger: +def _call(memory_logger): prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") model = ChatOpenAI( model="gpt-4o-mini", @@ -48,4 +36,18 @@ spans = memory_logger.pop() assert len(spans) > 0 + +run_auto_smoke( + "langchain", + is_patched=_is_patched, + cassette="test_global_handler", + integration="langchain", + run=_call, +) + +# The handler installed via auto_instrument must also flow through CallbackManager.configure(). +handler = get_global_handler() +manager = CallbackManager.configure() +assert next((h for h in manager.handlers if isinstance(h, BraintrustCallbackHandler)), None) is handler + print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py index d5cdd667e..d45951789 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py @@ -1,28 +1,15 @@ """Test auto_instrument for LiteLLM.""" import litellm -from braintrust.auto import auto_instrument from braintrust.integrations.litellm import LiteLLMIntegration -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke -# 1. Verify not patched initially -assert not LiteLLMIntegration.patchers[0].is_patched(litellm, None) +def _is_patched() -> bool: + return LiteLLMIntegration.patchers[0].is_patched(litellm, None) -# 2. Instrument -# Disable OpenAI auto-instrumentation here because LiteLLM's OpenAI-backed -# chat path can otherwise produce both a LiteLLM span and an OpenAI span. -# This test is meant to validate LiteLLM instrumentation in isolation. -results = auto_instrument(openai=False) -assert results.get("litellm") == True -assert LiteLLMIntegration.patchers[0].is_patched(litellm, None) - -# 3. Idempotent -results2 = auto_instrument(openai=False) -assert results2.get("litellm") == True -# 4. Make API call and verify span -with autoinstrument_test_context("test_auto_litellm", integration="litellm") as memory_logger: +def _call(memory_logger): response = litellm.completion( model="gpt-4o-mini", messages=[{"role": "user", "content": "Say hi"}], @@ -34,4 +21,15 @@ span = spans[0] assert span["metadata"]["provider"] == "openai" + +# Disable OpenAI auto-instrumentation here because LiteLLM's OpenAI-backed +# chat path can otherwise produce both a LiteLLM span and an OpenAI span. +# This test is meant to validate LiteLLM instrumentation in isolation. +run_auto_smoke( + "litellm", + auto_instrument_kwargs={"openai": False}, + is_patched=_is_patched, + integration="litellm", + run=_call, +) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py index 891bb2954..9ee45941a 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py @@ -2,13 +2,7 @@ import inspect -from braintrust.auto import auto_instrument -from wrapt import FunctionWrapper - - -def _is_braintrust_wrapped(target, attr: str) -> bool: - return isinstance(inspect.getattr_static(target, attr, None), FunctionWrapper) - +from braintrust.integrations.test_utils import run_auto_smoke # Import the provider classes before auto-instrumentation to verify setup handles # normal user import order in a fresh process. @@ -18,30 +12,29 @@ def _is_braintrust_wrapped(target, attr: str) -> bool: from livekit.agents.tts import TTS # noqa: E402 from livekit.agents.voice import generation # noqa: E402 from livekit.agents.voice.io import AudioOutput # noqa: E402 +from wrapt import FunctionWrapper + + +def _attr_wrapped(target, attr: str) -> bool: + return isinstance(inspect.getattr_static(target, attr, None), FunctionWrapper) + + +_WRAP_TARGETS = ( + (AgentSession, "run"), + (AgentSession, "_on_audio_output_changed"), + (AgentSession, "_update_user_state"), + (LLMStream, "_run"), + (STT, "recognize"), + (TTS, "synthesize"), + (AudioOutput, "capture_frame"), +) + +def _is_patched() -> bool: + return all(_attr_wrapped(target, attr) for target, attr in _WRAP_TARGETS) and isinstance( + generation._execute_tools_task, FunctionWrapper + ) -assert not _is_braintrust_wrapped(AgentSession, "run") -assert not _is_braintrust_wrapped(AgentSession, "_on_audio_output_changed") -assert not _is_braintrust_wrapped(AgentSession, "_update_user_state") -assert not isinstance(generation._execute_tools_task, FunctionWrapper) -assert not _is_braintrust_wrapped(LLMStream, "_run") -assert not _is_braintrust_wrapped(STT, "recognize") -assert not _is_braintrust_wrapped(TTS, "synthesize") -assert not _is_braintrust_wrapped(AudioOutput, "capture_frame") - -results = auto_instrument() -assert results.get("livekit_agents") is True -assert _is_braintrust_wrapped(AgentSession, "run") -assert _is_braintrust_wrapped(AgentSession, "_on_audio_output_changed") -assert _is_braintrust_wrapped(AgentSession, "_update_user_state") -assert isinstance(generation._execute_tools_task, FunctionWrapper) -assert _is_braintrust_wrapped(LLMStream, "_run") -assert _is_braintrust_wrapped(STT, "recognize") -assert _is_braintrust_wrapped(TTS, "synthesize") -assert _is_braintrust_wrapped(AudioOutput, "capture_frame") - -# Idempotent. -results2 = auto_instrument() -assert results2.get("livekit_agents") is True +run_auto_smoke("livekit_agents", is_patched=_is_patched) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py index 7400975e2..39ea59a80 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py @@ -2,8 +2,7 @@ import os -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke try: @@ -12,14 +11,7 @@ from mistralai import Mistral -results = auto_instrument() -assert results.get("mistral") == True - -results2 = auto_instrument() -assert results2.get("mistral") == True - - -with autoinstrument_test_context("test_auto_mistral", integration="mistral") as memory_logger: +def _call(memory_logger): client = Mistral(api_key=os.environ.get("MISTRAL_API_KEY")) response = client.chat.complete( model="mistral-small-latest", @@ -35,4 +27,6 @@ assert span["metadata"]["model"] == "mistral-small-latest" assert "4" in str(span["output"]) + +run_auto_smoke("mistral", integration="mistral", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py index 67dd73574..2d98e0674 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py @@ -3,30 +3,16 @@ import inspect import openai -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke from wrapt import FunctionWrapper -def _is_braintrust_wrapped() -> bool: +def _is_patched() -> bool: attr = inspect.getattr_static(openai.resources.chat.completions.Completions, "create", None) return isinstance(attr, FunctionWrapper) -# 1. Verify not patched initially -assert not _is_braintrust_wrapped() - -# 2. Instrument -results = auto_instrument() -assert results.get("openai") == True -assert _is_braintrust_wrapped() - -# 3. Idempotent -results2 = auto_instrument() -assert results2.get("openai") == True - -# 4. Make API call and verify span -with autoinstrument_test_context("test_auto_openai", integration="openai") as memory_logger: +def _call(memory_logger): client = openai.OpenAI() response = client.chat.completions.create( model="gpt-4o-mini", @@ -40,4 +26,6 @@ def _is_braintrust_wrapped() -> bool: assert span["metadata"]["provider"] == "openai" assert "gpt-4o-mini" in span["metadata"]["model"] + +run_auto_smoke("openai", is_patched=_is_patched, integration="openai", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py index a4e3f37bb..230c43333 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py @@ -3,9 +3,8 @@ import asyncio import agents -from braintrust.auto import auto_instrument from braintrust.integrations.openai_agents import BraintrustTracingProcessor -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke TEST_MODEL = "gpt-4o-mini" @@ -13,21 +12,13 @@ TEST_AGENT_INSTRUCTIONS = "You are a helpful assistant. Be very concise." -def _has_braintrust_processor() -> bool: +def _is_patched() -> bool: provider = agents.tracing.get_trace_provider() processors = getattr(getattr(provider, "_multi_processor", None), "_processors", ()) return any(isinstance(processor, BraintrustTracingProcessor) for processor in processors) -results = auto_instrument() -assert results.get("openai_agents") == True -assert _has_braintrust_processor() - -results2 = auto_instrument() -assert results2.get("openai_agents") == True -assert _has_braintrust_processor() - -with autoinstrument_test_context("test_auto_openai_agents", integration="openai_agents") as memory_logger: +def _call(memory_logger): from agents import Agent from agents.run import AgentRunner @@ -42,4 +33,6 @@ async def run_agent(): spans = memory_logger.pop() assert len(spans) >= 2, f"Expected at least 2 spans, got {len(spans)}" + +run_auto_smoke("openai_agents", is_patched=_is_patched, integration="openai_agents", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py index 2b601dc4f..d99c3ebfb 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py @@ -3,17 +3,10 @@ import os import openrouter -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke -results = auto_instrument() -assert results.get("openrouter") == True - -results2 = auto_instrument() -assert results2.get("openrouter") == True - -with autoinstrument_test_context("test_auto_openrouter", integration="openrouter") as memory_logger: +def _call(memory_logger): client = openrouter.OpenRouter(api_key=os.environ.get("OPENROUTER_API_KEY")) response = client.chat.send( model="openai/gpt-4o-mini", @@ -29,4 +22,6 @@ assert span["metadata"]["model"] == "gpt-4o-mini" assert "4" in span["output"][0]["message"]["content"] + +run_auto_smoke("openrouter", integration="openrouter", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py index 673b6b85b..1e770edaf 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py @@ -1,19 +1,11 @@ """Test auto_instrument for Pydantic AI (no uninstrument available).""" -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +import asyncio +from braintrust.integrations.test_utils import run_auto_smoke -# 1. Instrument -results = auto_instrument() -assert results.get("pydantic_ai") == True -# 2. Idempotent -results2 = auto_instrument() -assert results2.get("pydantic_ai") == True - -# 3. Make API call and verify span -with autoinstrument_test_context("test_auto_pydantic_ai", integration="pydantic_ai") as memory_logger: +def _call(memory_logger): from pydantic_ai import Agent from pydantic_ai.models.openai import OpenAIChatModel from pydantic_ai.settings import ModelSettings @@ -23,15 +15,14 @@ model_settings=ModelSettings(max_tokens=100), ) - import asyncio - result = asyncio.run(agent.run("Say hi")) assert result.output spans = memory_logger.pop() assert len(spans) >= 1, f"Expected at least 1 span, got {len(spans)}" - # Find the agent_run span agent_spans = [s for s in spans if "agent_run" in s["span_attributes"]["name"]] assert len(agent_spans) >= 1, f"Expected agent_run span, got {[s['span_attributes']['name'] for s in spans]}" + +run_auto_smoke("pydantic_ai", integration="pydantic_ai", run=_call) print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py index 091ad7e76..020215443 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py @@ -1,14 +1,10 @@ """Test auto_instrument for Temporal.""" -from braintrust.auto import auto_instrument from braintrust.integrations.temporal import BraintrustPlugin, setup_temporal +from braintrust.integrations.test_utils import run_auto_smoke -results = auto_instrument() -assert results.get("temporal") == True - -results2 = auto_instrument() -assert results2.get("temporal") == True +run_auto_smoke("temporal") assert setup_temporal() == True assert BraintrustPlugin is not None diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py index 64401b67a..b98a673a7 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py @@ -3,18 +3,12 @@ # Keep the large Transformers/PyTorch dependencies isolated to their nox job. # pylint: disable=import-error -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.test_utils import run_auto_smoke -results = auto_instrument(transformers=True) -assert results.get("transformers") is True -assert auto_instrument(transformers=True).get("transformers") is True +def _call(memory_logger): + from transformers import pipeline -from transformers import pipeline # noqa: E402 - - -with autoinstrument_test_context("test_auto_transformers", use_vcr=False) as memory_logger: generator = pipeline( "text-generation", model="hf-internal-testing/tiny-random-LlamaForCausalLM", @@ -29,4 +23,11 @@ assert span["span_attributes"]["name"] == "huggingface.transformers.text_generation" assert span["context"]["span_origin"]["instrumentation"]["name"] == "transformers-auto" + +run_auto_smoke( + "transformers", + auto_instrument_kwargs={"transformers": True}, + use_vcr=False, + run=_call, +) print("SUCCESS") diff --git a/py/src/braintrust/integrations/test_utils.py b/py/src/braintrust/integrations/test_utils.py index 181d4c7d7..a5859149e 100644 --- a/py/src/braintrust/integrations/test_utils.py +++ b/py/src/braintrust/integrations/test_utils.py @@ -3,8 +3,10 @@ import sys import textwrap import unittest.mock +from collections.abc import Callable, Mapping from contextlib import contextmanager from pathlib import Path +from typing import Any import pytest import vcr @@ -135,6 +137,57 @@ def verify_autoinstrument_script(script_name: str, timeout: int = 30) -> subproc return result +def run_auto_smoke( + name: str, + *, + auto_instrument_kwargs: Mapping[str, object] | None = None, + is_patched: Callable[[], bool] | None = None, + cassette: str | None = None, + integration: str | None = None, + use_vcr: bool = True, + vcr_config: dict | None = None, + run: Callable[[Any], None] | None = None, +) -> None: + """Run the standard ``auto_instrument()`` smoke pattern. + + Encodes the contract shared by scripts under ``auto_test_scripts/``: + + 1. Optional pre-check: ``is_patched()`` returns False before patching. + 2. ``auto_instrument(**auto_instrument_kwargs)`` returns ``{name: True, ...}``. + 3. Optional post-check: ``is_patched()`` returns True after patching. + 4. A second ``auto_instrument`` call still returns ``{name: True, ...}`` (idempotent). + 5. If ``run`` is given, open ``autoinstrument_test_context`` (defaults cassette + name to ``f"test_auto_{name}"``) and delegate to ``run(memory_logger)``. + + Callers assert their own API-call and span-shape expectations inside ``run``. + """ + from braintrust.auto import auto_instrument + + kwargs = dict(auto_instrument_kwargs or {}) + + if is_patched is not None: + assert not is_patched(), f"{name!r} already patched before auto_instrument()" + + first = auto_instrument(**kwargs) + assert first.get(name) is True, f"auto_instrument returned {first!r}" + if is_patched is not None: + assert is_patched(), f"{name!r} not patched after auto_instrument()" + + second = auto_instrument(**kwargs) + assert second.get(name) is True, f"auto_instrument (2nd call) returned {second!r}" + + if run is None: + return + + ctx_kwargs: dict[str, Any] = {"use_vcr": use_vcr} + if integration is not None: + ctx_kwargs["integration"] = integration + if vcr_config is not None: + ctx_kwargs["vcr_config"] = vcr_config + with autoinstrument_test_context(cassette or f"test_auto_{name}", **ctx_kwargs) as memory_logger: + run(memory_logger) + + def assert_metrics_are_valid(metrics, start=None, end=None): assert metrics # assert 0 < metrics["time_to_first_token"] From 089615aae17c12fb2cc5d35ccc1eb8e12dbb6980 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 4 Sep 2026 19:31:33 +0000 Subject: [PATCH 2/3] refactor(integrations): collapse auto_test_scripts to a single registry-driven smoke MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace 26 near-identical per-provider `test_auto_*.py` scripts with one runner (`_run_smoke.py`) driven by an inline SMOKES registry, invoked from each `TestAutoInstrument*` via a new `verify_autoinstrument_smoke(name)` helper. The scripts were doing far more than a subprocess sanity check requires. The only bug class a fresh subprocess uniquely proves is "from a cold Python process, `auto_instrument()` successfully sets up this integration"; span shape, provider metadata, patching topology, and real API calls all live in the in-process `test_*.py` files for each integration. The runner asserts exactly that minimum: `auto_instrument(**kwargs).get(name) is True`, twice (idempotent). Net: −1222 LOC. The `test_patch_litellm_*` scripts stay put — they exercise `patch_litellm()` directly rather than `auto_instrument()`. The `run_auto_smoke` helper added in the previous commit is removed along with the scripts that used it. Verified: `nox -s pylint`, `nox -s "test_litellm(latest)" -- -k "test_auto_instrument_litellm or test_patch_litellm"` (3 passed), and pre-commit hooks on all changed files. Co-Authored-By: Claude Opus 4.7 --- .../braintrust/integrations/adk/test_adk.py | 4 +- .../agentscope/test_agentscope.py | 4 +- .../braintrust/integrations/agno/test_agno.py | 4 +- .../integrations/ai_sdk/test_ai_sdk.py | 4 +- .../integrations/anthropic/test_anthropic.py | 4 +- .../auto_test_scripts/_run_smoke.py | 75 +++++++++ .../auto_test_scripts/test_auto_adk.py | 47 ------ .../auto_test_scripts/test_auto_agentscope.py | 104 ------------ .../auto_test_scripts/test_auto_agno.py | 157 ------------------ .../auto_test_scripts/test_auto_ai_sdk.py | 36 ---- .../auto_test_scripts/test_auto_anthropic.py | 41 ----- .../auto_test_scripts/test_auto_autogen.py | 30 ---- .../test_auto_bedrock_runtime.py | 39 ----- .../test_auto_claude_agent_sdk.py | 38 ----- .../auto_test_scripts/test_auto_cohere.py | 41 ----- .../auto_test_scripts/test_auto_crewai.py | 35 ---- .../auto_test_scripts/test_auto_cursor_sdk.py | 40 ----- .../auto_test_scripts/test_auto_dspy.py | 27 --- .../test_auto_google_genai.py | 31 ---- .../test_auto_huggingface_hub.py | 37 ----- .../auto_test_scripts/test_auto_instructor.py | 48 ------ .../auto_test_scripts/test_auto_langchain.py | 53 ------ .../auto_test_scripts/test_auto_litellm.py | 35 ---- .../test_auto_livekit_agents.py | 40 ----- .../auto_test_scripts/test_auto_mistral.py | 32 ---- .../auto_test_scripts/test_auto_openai.py | 31 ---- .../test_auto_openai_agents.py | 38 ----- .../auto_test_scripts/test_auto_openrouter.py | 27 --- .../auto_test_scripts/test_auto_pipecat.py | 102 ------------ .../test_auto_pydantic_ai.py | 28 ---- .../auto_test_scripts/test_auto_temporal.py | 12 -- .../test_auto_transformers.py | 33 ---- .../integrations/autogen/test_autogen.py | 4 +- .../bedrock_runtime/test_bedrock_runtime.py | 4 +- .../claude_agent_sdk/test_claude_agent_sdk.py | 4 +- .../integrations/cohere/test_cohere.py | 4 +- .../integrations/crewai/test_crewai.py | 4 +- .../cursor_sdk/test_cursor_sdk.py | 4 +- .../braintrust/integrations/dspy/test_dspy.py | 4 +- .../google_genai/test_google_genai.py | 4 +- .../huggingface_hub/test_huggingface_hub.py | 4 +- .../instructor/test_instructor.py | 4 +- .../integrations/langchain/test_context.py | 4 +- .../integrations/litellm/test_litellm.py | 4 +- .../livekit_agents/test_livekit_agents.py | 4 +- .../integrations/mistral/test_mistral.py | 4 +- .../integrations/openai/test_openai.py | 4 +- .../openai_agents/test_openai_agents.py | 4 +- .../openrouter/test_openrouter.py | 4 +- .../integrations/pipecat/test_pipecat.py | 4 +- .../test_pydantic_ai_integration.py | 4 +- .../integrations/temporal/test_temporal.py | 4 +- py/src/braintrust/integrations/test_utils.py | 62 ++----- .../transformers/test_transformers.py | 4 +- 54 files changed, 138 insertions(+), 1285 deletions(-) create mode 100644 py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_agentscope.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_agno.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py delete mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py diff --git a/py/src/braintrust/integrations/adk/test_adk.py b/py/src/braintrust/integrations/adk/test_adk.py index 84d57955b..7879733f2 100644 --- a/py/src/braintrust/integrations/adk/test_adk.py +++ b/py/src/braintrust/integrations/adk/test_adk.py @@ -1164,6 +1164,6 @@ class TestAutoInstrumentADK: def test_auto_instrument_adk(self): """Test auto_instrument patches ADK classes and is idempotent.""" - from braintrust.integrations.test_utils import verify_autoinstrument_script + from braintrust.integrations.test_utils import verify_autoinstrument_smoke - verify_autoinstrument_script("test_auto_adk.py") + verify_autoinstrument_smoke("adk") diff --git a/py/src/braintrust/integrations/agentscope/test_agentscope.py b/py/src/braintrust/integrations/agentscope/test_agentscope.py index ed58819b6..692907385 100644 --- a/py/src/braintrust/integrations/agentscope/test_agentscope.py +++ b/py/src/braintrust/integrations/agentscope/test_agentscope.py @@ -7,7 +7,7 @@ import pytest from braintrust import logger from braintrust.integrations.agentscope import setup_agentscope -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger from packaging.version import Version @@ -326,4 +326,4 @@ def test_setup_agentscope_is_idempotent(): class TestAutoInstrumentAgentScope: def test_auto_instrument_agentscope(self): - verify_autoinstrument_script("test_auto_agentscope.py") + verify_autoinstrument_smoke("agentscope") diff --git a/py/src/braintrust/integrations/agno/test_agno.py b/py/src/braintrust/integrations/agno/test_agno.py index dd22859d8..ee6de7915 100644 --- a/py/src/braintrust/integrations/agno/test_agno.py +++ b/py/src/braintrust/integrations/agno/test_agno.py @@ -9,7 +9,7 @@ from braintrust.integrations.agno import setup_agno from braintrust.integrations.agno import tracing as agno_tracing_module from braintrust.integrations.agno.patchers import wrap_agent, wrap_team -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.logger import Attachment, start_span from braintrust.test_helpers import init_test_logger @@ -232,7 +232,7 @@ def get_provider(self): class TestAutoInstrumentAgno: def test_auto_instrument_agno(self): - verify_autoinstrument_script("test_auto_agno.py") + verify_autoinstrument_smoke("agno") @pytest.mark.parametrize( diff --git a/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py b/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py index f2b31c8c6..40f531f9f 100644 --- a/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py +++ b/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py @@ -9,7 +9,7 @@ from braintrust import Attachment, logger, setup_ai_sdk from braintrust.integrations.ai_sdk import patch_ai_sdk, unpatch_ai_sdk from braintrust.integrations.ai_sdk.tracing import _shape_input_messages -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger from pydantic import BaseModel @@ -254,4 +254,4 @@ def test_registration_and_unregistration_are_idempotent(): def test_auto_instrument_ai_sdk_subprocess(): - verify_autoinstrument_script("test_auto_ai_sdk.py") + verify_autoinstrument_smoke("ai_sdk") diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 6c25c4a04..35890c04f 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -19,7 +19,7 @@ _get_metadata_from_kwargs, _log_message_to_span, ) -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger from pydantic import BaseModel @@ -1501,7 +1501,7 @@ def test_setup_creates_spans(memory_logger): class TestAutoInstrumentAnthropic: def test_auto_instrument_anthropic(self): - verify_autoinstrument_script("test_auto_anthropic.py") + verify_autoinstrument_smoke("anthropic") def test_extract_anthropic_usage_preserves_nested_numeric_fields(): diff --git a/py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py b/py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py new file mode 100644 index 000000000..d33977e8b --- /dev/null +++ b/py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py @@ -0,0 +1,75 @@ +"""Fresh-subprocess sanity check for :func:`braintrust.auto.auto_instrument`. + +Usage: ``python _run_smoke.py `` + +Runs ``auto_instrument(**kwargs)`` twice in a clean Python process and asserts +that the target integration is reported as successfully instrumented on both +calls (i.e. patching works and is idempotent). + +Deliberately minimal — span shape, provider metadata, patching topology, and +real API calls are all covered by the in-process ``test_*.py`` files for each +integration. The only thing a subprocess uniquely proves is: from a cold +Python process, ``auto_instrument()`` successfully sets up this integration. +""" + +import sys + + +# ``auto_instrument`` accepts a bool kwarg per integration and returns a dict +# keyed by a display name (usually identical to the kwarg — ``bedrock_runtime`` +# is the one exception, where the kwarg is ``bedrock``). Each entry below maps +# the display name to any kwarg overrides on top of the all-True defaults. +SMOKES: dict[str, dict[str, bool]] = { + "adk": {}, + "agentscope": {}, + "agno": {}, + "ai_sdk": {}, + "anthropic": {}, + "autogen": {}, + "bedrock_runtime": {}, + "claude_agent_sdk": {}, + "cohere": {}, + "crewai": {}, + "cursor_sdk": {}, + "dspy": {}, + "google_genai": {}, + "huggingface_hub": {}, + "instructor": {}, + "langchain": {}, + # LiteLLM's OpenAI-backed chat path would otherwise produce both a LiteLLM + # span and an OpenAI span; disable OpenAI so this smoke stays scoped. + "litellm": {"openai": False}, + "livekit_agents": {}, + "mistral": {}, + "openai": {}, + "openai_agents": {}, + "openrouter": {}, + "pipecat": {}, + "pydantic_ai": {}, + "temporal": {}, + "transformers": {}, +} + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(f"usage: {sys.argv[0]} ") + name = sys.argv[1] + if name not in SMOKES: + raise SystemExit(f"unknown integration: {name!r}. Add to SMOKES in {__file__}.") + + from braintrust.auto import auto_instrument + + kwargs = SMOKES[name] + + first = auto_instrument(**kwargs) + assert first.get(name) is True, f"auto_instrument returned {first!r} for {name!r}" + + second = auto_instrument(**kwargs) + assert second.get(name) is True, f"auto_instrument (2nd call) returned {second!r} for {name!r}" + + print("SUCCESS") + + +if __name__ == "__main__": + main() diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py deleted file mode 100644 index e14498847..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Test auto_instrument for Google ADK.""" - -import importlib -from importlib.metadata import version as pkg_version - -from braintrust.integrations.adk.patchers import ( - AgentRunAsyncPatcher, - AgentRunPatcher, - _RunnerRunAsyncSubPatcher, - _ThreadBridgePlatformSubPatcher, - _ThreadBridgeRunnersSubPatcher, -) -from braintrust.integrations.test_utils import run_auto_smoke -from google.adk import runners as adk_runners -from google.adk.agents import BaseAgent -from google.adk.runners import Runner - - -platform_thread = importlib.import_module("google.adk.platform.thread") -base_node = importlib.import_module("google.adk.workflow._base_node") if hasattr(BaseAgent, "run") else None -assert importlib.import_module("google.adk").__name__ == "google.adk" -assert pkg_version("google-adk") - - -agent_run_target = base_node.BaseNode.run if base_node is not None else BaseAgent.run_async -agent_run_patcher = AgentRunPatcher if base_node is not None else AgentRunAsyncPatcher - - -def _marker(target, patcher) -> bool: - return bool(getattr(target, patcher.patch_marker_attr(), False)) - - -def _is_patched() -> bool: - return ( - _marker(agent_run_target, agent_run_patcher) - and _marker(Runner.run_async, _RunnerRunAsyncSubPatcher) - and _marker(platform_thread.create_thread, _ThreadBridgePlatformSubPatcher) - and _marker(adk_runners.create_thread, _ThreadBridgeRunnersSubPatcher) - ) - - -run_auto_smoke("adk", is_patched=_is_patched) - -# Runner.run must stay unpatched even after auto_instrument — only run_async is instrumented. -assert not _marker(Runner.run, _RunnerRunAsyncSubPatcher) - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_agentscope.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_agentscope.py deleted file mode 100644 index 0347aeada..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_agentscope.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Test auto_instrument for AgentScope.""" -# pylint: disable=import-error,no-name-in-module,no-value-for-parameter,no-member - -import asyncio -import importlib - -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context - - -results = auto_instrument(openai=False) -assert results.get("agentscope") == True, "auto_instrument should return True for agentscope" - -results2 = auto_instrument(openai=False) -assert results2.get("agentscope") == True, "auto_instrument should still return True on second call" - -agent_module = importlib.import_module("agentscope.agent") -HAS_AGENT_REPLY_API = hasattr(agent_module, "Agent") - -if HAS_AGENT_REPLY_API: - from agentscope.agent import Agent - from agentscope.credential import OpenAICredential - from agentscope.message import UserMsg - from agentscope.model import OpenAIChatModel - from agentscope.tool import Toolkit - - assert hasattr(Agent.reply, "__wrapped__"), "Agent.reply should be wrapped" - assert hasattr(Toolkit.call_tool, "__wrapped__"), "Toolkit.call_tool should be wrapped" - assert hasattr(OpenAIChatModel.__call__, "__wrapped__"), "OpenAIChatModel.__call__ should be wrapped" - - model = OpenAIChatModel( - credential=OpenAICredential(api_key="test-api-key"), - model="gpt-4o-mini", - parameters=OpenAIChatModel.Parameters(temperature=0), - stream=False, - max_retries=0, - ) - agent = Agent( - name="Test Agent", - system_prompt="You are a helpful assistant. Be brief.", - model=model, - toolkit=Toolkit(), - ) - message = UserMsg("user", "Say hello in exactly two words.") -else: - from agentscope.agent import AgentBase, ReActAgent - from agentscope.formatter import OpenAIChatFormatter - from agentscope.memory import InMemoryMemory - from agentscope.message import Msg - from agentscope.model import OpenAIChatModel - from agentscope.pipeline import sequential_pipeline - from agentscope.tool import Toolkit - - try: - from agentscope.pipeline import fanout_pipeline - except ImportError: - fanout_pipeline = None - - assert hasattr(AgentBase.__call__, "__wrapped__"), "AgentBase.__call__ should be wrapped" - assert hasattr(sequential_pipeline, "__wrapped__"), "sequential_pipeline should be wrapped" - if fanout_pipeline is not None: - assert hasattr(fanout_pipeline, "__wrapped__"), "fanout_pipeline should be wrapped" - assert hasattr(Toolkit.call_tool_function, "__wrapped__"), "Toolkit.call_tool_function should be wrapped" - assert hasattr(OpenAIChatModel.__call__, "__wrapped__"), "OpenAIChatModel.__call__ should be wrapped" - - agent = ReActAgent( - name="Test Agent", - sys_prompt="You are a helpful assistant. Be brief.", - model=OpenAIChatModel( - model_name="gpt-4o-mini", - generate_kwargs={"temperature": 0}, - ), - formatter=OpenAIChatFormatter(), - toolkit=Toolkit(), - memory=InMemoryMemory(), - ) - message = Msg( - name="user", - content="Say hello in exactly two words.", - role="user", - ) - -if hasattr(agent, "set_console_output_enabled"): - agent.set_console_output_enabled(False) -elif hasattr(agent, "disable_console_output"): - agent.disable_console_output() - -with autoinstrument_test_context("test_auto_agentscope", integration="agentscope") as memory_logger: - result = asyncio.run(agent.reply(message) if HAS_AGENT_REPLY_API else agent(message)) - assert result is not None - - spans = memory_logger.pop() - assert len(spans) >= 2, f"Expected at least 2 spans (agent + model), got {len(spans)}" - - agent_span = next(span for span in spans if span["span_attributes"]["name"] == "Test Agent.reply") - llm_spans = [span for span in spans if span["span_attributes"]["type"].value == "llm"] - - assert agent_span["span_attributes"]["type"].value == "task" - assert llm_spans, "Should have at least one LLM span" - assert llm_spans[0]["metadata"]["model"] == "gpt-4o-mini" - assert llm_spans[0]["metadata"]["provider"] == "openai" - assert agent_span["span_id"] in llm_spans[0]["span_parents"] - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_agno.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_agno.py deleted file mode 100644 index 7cdfa01a3..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_agno.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Test auto_instrument for Agno (no uninstrument available).""" - -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context - - -# 1. Instrument -results = auto_instrument() -assert results.get("agno") == True, "auto_instrument should return True for agno" - -# 2. Idempotent -results2 = auto_instrument() -assert results2.get("agno") == True, "auto_instrument should still return True on second call" - -# 3. Verify methods are wrapped -from agno.agent import Agent -from agno.models.base import Model -from agno.team import Team -from agno.tools.function import FunctionCall - - -def check_wrapped(klass, private_method, public_method, required=True): - """Check if at least one method (private or public) is wrapped.""" - wrapped = False - if private_method and hasattr(klass, private_method): - if hasattr(getattr(klass, private_method), "__wrapped__"): - wrapped = True - if not wrapped and public_method and hasattr(klass, public_method): - if hasattr(getattr(klass, public_method), "__wrapped__"): - wrapped = True - - if required: - assert wrapped, f"{klass.__name__} should have {private_method or public_method} wrapped" - # If not required and nothing is wrapped, that's okay (method doesn't exist in this version) - - -# Agent methods -check_wrapped(Agent, "_run", "run", required=True) -check_wrapped(Agent, "_arun", "arun", required=True) -check_wrapped(Agent, "_run_stream", None, required=False) # Optional - only in 2.4.0 -check_wrapped(Agent, "_arun_stream", None, required=False) # Optional - only in 2.4.0 - -# Team methods -check_wrapped(Team, "_run", "run", required=True) -check_wrapped(Team, "_arun", "arun", required=True) -check_wrapped(Team, "_run_stream", None, required=False) -check_wrapped(Team, "_arun_stream", None, required=False) - -# Model methods (all public, all required) -assert hasattr(Model, "invoke") and hasattr(Model.invoke, "__wrapped__"), "Model.invoke should be wrapped" -assert hasattr(Model, "ainvoke") and hasattr(Model.ainvoke, "__wrapped__"), "Model.ainvoke should be wrapped" -assert hasattr(Model, "invoke_stream") and hasattr(Model.invoke_stream, "__wrapped__"), ( - "Model.invoke_stream should be wrapped" -) -assert hasattr(Model, "ainvoke_stream") and hasattr(Model.ainvoke_stream, "__wrapped__"), ( - "Model.ainvoke_stream should be wrapped" -) -assert hasattr(Model, "response") and hasattr(Model.response, "__wrapped__"), "Model.response should be wrapped" -assert hasattr(Model, "aresponse") and hasattr(Model.aresponse, "__wrapped__"), "Model.aresponse should be wrapped" -assert hasattr(Model, "response_stream") and hasattr(Model.response_stream, "__wrapped__"), ( - "Model.response_stream should be wrapped" -) -assert hasattr(Model, "aresponse_stream") and hasattr(Model.aresponse_stream, "__wrapped__"), ( - "Model.aresponse_stream should be wrapped" -) - -# FunctionCall methods (all public, all required) -assert hasattr(FunctionCall, "execute") and hasattr(FunctionCall.execute, "__wrapped__"), ( - "FunctionCall.execute should be wrapped" -) -assert hasattr(FunctionCall, "aexecute") and hasattr(FunctionCall.aexecute, "__wrapped__"), ( - "FunctionCall.aexecute should be wrapped" -) - -# Eval classes (agno.eval). These live in submodules the eval package imports lazily, -# so this also verifies auto_instrument imports and patches them in a fresh process. -# agent_as_judge (agno >= 2.4) and suite (agno >= 2.9) are absent in older versions. -import importlib - - -def check_eval_targets_wrapped(module_path, targets): - try: - module = importlib.import_module(module_path) - except ImportError: - print(f"{module_path} not present in this agno version, skipped") - return - for target in targets: - obj = module - for part in target.split("."): - obj = getattr(obj, part) - assert hasattr(obj, "__wrapped__"), f"{module_path}.{target} should be wrapped" - print(f"{module_path} wrapped: {', '.join(targets)}") - - -check_eval_targets_wrapped( - "agno.eval.accuracy", - ["AccuracyEval.run", "AccuracyEval.arun", "AccuracyEval.evaluate_answer", "AccuracyEval.aevaluate_answer"], -) -check_eval_targets_wrapped( - "agno.eval.agent_as_judge", - [ - "AgentAsJudgeEval.run", - "AgentAsJudgeEval.arun", - "AgentAsJudgeEval.post_check", - "AgentAsJudgeEval.async_post_check", - ], -) -check_eval_targets_wrapped("agno.eval.reliability", ["ReliabilityEval.run", "ReliabilityEval.arun"]) -check_eval_targets_wrapped("agno.eval.performance", ["PerformanceEval.run", "PerformanceEval.arun"]) -check_eval_targets_wrapped("agno.eval.suite", ["arun_cases", "_arun_case"]) - -# 4. Make API call and verify spans -with autoinstrument_test_context("test_auto_agno", integration="agno") as memory_logger: - from agno.models.openai import OpenAIChat - - agent = Agent( - name="Test Agent", - model=OpenAIChat(id="gpt-4o-mini"), - instructions="You are a helpful assistant. Be brief.", - ) - - response = agent.run("Say hi") - assert response, "Agent should return a response" - assert response.content, "Response should have content" - - spans = memory_logger.pop() - assert len(spans) >= 2, f"Expected at least 2 spans (agent + model), got {len(spans)}" - - # Verify we have an agent span (type: task) - agent_spans = [s for s in spans if "Test Agent" in s.get("span_attributes", {}).get("name", "")] - assert len(agent_spans) >= 1, "Should have at least one agent span" - - # Verify agent span is type TASK - agent_span = agent_spans[0] - assert agent_span.get("span_attributes", {}).get("type", {}).value == "task", "Agent span should be type 'task'" - - # Verify we have a model span (type: llm) - llm_spans = [s for s in spans if s.get("span_attributes", {}).get("type", {}).value == "llm"] - assert len(llm_spans) >= 1, f"Should have at least one LLM span, got {len(llm_spans)}" - - # Verify model span has expected attributes - llm_span = llm_spans[0] - assert "OpenAI" in llm_span.get("span_attributes", {}).get("name", ""), "LLM span should contain 'OpenAI'" - assert llm_span.get("metadata", {}).get("provider") == "OpenAI", "LLM span should have OpenAI provider" - - # Verify span hierarchy - LLM span should be child of agent span - llm_parents = llm_span.get("span_parents", []) - agent_span_id = agent_span.get("span_id") - assert agent_span_id in llm_parents, ( - f"LLM span should be child of agent span. Agent ID: {agent_span_id}, LLM parents: {llm_parents}" - ) - - print("Agent span created (type: task)") - print("Model span created (type: llm)") - print("Span hierarchy verified (model is child of agent)") - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py deleted file mode 100644 index 1b69fab5b..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Verify Vercel AI SDK instrumentation when ai is imported first.""" -# pylint: disable=import-error,no-member - -import asyncio - -import ai -from braintrust.integrations.test_utils import run_auto_smoke - - -def _call(memory_logger): - async def drive(): - async with ai.stream( - ai.get_model("openai:gpt-4o-mini"), - [ai.user_message("Reply with the single word hello.")], - ) as stream: - async for _ in stream: - pass - - asyncio.run(drive()) - - spans = memory_logger.pop() - assert len(spans) == 2, f"Expected AI SDK and provider spans, got: {spans!r}" - ai_span = next(span for span in spans if span["span_attributes"]["name"] == "ai.stream") - provider_span = next(span for span in spans if span["span_attributes"].get("type") == "llm") - assert ai_span["span_attributes"]["type"] == "task" - for token_metric in ("tokens", "prompt_tokens", "completion_tokens"): - assert token_metric not in ai_span["metrics"] - assert provider_span["metrics"]["tokens"] > 0 - assert provider_span["metrics"]["completion_reasoning_tokens"] >= 0 - assert provider_span["metrics"]["prompt_cached_tokens"] >= 0 - assert ai_span["context"]["span_origin"]["instrumentation"]["name"] == "ai-sdk-auto" - assert provider_span["context"]["span_origin"]["instrumentation"]["name"] == "openai-auto" - - -run_auto_smoke("ai_sdk", integration="ai_sdk", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py deleted file mode 100644 index 3905e9c16..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_anthropic.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Test auto_instrument for Anthropic.""" - -import os - -import anthropic -from braintrust.integrations.test_utils import run_auto_smoke - - -_TRACING_MODULE = "braintrust.integrations.anthropic.tracing" - - -def _is_patched() -> bool: - return ( - type(anthropic.Anthropic(api_key="test-key").messages).__module__ == _TRACING_MODULE - and type(anthropic.AsyncAnthropic(api_key="test-key").messages).__module__ == _TRACING_MODULE - ) - - -def _call(memory_logger): - model = ( - "claude-haiku-4-5-20251001" - if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") == "latest" - else "claude-3-haiku-20240307" - ) - client = anthropic.Anthropic() - response = client.messages.create( - model=model, - max_tokens=100, - messages=[{"role": "user", "content": "Say hi"}], - ) - assert response.content[0].text - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert span["metadata"]["provider"] == "anthropic" - assert "claude" in span["metadata"]["model"] - - -run_auto_smoke("anthropic", is_patched=_is_patched, integration="anthropic", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py deleted file mode 100644 index 9fd6f1f7f..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_autogen.py +++ /dev/null @@ -1,30 +0,0 @@ -from autogen_agentchat.agents import AssistantAgent, BaseChatAgent -from autogen_agentchat.teams import BaseGroupChat -from autogen_core.tools import FunctionTool -from braintrust.integrations.test_utils import run_auto_smoke - - -_PATCHED_MARKERS = { - BaseChatAgent.run: "__braintrust_patched_autogen_chat_agent_run__", - AssistantAgent.on_messages_stream: "__braintrust_patched_autogen_chat_agent_assistant_on_messages_stream__", - BaseGroupChat.run: "__braintrust_patched_autogen_team_run__", - FunctionTool.run: "__braintrust_patched_autogen_function_tool_run__", -} - - -def _is_patched() -> bool: - return all(getattr(target, marker, False) for target, marker in _PATCHED_MARKERS.items()) - - -run_auto_smoke("autogen", is_patched=_is_patched) - -# Additional marker checks not covered by the shared runner. -assert getattr(BaseChatAgent.run_stream, "__braintrust_patched_autogen_chat_agent_run_stream__", False) -assert getattr(BaseChatAgent.on_messages, "__braintrust_patched_autogen_chat_agent_base_on_messages__", False) -assert getattr( - BaseChatAgent.on_messages_stream, "__braintrust_patched_autogen_chat_agent_base_on_messages_stream__", False -) -assert getattr(AssistantAgent.on_messages, "__braintrust_patched_autogen_chat_agent_assistant_on_messages__", False) -assert getattr(BaseGroupChat.run_stream, "__braintrust_patched_autogen_team_run_stream__", False) - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py deleted file mode 100644 index 60e0a7b00..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_bedrock_runtime.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Test auto_instrument for boto3 Bedrock Runtime.""" - -import os - - -os.environ.setdefault("AWS_EC2_METADATA_DISABLED", "true") -if not os.environ.get("AWS_PROFILE") and not os.environ.get("AWS_BEARER_TOKEN_BEDROCK"): - os.environ.setdefault("AWS_ACCESS_KEY_ID", "testing") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "testing") - os.environ.setdefault("AWS_SESSION_TOKEN", "testing") - -import boto3 -from braintrust.integrations.test_utils import run_auto_smoke - - -MODEL = os.getenv("BRAINTRUST_BEDROCK_CONVERSE_MODEL", "us.amazon.nova-lite-v1:0") -REGION = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-east-1" - - -def _call(memory_logger): - client = boto3.client("bedrock-runtime", region_name=REGION) - response = client.converse( - modelId=MODEL, - messages=[{"role": "user", "content": [{"text": "Say hello in one word."}]}], - inferenceConfig={"maxTokens": 20, "temperature": 0}, - ) - assert response["output"]["message"]["role"] == "assistant" - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert span["metadata"]["provider"] == "bedrock" - assert span["metadata"]["model"] == MODEL - assert span["metadata"]["endpoint"] == "converse" - assert span["span_attributes"]["name"] == "bedrock.converse" - - -run_auto_smoke("bedrock_runtime", integration="bedrock_runtime", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py deleted file mode 100644 index b8a6e5271..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_claude_agent_sdk.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Test auto_instrument for Claude Agent SDK (no uninstrument available).""" - -import asyncio - -from braintrust.integrations.claude_agent_sdk._test_transport import make_cassette_transport -from braintrust.integrations.test_utils import run_auto_smoke - - -def _call(memory_logger): - import claude_agent_sdk # pylint: disable=import-error - - options = claude_agent_sdk.ClaudeAgentOptions( - model="claude-3-5-haiku-20241022", - permission_mode="bypassPermissions", - ) - transport = make_cassette_transport( - cassette_name="test_auto_claude_agent_sdk", - prompt="", - options=options, - ) - - async def run_agent(): - async with claude_agent_sdk.ClaudeSDKClient(options=options, transport=transport) as client: - await client.query("Say hi") - async for message in client.receive_response(): - if type(message).__name__ == "ResultMessage": - return message - return None - - result = asyncio.run(run_agent()) - assert result is not None - - spans = memory_logger.pop() - assert len(spans) >= 1, f"Expected at least 1 span, got {len(spans)}" - - -run_auto_smoke("claude_agent_sdk", use_vcr=False, run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py deleted file mode 100644 index 46c2a963b..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cohere.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Test auto_instrument for Cohere.""" - -import os - - -os.environ.setdefault("CO_API_KEY", "co-test-dummy-api-key-for-vcr-tests") -os.environ.setdefault("COHERE_API_KEY", os.environ["CO_API_KEY"]) - -import cohere -from braintrust.integrations.test_utils import run_auto_smoke - - -def _call(memory_logger): - use_v2 = hasattr(cohere, "ClientV2") and hasattr(cohere.ClientV2, "chat") - if use_v2: - client = cohere.ClientV2(api_key=os.environ["CO_API_KEY"]) - response = client.chat( - model="command-a-03-2025", - messages=[{"role": "user", "content": "Say hi in one word."}], - max_tokens=10, - ) - assert response.message.role == "assistant" - else: - client = cohere.Client(api_key=os.environ["CO_API_KEY"]) - response = client.chat( - model="command-a-03-2025", - message="Say hi in one word.", - max_tokens=10, - ) - assert isinstance(response.text, str) - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert span["metadata"]["provider"] == "cohere" - assert span["metadata"]["model"] == "command-a-03-2025" - assert span["span_attributes"]["name"] == "cohere.chat" - - -run_auto_smoke("cohere", integration="cohere", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py deleted file mode 100644 index e936c85a0..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_crewai.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Test auto_instrument for CrewAI. - -Verifies that ``auto_instrument(crewai=True)`` registers the Braintrust -CrewAI listener on ``crewai_event_bus`` and is idempotent across repeated -calls. Full span-shape coverage lives in ``test_crewai.py``. -""" - -# pylint: disable=import-error - -from braintrust.integrations.crewai import BraintrustCrewAIListener -from braintrust.integrations.crewai.patchers import _get_registered_listener -from braintrust.integrations.test_utils import run_auto_smoke - - -def _is_patched() -> bool: - listener = _get_registered_listener() - return isinstance(listener, BraintrustCrewAIListener) - - -run_auto_smoke("crewai", is_patched=_is_patched) - -# Listener stays the same across the two auto_instrument calls. -listener = _get_registered_listener() -assert listener is not None -assert isinstance(listener, BraintrustCrewAIListener) - -# Listener is actually subscribed on the CrewAI event bus. -from crewai.events import CrewKickoffStartedEvent -from crewai.events.event_bus import crewai_event_bus - - -sync_handlers = crewai_event_bus._sync_handlers.get(CrewKickoffStartedEvent, frozenset()) -assert sync_handlers, "Expected at least one sync handler registered for CrewKickoffStartedEvent" - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py deleted file mode 100644 index 9fd491d21..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_cursor_sdk.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Subprocess coverage for Cursor SDK auto-instrumentation and import order.""" - -# pylint: disable=import-error - -import os -import tempfile -from pathlib import Path - -import cursor_sdk -from braintrust.integrations.cursor_sdk._test_vcr import cursor_vcr_config -from braintrust.integrations.test_utils import run_auto_smoke -from braintrust.span_types import SpanTypeAttribute -from braintrust.test_helpers import find_spans_by_type - - -def _call(memory_logger): - with tempfile.TemporaryDirectory() as workspace: - Path(workspace, "README.md").write_text("Cursor auto-instrumentation workspace.\n", encoding="utf-8") - with cursor_sdk.CursorClient.launch_bridge(workspace=workspace) as client: - with client.agents.create( - model="composer-2.5", - api_key=os.environ.get("CURSOR_API_KEY", "crsr_test_key_for_cassette_playback"), - local=cursor_sdk.LocalAgentOptions(cwd=workspace), - ) as agent: - result = agent.send("Reply with exactly: cursor tracing complete").wait() - - assert result.status == "finished" - spans = memory_logger.pop() - assert find_spans_by_type(spans, SpanTypeAttribute.TASK) - assert find_spans_by_type(spans, SpanTypeAttribute.LLM) - assert all(span["context"]["span_origin"]["instrumentation"]["name"] == "cursor-sdk-auto" for span in spans) - - -run_auto_smoke( - "cursor_sdk", - integration="cursor_sdk", - vcr_config=cursor_vcr_config(), - run=_call, -) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py deleted file mode 100644 index 3bc0c0b2a..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Test auto_instrument for DSPy. - -Note: This test focuses on patching behavior only. Span verification for DSPy -is done in test_dspy.py::test_dspy_callback which uses pytest-vcr (supports httpx). -The standalone VCR in test_utils doesn't capture httpx used by litellm/dspy. -""" - -import dspy -from braintrust.integrations.dspy import BraintrustDSpyCallback -from braintrust.integrations.test_utils import run_auto_smoke - - -def _is_patched() -> bool: - return bool(getattr(dspy.configure, "__braintrust_patched_dspy_configure__", False)) - - -run_auto_smoke("dspy", is_patched=_is_patched) - -# Verify callback is added when configure() is called. -dspy.configure(lm=None) -from dspy.dsp.utils.settings import settings - - -has_bt_callback = any(isinstance(cb, BraintrustDSpyCallback) for cb in settings.callbacks) -assert has_bt_callback, "Expected BraintrustDSpyCallback in callbacks after configure()" - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py deleted file mode 100644 index 66df92577..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_genai.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Test auto_instrument for Google GenAI (no uninstrument available).""" - -import os - -from braintrust.integrations.test_utils import run_auto_smoke - - -def _call(memory_logger): - from google.genai import types - from google.genai.client import Client - - client = Client() - response = client.models.generate_content( - model=( - "gemini-2.5-flash-lite" - if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") == "latest" - else "gemini-2.0-flash-001" - ), - contents="Say hi", - config=types.GenerateContentConfig(max_output_tokens=100), - ) - assert response.text - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert "gemini" in span["metadata"]["model"] - - -run_auto_smoke("google_genai", integration="google_genai", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py deleted file mode 100644 index 277c91401..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_huggingface_hub.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Test auto_instrument for HuggingFace Hub.""" - -import os - - -# Dummy token must start with ``hf_`` so the HuggingFace SDK accepts it for -# ``provider="auto"`` routing (validated locally before any HTTP request). -os.environ.setdefault("HF_TOKEN", "hf_test_dummy_api_key_for_vcr_tests") - -from braintrust.integrations.test_utils import run_auto_smoke -from huggingface_hub import InferenceClient - - -CHAT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" - - -def _call(memory_logger): - # ``provider="cerebras"`` hosts ``meta-llama/Llama-3.1-8B-Instruct`` across - # the matrix; ``hf-inference`` no longer hosts most conversational checkpoints. - client = InferenceClient(model=CHAT_MODEL, provider="cerebras", token=os.environ["HF_TOKEN"]) - response = client.chat_completion( - messages=[{"role": "user", "content": "Say hi in one word."}], - max_tokens=10, - ) - assert response.choices - assert response.choices[0].message.role == "assistant" - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - # User-supplied ``provider`` overrides the default "huggingface" identity. - assert span["metadata"]["provider"] == "cerebras" - assert span["span_attributes"]["name"] == "huggingface.chat_completion" - - -run_auto_smoke("huggingface_hub", integration="huggingface_hub", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py deleted file mode 100644 index 4d1bb5ee5..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_instructor.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Test auto_instrument for Instructor.""" - -import instructor -import openai -from braintrust.integrations.test_utils import run_auto_smoke -from pydantic import BaseModel - - -class Person(BaseModel): - name: str - age: int - - -def _call(memory_logger): - # Drive a real instructor.from_openai call against a recorded cassette and - # verify a parent task-typed Instructor span shows up alongside the OpenAI - # llm child span. Cassette is shared with the in-process test suite under - # integrations/instructor/cassettes//. - client = openai.OpenAI(api_key="sk-test-dummy-api-key-for-vcr-tests") - patched = instructor.from_openai(client, mode=instructor.Mode.TOOLS) - result = patched.chat.completions.create( - model="gpt-4o-mini", - response_model=Person, - max_retries=3, - messages=[{"role": "user", "content": "Extract Grace, age 45."}], - ) - assert isinstance(result, Person) - assert result.model_dump() == {"name": "Grace", "age": 45} - - raw = memory_logger.pop() - spans = [] - for s in raw: - if isinstance(s, list): - spans.extend(s) - else: - spans.append(s) - types = [s["span_attributes"].get("type") for s in spans] - assert "task" in types, f"missing instructor parent (task) span: {types}" - assert "llm" in types, f"missing openai child (llm) span: {types}" - - -run_auto_smoke( - "instructor", - cassette="TestInstructorOpenAISpans.test_instructor_openai_single_success", - integration="instructor", - run=_call, -) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py deleted file mode 100644 index 3583ec15c..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_langchain.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Test auto_instrument for LangChain.""" - -from braintrust.integrations.langchain import BraintrustCallbackHandler -from braintrust.integrations.langchain.context import clear_global_handler, get_global_handler -from braintrust.integrations.test_utils import run_auto_smoke -from langchain_core.callbacks import CallbackManager -from langchain_core.prompts import ChatPromptTemplate -from langchain_openai import ChatOpenAI - - -# Ensure a clean starting state so the pre-check reflects a fresh process. -clear_global_handler() -manager = CallbackManager.configure() -assert next((h for h in manager.handlers if isinstance(h, BraintrustCallbackHandler)), None) is None - - -def _is_patched() -> bool: - return isinstance(get_global_handler(), BraintrustCallbackHandler) - - -def _call(memory_logger): - prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") - model = ChatOpenAI( - model="gpt-4o-mini", - temperature=1, - top_p=1, - frequency_penalty=0, - presence_penalty=0, - n=1, - ) - chain = prompt.pipe(model) - - message = chain.invoke({"number": "2"}) - assert message.content == "1 + 2 equals 3." - - spans = memory_logger.pop() - assert len(spans) > 0 - - -run_auto_smoke( - "langchain", - is_patched=_is_patched, - cassette="test_global_handler", - integration="langchain", - run=_call, -) - -# The handler installed via auto_instrument must also flow through CallbackManager.configure(). -handler = get_global_handler() -manager = CallbackManager.configure() -assert next((h for h in manager.handlers if isinstance(h, BraintrustCallbackHandler)), None) is handler - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py deleted file mode 100644 index d45951789..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Test auto_instrument for LiteLLM.""" - -import litellm -from braintrust.integrations.litellm import LiteLLMIntegration -from braintrust.integrations.test_utils import run_auto_smoke - - -def _is_patched() -> bool: - return LiteLLMIntegration.patchers[0].is_patched(litellm, None) - - -def _call(memory_logger): - response = litellm.completion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "Say hi"}], - ) - assert response.choices[0].message.content - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert span["metadata"]["provider"] == "openai" - - -# Disable OpenAI auto-instrumentation here because LiteLLM's OpenAI-backed -# chat path can otherwise produce both a LiteLLM span and an OpenAI span. -# This test is meant to validate LiteLLM instrumentation in isolation. -run_auto_smoke( - "litellm", - auto_instrument_kwargs={"openai": False}, - is_patched=_is_patched, - integration="litellm", - run=_call, -) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py deleted file mode 100644 index 9ee45941a..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Test auto_instrument for LiveKit Agents.""" - -import inspect - -from braintrust.integrations.test_utils import run_auto_smoke - -# Import the provider classes before auto-instrumentation to verify setup handles -# normal user import order in a fresh process. -from livekit.agents import AgentSession # noqa: E402 -from livekit.agents.inference.llm import LLMStream # noqa: E402 -from livekit.agents.stt import STT # noqa: E402 -from livekit.agents.tts import TTS # noqa: E402 -from livekit.agents.voice import generation # noqa: E402 -from livekit.agents.voice.io import AudioOutput # noqa: E402 -from wrapt import FunctionWrapper - - -def _attr_wrapped(target, attr: str) -> bool: - return isinstance(inspect.getattr_static(target, attr, None), FunctionWrapper) - - -_WRAP_TARGETS = ( - (AgentSession, "run"), - (AgentSession, "_on_audio_output_changed"), - (AgentSession, "_update_user_state"), - (LLMStream, "_run"), - (STT, "recognize"), - (TTS, "synthesize"), - (AudioOutput, "capture_frame"), -) - - -def _is_patched() -> bool: - return all(_attr_wrapped(target, attr) for target, attr in _WRAP_TARGETS) and isinstance( - generation._execute_tools_task, FunctionWrapper - ) - - -run_auto_smoke("livekit_agents", is_patched=_is_patched) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py deleted file mode 100644 index 39ea59a80..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_mistral.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Test auto_instrument for Mistral.""" - -import os - -from braintrust.integrations.test_utils import run_auto_smoke - - -try: - from mistralai.client import Mistral -except ImportError: - from mistralai import Mistral - - -def _call(memory_logger): - client = Mistral(api_key=os.environ.get("MISTRAL_API_KEY")) - response = client.chat.complete( - model="mistral-small-latest", - messages=[{"role": "user", "content": "What is 2+2? Reply with just the number."}], - max_tokens=10, - ) - assert "4" in str(response.choices[0].message.content) - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert span["metadata"]["provider"] == "mistral" - assert span["metadata"]["model"] == "mistral-small-latest" - assert "4" in str(span["output"]) - - -run_auto_smoke("mistral", integration="mistral", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py deleted file mode 100644 index 2d98e0674..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Test auto_instrument for OpenAI.""" - -import inspect - -import openai -from braintrust.integrations.test_utils import run_auto_smoke -from wrapt import FunctionWrapper - - -def _is_patched() -> bool: - attr = inspect.getattr_static(openai.resources.chat.completions.Completions, "create", None) - return isinstance(attr, FunctionWrapper) - - -def _call(memory_logger): - client = openai.OpenAI() - response = client.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "Say hi"}], - ) - assert response.choices[0].message.content - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert span["metadata"]["provider"] == "openai" - assert "gpt-4o-mini" in span["metadata"]["model"] - - -run_auto_smoke("openai", is_patched=_is_patched, integration="openai", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py deleted file mode 100644 index 230c43333..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai_agents.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Test auto_instrument for the OpenAI Agents SDK.""" - -import asyncio - -import agents -from braintrust.integrations.openai_agents import BraintrustTracingProcessor -from braintrust.integrations.test_utils import run_auto_smoke - - -TEST_MODEL = "gpt-4o-mini" -TEST_PROMPT = "What is 2+2? Just the number." -TEST_AGENT_INSTRUCTIONS = "You are a helpful assistant. Be very concise." - - -def _is_patched() -> bool: - provider = agents.tracing.get_trace_provider() - processors = getattr(getattr(provider, "_multi_processor", None), "_processors", ()) - return any(isinstance(processor, BraintrustTracingProcessor) for processor in processors) - - -def _call(memory_logger): - from agents import Agent - from agents.run import AgentRunner - - async def run_agent(): - agent = Agent(name="test-agent", model=TEST_MODEL, instructions=TEST_AGENT_INSTRUCTIONS) - return await AgentRunner().run(agent, TEST_PROMPT) - - result = asyncio.run(run_agent()) - assert result is not None - assert hasattr(result, "final_output") or hasattr(result, "output") - - spans = memory_logger.pop() - assert len(spans) >= 2, f"Expected at least 2 spans, got {len(spans)}" - - -run_auto_smoke("openai_agents", is_patched=_is_patched, integration="openai_agents", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py deleted file mode 100644 index d99c3ebfb..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Test auto_instrument for OpenRouter.""" - -import os - -import openrouter -from braintrust.integrations.test_utils import run_auto_smoke - - -def _call(memory_logger): - client = openrouter.OpenRouter(api_key=os.environ.get("OPENROUTER_API_KEY")) - response = client.chat.send( - model="openai/gpt-4o-mini", - messages=[{"role": "user", "content": "What is 2+2? Reply with just the number."}], - max_tokens=10, - ) - assert "4" in response.choices[0].message.content - - spans = memory_logger.pop() - assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" - span = spans[0] - assert span["metadata"]["provider"] == "openai" - assert span["metadata"]["model"] == "gpt-4o-mini" - assert "4" in span["output"][0]["message"]["content"] - - -run_auto_smoke("openrouter", integration="openrouter", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py deleted file mode 100644 index e32d8d541..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py +++ /dev/null @@ -1,102 +0,0 @@ -import asyncio -import importlib -import inspect -import os -import tempfile -from pathlib import Path - -from braintrust.auto import auto_instrument -from braintrust.integrations.test_utils import autoinstrument_test_context - - -def _ensure_nltk_punkt_tab(): - data_dir = Path(tempfile.gettempdir()) / "braintrust-pipecat-nltk-data" - punkt_tab = data_dir / "tokenizers" / "punkt_tab" - punkt_tab.mkdir(parents=True, exist_ok=True) - os.environ.setdefault("NLTK_DATA", str(data_dir)) - - -def _import(path): - _ensure_nltk_punkt_tab() - module_name, attr = path.rsplit(".", 1) - return getattr(importlib.import_module(module_name), attr) - - -def _worker_kwargs(**overrides): - PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") - signature = inspect.signature(PipelineWorker) - kwargs = {"idle_timeout_secs": None} - for name, value in { - "enable_turn_tracking": False, - "enable_rtvi": False, - "check_dangling_tasks": False, - }.items(): - if name in signature.parameters: - kwargs[name] = value - kwargs.update(overrides) - return kwargs - - -def _runner_kwargs(**overrides): - WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") - signature = inspect.signature(WorkerRunner) - kwargs = {"handle_sigint": False} - if "check_dangling_tasks" in signature.parameters: - kwargs["check_dangling_tasks"] = False - kwargs.update(overrides) - return kwargs - - -async def main(): - with autoinstrument_test_context("test_auto_pipecat", integration="pipecat") as memory_logger: - _ensure_nltk_punkt_tab() - results = auto_instrument() - assert results.get("pipecat") is True - - EndFrame = _import("pipecat.frames.frames.EndFrame") - LLMContextFrame = _import("pipecat.frames.frames.LLMContextFrame") - Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") - PipelineParams = _import("pipecat.pipeline.worker.PipelineParams") - PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") - LLMContext = _import("pipecat.processors.aggregators.llm_context.LLMContext") - OpenAILLMService = _import("pipecat.services.openai.llm.OpenAILLMService") - WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") - - llm = OpenAILLMService( - api_key=os.environ["OPENAI_API_KEY"], - settings=OpenAILLMService.Settings( - model="gpt-4o-mini", - temperature=0.0, - max_completion_tokens=20, - ), - ) - worker = PipelineWorker( - Pipeline([llm]), - **_worker_kwargs( - name="bt-auto-pipecat-worker", - params=PipelineParams(enable_metrics=True, enable_usage_metrics=True), - ), - ) - context = LLMContext( - messages=[ - {"role": "developer", "content": "Answer with exactly the requested text and no punctuation."}, - {"role": "user", "content": "Say: braintrust auto pipecat"}, - ] - ) - - @worker.event_handler("on_pipeline_started") - async def on_pipeline_started(_worker, _frame): - await worker.queue_frames([LLMContextFrame(context), EndFrame()]) - - runner = WorkerRunner(**_runner_kwargs()) - await runner.add_workers(worker) - await asyncio.wait_for(runner.run(), timeout=20) - - logs = memory_logger.pop() - names = {log.get("span_attributes", {}).get("name") for log in logs} - assert "pipecat_pipeline" in names - assert "pipecat_llm_response" in names - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py deleted file mode 100644 index 1e770edaf..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Test auto_instrument for Pydantic AI (no uninstrument available).""" - -import asyncio - -from braintrust.integrations.test_utils import run_auto_smoke - - -def _call(memory_logger): - from pydantic_ai import Agent - from pydantic_ai.models.openai import OpenAIChatModel - from pydantic_ai.settings import ModelSettings - - agent = Agent( - OpenAIChatModel("gpt-4o-mini"), - model_settings=ModelSettings(max_tokens=100), - ) - - result = asyncio.run(agent.run("Say hi")) - assert result.output - - spans = memory_logger.pop() - assert len(spans) >= 1, f"Expected at least 1 span, got {len(spans)}" - agent_spans = [s for s in spans if "agent_run" in s["span_attributes"]["name"]] - assert len(agent_spans) >= 1, f"Expected agent_run span, got {[s['span_attributes']['name'] for s in spans]}" - - -run_auto_smoke("pydantic_ai", integration="pydantic_ai", run=_call) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py deleted file mode 100644 index 020215443..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_temporal.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Test auto_instrument for Temporal.""" - -from braintrust.integrations.temporal import BraintrustPlugin, setup_temporal -from braintrust.integrations.test_utils import run_auto_smoke - - -run_auto_smoke("temporal") - -assert setup_temporal() == True -assert BraintrustPlugin is not None - -print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py deleted file mode 100644 index b98a673a7..000000000 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_transformers.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Subprocess coverage for Transformers auto-instrumentation.""" - -# Keep the large Transformers/PyTorch dependencies isolated to their nox job. -# pylint: disable=import-error - -from braintrust.integrations.test_utils import run_auto_smoke - - -def _call(memory_logger): - from transformers import pipeline - - generator = pipeline( - "text-generation", - model="hf-internal-testing/tiny-random-LlamaForCausalLM", - device=-1, - ) - response = generator("Hello", do_sample=False, max_new_tokens=1) - assert response - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span["span_attributes"]["name"] == "huggingface.transformers.text_generation" - assert span["context"]["span_origin"]["instrumentation"]["name"] == "transformers-auto" - - -run_auto_smoke( - "transformers", - auto_instrument_kwargs={"transformers": True}, - use_vcr=False, - run=_call, -) -print("SUCCESS") diff --git a/py/src/braintrust/integrations/autogen/test_autogen.py b/py/src/braintrust/integrations/autogen/test_autogen.py index 706301509..ace933a8f 100644 --- a/py/src/braintrust/integrations/autogen/test_autogen.py +++ b/py/src/braintrust/integrations/autogen/test_autogen.py @@ -1,7 +1,7 @@ import pytest from braintrust import logger from braintrust.integrations.autogen import setup_autogen -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger @@ -122,4 +122,4 @@ async def test_autogen_team_run_creates_team_span(memory_logger): def test_autogen_auto_instrument_subprocess(): - verify_autoinstrument_script("test_auto_autogen.py") + verify_autoinstrument_smoke("autogen") diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index ee873f6b9..35ebb3bcd 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -13,7 +13,7 @@ BedrockClientCreatorPatcher, BedrockRuntimeClientMethodsPatcher, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script +from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -435,4 +435,4 @@ def test_wrap_bedrock_converse_with_document_input_materializes_attachment(memor @pytest.mark.vcr def test_auto_instrument_bedrock_runtime_subprocess(): - verify_autoinstrument_script("test_auto_bedrock_runtime.py", timeout=60) + verify_autoinstrument_smoke("bedrock_runtime", timeout=60) diff --git a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py index 768954acf..45a4603c5 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py @@ -43,7 +43,7 @@ _serialize_tool_result_output, _thread_local, ) -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.logger import start_span from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger @@ -2548,7 +2548,7 @@ class TestAutoInstrumentClaudeAgentSDK: @pytest.mark.skipif(not CLAUDE_SDK_AVAILABLE, reason="Claude Agent SDK not installed") def test_auto_instrument_claude_agent_sdk(self): """Test auto_instrument patches Claude Agent SDK and creates spans.""" - verify_autoinstrument_script("test_auto_claude_agent_sdk.py") + verify_autoinstrument_smoke("claude_agent_sdk") @pytest.mark.skipif(not CLAUDE_SDK_AVAILABLE, reason="Claude Agent SDK not installed") diff --git a/py/src/braintrust/integrations/cohere/test_cohere.py b/py/src/braintrust/integrations/cohere/test_cohere.py index 0a99b639a..b33c21d1c 100644 --- a/py/src/braintrust/integrations/cohere/test_cohere.py +++ b/py/src/braintrust/integrations/cohere/test_cohere.py @@ -29,7 +29,7 @@ V2EmbedPatcher, V2RerankPatcher, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script +from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_spans_by_type, init_test_logger @@ -816,4 +816,4 @@ def test_cohere_integration_setup_patches_audio_transcriptions(memory_logger, cl class TestAutoInstrumentCohere: def test_auto_instrument_cohere(self): - verify_autoinstrument_script("test_auto_cohere.py") + verify_autoinstrument_smoke("cohere") diff --git a/py/src/braintrust/integrations/crewai/test_crewai.py b/py/src/braintrust/integrations/crewai/test_crewai.py index 50c798e56..ed16547e8 100644 --- a/py/src/braintrust/integrations/crewai/test_crewai.py +++ b/py/src/braintrust/integrations/crewai/test_crewai.py @@ -31,7 +31,7 @@ setup_crewai, ) from braintrust.integrations.crewai.patchers import _get_registered_listener, _reset_for_testing -from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_script +from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_smoke from braintrust.logger import Attachment, start_span from braintrust.test_helpers import init_test_logger from braintrust.util import LazyValue @@ -585,7 +585,7 @@ def test_setup_crewai_returns_true_under_active_logger(): class TestAutoInstrumentCrewAI: def test_auto_instrument_crewai(self): - verify_autoinstrument_script("test_auto_crewai.py") + verify_autoinstrument_smoke("crewai") def test_patch_crewai_subprocess(self): result = run_in_subprocess( diff --git a/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py b/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py index 040c1e289..41a65f695 100644 --- a/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py +++ b/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py @@ -13,7 +13,7 @@ from braintrust.integrations.cursor_sdk import CursorSDKIntegration, setup_cursor_sdk from braintrust.integrations.cursor_sdk._test_vcr import cursor_vcr_config from braintrust.integrations.openai import OpenAIIntegration -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_spans_by_type, init_test_logger @@ -76,7 +76,7 @@ def test_setup_cursor_sdk_is_idempotent(): @_REQUIRES_SYNC_BRIDGE @pytest.mark.vcr def test_auto_instrument_cursor_sdk_subprocess(): - verify_autoinstrument_script("test_auto_cursor_sdk.py", timeout=45) + verify_autoinstrument_smoke("cursor_sdk", timeout=45) @_REQUIRES_SYNC_BRIDGE diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index f76a246df..3954b5155 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -6,7 +6,7 @@ import pytest from braintrust import logger from braintrust.integrations.dspy import BraintrustDSpyCallback -from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_script +from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -236,4 +236,4 @@ class TestAutoInstrumentDSPy: def test_auto_instrument_dspy(self): """Test auto_instrument patches DSPy, creates spans, and uninstrument works.""" - verify_autoinstrument_script("test_auto_dspy.py") + verify_autoinstrument_smoke("dspy") diff --git a/py/src/braintrust/integrations/google_genai/test_google_genai.py b/py/src/braintrust/integrations/google_genai/test_google_genai.py index f6350ff0e..0f31eb134 100644 --- a/py/src/braintrust/integrations/google_genai/test_google_genai.py +++ b/py/src/braintrust/integrations/google_genai/test_google_genai.py @@ -8,7 +8,7 @@ import pytest from braintrust import logger from braintrust.integrations.google_genai import setup_genai -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.integrations.versioning import detect_module_version, version_satisfies from braintrust.logger import Attachment from braintrust.span_types import SpanTypeAttribute @@ -1915,4 +1915,4 @@ class TestAutoInstrumentGoogleGenAI: def test_auto_instrument_google_genai(self): """Test auto_instrument patches Google GenAI and creates spans.""" - verify_autoinstrument_script("test_auto_google_genai.py") + verify_autoinstrument_smoke("google_genai") diff --git a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py index b1ae4985c..07cd5f6d4 100644 --- a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py +++ b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py @@ -18,7 +18,7 @@ SentenceSimilarityPatcher, TextGenerationPatcher, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script +from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -759,4 +759,4 @@ async def _run(): class TestAutoInstrumentHuggingFaceHub: def test_auto_instrument_huggingface_hub(self): - verify_autoinstrument_script("test_auto_huggingface_hub.py") + verify_autoinstrument_smoke("huggingface_hub") diff --git a/py/src/braintrust/integrations/instructor/test_instructor.py b/py/src/braintrust/integrations/instructor/test_instructor.py index 27c359a3a..79849cc1f 100644 --- a/py/src/braintrust/integrations/instructor/test_instructor.py +++ b/py/src/braintrust/integrations/instructor/test_instructor.py @@ -232,9 +232,9 @@ class TestInstructorAutoInstrumentSubprocess: """auto_instrument() must instrument Instructor in a fresh subprocess too.""" def test_subprocess_auto_instrument_instructor(self): - from braintrust.integrations.test_utils import verify_autoinstrument_script + from braintrust.integrations.test_utils import verify_autoinstrument_smoke - verify_autoinstrument_script("test_auto_instructor.py", timeout=30) + verify_autoinstrument_smoke("instructor", timeout=30) class TestInstructorParentIsNotLLM: diff --git a/py/src/braintrust/integrations/langchain/test_context.py b/py/src/braintrust/integrations/langchain/test_context.py index b776eee49..09f13b1e8 100644 --- a/py/src/braintrust/integrations/langchain/test_context.py +++ b/py/src/braintrust/integrations/langchain/test_context.py @@ -4,7 +4,7 @@ import pytest from braintrust import logger from braintrust.integrations.langchain import BraintrustCallbackHandler, set_global_handler, setup_langchain -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger from langchain_core.callbacks import CallbackManager from langchain_core.messages import BaseMessage @@ -178,4 +178,4 @@ def test_setup_langchain_installs_default_handler(): class TestAutoInstrumentLangChain: def test_auto_instrument_langchain(self): - verify_autoinstrument_script("test_auto_langchain.py") + verify_autoinstrument_smoke("langchain") diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 373dc8113..5515bbd1c 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -9,7 +9,7 @@ from braintrust.integrations.litellm import patch_litellm from braintrust.integrations.test_utils import ( assert_metrics_are_valid, - verify_autoinstrument_script, + verify_autoinstrument_script, verify_autoinstrument_smoke, ) from braintrust.test_helpers import assert_dict_matches, init_test_logger @@ -1085,4 +1085,4 @@ class TestAutoInstrumentLiteLLM: def test_auto_instrument_litellm(self): """Test auto_instrument patches LiteLLM, creates spans, and uninstrument works.""" - verify_autoinstrument_script("test_auto_litellm.py") + verify_autoinstrument_smoke("litellm") diff --git a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py index 1512c7b91..6b618bc7c 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py +++ b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py @@ -20,7 +20,7 @@ traced_llm_stream_run, traced_session_start, ) -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -188,7 +188,7 @@ def test_livekit_agents_integration_min_version(): def test_auto_instrument_livekit_agents_subprocess(): pytest.importorskip("livekit.agents") - verify_autoinstrument_script("test_auto_livekit_agents.py") + verify_autoinstrument_smoke("livekit_agents") def test_wrap_livekit_agents_wraps_real_agent_session(): diff --git a/py/src/braintrust/integrations/mistral/test_mistral.py b/py/src/braintrust/integrations/mistral/test_mistral.py index 298fffc4f..250a26fc7 100644 --- a/py/src/braintrust/integrations/mistral/test_mistral.py +++ b/py/src/braintrust/integrations/mistral/test_mistral.py @@ -17,7 +17,7 @@ _normalize_mistral_multimodal_value, _ocr_process_wrapper, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script +from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_spans_by_type, init_test_logger @@ -1464,4 +1464,4 @@ def test_aggregate_completion_events_merges_tool_calls_and_content(): class TestAutoInstrumentMistral: def test_auto_instrument_mistral(self): - verify_autoinstrument_script("test_auto_mistral.py") + verify_autoinstrument_smoke("mistral") diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 87530ff3f..d59427a2f 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -17,7 +17,7 @@ _materialize_logged_file_input, _process_attachments_in_chat_output, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script +from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke from braintrust.integrations.utils import _try_to_dict from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import assert_dict_matches, init_test_logger @@ -2797,7 +2797,7 @@ class TestAutoInstrumentOpenAI: def test_auto_instrument_openai(self): """Test auto_instrument patches OpenAI, creates spans, and uninstrument works.""" - verify_autoinstrument_script("test_auto_openai.py") + verify_autoinstrument_smoke("openai") def test_wrap_openai_wraps_images_methods(): diff --git a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py index 51f442cdc..3c92006e9 100644 --- a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py +++ b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py @@ -4,7 +4,7 @@ import pytest from braintrust import logger from braintrust.integrations.openai_agents import BraintrustTracingProcessor, OpenAIAgentsIntegration -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -327,4 +327,4 @@ class TestAutoInstrumentOpenAIAgents: """Tests for auto_instrument() with the OpenAI Agents SDK.""" def test_auto_instrument_openai_agents(self): - verify_autoinstrument_script("test_auto_openai_agents.py") + verify_autoinstrument_smoke("openai_agents") diff --git a/py/src/braintrust/integrations/openrouter/test_openrouter.py b/py/src/braintrust/integrations/openrouter/test_openrouter.py index fa1ec892c..fd66f9ac2 100644 --- a/py/src/braintrust/integrations/openrouter/test_openrouter.py +++ b/py/src/braintrust/integrations/openrouter/test_openrouter.py @@ -5,7 +5,7 @@ import pytest from braintrust import logger from braintrust.integrations.openrouter import OpenRouterIntegration, wrap_openrouter -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script +from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -277,4 +277,4 @@ def test_openrouter_integration_setup_is_idempotent(monkeypatch): class TestAutoInstrumentOpenRouter: def test_auto_instrument_openrouter(self): - verify_autoinstrument_script("test_auto_openrouter.py") + verify_autoinstrument_smoke("openrouter") diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index b42764041..46a62b2cc 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -15,7 +15,7 @@ setup_pipecat, wrap_pipeline_worker, ) -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.logger import Attachment from braintrust.test_helpers import init_test_logger @@ -300,4 +300,4 @@ def test_setup_and_wrap_pipeline_worker_are_idempotent(): @pytest.mark.skipif(__import__("sys").version_info < (3, 11), reason="Pipecat AI 1.x requires Python 3.11+") def test_auto_instrument_pipecat_subprocess(): pytest.importorskip("pipecat") - verify_autoinstrument_script("test_auto_pipecat.py") + verify_autoinstrument_smoke("pipecat") diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index 42f4052ae..af64a164c 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -8,7 +8,7 @@ import pytest from braintrust import logger, setup_pydantic_ai, traced -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger from pydantic import BaseModel @@ -2918,7 +2918,7 @@ class TestAutoInstrumentPydanticAI: def test_auto_instrument_pydantic_ai(self): """Test auto_instrument patches Pydantic AI and creates spans.""" - verify_autoinstrument_script("test_auto_pydantic_ai.py") + verify_autoinstrument_smoke("pydantic_ai") @pytest.mark.vcr diff --git a/py/src/braintrust/integrations/temporal/test_temporal.py b/py/src/braintrust/integrations/temporal/test_temporal.py index c726e865a..3f66ca68a 100644 --- a/py/src/braintrust/integrations/temporal/test_temporal.py +++ b/py/src/braintrust/integrations/temporal/test_temporal.py @@ -9,7 +9,7 @@ import pytest import pytest_asyncio -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke pytest.importorskip("temporalio") @@ -321,7 +321,7 @@ class TestAutoInstrumentation: """Tests for Temporal auto-instrumentation helpers.""" def test_auto_instrument_temporal_subprocess(self): - verify_autoinstrument_script("test_auto_temporal.py") + verify_autoinstrument_smoke("temporal") def test_contrib_temporal_compat_import_deprecated(self): with pytest.warns(DeprecationWarning, match="braintrust.contrib.temporal is deprecated"): diff --git a/py/src/braintrust/integrations/test_utils.py b/py/src/braintrust/integrations/test_utils.py index a5859149e..846551535 100644 --- a/py/src/braintrust/integrations/test_utils.py +++ b/py/src/braintrust/integrations/test_utils.py @@ -3,10 +3,8 @@ import sys import textwrap import unittest.mock -from collections.abc import Callable, Mapping from contextlib import contextmanager from pathlib import Path -from typing import Any import pytest import vcr @@ -112,7 +110,11 @@ def run_in_subprocess(code: str, timeout: int = 30, env: dict[str, str] | None = ) -def verify_autoinstrument_script(script_name: str, timeout: int = 30) -> subprocess.CompletedProcess: +def verify_autoinstrument_script( + script_name: str, + timeout: int = 30, + args: list[str] | None = None, +) -> subprocess.CompletedProcess: """Run a test script from the integrations auto_test_scripts directory. Raises AssertionError if the script exits with non-zero code. @@ -127,7 +129,7 @@ def verify_autoinstrument_script(script_name: str, timeout: int = 30) -> subproc Path(_versioned_cassette_dir(str(_INTEGRATIONS_DIR / "claude_agent_sdk" / "cassettes"))) ) result = subprocess.run( - [sys.executable, str(script_path)], + [sys.executable, str(script_path), *(args or [])], capture_output=True, text=True, timeout=timeout, @@ -137,55 +139,13 @@ def verify_autoinstrument_script(script_name: str, timeout: int = 30) -> subproc return result -def run_auto_smoke( - name: str, - *, - auto_instrument_kwargs: Mapping[str, object] | None = None, - is_patched: Callable[[], bool] | None = None, - cassette: str | None = None, - integration: str | None = None, - use_vcr: bool = True, - vcr_config: dict | None = None, - run: Callable[[Any], None] | None = None, -) -> None: - """Run the standard ``auto_instrument()`` smoke pattern. - - Encodes the contract shared by scripts under ``auto_test_scripts/``: - - 1. Optional pre-check: ``is_patched()`` returns False before patching. - 2. ``auto_instrument(**auto_instrument_kwargs)`` returns ``{name: True, ...}``. - 3. Optional post-check: ``is_patched()`` returns True after patching. - 4. A second ``auto_instrument`` call still returns ``{name: True, ...}`` (idempotent). - 5. If ``run`` is given, open ``autoinstrument_test_context`` (defaults cassette - name to ``f"test_auto_{name}"``) and delegate to ``run(memory_logger)``. +def verify_autoinstrument_smoke(name: str, timeout: int = 30) -> subprocess.CompletedProcess: + """Fresh-subprocess sanity check for ``auto_instrument(name=True)``. - Callers assert their own API-call and span-shape expectations inside ``run``. + See ``auto_test_scripts/_run_smoke.py`` for what the check actually asserts. + Raises AssertionError if the check fails. """ - from braintrust.auto import auto_instrument - - kwargs = dict(auto_instrument_kwargs or {}) - - if is_patched is not None: - assert not is_patched(), f"{name!r} already patched before auto_instrument()" - - first = auto_instrument(**kwargs) - assert first.get(name) is True, f"auto_instrument returned {first!r}" - if is_patched is not None: - assert is_patched(), f"{name!r} not patched after auto_instrument()" - - second = auto_instrument(**kwargs) - assert second.get(name) is True, f"auto_instrument (2nd call) returned {second!r}" - - if run is None: - return - - ctx_kwargs: dict[str, Any] = {"use_vcr": use_vcr} - if integration is not None: - ctx_kwargs["integration"] = integration - if vcr_config is not None: - ctx_kwargs["vcr_config"] = vcr_config - with autoinstrument_test_context(cassette or f"test_auto_{name}", **ctx_kwargs) as memory_logger: - run(memory_logger) + return verify_autoinstrument_script("_run_smoke.py", timeout=timeout, args=[name]) def assert_metrics_are_valid(metrics, start=None, end=None): diff --git a/py/src/braintrust/integrations/transformers/test_transformers.py b/py/src/braintrust/integrations/transformers/test_transformers.py index 2fb896c6c..644e92e3e 100644 --- a/py/src/braintrust/integrations/transformers/test_transformers.py +++ b/py/src/braintrust/integrations/transformers/test_transformers.py @@ -7,7 +7,7 @@ import pytest from braintrust import logger -from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.integrations.transformers import TransformersIntegration, setup_transformers, wrap_transformers from braintrust.integrations.transformers.patchers import PIPELINE_PATCHERS from braintrust.integrations.transformers.tracing import _input, _metadata @@ -350,4 +350,4 @@ def test_streamer_call_produces_no_span(text_generation_pipeline, memory_logger, def test_auto_instrument_transformers(): - verify_autoinstrument_script("test_auto_transformers.py", timeout=120) + verify_autoinstrument_smoke("transformers", timeout=120) From 39a0b5ce54ddb5fc338c9ff32cbf851d77ef2081 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 4 Sep 2026 19:48:47 +0000 Subject: [PATCH 3/3] refactor(integrations): move auto_instrument smoke into nox, drop per-provider tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the previous commit: instead of every provider test file carrying a tiny `TestAutoInstrument*` class that spawns the same subprocess, hoist the smoke into `_run_tests` in the noxfile. Each integration session already knows which provider it's testing (via the test path), so it invokes python -m braintrust.integrations.auto_test_scripts._run_smoke once before pytest starts. This works uniformly in src and wheel modes. Deletes: - 26 `TestAutoInstrument*` classes/functions and their imports across every provider test file - `verify_autoinstrument_smoke` from `test_utils` (no callers left) - The `args` parameter I added to `verify_autoinstrument_script` in the last commit (only `verify_autoinstrument_smoke` used it; reverted to the original signature) Adds: - `_maybe_run_autoinstrument_smoke(session, test_path)` in noxfile.py — infers the integration name from the test path, dedupes per (session, integration), skips paths that aren't per-provider (e.g. test_core, test_versioning). - `llamaindex` and `strands` to the SMOKES registry — they had no smoke scripts before but do have `auto_instrument` entries, so they get free coverage now. Cost: the smoke no longer runs under raw `pytest test_foo.py` — only under `nox -s "test_foo(latest)"`. CI runs everything through nox so CI coverage is unchanged; devs running pytest directly lose the local sanity check, which is a fair trade for the −135 net LOC and one-line "add a new integration" story. Verified: `nox -s "test_litellm(latest)" -- -k test_patch_litellm` prints "python -m ..._run_smoke litellm\nSUCCESS" before pytest runs (2 passed), `nox -s pylint` clean, `nox -s test_core` clean. Co-Authored-By: Claude Opus 4.7 --- py/noxfile.py | 37 +++++++++++++++++++ .../braintrust/integrations/adk/test_adk.py | 8 ---- .../agentscope/test_agentscope.py | 4 -- .../braintrust/integrations/agno/test_agno.py | 6 --- .../integrations/ai_sdk/test_ai_sdk.py | 3 -- .../integrations/anthropic/test_anthropic.py | 6 --- .../auto_test_scripts/_run_smoke.py | 2 + .../integrations/autogen/test_autogen.py | 3 -- .../bedrock_runtime/test_bedrock_runtime.py | 7 +--- .../claude_agent_sdk/test_claude_agent_sdk.py | 10 ----- .../integrations/cohere/test_cohere.py | 5 +-- .../integrations/crewai/test_crewai.py | 19 +--------- .../cursor_sdk/test_cursor_sdk.py | 7 ---- .../braintrust/integrations/dspy/test_dspy.py | 10 +---- .../google_genai/test_google_genai.py | 7 ---- .../huggingface_hub/test_huggingface_hub.py | 5 +-- .../instructor/test_instructor.py | 9 ----- .../integrations/langchain/test_context.py | 4 -- .../integrations/litellm/test_litellm.py | 8 +--- .../livekit_agents/test_livekit_agents.py | 6 --- .../integrations/mistral/test_mistral.py | 5 +-- .../integrations/openai/test_openai.py | 10 +---- .../openai_agents/test_openai_agents.py | 6 --- .../openrouter/test_openrouter.py | 5 +-- .../integrations/pipecat/test_pipecat.py | 8 ---- .../test_pydantic_ai_integration.py | 9 ----- .../integrations/temporal/test_temporal.py | 6 --- py/src/braintrust/integrations/test_utils.py | 17 +-------- .../transformers/test_transformers.py | 3 -- 29 files changed, 50 insertions(+), 185 deletions(-) diff --git a/py/noxfile.py b/py/noxfile.py index 351d9383b..fbede43ff 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -925,6 +925,42 @@ def _run_core_tests(session): ) +_SMOKE_DONE: set[str] = set() + + +def _maybe_run_autoinstrument_smoke(session, test_path): + """Run the fresh-subprocess ``auto_instrument`` sanity check once per + (session, integration). + + Infers the integration name from ``test_path`` (``braintrust/integrations/ + /test_*.py`` → ````) and invokes + ``auto_test_scripts/_run_smoke.py`` via ``-m`` so it works in both src and + wheel modes. Silently skips paths that aren't under a per-provider + subdirectory (e.g. ``braintrust`` itself for ``test_core``, or + ``braintrust/integrations/test_versioning.py``). + """ + parts = pathlib.Path(test_path).parts + try: + idx = parts.index("integrations") + except ValueError: + return + if idx + 1 >= len(parts): + return + name = parts[idx + 1] + if name.endswith(".py"): + return # e.g. integrations/test_versioning.py — shared, not per-provider + key = f"{session.name}::{name}" + if key in _SMOKE_DONE: + return + _SMOKE_DONE.add(key) + session.run( + "python", + "-m", + "braintrust.integrations.auto_test_scripts._run_smoke", + name, + ) + + def _run_tests( session, test_path, @@ -935,6 +971,7 @@ def _run_tests( run_from_temp_dir=False, ): """Run tests against a wheel or the source code. Paths should be relative and start with braintrust.""" + _maybe_run_autoinstrument_smoke(session, test_path) env = env.copy() if env else {} if version: env["BRAINTRUST_TEST_PACKAGE_VERSION"] = version diff --git a/py/src/braintrust/integrations/adk/test_adk.py b/py/src/braintrust/integrations/adk/test_adk.py index 7879733f2..4a40b5ae6 100644 --- a/py/src/braintrust/integrations/adk/test_adk.py +++ b/py/src/braintrust/integrations/adk/test_adk.py @@ -1159,11 +1159,3 @@ async def test_adk_response_json_schema_dict(memory_logger): assert output["avg_logprobs"] is not None -class TestAutoInstrumentADK: - """Tests for auto_instrument() with Google ADK.""" - - def test_auto_instrument_adk(self): - """Test auto_instrument patches ADK classes and is idempotent.""" - from braintrust.integrations.test_utils import verify_autoinstrument_smoke - - verify_autoinstrument_smoke("adk") diff --git a/py/src/braintrust/integrations/agentscope/test_agentscope.py b/py/src/braintrust/integrations/agentscope/test_agentscope.py index 692907385..eb6cfd371 100644 --- a/py/src/braintrust/integrations/agentscope/test_agentscope.py +++ b/py/src/braintrust/integrations/agentscope/test_agentscope.py @@ -7,7 +7,6 @@ import pytest from braintrust import logger from braintrust.integrations.agentscope import setup_agentscope -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger from packaging.version import Version @@ -324,6 +323,3 @@ def test_setup_agentscope_is_idempotent(): assert inspect.getattr_static(OpenAIChatModel, "__call__") is wrapped -class TestAutoInstrumentAgentScope: - def test_auto_instrument_agentscope(self): - verify_autoinstrument_smoke("agentscope") diff --git a/py/src/braintrust/integrations/agno/test_agno.py b/py/src/braintrust/integrations/agno/test_agno.py index ee6de7915..ecbc6e2f5 100644 --- a/py/src/braintrust/integrations/agno/test_agno.py +++ b/py/src/braintrust/integrations/agno/test_agno.py @@ -9,7 +9,6 @@ from braintrust.integrations.agno import setup_agno from braintrust.integrations.agno import tracing as agno_tracing_module from braintrust.integrations.agno.patchers import wrap_agent, wrap_team -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.logger import Attachment, start_span from braintrust.test_helpers import init_test_logger @@ -230,11 +229,6 @@ def get_provider(self): assert agno_tracing_module._get_model_name(FakeModel()) == "OpenAI" -class TestAutoInstrumentAgno: - def test_auto_instrument_agno(self): - verify_autoinstrument_smoke("agno") - - @pytest.mark.parametrize( "wrapper,name", [ diff --git a/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py b/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py index 40f531f9f..7e13733d9 100644 --- a/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py +++ b/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py @@ -9,7 +9,6 @@ from braintrust import Attachment, logger, setup_ai_sdk from braintrust.integrations.ai_sdk import patch_ai_sdk, unpatch_ai_sdk from braintrust.integrations.ai_sdk.tracing import _shape_input_messages -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger from pydantic import BaseModel @@ -253,5 +252,3 @@ def test_registration_and_unregistration_are_idempotent(): assert patch_ai_sdk() -def test_auto_instrument_ai_sdk_subprocess(): - verify_autoinstrument_smoke("ai_sdk") diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 35890c04f..5236cf416 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -19,7 +19,6 @@ _get_metadata_from_kwargs, _log_message_to_span, ) -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger from pydantic import BaseModel @@ -1499,11 +1498,6 @@ def test_setup_creates_spans(memory_logger): assert "service_tier" not in metrics -class TestAutoInstrumentAnthropic: - def test_auto_instrument_anthropic(self): - verify_autoinstrument_smoke("anthropic") - - def test_extract_anthropic_usage_preserves_nested_numeric_fields(): usage = { "input_tokens": 8, diff --git a/py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py b/py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py index d33977e8b..f20e89b4a 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py +++ b/py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py @@ -40,12 +40,14 @@ # span and an OpenAI span; disable OpenAI so this smoke stays scoped. "litellm": {"openai": False}, "livekit_agents": {}, + "llamaindex": {}, "mistral": {}, "openai": {}, "openai_agents": {}, "openrouter": {}, "pipecat": {}, "pydantic_ai": {}, + "strands": {}, "temporal": {}, "transformers": {}, } diff --git a/py/src/braintrust/integrations/autogen/test_autogen.py b/py/src/braintrust/integrations/autogen/test_autogen.py index ace933a8f..ddfd2da5e 100644 --- a/py/src/braintrust/integrations/autogen/test_autogen.py +++ b/py/src/braintrust/integrations/autogen/test_autogen.py @@ -1,7 +1,6 @@ import pytest from braintrust import logger from braintrust.integrations.autogen import setup_autogen -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger @@ -121,5 +120,3 @@ async def test_autogen_team_run_creates_team_span(memory_logger): assert team_span_ids.intersection(agent_span["span_parents"]) -def test_autogen_auto_instrument_subprocess(): - verify_autoinstrument_smoke("autogen") diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index 35ebb3bcd..aab4bbd7f 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -13,7 +13,7 @@ BedrockClientCreatorPatcher, BedrockRuntimeClientMethodsPatcher, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.test_helpers import init_test_logger @@ -431,8 +431,3 @@ def test_wrap_bedrock_converse_with_document_input_materializes_attachment(memor assert isinstance(file_part["file"]["file_data"], Attachment) assert file_part["file"]["file_data"].reference["content_type"] == "application/pdf" assert "source" not in file_part - - -@pytest.mark.vcr -def test_auto_instrument_bedrock_runtime_subprocess(): - verify_autoinstrument_smoke("bedrock_runtime", timeout=60) diff --git a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py index 45a4603c5..604e1303d 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py @@ -43,7 +43,6 @@ _serialize_tool_result_output, _thread_local, ) -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.logger import start_span from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger @@ -2542,15 +2541,6 @@ async def calculator_handler(args): ) -class TestAutoInstrumentClaudeAgentSDK: - """Tests for auto_instrument() with Claude Agent SDK.""" - - @pytest.mark.skipif(not CLAUDE_SDK_AVAILABLE, reason="Claude Agent SDK not installed") - def test_auto_instrument_claude_agent_sdk(self): - """Test auto_instrument patches Claude Agent SDK and creates spans.""" - verify_autoinstrument_smoke("claude_agent_sdk") - - @pytest.mark.skipif(not CLAUDE_SDK_AVAILABLE, reason="Claude Agent SDK not installed") @pytest.mark.asyncio async def test_setup_claude_agent_sdk_repro_import_before_setup(memory_logger, monkeypatch): diff --git a/py/src/braintrust/integrations/cohere/test_cohere.py b/py/src/braintrust/integrations/cohere/test_cohere.py index b33c21d1c..37c10d6fc 100644 --- a/py/src/braintrust/integrations/cohere/test_cohere.py +++ b/py/src/braintrust/integrations/cohere/test_cohere.py @@ -29,7 +29,7 @@ V2EmbedPatcher, V2RerankPatcher, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_spans_by_type, init_test_logger @@ -814,6 +814,3 @@ def test_cohere_integration_setup_patches_audio_transcriptions(memory_logger, cl assert spans[0]["metadata"]["model"] == TRANSCRIBE_MODEL -class TestAutoInstrumentCohere: - def test_auto_instrument_cohere(self): - verify_autoinstrument_smoke("cohere") diff --git a/py/src/braintrust/integrations/crewai/test_crewai.py b/py/src/braintrust/integrations/crewai/test_crewai.py index ed16547e8..b61698c5e 100644 --- a/py/src/braintrust/integrations/crewai/test_crewai.py +++ b/py/src/braintrust/integrations/crewai/test_crewai.py @@ -31,7 +31,7 @@ setup_crewai, ) from braintrust.integrations.crewai.patchers import _get_registered_listener, _reset_for_testing -from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import run_in_subprocess from braintrust.logger import Attachment, start_span from braintrust.test_helpers import init_test_logger from braintrust.util import LazyValue @@ -583,20 +583,3 @@ def test_setup_crewai_returns_true_under_active_logger(): # --------------------------------------------------------------------------- -class TestAutoInstrumentCrewAI: - def test_auto_instrument_crewai(self): - verify_autoinstrument_smoke("crewai") - - def test_patch_crewai_subprocess(self): - result = run_in_subprocess( - """ - from braintrust.integrations.crewai import patch_crewai - from braintrust.integrations.crewai.patchers import _get_registered_listener - assert patch_crewai() - assert _get_registered_listener() is not None - assert patch_crewai() # idempotent - print("SUCCESS") - """ - ) - assert result.returncode == 0, f"Failed: {result.stderr}" - assert "SUCCESS" in result.stdout diff --git a/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py b/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py index 41a65f695..5f2273e70 100644 --- a/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py +++ b/py/src/braintrust/integrations/cursor_sdk/test_cursor_sdk.py @@ -13,7 +13,6 @@ from braintrust.integrations.cursor_sdk import CursorSDKIntegration, setup_cursor_sdk from braintrust.integrations.cursor_sdk._test_vcr import cursor_vcr_config from braintrust.integrations.openai import OpenAIIntegration -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_spans_by_type, init_test_logger @@ -73,12 +72,6 @@ def test_setup_cursor_sdk_is_idempotent(): assert setup_cursor_sdk(project=PROJECT_NAME) -@_REQUIRES_SYNC_BRIDGE -@pytest.mark.vcr -def test_auto_instrument_cursor_sdk_subprocess(): - verify_autoinstrument_smoke("cursor_sdk", timeout=45) - - @_REQUIRES_SYNC_BRIDGE @pytest.mark.vcr def test_characterize_no_downstream_provider_spans(memory_logger, tmp_path): diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index 3954b5155..d49684380 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -6,7 +6,7 @@ import pytest from braintrust import logger from braintrust.integrations.dspy import BraintrustDSpyCallback -from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import run_in_subprocess from braintrust.test_helpers import init_test_logger @@ -229,11 +229,3 @@ def test_legacy_wrapper_import_still_works(self): """) assert result.returncode == 0, f"Failed: {result.stderr}" assert "SUCCESS" in result.stdout - - -class TestAutoInstrumentDSPy: - """Tests for auto_instrument() with DSPy.""" - - def test_auto_instrument_dspy(self): - """Test auto_instrument patches DSPy, creates spans, and uninstrument works.""" - verify_autoinstrument_smoke("dspy") diff --git a/py/src/braintrust/integrations/google_genai/test_google_genai.py b/py/src/braintrust/integrations/google_genai/test_google_genai.py index 0f31eb134..2997379aa 100644 --- a/py/src/braintrust/integrations/google_genai/test_google_genai.py +++ b/py/src/braintrust/integrations/google_genai/test_google_genai.py @@ -8,7 +8,6 @@ import pytest from braintrust import logger from braintrust.integrations.google_genai import setup_genai -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.integrations.versioning import detect_module_version, version_satisfies from braintrust.logger import Attachment from braintrust.span_types import SpanTypeAttribute @@ -1910,9 +1909,3 @@ async def test_interactions_async_stream(memory_logger): ) -class TestAutoInstrumentGoogleGenAI: - """Tests for auto_instrument() with Google GenAI.""" - - def test_auto_instrument_google_genai(self): - """Test auto_instrument patches Google GenAI and creates spans.""" - verify_autoinstrument_smoke("google_genai") diff --git a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py index 07cd5f6d4..9384a3397 100644 --- a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py +++ b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py @@ -18,7 +18,7 @@ SentenceSimilarityPatcher, TextGenerationPatcher, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.test_helpers import init_test_logger @@ -757,6 +757,3 @@ async def _run(): # --------------------------------------------------------------------------- -class TestAutoInstrumentHuggingFaceHub: - def test_auto_instrument_huggingface_hub(self): - verify_autoinstrument_smoke("huggingface_hub") diff --git a/py/src/braintrust/integrations/instructor/test_instructor.py b/py/src/braintrust/integrations/instructor/test_instructor.py index 79849cc1f..187576f74 100644 --- a/py/src/braintrust/integrations/instructor/test_instructor.py +++ b/py/src/braintrust/integrations/instructor/test_instructor.py @@ -228,15 +228,6 @@ def test_setup_is_idempotent(self): assert callable(patched.chat.completions.create) -class TestInstructorAutoInstrumentSubprocess: - """auto_instrument() must instrument Instructor in a fresh subprocess too.""" - - def test_subprocess_auto_instrument_instructor(self): - from braintrust.integrations.test_utils import verify_autoinstrument_smoke - - verify_autoinstrument_smoke("instructor", timeout=30) - - class TestInstructorParentIsNotLLM: """Span-type invariant: Instructor parent is never typed as `llm`.""" diff --git a/py/src/braintrust/integrations/langchain/test_context.py b/py/src/braintrust/integrations/langchain/test_context.py index 09f13b1e8..cc5b3f650 100644 --- a/py/src/braintrust/integrations/langchain/test_context.py +++ b/py/src/braintrust/integrations/langchain/test_context.py @@ -4,7 +4,6 @@ import pytest from braintrust import logger from braintrust.integrations.langchain import BraintrustCallbackHandler, set_global_handler, setup_langchain -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger from langchain_core.callbacks import CallbackManager from langchain_core.messages import BaseMessage @@ -176,6 +175,3 @@ def test_setup_langchain_installs_default_handler(): assert get_global_handler() is handler -class TestAutoInstrumentLangChain: - def test_auto_instrument_langchain(self): - verify_autoinstrument_smoke("langchain") diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 5515bbd1c..db1e71cce 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -9,7 +9,7 @@ from braintrust.integrations.litellm import patch_litellm from braintrust.integrations.test_utils import ( assert_metrics_are_valid, - verify_autoinstrument_script, verify_autoinstrument_smoke, + verify_autoinstrument_script, ) from braintrust.test_helpers import assert_dict_matches, init_test_logger @@ -1080,9 +1080,3 @@ def test_litellm_extract_rerank_output_drops_document(): ] -class TestAutoInstrumentLiteLLM: - """Tests for auto_instrument() with LiteLLM.""" - - def test_auto_instrument_litellm(self): - """Test auto_instrument patches LiteLLM, creates spans, and uninstrument works.""" - verify_autoinstrument_smoke("litellm") diff --git a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py index 6b618bc7c..2eb37ca50 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py +++ b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py @@ -20,7 +20,6 @@ traced_llm_stream_run, traced_session_start, ) -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -186,11 +185,6 @@ def test_livekit_agents_integration_min_version(): assert LiveKitAgentsIntegration.min_version == "1.3.1" -def test_auto_instrument_livekit_agents_subprocess(): - pytest.importorskip("livekit.agents") - verify_autoinstrument_smoke("livekit_agents") - - def test_wrap_livekit_agents_wraps_real_agent_session(): pytest.importorskip("livekit.agents") diff --git a/py/src/braintrust/integrations/mistral/test_mistral.py b/py/src/braintrust/integrations/mistral/test_mistral.py index 250a26fc7..3bc44456e 100644 --- a/py/src/braintrust/integrations/mistral/test_mistral.py +++ b/py/src/braintrust/integrations/mistral/test_mistral.py @@ -17,7 +17,7 @@ _normalize_mistral_multimodal_value, _ocr_process_wrapper, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_spans_by_type, init_test_logger @@ -1462,6 +1462,3 @@ def test_aggregate_completion_events_merges_tool_calls_and_content(): assert tool_call["function"]["arguments"] == '{"city":"San Francisco"}' -class TestAutoInstrumentMistral: - def test_auto_instrument_mistral(self): - verify_autoinstrument_smoke("mistral") diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index d59427a2f..0471bdafd 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -17,7 +17,7 @@ _materialize_logged_file_input, _process_attachments_in_chat_output, ) -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.integrations.utils import _try_to_dict from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import assert_dict_matches, init_test_logger @@ -2792,14 +2792,6 @@ async def test_setup_async_creates_spans(self, memory_logger): assert span["input"] -class TestAutoInstrumentOpenAI: - """Tests for auto_instrument() with OpenAI.""" - - def test_auto_instrument_openai(self): - """Test auto_instrument patches OpenAI, creates spans, and uninstrument works.""" - verify_autoinstrument_smoke("openai") - - def test_wrap_openai_wraps_images_methods(): """wrap_openai() should instrument every OpenAI images resource method.""" import inspect diff --git a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py index 3c92006e9..46244903b 100644 --- a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py +++ b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py @@ -4,7 +4,6 @@ import pytest from braintrust import logger from braintrust.integrations.openai_agents import BraintrustTracingProcessor, OpenAIAgentsIntegration -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.test_helpers import init_test_logger @@ -323,8 +322,3 @@ async def test_openai_agents_task_and_turn_span_types(memory_logger): assert turn_span.get("metadata", {}).get("agent_name") is not None -class TestAutoInstrumentOpenAIAgents: - """Tests for auto_instrument() with the OpenAI Agents SDK.""" - - def test_auto_instrument_openai_agents(self): - verify_autoinstrument_smoke("openai_agents") diff --git a/py/src/braintrust/integrations/openrouter/test_openrouter.py b/py/src/braintrust/integrations/openrouter/test_openrouter.py index fd66f9ac2..00a522fc7 100644 --- a/py/src/braintrust/integrations/openrouter/test_openrouter.py +++ b/py/src/braintrust/integrations/openrouter/test_openrouter.py @@ -5,7 +5,7 @@ import pytest from braintrust import logger from braintrust.integrations.openrouter import OpenRouterIntegration, wrap_openrouter -from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_smoke +from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.test_helpers import init_test_logger @@ -275,6 +275,3 @@ def test_openrouter_integration_setup_is_idempotent(monkeypatch): monkeypatch.setattr(Responses, "send", first_responses_send) -class TestAutoInstrumentOpenRouter: - def test_auto_instrument_openrouter(self): - verify_autoinstrument_smoke("openrouter") diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 46a62b2cc..266b79076 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -15,7 +15,6 @@ setup_pipecat, wrap_pipeline_worker, ) -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.logger import Attachment from braintrust.test_helpers import init_test_logger @@ -294,10 +293,3 @@ def test_setup_and_wrap_pipeline_worker_are_idempotent(): observers = getattr(worker_observer, "_observers") braintrust_observers = [observer for observer in observers if isinstance(observer, BraintrustPipecatObserver)] assert braintrust_observers == [explicit_observer] - - -@pytest.mark.vcr -@pytest.mark.skipif(__import__("sys").version_info < (3, 11), reason="Pipecat AI 1.x requires Python 3.11+") -def test_auto_instrument_pipecat_subprocess(): - pytest.importorskip("pipecat") - verify_autoinstrument_smoke("pipecat") diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index af64a164c..cd18efa44 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -8,7 +8,6 @@ import pytest from braintrust import logger, setup_pydantic_ai, traced -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger from pydantic import BaseModel @@ -2913,14 +2912,6 @@ async def test_no_model_agent_run(memory_logger): assert chat_span["metadata"]["model"] == "gpt-4o-mini" -class TestAutoInstrumentPydanticAI: - """Tests for auto_instrument() with Pydantic AI.""" - - def test_auto_instrument_pydantic_ai(self): - """Test auto_instrument patches Pydantic AI and creates spans.""" - verify_autoinstrument_smoke("pydantic_ai") - - @pytest.mark.vcr def test_model_request_stream_sync_thread_context_propagation(memory_logger, direct): """Test that Braintrust context propagates into the background thread created by model_request_stream_sync. diff --git a/py/src/braintrust/integrations/temporal/test_temporal.py b/py/src/braintrust/integrations/temporal/test_temporal.py index 3f66ca68a..d7a6b1c53 100644 --- a/py/src/braintrust/integrations/temporal/test_temporal.py +++ b/py/src/braintrust/integrations/temporal/test_temporal.py @@ -9,9 +9,6 @@ import pytest import pytest_asyncio -from braintrust.integrations.test_utils import verify_autoinstrument_smoke - - pytest.importorskip("temporalio") import braintrust @@ -320,9 +317,6 @@ def state(self) -> str: class TestAutoInstrumentation: """Tests for Temporal auto-instrumentation helpers.""" - def test_auto_instrument_temporal_subprocess(self): - verify_autoinstrument_smoke("temporal") - def test_contrib_temporal_compat_import_deprecated(self): with pytest.warns(DeprecationWarning, match="braintrust.contrib.temporal is deprecated"): import importlib diff --git a/py/src/braintrust/integrations/test_utils.py b/py/src/braintrust/integrations/test_utils.py index 846551535..181d4c7d7 100644 --- a/py/src/braintrust/integrations/test_utils.py +++ b/py/src/braintrust/integrations/test_utils.py @@ -110,11 +110,7 @@ def run_in_subprocess(code: str, timeout: int = 30, env: dict[str, str] | None = ) -def verify_autoinstrument_script( - script_name: str, - timeout: int = 30, - args: list[str] | None = None, -) -> subprocess.CompletedProcess: +def verify_autoinstrument_script(script_name: str, timeout: int = 30) -> subprocess.CompletedProcess: """Run a test script from the integrations auto_test_scripts directory. Raises AssertionError if the script exits with non-zero code. @@ -129,7 +125,7 @@ def verify_autoinstrument_script( Path(_versioned_cassette_dir(str(_INTEGRATIONS_DIR / "claude_agent_sdk" / "cassettes"))) ) result = subprocess.run( - [sys.executable, str(script_path), *(args or [])], + [sys.executable, str(script_path)], capture_output=True, text=True, timeout=timeout, @@ -139,15 +135,6 @@ def verify_autoinstrument_script( return result -def verify_autoinstrument_smoke(name: str, timeout: int = 30) -> subprocess.CompletedProcess: - """Fresh-subprocess sanity check for ``auto_instrument(name=True)``. - - See ``auto_test_scripts/_run_smoke.py`` for what the check actually asserts. - Raises AssertionError if the check fails. - """ - return verify_autoinstrument_script("_run_smoke.py", timeout=timeout, args=[name]) - - def assert_metrics_are_valid(metrics, start=None, end=None): assert metrics # assert 0 < metrics["time_to_first_token"] diff --git a/py/src/braintrust/integrations/transformers/test_transformers.py b/py/src/braintrust/integrations/transformers/test_transformers.py index 644e92e3e..01ffc635b 100644 --- a/py/src/braintrust/integrations/transformers/test_transformers.py +++ b/py/src/braintrust/integrations/transformers/test_transformers.py @@ -7,7 +7,6 @@ import pytest from braintrust import logger -from braintrust.integrations.test_utils import verify_autoinstrument_smoke from braintrust.integrations.transformers import TransformersIntegration, setup_transformers, wrap_transformers from braintrust.integrations.transformers.patchers import PIPELINE_PATCHERS from braintrust.integrations.transformers.tracing import _input, _metadata @@ -349,5 +348,3 @@ def test_streamer_call_produces_no_span(text_generation_pipeline, memory_logger, capsys.readouterr() -def test_auto_instrument_transformers(): - verify_autoinstrument_smoke("transformers", timeout=120)