ref(integrations): collapse auto_test_scripts to a nox-level smoke

py/noxfile.py

@@ -925,6 +925,42 @@ def _run_core_tests(session):
     )
 
 
+_SMOKE_DONE: set[str] = set()
+
+
+def _maybe_run_autoinstrument_smoke(session, test_path):
+    """Run the fresh-subprocess ``auto_instrument`` sanity check once per
+    (session, integration).
+
+    Infers the integration name from ``test_path`` (``braintrust/integrations/
+    <name>/test_*.py`` → ``<name>``) and invokes
+    ``auto_test_scripts/_run_smoke.py`` via ``-m`` so it works in both src and
+    wheel modes.  Silently skips paths that aren't under a per-provider
+    subdirectory (e.g. ``braintrust`` itself for ``test_core``, or
+    ``braintrust/integrations/test_versioning.py``).
+    """
+    parts = pathlib.Path(test_path).parts
+    try:
+        idx = parts.index("integrations")
+    except ValueError:
+        return
+    if idx + 1 >= len(parts):
+        return
+    name = parts[idx + 1]
+    if name.endswith(".py"):
+        return  # e.g. integrations/test_versioning.py — shared, not per-provider
+    key = f"{session.name}::{name}"
+    if key in _SMOKE_DONE:
+        return
+    _SMOKE_DONE.add(key)
+    session.run(
+        "python",
+        "-m",
+        "braintrust.integrations.auto_test_scripts._run_smoke",
+        name,
+    )
+
+
 def _run_tests(
     session,
     test_path,
@@ -935,6 +971,7 @@ def _run_tests(
     run_from_temp_dir=False,
 ):
     """Run tests against a wheel or the source code. Paths should be relative and start with braintrust."""
+    _maybe_run_autoinstrument_smoke(session, test_path)
     env = env.copy() if env else {}
     if version:
         env["BRAINTRUST_TEST_PACKAGE_VERSION"] = version

py/src/braintrust/integrations/adk/test_adk.py

@@ -1159,11 +1159,3 @@ async def test_adk_response_json_schema_dict(memory_logger):
         assert output["avg_logprobs"] is not None
 
 
-class TestAutoInstrumentADK:
-    """Tests for auto_instrument() with Google ADK."""
-
-    def test_auto_instrument_adk(self):
-        """Test auto_instrument patches ADK classes and is idempotent."""
-        from braintrust.integrations.test_utils import verify_autoinstrument_script
-
-        verify_autoinstrument_script("test_auto_adk.py")

py/src/braintrust/integrations/agentscope/test_agentscope.py

@@ -7,7 +7,6 @@
 import pytest
 from braintrust import logger
 from braintrust.integrations.agentscope import setup_agentscope
-from braintrust.integrations.test_utils import verify_autoinstrument_script
 from braintrust.span_types import SpanTypeAttribute
 from braintrust.test_helpers import init_test_logger
 from packaging.version import Version
@@ -324,6 +323,3 @@ def test_setup_agentscope_is_idempotent():
     assert inspect.getattr_static(OpenAIChatModel, "__call__") is wrapped
 
 
-class TestAutoInstrumentAgentScope:
-    def test_auto_instrument_agentscope(self):
-        verify_autoinstrument_script("test_auto_agentscope.py")

py/src/braintrust/integrations/agno/test_agno.py

@@ -9,7 +9,6 @@
 from braintrust.integrations.agno import setup_agno
 from braintrust.integrations.agno import tracing as agno_tracing_module
 from braintrust.integrations.agno.patchers import wrap_agent, wrap_team
-from braintrust.integrations.test_utils import verify_autoinstrument_script
 from braintrust.logger import Attachment, start_span
 from braintrust.test_helpers import init_test_logger
 
@@ -230,11 +229,6 @@ def get_provider(self):
     assert agno_tracing_module._get_model_name(FakeModel()) == "OpenAI"
 
 
-class TestAutoInstrumentAgno:
-    def test_auto_instrument_agno(self):
-        verify_autoinstrument_script("test_auto_agno.py")
-
-
 @pytest.mark.parametrize(
     "wrapper,name",
     [

py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py

@@ -9,7 +9,6 @@
 from braintrust import Attachment, logger, setup_ai_sdk
 from braintrust.integrations.ai_sdk import patch_ai_sdk, unpatch_ai_sdk
 from braintrust.integrations.ai_sdk.tracing import _shape_input_messages
-from braintrust.integrations.test_utils import verify_autoinstrument_script
 from braintrust.span_types import SpanTypeAttribute
 from braintrust.test_helpers import init_test_logger
 from pydantic import BaseModel
@@ -253,5 +252,3 @@ def test_registration_and_unregistration_are_idempotent():
     assert patch_ai_sdk()
 
 
-def test_auto_instrument_ai_sdk_subprocess():
-    verify_autoinstrument_script("test_auto_ai_sdk.py")

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -19,7 +19,6 @@
     _get_metadata_from_kwargs,
     _log_message_to_span,
 )
-from braintrust.integrations.test_utils import verify_autoinstrument_script
 from braintrust.span_types import SpanTypeAttribute
 from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger
 from pydantic import BaseModel
@@ -1499,11 +1498,6 @@ def test_setup_creates_spans(memory_logger):
     assert "service_tier" not in metrics
 
 
-class TestAutoInstrumentAnthropic:
-    def test_auto_instrument_anthropic(self):
-        verify_autoinstrument_script("test_auto_anthropic.py")
-
-
 def test_extract_anthropic_usage_preserves_nested_numeric_fields():
     usage = {
         "input_tokens": 8,

py/src/braintrust/integrations/auto_test_scripts/_run_smoke.py

@@ -0,0 +1,77 @@
+"""Fresh-subprocess sanity check for :func:`braintrust.auto.auto_instrument`.
+
+Usage: ``python _run_smoke.py <integration_name>``
+
+Runs ``auto_instrument(**kwargs)`` twice in a clean Python process and asserts
+that the target integration is reported as successfully instrumented on both
+calls (i.e. patching works and is idempotent).
+
+Deliberately minimal — span shape, provider metadata, patching topology, and
+real API calls are all covered by the in-process ``test_*.py`` files for each
+integration.  The only thing a subprocess uniquely proves is: from a cold
+Python process, ``auto_instrument()`` successfully sets up this integration.
+"""
+
+import sys
+
+
+# ``auto_instrument`` accepts a bool kwarg per integration and returns a dict
+# keyed by a display name (usually identical to the kwarg — ``bedrock_runtime``
+# is the one exception, where the kwarg is ``bedrock``).  Each entry below maps
+# the display name to any kwarg overrides on top of the all-True defaults.
+SMOKES: dict[str, dict[str, bool]] = {
+    "adk": {},
+    "agentscope": {},
+    "agno": {},
+    "ai_sdk": {},
+    "anthropic": {},
+    "autogen": {},
+    "bedrock_runtime": {},
+    "claude_agent_sdk": {},
+    "cohere": {},
+    "crewai": {},
+    "cursor_sdk": {},
+    "dspy": {},
+    "google_genai": {},
+    "huggingface_hub": {},
+    "instructor": {},
+    "langchain": {},
+    # LiteLLM's OpenAI-backed chat path would otherwise produce both a LiteLLM
+    # span and an OpenAI span; disable OpenAI so this smoke stays scoped.
+    "litellm": {"openai": False},
+    "livekit_agents": {},
+    "llamaindex": {},
+    "mistral": {},
+    "openai": {},
+    "openai_agents": {},
+    "openrouter": {},
+    "pipecat": {},
+    "pydantic_ai": {},
+    "strands": {},
+    "temporal": {},
+    "transformers": {},
+}
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        raise SystemExit(f"usage: {sys.argv[0]} <integration_name>")
+    name = sys.argv[1]
+    if name not in SMOKES:
+        raise SystemExit(f"unknown integration: {name!r}. Add to SMOKES in {__file__}.")
+
+    from braintrust.auto import auto_instrument
+
+    kwargs = SMOKES[name]
+
+    first = auto_instrument(**kwargs)
+    assert first.get(name) is True, f"auto_instrument returned {first!r} for {name!r}"
+
+    second = auto_instrument(**kwargs)
+    assert second.get(name) is True, f"auto_instrument (2nd call) returned {second!r} for {name!r}"
+
+    print("SUCCESS")
+
+
+if __name__ == "__main__":
+    main()