diff --git a/nemoguardrails/base_guardrails.py b/nemoguardrails/base_guardrails.py index 359a03f16e..599e65649a 100644 --- a/nemoguardrails/base_guardrails.py +++ b/nemoguardrails/base_guardrails.py @@ -27,9 +27,10 @@ """ from abc import ABC, abstractmethod -from typing import Any, AsyncIterator +from typing import Any, AsyncIterator, Optional, Union from nemoguardrails.rails.llm.config import RailsConfig +from nemoguardrails.rails.llm.options import GenerationOptions class BaseGuardrails(ABC): @@ -44,12 +45,14 @@ class BaseGuardrails(ABC): config: RailsConfig @abstractmethod - def generate(self, *args: Any, **kwargs: Any) -> Any: + def generate(self, *args: Any, options: Optional[Union[dict, GenerationOptions]] = None, **kwargs: Any) -> Any: """Generate an LLM response synchronously with guardrails applied.""" ... @abstractmethod - async def generate_async(self, *args: Any, **kwargs: Any) -> Any: + async def generate_async( + self, *args: Any, options: Optional[Union[dict, GenerationOptions]] = None, **kwargs: Any + ) -> Any: """Generate an LLM response asynchronously with guardrails applied.""" ... diff --git a/nemoguardrails/guardrails/actions/content_safety_action.py b/nemoguardrails/guardrails/actions/content_safety_action.py index eb9a15e034..ce0229dc89 100644 --- a/nemoguardrails/guardrails/actions/content_safety_action.py +++ b/nemoguardrails/guardrails/actions/content_safety_action.py @@ -121,10 +121,13 @@ def _content_safety_to_rail_result(parsed: object) -> RailResult: """ if isinstance(parsed, (list, tuple)): if parsed and parsed[0] is True: - return RailResult(is_safe=True) + return RailResult(is_safe=True, return_value={"allowed": True, "policy_violations": []}) if parsed and parsed[0] is False: - if len(parsed) > 1: - categories = ", ".join(str(c) for c in parsed[1:]) - return RailResult(is_safe=False, reason=f"Safety categories: {categories}") - return RailResult(is_safe=False, reason="Unknown") + violations = [str(c) for c in parsed[1:]] + verdict = {"allowed": False, "policy_violations": violations} + if violations: + return RailResult( + is_safe=False, reason=f"Safety categories: {', '.join(violations)}", return_value=verdict + ) + return RailResult(is_safe=False, reason="Unknown", return_value=verdict) raise RuntimeError(f"Unexpected content safety parse result: {parsed}") diff --git a/nemoguardrails/guardrails/actions/jailbreak_detection_action.py b/nemoguardrails/guardrails/actions/jailbreak_detection_action.py index edc806573a..2c974d79a0 100644 --- a/nemoguardrails/guardrails/actions/jailbreak_detection_action.py +++ b/nemoguardrails/guardrails/actions/jailbreak_detection_action.py @@ -41,6 +41,7 @@ def _parse_response(self, response: Any) -> RailResult: raise RuntimeError(f"Jailbreak response missing 'jailbreak' field: {response}") score = response.get("score", "unknown") - if response["jailbreak"]: - return RailResult(is_safe=False, reason=f"Score: {score}") - return RailResult(is_safe=True, reason=f"Score: {score}") + is_jailbreak = response["jailbreak"] + if is_jailbreak: + return RailResult(is_safe=False, reason=f"Score: {score}", return_value=is_jailbreak) + return RailResult(is_safe=True, reason=f"Score: {score}", return_value=is_jailbreak) diff --git a/nemoguardrails/guardrails/actions/topic_safety_action.py b/nemoguardrails/guardrails/actions/topic_safety_action.py index 076824fa02..cee15f0fe6 100644 --- a/nemoguardrails/guardrails/actions/topic_safety_action.py +++ b/nemoguardrails/guardrails/actions/topic_safety_action.py @@ -63,5 +63,5 @@ async def _get_response(self, model_type: Optional[str], prompt: Any) -> str: def _parse_response(self, response: Any) -> RailResult: if response.lower().strip() == "off-topic": - return RailResult(is_safe=False, reason="Topic safety: off-topic") - return RailResult(is_safe=True) + return RailResult(is_safe=False, reason="Topic safety: off-topic", return_value={"on_topic": False}) + return RailResult(is_safe=True, return_value={"on_topic": True}) diff --git a/nemoguardrails/guardrails/engine_registry.py b/nemoguardrails/guardrails/engine_registry.py index a9bf7ef254..1b7ff6f242 100644 --- a/nemoguardrails/guardrails/engine_registry.py +++ b/nemoguardrails/guardrails/engine_registry.py @@ -172,6 +172,10 @@ def _get_engine(self, name: str, expected_type: type[_EngineT]) -> _EngineT: raise TypeError(f"Engine '{name}' is {type(engine).__name__}, expected {expected_type.__name__}") return engine + def provider_name(self, model_type: str) -> str: + """Return the provider/engine name (e.g. 'nim', 'openai') for a model engine.""" + return self._get_engine(model_type, ModelEngine).model_config.engine or "unknown" + async def model_call(self, model_type: str, messages: list[dict], **kwargs: Any) -> LLMResponse: """Route a chat completion request to the named model engine. diff --git a/nemoguardrails/guardrails/guardrails.py b/nemoguardrails/guardrails/guardrails.py index 9579f55333..6a438d8dc7 100644 --- a/nemoguardrails/guardrails/guardrails.py +++ b/nemoguardrails/guardrails/guardrails.py @@ -23,7 +23,7 @@ import logging import warnings -from typing import Any, AsyncIterator, Callable, List, Optional, Tuple, Type, Union, cast, overload +from typing import Any, AsyncIterator, Callable, List, Optional, Tuple, Type, Union, cast from typing_extensions import Self @@ -38,7 +38,7 @@ from nemoguardrails.logging.explain import ExplainInfo from nemoguardrails.rails.llm.config import RailsConfig from nemoguardrails.rails.llm.llmrails import LLMRails -from nemoguardrails.rails.llm.options import GenerationResponse, RailsResult, RailType +from nemoguardrails.rails.llm.options import GenerationOptions, GenerationResponse, RailsResult, RailType from nemoguardrails.types import LLMModel log = logging.getLogger(__name__) @@ -181,74 +181,44 @@ def passthrough_fn(self, fn: Optional[Callable]) -> None: llmrails = cast(LLMRails, self.rails_engine) llmrails.passthrough_fn = fn - @staticmethod - def _convert_to_messages(prompt: str | None = None, messages: LLMMessages | None = None) -> LLMMessages: - """Return messages in standard format, converting a prompt string if needed. - - If messages is provided, returns it as-is. - If prompt is provided, wraps it as [{"role": "user", "content": prompt}]. - """ - - # Priority: messages first, then prompt - if messages: - return messages - - if prompt: - # Convert string prompt to standard format - return [{"role": "user", "content": prompt}] - - raise ValueError("Neither prompt nor messages provided for generation") - async def _ensure_started(self) -> None: """Lazy initialization: call startup() on first use if not already started.""" if not self._started: await self.startup() def generate( - self, prompt: str | None = None, messages: LLMMessages | None = None, **kwargs + self, + prompt: str | None = None, + messages: LLMMessages | None = None, + options: Optional[Union[dict, GenerationOptions]] = None, + **kwargs, ) -> Union[str, dict, GenerationResponse, Tuple[dict, dict]]: """Generate an LLM response synchronously with guardrails applied. Supported in both IORails and LLMRails """ + return self.rails_engine.generate(prompt=prompt, messages=messages, options=options, **kwargs) - generate_messages = self._convert_to_messages(prompt, messages) - return self.rails_engine.generate(messages=generate_messages, **kwargs) - - @overload - async def generate_async(self, prompt: str | None = None, messages: LLMMessages | None = None, **kwargs) -> str: ... - - @overload async def generate_async( - self, prompt: str | None = None, messages: LLMMessages | None = None, **kwargs - ) -> dict: ... - - @overload - async def generate_async( - self, prompt: str | None = None, messages: LLMMessages | None = None, **kwargs - ) -> GenerationResponse: ... - - @overload - async def generate_async( - self, prompt: str | None = None, messages: LLMMessages | None = None, **kwargs - ) -> tuple[dict, dict]: ... - - async def generate_async( - self, prompt: str | None = None, messages: LLMMessages | None = None, **kwargs + self, + prompt: str | None = None, + messages: LLMMessages | None = None, + options: Optional[Union[dict, GenerationOptions]] = None, + **kwargs, ) -> str | dict | GenerationResponse | tuple[dict, dict]: """Generate an LLM response asynchronously with guardrails applied. Supported by both LLMRails and IORails """ await self._ensure_started() - generate_messages = self._convert_to_messages(prompt, messages) - return await self.rails_engine.generate_async(messages=generate_messages, **kwargs) + return await self.rails_engine.generate_async(prompt=prompt, messages=messages, options=options, **kwargs) def stream_async( self, prompt: str | None = None, messages: LLMMessages | None = None, **kwargs ) -> AsyncIterator[str | dict]: """Generate an LLM response asynchronously with streaming support.""" - stream_messages = self._convert_to_messages(prompt, messages) + # TODO: Move prompt/message normalization into IORails when streaming GenerationOptions added + stream_messages = IORails._convert_to_messages(prompt, messages) async def _with_startup(iterator: AsyncIterator[str | dict]) -> AsyncIterator[str | dict]: await self._ensure_started() diff --git a/nemoguardrails/guardrails/guardrails_types.py b/nemoguardrails/guardrails/guardrails_types.py index 546f66361c..e7ab281f59 100644 --- a/nemoguardrails/guardrails/guardrails_types.py +++ b/nemoguardrails/guardrails/guardrails_types.py @@ -16,9 +16,11 @@ import secrets from contextvars import ContextVar, Token -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, TypeAlias +from typing import Any, Optional, TypeAlias + +from nemoguardrails.types import LLMResponse, UsageInfo # LLMMessage can contain role/content, plus optional tool_calls / tool_call_id / name; content may be None LLMMessage: TypeAlias = dict[str, Any] @@ -32,13 +34,70 @@ class RailDirection(Enum): OUTPUT = "Output" +@dataclass(frozen=True, slots=True) +class RailCallRecord: + """One rail's execution record, carried on RailResult for GenerationLog synthesis. + + Captures what a single rail did — its verdict and the (at most one) model call it + made — as engine-neutral data. IORails maps a ``RailCallRecord`` to an + ``ActivatedRail`` (with a single synthetic ``ExecutedAction`` and ``LLMCallInfo``); + the raw ``usage``/timing is kept here so this module stays free of the pydantic + ``GenerationLog`` types. Tool rails that make no model call leave ``usage`` None. + """ + + flow: str + rail_type: str + is_safe: bool + made_call: bool = False + action_name: Optional[str] = None + return_value: Any = None + task: Optional[str] = None + request_id: Optional[str] = None + usage: Optional[UsageInfo] = None + llm_model_name: Optional[str] = None + llm_provider_name: Optional[str] = None + prompt: Optional[str] = None + completion: Optional[str] = None + started_at: Optional[float] = None + finished_at: Optional[float] = None + duration: Optional[float] = None + + @dataclass(frozen=True, slots=True) class RailResult: - """Result of a rail safety check.""" + """Result of a rail safety check. + + ``records`` carries the per-rail execution records for every rail that ran in this + check (not just the blocking one), so IORails can synthesize a ``GenerationLog``. + It is empty unless log collection is active. ``return_value`` is the rail's + structured verdict (e.g. ``{"allowed": ..., "policy_violations": [...]}``) when the + action supplies one, used as the log's ``ExecutedAction.return_value``. + + ``records`` and ``return_value`` are log-capture metadata, not part of the safety + verdict, so they are excluded from equality and hashing (``compare=False``) — two + results with the same ``is_safe``/``reason``/``triggered_rail`` compare equal + regardless of captured log data. + """ is_safe: bool reason: str | None = None triggered_rail: str | None = None + records: tuple[RailCallRecord, ...] = field(default=(), compare=False) + return_value: Any = field(default=None, compare=False) + + +@dataclass(frozen=True, slots=True) +class TimedLLMResponse: + """An LLM response paired with wall-clock start/finish timestamps and a monotonic duration. + + Returned by IORails' main-model call helper so the sequential and speculative paths both + carry real timing into the generation ``RailCallRecord``. + """ + + response: LLMResponse + started_at: float + finished_at: float + duration: float # Default max character length for truncate(). Used to keep DEBUG log lines short. @@ -84,3 +143,22 @@ def truncate(text: object, max_len: int | None = None) -> str: if len(s) <= limit: return s return s[:limit] + "..." + + +def serialize_prompt(messages: list[dict]) -> str: + """Render a chat message list to a role-labeled string for GenerationLog's ``prompt``. + + Content parity with LLMRails' logged prompt, not byte-for-byte format parity: each + message becomes ``": "``. Non-content fields present on the message + (``name``, ``tool_call_id``, ``tool_calls``, ``reasoning``) are appended as a compact + ``[key=value, ...]`` suffix so tool-call and reasoning-only turns are preserved rather + than dropped. Messages are blank-line separated. + """ + lines = [] + for m in messages: + line = f"{m.get('role', '')}: {m.get('content') or ''}" + extras = [f"{key}={m[key]}" for key in ("name", "tool_call_id", "tool_calls", "reasoning") if m.get(key)] + if extras: + line += f" [{', '.join(extras)}]" + lines.append(line) + return "\n\n".join(lines) diff --git a/nemoguardrails/guardrails/iorails.py b/nemoguardrails/guardrails/iorails.py index 4b8df06120..05922c3614 100644 --- a/nemoguardrails/guardrails/iorails.py +++ b/nemoguardrails/guardrails/iorails.py @@ -37,8 +37,11 @@ from nemoguardrails.guardrails.guardrails_types import ( LLMMessage, LLMMessages, + RailCallRecord, RailDirection, + TimedLLMResponse, get_request_id, + serialize_prompt, truncate, ) from nemoguardrails.guardrails.rails_manager import RailsManager @@ -60,10 +63,21 @@ traced_request, ) from nemoguardrails.llm.taskmanager import LLMTaskManager +from nemoguardrails.logging.explain import LLMCallInfo from nemoguardrails.patch_asyncio import check_sync_call_from_async_loop from nemoguardrails.rails.llm.buffer import get_buffer_strategy from nemoguardrails.rails.llm.config import RailsConfig, _get_flow_name -from nemoguardrails.rails.llm.options import GenerationOptions, RailsResult, RailStatus, RailType +from nemoguardrails.rails.llm.options import ( + ActivatedRail, + ExecutedAction, + GenerationLog, + GenerationOptions, + GenerationResponse, + GenerationStats, + RailsResult, + RailStatus, + RailType, +) from nemoguardrails.streaming import END_OF_STREAM, StreamingHandler from nemoguardrails.tracing.constants import GuardrailsAttributes from nemoguardrails.types import LLMModel, LLMResponse, ToolCall @@ -170,6 +184,223 @@ def _build_assistant_message(content: str, tool_calls: Optional[list[ToolCall]]) } +def _build_llm_metadata(response: LLMResponse) -> Optional[dict]: + """Return the main call's ``provider_metadata`` verbatim (LLMRails mirror). + + A pure passthrough — token usage lives in ``log`` (``log.llm_calls`` / ``log.stats``), + not here, matching LLMRails. Returns ``None`` when the main call carried no metadata. + """ + return dict(response.provider_metadata or {}) or None + + +def _make_generation_record( + timed: TimedLLMResponse, + provider_name: Optional[str], + prompt: Optional[str], +) -> RailCallRecord: + """Represent the main generation call as a ``RailCallRecord`` (rail_type "generation"). + + Unifies the main call with the rail calls so the log builder can treat every LLM call + the same way. ``prompt`` is the serialized main-model messages; ``completion`` is the + response content. Timing comes from ``timed``: ``started_at``/``finished_at`` are + wall-clock timestamps and ``duration`` is a monotonic delta. + """ + response = timed.response + return RailCallRecord( + flow="generation", + rail_type="generation", + is_safe=True, + made_call=True, + action_name="generate_bot_message", + task="general", + request_id=response.request_id, + usage=response.usage, + llm_model_name=response.model, + llm_provider_name=provider_name, + prompt=prompt, + completion=response.content, + started_at=timed.started_at, + finished_at=timed.finished_at, + duration=timed.duration, + ) + + +def _record_has_llm_call(record: RailCallRecord) -> bool: + """True when a record reflects a model/API call (made a call, or carries usage/model).""" + return record.made_call or record.usage is not None or record.llm_model_name is not None + + +def _call_info(record: RailCallRecord) -> LLMCallInfo: + """Map a ``RailCallRecord`` to a ``LLMCallInfo`` for the log.""" + usage = record.usage + return LLMCallInfo( + task=record.task, + duration=record.duration, + total_tokens=usage.total_tokens if usage else None, + prompt_tokens=usage.input_tokens if usage else None, + completion_tokens=usage.output_tokens if usage else None, + started_at=record.started_at, + finished_at=record.finished_at, + id=record.request_id, + prompt=record.prompt, + completion=record.completion, + llm_model_name=record.llm_model_name or "unknown", + llm_provider_name=record.llm_provider_name or "unknown", + ) + + +def _activated_rail(record: RailCallRecord) -> ActivatedRail: + """Map a ``RailCallRecord`` to an ``ActivatedRail`` with one synthetic ``ExecutedAction``. + + IORails runs one model-backed check per rail (not a Colang action chain), so each rail + gets a single ``ExecutedAction`` carrying the rail's structured verdict as + ``return_value`` and its (at most one) LLM call. + """ + action = ExecutedAction( + action_name=record.action_name or record.flow, + action_params={}, + return_value=record.return_value, + llm_calls=[_call_info(record)] if _record_has_llm_call(record) else [], + started_at=record.started_at, + finished_at=record.finished_at, + duration=record.duration, + ) + return ActivatedRail( + type=record.rail_type, + name=record.flow, + executed_actions=[action], + stop=not record.is_safe, + started_at=record.started_at, + finished_at=record.finished_at, + duration=record.duration, + ) + + +def _build_generation_stats( + records: list[RailCallRecord], call_records: list[RailCallRecord], total_duration: Optional[float] +) -> GenerationStats: + """Aggregate per-phase durations and token totals across every recorded call.""" + + def _phase_duration(*rail_types: str) -> Optional[float]: + total = sum((r.duration or 0.0) for r in records if r.rail_type in rail_types) + return total or None + + return GenerationStats( + input_rails_duration=_phase_duration("input", "tool_input"), + output_rails_duration=_phase_duration("output", "tool_output"), + generation_rails_duration=_phase_duration("generation"), + total_duration=total_duration, + llm_calls_duration=sum((r.duration or 0.0) for r in call_records) or None, + llm_calls_count=len(call_records), + llm_calls_total_prompt_tokens=sum(r.usage.input_tokens for r in call_records if r.usage), + llm_calls_total_completion_tokens=sum(r.usage.output_tokens for r in call_records if r.usage), + llm_calls_total_tokens=sum(r.usage.total_tokens for r in call_records if r.usage), + ) + + +def _build_generation_log( + records: list[RailCallRecord], options: Optional[GenerationOptions], total_duration: Optional[float] +) -> Optional[GenerationLog]: + """Synthesize a ``GenerationLog`` from collected rail + generation records. + + Returns ``None`` unless ``options.log`` requests ``activated_rails`` or ``llm_calls``. + ``stats`` is always included when either is requested (matching LLMRails); the + per-rail ``activated_rails`` and flat ``llm_calls`` are gated on their own flags. + ``internal_events`` / ``colang_history`` are rejected earlier (Colang-only). + """ + log_options = options.log if options else None + if not log_options or not (log_options.activated_rails or log_options.llm_calls): + return None + + call_records = [record for record in records if _record_has_llm_call(record)] + generation_log = GenerationLog() + generation_log.stats = _build_generation_stats(records, call_records, total_duration) + if log_options.activated_rails: + generation_log.activated_rails = [_activated_rail(record) for record in records] + if log_options.llm_calls: + generation_log.llm_calls = [_call_info(record) for record in call_records] + return generation_log + + +def _build_generation_response( + response_text: str, + reasoning_content: Optional[str], + response: LLMResponse, + log: Optional[GenerationLog] = None, +) -> GenerationResponse: + """Build the structured ``GenerationResponse`` returned when ``options`` are supplied. + + Reasoning goes to the ``reasoning_content`` field with clean message content (no + inline ```` prefix — that is the bare-return shape). Tool calls use the + canonical ``ToolCall.to_dict()`` shape (dict arguments), matching LLMRails' + ``GenerationResponse.tool_calls`` rather than the OpenAI-wire JSON-string shape + used for the bare message. ``llm_output`` stays ``None`` to match LLMRails, whose + ``raw_response`` source is never populated. ``log`` is set when the caller requested + log details via ``options.log``. + """ + result = GenerationResponse(response=[{"role": "assistant", "content": response_text}]) + if reasoning_content: + result.reasoning_content = reasoning_content + if response.tool_calls: + result.tool_calls = [tool_call.to_dict() for tool_call in response.tool_calls] + llm_metadata = _build_llm_metadata(response) + if llm_metadata: + result.llm_metadata = llm_metadata + if log is not None: + result.log = log + return result + + +def _finalize_refusal(structured: bool, log: Optional[GenerationLog] = None) -> Union[LLMMessage, GenerationResponse]: + """Shape the refusal message for the active return contract (structured vs bare). + + On the structured path the collected ``log`` (rails that ran up to the block) is + attached when the caller requested it. + """ + message: LLMMessage = {"role": "assistant", "content": REFUSAL_MESSAGE} + if not structured: + return message + result = GenerationResponse(response=[message]) + if log is not None: + result.log = log + return result + + +def _response_content_for_capture(result: Union[LLMMessage, GenerationResponse]) -> Optional[str]: + """Extract the assistant content for content capture from either return shape.""" + if isinstance(result, GenerationResponse): + response = result.response + if isinstance(response, list) and response: + last = response[-1] + content = last.get("content") if isinstance(last, dict) else None + return content if isinstance(content, str) else None + return response if isinstance(response, str) else None + return result.get("content") + + +def _raise_on_unsupported_options(options: Optional[GenerationOptions], state: object) -> None: + """Raise for ``GenerationOptions``/``state`` features IORails cannot honor. + + ``state`` and ``output_vars``/``output_data`` require Colang runtime state that + IORails does not have and raise ``ValueError``. ``log.internal_events`` / + ``log.colang_history`` are Colang-runtime-only and raise ``NotImplementedError``; + ``log.activated_rails`` / ``log.llm_calls`` are supported. These raise rather than + silently returning empty data, mirroring how LLMRails rejects options it cannot fulfill. + """ + if state is not None: + raise ValueError("state is not supported by IORails; it is a stateless input/output rails engine") + if options is None: + return + if options.output_vars: + raise ValueError("output_vars/output_data is not supported by IORails; it has no Colang context to return") + log_options = options.log + if log_options and (log_options.internal_events or log_options.colang_history): + raise NotImplementedError( + "GenerationLog `internal_events` and `colang_history` are not supported by IORails " + "(no Colang runtime); use `activated_rails` and/or `llm_calls`" + ) + + def _coerce_generation_options(options: Optional[Union[dict, GenerationOptions]]) -> Optional[GenerationOptions]: """Normalize the request ``options`` argument into a ``GenerationOptions`` or None.""" if isinstance(options, GenerationOptions): @@ -464,7 +695,36 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): """Context manager (used for testing rather than long-lived instance)""" await self.stop() - def generate(self, messages: LLMMessages, **kwargs) -> LLMMessage: + @staticmethod + def _convert_to_messages( + prompt: Optional[Union[str, LLMMessages]] = None, + messages: Optional[Union[LLMMessages, str]] = None, + ) -> LLMMessages: + """Normalize a prompt string or a message list into the standard messages format. + + Argument order mirrors the ``Guardrails`` facade and LLMRails (prompt first). + ``messages`` takes priority when both are supplied; a prompt string becomes a + single user turn. A wrong-typed value — a list passed as ``prompt`` or a string + passed as ``messages`` (the common positional mix-ups) — raises ``TypeError``; + neither provided raises ``ValueError``. + """ + if messages is not None and not isinstance(messages, list): + raise TypeError("messages must be a list of {'role', 'content'} dicts; pass a string via prompt=") + if prompt is not None and not isinstance(prompt, str): + raise TypeError("prompt must be a string; pass a message list via messages=") + if messages: + return messages + if prompt: + return [{"role": "user", "content": prompt}] + raise ValueError("Neither prompt nor messages provided for generation") + + def generate( + self, + prompt: Optional[str] = None, + messages: Optional[LLMMessages] = None, + options: Optional[Union[dict, GenerationOptions]] = None, + **kwargs, + ) -> Union[LLMMessage, GenerationResponse]: """Synchronous version of generate_async. Telemetry is disabled for the ephemeral IORails object used for @@ -472,6 +732,7 @@ def generate(self, messages: LLMMessages, **kwargs) -> LLMMessage: `generate_async()` and `stream_async()` methods for non-streaming and streaming requests respectively. """ + messages = self._convert_to_messages(prompt, messages) # Disable tracing and metrics for synchronous generation calls sync_config = self.config.model_copy(deep=True) @@ -484,11 +745,17 @@ async def _run_sync_iorails(): """Spin up a short-lived IORails engine for one synchronous generate call.""" # Avoid counting this sync-API bridge as a separate user-created IORails instance. async with IORails(sync_config, _report_usage=False) as iorails_engine: - return await iorails_engine.generate_async(messages, **kwargs) + return await iorails_engine.generate_async(messages=messages, options=options, **kwargs) return asyncio.run(_run_sync_iorails()) - async def generate_async(self, messages: LLMMessages, **kwargs) -> LLMMessage: + async def generate_async( + self, + prompt: Optional[str] = None, + messages: Optional[LLMMessages] = None, + options: Optional[Union[dict, GenerationOptions]] = None, + **kwargs, + ) -> Union[LLMMessage, GenerationResponse]: """Public entry: submit the request to the internal work queue. The queue enforces non-streaming concurrency limits @@ -504,17 +771,23 @@ async def generate_async(self, messages: LLMMessages, **kwargs) -> LLMMessage: ``requests.errors{error.type=QueueFull}`` and ``nonstream.rejections`` — honest dual-signal reporting. """ + messages = self._convert_to_messages(prompt, messages) await self.start() metrics_ctx = request_metrics() if self._metrics_enabled else nullcontext() with metrics_ctx: try: - return await self._generate_async_queue.submit(self._run_generate, messages, **kwargs) + return await self._generate_async_queue.submit(self._run_generate, messages, options=options, **kwargs) except asyncio.QueueFull: if self._metrics_enabled: record_nonstream_rejected() raise - async def _run_generate(self, messages: LLMMessages, **kwargs) -> LLMMessage: + async def _run_generate( + self, + messages: LLMMessages, + options: Optional[Union[dict, GenerationOptions]] = None, + **kwargs, + ) -> Union[LLMMessage, GenerationResponse]: """Runs inside a queue worker task. Wraps the pipeline in ``traced_request`` so each request gets its own span + request ID, then delegates to ``_do_generate`` for the actual input rails → @@ -525,7 +798,7 @@ async def _run_generate(self, messages: LLMMessages, **kwargs) -> LLMMessage: with traced_request(tracer) as (request_span, req_id): t0 = time.monotonic() try: - result = await self._do_generate(messages, req_id, request_span, **kwargs) + result = await self._do_generate(messages, req_id, request_span, options=options, **kwargs) except Exception: elapsed_ms = (time.monotonic() - t0) * 1000 log.error("[%s] generate_async failed time=%.1fms", req_id, elapsed_ms, exc_info=True) @@ -533,7 +806,7 @@ async def _run_generate(self, messages: LLMMessages, **kwargs) -> LLMMessage: # Capture content once here at the traced_request boundary so any # future early-return added to _do_generate is covered automatically. if self._content_capture_enabled: - set_request_content(request_span, messages, result.get("content")) + set_request_content(request_span, messages, _response_content_for_capture(result)) elapsed_ms = (time.monotonic() - t0) * 1000 log.info("[%s] generate_async completed time=%.1fms", req_id, elapsed_ms) return result @@ -559,13 +832,23 @@ def _guardrails_violation_payload(message: str, param: str) -> str: ) async def _do_generate( - self, messages: LLMMessages, req_id: str, request_span: Optional["Span"] = None, **kwargs - ) -> LLMMessage: + self, + messages: LLMMessages, + req_id: str, + request_span: Optional["Span"] = None, + *, + options: Optional[Union[dict, GenerationOptions]] = None, + **kwargs, + ) -> Union[LLMMessage, GenerationResponse]: """Core pipeline: tool-result rails -> input rails -> LLM call -> tool-call + output rails.""" log.info("[%s] generate_async called", req_id) log.debug("[%s] generate_async messages=%s", req_id, truncate(messages)) - options = _coerce_generation_options(kwargs.get("options")) + options = _coerce_generation_options(options) + _raise_on_unsupported_options(options, kwargs.get("state")) + # When options are supplied we return a structured GenerationResponse + # (mirroring LLMRails' `if gen_options:` branch); otherwise a bare LLMMessage. + has_generation_options = options is not None # Pass llm_params (including tool definitions) unchanged to the LLM call. llm_kwargs = options.llm_params if (options and options.llm_params) else {} input_enabled = options.rails.input if options else True @@ -573,25 +856,40 @@ async def _do_generate( tool_input_enabled = options.rails.tool_input if options else True tool_output_enabled = options.rails.tool_output if options else True + # Per-rail + generation records accumulated for the GenerationLog (built only when + # options.log requests it). Each rail check and the main call append their record. + t_start = time.monotonic() + records: list[RailCallRecord] = [] + + def _blocked_return() -> Union[LLMMessage, GenerationResponse]: + """Refusal shaped for the return contract, carrying the log of rails run so far.""" + log_obj = ( + _build_generation_log(records, options, time.monotonic() - t_start) if has_generation_options else None + ) + return _finalize_refusal(has_generation_options, log_obj) + # Agent/client executes tool-calls and sends results to Main LLM with prior conversation history. # Symmetric with INPUT rails log.info("[%s] Running tool result rails", req_id) tool_result = await self.rails_manager.are_tool_results_safe(messages, enabled=tool_input_enabled) + records.extend(tool_result.records) if not tool_result.is_safe: log.info("[%s] Tool result blocked: %s", req_id, tool_result.reason) if self._metrics_enabled: record_request_blocked(RailDirection.INPUT) - return {"role": "assistant", "content": REFUSAL_MESSAGE} + return _blocked_return() if self._speculative_generation: response = await self._do_generate_speculative( - messages, req_id, llm_kwargs, request_span, input_enabled=input_enabled + messages, req_id, llm_kwargs, request_span, input_enabled=input_enabled, records_out=records ) else: - response = await self._do_generate_sequential(messages, req_id, llm_kwargs, input_enabled=input_enabled) + response = await self._do_generate_sequential( + messages, req_id, llm_kwargs, input_enabled=input_enabled, records_out=records + ) if response is None: - return {"role": "assistant", "content": REFUSAL_MESSAGE} + return _blocked_return() # Log raw content before reasoning extraction and think-token removal log.debug("[%s] Raw LLM response: %s", req_id, truncate(response.content)) @@ -608,11 +906,12 @@ async def _do_generate( tool_call = await self.rails_manager.are_tool_calls_safe( response.tool_calls, llm_kwargs, enabled=tool_output_enabled ) + records.extend(tool_call.records) if not tool_call.is_safe: log.info("[%s] Tool call blocked: %s", req_id, tool_call.reason) if self._metrics_enabled: record_request_blocked(RailDirection.OUTPUT) - return {"role": "assistant", "content": REFUSAL_MESSAGE} + return _blocked_return() # Output rails check the final answer, not reasoning traces. # Reasoning is re-attached as tags only below so reasoning intentionally bypasses output @@ -623,25 +922,55 @@ async def _do_generate( if not is_tool_call_only: log.info("[%s] Running output rails", req_id) output_result = await self.rails_manager.is_output_safe(messages, response_text, enabled=output_enabled) + records.extend(output_result.records) if not output_result.is_safe: log.info("[%s] Output blocked: %s", req_id, output_result.reason) if self._metrics_enabled: record_request_blocked(RailDirection.OUTPUT) - return {"role": "assistant", "content": REFUSAL_MESSAGE} + return _blocked_return() + + if has_generation_options: + log_obj = _build_generation_log(records, options, time.monotonic() - t_start) + return _build_generation_response(response_text, reasoning_content, response, log_obj) - # TODO: Support returning GenerationResponse `reasoning_content` to match LLMRails - # For now, embed the reasoning on the content with think-tags + # Bare return path: reasoning is delivered inline as a prefix + # (LLMRails legacy shape). The structured path above instead puts it in the + # reasoning_content field and keeps the message content clean. if reasoning_content: response_text = f"{reasoning_content}\n" + response_text return _build_assistant_message(response_text, response.tool_calls) + async def _timed_main_call(self, messages: LLMMessages, llm_kwargs: dict) -> TimedLLMResponse: + """Call the main model, returning the response with wall-clock start/finish and a monotonic duration. + + Shared by the sequential and speculative paths so the main call's ``RailCallRecord`` + always carries real timing (the speculative path previously left it None). + """ + started_at = time.time() + t0 = time.monotonic() + response = await self.engine_registry.model_call("main", messages, **llm_kwargs) + return TimedLLMResponse( + response=response, + started_at=started_at, + finished_at=time.time(), + duration=time.monotonic() - t0, + ) + async def _do_generate_sequential( - self, messages: LLMMessages, req_id: str, llm_kwargs: dict, *, input_enabled: Union[bool, list[str]] = True + self, + messages: LLMMessages, + req_id: str, + llm_kwargs: dict, + *, + input_enabled: Union[bool, list[str]] = True, + records_out: Optional[list[RailCallRecord]] = None, ) -> Optional[LLMResponse]: """Sequential path: input rails block before LLM generation starts.""" log.info("[%s] Running input rails", req_id) input_result = await self.rails_manager.is_input_safe(messages, enabled=input_enabled) + if records_out is not None: + records_out.extend(input_result.records) if not input_result.is_safe: log.info("[%s] Input blocked: %s", req_id, input_result.reason) if self._metrics_enabled: @@ -649,7 +978,12 @@ async def _do_generate_sequential( return None log.info("[%s] Calling main LLM", req_id) - return await self.engine_registry.model_call("main", messages, **llm_kwargs) + timed_llm_response = await self._timed_main_call(messages, llm_kwargs) + if records_out is not None: + provider = self.engine_registry.provider_name("main") + prompt = serialize_prompt(messages) + records_out.append(_make_generation_record(timed_llm_response, provider, prompt)) + return timed_llm_response.response async def _do_generate_speculative( self, @@ -659,16 +993,22 @@ async def _do_generate_speculative( request_span: Optional["Span"] = None, *, input_enabled: Union[bool, list[str]] = True, + records_out: Optional[list[RailCallRecord]] = None, ) -> Optional[LLMResponse]: """Speculative path: input rails and LLM generation race concurrently.""" log.info("[%s] Speculative generation: launching input rails + LLM concurrently", req_id) rails_task = asyncio.create_task(self.rails_manager.is_input_safe(messages, enabled=input_enabled)) - gen_task = asyncio.create_task(self.engine_registry.model_call("main", messages, **llm_kwargs)) + gen_task = asyncio.create_task(self._timed_main_call(messages, llm_kwargs)) try: response = await self._parallel_input_rail_and_response_generation( - rails_task, gen_task, req_id, request_span + rails_task, + gen_task, + req_id, + request_span, + records_out=records_out, + main_prompt=serialize_prompt(messages), ) except BaseException as outer_exc: for t in (rails_task, gen_task): @@ -701,8 +1041,17 @@ async def _parallel_input_rail_and_response_generation( gen_task: asyncio.Task, req_id: str, request_span: Optional["Span"] = None, + *, + records_out: Optional[list[RailCallRecord]] = None, + main_prompt: str = "", ) -> Optional[LLMResponse]: """Race input rails against LLM generation, return LLMResponse or None (rejected).""" + + def _record_generation(timed: TimedLLMResponse) -> None: + if records_out is not None: + provider = self.engine_registry.provider_name("main") + records_out.append(_make_generation_record(timed, provider, main_prompt)) + done, _ = await asyncio.wait({rails_task, gen_task}, return_when=asyncio.FIRST_COMPLETED) first_completed = ( @@ -713,6 +1062,8 @@ async def _parallel_input_rail_and_response_generation( if rails_task in done: input_result = rails_task.result() + if records_out is not None: + records_out.extend(input_result.records) if not input_result.is_safe: log.info("[%s] Input blocked (speculative): %s", req_id, input_result.reason) @@ -721,7 +1072,10 @@ async def _parallel_input_rail_and_response_generation( # tasks finish simultaneously, gen_task may hold a stored exception that # would leak through suppress(CancelledError). gather drains it safely. gen_result = (await asyncio.gather(gen_task, return_exceptions=True))[0] - if isinstance(gen_result, BaseException) and not isinstance(gen_result, asyncio.CancelledError): + if isinstance(gen_result, TimedLLMResponse): + # Generation completed before cancellation took effect — record the call that was made. + _record_generation(gen_result) + elif isinstance(gen_result, BaseException) and not isinstance(gen_result, asyncio.CancelledError): log.warning("[%s] LLM generation error suppressed: %s", req_id, gen_result) if self._metrics_enabled: record_request_blocked(RailDirection.INPUT) @@ -731,16 +1085,19 @@ async def _parallel_input_rail_and_response_generation( return None # Rails passed — wait for generation to finish - response = await gen_task + timed = await gen_task set_speculative_span_attrs(request_span, first_completed, "none") else: # Generation finished first — wait for rails verdict - response = gen_task.result() + timed = gen_task.result() input_result = await rails_task + if records_out is not None: + records_out.extend(input_result.records) if not input_result.is_safe: log.info("[%s] Input blocked (speculative, gen-first): %s", req_id, input_result.reason) + _record_generation(timed) if self._metrics_enabled: record_request_blocked(RailDirection.INPUT) set_speculative_span_attrs( @@ -750,8 +1107,9 @@ async def _parallel_input_rail_and_response_generation( set_speculative_span_attrs(request_span, first_completed, "none") - log.debug("[%s] Main LLM response: %s", req_id, truncate(response.content)) - return response + log.debug("[%s] Main LLM response: %s", req_id, truncate(timed.response.content)) + _record_generation(timed) + return timed.response def check(self, messages: LLMMessages, rail_types: Optional[list[RailType]] = None) -> RailsResult: """Synchronous version of ``check_async``. diff --git a/nemoguardrails/guardrails/rail_action.py b/nemoguardrails/guardrails/rail_action.py index 5d4a74ade1..fa0cd7eb5d 100644 --- a/nemoguardrails/guardrails/rail_action.py +++ b/nemoguardrails/guardrails/rail_action.py @@ -23,7 +23,10 @@ from __future__ import annotations import logging +import time from abc import ABC, abstractmethod +from contextvars import ContextVar +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional, Union from nemoguardrails.guardrails.engine_registry import EngineRegistry @@ -31,12 +34,13 @@ LLMMessages, RailResult, get_request_id, + serialize_prompt, truncate, ) from nemoguardrails.guardrails.telemetry import action_span, record_span_error from nemoguardrails.llm.taskmanager import LLMTaskManager from nemoguardrails.rails.llm.config import _get_flow_model, _get_flow_name -from nemoguardrails.types import LLMResponse +from nemoguardrails.types import LLMResponse, UsageInfo if TYPE_CHECKING: from opentelemetry.trace import Tracer @@ -44,6 +48,40 @@ log = logging.getLogger(__name__) +@dataclass(frozen=True) +class RailLLMCall: + """Usage/model/timing captured for the call a rail made, for GenerationLog. + + Set by :meth:`RailAction._get_llm_response` (LLM rails) or + :meth:`RailAction._get_api_response` (API rails, e.g. jailbreak — usage/model None), + and read by RailsManager right after the rail runs (same async task), so the caller + can build the per-rail ``RailCallRecord``. ``started_at``/``finished_at`` are + wall-clock (``time.time()``) timestamps; ``duration`` is a monotonic delta. + """ + + usage: Optional[UsageInfo] + llm_model_name: Optional[str] + request_id: Optional[str] + provider_name: Optional[str] + prompt: Optional[str] + completion: Optional[str] + started_at: float + finished_at: float + duration: float + + +# Request-scoped: the last LLM call a rail made. ``RailAction.run`` clears it at the +# start of each rail so a rail that makes no model call leaves it None. +_rail_llm_call_var: ContextVar[Optional[RailLLMCall]] = ContextVar("rail_llm_call", default=None) + + +def get_and_clear_rail_llm_call_contextvar() -> Optional[RailLLMCall]: + """Return and clear the LLM call captured for the rail that just ran.""" + call = _rail_llm_call_var.get() + _rail_llm_call_var.set(None) + return call + + class RailAction(ABC): """Base class for all IORails rail actions. @@ -81,6 +119,9 @@ async def run( bot_response: Optional[str] = None, ) -> RailResult: """Execute the full rail pipeline and return a safety result.""" + # Clear any capture from a prior rail on this task; a rail that makes no model + # call then leaves it None and produces a record with no LLM call. + _rail_llm_call_var.set(None) with action_span(self._tracer, self.action_name) as span: req_id = get_request_id() base_flow = _get_flow_name(flow) @@ -153,10 +194,42 @@ async def _get_llm_response( messages: list[dict], **kwargs: Any, ) -> LLMResponse: - """Call an LLM via EngineRegistry and return the structured response.""" + """Call an LLM via EngineRegistry and return the structured response. + + Captures usage/model/timing into a request-scoped contextvar so RailsManager can + record this call in the GenerationLog after the rail runs. The capture happens in a + ``finally`` so a call that raises is still recorded as an attempt (usage/model/ + completion left None), matching LLMRails counting a failed call. + """ if not model_type: raise RuntimeError("model_type is required for LLM calls") - return await self.engine_registry.model_call(model_type, messages, **kwargs) + started_at = time.time() + t0 = time.monotonic() + usage: Optional[UsageInfo] = None + model_name: Optional[str] = None + request_id: Optional[str] = None + completion: Optional[str] = None + try: + response = await self.engine_registry.model_call(model_type, messages, **kwargs) + usage = response.usage + model_name = response.model + request_id = response.request_id + completion = response.content + return response + finally: + _rail_llm_call_var.set( + RailLLMCall( + usage=usage, + llm_model_name=model_name, + request_id=request_id, + provider_name=self.engine_registry.provider_name(model_type), + prompt=serialize_prompt(messages), + completion=completion, + started_at=started_at, + finished_at=time.time(), + duration=time.monotonic() - t0, + ) + ) async def _get_api_response( self, @@ -164,8 +237,31 @@ async def _get_api_response( body: dict[str, Any], **kwargs: Any, ) -> dict[str, Any]: - """Call an API endpoint via EngineRegistry and return the response dict.""" - return await self.engine_registry.api_call(api_name, body, **kwargs) + """Call an API endpoint via EngineRegistry and return the response dict. + + Records the call (with no token usage or model name) so API-backed rails such as + jailbreak detection still appear in the GenerationLog's ``llm_calls`` and counts, + matching LLMRails. Recorded in a ``finally`` so a call that raises is still counted + as an attempt. + """ + started_at = time.time() + t0 = time.monotonic() + try: + return await self.engine_registry.api_call(api_name, body, **kwargs) + finally: + _rail_llm_call_var.set( + RailLLMCall( + usage=None, + llm_model_name=None, + request_id=None, + provider_name=None, + prompt=None, + completion=None, + started_at=started_at, + finished_at=time.time(), + duration=time.monotonic() - t0, + ) + ) async def _get_local_response(self, **kwargs: Any) -> Any: """Run a local/in-process check. Override in subclasses that need it.""" diff --git a/nemoguardrails/guardrails/rails_manager.py b/nemoguardrails/guardrails/rails_manager.py index d296289ff1..633001ab85 100644 --- a/nemoguardrails/guardrails/rails_manager.py +++ b/nemoguardrails/guardrails/rails_manager.py @@ -37,16 +37,17 @@ from nemoguardrails.guardrails.actions.topic_safety_action import TopicSafetyInputAction from nemoguardrails.guardrails.engine_registry import EngineRegistry from nemoguardrails.guardrails.guardrails_types import ( + RailCallRecord, RailDirection, RailResult, get_request_id, ) -from nemoguardrails.guardrails.rail_action import RailAction +from nemoguardrails.guardrails.rail_action import RailAction, RailLLMCall, get_and_clear_rail_llm_call_contextvar from nemoguardrails.guardrails.telemetry import mark_rail_stop, rail_span, set_rail_content from nemoguardrails.guardrails.tool_rail_action import ToolRailAction from nemoguardrails.guardrails.tool_schema import ToolExchange, Toolset from nemoguardrails.llm.taskmanager import LLMTaskManager -from nemoguardrails.rails.llm.config import _get_flow_name +from nemoguardrails.rails.llm.config import _get_flow_model, _get_flow_name from nemoguardrails.types import ToolCall if TYPE_CHECKING: @@ -79,6 +80,39 @@ _ToolActionT = TypeVar("_ToolActionT", bound=ToolRailAction) +def _rail_call_record(flow: str, rail_type: str, result: RailResult, call: Optional[RailLLMCall]) -> RailCallRecord: + """Build the per-rail GenerationLog record from a rail's result + its captured LLM call. + + ``call`` is None for model-free rails (e.g. tool validators); ``return_value`` uses the + action's structured verdict when present, else a minimal ``{"allowed": is_safe}``. + """ + verdict = result.return_value if result.return_value is not None else {"allowed": result.is_safe} + # GenerationLog parity with LLMRails: action_name/task use the prompt-template key + # (underscores) rather than the space-separated Colang flow name; ``flow`` keeps spaces. + base_name = _get_flow_name(flow) or flow + model = _get_flow_model(flow) + action_name = base_name.replace(" ", "_") + task = f"{action_name} $model={model}" if model else action_name + return RailCallRecord( + flow=flow, + rail_type=rail_type, + is_safe=result.is_safe, + made_call=call is not None, + action_name=action_name, + return_value=verdict, + task=task, + request_id=call.request_id if call else None, + usage=call.usage if call else None, + llm_model_name=call.llm_model_name if call else None, + llm_provider_name=call.provider_name if call else None, + prompt=call.prompt if call else None, + completion=call.completion if call else None, + started_at=call.started_at if call else None, + finished_at=call.finished_at if call else None, + duration=call.duration if call else None, + ) + + class RailsManager: """Orchestrates input and output safety checks for IORails. @@ -304,8 +338,10 @@ async def _run_rail( with rail_span(self._tracer, flow, direction) as span: action = self._actions[flow] result = await action.run(flow, messages, bot_response) + call = get_and_clear_rail_llm_call_contextvar() if not result.is_safe and result.triggered_rail is None: result = replace(result, triggered_rail=_get_flow_name(flow) or flow) + result = replace(result, records=(_rail_call_record(flow, direction.value.lower(), result, call),)) mark_rail_stop(span, result.is_safe) # Capture rail input + block reason after the action runs. # RailAction.run() catches its own exceptions and returns @@ -324,6 +360,9 @@ async def _run_tool_call_rail(self, flow: str, tool_calls: list[ToolCall], tools """Dispatch a single tool-call rail to its action, wrapped in an OUTPUT rail span.""" with rail_span(self._tracer, flow, RailDirection.OUTPUT) as span: result = await self._tool_call_actions[flow].run(toolset, tool_calls) + # Tool rails are model-free, so no LLM call is captured (usage stays None). + call = get_and_clear_rail_llm_call_contextvar() + result = replace(result, records=(_rail_call_record(flow, "tool_output", result, call),)) mark_rail_stop(span, result.is_safe) if self._content_capture_enabled: set_rail_content( @@ -346,6 +385,9 @@ async def _run_tool_result_rail(self, flow: str, exchanges: list[ToolExchange]) result = await action.run(exchange.results, exchange.calls) if not result.is_safe: break + # Tool rails are model-free, so no LLM call is captured (usage stays None). + call = get_and_clear_rail_llm_call_contextvar() + result = replace(result, records=(_rail_call_record(flow, "tool_input", result, call),)) mark_rail_stop(span, result.is_safe) if self._content_capture_enabled: all_results = [r for exchange in exchanges for r in exchange.results] @@ -368,14 +410,16 @@ async def _run_rails_sequential( """Run rail coroutines sequentially, short-circuiting on first unsafe result.""" req_id = get_request_id() remaining = iter(rails.items()) + collected: list[RailCallRecord] = [] try: for flow, coro in remaining: result = await coro + collected.extend(result.records) log.debug("[%s] %s flow %s result %s", req_id, direction.value, flow, result) if not result.is_safe: log.info("[%s] %s flow %s blocked", req_id, direction.value, flow) - return result - return RailResult(is_safe=True) + return replace(result, records=tuple(collected)) + return RailResult(is_safe=True, records=tuple(collected)) finally: for _, coro in remaining: coro.close() @@ -385,34 +429,36 @@ async def _run_rails_parallel( rails: Mapping[str, Coroutine[Any, Any, RailResult]], direction: RailDirection, ) -> RailResult: - """Run rail coroutines concurrently, cancelling remaining on first unsafe result.""" + """Run rail coroutines concurrently; on the first unsafe result, finish draining its + completion batch (so rails that finished alongside it still contribute their records) + before cancelling the rest.""" req_id = get_request_id() task_to_flow: dict[asyncio.Task, str] = {asyncio.create_task(coro): flow for flow, coro in rails.items()} tasks = list(task_to_flow.keys()) task_order = {task: i for i, task in enumerate(tasks)} pending_tasks: set[asyncio.Task] = set(tasks) + collected: list[RailCallRecord] = [] try: while pending_tasks: done, pending_tasks = await asyncio.wait(pending_tasks, return_when=asyncio.FIRST_COMPLETED) + first_unsafe: Optional[RailResult] = None for task in sorted(done, key=lambda t: task_order[t]): result = task.result() + collected.extend(result.records) flow = task_to_flow[task] log.debug("[%s] %s flow %s result %s", req_id, direction.value, flow, result) - if not result.is_safe: - log.info( - "[%s] %s flow %s blocked (cancelling %d remaining)", - req_id, - direction.value, - flow, - len(pending_tasks), - ) + if not result.is_safe and first_unsafe is None: + first_unsafe = result + log.info("[%s] %s flow %s blocked", req_id, direction.value, flow) + if first_unsafe is not None: + if pending_tasks: + log.info("[%s] %s cancelling %d remaining", req_id, direction.value, len(pending_tasks)) for t in pending_tasks: t.cancel() - if pending_tasks: - await asyncio.wait(pending_tasks) - return result - return RailResult(is_safe=True) + await asyncio.wait(pending_tasks) + return replace(first_unsafe, records=tuple(collected)) + return RailResult(is_safe=True, records=tuple(collected)) except BaseException: for t in tasks: if not t.done(): diff --git a/tests/guardrails/test_guardrails.py b/tests/guardrails/test_guardrails.py index d573fb0daa..865aed12b5 100644 --- a/tests/guardrails/test_guardrails.py +++ b/tests/guardrails/test_guardrails.py @@ -123,8 +123,8 @@ async def mock_stream(): guardrails.update_llm(mock_new_llm) # Verify all calls went to LLMRails - guardrails.rails_engine.generate.assert_called_once_with(messages=messages) - guardrails.rails_engine.generate_async.assert_called_once_with(messages=messages) + guardrails.rails_engine.generate.assert_called_once_with(prompt=None, messages=messages, options=None) + guardrails.rails_engine.generate_async.assert_called_once_with(prompt=None, messages=messages, options=None) guardrails.rails_engine.stream_async.assert_called_once_with(messages=messages) guardrails.rails_engine.explain.assert_called_once() guardrails.rails_engine.update_llm.assert_called_once_with(mock_new_llm) @@ -178,8 +178,8 @@ async def mock_stream(): with pytest.raises(NotImplementedError, match="IORails doesn't support update_llm()"): guardrails.update_llm(mock_new_llm) - guardrails.rails_engine.generate.assert_called_once_with(messages=messages) - guardrails.rails_engine.generate_async.assert_called_once_with(messages=messages) + guardrails.rails_engine.generate.assert_called_once_with(prompt=None, messages=messages, options=None) + guardrails.rails_engine.generate_async.assert_called_once_with(prompt=None, messages=messages, options=None) guardrails.rails_engine.stream_async.assert_called_once_with( messages=messages, options=None, @@ -257,8 +257,8 @@ async def mock_stream(): guardrails.update_llm(mock_new_llm) # Verify all calls went to LLMRails - guardrails.rails_engine.generate.assert_called_once_with(messages=messages) - guardrails.rails_engine.generate_async.assert_called_once_with(messages=messages) + guardrails.rails_engine.generate.assert_called_once_with(prompt=None, messages=messages, options=None) + guardrails.rails_engine.generate_async.assert_called_once_with(prompt=None, messages=messages, options=None) guardrails.rails_engine.stream_async.assert_called_once_with(messages=messages) guardrails.rails_engine.explain.assert_called_once() guardrails.rails_engine.update_llm.assert_called_once_with(mock_new_llm) @@ -553,93 +553,13 @@ def test_use_iorails_false_overrides_require_iorails( mock_llmrails_class.assert_called_once_with(_content_safety_rails_config, None, False) -class TestConvertToMessages: - """Tests for the _convert_to_messages static method.""" - - def test_prompt_string(self): - """Test conversion of string prompt to LLMMessages.""" - result = Guardrails._convert_to_messages(prompt="Hello, how are you?") - - expected = [{"role": "user", "content": "Hello, how are you?"}] - assert result == expected - - def test_empty_string_prompt(self): - """Test conversion of empty string prompt raises ValueError.""" - # Empty string is falsy, so it should raise an error - with pytest.raises(ValueError, match="Neither prompt nor messages provided"): - Guardrails._convert_to_messages(prompt="") - - def test_messages_single_message(self): - """Test conversion with single message.""" - messages = [{"role": "user", "content": "What is the weather?"}] - result = Guardrails._convert_to_messages(messages=messages) - assert result == messages - - def test_messages_multiple_messages(self): - """Test conversion with multiple messages.""" - messages = [ - {"role": "user", "content": "What is AI?"}, - {"role": "assistant", "content": "AI is artificial intelligence."}, - {"role": "user", "content": "Tell me more."}, - ] - result = Guardrails._convert_to_messages(messages=messages) - - assert result == messages - - def test_messages_with_system_message(self): - """Test conversion with system message.""" - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ] - result = Guardrails._convert_to_messages(messages=messages) - - assert result == messages - - def test_empty_messages_list(self): - """Test conversion with empty messages list raises ValueError.""" - # Empty list is falsy, so it should raise an error - messages = [] - with pytest.raises(ValueError, match="Neither prompt nor messages provided"): - Guardrails._convert_to_messages(messages=messages) - - def test_messages_take_priority_over_prompt(self): - """Test that messages parameter takes priority when both are provided.""" - messages = [{"role": "user", "content": "From messages"}] - result = Guardrails._convert_to_messages(prompt="From prompt", messages=messages) - assert result == messages - - def test_neither_prompt_nor_messages_raises_error(self): - """Test that providing neither prompt nor messages raises ValueError.""" - with pytest.raises(ValueError, match="Neither prompt nor messages provided"): - Guardrails._convert_to_messages() - - def test_multiline_string_prompt(self): - """Test conversion of multiline string prompt.""" - multiline_prompt = """Line 1 -Line 2 -Line 3""" - result = Guardrails._convert_to_messages(prompt=multiline_prompt) - - expected = [{"role": "user", "content": multiline_prompt}] - assert result == expected - - def test_string_prompt_with_special_characters(self): - """Test conversion of string prompt with special characters.""" - special_prompt = "Hello! @#$%^&*() How's the weather? \"quoted\" 'text'" - result = Guardrails._convert_to_messages(prompt=special_prompt) - - expected = [{"role": "user", "content": special_prompt}] - assert result == expected - - class TestGenerateAsync: """Tests for the asynchronous generate_async method.""" @pytest.mark.asyncio @patch("nemoguardrails.guardrails.guardrails.LLMRails") async def test_generate_async_with_string_prompt(self, mock_llmrails_class, _nemoguards_rails_config): - """Test generate_async method with a string prompt using context manager.""" + """generate_async passes a string prompt through to the engine unchanged (facade is a passthrough).""" mock_llmrails_instance = MagicMock() mock_llmrails_class.return_value = mock_llmrails_instance mock_llmrails_instance.generate_async = AsyncMock(return_value="Async response") @@ -647,9 +567,9 @@ async def test_generate_async_with_string_prompt(self, mock_llmrails_class, _nem async with Guardrails(config=_nemoguards_rails_config, use_iorails=False) as guardrails: result = await guardrails.generate_async(prompt="Hello async!") - # Verify generate_async was called with correct messages - expected_messages = [{"role": "user", "content": "Hello async!"}] - mock_llmrails_instance.generate_async.assert_awaited_once_with(messages=expected_messages) + mock_llmrails_instance.generate_async.assert_awaited_once_with( + prompt="Hello async!", messages=None, options=None + ) assert result == "Async response" @pytest.mark.asyncio @@ -668,7 +588,7 @@ async def test_generate_async_with_messages(self, mock_llmrails_class, _nemoguar ] result = await guardrails.generate_async(messages=messages) - mock_llmrails_instance.generate_async.assert_awaited_once_with(messages=messages) + mock_llmrails_instance.generate_async.assert_awaited_once_with(prompt=None, messages=messages, options=None) assert result == "Async conversation response" @pytest.mark.asyncio @@ -682,10 +602,8 @@ async def test_generate_async_with_kwargs(self, mock_llmrails_class, _nemoguards async with Guardrails(config=_nemoguards_rails_config, use_iorails=False) as guardrails: result = await guardrails.generate_async(prompt="Test", temperature=0.5, top_p=0.9) - # Verify kwargs were passed through - expected_messages = [{"role": "user", "content": "Test"}] mock_llmrails_instance.generate_async.assert_awaited_once_with( - messages=expected_messages, temperature=0.5, top_p=0.9 + prompt="Test", messages=None, options=None, temperature=0.5, top_p=0.9 ) assert result == "Response" @@ -897,10 +815,11 @@ def test_generate_with_additional_parameters(self, mock_llmrails_class, _nemogua top_p=0.9, ) - # Verify all kwargs were passed through - expected_messages = [{"role": "user", "content": "Test"}] + # The facade is a passthrough; prompt/messages are forwarded to the engine verbatim. mock_llmrails_instance.generate.assert_called_once_with( - messages=expected_messages, + prompt="Test", + messages=None, + options=None, temperature=0.7, max_tokens=100, top_p=0.9, @@ -2058,3 +1977,24 @@ async def test_jailbreak_input_check_blocked(self): assert result.status == RailStatus.BLOCKED assert result.rail == "jailbreak detection model" assert result.content == REFUSAL_MESSAGE + + +class TestOptionsForwarding: + """The facade forwards a caller-supplied ``options`` object to the engine verbatim.""" + + @pytest.mark.asyncio + @patch.object(LLMRails, "__init__", return_value=None) + async def test_options_forwarded_unchanged(self, _mock_init, _content_safety_rails_config): + """A non-None options object reaches both generate and generate_async as the same instance.""" + options = GenerationOptions() + messages = [{"role": "user", "content": "hi"}] + + async with Guardrails(config=_content_safety_rails_config, use_iorails=False) as guardrails: + guardrails.rails_engine.generate = MagicMock(return_value="sync") + guardrails.rails_engine.generate_async = AsyncMock(return_value="async") + + guardrails.generate(messages=messages, options=options) + await guardrails.generate_async(messages=messages, options=options) + + assert guardrails.rails_engine.generate.call_args.kwargs["options"] is options + assert guardrails.rails_engine.generate_async.call_args.kwargs["options"] is options diff --git a/tests/guardrails/test_iorails.py b/tests/guardrails/test_iorails.py index ce287a47b0..11503da8e6 100644 --- a/tests/guardrails/test_iorails.py +++ b/tests/guardrails/test_iorails.py @@ -26,7 +26,7 @@ from nemoguardrails.guardrails.iorails import REFUSAL_MESSAGE, IORails from nemoguardrails.guardrails.model_engine import ModelEngine from nemoguardrails.rails.llm.config import RailsConfig -from nemoguardrails.rails.llm.options import GenerationOptions +from nemoguardrails.rails.llm.options import GenerationOptions, GenerationResponse from nemoguardrails.types import LLMResponse, LLMResponseChunk, ToolCall, ToolCallFunction from tests.guardrails.test_data import CONTENT_SAFETY_CONFIG, NEMOGUARDS_CONFIG @@ -81,7 +81,7 @@ async def test_safe_input_and_output(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content=llm_response)) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": llm_response} iorails.rails_manager.is_input_safe.assert_called_once_with(messages, enabled=True) @@ -90,7 +90,7 @@ async def test_safe_input_and_output(self, iorails): @pytest.mark.asyncio async def test_safe_input_and_output_with_generation_options(self, iorails): - """Returns LLM response when both input and output rails pass.""" + """Passing options returns a GenerationResponse wrapping the LLM response.""" messages = [{"role": "user", "content": "hi"}] llm_response = "Hello from LLM" @@ -101,9 +101,10 @@ async def test_safe_input_and_output_with_generation_options(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content=llm_response)) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails.generate_async(messages, options=options) + result = await iorails.generate_async(messages=messages, options=options) - assert result == {"role": "assistant", "content": llm_response} + assert isinstance(result, GenerationResponse) + assert result.response == [{"role": "assistant", "content": llm_response}] iorails.rails_manager.is_input_safe.assert_called_once_with(messages, enabled=True) iorails.engine_registry.model_call.assert_called_once_with("main", messages, **llm_params) iorails.rails_manager.is_output_safe.assert_called_once_with(messages, llm_response, enabled=True) @@ -129,7 +130,7 @@ async def mock_output_safe(messages, response, *, enabled=True): iorails.engine_registry.model_call = mock_generate iorails.rails_manager.is_output_safe = mock_output_safe - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert call_order == ["input", "generate", "output"] @pytest.mark.asyncio @@ -140,7 +141,7 @@ async def test_unsafe_input(self, iorails): iorails.rails_manager.is_output_safe = AsyncMock() messages = [{"role": "user", "content": "bad input"}] - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} iorails.rails_manager.is_input_safe.assert_called_once_with(messages, enabled=True) @@ -157,7 +158,7 @@ async def test_unsafe_output(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content=llm_response)) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=False, reason="blocked")) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} iorails.rails_manager.is_input_safe.assert_called_once_with(messages, enabled=True) @@ -175,7 +176,7 @@ async def test_unsafe_output_records_block_metric_when_metrics_enabled(self, ior iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=False, reason="blocked")) with patch("nemoguardrails.guardrails.iorails.record_request_blocked") as record_blocked: - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} record_blocked.assert_called_once_with(RailDirection.OUTPUT) @@ -190,7 +191,7 @@ async def test_dict_options_forwarded(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="ok")) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - await iorails.generate_async(messages, options={"llm_params": llm_params}) + await iorails.generate_async(messages=messages, options={"llm_params": llm_params}) iorails.engine_registry.model_call.assert_called_once_with("main", messages, **llm_params) @@ -201,7 +202,7 @@ async def test_generate_async_propagates_exception(self, iorails): iorails.engine_registry.model_call = AsyncMock(side_effect=RuntimeError("LLM internal error")) with pytest.raises(RuntimeError, match="LLM internal error"): - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) class TestToolCalling: @@ -218,7 +219,7 @@ async def test_tools_in_llm_params_forwarded_to_model(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="ok")) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - await iorails.generate_async(messages, options=options) + await iorails.generate_async(messages=messages, options=options) iorails.engine_registry.model_call.assert_called_once_with( "main", messages, tools=[tool], tool_choice="auto", parallel_tool_calls=True @@ -240,7 +241,7 @@ async def test_tool_calls_returned_on_assistant_message(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="", tool_calls=tool_calls)) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result["role"] == "assistant" assert result["content"] is None @@ -258,7 +259,7 @@ async def test_output_rails_skipped_for_tool_call_only_response(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="", tool_calls=tool_calls)) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) iorails.rails_manager.is_output_safe.assert_not_called() assert result["tool_calls"][0]["function"]["name"] == "f" @@ -275,7 +276,7 @@ async def test_text_with_tool_calls_runs_output_rails_and_returns_both(self, ior ) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) iorails.rails_manager.is_output_safe.assert_called_once_with(messages, "some text", enabled=True) assert result["content"] == "some text" @@ -667,7 +668,7 @@ async def test_generate_async_calls_start(self, iorails): iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) assert not iorails._running - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) iorails.engine_registry.start.assert_called_once() assert iorails._running @@ -680,8 +681,8 @@ async def test_generate_async_start_is_idempotent(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="ok")) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - await iorails.generate_async([{"role": "user", "content": "hi"}]) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) iorails.engine_registry.start.assert_called_once() @@ -745,7 +746,7 @@ def test_generate_delegates_to_generate_async(self, iorails_sync): # Patch IORails so the temp instance inside generate() uses our mocked iorails with patch("nemoguardrails.guardrails.iorails.IORails", return_value=iorails): - result = iorails.generate(messages) + result = iorails.generate(messages=messages) assert result == expected @@ -760,7 +761,7 @@ def test_generate_passes_kwargs(self, iorails_sync): iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) with patch("nemoguardrails.guardrails.iorails.IORails", return_value=iorails): - iorails.generate(messages, options=options) + iorails.generate(messages=messages, options=options) iorails.engine_registry.model_call.assert_called_once_with("main", messages, temperature=0.5) @@ -774,7 +775,7 @@ def test_generate_marks_temp_engine_as_internal(self, iorails_sync): iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) with patch("nemoguardrails.guardrails.iorails.IORails", return_value=iorails) as mock_iorails: - iorails.generate(messages) + iorails.generate(messages=messages) mock_iorails.assert_called_once() assert mock_iorails.call_args.kwargs == {"_report_usage": False} @@ -783,7 +784,7 @@ def test_generate_raises_when_called_from_async_loop(self, iorails_sync): """generate() raises RuntimeError when called inside a running event loop.""" async def call_generate(): - iorails_sync.generate([{"role": "user", "content": "hi"}]) + iorails_sync.generate(messages=[{"role": "user", "content": "hi"}]) with pytest.raises(RuntimeError): asyncio.run(call_generate()) @@ -849,7 +850,7 @@ async def test_main_model_url_not_doubled(self, base_url_input): main_engine._running = True try: - await iorails.generate_async([{"role": "user", "content": "Hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "Hi"}]) finally: await iorails.stop() diff --git a/tests/guardrails/test_iorails_generation_log.py b/tests/guardrails/test_iorails_generation_log.py new file mode 100644 index 0000000000..cbcbdf8c22 --- /dev/null +++ b/tests/guardrails/test_iorails_generation_log.py @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GenerationLog + token usage for IORails (PR B), matching LLMRails' log contract. + +When ``options.log`` requests ``activated_rails`` and/or ``llm_calls``, IORails +synthesizes a ``GenerationLog`` from the per-rail ``RailCallRecord``s carried on each +``RailResult`` plus the main generation call: ``activated_rails`` (one synthetic +``ExecutedAction`` per rail carrying the real verdict as ``return_value``), a flat +``llm_calls`` list (main + every rail), and aggregate ``stats``. ``internal_events`` +and ``colang_history`` are Colang-runtime-only and raise ``NotImplementedError``. +Token usage lives only in ``log`` — it is no longer surfaced under ``llm_metadata``. +""" + +from unittest.mock import AsyncMock + +import pytest +import pytest_asyncio + +from nemoguardrails.guardrails.guardrails_types import RailCallRecord, RailResult +from nemoguardrails.guardrails.iorails import IORails, _activated_rail +from nemoguardrails.rails.llm.options import GenerationResponse +from nemoguardrails.types import LLMResponse, UsageInfo +from tests.guardrails.async_helpers import started_iorails +from tests.guardrails.test_data import NEMOGUARDS_CONFIG + +_USER = [{"role": "user", "content": "hi"}] + +_CS_INPUT_FLOW = "content safety check input $model=content_safety" +_CS_OUTPUT_FLOW = "content safety check output $model=content_safety" + + +@pytest_asyncio.fixture +async def iorails(): + """Started IORails instance with worker-queue teardown after each test.""" + async with started_iorails(NEMOGUARDS_CONFIG) as iorails: + yield iorails + + +def _input_record(*, is_safe: bool = True, return_value=None) -> RailCallRecord: + """A content-safety input-rail record with a NeMoGuard-style verdict + usage.""" + return RailCallRecord( + flow=_CS_INPUT_FLOW, + rail_type="input", + is_safe=is_safe, + action_name="content_safety_check_input", + return_value=return_value if return_value is not None else {"allowed": is_safe, "policy_violations": []}, + task="content_safety_check_input $model=content_safety", + usage=UsageInfo(input_tokens=762, output_tokens=8, total_tokens=770), + llm_model_name="nvidia/llama-3.1-nemoguard-8b-content-safety", + llm_provider_name="nim", + started_at=1.0, + finished_at=1.5, + duration=0.5, + ) + + +def _output_record() -> RailCallRecord: + """A content-safety output-rail record with usage.""" + return RailCallRecord( + flow=_CS_OUTPUT_FLOW, + rail_type="output", + is_safe=True, + action_name="content_safety_check_output", + return_value={"allowed": True, "policy_violations": []}, + task="content_safety_check_output $model=content_safety", + usage=UsageInfo(input_tokens=855, output_tokens=15, total_tokens=870), + llm_model_name="nvidia/llama-3.1-nemoguard-8b-content-safety", + llm_provider_name="nim", + started_at=3.0, + finished_at=3.4, + duration=0.4, + ) + + +def _stub_pipeline(iorails: IORails, *, input_records=(), output_records=(), input_safe=True) -> None: + """Stub input/output rails to return given records, and a main call with usage.""" + iorails.rails_manager.is_input_safe = AsyncMock( + return_value=RailResult( + is_safe=input_safe, + reason=None if input_safe else "unsafe", + triggered_rail=None if input_safe else _CS_INPUT_FLOW, + records=tuple(input_records), + ) + ) + iorails.rails_manager.is_output_safe = AsyncMock( + return_value=RailResult(is_safe=True, records=tuple(output_records)) + ) + iorails.engine_registry.model_call = AsyncMock( + return_value=LLMResponse(content="Hi", usage=UsageInfo(input_tokens=99, output_tokens=50, total_tokens=149)) + ) + + +class TestLogGating: + """`log` is only built when explicitly requested.""" + + @pytest.mark.asyncio + async def test_log_none_when_not_requested(self, iorails): + """With options but no log flags, `res.log` stays None.""" + _stub_pipeline(iorails, input_records=[_input_record()]) + + result = await iorails.generate_async(messages=_USER, options={}) + + assert isinstance(result, GenerationResponse) + assert result.log is None + + +class TestLlmCallsAndStats: + """Flat `llm_calls` list and aggregate `stats` cover the main call plus every rail.""" + + @pytest.mark.asyncio + async def test_llm_calls_and_stats_aggregate_main_and_rails(self, iorails): + """`log.llm_calls` lists main + each rail call; `log.stats` sums their tokens.""" + _stub_pipeline(iorails, input_records=[_input_record()], output_records=[_output_record()]) + + result = await iorails.generate_async(messages=_USER, options={"log": {"llm_calls": True}}) + + assert result.log is not None + stats = result.log.stats + assert stats.llm_calls_count == 3 + assert stats.llm_calls_total_prompt_tokens == 762 + 855 + 99 + assert stats.llm_calls_total_completion_tokens == 8 + 15 + 50 + assert stats.llm_calls_total_tokens == 770 + 870 + 149 + + totals = sorted(call.total_tokens for call in result.log.llm_calls) + assert totals == [149, 770, 870] + + +class TestActivatedRails: + """`activated_rails` carries one synthetic action per rail with the real verdict.""" + + @pytest.mark.asyncio + async def test_activated_rail_carries_verdict_as_return_value(self, iorails): + """The rail's structured verdict is preserved as executed_actions[0].return_value.""" + verdict = {"allowed": False, "policy_violations": ["Violence", "Criminal Planning/Confessions"]} + _stub_pipeline(iorails, input_records=[_input_record(is_safe=False, return_value=verdict)], input_safe=False) + + result = await iorails.generate_async(messages=_USER, options={"log": {"activated_rails": True}}) + + assert result.log is not None + rail = next(r for r in result.log.activated_rails if r.name == _CS_INPUT_FLOW) + assert rail.type == "input" + assert rail.executed_actions[0].action_name == "content_safety_check_input" + assert rail.executed_actions[0].return_value == verdict + + @pytest.mark.asyncio + async def test_stop_set_on_blocking_rail(self, iorails): + """A blocking input rail is marked stop=True, and the blocked request still logs it.""" + _stub_pipeline(iorails, input_records=[_input_record(is_safe=False)], input_safe=False) + + result = await iorails.generate_async(messages=_USER, options={"log": {"activated_rails": True}}) + + assert isinstance(result, GenerationResponse) + assert result.log is not None + rail = next(r for r in result.log.activated_rails if r.name == _CS_INPUT_FLOW) + assert rail.stop is True + + +class TestUnsupportedLogOptions: + """Colang-runtime-only log fields raise NotImplementedError.""" + + @pytest.mark.asyncio + async def test_internal_events_raises(self, iorails): + """Requesting internal_events raises — IORails runs no Colang event stream.""" + _stub_pipeline(iorails, input_records=[_input_record()]) + + with pytest.raises(NotImplementedError): + await iorails.generate_async(messages=_USER, options={"log": {"internal_events": True}}) + + @pytest.mark.asyncio + async def test_colang_history_raises(self, iorails): + """Requesting colang_history raises — IORails produces no Colang transcript.""" + _stub_pipeline(iorails, input_records=[_input_record()]) + + with pytest.raises(NotImplementedError): + await iorails.generate_async(messages=_USER, options={"log": {"colang_history": True}}) + + +class TestUsageRemovedFromLlmMetadata: + """Token usage moved to `log`; `llm_metadata` is a pure provider_metadata passthrough.""" + + @pytest.mark.asyncio + async def test_llm_metadata_has_no_usage_key(self, iorails): + """provider_metadata is surfaced verbatim; no `usage` sub-key is added.""" + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = AsyncMock( + return_value=LLMResponse( + content="Hi", + provider_metadata={"response_headers": {"nvcf-status": "fulfilled"}}, + usage=UsageInfo(input_tokens=10, output_tokens=5, total_tokens=15), + ) + ) + + result = await iorails.generate_async(messages=_USER, options={}) + + assert result.llm_metadata == {"response_headers": {"nvcf-status": "fulfilled"}} + + @pytest.mark.asyncio + async def test_llm_metadata_none_when_only_usage(self, iorails): + """With usage but no provider_metadata, llm_metadata is None (usage no longer graft-in).""" + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = AsyncMock( + return_value=LLMResponse(content="Hi", usage=UsageInfo(input_tokens=10, output_tokens=5, total_tokens=15)) + ) + + result = await iorails.generate_async(messages=_USER, options={}) + + assert result.llm_metadata is None + + +class TestActivatedRailHelper: + """`_activated_rail` maps a RailCallRecord to an ActivatedRail + synthetic ExecutedAction.""" + + def test_record_with_llm_call(self): + """A model-backed rail yields one ExecutedAction with its verdict and a single LLMCallInfo.""" + record = RailCallRecord( + flow=_CS_INPUT_FLOW, + rail_type="input", + is_safe=False, + action_name="content_safety_check_input", + return_value={"allowed": False, "policy_violations": ["Violence"]}, + task="content_safety_check_input $model=content_safety", + usage=UsageInfo(input_tokens=762, output_tokens=22, total_tokens=784), + llm_model_name="nvidia/llama-3.1-nemoguard-8b-content-safety", + llm_provider_name="nim", + started_at=1.0, + finished_at=1.5, + duration=0.5, + ) + + rail = _activated_rail(record) + + assert rail.type == "input" + assert rail.name == _CS_INPUT_FLOW + assert rail.stop is True + assert rail.duration == 0.5 + action = rail.executed_actions[0] + assert action.action_name == "content_safety_check_input" + assert action.return_value == {"allowed": False, "policy_violations": ["Violence"]} + assert len(action.llm_calls) == 1 + assert action.llm_calls[0].total_tokens == 784 + assert action.llm_calls[0].llm_model_name == "nvidia/llama-3.1-nemoguard-8b-content-safety" + + def test_record_without_llm_call(self): + """A model-free rail (usage=None) yields an empty llm_calls list; action_name falls back to flow.""" + record = RailCallRecord( + flow="tool call validation", + rail_type="tool_output", + is_safe=True, + action_name=None, + return_value={"allowed": True}, + ) + + rail = _activated_rail(record) + + assert rail.type == "tool_output" + assert rail.stop is False + action = rail.executed_actions[0] + assert action.action_name == "tool call validation" + assert action.llm_calls == [] + assert action.return_value == {"allowed": True} + + def test_api_rail_made_call_without_usage_counts(self): + """An API rail (made_call=True, usage=None, e.g. jailbreak) still yields one llm_call.""" + record = RailCallRecord( + flow="jailbreak detection model", + rail_type="input", + is_safe=True, + made_call=True, + action_name="jailbreak detection model", + return_value=False, + task="jailbreak detection model", + ) + + rail = _activated_rail(record) + + assert len(rail.executed_actions[0].llm_calls) == 1 + assert rail.executed_actions[0].llm_calls[0].total_tokens is None + assert rail.executed_actions[0].return_value is False + + def test_record_maps_prompt_and_completion(self): + """A record's captured prompt/completion surface on the mapped LLMCallInfo.""" + record = RailCallRecord( + flow=_CS_INPUT_FLOW, + rail_type="input", + is_safe=True, + made_call=True, + action_name="content_safety_check_input", + return_value={"allowed": True}, + task="content_safety_check_input $model=content_safety", + usage=UsageInfo(input_tokens=10, output_tokens=2, total_tokens=12), + prompt="user: hi", + completion='{"User Safety": "safe"}', + ) + + call = _activated_rail(record).executed_actions[0].llm_calls[0] + + assert call.prompt == "user: hi" + assert call.completion == '{"User Safety": "safe"}' + + def test_api_rail_record_has_no_prompt_or_completion(self): + """An API rail (jailbreak) captures no content, so prompt/completion map to None.""" + record = RailCallRecord( + flow="jailbreak detection model", + rail_type="input", + is_safe=True, + made_call=True, + action_name="jailbreak_detection_model", + return_value=False, + task="jailbreak_detection_model", + ) + + call = _activated_rail(record).executed_actions[0].llm_calls[0] + + assert call.prompt is None + assert call.completion is None diff --git a/tests/guardrails/test_iorails_generation_log_capture.py b/tests/guardrails/test_iorails_generation_log_capture.py new file mode 100644 index 0000000000..01289154fc --- /dev/null +++ b/tests/guardrails/test_iorails_generation_log_capture.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real rail-call capture → GenerationLog for IORails (PR B, Phase 2b). + +Exercises the ACTUAL RailAction + RailsManager pipeline (not stubbed at the +rails-manager level): a rail's model call is captured (usage/model/timing) and its +verdict preserved onto the RailResult, then surfaced as per-rail RailCallRecords that +IORails turns into GenerationLog entries. Only the engine's ``model_call`` is mocked, +so the real content-safety actions parse the responses and produce real records. +""" + +import json +from unittest.mock import AsyncMock + +import pytest +import pytest_asyncio + +from nemoguardrails.guardrails.iorails import REFUSAL_MESSAGE +from nemoguardrails.rails.llm.options import GenerationResponse +from nemoguardrails.types import LLMResponse, UsageInfo +from tests.guardrails.async_helpers import started_iorails +from tests.guardrails.test_data import CONTENT_SAFETY_CONFIG + +_USER = [{"role": "user", "content": "hi"}] +_CS_INPUT = "content safety check input $model=content_safety" +_CS_MODEL = "nvidia/llama-3.1-nemoguard-8b-content-safety" +_MAIN_MODEL = "meta/llama-3.3-70b-instruct" + +# {"User Safety": "safe", "Response Safety": "safe"} parses safe for both the input +# parser (reads User Safety) and the output parser (reads Response Safety). +_SAFE_BOTH = json.dumps({"User Safety": "safe", "Response Safety": "safe"}) +_UNSAFE_INPUT = json.dumps({"User Safety": "unsafe", "Safety Categories": "S1: Violence"}) + + +def _cs_and_main_model_call(*, cs_request_id=None, main_request_id=None): + """Build a ``model_call`` side_effect: content-safety returns _SAFE_BOTH, the main model returns "Hi".""" + + async def _model_call(model_type, messages, **kwargs): + if model_type == "content_safety": + return LLMResponse( + content=_SAFE_BOTH, + usage=UsageInfo(input_tokens=100, output_tokens=10, total_tokens=110), + model=_CS_MODEL, + request_id=cs_request_id, + ) + return LLMResponse( + content="Hi", + usage=UsageInfo(input_tokens=20, output_tokens=5, total_tokens=25), + model=_MAIN_MODEL, + request_id=main_request_id, + ) + + return _model_call + + +@pytest_asyncio.fixture +async def iorails(): + """Started IORails on a content-safety-only config (input + output rails).""" + async with started_iorails(CONTENT_SAFETY_CONFIG) as engine: + yield engine + + +class TestRailRecordCapture: + """A real rail run captures its LLM call + verdict onto RailResult.records.""" + + @pytest.mark.asyncio + async def test_is_input_safe_captures_usage_and_verdict(self, iorails): + """is_input_safe returns a record carrying the rail's tokens, model, and verdict.""" + iorails.engine_registry.model_call = AsyncMock( + return_value=LLMResponse( + content=_SAFE_BOTH, + usage=UsageInfo(input_tokens=762, output_tokens=8, total_tokens=770), + model=_CS_MODEL, + request_id="req-cs-input", + ) + ) + + result = await iorails.rails_manager.is_input_safe(_USER) + + assert result.is_safe is True + assert len(result.records) == 1 + record = result.records[0] + assert record.flow == _CS_INPUT + assert record.rail_type == "input" + assert record.made_call is True + assert record.usage.total_tokens == 770 + assert record.llm_model_name == _CS_MODEL + assert record.llm_provider_name == "nim" + assert record.request_id == "req-cs-input" + assert record.return_value == {"allowed": True, "policy_violations": []} + + @pytest.mark.asyncio + async def test_failed_model_call_still_records_the_attempt(self, iorails): + """A rail whose model call raises still yields a record marked made_call=True (usage/model None).""" + iorails.engine_registry.model_call = AsyncMock(side_effect=RuntimeError("provider down")) + + result = await iorails.rails_manager.is_input_safe(_USER) + + assert result.is_safe is False + assert len(result.records) == 1 + record = result.records[0] + assert record.made_call is True + assert record.usage is None + assert record.llm_model_name is None + assert record.llm_provider_name == "nim" + assert record.duration is not None + + +class TestGenerationLogEndToEnd: + """Full generate_async path builds a GenerationLog from real rail + main-call records.""" + + @pytest.mark.asyncio + async def test_stats_and_activated_rails(self, iorails): + """log covers input CS + main + output CS calls, with the content-safety verdict.""" + iorails.engine_registry.model_call = AsyncMock( + side_effect=_cs_and_main_model_call(cs_request_id="req-cs", main_request_id="req-main") + ) + + result = await iorails.generate_async( + messages=_USER, options={"log": {"llm_calls": True, "activated_rails": True}} + ) + + assert isinstance(result, GenerationResponse) + assert result.log is not None + stats = result.log.stats + assert stats.llm_calls_count == 3 + assert stats.llm_calls_total_prompt_tokens == 100 + 20 + 100 + assert stats.llm_calls_total_completion_tokens == 10 + 5 + 10 + assert stats.llm_calls_total_tokens == 110 + 25 + 110 + + cs_rail = next(rail for rail in result.log.activated_rails if rail.name == _CS_INPUT) + assert cs_rail.type == "input" + assert cs_rail.executed_actions[0].return_value == {"allowed": True, "policy_violations": []} + # id + provider are threaded through the capture (parity fixes). + cs_call = cs_rail.executed_actions[0].llm_calls[0] + assert cs_call.id == "req-cs" + assert cs_call.llm_provider_name == "nim" + assert any(rail.type == "generation" for rail in result.log.activated_rails) + + @pytest.mark.asyncio + async def test_llm_calls_capture_prompt_and_completion(self, iorails): + """Each LLM-backed call in the log carries its serialized prompt and raw completion.""" + iorails.engine_registry.model_call = AsyncMock(side_effect=_cs_and_main_model_call()) + + result = await iorails.generate_async(messages=_USER, options={"log": {"llm_calls": True}}) + + assert isinstance(result, GenerationResponse) + assert result.log is not None + llm_calls = result.log.llm_calls or [] + cs_call = next(c for c in llm_calls if c.task and "content_safety_check_input" in c.task) + main_call = next(c for c in llm_calls if c.task == "general") + + assert cs_call.completion == _SAFE_BOTH + assert cs_call.prompt is not None + assert "hi" in cs_call.prompt + assert main_call.completion == "Hi" + assert main_call.prompt is not None + assert "hi" in main_call.prompt + + @pytest.mark.asyncio + async def test_prompt_serializes_role_and_content(self, iorails): + """The serialized prompt is role-labeled so a reader can tell system from user turns.""" + iorails.engine_registry.model_call = AsyncMock( + return_value=LLMResponse( + content=_SAFE_BOTH, + usage=UsageInfo(input_tokens=100, output_tokens=10, total_tokens=110), + model=_CS_MODEL, + ) + ) + + result = await iorails.generate_async(messages=_USER, options={"log": {"llm_calls": True}}) + + assert result.log is not None + llm_calls = result.log.llm_calls or [] + cs_call = next(c for c in llm_calls if c.task and "content_safety_check_input" in c.task) + assert cs_call.prompt is not None + assert "user:" in cs_call.prompt or "system:" in cs_call.prompt + + @pytest.mark.asyncio + async def test_blocked_input_logs_verdict_and_stop(self, iorails): + """A blocked input rail logs its unsafe verdict + stop, and the request refuses.""" + iorails.engine_registry.model_call = AsyncMock( + return_value=LLMResponse( + content=_UNSAFE_INPUT, + usage=UsageInfo(input_tokens=762, output_tokens=22, total_tokens=784), + model=_CS_MODEL, + ) + ) + + result = await iorails.generate_async(messages=_USER, options={"log": {"activated_rails": True}}) + + assert isinstance(result, GenerationResponse) + assert result.response == [{"role": "assistant", "content": REFUSAL_MESSAGE}] + assert result.log is not None + cs_rail = next(rail for rail in result.log.activated_rails if rail.name == _CS_INPUT) + assert cs_rail.stop is True + verdict = cs_rail.executed_actions[0].return_value + assert verdict["allowed"] is False + assert len(verdict["policy_violations"]) >= 1 diff --git a/tests/guardrails/test_iorails_generation_response.py b/tests/guardrails/test_iorails_generation_response.py new file mode 100644 index 0000000000..3e029a4d2d --- /dev/null +++ b/tests/guardrails/test_iorails_generation_response.py @@ -0,0 +1,417 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Structured GenerationResponse return from IORails, matching LLMRails. + +When ``options`` is supplied, ``generate_async``/``generate`` return a +``GenerationResponse`` instead of a bare ``LLMMessage`` dict, mirroring LLMRails' +``if gen_options:`` branch. When ``options`` is absent the bare dict is returned +unchanged. The structured path populates ``response``, ``reasoning_content``, +``tool_calls`` (as ``ToolCall.to_dict()`` with dict arguments), ``log`` (from +per-rail records), and ``llm_metadata`` (main-call ``provider_metadata`` only — +token usage lives in ``log``); ``llm_output`` is always ``None`` (parity with +LLMRails' unwired ``raw_response``). ``output_vars``/``state`` raise ``ValueError`` +and ``log.internal_events``/``log.colang_history`` raise ``NotImplementedError``. +""" + +from unittest.mock import AsyncMock, patch + +import pytest +import pytest_asyncio + +from nemoguardrails.guardrails.guardrails_types import RailResult +from nemoguardrails.guardrails.iorails import REFUSAL_MESSAGE, IORails, _response_content_for_capture +from nemoguardrails.rails.llm.config import RailsConfig +from nemoguardrails.rails.llm.options import GenerationOptions, GenerationResponse +from nemoguardrails.types import LLMResponse, ToolCall, ToolCallFunction, UsageInfo +from tests.guardrails.async_helpers import started_iorails +from tests.guardrails.test_data import NEMOGUARDS_CONFIG + + +@pytest_asyncio.fixture +async def iorails(): + """Started IORails instance with worker-queue teardown after each test.""" + async with started_iorails(NEMOGUARDS_CONFIG) as iorails: + yield iorails + + +@pytest.fixture +def iorails_sync(): + """Unstarted IORails instance for driving the synchronous ``generate`` path.""" + return IORails(RailsConfig.from_content(config=NEMOGUARDS_CONFIG)) + + +def _stub_safe_rails(iorails: IORails) -> None: + """Default-safe input, output, and tool-call rails so tests focus on the LLM response.""" + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.rails_manager.are_tool_calls_safe = AsyncMock(return_value=RailResult(is_safe=True)) + + +def _stub_model(iorails: IORails, response: LLMResponse) -> None: + """Make the main-model call return a fixed structured LLMResponse.""" + iorails.engine_registry.model_call = AsyncMock(return_value=response) + + +_USER = [{"role": "user", "content": "hi"}] + +_WEATHER_TOOL_CALL = ToolCall( + id="call_1", + type="function", + function=ToolCallFunction(name="get_weather", arguments={"city": "SF"}), +) + + +async def _generate_structured(iorails: IORails, response: LLMResponse, *, options=None, **kwargs): + """Stub safe rails + a fixed model response, then run the structured (``options``) path.""" + _stub_safe_rails(iorails) + _stub_model(iorails, response) + return await iorails.generate_async(messages=_USER, options={} if options is None else options, **kwargs) + + +class TestStructuredResponseTrigger: + """``options`` presence decides GenerationResponse vs. bare dict.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "options", + [{"llm_params": {"temperature": 0.5}}, GenerationOptions()], + ids=["dict", "GenerationOptions"], + ) + async def test_options_returns_generation_response(self, iorails, options): + """Any ``options`` value (dict or GenerationOptions) switches the return type to GenerationResponse.""" + result = await _generate_structured(iorails, LLMResponse(content="Hello"), options=options) + + assert isinstance(result, GenerationResponse) + + @pytest.mark.asyncio + async def test_no_generation_options_returns_messages(self, iorails): + """Without ``options`` the return stays the bare assistant-message dict.""" + _stub_safe_rails(iorails) + _stub_model(iorails, LLMResponse(content="Hello")) + + result = await iorails.generate_async(messages=_USER) + + assert not isinstance(result, GenerationResponse) + assert result == {"role": "assistant", "content": "Hello"} + + +class TestResponseField: + """The ``response`` field wraps the assistant message in a one-element list.""" + + @pytest.mark.asyncio + async def test_plain_content_wrapped_in_list(self, iorails): + """``response`` is ``[{"role":"assistant","content": text}]``.""" + result = await _generate_structured(iorails, LLMResponse(content="Hello there")) + + assert isinstance(result, GenerationResponse) + assert result.response == [{"role": "assistant", "content": "Hello there"}] + + +class TestReasoningContent: + """Reasoning goes to ``reasoning_content`` with clean content (no inline ).""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "response", + [ + LLMResponse(content="Hello", reasoning="thinking step"), + LLMResponse(content="thinking stepHello"), + ], + ids=["native-reasoning-field", "inline-think-tags"], + ) + async def test_reasoning_extracted_content_clean(self, iorails, response): + """Reasoning (native field or inline ) goes to ``reasoning_content``; response content stays clean and output rails see only the clean text.""" + result = await _generate_structured(iorails, response) + + assert isinstance(result, GenerationResponse) + assert result.reasoning_content == "thinking step" + assert result.response == [{"role": "assistant", "content": "Hello"}] + iorails.rails_manager.is_output_safe.assert_called_once_with(_USER, "Hello", enabled=True) + + @pytest.mark.asyncio + async def test_no_reasoning_field_is_none(self, iorails): + """Absent reasoning leaves ``reasoning_content`` as None.""" + result = await _generate_structured(iorails, LLMResponse(content="plain answer")) + + assert isinstance(result, GenerationResponse) + assert result.reasoning_content is None + assert result.response == [{"role": "assistant", "content": "plain answer"}] + + +class TestToolCalls: + """``tool_calls`` use the LLMRails ``ToolCall.to_dict()`` shape (dict arguments).""" + + @pytest.mark.asyncio + async def test_tool_calls_serialized_with_dict_arguments(self, iorails): + """``tool_calls`` is a list of ``to_dict()`` entries whose ``arguments`` stay a dict, not a JSON string.""" + result = await _generate_structured(iorails, LLMResponse(content="", tool_calls=[_WEATHER_TOOL_CALL])) + + assert isinstance(result, GenerationResponse) + assert result.tool_calls == [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": {"city": "SF"}}, + } + ] + + @pytest.mark.asyncio + async def test_response_message_has_no_tool_calls_key(self, iorails): + """In the structured path tool calls live only in the top-level field, not on the message.""" + result = await _generate_structured(iorails, LLMResponse(content="", tool_calls=[_WEATHER_TOOL_CALL])) + + assert isinstance(result, GenerationResponse) + assert result.response == [{"role": "assistant", "content": ""}] + assert "tool_calls" not in result.response[0] + + @pytest.mark.asyncio + async def test_no_tool_calls_field_is_none(self, iorails): + """A text-only response leaves ``tool_calls`` as None.""" + result = await _generate_structured(iorails, LLMResponse(content="Hello")) + + assert isinstance(result, GenerationResponse) + assert result.tool_calls is None + + +class TestLLMMetadata: + """``llm_metadata`` is the main-call ``provider_metadata`` verbatim; usage lives in ``log``.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "response, expected", + [ + ( + LLMResponse( + content="Hello", + provider_metadata={"response_headers": {"nvcf-status": "fulfilled"}}, + usage=UsageInfo(input_tokens=10, output_tokens=5, total_tokens=15), + ), + {"response_headers": {"nvcf-status": "fulfilled"}}, + ), + (LLMResponse(content="Hello", usage=UsageInfo(input_tokens=3, output_tokens=4, total_tokens=7)), None), + (LLMResponse(content="Hello"), None), + ], + ids=["provider-metadata-passthrough", "usage-only-none", "nothing-none"], + ) + async def test_llm_metadata_is_provider_metadata_only(self, iorails, response, expected): + """llm_metadata is the main-call provider_metadata verbatim (else None); token usage is never grafted under ``usage``.""" + result = await _generate_structured(iorails, response) + + assert isinstance(result, GenerationResponse) + assert result.llm_metadata == expected + + +class TestLLMOutput: + """``llm_output`` is always None, matching LLMRails' unwired ``raw_response``.""" + + @pytest.mark.asyncio + async def test_llm_output_none_even_when_requested(self, iorails): + """``options={"llm_output": True}`` is accepted but the field stays None.""" + response = LLMResponse(content="Hello", provider_metadata={"response_headers": {"nvcf-status": "fulfilled"}}) + result = await _generate_structured(iorails, response, options={"llm_output": True}) + + assert isinstance(result, GenerationResponse) + assert result.llm_output is None + + +class TestUnsupportedOptionGuards: + """Colang-coupled options IORails cannot honor raise.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "gen_kwargs", + [ + {"options": {"output_vars": True}}, + {"options": {"output_vars": ["relevant_chunks"]}}, + {"options": {}, "state": {"conversation": []}}, + ], + ids=["output_vars-true", "output_vars-list", "state-arg"], + ) + async def test_colang_state_option_raises_value_error(self, iorails, gen_kwargs): + """``output_vars`` (any form) and ``state`` need Colang runtime context IORails lacks — ValueError.""" + _stub_safe_rails(iorails) + _stub_model(iorails, LLMResponse(content="Hello")) + + with pytest.raises(ValueError): + await iorails.generate_async(messages=_USER, **gen_kwargs) + + @pytest.mark.asyncio + async def test_colang_only_log_flag_raises(self, iorails): + """Colang-runtime-only log details raise NotImplementedError. + + ``activated_rails``/``llm_calls`` are supported and produce a GenerationLog; + only ``internal_events`` and ``colang_history`` (which need the Colang runtime) + are rejected. + """ + _stub_safe_rails(iorails) + _stub_model(iorails, LLMResponse(content="Hello")) + + with pytest.raises(NotImplementedError): + await iorails.generate_async(messages=_USER, options={"log": {"internal_events": True}}) + + +class TestBlockedStructuredResponse: + """A blocked request returns the refusal in ``response`` with the other fields empty.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "input_safe, output_safe", [(False, True), (True, False)], ids=["input-block", "output-block"] + ) + async def test_block_returns_refusal_response(self, iorails, input_safe, output_safe): + """A block at either the input or output rail yields a GenerationResponse whose response is the refusal and whose other fields are empty.""" + iorails.rails_manager.is_input_safe = AsyncMock( + return_value=RailResult(is_safe=input_safe, reason=None if input_safe else "unsafe") + ) + iorails.rails_manager.is_output_safe = AsyncMock( + return_value=RailResult(is_safe=output_safe, reason=None if output_safe else "unsafe") + ) + _stub_model(iorails, LLMResponse(content="bad answer")) + + result = await iorails.generate_async(messages=_USER, options={}) + + assert isinstance(result, GenerationResponse) + assert result.response == [{"role": "assistant", "content": REFUSAL_MESSAGE}] + assert result.tool_calls is None + assert result.reasoning_content is None + assert result.llm_metadata is None + + +class TestBarePathUnchanged: + """The optionless bare-dict path keeps its existing behavior.""" + + @pytest.mark.asyncio + async def test_bare_path_inlines_reasoning_prefix(self, iorails): + """Without ``options`` reasoning is still delivered inline as a prefix.""" + _stub_safe_rails(iorails) + _stub_model(iorails, LLMResponse(content="Hello", reasoning="thinking step")) + + result = await iorails.generate_async(messages=_USER) + + assert result == {"role": "assistant", "content": "thinking step\nHello"} + + +class TestSyncGenerateStructured: + """The synchronous ``generate`` mirrors the async structured return.""" + + def test_sync_generate_with_options_returns_generation_response(self, iorails_sync): + """``generate(options=...)`` returns a GenerationResponse from the ephemeral engine.""" + iorails_sync.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails_sync.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails_sync.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="Hello")) + + with patch("nemoguardrails.guardrails.iorails.IORails", return_value=iorails_sync): + result = iorails_sync.generate(messages=_USER, options={}) + + assert isinstance(result, GenerationResponse) + + +class TestPromptAndMessages: + """IORails accepts a keyword-only ``prompt`` (converted to messages) or a ``messages`` list.""" + + @pytest.mark.asyncio + async def test_prompt_string_generates(self, iorails): + """A keyword prompt string is converted to a user message and generates normally.""" + _stub_safe_rails(iorails) + _stub_model(iorails, LLMResponse(content="Hi")) + + result = await iorails.generate_async(prompt="Hello", options={}) + + assert isinstance(result, GenerationResponse) + assert result.response == [{"role": "assistant", "content": "Hi"}] + + @pytest.mark.asyncio + async def test_neither_prompt_nor_messages_raises(self, iorails): + """generate_async with neither prompt nor messages raises ValueError.""" + with pytest.raises(ValueError): + await iorails.generate_async() + + def test_sync_generate_neither_raises(self, iorails_sync): + """The sync generate with neither prompt nor messages raises ValueError.""" + with pytest.raises(ValueError): + iorails_sync.generate() + + @pytest.mark.asyncio + async def test_list_passed_positionally_as_prompt_raises_typeerror(self, iorails): + """A message list in the positional prompt slot raises TypeError, not a silent misparse.""" + with pytest.raises(TypeError): + await iorails.generate_async([{"role": "user", "content": "hi"}]) + + +class TestConvertToMessages: + """``IORails._convert_to_messages`` normalizes prompt/messages (moved from the facade).""" + + def test_messages_passthrough(self): + """A multi-turn message list (system + user + assistant) is returned unchanged.""" + msgs = [ + {"role": "system", "content": "be nice"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] + assert IORails._convert_to_messages(messages=msgs) is msgs + + def test_prompt_wrapped_as_user_turn(self): + """A prompt string (content preserved verbatim) becomes a single user message.""" + prompt = 'Line 1\nLine 2 with "quotes" & symbols' + assert IORails._convert_to_messages(prompt=prompt) == [{"role": "user", "content": prompt}] + + def test_messages_win_when_both_supplied(self): + """When both are given, messages takes priority and prompt is ignored.""" + msgs = [{"role": "user", "content": "hi"}] + assert IORails._convert_to_messages(prompt="ignored", messages=msgs) is msgs + + def test_string_messages_raises_typeerror(self): + """A string in the messages slot raises TypeError pointing at prompt=.""" + with pytest.raises(TypeError): + IORails._convert_to_messages(messages="hi") + + def test_list_prompt_raises_typeerror(self): + """A message list in the prompt slot raises TypeError pointing at messages=.""" + with pytest.raises(TypeError): + IORails._convert_to_messages(prompt=[{"role": "user", "content": "hi"}]) + + def test_empty_string_prompt_raises_valueerror(self): + """An empty prompt string is falsy, so it counts as 'neither provided'.""" + with pytest.raises(ValueError): + IORails._convert_to_messages(prompt="") + + def test_empty_messages_list_raises_valueerror(self): + """An empty messages list is falsy, so it counts as 'neither provided'.""" + with pytest.raises(ValueError): + IORails._convert_to_messages(messages=[]) + + def test_neither_raises_valueerror(self): + """Neither prompt nor messages raises ValueError.""" + with pytest.raises(ValueError): + IORails._convert_to_messages() + + +class TestResponseContentForCapture: + """`_response_content_for_capture` extracts assistant text from either return shape.""" + + @pytest.mark.parametrize( + "result, expected", + [ + (GenerationResponse(response=[{"role": "assistant", "content": "hi there"}]), "hi there"), + (GenerationResponse(response="hi there"), "hi there"), + (GenerationResponse(response=[]), None), + (GenerationResponse(response=[{"role": "assistant", "content": None}]), None), + ({"role": "assistant", "content": "hi there"}, "hi there"), + ], + ids=["structured-list", "structured-str", "empty-list", "non-str-content", "bare-dict"], + ) + def test_capture_content(self, result, expected): + """Assistant content is pulled from structured list/str responses and from the bare-dict path; non-str/absent content yields None.""" + assert _response_content_for_capture(result) == expected diff --git a/tests/guardrails/test_iorails_reasoning.py b/tests/guardrails/test_iorails_reasoning.py index 083878c77b..975dcf38bd 100644 --- a/tests/guardrails/test_iorails_reasoning.py +++ b/tests/guardrails/test_iorails_reasoning.py @@ -138,7 +138,7 @@ async def test_native_reasoning_field_prefixes_content(self, iorails): return_value=LLMResponse(content="Hello", reasoning="thinking step") ) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": "thinking step\nHello"} # Output rails see the original content unchanged when reasoning came from @@ -154,7 +154,7 @@ async def test_inline_think_tags_extracted_and_stripped(self, iorails): return_value=LLMResponse(content="thinking stepHello") ) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": "thinking step\nHello"} # Output rails MUST receive content with tags stripped — this is @@ -178,7 +178,7 @@ async def test_native_reasoning_wins_over_inline_tags(self, iorails): ) ) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == { "role": "assistant", @@ -195,7 +195,7 @@ async def test_malformed_think_tag_opener_only(self, iorails): _stub_safe_rails(iorails) iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="incomplete reasoning")) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": "incomplete reasoning"} iorails.rails_manager.is_output_safe.assert_called_once_with( @@ -209,7 +209,7 @@ async def test_malformed_think_tag_closer_only(self, iorails): _stub_safe_rails(iorails) iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="orphan reply")) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": "orphan reply"} iorails.rails_manager.is_output_safe.assert_called_once_with(messages, "orphan reply", enabled=True) @@ -224,7 +224,7 @@ async def test_output_rail_block_with_native_reasoning(self, iorails): return_value=LLMResponse(content="bad answer", reasoning="reasoning step") ) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} @@ -238,7 +238,7 @@ async def test_output_rail_block_with_inline_tags(self, iorails): return_value=LLMResponse(content="thinkingbad answer") ) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} # Output rails see only the stripped content — they're judging the model @@ -254,7 +254,7 @@ async def test_empty_string_reasoning_falls_through_to_inline_extraction(self, i return_value=LLMResponse(content="fallback reasoningHi", reasoning="") ) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": "fallback reasoning\nHi"} iorails.rails_manager.is_output_safe.assert_called_once_with(messages, "Hi", enabled=True) @@ -266,7 +266,7 @@ async def test_no_reasoning_returns_unchanged_content(self, iorails): _stub_safe_rails(iorails) iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="plain answer")) - result = await iorails.generate_async(messages) + result = await iorails.generate_async(messages=messages) assert result == {"role": "assistant", "content": "plain answer"} iorails.rails_manager.is_output_safe.assert_called_once_with(messages, "plain answer", enabled=True) diff --git a/tests/guardrails/test_iorails_telemetry.py b/tests/guardrails/test_iorails_telemetry.py index cd2980730e..b9c42056a4 100644 --- a/tests/guardrails/test_iorails_telemetry.py +++ b/tests/guardrails/test_iorails_telemetry.py @@ -140,7 +140,7 @@ class TestGenerateAsyncWithTracing: async def test_creates_span(self, iorails_tracing, exporter): _stub_safe_pipeline(iorails_tracing) - result = await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result == {"role": "assistant", "content": "Hello"} spans = exporter.get_finished_spans() @@ -152,7 +152,7 @@ async def test_creates_span(self, iorails_tracing, exporter): async def test_span_has_required_attributes(self, iorails_tracing, exporter): _stub_safe_pipeline(iorails_tracing) - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() attrs = dict(spans[0].attributes) @@ -173,7 +173,7 @@ async def capture_req_id(messages, *, enabled=True): _stub_safe_pipeline(iorails_tracing) iorails_tracing.rails_manager.is_input_safe = capture_req_id - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() span_req_id = spans[0].attributes["request.id"] @@ -185,7 +185,7 @@ async def test_span_records_exception(self, iorails_tracing, exporter): iorails_tracing.engine_registry.model_call = AsyncMock(side_effect=RuntimeError("LLM failed")) with pytest.raises(RuntimeError, match="LLM failed"): - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() assert len(spans) == 1 @@ -198,7 +198,7 @@ async def test_span_created_on_input_block(self, iorails_tracing, exporter): """A span is still created and completed even when input rails block.""" iorails_tracing.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=False, reason="unsafe")) - result = await iorails_tracing.generate_async([{"role": "user", "content": "bad"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "bad"}]) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} spans = exporter.get_finished_spans() @@ -211,7 +211,7 @@ class TestGenerateAsyncWithoutTracing: async def test_no_spans_exported(self, iorails_no_tracing, exporter): _stub_safe_pipeline(iorails_no_tracing) - result = await iorails_no_tracing.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails_no_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result == {"role": "assistant", "content": "Hello"} assert len(exporter.get_finished_spans()) == 0 @@ -229,7 +229,7 @@ async def capture_req_id(messages, *, enabled=True): _stub_safe_pipeline(iorails_no_tracing) iorails_no_tracing.rails_manager.is_input_safe = capture_req_id - await iorails_no_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_no_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert captured_req_id is not None assert len(captured_req_id) == REQUEST_ID_HEX_CHARS @@ -264,7 +264,7 @@ async def capturing_output_check(messages, response, *, enabled=True): iorails_tracing.rails_manager.is_output_safe = capturing_output_check messages = [{"role": "user", "content": "hello"}] - result = await iorails_tracing.generate_async(messages) + result = await iorails_tracing.generate_async(messages=messages) # Response correctness assert result == {"role": "assistant", "content": "Generated response"} @@ -317,8 +317,8 @@ async def record_req_id(messages, *, enabled=True): messages = [{"role": "user", "content": "hi"}] await asyncio.gather( - iorails_tracing.generate_async(messages), - iorails_tracing.generate_async(messages), + iorails_tracing.generate_async(messages=messages), + iorails_tracing.generate_async(messages=messages), ) spans = exporter.get_finished_spans() @@ -347,7 +347,7 @@ async def capture_then_pass(messages, *, enabled=True): iorails_tracing.engine_registry.model_call = AsyncMock(side_effect=RuntimeError("connection refused")) with pytest.raises(RuntimeError, match="connection refused"): - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() assert len(spans) == 1 @@ -412,7 +412,7 @@ async def test_span_hierarchy_on_safe_request(self, iorails_tracing, exporter): """Full safe request produces: request → rail → action → LLM/API spans.""" _stub_deep_pipeline(iorails_tracing) - result = await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result["content"] == "Hello" spans = exporter.get_finished_spans() @@ -445,7 +445,7 @@ async def test_rail_span_attributes(self, iorails_tracing, exporter): """Rail spans have correct rail.type and rail.name attributes.""" _stub_deep_pipeline(iorails_tracing) - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() rail_spans = [s for s in spans if s.name == "guardrails.rail"] @@ -466,7 +466,7 @@ async def test_rail_stop_attribute_on_block(self, iorails_tracing, exporter): """When a rail blocks, its span has rail.stop=True.""" _stub_deep_pipeline(iorails_tracing, input_safe=False) - result = await iorails_tracing.generate_async([{"role": "user", "content": "bad"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "bad"}]) assert result["content"] == REFUSAL_MESSAGE spans = exporter.get_finished_spans() @@ -480,7 +480,7 @@ async def test_span_hierarchy_on_unsafe_request(self, iorails_tracing, exporter) """Unsafe request: main LLM and output rails never run, request span still completes cleanly.""" _stub_deep_pipeline(iorails_tracing, input_safe=False) - result = await iorails_tracing.generate_async([{"role": "user", "content": "bad"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "bad"}]) assert result["content"] == REFUSAL_MESSAGE spans = exporter.get_finished_spans() @@ -521,7 +521,7 @@ async def test_action_span_records_engine_error(self, iorails_tracing, exporter) elif isinstance(engine, ModelEngine): engine.chat_completion = AsyncMock(return_value=LLMResponse(content=SAFE_INPUT_JSON)) - result = await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result["content"] == REFUSAL_MESSAGE spans = exporter.get_finished_spans() @@ -556,7 +556,7 @@ async def test_span_tree_parent_child_links(self, iorails_tracing, exporter): """ _stub_deep_pipeline(iorails_tracing) - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() @@ -624,7 +624,7 @@ async def test_action_span_attributes(self, iorails_tracing, exporter): """Action spans have correct action.name attributes.""" _stub_deep_pipeline(iorails_tracing) - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() action_spans = [s for s in spans if s.name == "guardrails.action"] @@ -639,7 +639,7 @@ async def test_llm_span_attributes(self, iorails_tracing, exporter): """LLM spans have GenAI semantic convention attributes.""" _stub_deep_pipeline(iorails_tracing) - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() llm_spans = [s for s in spans if "gen_ai.request.model" in (s.attributes or {})] @@ -661,7 +661,7 @@ async def test_no_child_spans_when_tracing_disabled(self, iorails_no_tracing, ex """ _stub_deep_pipeline(iorails_no_tracing) - await iorails_no_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_no_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert len(exporter.get_finished_spans()) == 0 @@ -686,7 +686,7 @@ async def test_no_orphaned_child_spans_with_global_provider(self, tracer_from_pr async with iorails: _stub_deep_pipeline(iorails) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) # Zero spans of any kind assert exporter.get_finished_spans() == () @@ -1009,7 +1009,7 @@ async def test_falls_back_gracefully(self, exporter): async with iorails: _stub_safe_pipeline(iorails) - result = await iorails.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result == {"role": "assistant", "content": "Hello"} assert iorails._tracing_enabled is False @@ -1071,7 +1071,7 @@ async def test_emits_counter_and_duration_on_safe_request(self, iorails_tracing, no errors, requests.active back to 0.""" _stub_safe_pipeline(iorails_tracing) - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert points["guardrails.requests"][0].value == 1 @@ -1087,7 +1087,7 @@ async def test_emits_errors_counter_on_exception(self, iorails_tracing, metric_r iorails_tracing.engine_registry.model_call = AsyncMock(side_effect=RuntimeError("LLM failed")) with pytest.raises(RuntimeError, match="LLM failed"): - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert points["guardrails.requests"][0].value == 1 @@ -1100,7 +1100,7 @@ async def test_emits_errors_counter_on_exception(self, iorails_tracing, metric_r async def test_no_metrics_emitted_when_metrics_disabled(self, iorails_no_tracing, metric_reader): _stub_safe_pipeline(iorails_no_tracing) - await iorails_no_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_no_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert points == {} @@ -1110,7 +1110,7 @@ async def test_emits_blocked_counter_on_input_block(self, iorails_tracing, metri _stub_safe_pipeline(iorails_tracing) iorails_tracing.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=False, reason="unsafe")) - result = await iorails_tracing.generate_async([{"role": "user", "content": "bad"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "bad"}]) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} points = collect_metric_points(metric_reader) @@ -1127,7 +1127,7 @@ async def test_emits_blocked_counter_on_output_block(self, iorails_tracing, metr return_value=RailResult(is_safe=False, reason="unsafe response") ) - result = await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} points = collect_metric_points(metric_reader) @@ -1143,7 +1143,7 @@ async def test_no_blocked_counter_emitted_when_tracing_disabled(self, iorails_no return_value=RailResult(is_safe=False, reason="unsafe") ) - result = await iorails_no_tracing.generate_async([{"role": "user", "content": "bad"}]) + result = await iorails_no_tracing.generate_async(messages=[{"role": "user", "content": "bad"}]) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} points = collect_metric_points(metric_reader) @@ -1168,7 +1168,7 @@ async def test_nonstream_rejections_counter_on_queue_full(self, iorails_tracing, iorails_tracing._generate_async_queue.submit = AsyncMock(side_effect=asyncio.QueueFull("admission queue full")) with pytest.raises(asyncio.QueueFull, match="admission queue full"): - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert points["guardrails.nonstream.rejections"][0].value == 1 @@ -1186,7 +1186,7 @@ async def test_queuefull_bumps_both_errors_and_nonstream_rejections(self, iorail iorails_tracing._generate_async_queue.submit = AsyncMock(side_effect=asyncio.QueueFull("admission queue full")) with pytest.raises(asyncio.QueueFull): - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert points["guardrails.nonstream.rejections"][0].value == 1 @@ -1221,7 +1221,7 @@ async def test_request_duration_includes_queue_wait(self, metric_reader): iorails._do_generate = _gated_generate(gate) tasks = [ - asyncio.create_task(iorails.generate_async([{"role": "user", "content": f"m{i}"}])) + asyncio.create_task(iorails.generate_async(messages=[{"role": "user", "content": f"m{i}"}])) for i in range(2) ] # Wait until exactly one is executing and one is queued. @@ -1249,7 +1249,7 @@ async def test_no_nonstream_rejections_counter_when_metrics_disabled(self, iorai ) with pytest.raises(asyncio.QueueFull): - await iorails_no_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_no_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert "guardrails.nonstream.rejections" not in points @@ -1577,7 +1577,7 @@ async def test_metrics_only_emits_metrics_but_no_spans(self, metric_reader, expo iorails = IORails(RailsConfig.from_content(config=_make_metrics_only_config())) async with iorails: _stub_safe_pipeline(iorails) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert points["guardrails.requests"][0].value == 1 @@ -1594,7 +1594,7 @@ async def test_tracing_only_emits_spans_but_no_metrics(self, tracer_from_provide iorails = IORails(RailsConfig.from_content(config=_make_tracing_only_config())) async with iorails: _stub_safe_pipeline(iorails) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() assert any(s.name == "guardrails.request" for s in spans) @@ -1635,7 +1635,7 @@ async def test_nonstream_active_reads_one_while_worker_is_busy(self, metric_read async with iorails: iorails._do_generate = _gated_generate(gate) - task = asyncio.create_task(iorails.generate_async([{"role": "user", "content": "hi"}])) + task = asyncio.create_task(iorails.generate_async(messages=[{"role": "user", "content": "hi"}])) # Wait for the worker to pick up the item and enter # the gated generate (busy_count=1, pending=0). await wait_for_queue_state(iorails._generate_async_queue, busy=1, pending=0) @@ -1670,7 +1670,7 @@ async def test_nonstream_queued_reflects_backlog_past_worker_capacity(self, metr iorails._do_generate = _gated_generate(gate) tasks = [ - asyncio.create_task(iorails.generate_async([{"role": "user", "content": f"m{i}"}])) + asyncio.create_task(iorails.generate_async(messages=[{"role": "user", "content": f"m{i}"}])) for i in range(3) ] # Wait for one worker to pick up an item and the other two @@ -1782,7 +1782,7 @@ async def test_captures_queued_nonstream_request_mid_flight(self, metric_reader) async with iorails: iorails._do_generate = _gated_generate(gate) tasks = [ - asyncio.create_task(iorails.generate_async([{"role": "user", "content": f"m{i}"}])) + asyncio.create_task(iorails.generate_async(messages=[{"role": "user", "content": f"m{i}"}])) for i in range(2) ] await wait_for_queue_state(iorails._generate_async_queue, busy=1, pending=1) @@ -1850,7 +1850,7 @@ async def test_invariant_aggregate_equals_component_sum(self, metric_reader): # Launch one executing + one queued non-streaming request. nonstream_tasks = [ - asyncio.create_task(iorails.generate_async([{"role": "user", "content": f"n{i}"}])) + asyncio.create_task(iorails.generate_async(messages=[{"role": "user", "content": f"n{i}"}])) for i in range(2) ] await wait_for_queue_state(iorails._generate_async_queue, busy=1, pending=1) @@ -1941,7 +1941,7 @@ async def test_emits_token_and_duration_metrics_per_model(self, iorails_tracing, """ _stub_deep_pipeline_with_usage(iorails_tracing) - result = await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result == {"role": "assistant", "content": "Hello"} points = collect_metric_points(metric_reader) @@ -1985,7 +1985,7 @@ async def test_no_llm_metrics_when_metrics_disabled(self, iorails_no_tracing, me """ _stub_deep_pipeline_with_usage(iorails_no_tracing) - await iorails_no_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_no_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) points = collect_metric_points(metric_reader) assert "gen_ai.client.token.usage" not in points @@ -2081,7 +2081,7 @@ async def test_no_content_on_any_span(self, iorails_tracing, exporter): """Capture off → request, LLM, and rail spans carry no captured content.""" _stub_deep_pipeline(iorails_tracing) - await iorails_tracing.generate_async([{"role": "user", "content": "hi"}]) + await iorails_tracing.generate_async(messages=[{"role": "user", "content": "hi"}]) spans = exporter.get_finished_spans() @@ -2108,7 +2108,7 @@ async def test_request_span_carries_guardrails_attrs(self, iorails_content_captu """Request span carries guardrails.request.input/output attrs, not gen_ai.* events.""" _stub_deep_pipeline(iorails_content_capture, main_llm_response="Hi back") - await iorails_content_capture.generate_async([{"role": "user", "content": "hello"}]) + await iorails_content_capture.generate_async(messages=[{"role": "user", "content": "hello"}]) span = _request_span(exporter.get_finished_spans()) assert json.loads(span.attributes[GuardrailsAttributes.REQUEST_INPUT]) == [{"role": "user", "content": "hello"}] @@ -2126,7 +2126,7 @@ async def test_legacy_events_on_main_llm_span(self, iorails_content_capture, exp """ _stub_deep_pipeline(iorails_content_capture) - await iorails_content_capture.generate_async([{"role": "user", "content": "hello"}]) + await iorails_content_capture.generate_async(messages=[{"role": "user", "content": "hello"}]) span = _main_llm_span(exporter.get_finished_spans()) event_names = [e.name for e in span.events] @@ -2138,7 +2138,7 @@ async def test_refusal_message_captured_on_blocked_input(self, iorails_content_c """A blocked input records REFUSAL_MESSAGE as guardrails.request.output.""" _stub_deep_pipeline(iorails_content_capture, input_safe=False) - result = await iorails_content_capture.generate_async([{"role": "user", "content": "bad"}]) + result = await iorails_content_capture.generate_async(messages=[{"role": "user", "content": "bad"}]) assert result["content"] == REFUSAL_MESSAGE span = _request_span(exporter.get_finished_spans()) @@ -2159,7 +2159,7 @@ async def test_refusal_message_captured_on_blocked_output(self, iorails_content_ return_value=RailResult(is_safe=False, reason="unsafe response") ) - result = await iorails.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result["content"] == REFUSAL_MESSAGE span = _request_span(exporter.get_finished_spans()) @@ -2187,7 +2187,7 @@ async def test_llm_and_request_spans_diverge_on_output_block(self, iorails_conte return_value=RailResult(is_safe=False, reason="unsafe response") ) - result = await iorails.generate_async([{"role": "user", "content": "hi"}]) + result = await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert result["content"] == REFUSAL_MESSAGE spans = exporter.get_finished_spans() @@ -2215,7 +2215,7 @@ async def test_request_span_carries_guardrails_attrs(self, iorails_content_captu """Opt-in set → request span carries guardrails.request.* attrs (plain strings).""" _stub_deep_pipeline(iorails_content_capture, main_llm_response="The answer") - await iorails_content_capture.generate_async([{"role": "user", "content": "hello"}]) + await iorails_content_capture.generate_async(messages=[{"role": "user", "content": "hello"}]) span = _request_span(exporter.get_finished_spans()) assert json.loads(span.attributes[GuardrailsAttributes.REQUEST_INPUT]) == [{"role": "user", "content": "hello"}] @@ -2233,7 +2233,7 @@ async def test_system_instructions_on_llm_span(self, iorails_content_capture, ex {"role": "system", "content": "be helpful"}, {"role": "user", "content": "hi"}, ] - await iorails_content_capture.generate_async(messages) + await iorails_content_capture.generate_async(messages=messages) llm_span = _main_llm_span(exporter.get_finished_spans()) sysinst = json.loads(llm_span.attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS]) @@ -2251,7 +2251,7 @@ async def test_json_attrs_on_main_llm_span(self, iorails_content_capture, export """Opt-in set → the main LLM span carries JSON input/output message attrs.""" _stub_deep_pipeline(iorails_content_capture) - await iorails_content_capture.generate_async([{"role": "user", "content": "hello"}]) + await iorails_content_capture.generate_async(messages=[{"role": "user", "content": "hello"}]) span = _main_llm_span(exporter.get_finished_spans()) assert GenAIAttributes.GEN_AI_INPUT_MESSAGES in span.attributes @@ -2273,7 +2273,7 @@ async def test_env_var_enables_capture_when_config_unset(self, monkeypatch, trac iorails = IORails(config) async with iorails: _stub_deep_pipeline(iorails) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) span = _request_span(exporter.get_finished_spans()) # Request span carries guardrails.request.* attrs @@ -2297,7 +2297,7 @@ async def test_env_var_false_disables_capture_when_config_true(self, monkeypatch iorails = IORails(config) async with iorails: _stub_deep_pipeline(iorails) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) span = _request_span(exporter.get_finished_spans()) # No content attrs despite config=True — env=false wins @@ -2314,7 +2314,7 @@ async def test_rail_input_recorded_on_passing_rail(self, iorails_content_capture """Every rail span records its rail.input; passing rails carry no rail.reason.""" _stub_deep_pipeline(iorails_content_capture) - await iorails_content_capture.generate_async([{"role": "user", "content": "hi"}]) + await iorails_content_capture.generate_async(messages=[{"role": "user", "content": "hi"}]) rail_spans = [s for s in exporter.get_finished_spans() if s.name == "guardrails.rail"] assert len(rail_spans) >= 1 @@ -2330,7 +2330,7 @@ async def test_rail_reason_set_on_blocked_rail_only(self, iorails_content_captur """When an input rail blocks, its span gets a reason; later rails never run.""" _stub_deep_pipeline(iorails_content_capture, input_safe=False) - await iorails_content_capture.generate_async([{"role": "user", "content": "bad"}]) + await iorails_content_capture.generate_async(messages=[{"role": "user", "content": "bad"}]) rail_spans = [s for s in exporter.get_finished_spans() if s.name == "guardrails.rail"] blocked = [s for s in rail_spans if GuardrailsAttributes.RAIL_REASON in s.attributes] diff --git a/tests/guardrails/test_jailbreak_detection_iorails_actions.py b/tests/guardrails/test_jailbreak_detection_iorails_actions.py index 57c00d38cc..59837b8560 100644 --- a/tests/guardrails/test_jailbreak_detection_iorails_actions.py +++ b/tests/guardrails/test_jailbreak_detection_iorails_actions.py @@ -52,14 +52,14 @@ def test_creates_api_payload(self, action): class TestJailbreakParseResponse: def test_safe(self, action): - assert action._parse_response({"jailbreak": False, "score": 0.1}) == RailResult( - is_safe=True, reason="Score: 0.1" - ) + result = action._parse_response({"jailbreak": False, "score": 0.1}) + assert result == RailResult(is_safe=True, reason="Score: 0.1") + assert result.return_value is False def test_jailbreak_detected(self, action): - assert action._parse_response({"jailbreak": True, "score": 0.95}) == RailResult( - is_safe=False, reason="Score: 0.95" - ) + result = action._parse_response({"jailbreak": True, "score": 0.95}) + assert result == RailResult(is_safe=False, reason="Score: 0.95") + assert result.return_value is True def test_missing_jailbreak_field_raises(self, action): with pytest.raises(RuntimeError, match="missing 'jailbreak' field"): diff --git a/tests/guardrails/test_rails_manager.py b/tests/guardrails/test_rails_manager.py index f3957d0697..89d743eb29 100644 --- a/tests/guardrails/test_rails_manager.py +++ b/tests/guardrails/test_rails_manager.py @@ -31,8 +31,8 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from nemoguardrails.guardrails.engine_registry import EngineRegistry -from nemoguardrails.guardrails.guardrails_types import RailDirection, RailResult -from nemoguardrails.guardrails.rails_manager import RailsManager +from nemoguardrails.guardrails.guardrails_types import RailCallRecord, RailDirection, RailResult, serialize_prompt +from nemoguardrails.guardrails.rails_manager import RailsManager, _rail_call_record from nemoguardrails.llm.taskmanager import LLMTaskManager from nemoguardrails.rails.llm.config import RailsConfig from nemoguardrails.tracing.constants import GuardrailsAttributes @@ -971,3 +971,102 @@ async def test_safe_result_has_no_triggered_rail(self, content_safety_rails_mana result = await content_safety_rails_manager.is_input_safe(MESSAGES) assert result.is_safe assert result.triggered_rail is None + + +class TestRailCallRecordNaming: + """`_rail_call_record` names task/action_name in LLMRails' underscore form. + + The GenerationLog's ``executed_actions[].action_name`` and ``llm_calls[].task`` + must match LLMRails, which uses the prompt-template key (underscores) rather than + the space-separated Colang flow name. ``flow`` itself keeps the space form. + """ + + @pytest.mark.parametrize( + "flow, action_name, task", + [ + ( + "content safety check input $model=content_safety", + "content_safety_check_input", + "content_safety_check_input $model=content_safety", + ), + ("jailbreak detection model", "jailbreak_detection_model", "jailbreak_detection_model"), + ], + ids=["modelled", "modelless"], + ) + def test_underscore_task_and_action_name(self, flow, action_name, task): + """action_name/task use the underscore prompt-template key; ``flow`` keeps its space form.""" + record = _rail_call_record(flow=flow, rail_type="input", result=RailResult(is_safe=True), call=None) + + assert record.flow == flow + assert record.action_name == action_name + assert record.task == task + + +class TestSerializePrompt: + """`serialize_prompt` renders a message list to a role-labeled string for the log.""" + + def test_role_labeled_join(self): + """Each message renders as ': ', blank-line separated.""" + out = serialize_prompt( + [ + {"role": "system", "content": "be nice"}, + {"role": "user", "content": "hi"}, + ] + ) + assert out == "system: be nice\n\nuser: hi" + + def test_missing_content_renders_empty(self): + """A message with no content and no other fields renders as just the role label.""" + out = serialize_prompt([{"role": "assistant", "content": None}]) + assert out == "assistant: " + + def test_tool_calls_preserved(self): + """An assistant tool-call turn keeps its tool_calls rather than rendering blank.""" + out = serialize_prompt( + [ + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call_1", "function": {"name": "get_weather"}}], + } + ] + ) + assert "call_1" in out + assert "get_weather" in out + + def test_tool_result_fields_preserved(self): + """A tool-result turn keeps its tool_call_id and name alongside the content.""" + out = serialize_prompt([{"role": "tool", "content": "sunny", "tool_call_id": "call_1", "name": "get_weather"}]) + assert "sunny" in out + assert "call_1" in out + assert "get_weather" in out + + def test_reasoning_preserved(self): + """A reasoning-only turn keeps its reasoning text instead of dropping it.""" + out = serialize_prompt([{"role": "assistant", "content": None, "reasoning": "thinking hard"}]) + assert "thinking hard" in out + + +class TestParallelBatchDrainsRecords: + """`_run_rails_parallel` keeps records from every task that completed in a batch.""" + + @pytest.mark.asyncio + async def test_unsafe_first_does_not_drop_later_safe_records(self): + """When an unsafe rail sorts before a safe one that finished in the same wait batch, the safe rail's records survive.""" + manager = _make_rails_manager(RailsConfig.from_content(config=CONTENT_SAFETY_CONFIG)) + + unsafe_record = RailCallRecord(flow="jailbreak detection model", rail_type="input", is_safe=False) + safe_record = RailCallRecord(flow="content safety check input", rail_type="input", is_safe=True) + + async def _unsafe(): + return RailResult(is_safe=False, reason="blocked", records=(unsafe_record,)) + + async def _safe(): + return RailResult(is_safe=True, records=(safe_record,)) + + # Insertion order sets task_order, so the unsafe rail sorts first in the done batch. + rails = {"unsafe": _unsafe(), "safe": _safe()} + result = await manager._run_rails_parallel(rails, RailDirection.INPUT) + + assert result.is_safe is False + assert {r.flow for r in result.records} == {"jailbreak detection model", "content safety check input"} diff --git a/tests/guardrails/test_request_id.py b/tests/guardrails/test_request_id.py index 0cefcaa602..633b9cd25b 100644 --- a/tests/guardrails/test_request_id.py +++ b/tests/guardrails/test_request_id.py @@ -123,7 +123,7 @@ async def test_request_id_is_valid_hex(self, iorails): iorails.engine_registry.model_call = _make_capturing_mock(captured_ids, "llm", LLMResponse(content="Hello")) iorails.rails_manager.is_output_safe = _make_capturing_mock(captured_ids, "output", RailResult(is_safe=True)) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert len(captured_ids) == 3 for _, rid in captured_ids: @@ -138,7 +138,7 @@ async def test_same_id_across_all_layers(self, iorails): iorails.engine_registry.model_call = _make_capturing_mock(captured_ids, "llm", LLMResponse(content="Hello")) iorails.rails_manager.is_output_safe = _make_capturing_mock(captured_ids, "output", RailResult(is_safe=True)) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) ids = [rid for _, rid in captured_ids] assert ids[0] == ids[1] == ids[2] @@ -150,7 +150,7 @@ async def test_request_id_reset_after_completion(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="Hello")) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert get_request_id() == "no-req-id" @@ -159,7 +159,7 @@ async def test_request_id_reset_after_input_blocked(self, iorails): """ContextVar is reset even when the request is blocked at input.""" iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=False, reason="blocked")) - await iorails.generate_async([{"role": "user", "content": "bad"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "bad"}]) assert get_request_id() == "no-req-id" @@ -170,7 +170,7 @@ async def test_request_id_reset_after_output_blocked(self, iorails): iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="bad response")) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=False, reason="blocked")) - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert get_request_id() == "no-req-id" @@ -180,7 +180,7 @@ async def test_request_id_reset_after_exception(self, iorails): iorails.rails_manager.is_input_safe = AsyncMock(side_effect=RuntimeError("boom")) with pytest.raises(RuntimeError, match="boom"): - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert get_request_id() == "no-req-id" @@ -202,7 +202,7 @@ async def capture_input(*args, **kwargs): iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) for _ in range(5): - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) assert len(ids_per_request) == 5 assert len(set(ids_per_request)) == 5, f"Expected 5 unique IDs, got: {ids_per_request}" @@ -229,7 +229,7 @@ async def capture_output(*args, **kwargs): iorails.rails_manager.is_output_safe = capture_output for _ in range(3): - await iorails.generate_async([{"role": "user", "content": "hi"}]) + await iorails.generate_async(messages=[{"role": "user", "content": "hi"}]) # 3 calls per request × 3 requests = 9 captures assert len(request_snapshots) == 9 @@ -277,9 +277,9 @@ async def capture_output(*args, **kwargs): messages = [{"role": "user", "content": "hi"}] results = await asyncio.gather( - iorails.generate_async(messages), - iorails.generate_async(messages), - iorails.generate_async(messages), + iorails.generate_async(messages=messages), + iorails.generate_async(messages=messages), + iorails.generate_async(messages=messages), ) # All 3 requests completed successfully @@ -324,7 +324,7 @@ async def capture_output(*args, **kwargs): engine.engine_registry.model_call = capture_llm engine.rails_manager.is_output_safe = capture_output - await engine.generate_async([{"role": "user", "content": "hi"}]) + await engine.generate_async(messages=[{"role": "user", "content": "hi"}]) task_ids[task_num] = captured await asyncio.gather( @@ -351,8 +351,8 @@ async def test_contextvar_reset_after_concurrent_requests(self, iorails): messages = [{"role": "user", "content": "hi"}] await asyncio.gather( - iorails.generate_async(messages), - iorails.generate_async(messages), + iorails.generate_async(messages=messages), + iorails.generate_async(messages=messages), ) assert get_request_id() == "no-req-id" @@ -402,7 +402,7 @@ async def capturing_call(self_engine, messages, **kwargs): for model_engine in engine.engine_registry._engines.values(): model_engine._running = True - await engine.generate_async([{"role": "user", "content": "hello"}]) + await engine.generate_async(messages=[{"role": "user", "content": "hello"}]) # We expect 5 captures: # 1. rails_manager_input diff --git a/tests/guardrails/test_speculative_generation.py b/tests/guardrails/test_speculative_generation.py index f4e296e771..5d568a3da0 100644 --- a/tests/guardrails/test_speculative_generation.py +++ b/tests/guardrails/test_speculative_generation.py @@ -30,13 +30,32 @@ from nemoguardrails.guardrails.guardrails_types import RailResult from nemoguardrails.guardrails.iorails import REFUSAL_MESSAGE, IORails from nemoguardrails.rails.llm.config import RailsConfig -from nemoguardrails.types import LLMResponse +from nemoguardrails.rails.llm.options import GenerationResponse +from nemoguardrails.types import LLMResponse, UsageInfo from tests.guardrails.async_helpers import started_iorails from tests.guardrails.test_data import NEMOGUARDS_CONFIG, NEMOGUARDS_SPECULATIVE_CONFIG MESSAGES = [{"role": "user", "content": "hi"}] +def _hi_model() -> AsyncMock: + """Main-model mock returning a fixed 'Hi' response with usage.""" + return AsyncMock( + return_value=LLMResponse(content="Hi", usage=UsageInfo(input_tokens=5, output_tokens=3, total_tokens=8)) + ) + + +async def _slow_reject(messages, *, enabled=True): + """Input rails that reject after a short delay (so generation finishes first).""" + await asyncio.sleep(0.05) + return RailResult(is_safe=False, reason="unsafe") + + +async def _immediate_reject(messages, *, enabled=True): + """Input rails that reject immediately (so rails and generation finish in the same tick).""" + return RailResult(is_safe=False, reason="unsafe") + + @pytest_asyncio.fixture async def iorails(): async with started_iorails(NEMOGUARDS_SPECULATIVE_CONFIG) as instance: @@ -89,7 +108,7 @@ async def slow_llm(model_type, messages): iorails.engine_registry.model_call = slow_llm iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails.generate_async(MESSAGES) + result = await iorails.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": "Hello from LLM"} @@ -113,7 +132,7 @@ async def slow_llm(model_type, messages): iorails.engine_registry.model_call = slow_llm iorails.rails_manager.is_output_safe = AsyncMock() - result = await iorails.generate_async(MESSAGES) + result = await iorails.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} iorails.rails_manager.is_output_safe.assert_not_called() @@ -138,7 +157,7 @@ async def fast_llm(model_type, messages): iorails.engine_registry.model_call = fast_llm iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails.generate_async(MESSAGES) + result = await iorails.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": "Fast LLM response"} @@ -157,7 +176,7 @@ async def fast_llm(model_type, messages): iorails.engine_registry.model_call = fast_llm iorails.rails_manager.is_output_safe = AsyncMock() - result = await iorails.generate_async(MESSAGES) + result = await iorails.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} iorails.rails_manager.is_output_safe.assert_not_called() @@ -174,7 +193,7 @@ async def slow_rails(messages, *, enabled=True): iorails.engine_registry.model_call = AsyncMock(side_effect=RuntimeError("LLM crashed")) with pytest.raises(RuntimeError, match="LLM crashed"): - await iorails.generate_async(MESSAGES) + await iorails.generate_async(messages=MESSAGES) @pytest.mark.asyncio async def test_rails_error_cancels_generation(self, iorails): @@ -188,7 +207,7 @@ async def slow_llm(model_type, messages): iorails.engine_registry.model_call = slow_llm with pytest.raises(RuntimeError, match="Rails crashed"): - await iorails.generate_async(MESSAGES) + await iorails.generate_async(messages=MESSAGES) @pytest.mark.asyncio async def test_rails_reject_with_simultaneous_llm_exception(self, iorails, caplog_iorails): @@ -209,7 +228,7 @@ async def slow_raises(model_type, messages): iorails.rails_manager.is_output_safe = AsyncMock() with caplog_iorails.at_level("WARNING", logger="nemoguardrails.guardrails.iorails"): - result = await iorails.generate_async(MESSAGES) + result = await iorails.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} iorails.rails_manager.is_output_safe.assert_not_called() @@ -231,7 +250,7 @@ async def gen_raises(model_type, messages): with caplog_iorails.at_level("WARNING", logger="nemoguardrails.guardrails.iorails"): with pytest.raises(RuntimeError): - await iorails.generate_async(MESSAGES) + await iorails.generate_async(messages=MESSAGES) assert any("task error discarded during cleanup" in rec.message for rec in caplog_iorails.records) @@ -254,7 +273,7 @@ async def slow_llm(model_type, messages): iorails.rails_manager.is_output_safe = AsyncMock() with patch("nemoguardrails.guardrails.iorails.record_request_blocked") as record_mock: - result = await iorails.generate_async(MESSAGES) + result = await iorails.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} record_mock.assert_called_once() @@ -279,7 +298,7 @@ async def fast_llm(model_type, messages): iorails.rails_manager.is_output_safe = AsyncMock() with patch("nemoguardrails.guardrails.iorails.record_request_blocked") as record_mock: - result = await iorails.generate_async(MESSAGES) + result = await iorails.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} record_mock.assert_called_once() @@ -306,7 +325,7 @@ async def mock_output(messages, response, *, enabled=True): iorails_sequential.engine_registry.model_call = mock_generate iorails_sequential.rails_manager.is_output_safe = mock_output - await iorails_sequential.generate_async(MESSAGES) + await iorails_sequential.generate_async(messages=MESSAGES) assert call_order == ["input", "generate", "output"] @@ -360,7 +379,7 @@ async def slow_llm(model_type, messages): iorails_speculative_tracing.engine_registry.model_call = slow_llm iorails_speculative_tracing.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails_speculative_tracing.generate_async(MESSAGES) + result = await iorails_speculative_tracing.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": "Hello from LLM"} spans = span_exporter.get_finished_spans() @@ -386,7 +405,7 @@ async def slow_llm(model_type, messages): iorails_speculative_tracing.engine_registry.model_call = slow_llm iorails_speculative_tracing.rails_manager.is_output_safe = AsyncMock() - result = await iorails_speculative_tracing.generate_async(MESSAGES) + result = await iorails_speculative_tracing.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} spans = span_exporter.get_finished_spans() @@ -412,7 +431,7 @@ async def fast_llm(model_type, messages): iorails_speculative_tracing.engine_registry.model_call = fast_llm iorails_speculative_tracing.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - result = await iorails_speculative_tracing.generate_async(MESSAGES) + result = await iorails_speculative_tracing.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": "Fast LLM response"} spans = span_exporter.get_finished_spans() @@ -438,7 +457,7 @@ async def fast_llm(model_type, messages): iorails_speculative_tracing.engine_registry.model_call = fast_llm iorails_speculative_tracing.rails_manager.is_output_safe = AsyncMock() - result = await iorails_speculative_tracing.generate_async(MESSAGES) + result = await iorails_speculative_tracing.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} spans = span_exporter.get_finished_spans() @@ -462,7 +481,7 @@ async def test_sequential_mode_has_no_speculative_attrs(self, test_tracer, span_ iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="response")) iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) - await iorails.generate_async(MESSAGES) + await iorails.generate_async(messages=MESSAGES) spans = span_exporter.get_finished_spans() request_spans = [s for s in spans if s.name == "guardrails.request"] @@ -471,3 +490,41 @@ async def test_sequential_mode_has_no_speculative_attrs(self, test_tracer, span_ assert "speculative_generation.mode_active" not in attrs assert "speculative_generation.first_completed" not in attrs assert "speculative_generation.first_rejector" not in attrs + + +class TestSpeculativeGenerationTiming: + """The speculative path times the main call so its record and stats aren't left empty.""" + + @pytest.mark.asyncio + async def test_main_call_record_carries_timing(self, iorails): + """On the speculative path, the generation call in the log has real timestamps and a duration.""" + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = _hi_model() + + result = await iorails.generate_async(messages=MESSAGES, options={"log": {"llm_calls": True}}) + + assert isinstance(result, GenerationResponse) + assert result.log is not None + gen_call = next(c for c in (result.log.llm_calls or []) if c.task == "general") + assert gen_call.started_at is not None + assert gen_call.finished_at is not None + assert gen_call.duration is not None + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "reject_rails", [_slow_reject, _immediate_reject], ids=["gen-first", "rails-first-simultaneous"] + ) + async def test_blocked_speculative_records_completed_call(self, iorails, reject_rails): + """A speculative main call that completed before the input rails blocked is still logged — + whether generation finished first or both finished in the same tick.""" + iorails.rails_manager.is_input_safe = reject_rails + iorails.engine_registry.model_call = _hi_model() + + result = await iorails.generate_async(messages=MESSAGES, options={"log": {"llm_calls": True}}) + + assert isinstance(result, GenerationResponse) + assert result.log is not None + gen_calls = [c for c in (result.log.llm_calls or []) if c.task == "general"] + assert len(gen_calls) == 1 + assert gen_calls[0].duration is not None diff --git a/tests/guardrails/test_tool_rails_iorails.py b/tests/guardrails/test_tool_rails_iorails.py index d8593c6c56..62d8141d04 100644 --- a/tests/guardrails/test_tool_rails_iorails.py +++ b/tests/guardrails/test_tool_rails_iorails.py @@ -308,21 +308,21 @@ class TestNonStreamingToolCalls: @pytest.mark.asyncio async def test_allowed_tool_call_passes(self, iorails): _inject_json_response(iorails, _tool_call_payload("get_weather", '{"city": "Paris"}')) - result = await iorails.generate_async(MESSAGES, options={"llm_params": LLM_PARAMS}) - assert result["tool_calls"][0]["function"]["name"] == "get_weather" + result = await iorails.generate_async(messages=MESSAGES, options={"llm_params": LLM_PARAMS}) + assert result.tool_calls[0]["function"]["name"] == "get_weather" @pytest.mark.asyncio async def test_undeclared_tool_call_blocked(self, iorails): _inject_json_response(iorails, _tool_call_payload("rm_rf", "{}")) - result = await iorails.generate_async(MESSAGES, options={"llm_params": LLM_PARAMS}) - assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} + result = await iorails.generate_async(messages=MESSAGES, options={"llm_params": LLM_PARAMS}) + assert result.response == [{"role": "assistant", "content": REFUSAL_MESSAGE}] @pytest.mark.asyncio async def test_invalid_arguments_blocked(self, iorails): # Missing the required "city" argument violates the declared schema. _inject_json_response(iorails, _tool_call_payload("get_weather", "{}")) - result = await iorails.generate_async(MESSAGES, options={"llm_params": LLM_PARAMS}) - assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} + result = await iorails.generate_async(messages=MESSAGES, options={"llm_params": LLM_PARAMS}) + assert result.response == [{"role": "assistant", "content": REFUSAL_MESSAGE}] class TestSpeculativeToolCalls: @@ -336,14 +336,14 @@ async def speculative_iorails(self): @pytest.mark.asyncio async def test_undeclared_tool_call_blocked(self, speculative_iorails): _inject_json_response(speculative_iorails, _tool_call_payload("rm_rf", "{}")) - result = await speculative_iorails.generate_async(MESSAGES, options={"llm_params": LLM_PARAMS}) - assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} + result = await speculative_iorails.generate_async(messages=MESSAGES, options={"llm_params": LLM_PARAMS}) + assert result.response == [{"role": "assistant", "content": REFUSAL_MESSAGE}] @pytest.mark.asyncio async def test_allowed_tool_call_passes(self, speculative_iorails): _inject_json_response(speculative_iorails, _tool_call_payload("get_weather", '{"city": "Paris"}')) - result = await speculative_iorails.generate_async(MESSAGES, options={"llm_params": LLM_PARAMS}) - assert result["tool_calls"][0]["function"]["name"] == "get_weather" + result = await speculative_iorails.generate_async(messages=MESSAGES, options={"llm_params": LLM_PARAMS}) + assert result.tool_calls[0]["function"]["name"] == "get_weather" @pytest.mark.asyncio async def test_input_toggle_forwarded_to_input_rails(self, speculative_iorails): @@ -351,7 +351,7 @@ async def test_input_toggle_forwarded_to_input_rails(self, speculative_iorails): spy = AsyncMock(wraps=speculative_iorails.rails_manager.is_input_safe) speculative_iorails.rails_manager.is_input_safe = spy _inject_json_response(speculative_iorails, _text_payload("ok")) - await speculative_iorails.generate_async(MESSAGES, options={"rails": {"input": False}}) + await speculative_iorails.generate_async(messages=MESSAGES, options={"rails": {"input": False}}) assert spy.await_args.kwargs.get("enabled") is False @@ -368,14 +368,14 @@ async def iorails_config_tools(self): async def test_config_declared_tool_call_passes(self, iorails_config_tools): """A call to a config-declared tool passes the rail when the request carries no llm_params.""" _inject_json_response(iorails_config_tools, _tool_call_payload("get_weather", '{"city": "Paris"}')) - result = await iorails_config_tools.generate_async(MESSAGES) + result = await iorails_config_tools.generate_async(messages=MESSAGES) assert result["tool_calls"][0]["function"]["name"] == "get_weather" @pytest.mark.asyncio async def test_undeclared_tool_call_blocked_against_config_tools(self, iorails_config_tools): """A call to a tool absent from the config toolset is blocked when the request carries no llm_params.""" _inject_json_response(iorails_config_tools, _tool_call_payload("rm_rf", "{}")) - result = await iorails_config_tools.generate_async(MESSAGES) + result = await iorails_config_tools.generate_async(messages=MESSAGES) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} @pytest.mark.asyncio @@ -392,7 +392,7 @@ class TestNonStreamingToolResults: @pytest.mark.asyncio async def test_linked_tool_result_passes(self, iorails): _inject_json_response(iorails, _text_payload("It is sunny.")) - result = await iorails.generate_async(make_tool_conversation(result_call_id="call_1")) + result = await iorails.generate_async(messages=make_tool_conversation(result_call_id="call_1")) assert result == {"role": "assistant", "content": "It is sunny."} @pytest.mark.asyncio @@ -400,7 +400,7 @@ async def test_unlinked_tool_result_blocked_before_generation(self, iorails): # A blocked tool result must short-circuit before the model is ever called: # wire a transport that fails if used, and assert it stayed untouched. forbidden_post = _inject_forbidden_transport(iorails) - result = await iorails.generate_async(make_tool_conversation(result_call_id="call_999")) + result = await iorails.generate_async(messages=make_tool_conversation(result_call_id="call_999")) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} forbidden_post.assert_not_called() @@ -409,14 +409,14 @@ async def test_recycled_call_ids_across_turns_not_blocked(self, iorails): """Multi-turn conversation reusing the same call_id across turns is valid""" _inject_json_response(iorails, _text_payload("It's 12C in London.")) - result = await iorails.generate_async(multi_turn_reused_call_id_messages()) + result = await iorails.generate_async(messages=multi_turn_reused_call_id_messages()) assert result == {"role": "assistant", "content": "It's 12C in London."} @pytest.mark.asyncio async def test_malformed_prior_tool_call_does_not_block(self, iorails): """Malformed tool-call in prior turns must not block request.""" _inject_json_response(iorails, _text_payload("It's 12C in London.")) - result = await iorails.generate_async(malformed_prior_tool_call_messages()) + result = await iorails.generate_async(messages=malformed_prior_tool_call_messages()) assert result == {"role": "assistant", "content": "It's 12C in London."} @@ -473,18 +473,18 @@ async def test_tool_output_disabled_skips_tool_call_rail(self, iorails): # An undeclared tool call would normally block; disabling tool_output lets it through. _inject_json_response(iorails, _tool_call_payload("rm_rf", "{}")) options = {"llm_params": LLM_PARAMS, "rails": {"tool_output": False}} - result = await iorails.generate_async(MESSAGES, options=options) - assert result["tool_calls"][0]["function"]["name"] == "rm_rf" + result = await iorails.generate_async(messages=MESSAGES, options=options) + assert result.tool_calls[0]["function"]["name"] == "rm_rf" @pytest.mark.asyncio async def test_tool_input_disabled_skips_tool_result_rail(self, iorails): # An unlinked tool result would normally block; disabling tool_input lets it through. _inject_json_response(iorails, _text_payload("ok")) result = await iorails.generate_async( - make_tool_conversation(result_call_id="call_999"), + messages=make_tool_conversation(result_call_id="call_999"), options={"rails": {"tool_input": False}}, ) - assert result == {"role": "assistant", "content": "ok"} + assert result.response == [{"role": "assistant", "content": "ok"}] @pytest.mark.asyncio async def test_input_toggle_forwarded_to_input_rails(self, iorails): @@ -492,7 +492,7 @@ async def test_input_toggle_forwarded_to_input_rails(self, iorails): spy = AsyncMock(wraps=iorails.rails_manager.is_input_safe) iorails.rails_manager.is_input_safe = spy _inject_json_response(iorails, _text_payload("ok")) - await iorails.generate_async(MESSAGES, options={"rails": {"input": False}}) + await iorails.generate_async(messages=MESSAGES, options={"rails": {"input": False}}) assert spy.await_args.kwargs.get("enabled") is False @pytest.mark.asyncio @@ -501,7 +501,7 @@ async def test_output_toggle_forwarded_to_output_rails(self, iorails): spy = AsyncMock(wraps=iorails.rails_manager.is_output_safe) iorails.rails_manager.is_output_safe = spy _inject_json_response(iorails, _text_payload("ok")) - await iorails.generate_async(MESSAGES, options={"rails": {"output": False}}) + await iorails.generate_async(messages=MESSAGES, options={"rails": {"output": False}}) assert spy.await_args.kwargs.get("enabled") is False @pytest.mark.asyncio @@ -530,8 +530,8 @@ async def test_duplicate_tool_definitions_block(self, iorails): # parse_tools raises on a duplicate tool name; the request must fail closed. _inject_json_response(iorails, _tool_call_payload("get_weather", '{"city": "Paris"}')) dup_params = {"tools": [WEATHER_TOOL, WEATHER_TOOL]} - result = await iorails.generate_async(MESSAGES, options={"llm_params": dup_params}) - assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} + result = await iorails.generate_async(messages=MESSAGES, options={"llm_params": dup_params}) + assert result.response == [{"role": "assistant", "content": REFUSAL_MESSAGE}] class TestToolRailBlockMetrics: @@ -541,7 +541,7 @@ class TestToolRailBlockMetrics: async def test_nonstream_tool_result_block_records_input_metric(self, iorails): iorails._metrics_enabled = True with patch("nemoguardrails.guardrails.iorails.record_request_blocked") as record_blocked: - result = await iorails.generate_async(make_tool_conversation(result_call_id="call_999")) + result = await iorails.generate_async(messages=make_tool_conversation(result_call_id="call_999")) assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} record_blocked.assert_called_once_with(RailDirection.INPUT) @@ -550,8 +550,8 @@ async def test_nonstream_tool_call_block_records_output_metric(self, iorails): iorails._metrics_enabled = True _inject_json_response(iorails, _tool_call_payload("rm_rf", "{}")) with patch("nemoguardrails.guardrails.iorails.record_request_blocked") as record_blocked: - result = await iorails.generate_async(MESSAGES, options={"llm_params": LLM_PARAMS}) - assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} + result = await iorails.generate_async(messages=MESSAGES, options={"llm_params": LLM_PARAMS}) + assert result.response == [{"role": "assistant", "content": REFUSAL_MESSAGE}] record_blocked.assert_called_once_with(RailDirection.OUTPUT) @pytest.mark.asyncio diff --git a/tests/guardrails/test_topic_safety_iorails_actions.py b/tests/guardrails/test_topic_safety_iorails_actions.py index 6504552dd8..7abbfd9a9b 100644 --- a/tests/guardrails/test_topic_safety_iorails_actions.py +++ b/tests/guardrails/test_topic_safety_iorails_actions.py @@ -92,10 +92,14 @@ def test_multi_turn_messages_included(self, action): class TestTopicSafetyParseResponse: def test_on_topic(self, action): - assert action._parse_response("on-topic") == RailResult(is_safe=True) + result = action._parse_response("on-topic") + assert result == RailResult(is_safe=True) + assert result.return_value == {"on_topic": True} def test_off_topic(self, action): - assert action._parse_response("off-topic") == RailResult(is_safe=False, reason="Topic safety: off-topic") + result = action._parse_response("off-topic") + assert result == RailResult(is_safe=False, reason="Topic safety: off-topic") + assert result.return_value == {"on_topic": False} @pytest.mark.parametrize("text", ["Off-Topic", " off-topic \n", "OFF-TOPIC"]) def test_off_topic_variants(self, action, text):