From f92bc139bb0704079ae6e4b8339a033977b0317b Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 14:00:57 +0200 Subject: [PATCH 1/5] feat(gooddata-eval): capture agent reasoning steps in ChatResult The SSE reasoning events were already being read to produce reasoning_step_count, but the step text itself was discarded. Keep it as reasoning_steps on ChatResult/ItemReport and surface it in the JSON report so eval consumers can inspect the agent's actual reasoning trace, not just how many steps it took. --- .../src/gooddata_eval/core/chat/sse_client.py | 1 + .../src/gooddata_eval/core/models.py | 1 + .../core/reporting/json_report.py | 1 + .../src/gooddata_eval/core/runner.py | 2 ++ .../gooddata-eval/tests/test_reporting.py | 3 +++ packages/gooddata-eval/tests/test_runner.py | 19 +++++++++++++++++++ .../gooddata-eval/tests/test_sse_client.py | 8 ++++++++ 7 files changed, 35 insertions(+) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index d1e375c97..eb85a5f28 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -188,6 +188,7 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult: "alertProposals": acc.alert_proposals, "toolCallEvents": acc.tool_call_events, "reasoningStepCount": len(acc.reasoning_steps), + "reasoningSteps": [step["summary"] for step in acc.reasoning_steps], } if acc.visualizations: payload["createdVisualizations"] = { diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index ee58dcc80..40f7ede96 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -98,6 +98,7 @@ class ChatResult(BaseModel): alert_proposals: list[dict] = Field(default_factory=list, alias="alertProposals") tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents") reasoning_step_count: int = Field(default=0, alias="reasoningStepCount") + reasoning_steps: list[str] = Field(default_factory=list, alias="reasoningSteps") conversation_id: str | None = Field(default=None, alias="conversationId") response_id: str | None = Field(default=None, alias="responseId") # True once gen-ai's response_ended event arrived. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index d4c7b4a3e..1a28e0001 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -36,6 +36,7 @@ def _build_run_dict(report: EvalReport) -> dict: "detail": item.best_detail, "conversation_id": item.conversation_id, "response_id": item.response_id, + "reasoning": item.reasoning_steps, } for item in report.items }, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 5bd56c9bf..0161cc3f8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -33,6 +33,7 @@ class ItemReport: best_detail: dict = field(default_factory=dict) conversation_id: str | None = None response_id: str | None = None + reasoning_steps: list[str] = field(default_factory=list) @property def avg_latency_s(self) -> float: @@ -116,6 +117,7 @@ def _run_one_item( chat_result = backend.ask(item) report.conversation_id = getattr(chat_result, "conversation_id", None) or report.conversation_id report.response_id = getattr(chat_result, "response_id", None) or report.response_id + report.reasoning_steps = getattr(chat_result, "reasoning_steps", None) or report.reasoning_steps evaluation = evaluator.evaluate(item, chat_result) latency = time.perf_counter() - t0 report.runs += 1 diff --git a/packages/gooddata-eval/tests/test_reporting.py b/packages/gooddata-eval/tests/test_reporting.py index f191de38e..467fda7fb 100644 --- a/packages/gooddata-eval/tests/test_reporting.py +++ b/packages/gooddata-eval/tests/test_reporting.py @@ -23,6 +23,7 @@ def _report() -> EvalReport: pass_at_k=True, runs=2, latency_s=2.5, + reasoning_steps=["step one", "step two"], ), ItemReport( id="i2", @@ -48,6 +49,8 @@ def test_build_json_report_keyed_by_item_id(): assert data["items"]["i1"]["pass_at_k"] is True assert data["items"]["i1"]["latency_s"] == 2.5 assert data["items"]["i1"]["avg_latency_s"] == 1.25 + assert data["items"]["i1"]["reasoning"] == ["step one", "step two"] + assert data["items"]["i2"]["reasoning"] == [] def test_write_json_report_creates_file(tmp_path): diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index a3f1742b1..23ff8c64d 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -258,6 +258,25 @@ def ask(self, item: DatasetItem) -> ChatResult: assert "conversation_id" not in report.items[0].error +def test_run_items_carries_reasoning_steps_from_chat_result(): + """reasoning_steps from the ChatResult surfaces on the item report, same as conversation_id.""" + + class _ReasoningBackend: + def ask(self, item: DatasetItem) -> ChatResult: + return ChatResult.model_validate( + {"textResponse": "which metric?", "reasoningSteps": ["step one", "step two"]} + ) + + report = run_items([_item()], _ReasoningBackend(), runs=1) + assert report.items[0].reasoning_steps == ["step one", "step two"] + + +def test_run_items_reasoning_steps_empty_when_chat_result_has_none(): + backend = _FakeBackend([_empty_chat()]) + report = run_items([_item()], backend, runs=1) + assert report.items[0].reasoning_steps == [] + + def test_run_items_callback_exception_is_logged_not_swallowed(capsys): """A raising callback prints a traceback to stderr but the run continues.""" backend = _FakeBackend([_chat_with(_viz_obj())] * 2) diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 6361410ba..c02b1466f 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -175,9 +175,17 @@ def test_parse_sse_lines_counts_reasoning_steps(): ] result = parse_sse_lines(lines) assert result.reasoning_step_count == 2 + assert result.reasoning_steps == ["step one", "step two"] assert result.text_response == "Done" +def test_parse_sse_lines_reasoning_steps_empty_when_no_reasoning_events(): + lines = ['data: {"item": {"role": "assistant", "content": {"type": "text", "text": "Done"}}}'] + result = parse_sse_lines(lines) + assert result.reasoning_step_count == 0 + assert result.reasoning_steps == [] + + def test_parse_sse_lines_prefers_multipart_viz_over_adhoc_fallback(): """Real multipart visualization takes priority over adhoc tool call stash.""" From d8d656a1540848a190a2ef35c3f90d0f04d7ee26 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 14:12:51 +0200 Subject: [PATCH 2/5] test(gooddata-eval): cover multi-run reasoning_steps retention Addresses CodeRabbit nitpick on PR #1708: a later run returning no reasoning events must not clobber an earlier run's captured steps (runner.py:120's `or` pattern, same as conversation_id/response_id). --- packages/gooddata-eval/tests/test_runner.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index 23ff8c64d..925c214a6 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -277,6 +277,22 @@ def test_run_items_reasoning_steps_empty_when_chat_result_has_none(): assert report.items[0].reasoning_steps == [] +def test_run_items_reasoning_steps_keeps_earlier_run_when_later_run_is_empty(): + """A later run with no reasoning events must not clobber an earlier run's steps (runner.py:120's `or`).""" + + class _MixedReasoningBackend: + def __init__(self): + self.calls = 0 + + def ask(self, item: DatasetItem) -> ChatResult: + self.calls += 1 + steps = ["step one", "step two"] if self.calls == 1 else [] + return ChatResult.model_validate({"textResponse": "answer", "reasoningSteps": steps}) + + report = run_items([_item()], _MixedReasoningBackend(), runs=2) + assert report.items[0].reasoning_steps == ["step one", "step two"] + + def test_run_items_callback_exception_is_logged_not_swallowed(capsys): """A raising callback prints a traceback to stderr but the run continues.""" backend = _FakeBackend([_chat_with(_viz_obj())] * 2) From 7d4496ff9ea3fea71e47a1a98ad6cc94ce672285 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Wed, 5 Aug 2026 09:13:35 +0200 Subject: [PATCH 3/5] feat(gooddata-eval): capture reasoning_steps through the agentic-CLI path 6001d2fd wired ChatResult.reasoning_steps through runner.py's generic single-turn path only. The agentic-CLI path (cli/agentic_runner.py -> evaluate_agentic_*) builds its own ItemReport and never touched it, so agentic_alert_skill/agentic_metric_skill/agentic_conversation items could never produce a reasoning trace, no matter what the platform emitted. Accumulates reasoning_steps across every send_message call in each of the three evaluators' run loops, attaches it to the run/turn result, and surfaces it from evaluate_agentic_* either as the return value (pass) or as an attribute on the raised exception (fail) -- mirroring the existing conversation_id-on-exception idiom in ChatClient.ask(). run_agentic_items picks it up from either path onto ItemReport.reasoning_steps, which json_report.py already serializes unconditionally. general_question/guardrail/search_tool/visualization are left untouched -- their evaluate_agentic_* functions still return None, unchanged. --- .../src/gooddata_eval/cli/agentic_runner.py | 26 ++-- .../gooddata_eval/core/agentic/alert_skill.py | 21 ++- .../core/agentic/conversation.py | 22 ++- .../core/agentic/metric_skill.py | 21 ++- .../tests/test_agentic_alert_skill.py | 115 ++++++++++++++++ .../tests/test_agentic_conversation.py | 127 ++++++++++++++++++ .../tests/test_agentic_metric_skill.py | 99 ++++++++++++++ .../tests/test_agentic_runner.py | 74 +++++++++- 8 files changed, 482 insertions(+), 23 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index b379fadd9..1526c309d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -86,8 +86,12 @@ def _dispatch_agentic( model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, -) -> None: - """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" +) -> list[str] | None: + """Call the appropriate evaluate_agentic_* function for the item's test_kind. + + Returns whatever that function returns -- only alert_skill/metric_skill/conversation + currently return their reasoning_steps; the rest still return None (unchanged). + """ kind = item.test_kind eo = item.expected_output lf_kw: _LfKw = { @@ -100,7 +104,7 @@ def _dispatch_agentic( } if kind in ("vis_agentic", "agentic_visualization"): - evaluate_agentic_visualization( + return evaluate_agentic_visualization( host=host, token=token, workspace_id=workspace_id, @@ -111,7 +115,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_metric_skill": - evaluate_agentic_metric_skill( + return evaluate_agentic_metric_skill( host=host, token=token, workspace_id=workspace_id, @@ -122,7 +126,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_alert_skill": - evaluate_agentic_alert_skill( + return evaluate_agentic_alert_skill( host=host, token=token, workspace_id=workspace_id, @@ -136,7 +140,7 @@ def _dispatch_agentic( eo_dict = eo if isinstance(eo, dict) else {} tool_call = eo_dict.get("tool_call", {}) expected_args = tool_call.get("function_arguments", eo_dict) - evaluate_agentic_search_tool( + return evaluate_agentic_search_tool( host=host, token=token, workspace_id=workspace_id, @@ -147,7 +151,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_general_question": - evaluate_agentic_general_question( + return evaluate_agentic_general_question( host=host, token=token, workspace_id=workspace_id, @@ -158,7 +162,7 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_guardrail": - evaluate_agentic_guardrail( + return evaluate_agentic_guardrail( host=host, token=token, workspace_id=workspace_id, @@ -180,7 +184,7 @@ def _dispatch_agentic( ) elif kind == "agentic_conversation": fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {} - evaluate_agentic_conversation( + return evaluate_agentic_conversation( host=host, token=token, workspace_id=workspace_id, @@ -228,14 +232,16 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic( + reasoning_steps = _dispatch_agentic( item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id ) item_report.pass_at_k = True item_report.runs = k + item_report.reasoning_steps = reasoning_steps or [] except AssertionError as exc: item_report.pass_at_k = False item_report.runs = k + item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] print(f"[agentic] {item.id} FAIL: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index e94584f39..532abced8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -6,7 +6,7 @@ import json import os import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from gooddata_sdk import GoodDataSdk @@ -302,6 +302,7 @@ class AlertRunResult: alert_id: str | None eval: AlertEvaluation actual_alert_arguments: dict + reasoning_steps: list[str] = field(default_factory=list) @dataclass @@ -450,6 +451,7 @@ def _run_once(conv_id: str) -> AlertRunResult: alert_id: str | None = None actual_args: dict = {} tool_called = False + reasoning_steps: list[str] = [] # conversation_history stores prior turns for GPT-4o context. # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply. conversation_history: list = [] @@ -457,6 +459,7 @@ def _run_once(conv_id: str) -> AlertRunResult: for _iteration in range(max_iterations): chat_result = client.send_message(conv_id, current_question) + reasoning_steps.extend(chat_result.reasoning_steps or []) alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or []) if tool_called: alert_id_to_delete = alert_id @@ -492,6 +495,7 @@ def _run_once(conv_id: str) -> AlertRunResult: alert_id=alert_id, eval=ev, actual_alert_arguments=actual_args, + reasoning_steps=reasoning_steps, ) finally: if alert_id_to_delete: @@ -542,6 +546,7 @@ class AlertSkillAssertionError(AssertionError): """Raised when an alert-skill evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] def evaluate_agentic_alert_skill( @@ -561,8 +566,13 @@ def evaluate_agentic_alert_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.""" +) -> list[str]: + """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure. + + Returns the best run's reasoning_steps on success; on failure the same list is attached + to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception + idiom in `ChatClient.ask()`) so callers can retrieve it either way. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -636,7 +646,7 @@ def evaluate_agentic_alert_skill( if not summary.pass_at_k: best = summary.best ev = best.eval - raise AlertSkillAssertionError( + exc = AlertSkillAssertionError( f"Alert skill assertion failed. strict_pass={ev.strict_pass}. " f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, " @@ -644,3 +654,6 @@ def evaluate_agentic_alert_skill( f"recipients_correct={ev.recipients_correct}. " f"Actual args: {best.actual_alert_arguments}" ) + exc.reasoning_steps = best.reasoning_steps + raise exc + return summary.best.reasoning_steps diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 1e50352c2..51fd1d4e9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -5,7 +5,7 @@ import json import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Literal from gooddata_sdk import GoodDataSdk @@ -265,6 +265,7 @@ class ConversationResult: full_skill_coverage: bool conversation_success: bool total_clarification_turns: int + reasoning_steps: list[str] = field(default_factory=list) def run_agentic_conversation( @@ -296,6 +297,7 @@ def run_agentic_conversation( # not persist in the (shared) workspace and get reused by a later test. Deferred to # the end — a later turn may $ref a metric an earlier turn created. created_metric_ids: list[str] = [] + reasoning_steps: list[str] = [] try: if initial_conversation_id is not None: @@ -332,6 +334,7 @@ def run_agentic_conversation( chat_result = client.send_message(conversation_id, current_message) final_result = chat_result all_tool_calls.extend(chat_result.tool_call_events or []) + reasoning_steps.extend(chat_result.reasoning_steps or []) if _check_output_present(resolved_turn, chat_result): break @@ -395,6 +398,7 @@ def run_agentic_conversation( full_skill_coverage=full_skill_coverage, conversation_success=conversation_success, total_clarification_turns=total_clarification_turns, + reasoning_steps=reasoning_steps, ) @@ -402,6 +406,7 @@ class ConversationAssertionError(AssertionError): """Raised when a conversation evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] def evaluate_agentic_conversation( @@ -419,8 +424,14 @@ def evaluate_agentic_conversation( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run conversation evaluation, log to Langfuse, and raise on failure.""" +) -> list[str]: + """Run conversation evaluation, log to Langfuse, and raise on failure. + + Returns the conversation's reasoning_steps on success; on failure the same list is + attached to the raised exception as ``.reasoning_steps`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve it + either way. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -497,8 +508,11 @@ def evaluate_agentic_conversation( if not result.conversation_success: failed_turns = [tr for tr in result.turn_results if not tr.skill_success] - raise ConversationAssertionError( + exc = ConversationAssertionError( f"Conversation assertion failed. " f"full_skill_coverage={result.full_skill_coverage}. " f"Failed turns: {[t.turn_id for t in failed_turns]}" ) + exc.reasoning_steps = result.reasoning_steps + raise exc + return result.reasoning_steps diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 34d3f0324..2db890209 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -5,7 +5,7 @@ import os import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from gooddata_sdk import GoodDataSdk @@ -129,6 +129,7 @@ class MetricRunResult: actual_maql: str maql_correct: bool total_turns: float + reasoning_steps: list[str] = field(default_factory=list) @dataclass @@ -205,11 +206,13 @@ def _execute_single_metric_run( metric_id_to_delete: str | None = None turns = 0 current_question = question + reasoning_steps: list[str] = [] try: for _iteration in range(max_iterations): turns += 1 chat_result = client.send_message(conversation_id, current_question) + reasoning_steps.extend(chat_result.reasoning_steps or []) candidate = _extract_metric_result(chat_result.tool_call_events or []) if candidate is not None: metric_result = candidate @@ -236,6 +239,7 @@ def _execute_single_metric_run( actual_maql=actual_maql, maql_correct=maql_correct, total_turns=float(turns), + reasoning_steps=reasoning_steps, ) finally: if metric_id_to_delete: @@ -306,6 +310,7 @@ class MetricSkillAssertionError(AssertionError): """Raised when a metric-skill evaluation fails.""" __tracebackhide__ = True + reasoning_steps: list[str] def evaluate_agentic_metric_skill( @@ -325,8 +330,13 @@ def evaluate_agentic_metric_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> None: - """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.""" +) -> list[str]: + """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure. + + Returns the best run's reasoning_steps on success; on failure the same list is attached + to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception + idiom in `ChatClient.ask()`) so callers can retrieve it either way. + """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -391,9 +401,12 @@ def evaluate_agentic_metric_skill( best = summary.best expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) - raise MetricSkillAssertionError( + exc = MetricSkillAssertionError( f"Metric skill assertion failed. " f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " f"Actual MAQL: {best.actual_maql}." ) + exc.reasoning_steps = best.reasoning_steps + raise exc + return summary.best.reasoning_steps diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 13c94c2bb..140c62fd4 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -2,13 +2,16 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import MagicMock, patch +import pytest from gooddata_eval.core.agentic.alert_skill import ( AlertEvaluation, + AlertSkillAssertionError, _check_filters, _check_trigger, _deep_subset, _normalize_expected_output, _to_number, + evaluate_agentic_alert_skill, generate_simulated_alert_response, render_alert_proposal, run_agentic_alert_skill, @@ -489,3 +492,115 @@ def test_run_agentic_alert_skill_answers_proposal_only_confirmation_turn(): assert "admin@gooddata.com" in agent_message assert summary.best.eval.alert_created is True assert summary.best.alert_id == "alert-1" + + +def test_run_agentic_alert_skill_accumulates_reasoning_steps_across_iterations(): + proposal_turn = ChatResult.model_validate( + { + "text_response": None, + "alertProposals": [_PROPOSAL], + "toolCallEvents": [ + {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None} + ], + "reasoningSteps": ["step one"], + } + ) + created_turn = ChatResult.model_validate( + { + "text_response": "Alert created.", + "toolCallEvents": [ + { + "functionName": "create_metric_alert", + "functionArguments": '{"operator": "GREATER_THAN", "threshold": 500}', + "result": '{"id": "alert-1"}', + } + ], + "reasoningSteps": ["step two"], + } + ) + mock_client = MagicMock() + mock_client.send_message.side_effect = [proposal_turn, created_turn] + + with ( + patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), + patch( + "gooddata_eval.core.agentic.alert_skill.generate_simulated_alert_response", + return_value="Yes, please proceed to create the alert.", + ), + patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), + ): + summary = run_agentic_alert_skill( + host="http://host", + token="tok", + workspace_id="ws1", + question="Notify me whenever the number of orders goes above 500", + expected_output={"operator": "GREATER_THAN", "threshold": 500}, + k=1, + max_iterations=6, + initial_conversation_id="conv-1", + ) + + assert summary.best.reasoning_steps == ["step one", "step two"] + + +def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): + chat_result = ChatResult.model_validate( + { + "text_response": "Alert created.", + "toolCallEvents": [ + { + "functionName": "create_metric_alert", + "functionArguments": '{"operator": "GREATER_THAN", "threshold": 500}', + "result": '{"id": "alert-1"}', + } + ], + "reasoningSteps": ["thinking about it"], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = chat_result + + with ( + patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), + ): + reasoning = evaluate_agentic_alert_skill( + host="http://host", + token="tok", + workspace_id="ws1", + question="Notify me whenever the number of orders goes above 500", + expected_output={"operator": "GREATER_THAN", "threshold": 500}, + k=1, + max_iterations=1, + ) + + assert reasoning == ["thinking about it"] + + +def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail(): + chat_result = ChatResult.model_validate( + { + "text_response": "I cannot create the alert", + "toolCallEvents": [], + "reasoningSteps": ["confused thinking"], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = chat_result + + with ( + patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), + pytest.raises(AlertSkillAssertionError) as exc_info, + ): + evaluate_agentic_alert_skill( + host="http://host", + token="tok", + workspace_id="ws1", + question="Create alert", + expected_output={"operator": "GREATER_THAN", "threshold": 100}, + k=1, + max_iterations=1, + ) + assert exc_info.value.reasoning_steps == ["confused thinking"] diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index cb2970272..cc4ab0791 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -4,10 +4,12 @@ import pytest from gooddata_eval.core.agentic.conversation import ( + ConversationAssertionError, ConversationFixture, TurnDefinition, TurnResult, _resolve_refs, + evaluate_agentic_conversation, run_agentic_conversation, ) from gooddata_eval.core.models import ChatResult, ToolCallEvent @@ -467,3 +469,128 @@ def test_run_agentic_conversation_records_a_failed_turn_when_a_ref_cannot_be_res assert result.turn_results[1].no_error is False assert result.turn_results[2].skill_success is True assert result.conversation_success is False + + +def test_run_agentic_conversation_accumulates_reasoning_steps_across_turns(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + tc = MagicMock(spec=ToolCallEvent) + tc.function_name = "set_skills" + tc.parsed_arguments = lambda: {"skills": ["visualization"]} + + turn1_result = MagicMock() + turn1_result.text_response = "Here is your visualization" + turn1_result.created_visualizations = [MagicMock()] + turn1_result.tool_call_events = [tc] + turn1_result.reasoning_steps = ["turn one reasoning"] + + turn2_result = MagicMock() + turn2_result.text_response = "Here is another visualization" + turn2_result.created_visualizations = [MagicMock()] + turn2_result.tool_call_events = [tc] + turn2_result.reasoning_steps = ["turn two reasoning"] + + mock_client.send_message.side_effect = [turn1_result, turn2_result] + + fixture = ConversationFixture( + id="test-reasoning", + expected_skills=["visualization"], + turns=[ + TurnDefinition( + turn_id="t1", + message="Make a chart", + expected_skill="visualization", + expected_output_type="visualization", + ), + TurnDefinition( + turn_id="t2", + message="Make another chart", + expected_skill="visualization", + expected_output_type="visualization", + ), + ], + ) + with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=fixture, + ) + + assert result.reasoning_steps == ["turn one reasoning", "turn two reasoning"] + + +def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + tc = MagicMock(spec=ToolCallEvent) + tc.function_name = "set_skills" + tc.parsed_arguments = lambda: {"skills": ["visualization"]} + chat_result = MagicMock() + chat_result.text_response = "Here is your visualization" + chat_result.created_visualizations = [MagicMock()] + chat_result.tool_call_events = [tc] + chat_result.reasoning_steps = ["thinking about it"] + mock_client.send_message.return_value = chat_result + + fixture = ConversationFixture( + id="test-1", + expected_skills=["visualization"], + turns=[ + TurnDefinition( + turn_id="t1", + message="Make a chart", + expected_skill="visualization", + expected_output_type="visualization", + ) + ], + ) + with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client): + reasoning = evaluate_agentic_conversation( + host="http://host", + token="tok", + workspace_id="ws1", + fixture=fixture, + ) + assert reasoning == ["thinking about it"] + + +def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + tc = MagicMock(spec=ToolCallEvent) + tc.function_name = "set_skills" + tc.parsed_arguments = lambda: {"skills": ["other_skill"]} + chat_result = MagicMock() + chat_result.text_response = "Here is something else" + chat_result.created_visualizations = None + chat_result.tool_call_events = [tc] + chat_result.alert_proposals = [] + chat_result.reasoning_steps = ["confused thinking"] + mock_client.send_message.return_value = chat_result + + fixture = ConversationFixture( + id="test-1", + expected_skills=["visualization"], + turns=[ + TurnDefinition( + turn_id="t1", + message="Make a chart", + expected_skill="visualization", + expected_output_type="visualization", + ) + ], + ) + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + pytest.raises(ConversationAssertionError) as exc_info, + ): + evaluate_agentic_conversation( + host="http://host", + token="tok", + workspace_id="ws1", + fixture=fixture, + max_clarification_turns=0, + ) + assert exc_info.value.reasoning_steps == ["confused thinking"] diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 5f9dec88a..ccc24299c 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -9,9 +9,11 @@ from gooddata_eval.core.agentic.metric_skill import ( AgenticMetricSummary, MetricRunResult, + MetricSkillAssertionError, SimulatedResponseError, _delete_metric, _normalize_maql, + evaluate_agentic_metric_skill, generate_simulated_response, run_agentic_metric_skill, ) @@ -318,3 +320,100 @@ def test_run_agentic_metric_skill_fails_the_run_when_the_simulated_reply_cannot_ assert summary.best.total_turns == 1.0 mock_client.close.assert_called_once() mock_sim.assert_called_once_with("Which brand field should I count?", {"maql": "SELECT {metric/foo}"}) + + +def test_run_agentic_metric_skill_accumulates_reasoning_steps_across_iterations(): + clarify_turn = ChatResult.model_validate( + { + "textResponse": "Could you clarify which foo you mean?", + "toolCallEvents": [], + "reasoningSteps": ["step one"], + } + ) + created_turn = ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + "reasoningSteps": ["step two"], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [clarify_turn, created_turn] + + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.metric_skill.generate_simulated_response", return_value="It's foo"), + ): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=2, + ) + + assert summary.best.reasoning_steps == ["step one", "step two"] + + +def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + "reasoningSteps": ["thinking about it"], + } + ) + with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): + reasoning = evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=1, + ) + assert reasoning == ["thinking about it"] + + +def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "I will work on that.", + "toolCallEvents": [], + "reasoningSteps": ["confused thinking"], + } + ) + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=1, + ) + assert exc_info.value.reasoning_steps == ["confused thinking"] diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 3cbff3137..fda604542 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1,8 +1,10 @@ # (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import patch import pytest -from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.cli.agentic_runner import _dispatch_agentic, run_agentic_items +from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError from gooddata_eval.core.models import DatasetItem @@ -91,3 +93,73 @@ def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_ agent_id="agent-1", ) assert mock_eval.call_args.kwargs["agent_id"] == "agent-1" + + +def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem: + return DatasetItem( + id="item-1", + dataset_name="d", + test_kind=test_kind, + question="Alert me when revenue drops below 100.", + expected_output={"operator": "LESS_THAN", "threshold": 100}, + ) + + +def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + return_value=["it created the alert"], + ): + report = run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].pass_at_k is True + assert report.items[0].reasoning_steps == ["it created the alert"] + + +def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): + exc = AlertSkillAssertionError("nope") + exc.reasoning_steps = ["it got confused"] + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].pass_at_k is False + assert report.items[0].reasoning_steps == ["it got confused"] + + +def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_none(): + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + side_effect=AlertSkillAssertionError("nope"), + ): + report = run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].reasoning_steps == [] + + +def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds(): + # general_question/guardrail/search_tool/visualization still return None -- unchanged. + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_guardrail", return_value=None): + report = run_agentic_items( + [_item(test_kind="agentic_guardrail")], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + assert report.items[0].pass_at_k is True + assert report.items[0].reasoning_steps == [] From 2b1e3ad901d15e1ce81a2af9843ba35e8f7f4c5a Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Wed, 5 Aug 2026 14:02:51 +0200 Subject: [PATCH 4/5] feat(gooddata-eval): capture conversation_id/response_id through the agentic-CLI path 8010bd44 wired reasoning_steps through cli/agentic_runner.py -> evaluate_agentic_*, but conversation_id/response_id stayed unset on ItemReport for every agentic kind (agentic_alert_skill/agentic_metric_skill/agentic_conversation) -- each ChatResult already carries both, and conversation_id was already threaded up to the Alert/Metric/ConversationRunResult layer, but neither ever reached the top-level evaluate_agentic_* return value or its failure exception, so run_agentic_items had nothing to read. Mirrors the reasoning_steps idiom exactly: widens each evaluate_agentic_*'s return from list[str] to (reasoning_steps, conversation_id, response_id), attaches all three to the raised exception on failure, and has run_agentic_items unpack either form (tuple or the untouched kinds' bare list/None) onto ItemReport.conversation_id /response_id. response_id is new at the RunResult layer for all three kinds -- captured as the last non-null value across a run's turns, same pattern already used for reasoning_steps accumulation. general_question/guardrail/search_tool/visualization untouched (already populated via the single-turn runner.py path, not this one). Co-Authored-By: Claude Opus 5 --- .../src/gooddata_eval/cli/agentic_runner.py | 16 +++++++++++---- .../gooddata_eval/core/agentic/alert_skill.py | 20 ++++++++++++++----- .../core/agentic/conversation.py | 19 +++++++++++++----- .../core/agentic/metric_skill.py | 20 ++++++++++++++----- .../tests/test_agentic_alert_skill.py | 6 +++++- .../tests/test_agentic_conversation.py | 8 +++++++- .../tests/test_agentic_metric_skill.py | 6 +++++- .../tests/test_agentic_runner.py | 12 ++++++++++- 8 files changed, 84 insertions(+), 23 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 1526c309d..02c599326 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -86,11 +86,12 @@ def _dispatch_agentic( model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, -) -> list[str] | None: +) -> tuple[list[str], str, str | None] | list[str] | None: """Call the appropriate evaluate_agentic_* function for the item's test_kind. - Returns whatever that function returns -- only alert_skill/metric_skill/conversation - currently return their reasoning_steps; the rest still return None (unchanged). + Returns whatever that function returns -- alert_skill/metric_skill/conversation return + their (reasoning_steps, conversation_id, response_id); the rest still return None + (unchanged). """ kind = item.test_kind eo = item.expected_output @@ -232,16 +233,23 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - reasoning_steps = _dispatch_agentic( + outcome = _dispatch_agentic( item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id ) + reasoning_steps, conversation_id, response_id = ( + outcome if isinstance(outcome, tuple) else (outcome, None, None) + ) item_report.pass_at_k = True item_report.runs = k item_report.reasoning_steps = reasoning_steps or [] + item_report.conversation_id = conversation_id + item_report.response_id = response_id except AssertionError as exc: item_report.pass_at_k = False item_report.runs = k item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] + item_report.conversation_id = getattr(exc, "conversation_id", None) + item_report.response_id = getattr(exc, "response_id", None) print(f"[agentic] {item.id} FAIL: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 532abced8..4eb4d4b5b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -303,6 +303,7 @@ class AlertRunResult: eval: AlertEvaluation actual_alert_arguments: dict reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -452,6 +453,7 @@ def _run_once(conv_id: str) -> AlertRunResult: actual_args: dict = {} tool_called = False reasoning_steps: list[str] = [] + response_id: str | None = None # conversation_history stores prior turns for GPT-4o context. # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply. conversation_history: list = [] @@ -460,6 +462,7 @@ def _run_once(conv_id: str) -> AlertRunResult: for _iteration in range(max_iterations): chat_result = client.send_message(conv_id, current_question) reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or []) if tool_called: alert_id_to_delete = alert_id @@ -496,6 +499,7 @@ def _run_once(conv_id: str) -> AlertRunResult: eval=ev, actual_alert_arguments=actual_args, reasoning_steps=reasoning_steps, + response_id=response_id, ) finally: if alert_id_to_delete: @@ -547,6 +551,8 @@ class AlertSkillAssertionError(AssertionError): __tracebackhide__ = True reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_alert_skill( @@ -566,12 +572,14 @@ def evaluate_agentic_alert_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> list[str]: +) -> tuple[list[str], str, str | None]: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure. - Returns the best run's reasoning_steps on success; on failure the same list is attached - to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception - idiom in `ChatClient.ask()`) so callers can retrieve it either way. + Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on + failure the same three values are attached to the raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them + either way. """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -655,5 +663,7 @@ def evaluate_agentic_alert_skill( f"Actual args: {best.actual_alert_arguments}" ) exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id raise exc - return summary.best.reasoning_steps + return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 51fd1d4e9..c0c00c362 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -266,6 +266,7 @@ class ConversationResult: conversation_success: bool total_clarification_turns: int reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None def run_agentic_conversation( @@ -298,6 +299,7 @@ def run_agentic_conversation( # the end — a later turn may $ref a metric an earlier turn created. created_metric_ids: list[str] = [] reasoning_steps: list[str] = [] + response_id: str | None = None try: if initial_conversation_id is not None: @@ -335,6 +337,7 @@ def run_agentic_conversation( final_result = chat_result all_tool_calls.extend(chat_result.tool_call_events or []) reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id if _check_output_present(resolved_turn, chat_result): break @@ -399,6 +402,7 @@ def run_agentic_conversation( conversation_success=conversation_success, total_clarification_turns=total_clarification_turns, reasoning_steps=reasoning_steps, + response_id=response_id, ) @@ -407,6 +411,8 @@ class ConversationAssertionError(AssertionError): __tracebackhide__ = True reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_conversation( @@ -424,12 +430,13 @@ def evaluate_agentic_conversation( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> list[str]: +) -> tuple[list[str], str, str | None]: """Run conversation evaluation, log to Langfuse, and raise on failure. - Returns the conversation's reasoning_steps on success; on failure the same list is - attached to the raised exception as ``.reasoning_steps`` (mirrors the - `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve it + Returns the conversation's (reasoning_steps, conversation_id, response_id) on success; + on failure the same three values are attached to the raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. """ from datetime import datetime as _dt # noqa: PLC0415 @@ -514,5 +521,7 @@ def evaluate_agentic_conversation( f"Failed turns: {[t.turn_id for t in failed_turns]}" ) exc.reasoning_steps = result.reasoning_steps + exc.conversation_id = result.conversation_id + exc.response_id = result.response_id raise exc - return result.reasoning_steps + return result.reasoning_steps, result.conversation_id, result.response_id diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 2db890209..813ce8e38 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -130,6 +130,7 @@ class MetricRunResult: maql_correct: bool total_turns: float reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -207,12 +208,14 @@ def _execute_single_metric_run( turns = 0 current_question = question reasoning_steps: list[str] = [] + response_id: str | None = None try: for _iteration in range(max_iterations): turns += 1 chat_result = client.send_message(conversation_id, current_question) reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id candidate = _extract_metric_result(chat_result.tool_call_events or []) if candidate is not None: metric_result = candidate @@ -240,6 +243,7 @@ def _execute_single_metric_run( maql_correct=maql_correct, total_turns=float(turns), reasoning_steps=reasoning_steps, + response_id=response_id, ) finally: if metric_id_to_delete: @@ -311,6 +315,8 @@ class MetricSkillAssertionError(AssertionError): __tracebackhide__ = True reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_metric_skill( @@ -330,12 +336,14 @@ def evaluate_agentic_metric_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> list[str]: +) -> tuple[list[str], str, str | None]: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure. - Returns the best run's reasoning_steps on success; on failure the same list is attached - to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception - idiom in `ChatClient.ask()`) so callers can retrieve it either way. + Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on + failure the same three values are attached to the raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them + either way. """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -408,5 +416,7 @@ def evaluate_agentic_metric_skill( f"Actual MAQL: {best.actual_maql}." ) exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id raise exc - return summary.best.reasoning_steps + return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 140c62fd4..0fc6a385f 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -565,7 +565,7 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), ): - reasoning = evaluate_agentic_alert_skill( + reasoning, conversation_id, response_id = evaluate_agentic_alert_skill( host="http://host", token="tok", workspace_id="ws1", @@ -576,6 +576,8 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): ) assert reasoning == ["thinking about it"] + assert conversation_id == "conv-1" + assert response_id is None def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail(): @@ -604,3 +606,5 @@ def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_f max_iterations=1, ) assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index cc4ab0791..3815e867c 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -532,6 +532,7 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): chat_result.created_visualizations = [MagicMock()] chat_result.tool_call_events = [tc] chat_result.reasoning_steps = ["thinking about it"] + chat_result.response_id = "resp-1" mock_client.send_message.return_value = chat_result fixture = ConversationFixture( @@ -547,13 +548,15 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): ], ) with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client): - reasoning = evaluate_agentic_conversation( + reasoning, conversation_id, response_id = evaluate_agentic_conversation( host="http://host", token="tok", workspace_id="ws1", fixture=fixture, ) assert reasoning == ["thinking about it"] + assert conversation_id == "conv-1" + assert response_id == "resp-1" def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail(): @@ -568,6 +571,7 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_ chat_result.tool_call_events = [tc] chat_result.alert_proposals = [] chat_result.reasoning_steps = ["confused thinking"] + chat_result.response_id = "resp-2" mock_client.send_message.return_value = chat_result fixture = ConversationFixture( @@ -594,3 +598,5 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_ max_clarification_turns=0, ) assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index ccc24299c..dd855d8f2 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -381,7 +381,7 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): } ) with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): - reasoning = evaluate_agentic_metric_skill( + reasoning, conversation_id, response_id = evaluate_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", workspace_id="ws1", @@ -391,6 +391,8 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): max_iterations=1, ) assert reasoning == ["thinking about it"] + assert conversation_id == "conv-1" + assert response_id is None def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail(): @@ -417,3 +419,5 @@ def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_ max_iterations=1, ) assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index fda604542..925d323d7 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -108,7 +108,7 @@ def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem: def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): with patch( "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", - return_value=["it created the alert"], + return_value=(["it created the alert"], "conv-1", "resp-1"), ): report = run_agentic_items( [_item()], @@ -119,11 +119,15 @@ def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): ) assert report.items[0].pass_at_k is True assert report.items[0].reasoning_steps == ["it created the alert"] + assert report.items[0].conversation_id == "conv-1" + assert report.items[0].response_id == "resp-1" def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): exc = AlertSkillAssertionError("nope") exc.reasoning_steps = ["it got confused"] + exc.conversation_id = "conv-2" + exc.response_id = "resp-2" with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): report = run_agentic_items( [_item()], @@ -134,6 +138,8 @@ def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): ) assert report.items[0].pass_at_k is False assert report.items[0].reasoning_steps == ["it got confused"] + assert report.items[0].conversation_id == "conv-2" + assert report.items[0].response_id == "resp-2" def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_none(): @@ -149,6 +155,8 @@ def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_ run_ts="2026-01-01", ) assert report.items[0].reasoning_steps == [] + assert report.items[0].conversation_id is None + assert report.items[0].response_id is None def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds(): @@ -163,3 +171,5 @@ def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds ) assert report.items[0].pass_at_k is True assert report.items[0].reasoning_steps == [] + assert report.items[0].conversation_id is None + assert report.items[0].response_id is None From 8e7d622e15cbd1ce1aa849b3df2256eb19f0b54a Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 20 Aug 2026 13:46:47 +0200 Subject: [PATCH 5/5] refactor(gooddata-eval): replace tuple return with AgenticEvalOutcome dataclass evaluate_agentic_alert_skill/metric_skill/conversation returned a bare tuple[list[str], str, str | None], and _dispatch_agentic's return type was a tuple | list[str] | None union unpacked via isinstance(outcome, tuple). Introduced AgenticEvalOutcome (reasoning_steps, conversation_id, response_id) in core/models.py and used it as the return type instead. Co-Authored-By: Claude Sonnet 5 --- .../src/gooddata_eval/cli/agentic_runner.py | 15 +++++++++------ .../src/gooddata_eval/core/agentic/alert_skill.py | 15 ++++++++++----- .../gooddata_eval/core/agentic/conversation.py | 15 ++++++++++----- .../gooddata_eval/core/agentic/metric_skill.py | 15 ++++++++++----- .../src/gooddata_eval/core/models.py | 8 ++++++++ .../tests/test_agentic_alert_skill.py | 8 ++++---- .../tests/test_agentic_conversation.py | 8 ++++---- .../tests/test_agentic_metric_skill.py | 8 ++++---- .../gooddata-eval/tests/test_agentic_runner.py | 6 ++++-- 9 files changed, 63 insertions(+), 35 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 02c599326..24fbad1a9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -16,7 +16,7 @@ from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import CreatedVisualization, DatasetItem +from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport @@ -86,11 +86,11 @@ def _dispatch_agentic( model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, -) -> tuple[list[str], str, str | None] | list[str] | None: +) -> AgenticEvalOutcome | list[str] | None: """Call the appropriate evaluate_agentic_* function for the item's test_kind. Returns whatever that function returns -- alert_skill/metric_skill/conversation return - their (reasoning_steps, conversation_id, response_id); the rest still return None + an AgenticEvalOutcome; the rest still return None (unchanged). """ kind = item.test_kind @@ -236,9 +236,12 @@ def run_agentic_items( outcome = _dispatch_agentic( item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id ) - reasoning_steps, conversation_id, response_id = ( - outcome if isinstance(outcome, tuple) else (outcome, None, None) - ) + if isinstance(outcome, AgenticEvalOutcome): + reasoning_steps = outcome.reasoning_steps + conversation_id = outcome.conversation_id + response_id = outcome.response_id + else: + reasoning_steps, conversation_id, response_id = outcome, None, None item_report.pass_at_k = True item_report.runs = k item_report.reasoning_steps = reasoning_steps or [] diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 4eb4d4b5b..e21737371 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -14,7 +14,7 @@ from gooddata_eval.core.agentic._catalog import CatalogMetricAlert from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent try: from openai import OpenAI as _OpenAI @@ -572,11 +572,12 @@ def evaluate_agentic_alert_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> tuple[list[str], str, str | None]: +) -> AgenticEvalOutcome: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure. - Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on - failure the same three values are attached to the raised exception as + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. @@ -666,4 +667,8 @@ def evaluate_agentic_alert_skill( exc.conversation_id = best.conversation_id exc.response_id = best.response_id raise exc - return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id + return AgenticEvalOutcome( + reasoning_steps=summary.best.reasoning_steps, + conversation_id=summary.best.conversation_id, + response_id=summary.best.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index c0c00c362..95821bb43 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -15,7 +15,7 @@ from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ChatResult, ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ChatResult, ToolCallEvent from gooddata_eval.core.scoring import ( check_filters, check_viz_type, @@ -430,11 +430,12 @@ def evaluate_agentic_conversation( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> tuple[list[str], str, str | None]: +) -> AgenticEvalOutcome: """Run conversation evaluation, log to Langfuse, and raise on failure. - Returns the conversation's (reasoning_steps, conversation_id, response_id) on success; - on failure the same three values are attached to the raised exception as + Returns the conversation's outcome (reasoning_steps, conversation_id, response_id) as + an AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. @@ -524,4 +525,8 @@ def evaluate_agentic_conversation( exc.conversation_id = result.conversation_id exc.response_id = result.response_id raise exc - return result.reasoning_steps, result.conversation_id, result.response_id + return AgenticEvalOutcome( + reasoning_steps=result.reasoning_steps, + conversation_id=result.conversation_id, + response_id=result.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 813ce8e38..a699ebe63 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -12,7 +12,7 @@ from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent try: from openai import OpenAI as _OpenAI @@ -336,11 +336,12 @@ def evaluate_agentic_metric_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> tuple[list[str], str, str | None]: +) -> AgenticEvalOutcome: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure. - Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on - failure the same three values are attached to the raised exception as + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. @@ -419,4 +420,8 @@ def evaluate_agentic_metric_skill( exc.conversation_id = best.conversation_id exc.response_id = best.response_id raise exc - return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id + return AgenticEvalOutcome( + reasoning_steps=summary.best.reasoning_steps, + conversation_id=summary.best.conversation_id, + response_id=summary.best.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 40f7ede96..0c44cc114 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -107,6 +107,14 @@ class ChatResult(BaseModel): turn_wall_clock_sec: float | None = None +class AgenticEvalOutcome(BaseModel): + """Reasoning trace and trace-lookup IDs returned by an evaluate_agentic_* call on success.""" + + reasoning_steps: list[str] = Field(default_factory=list) + conversation_id: str | None = None + response_id: str | None = None + + class SummaryInput(BaseModel): """Structured input for the `dashboard_summary` test kind. diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 0fc6a385f..acc0b5669 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -565,7 +565,7 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), ): - reasoning, conversation_id, response_id = evaluate_agentic_alert_skill( + outcome = evaluate_agentic_alert_skill( host="http://host", token="tok", workspace_id="ws1", @@ -575,9 +575,9 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): max_iterations=1, ) - assert reasoning == ["thinking about it"] - assert conversation_id == "conv-1" - assert response_id is None + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id is None def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail(): diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index 3815e867c..a6368683b 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -548,15 +548,15 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): ], ) with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client): - reasoning, conversation_id, response_id = evaluate_agentic_conversation( + outcome = evaluate_agentic_conversation( host="http://host", token="tok", workspace_id="ws1", fixture=fixture, ) - assert reasoning == ["thinking about it"] - assert conversation_id == "conv-1" - assert response_id == "resp-1" + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail(): diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index dd855d8f2..684436ee1 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -381,7 +381,7 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): } ) with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): - reasoning, conversation_id, response_id = evaluate_agentic_metric_skill( + outcome = evaluate_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", workspace_id="ws1", @@ -390,9 +390,9 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): k=1, max_iterations=1, ) - assert reasoning == ["thinking about it"] - assert conversation_id == "conv-1" - assert response_id is None + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id is None def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail(): diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 925d323d7..627afebd9 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -5,7 +5,7 @@ import pytest from gooddata_eval.cli.agentic_runner import _dispatch_agentic, run_agentic_items from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError -from gooddata_eval.core.models import DatasetItem +from gooddata_eval.core.models import AgenticEvalOutcome, DatasetItem def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): @@ -108,7 +108,9 @@ def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem: def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): with patch( "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", - return_value=(["it created the alert"], "conv-1", "resp-1"), + return_value=AgenticEvalOutcome( + reasoning_steps=["it created the alert"], conversation_id="conv-1", response_id="resp-1" + ), ): report = run_agentic_items( [_item()],