feat(gooddata-eval): capture reasoning_steps through the agentic-CLI path

6001d2f wired ChatResult.reasoning_steps through runner.py's generic
single-turn path only. The agentic-CLI path (cli/agentic_runner.py ->
evaluate_agentic_*) builds its own ItemReport and never touched it, so
agentic_alert_skill/agentic_metric_skill/agentic_conversation items could
never produce a reasoning trace, no matter what the platform emitted.

Accumulates reasoning_steps across every send_message call in each of the
three evaluators' run loops, attaches it to the run/turn result, and
surfaces it from evaluate_agentic_* either as the return value (pass) or
as an attribute on the raised exception (fail) -- mirroring the existing
conversation_id-on-exception idiom in ChatClient.ask(). run_agentic_items
picks it up from either path onto ItemReport.reasoning_steps, which
json_report.py already serializes unconditionally.

general_question/guardrail/search_tool/visualization are left untouched --
their evaluate_agentic_* functions still return None, unchanged.

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -86,8 +86,12 @@ def _dispatch_agentic(
     model_version_override: str | None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
-) -> None:
-    """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
+) -> list[str] | None:
+    """Call the appropriate evaluate_agentic_* function for the item's test_kind.
+
+    Returns whatever that function returns -- only alert_skill/metric_skill/conversation
+    currently return their reasoning_steps; the rest still return None (unchanged).
+    """
     kind = item.test_kind
     eo = item.expected_output
     lf_kw: _LfKw = {
@@ -100,7 +104,7 @@ def _dispatch_agentic(
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
-        evaluate_agentic_visualization(
+        return evaluate_agentic_visualization(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -111,7 +115,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
-        evaluate_agentic_metric_skill(
+        return evaluate_agentic_metric_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -122,7 +126,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
-        evaluate_agentic_alert_skill(
+        return evaluate_agentic_alert_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -136,7 +140,7 @@ def _dispatch_agentic(
         eo_dict = eo if isinstance(eo, dict) else {}
         tool_call = eo_dict.get("tool_call", {})
         expected_args = tool_call.get("function_arguments", eo_dict)
-        evaluate_agentic_search_tool(
+        return evaluate_agentic_search_tool(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -147,7 +151,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_general_question":
-        evaluate_agentic_general_question(
+        return evaluate_agentic_general_question(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -158,7 +162,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_guardrail":
-        evaluate_agentic_guardrail(
+        return evaluate_agentic_guardrail(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -180,7 +184,7 @@ def _dispatch_agentic(
         )
     elif kind == "agentic_conversation":
         fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {}
-        evaluate_agentic_conversation(
+        return evaluate_agentic_conversation(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -228,14 +232,16 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(
+            reasoning_steps = _dispatch_agentic(
                 item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id
             )
             item_report.pass_at_k = True
             item_report.runs = k
+            item_report.reasoning_steps = reasoning_steps or []
         except AssertionError as exc:
             item_report.pass_at_k = False
             item_report.runs = k
+            item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -6,7 +6,7 @@
 import json
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from gooddata_sdk import GoodDataSdk
@@ -302,6 +302,7 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    reasoning_steps: list[str] = field(default_factory=list)
 
 
 @dataclass
@@ -450,13 +451,15 @@ def _run_once(conv_id: str) -> AlertRunResult:
             alert_id: str | None = None
             actual_args: dict = {}
             tool_called = False
+            reasoning_steps: list[str] = []
             # conversation_history stores prior turns for GPT-4o context.
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
             current_question = question
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                reasoning_steps.extend(chat_result.reasoning_steps or [])
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -492,6 +495,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                reasoning_steps=reasoning_steps,
             )
         finally:
             if alert_id_to_delete:
@@ -542,6 +546,7 @@ class AlertSkillAssertionError(AssertionError):
     """Raised when an alert-skill evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
 
 
 def evaluate_agentic_alert_skill(
@@ -561,8 +566,13 @@ def evaluate_agentic_alert_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
+) -> list[str]:
+    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
+
+    Returns the best run's reasoning_steps on success; on failure the same list is attached
+    to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception
+    idiom in `ChatClient.ask()`) so callers can retrieve it either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -636,11 +646,14 @@ def evaluate_agentic_alert_skill(
     if not summary.pass_at_k:
         best = summary.best
         ev = best.eval
-        raise AlertSkillAssertionError(
+        exc = AlertSkillAssertionError(
             f"Alert skill assertion failed. strict_pass={ev.strict_pass}. "
             f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, "
             f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, "
             f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, "
             f"recipients_correct={ev.recipients_correct}. "
             f"Actual args: {best.actual_alert_arguments}"
         )
+        exc.reasoning_steps = best.reasoning_steps
+        raise exc
+    return summary.best.reasoning_steps

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -5,7 +5,7 @@
 
 import json
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Literal
 
 from gooddata_sdk import GoodDataSdk
@@ -265,6 +265,7 @@ class ConversationResult:
     full_skill_coverage: bool
     conversation_success: bool
     total_clarification_turns: int
+    reasoning_steps: list[str] = field(default_factory=list)
 
 
 def run_agentic_conversation(
@@ -296,6 +297,7 @@ def run_agentic_conversation(
     # not persist in the (shared) workspace and get reused by a later test. Deferred to
     # the end — a later turn may $ref a metric an earlier turn created.
     created_metric_ids: list[str] = []
+    reasoning_steps: list[str] = []
 
     try:
         if initial_conversation_id is not None:
@@ -332,6 +334,7 @@ def run_agentic_conversation(
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
                 all_tool_calls.extend(chat_result.tool_call_events or [])
+                reasoning_steps.extend(chat_result.reasoning_steps or [])
 
                 if _check_output_present(resolved_turn, chat_result):
                     break
@@ -395,13 +398,15 @@ def run_agentic_conversation(
         full_skill_coverage=full_skill_coverage,
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
+        reasoning_steps=reasoning_steps,
     )
 
 
 class ConversationAssertionError(AssertionError):
     """Raised when a conversation evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
 
 
 def evaluate_agentic_conversation(
@@ -419,8 +424,14 @@ def evaluate_agentic_conversation(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run conversation evaluation, log to Langfuse, and raise on failure."""
+) -> list[str]:
+    """Run conversation evaluation, log to Langfuse, and raise on failure.
+
+    Returns the conversation's reasoning_steps on success; on failure the same list is
+    attached to the raised exception as ``.reasoning_steps`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve it
+    either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -497,8 +508,11 @@ def evaluate_agentic_conversation(
 
     if not result.conversation_success:
         failed_turns = [tr for tr in result.turn_results if not tr.skill_success]
-        raise ConversationAssertionError(
+        exc = ConversationAssertionError(
             f"Conversation assertion failed. "
             f"full_skill_coverage={result.full_skill_coverage}. "
             f"Failed turns: {[t.turn_id for t in failed_turns]}"
         )
+        exc.reasoning_steps = result.reasoning_steps
+        raise exc
+    return result.reasoning_steps

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -5,7 +5,7 @@
 
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from gooddata_sdk import GoodDataSdk
@@ -129,6 +129,7 @@ class MetricRunResult:
     actual_maql: str
     maql_correct: bool
     total_turns: float
+    reasoning_steps: list[str] = field(default_factory=list)
 
 
 @dataclass
@@ -205,11 +206,13 @@ def _execute_single_metric_run(
     metric_id_to_delete: str | None = None
     turns = 0
     current_question = question
+    reasoning_steps: list[str] = []
 
     try:
         for _iteration in range(max_iterations):
             turns += 1
             chat_result = client.send_message(conversation_id, current_question)
+            reasoning_steps.extend(chat_result.reasoning_steps or [])
             candidate = _extract_metric_result(chat_result.tool_call_events or [])
             if candidate is not None:
                 metric_result = candidate
@@ -236,6 +239,7 @@ def _execute_single_metric_run(
             actual_maql=actual_maql,
             maql_correct=maql_correct,
             total_turns=float(turns),
+            reasoning_steps=reasoning_steps,
         )
     finally:
         if metric_id_to_delete:
@@ -306,6 +310,7 @@ class MetricSkillAssertionError(AssertionError):
     """Raised when a metric-skill evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
 
 
 def evaluate_agentic_metric_skill(
@@ -325,8 +330,13 @@ def evaluate_agentic_metric_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
+) -> list[str]:
+    """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.
+
+    Returns the best run's reasoning_steps on success; on failure the same list is attached
+    to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception
+    idiom in `ChatClient.ask()`) so callers can retrieve it either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -391,9 +401,12 @@ def evaluate_agentic_metric_skill(
         best = summary.best
         expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output]
         candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list)
-        raise MetricSkillAssertionError(
+        exc = MetricSkillAssertionError(
             f"Metric skill assertion failed. "
             f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. "
             f"Expected MAQL (candidates): {candidates_str}. "
             f"Actual MAQL: {best.actual_maql}."
         )
+        exc.reasoning_steps = best.reasoning_steps
+        raise exc
+    return summary.best.reasoning_steps