fix(gooddata-eval): a rejected draft attempt is not a refusal

`_extract_tool_result` takes only successful calls, which is right for the
normal path -- a failed call there is a retry the agent recovered from. But
a refusal case was scored on `tool_result is None` and `dashboard_part is
None`, and an agent whose draft the tool REJECTED leaves both empty. So a
run that tried to build and was stopped by a guardrail read as a clean
refusal, which credits the model for the guardrail.

`dashboard_not_drafted` now also requires that the producing tool was never
called at all. The flag is computed in the run loop over every tool-call
event, not from the extracted result, so it survives the filtering that
hides the failed call. Nothing outside the refusal shape reads it.

Found in review by CodeRabbit.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py

@@ -769,6 +769,10 @@ class DashboardEvaluation:
     titles_matched: bool = True
     forbidden_absent: bool = True
     answer_matched: bool = True
+    # Whether the producing tool was called AT ALL, successful or not. Only a refusal case
+    # reads it: everywhere else a failed call is a retry the agent recovered from, which is
+    # why `_extract_tool_result` skips it.
+    producing_tool_called: bool = False
     failures: list[str] = field(default_factory=list)
     notes: list[str] = field(default_factory=list)
 
@@ -781,10 +785,17 @@ def strict_checks(self) -> dict[str, bool]:
         if self.applies.refusal:
             # Named for what has to be true, not negated at read time: `dashboard_not_drafted`
             # false means a dashboard came back from a case that asked for none, which is the
-            # failure. Both halves are required -- a successful tool call with no part, or a
-            # part with no successful call, are each a dashboard the user would be shown.
+            # failure. A successful tool call with no part, or a part with no successful call,
+            # are each a dashboard the user would be shown.
+            #
+            # The third term is the one that is easy to miss: an agent that CALLED the tool and
+            # had the call rejected leaves no result and no part, so the first two would read it
+            # as a clean refusal. It is not one -- the agent tried to build and the tool stopped
+            # it, and scoring that as correct credits the model for a guardrail.
             checks = {
-                "dashboard_not_drafted": not self.drafted and not self.part_present,
+                "dashboard_not_drafted": (
+                    not self.drafted and not self.part_present and not self.producing_tool_called
+                ),
                 "dashboard_skill_activated": self.skill_activated,
             }
             if self.applies.answer:
@@ -870,6 +881,7 @@ def evaluate_dashboard_response(
     skill_activated: bool,
     patch_part: dict | None = None,
     answer_text: str = "",
+    producing_tool_called: bool = False,
 ) -> DashboardEvaluation:
     """Score one dashboard response against its expectation.
 
@@ -909,6 +921,7 @@ def evaluate_dashboard_response(
             new_visualizations_met=True,
             applies=applies,
             skill_activated=skill_activated,
+            producing_tool_called=producing_tool_called,
             answer_matched=not answer_failures,
             failures=[
                 *(
@@ -921,6 +934,11 @@ def evaluate_dashboard_response(
                     if dashboard_part is not None
                     else []
                 ),
+                *(
+                    [f"the case expects no dashboard, but the agent called {tool} (the call did not succeed)"]
+                    if producing_tool_called and not drafted
+                    else []
+                ),
                 *answer_failures,
             ],
         )
@@ -1198,6 +1216,8 @@ def _execute_single_dashboard_run(
             _skill_activated(all_tool_call_events, _required_skill(expected_output)),
             patch_part=patch_part,
             answer_text=answer_text,
+            # Any call, not just a successful one -- see DashboardEvaluation.
+            producing_tool_called=any(tc.function_name == tool for tc in all_tool_call_events),
         ),
         tool_result=tool_result,
         dashboard_part=dashboard_part,

packages/gooddata-eval/tests/test_agentic_dashboard_skill.py

@@ -1485,6 +1485,46 @@ def test_falling_over_silently_is_not_a_refusal(self):
         assert not ev.answer_matched
         assert not ev.strict_pass
 
+    def test_a_rejected_draft_attempt_is_not_a_refusal(self):
+        """The gap the first two checks leave: `_extract_tool_result` takes only SUCCESSFUL
+        calls, so an agent whose draft the tool rejected leaves no result and no part and
+        would read as a clean refusal. It is not one -- it tried to build and a guardrail
+        stopped it, and scoring that as correct credits the model for the guardrail."""
+        ev = evaluate_dashboard_response(
+            None,
+            None,
+            self._NO_DATA,
+            skill_activated=True,
+            answer_text="There is no fraud data here.",
+            producing_tool_called=True,
+        )
+        assert not ev.strict_checks["dashboard_not_drafted"]
+        assert not ev.strict_pass
+        assert any("did not succeed" in f for f in ev.failures)
+
+    def test_a_failed_draft_attempt_is_read_off_the_turn_not_the_result(self):
+        """End to end, because the flag is computed in the run loop rather than passed in by
+        a fixture: the tool call is present on the turn, its result is an error, so nothing
+        reaches the scorer except the fact that the call happened."""
+        client = MagicMock()
+        client.send_message.return_value = _chat_result(
+            tool_calls=[_tool_call("draft_dashboard", {"status": "error", "message": "no such metric"})],
+            text="There is no fraud data in this workspace, so I cannot build that.",
+        )
+        summary = _run_with(client, self._NO_DATA)
+        assert summary.best.evaluation.producing_tool_called
+        assert not summary.best.evaluation.strict_pass
+
+    def test_an_answer_alone_still_passes_when_nothing_was_called(self):
+        """The control for the two above: the flag must not fail a genuine refusal."""
+        client = MagicMock()
+        client.send_message.return_value = _chat_result(
+            text="There is no fraud data in this workspace, so I cannot build that."
+        )
+        summary = _run_with(client, self._NO_DATA)
+        assert not summary.best.evaluation.producing_tool_called
+        assert summary.best.evaluation.strict_pass
+
     def test_a_refusal_case_without_an_answer_assertion_is_rejected_up_front(self):
         client = MagicMock()
         expected = {"type": "dashboard", "expects_dashboard": False, "visualizations": []}