Skip to content

Commit 18d3def

Browse files
committed
fix(gooddata-eval): fix create/execute pairing, score names, dataset default
- _extract_kda_calls picked the last create and last execute call independently, not as a pair. create_1 -> execute_1(success) -> create_2 (never executed) would wrongly report create_2's args as executed/succeeded using execute_1's stale result. Fixed by clearing execute_result whenever a new create call is seen. - Score names executed/success/turn_completed now carry the kda_ prefix (kda_executed/kda_success/kda_turn_completed), matching kda_triggered's existing convention -- gdc-nas's combo_report.py verdict() must read the same names, updated in the same change. - dataset_name default corrected from "kda_skill" to "agent_kda_skill", matching the actual Langfuse dataset name used everywhere else. JIRA: QA-28800 risk: nonprod
1 parent 8557c38 commit 18d3def

2 files changed

Lines changed: 30 additions & 9 deletions

File tree

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -118,17 +118,20 @@ def generate_simulated_kda_response(agent_message: str, measure_candidates: dict
118118

119119

120120
def _extract_kda_calls(tool_call_events: list[ToolCallEvent]) -> tuple[dict | None, dict | None]:
121-
"""Return (create_args, execute_result): the arguments of the LAST
122-
`create_key_driver_analysis` call and the parsed result of the LAST
123-
`execute_key_driver_analysis` call. Taking the last (not first) attempt matches
124-
the observed retry-loop behaviour (kda_1 fails, kda_2 retries) -- the last
125-
attempt is what actually determined the answer the chatbot gave.
121+
"""Return (create_args, execute_result) for the LAST create/execute *pair* -- not the
122+
last create and last execute picked independently. Taking the last pair (not the
123+
first) matches the observed retry-loop behaviour (kda_1 fails, kda_2 retries): the
124+
last attempt is what actually determined the answer the chatbot gave. A new create
125+
call clears any earlier execute_result, since that result belongs to the create it
126+
followed, not to this one -- without that reset, `create_1 -> execute_1(success) ->
127+
create_2 (never executed)` would wrongly pair create_2's args with execute_1's result.
126128
"""
127129
create_args: dict | None = None
128130
execute_result: dict | None = None
129131
for tc in tool_call_events:
130132
if tc.function_name == "create_key_driver_analysis":
131133
create_args = tc.parsed_arguments()
134+
execute_result = None
132135
elif tc.function_name == "execute_key_driver_analysis" and tc.result:
133136
execute_result = tc.parsed_result()
134137
return create_args, execute_result
@@ -430,7 +433,7 @@ def evaluate_agentic_kda_skill(
430433
initial_conversation_id: str | None = None,
431434
langfuse: object | None = None,
432435
dataset_item_id: str = "",
433-
dataset_name: str = "kda_skill",
436+
dataset_name: str = "agent_kda_skill",
434437
run_timestamp: str | None = None,
435438
model_version_override: str | None = None,
436439
run_metadata_extra: dict | None = None,
@@ -502,9 +505,9 @@ def _select_and_record(found: list[Any]) -> Any | None:
502505
# Gates strict_pass -- current scope is completion only (see KdaEvaluation docstring).
503506
strict_checks = {
504507
"kda_triggered": ev.kda_triggered,
505-
"executed": ev.executed,
506-
"success": ev.success,
507-
"turn_completed": ev.turn_completed,
508+
"kda_executed": ev.executed,
509+
"kda_success": ev.success,
510+
"kda_turn_completed": ev.turn_completed,
508511
}
509512
# Informational only -- logged for visibility / a future correctness ticket,
510513
# NOT part of strict_checks/strict_pass. See KdaEvaluation docstring.

packages/gooddata-eval/tests/test_agentic_kda_skill.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,24 @@ def test_extract_kda_calls_takes_last_execute_on_retry():
154154
assert execute_result == {"success": True, "data": {"summary": {}}}
155155

156156

157+
def test_extract_kda_calls_does_not_pair_a_new_create_with_an_earlier_execute():
158+
# create_1 -> execute_1(success) -> create_2 (never executed): create_2's args must
159+
# not get paired with execute_1's stale result -- that would wrongly report the run
160+
# as executed/succeeded when the actual last attempt never ran.
161+
events = ChatResult.model_validate(
162+
{
163+
"toolCallEvents": [
164+
_tool_call("create_key_driver_analysis", arguments={"measure": {"type": "metric", "id": "a"}}),
165+
_tool_call("execute_key_driver_analysis", result={"success": True, "data": {"summary": {}}}),
166+
_tool_call("create_key_driver_analysis", arguments={"measure": {"type": "metric", "id": "b"}}),
167+
]
168+
}
169+
).tool_call_events
170+
create_args, execute_result = _extract_kda_calls(events)
171+
assert create_args == {"measure": {"type": "metric", "id": "b"}}
172+
assert execute_result is None
173+
174+
157175
def test_extract_kda_calls_none_when_no_tool_calls():
158176
create_args, execute_result = _extract_kda_calls([])
159177
assert create_args is None

0 commit comments

Comments
 (0)