From cf70e92bf14fe603f180336d4ba940eabaed881a Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 15 Jul 2026 11:05:24 -0600 Subject: [PATCH] update eval input/output/expected to conform to other SDKs --- lib/braintrust/eval/runner.rb | 8 ++++---- test/braintrust/eval/runner_test.rb | 14 +++++++------- test/braintrust/eval_test.rb | 5 +++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/braintrust/eval/runner.rb b/lib/braintrust/eval/runner.rb index 074415f..262c389 100644 --- a/lib/braintrust/eval/runner.rb +++ b/lib/braintrust/eval/runner.rb @@ -91,8 +91,8 @@ def run_eval_case(kase, errors) # Set attributes known before task execution eval_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval")) - set_json_attr(eval_span, "braintrust.input_json", {input: kase.input}) - set_json_attr(eval_span, "braintrust.expected", kase.expected) if kase.expected + set_json_attr(eval_span, "braintrust.input_json", kase.input) + set_json_attr(eval_span, "braintrust.expected_json", kase.expected) unless kase.expected.nil? set_json_attr(eval_span, "braintrust.metadata", kase.metadata) if kase.metadata eval_span.set_attribute("braintrust.tags", kase.tags) if kase.tags eval_span.set_attribute("braintrust.origin", kase.origin) if kase.origin @@ -103,7 +103,6 @@ def run_eval_case(kase, errors) rescue => e # Error already recorded on task span, set eval span status eval_span.status = OpenTelemetry::Trace::Status.error(e.message) - set_json_attr(eval_span, "braintrust.output_json", {output: nil}) errors << "Task failed for input '#{kase.input}': #{e.message}" report_progress(eval_span, kase, error: e.message) next @@ -134,7 +133,7 @@ def run_eval_case(kase, errors) end # Set output after task completes - set_json_attr(eval_span, "braintrust.output_json", {output: kase.output}) + set_json_attr(eval_span, "braintrust.output_json", kase.output) report_progress(eval_span, kase, data: kase.output) end @@ -151,6 +150,7 @@ def run_task(kase) task_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr set_json_attr(task_span, "braintrust.span_attributes", build_span_attributes("task")) set_json_attr(task_span, "braintrust.input_json", kase.input) + set_json_attr(task_span, "braintrust.expected_json", kase.expected) unless kase.expected.nil? begin output = eval_context.task.call( diff --git a/test/braintrust/eval/runner_test.rb b/test/braintrust/eval/runner_test.rb index 7b02123..d52f3ce 100644 --- a/test/braintrust/eval/runner_test.rb +++ b/test/braintrust/eval/runner_test.rb @@ -531,7 +531,7 @@ def test_runner_eval_span_has_case_metadata assert_equal "greeting", metadata["category"] end - def test_runner_eval_span_input_json_wrapped + def test_runner_eval_span_input_json_is_raw_value rig = setup_otel_test_rig context = Braintrust::Eval::Context.build( @@ -550,7 +550,7 @@ def test_runner_eval_span_input_json_wrapped eval_span = rig.exporter.finished_spans.find { |s| s.name == "eval" } input_json = JSON.parse(eval_span.attributes["braintrust.input_json"]) - assert_equal({"input" => "hello"}, input_json) + assert_equal "hello", input_json end def test_runner_eval_span_tags_as_array @@ -576,7 +576,7 @@ def test_runner_eval_span_tags_as_array assert_equal ["fast", "regression"], tags end - def test_runner_eval_span_output_json_null_on_task_error + def test_runner_eval_span_omits_output_json_on_task_error rig = setup_otel_test_rig context = Braintrust::Eval::Context.build( @@ -593,12 +593,11 @@ def test_runner_eval_span_output_json_null_on_task_error Braintrust::Eval::Runner.new(context).run eval_span = rig.exporter.finished_spans.find { |s| s.name == "eval" } - output_json = JSON.parse(eval_span.attributes["braintrust.output_json"]) - assert_equal({"output" => nil}, output_json) + assert_nil eval_span.attributes["braintrust.output_json"] end - def test_runner_eval_span_output_json_wrapped + def test_runner_eval_span_output_json_is_raw_value rig = setup_otel_test_rig context = Braintrust::Eval::Context.build( @@ -617,7 +616,7 @@ def test_runner_eval_span_output_json_wrapped eval_span = rig.exporter.finished_spans.find { |s| s.name == "eval" } output_json = JSON.parse(eval_span.attributes["braintrust.output_json"]) - assert_equal({"output" => "HELLO"}, output_json) + assert_equal "HELLO", output_json end def test_runner_eval_spans_are_independent_roots @@ -695,6 +694,7 @@ def test_runner_run_creates_task_spans assert_equal 1, task_spans.length assert_equal "experiment_id:exp-123", task_spans[0].attributes["braintrust.parent"] + assert_equal "HELLO", JSON.parse(task_spans[0].attributes["braintrust.expected_json"]) end def test_runner_run_creates_per_scorer_spans diff --git a/test/braintrust/eval_test.rb b/test/braintrust/eval_test.rb index 6a11d00..8a42314 100644 --- a/test/braintrust/eval_test.rb +++ b/test/braintrust/eval_test.rb @@ -358,13 +358,14 @@ def test_eval_run_with_tracing assert eval_span.attributes["braintrust.parent"] assert_match(/experiment_id:[0-9a-f-]{36}/, eval_span.attributes["braintrust.parent"]) input_json = JSON.parse(eval_span.attributes["braintrust.input_json"]) - assert_equal({"input" => "hello"}, input_json) + assert_equal "hello", input_json output_json = JSON.parse(eval_span.attributes["braintrust.output_json"]) - assert_equal({"output" => "HELLO"}, output_json) + assert_equal "HELLO", output_json # Verify task span assert task_span.attributes["braintrust.span_attributes"] assert_includes task_span.attributes["braintrust.span_attributes"], "task" + assert_equal "HELLO", JSON.parse(task_span.attributes["braintrust.expected_json"]) # Verify score span has scorer-specific attributes assert score_span.attributes["braintrust.scores"]