feat(eval): filter reported scores and metrics

.github/workflows/eval-py-uv.yml

@@ -1,9 +1,10 @@
 name: Run Python evals
 
 on:
+  pull_request:
   push:
-    # files:
-    #   - 'test-eval/**'
+    branches:
+      - main
 
 permissions:
   pull-requests: write

.github/workflows/eval-py.yml

@@ -1,9 +1,10 @@
 name: Run Python evals
 
 on:
+  pull_request:
   push:
-    # files:
-    #   - 'test-eval/**'
+    branches:
+      - main
 
 permissions:
   pull-requests: write

.github/workflows/eval-single.yaml

@@ -1,9 +1,10 @@
 name: Run evals
 
 on:
+  pull_request:
   push:
-    # files:
-    #   - 'test-eval/**'
+    branches:
+      - main
 
 permissions:
   pull-requests: write

.github/workflows/eval.yml

@@ -1,9 +1,10 @@
 name: Run evals
 
 on:
+  pull_request:
   push:
-    # files:
-    #   - 'test-eval/**'
+    branches:
+      - main
 
 permissions:
   pull-requests: write

README.md

@@ -50,6 +50,8 @@ jobs:
 | `package_manager`      | No       | `npm` or `pnpm` for Node; `pip` or `uv` for Python; `go` for Go. Can be omitted for the default package manager.                                            |
 | `use_proxy`            | No       | Set to `true` to use the Braintrust proxy at `https://braintrustproxy.com/v1`, which can cache repetitive LLM calls and speed up evals. Defaults to `true`. |
 | `terminate_on_failure` | No       | Set to `true` to stop the eval process when an error occurs. Defaults to `false`. Ignored for Go evals.                                                     |
+| `report_scores`        | No       | Comma- or newline-separated score names to include in the PR comment. Defaults to all available scores.                                                     |
+| `report_metrics`       | No       | Comma- or newline-separated metric names to include in the PR comment. Defaults to all available metrics.                                                  |
 | `github_token`         | No       | GitHub token used to create or update PR comments. Defaults to `${{ github.token }}`.                                                                       |
 
 ## Full example
@@ -129,14 +131,32 @@ For more fully configured workflows, see the `examples` directory:
   fmt.Println(string(b))
   ```
 
-The action creates or updates a single PR comment with a Braintrust link and
-result table. For example:
+The action creates or updates a single PR comment with a Braintrust link and a
+result table with score and metric sections. To show only selected results, set
+`report_scores` and `report_metrics` to their exact names:
+
+```yaml
+- uses: braintrustdata/eval-action@v2
+  with:
+    api_key: ${{ secrets.BRAINTRUST_API_KEY }}
+    runtime: node
+    report_scores: Levenshtein, Factuality
+    report_metrics: |
+      Duration
+      Cost
+```
+
+Each input accepts comma- or newline-separated names and filters its category
+independently. When an input is omitted or empty, all results in that category
+are included. For example:
 
 ### Example Braintrust eval report
 
 **[Say Hi Bot (HEAD-1714341466)](https://www.braintrustdata.com/app/braintrustdata.com/p/Say%20Hi%20Bot/experiments/HEAD-1714341466)**
 
-| Score       | Average    | Improvements | Regressions |
-| ----------- | ---------- | -----------: | ----------: |
-| Levenshtein | 83% (+3pp) |         8 🟢 |        4 🔴 |
-| Duration    | 1s (0s)    |        16 🟢 |        1 🔴 |
+| Name            | Average    | Improvements | Regressions |
+| --------------- | ---------- | -----------: | ----------: |
+| **Scores**      |            |              |             |
+| Levenshtein     | 83% (+3pp) |         8 🟢 |        4 🔴 |
+| **Metrics**     |            |              |             |
+| Duration        | 1s (0s)    |        16 🟢 |        1 🔴 |

action.yml

@@ -41,6 +41,18 @@ inputs:
       'true' or 'false'."
     required: false
     default: "false"
+  report_scores:
+    description:
+      "A comma- or newline-separated list of score names to include in the PR
+      comment. By default, all scores are included."
+    required: false
+    default: ""
+  report_metrics:
+    description:
+      "A comma- or newline-separated list of metric names to include in the PR
+      comment. By default, all metrics are included."
+    required: false
+    default: ""
   github_token:
     description: "Your GitHub token"
     required: true

eval/src/main.test.ts

@@ -0,0 +1,104 @@
+import type { ExperimentSummary } from "braintrust";
+import { describe, expect, it } from "vite-plus/test";
+
+import { formatSummary, parseReportNames } from "./main";
+
+const summary: ExperimentSummary = {
+  projectName: "Document processing",
+  experimentName: "pull-request-123",
+  experimentUrl: "https://example.com/experiment",
+  scores: {
+    Accuracy: {
+      name: "Accuracy",
+      score: 0.9,
+      diff: 0.05,
+      improvements: 3,
+      regressions: 1,
+    },
+    Completeness: {
+      name: "Completeness",
+      score: 0.8,
+      improvements: 0,
+      regressions: 2,
+    },
+  },
+  metrics: {
+    Duration: {
+      name: "Duration",
+      metric: 1.25,
+      unit: "s",
+      diff: -0.1,
+      improvements: 2,
+      regressions: 0,
+    },
+    Cost: {
+      name: "Cost",
+      metric: 0.02,
+      unit: "$",
+      improvements: 0,
+      regressions: 0,
+    },
+  },
+};
+
+describe("parseReportNames", () => {
+  it("parses comma- and newline-separated names", () => {
+    expect(
+      parseReportNames(" Accuracy, Duration\nCompleteness,Accuracy "),
+    ).toEqual(["Accuracy", "Duration", "Completeness"]);
+  });
+
+  it("uses an empty list to report every result in a category", () => {
+    expect(parseReportNames("  \n , ")).toEqual([]);
+  });
+});
+
+describe("formatSummary", () => {
+  it("reports scores and metrics as sections in one table by default", () => {
+    const result = formatSummary(summary);
+
+    expect(
+      result.match(/Name \| Average \| Improvements \| Regressions/g),
+    ).toHaveLength(1);
+    expect(result).toContain("**Scores** | | |");
+    expect(result).toContain("Accuracy | 90% (+5pp)");
+    expect(result).toContain("Completeness | 80%");
+    expect(result).toContain("**Metrics** | | |");
+    expect(result).toContain("Duration | 1.25s (-0.1s)");
+    expect(result).toContain("Cost | 0.02$");
+    expect(result.indexOf("**Metrics**")).toBeGreaterThan(
+      result.indexOf("Completeness | 80%"),
+    );
+  });
+
+  it("reports only selected scores and metrics", () => {
+    const result = formatSummary(summary, {
+      scores: ["Accuracy"],
+      metrics: ["Duration"],
+    });
+
+    expect(result).toContain("Accuracy | 90% (+5pp)");
+    expect(result).toContain("Duration | 1.25s (-0.1s)");
+    expect(result).not.toContain("Completeness");
+    expect(result).not.toContain("Cost");
+  });
+
+  it("filters scores and metrics independently", () => {
+    const result = formatSummary(summary, { scores: ["Accuracy"] });
+
+    expect(result).not.toContain("Completeness");
+    expect(result).toContain("Duration");
+    expect(result).toContain("Cost");
+  });
+
+  it("keeps the experiment link when no names match", () => {
+    expect(
+      formatSummary(summary, {
+        scores: ["Unknown score"],
+        metrics: ["Unknown metric"],
+      }),
+    ).toBe(
+      "**[Document processing (pull-request-123)](https://example.com/experiment)**",
+    );
+  });
+});