Evals harness

.github/workflows/skill-evals.yml

@@ -0,0 +1,92 @@
+name: skill-evals
+
+# Runs the flyte-agent-plugin skills eval harness. On PRs it runs only the
+# scenarios affected by the changed files (see evals/select.py); nightly it runs
+# the full matrix including the `real` tier.
+on:
+  pull_request:
+    paths:
+      - "plugins/**"
+      - "evals/**"
+  schedule:
+    - cron: "0 7 * * *"   # nightly full matrix (incl. real tier)
+  workflow_dispatch: {}
+
+env:
+  PYTHONPATH: ${{ github.workspace }}
+
+jobs:
+  # 1) Decide what to run from the diff.
+  select:
+    runs-on: ubuntu-latest
+    outputs:
+      skills: ${{ steps.sel.outputs.skills }}
+      run_all: ${{ steps.sel.outputs.run_all }}
+      run_kind: ${{ steps.sel.outputs.run_kind }}
+      run_real: ${{ steps.sel.outputs.run_real }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install pyyaml requests
+      - id: sel
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            OUT=$(python -m evals.select --base "origin/${{ github.base_ref }}")
+          else
+            # nightly / manual: run everything
+            OUT=$(python -m evals.select --changed evals/manifest.yaml)
+          fi
+          echo "$OUT"
+          python - "$OUT" <<'PY' >> "$GITHUB_OUTPUT"
+          import json, sys
+          d = json.loads(sys.argv[1])
+          print("skills=" + json.dumps(d["skills"]))
+          print("run_all=" + str(d["run_all"]).lower())
+          print("run_kind=" + str(d["run_kind"]).lower())
+          print("run_real=" + str(d["run_real"]).lower())
+          PY
+
+  # 2) Static + trajectory tiers, orchestrated on demo.hosted via Flyte.
+  flyte-evals:
+    needs: select
+    if: needs.select.outputs.skills != '[]'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install flyte>=2.5.0 pyyaml requests
+      - name: Configure Flyte auth
+        run: echo "auth via UNION_API_KEY secret"
+        env:
+          UNION_API_KEY: ${{ secrets.UNION_API_KEY }}
+      - name: Run evals on demo.hosted
+        env:
+          UNION_API_KEY: ${{ secrets.UNION_API_KEY }}
+          GLM_API_KEY: ${{ secrets.GLM_API_KEY }}
+          TIERS: ${{ github.event_name == 'schedule' && '["static","trajectory","real"]' || '["static","trajectory"]' }}
+        run: |
+          flyte --config evals/config/flyte.yaml run --copy-style loaded_modules \
+            evals/workflows/eval_wf.py main \
+            --skills '${{ needs.select.outputs.skills }}' \
+            --tiers "$TIERS"
+      # The workflow attaches the HTML scorecard to the run report; the run exits
+      # non-zero if any scenario fails (enforced inside aggregate/report).
+
+  # 3) Real kind-in-Docker smoke, only when a kind skill changed (privileged).
+  kind-smoke:
+    needs: select
+    if: needs.select.outputs.run_kind == 'true'
+    runs-on: ubuntu-latest   # GitHub-hosted runners allow privileged Docker/kind
+    steps:
+      - uses: actions/checkout@v4
+      - uses: helm/kind-action@v1
+        with:
+          install_only: true
+      - name: kind smoke
+        run: bash evals/kind_smoke/run.sh

README.md

@@ -233,8 +233,7 @@ scripts/smoke_test_mcp.py                   # end-to-end check of the local MCP
 
 Each harness consumes a different part of this. Claude Code and Codex read the plugin
 manifests, so the **plugin name** matters to them. Hermes, opencode, and pi install skills
-by **directory path**, so `plugins/flyte/skills/…` is their interface — which is why the
-rename from `flyte-skills` touched both.
+by **directory path**, so `plugins/flyte/skills/…` is their interface.
 
 The `.mcp.json` server is Claude Code-specific; the skills themselves stay portable across
 harnesses.

evals/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.pyc
+.pytest_cache/

evals/README.md

@@ -0,0 +1,84 @@
+# flyte agent plugin eval harness
+
+Automated testing & evaluation for the `flyte` agent skills. It runs a real
+agent harness (**opencode / pi / hermes**) against the union-hosted **GLM** endpoint,
+hands it a skill + a task, and scores what it produces — orchestrated as **Flyte
+workflows on `demo.hosted.unionai.cloud`**, with path-based selection so only the
+scenarios for changed skills run per PR.
+
+## Concepts
+
+- **Scenario** (`scenarios/<skill>/*.yaml`) — one declarative eval: a prompt, the
+  deterministic `checks`, an LLM-judge `rubric`, and (tier `real`) a `real_run`.
+- **Tiers** — `static` (lint the SKILL.md; no LLM), `trajectory` (run the agent with
+  side-effecting commands stubbed, judge the artifacts it produces), `real` (actually
+  `flyte run` SDK output on demo.hosted; kind stood up for real on a CI runner).
+- **Control arm** — every trajectory/real scenario runs twice: **treatment** (skill
+  installed) and **control** (skill absent). The headline metric is
+  **lift = treatment − control**, isolating the skill's contribution. A negative lift
+  is a regression signal.
+
+## Run locally
+
+```bash
+pip install pyyaml requests            # + the harness CLI(s) you want to exercise
+export PYTHONPATH=$(git rev-parse --show-toplevel)
+
+# Static lint of every skill — no LLM, no agent:
+python -m evals.harness.run --tier static
+
+# One trajectory scenario end-to-end (needs GLM_API_KEY + the harness CLI):
+export GLM_API_KEY=...     # token for the demo.hosted GLM app
+python -m evals.harness.run --scenario sdk-author-map-task --harness opencode
+
+# Everything for one skill, JSON out + scorecard:
+python -m evals.harness.run --skill flyte-sdk-author --json out.json
+python -m evals.report out.json --html scorecard.html
+```
+
+`GLM_BASE_URL` / `GLM_MODEL` / `GLM_API_KEY` configure the endpoint (see
+`harness/glm.py`). The `real` tier only submits remote runs when
+`FLYTE_EVALS_ENABLE_REAL=1` is set.
+
+## Run on demo.hosted (Flyte)
+
+```bash
+flyte --config evals/config/flyte.yaml run evals/workflows/eval_wf.py main \
+    --skills '["flyte-sdk-author"]' --tiers '["static","trajectory"]'
+```
+
+Fans out one action per (scenario × harness); the `aggregate` task attaches an HTML
+scorecard to the run's report tab. GLM creds come from the `glm-api-key` Flyte secret.
+
+## Selective execution
+
+```bash
+python -m evals.select --base origin/main        # changed files -> scenario subset
+```
+
+Emits `{run_all, skills, scenario_ids, run_kind, run_real}`. A change under a skill
+dir selects that skill's scenarios; a change to the engine (`harness/**`,
+`workflows/**`, `manifest.yaml`, …) forces the whole suite. CI wiring is in
+`.github/workflows/skill-evals.yml`.
+
+## Layout
+
+```
+manifest.yaml            cross-cutting config + shared-infra globs + skill classes
+scenarios/<skill>/*.yaml declarative eval specs
+harness/                 spec, checks, static_lint, sandbox, runners/, judge, evaluate, run
+workflows/               eval_wf.py (Flyte fan-out+aggregate), images.py
+config/flyte.yaml        demo.hosted admin/image/task config
+select.py  report.py     changed-files selector; JSON+HTML+markdown scorecard
+kind_smoke/run.sh        real kind-in-Docker smoke (privileged CI runner)
+tests/                   unit tests (no network/LLM)
+```
+
+## Status / open spikes
+
+- **GLM endpoint contract** — endpoint is live but auth-gated; confirm the exact
+  OpenAI-compatible route + auth header + model name, wire the key as a Flyte/GH
+  secret. Single point of change: `harness/glm.py`.
+- **Harness invocation** — the opencode adapter is complete; `pi` and `hermes`
+  adapters carry a best-effort headless invocation to confirm in the adapter spike
+  (`is_available()` gates uninstalled harnesses cleanly). See `harness/runners/`.

evals/__init__.py

@@ -0,0 +1 @@
+"""Testing & eval harness for the Flyte plugin skills."""

evals/config/flyte.yaml

@@ -0,0 +1,11 @@
+# Flyte config for running the eval harness on demo.hosted.unionai.cloud.
+# Referenced by evals/workflows/eval_wf.py and the GitHub Actions `flyte-evals` job.
+# Auth is supplied out-of-band (device flow / api-key env), never committed here.
+admin:
+  endpoint: dns:///demo.hosted.unionai.cloud
+image:
+  builder: remote
+task:
+  org: demo
+  project: flytesnacks
+  domain: development

evals/harness/__init__.py

@@ -0,0 +1 @@
+"""Core reusable engine: specs, checks, sandbox, runners, judge, scoring."""

evals/harness/checks.py

@@ -0,0 +1,191 @@
+"""Deterministic, LLM-free checks referenced by scenario specs.
+
+Each check kind is a function `(workspace: Path, params: dict) -> CheckResult`.
+Checks run against the workspace an agent produced (trajectory tier) or against a
+skill directory (static tier). They are pure w.r.t. the filesystem + subprocess
+only — no network, no LLM — so they are deterministic and unit-testable.
+
+Add a new kind by decorating a function with @check("my_kind").
+"""
+
+from __future__ import annotations
+
+import ast
+import glob as _glob
+import pathlib
+import re
+import subprocess
+from dataclasses import dataclass
+from typing import Any, Callable
+
+import yaml
+
+
+@dataclass(frozen=True)
+class CheckResult:
+    kind: str
+    passed: bool
+    detail: str
+
+    def __bool__(self) -> bool:  # allow `all(results)`
+        return self.passed
+
+
+CheckFn = Callable[[pathlib.Path, dict], CheckResult]
+_REGISTRY: dict[str, CheckFn] = {}
+
+
+def check(kind: str) -> Callable[[CheckFn], CheckFn]:
+    def deco(fn: CheckFn) -> CheckFn:
+        _REGISTRY[kind] = fn
+        return fn
+    return deco
+
+
+def run_checks(workspace: pathlib.Path, specs: list[dict[str, Any]]) -> list[CheckResult]:
+    results: list[CheckResult] = []
+    for spec in specs:
+        kind = spec.get("kind")
+        fn = _REGISTRY.get(kind)
+        if fn is None:
+            results.append(CheckResult(str(kind), False, f"unknown check kind {kind!r}"))
+            continue
+        try:
+            results.append(fn(workspace, spec))
+        except Exception as exc:  # a check crash is a failure, never a harness crash
+            results.append(CheckResult(str(kind), False, f"check raised: {exc!r}"))
+    return results
+
+
+def _files(workspace: pathlib.Path, pattern: str) -> list[pathlib.Path]:
+    return [pathlib.Path(p) for p in _glob.glob(str(workspace / pattern), recursive=True)]
+
+
+# ---------------------------------------------------------------------------- #
+# Check kinds
+# ---------------------------------------------------------------------------- #
+
+@check("file_glob")
+def _file_glob(ws: pathlib.Path, p: dict) -> CheckResult:
+    """At least `min_count` (default 1) files match `glob`."""
+    pattern = p["glob"]
+    minc = int(p.get("min_count", 1))
+    matches = [f for f in _files(ws, pattern) if f.is_file()]
+    ok = len(matches) >= minc
+    return CheckResult("file_glob", ok, f"{len(matches)} match(es) for {pattern!r} (need >= {minc})")
+
+
+@check("contains_regex")
+def _contains_regex(ws: pathlib.Path, p: dict) -> CheckResult:
+    """Some file matching `file_glob` contains `pattern`."""
+    pattern = re.compile(p["pattern"], re.MULTILINE)
+    files = [f for f in _files(ws, p.get("file_glob", "**/*")) if f.is_file()]
+    for f in files:
+        try:
+            if pattern.search(f.read_text(errors="ignore")):
+                return CheckResult("contains_regex", True, f"{p['pattern']!r} found in {f.name}")
+        except OSError:
+            continue
+    return CheckResult("contains_regex", False, f"{p['pattern']!r} not found in {len(files)} file(s)")
+
+
+@check("not_contains_regex")
+def _not_contains_regex(ws: pathlib.Path, p: dict) -> CheckResult:
+    """No file matching `file_glob` contains `pattern` (anti-pattern guard)."""
+    pattern = re.compile(p["pattern"], re.MULTILINE)
+    for f in _files(ws, p.get("file_glob", "**/*")):
+        if not f.is_file():
+            continue
+        try:
+            m = pattern.search(f.read_text(errors="ignore"))
+        except OSError:
+            continue
+        if m:
+            return CheckResult("not_contains_regex", False, f"forbidden {p['pattern']!r} in {f.name}")
+    return CheckResult("not_contains_regex", True, f"{p['pattern']!r} absent (good)")
+
+
+@check("python_imports")
+def _python_imports(ws: pathlib.Path, p: dict) -> CheckResult:
+    """Some produced .py file imports `module` (top-level name)."""
+    module = p["module"]
+    want = module.split(".")[0]
+    for f in _files(ws, p.get("file_glob", "**/*.py")):
+        if not f.is_file():
+            continue
+        try:
+            tree = ast.parse(f.read_text(errors="ignore"))
+        except SyntaxError:
+            continue
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Import) and any(a.name.split(".")[0] == want for a in node.names):
+                return CheckResult("python_imports", True, f"import {module} in {f.name}")
+            if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == want:
+                return CheckResult("python_imports", True, f"from {module} in {f.name}")
+    return CheckResult("python_imports", False, f"no import of {module!r} found")
+
+
+@check("python_parses")
+def _python_parses(ws: pathlib.Path, p: dict) -> CheckResult:
+    """Every .py file matching `file_glob` parses (valid syntax)."""
+    files = [f for f in _files(ws, p.get("file_glob", "**/*.py")) if f.is_file()]
+    if not files:
+        return CheckResult("python_parses", False, "no python files produced")
+    for f in files:
+        try:
+            ast.parse(f.read_text(errors="ignore"))
+        except SyntaxError as e:
+            return CheckResult("python_parses", False, f"syntax error in {f.name}: {e}")
+    return CheckResult("python_parses", True, f"{len(files)} python file(s) parse")
+
+
+@check("yaml_valid")
+def _yaml_valid(ws: pathlib.Path, p: dict) -> CheckResult:
+    """Every file matching `file_glob` is valid YAML."""
+    files = [f for f in _files(ws, p.get("file_glob", "**/*.y*ml")) if f.is_file()]
+    if not files and p.get("required", True):
+        return CheckResult("yaml_valid", False, "no yaml files produced")
+    for f in files:
+        try:
+            list(yaml.safe_load_all(f.read_text(errors="ignore")))
+        except yaml.YAMLError as e:
+            return CheckResult("yaml_valid", False, f"invalid yaml {f.name}: {e}")
+    return CheckResult("yaml_valid", True, f"{len(files)} yaml file(s) valid")
+
+
+@check("cmd_succeeds")
+def _cmd_succeeds(ws: pathlib.Path, p: dict) -> CheckResult:
+    """Run `cmd` in the workspace; pass iff exit code == expected (default 0)."""
+    cmd = p["cmd"]
+    expected = int(p.get("expect_code", 0))
+    timeout = int(p.get("timeout", 120))
+    try:
+        proc = subprocess.run(
+            cmd, shell=True, cwd=str(ws), capture_output=True, text=True, timeout=timeout
+        )
+    except subprocess.TimeoutExpired:
+        return CheckResult("cmd_succeeds", False, f"timeout after {timeout}s: {cmd}")
+    ok = proc.returncode == expected
+    tail = (proc.stderr or proc.stdout or "").strip().splitlines()[-3:]
+    return CheckResult("cmd_succeeds", ok, f"exit {proc.returncode} (want {expected}) :: {' / '.join(tail)}")
+
+
+@check("stub_called")
+def _stub_called(ws: pathlib.Path, p: dict) -> CheckResult:
+    """A stubbed binary (e.g. kubectl) was invoked with an argument matching `pattern`.
+
+    The stub-PATH bins append their argv to `.stublog` in the workspace (see
+    evals/harness/stubs/). This lets trajectory-tier deploy scenarios assert on
+    what the agent *would* have run, with zero real infra.
+    """
+    log = ws / ".stublog"
+    if not log.exists():
+        return CheckResult("stub_called", False, "no .stublog (no stubbed commands ran)")
+    text = log.read_text(errors="ignore")
+    pattern = re.compile(p["pattern"])
+    ok = bool(pattern.search(text))
+    return CheckResult("stub_called", ok, f"{p['pattern']!r} in stublog" if ok else f"{p['pattern']!r} not invoked")
+
+
+def registered_kinds() -> list[str]:
+    return sorted(_REGISTRY)