From 60e8302dc8935b6ecbec6171e2665041edcec868 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Tue, 21 Jul 2026 12:47:02 -0700 Subject: [PATCH 1/4] add the eval harness to flyte skills Signed-off-by: Niels Bantilan --- evals/__init__.py | 1 + evals/config/flyte.yaml | 11 + evals/harness/__init__.py | 1 + evals/harness/checks.py | 191 ++++++++++++++++++ evals/harness/evaluate.py | 149 ++++++++++++++ evals/harness/glm.py | 71 +++++++ evals/harness/judge.py | 94 +++++++++ evals/harness/run.py | 89 ++++++++ evals/harness/runners/__init__.py | 25 +++ evals/harness/runners/base.py | 108 ++++++++++ evals/harness/runners/hermes.py | 43 ++++ evals/harness/runners/opencode.py | 55 +++++ evals/harness/runners/pi.py | 46 +++++ evals/harness/sandbox.py | 100 +++++++++ evals/harness/spec.py | 174 ++++++++++++++++ evals/harness/static_lint.py | 102 ++++++++++ evals/manifest.yaml | 39 ++++ evals/pyproject.toml | 29 +++ evals/report.py | 113 +++++++++++ .../deploy-flyte-kind-vm/static.yaml | 5 + .../deploy-flyte-kind/kind-config.yaml | 38 ++++ evals/scenarios/deploy-flyte-kind/static.yaml | 5 + evals/scenarios/flyte-deploy-aws/static.yaml | 5 + evals/scenarios/flyte-mcp-server/static.yaml | 5 + evals/scenarios/flyte-sdk-agent/static.yaml | 5 + evals/scenarios/flyte-sdk-app/static.yaml | 5 + .../scenarios/flyte-sdk-author/map-task.yaml | 39 ++++ .../scenarios/flyte-sdk-author/real-run.yaml | 27 +++ evals/scenarios/flyte-sdk-author/static.yaml | 5 + .../flyte-sdk-data/etl-pipeline.yaml | 29 +++ evals/scenarios/flyte-sdk-data/static.yaml | 5 + evals/scenarios/flyte-sdk-eval/static.yaml | 5 + evals/scenarios/flyte-sdk-ml/static.yaml | 5 + .../scenarios/flyte-sdk-optimize/static.yaml | 5 + evals/scenarios/flyte-sdk-run/static.yaml | 5 + evals/scenarios/flyte-sdk-ship/static.yaml | 5 + evals/scenarios/flyte-sdk-types/static.yaml | 5 + evals/scenarios/start-dex-local/static.yaml | 5 + evals/select.py | 99 +++++++++ evals/static.yaml | 5 + evals/workflows/__init__.py | 1 + evals/workflows/eval_wf.py | 107 ++++++++++ evals/workflows/images.py | 31 +++ 43 files changed, 1892 insertions(+) create mode 100644 evals/__init__.py create mode 100644 evals/config/flyte.yaml create mode 100644 evals/harness/__init__.py create mode 100644 evals/harness/checks.py create mode 100644 evals/harness/evaluate.py create mode 100644 evals/harness/glm.py create mode 100644 evals/harness/judge.py create mode 100644 evals/harness/run.py create mode 100644 evals/harness/runners/__init__.py create mode 100644 evals/harness/runners/base.py create mode 100644 evals/harness/runners/hermes.py create mode 100644 evals/harness/runners/opencode.py create mode 100644 evals/harness/runners/pi.py create mode 100644 evals/harness/sandbox.py create mode 100644 evals/harness/spec.py create mode 100644 evals/harness/static_lint.py create mode 100644 evals/manifest.yaml create mode 100644 evals/pyproject.toml create mode 100644 evals/report.py create mode 100644 evals/scenarios/deploy-flyte-kind-vm/static.yaml create mode 100644 evals/scenarios/deploy-flyte-kind/kind-config.yaml create mode 100644 evals/scenarios/deploy-flyte-kind/static.yaml create mode 100644 evals/scenarios/flyte-deploy-aws/static.yaml create mode 100644 evals/scenarios/flyte-mcp-server/static.yaml create mode 100644 evals/scenarios/flyte-sdk-agent/static.yaml create mode 100644 evals/scenarios/flyte-sdk-app/static.yaml create mode 100644 evals/scenarios/flyte-sdk-author/map-task.yaml create mode 100644 evals/scenarios/flyte-sdk-author/real-run.yaml create mode 100644 evals/scenarios/flyte-sdk-author/static.yaml create mode 100644 evals/scenarios/flyte-sdk-data/etl-pipeline.yaml create mode 100644 evals/scenarios/flyte-sdk-data/static.yaml create mode 100644 evals/scenarios/flyte-sdk-eval/static.yaml create mode 100644 evals/scenarios/flyte-sdk-ml/static.yaml create mode 100644 evals/scenarios/flyte-sdk-optimize/static.yaml create mode 100644 evals/scenarios/flyte-sdk-run/static.yaml create mode 100644 evals/scenarios/flyte-sdk-ship/static.yaml create mode 100644 evals/scenarios/flyte-sdk-types/static.yaml create mode 100644 evals/scenarios/start-dex-local/static.yaml create mode 100644 evals/select.py create mode 100644 evals/static.yaml create mode 100644 evals/workflows/__init__.py create mode 100644 evals/workflows/eval_wf.py create mode 100644 evals/workflows/images.py diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 0000000..123f293 --- /dev/null +++ b/evals/__init__.py @@ -0,0 +1 @@ +"""Testing & eval harness for the flyte-skills agent skills.""" diff --git a/evals/config/flyte.yaml b/evals/config/flyte.yaml new file mode 100644 index 0000000..7c2a860 --- /dev/null +++ b/evals/config/flyte.yaml @@ -0,0 +1,11 @@ +# Flyte config for running the eval harness on demo.hosted.unionai.cloud. +# Referenced by evals/workflows/eval_wf.py and the GitHub Actions `flyte-evals` job. +# Auth is supplied out-of-band (device flow / api-key env), never committed here. +admin: + endpoint: dns:///demo.hosted.unionai.cloud +image: + builder: remote +task: + org: demo + project: flytesnacks + domain: development diff --git a/evals/harness/__init__.py b/evals/harness/__init__.py new file mode 100644 index 0000000..76f4fc6 --- /dev/null +++ b/evals/harness/__init__.py @@ -0,0 +1 @@ +"""Core reusable engine: specs, checks, sandbox, runners, judge, scoring.""" diff --git a/evals/harness/checks.py b/evals/harness/checks.py new file mode 100644 index 0000000..e92fe75 --- /dev/null +++ b/evals/harness/checks.py @@ -0,0 +1,191 @@ +"""Deterministic, LLM-free checks referenced by scenario specs. + +Each check kind is a function `(workspace: Path, params: dict) -> CheckResult`. +Checks run against the workspace an agent produced (trajectory tier) or against a +skill directory (static tier). They are pure w.r.t. the filesystem + subprocess +only — no network, no LLM — so they are deterministic and unit-testable. + +Add a new kind by decorating a function with @check("my_kind"). +""" + +from __future__ import annotations + +import ast +import glob as _glob +import pathlib +import re +import subprocess +from dataclasses import dataclass +from typing import Any, Callable + +import yaml + + +@dataclass(frozen=True) +class CheckResult: + kind: str + passed: bool + detail: str + + def __bool__(self) -> bool: # allow `all(results)` + return self.passed + + +CheckFn = Callable[[pathlib.Path, dict], CheckResult] +_REGISTRY: dict[str, CheckFn] = {} + + +def check(kind: str) -> Callable[[CheckFn], CheckFn]: + def deco(fn: CheckFn) -> CheckFn: + _REGISTRY[kind] = fn + return fn + return deco + + +def run_checks(workspace: pathlib.Path, specs: list[dict[str, Any]]) -> list[CheckResult]: + results: list[CheckResult] = [] + for spec in specs: + kind = spec.get("kind") + fn = _REGISTRY.get(kind) + if fn is None: + results.append(CheckResult(str(kind), False, f"unknown check kind {kind!r}")) + continue + try: + results.append(fn(workspace, spec)) + except Exception as exc: # a check crash is a failure, never a harness crash + results.append(CheckResult(str(kind), False, f"check raised: {exc!r}")) + return results + + +def _files(workspace: pathlib.Path, pattern: str) -> list[pathlib.Path]: + return [pathlib.Path(p) for p in _glob.glob(str(workspace / pattern), recursive=True)] + + +# ---------------------------------------------------------------------------- # +# Check kinds +# ---------------------------------------------------------------------------- # + +@check("file_glob") +def _file_glob(ws: pathlib.Path, p: dict) -> CheckResult: + """At least `min_count` (default 1) files match `glob`.""" + pattern = p["glob"] + minc = int(p.get("min_count", 1)) + matches = [f for f in _files(ws, pattern) if f.is_file()] + ok = len(matches) >= minc + return CheckResult("file_glob", ok, f"{len(matches)} match(es) for {pattern!r} (need >= {minc})") + + +@check("contains_regex") +def _contains_regex(ws: pathlib.Path, p: dict) -> CheckResult: + """Some file matching `file_glob` contains `pattern`.""" + pattern = re.compile(p["pattern"], re.MULTILINE) + files = [f for f in _files(ws, p.get("file_glob", "**/*")) if f.is_file()] + for f in files: + try: + if pattern.search(f.read_text(errors="ignore")): + return CheckResult("contains_regex", True, f"{p['pattern']!r} found in {f.name}") + except OSError: + continue + return CheckResult("contains_regex", False, f"{p['pattern']!r} not found in {len(files)} file(s)") + + +@check("not_contains_regex") +def _not_contains_regex(ws: pathlib.Path, p: dict) -> CheckResult: + """No file matching `file_glob` contains `pattern` (anti-pattern guard).""" + pattern = re.compile(p["pattern"], re.MULTILINE) + for f in _files(ws, p.get("file_glob", "**/*")): + if not f.is_file(): + continue + try: + m = pattern.search(f.read_text(errors="ignore")) + except OSError: + continue + if m: + return CheckResult("not_contains_regex", False, f"forbidden {p['pattern']!r} in {f.name}") + return CheckResult("not_contains_regex", True, f"{p['pattern']!r} absent (good)") + + +@check("python_imports") +def _python_imports(ws: pathlib.Path, p: dict) -> CheckResult: + """Some produced .py file imports `module` (top-level name).""" + module = p["module"] + want = module.split(".")[0] + for f in _files(ws, p.get("file_glob", "**/*.py")): + if not f.is_file(): + continue + try: + tree = ast.parse(f.read_text(errors="ignore")) + except SyntaxError: + continue + for node in ast.walk(tree): + if isinstance(node, ast.Import) and any(a.name.split(".")[0] == want for a in node.names): + return CheckResult("python_imports", True, f"import {module} in {f.name}") + if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == want: + return CheckResult("python_imports", True, f"from {module} in {f.name}") + return CheckResult("python_imports", False, f"no import of {module!r} found") + + +@check("python_parses") +def _python_parses(ws: pathlib.Path, p: dict) -> CheckResult: + """Every .py file matching `file_glob` parses (valid syntax).""" + files = [f for f in _files(ws, p.get("file_glob", "**/*.py")) if f.is_file()] + if not files: + return CheckResult("python_parses", False, "no python files produced") + for f in files: + try: + ast.parse(f.read_text(errors="ignore")) + except SyntaxError as e: + return CheckResult("python_parses", False, f"syntax error in {f.name}: {e}") + return CheckResult("python_parses", True, f"{len(files)} python file(s) parse") + + +@check("yaml_valid") +def _yaml_valid(ws: pathlib.Path, p: dict) -> CheckResult: + """Every file matching `file_glob` is valid YAML.""" + files = [f for f in _files(ws, p.get("file_glob", "**/*.y*ml")) if f.is_file()] + if not files and p.get("required", True): + return CheckResult("yaml_valid", False, "no yaml files produced") + for f in files: + try: + list(yaml.safe_load_all(f.read_text(errors="ignore"))) + except yaml.YAMLError as e: + return CheckResult("yaml_valid", False, f"invalid yaml {f.name}: {e}") + return CheckResult("yaml_valid", True, f"{len(files)} yaml file(s) valid") + + +@check("cmd_succeeds") +def _cmd_succeeds(ws: pathlib.Path, p: dict) -> CheckResult: + """Run `cmd` in the workspace; pass iff exit code == expected (default 0).""" + cmd = p["cmd"] + expected = int(p.get("expect_code", 0)) + timeout = int(p.get("timeout", 120)) + try: + proc = subprocess.run( + cmd, shell=True, cwd=str(ws), capture_output=True, text=True, timeout=timeout + ) + except subprocess.TimeoutExpired: + return CheckResult("cmd_succeeds", False, f"timeout after {timeout}s: {cmd}") + ok = proc.returncode == expected + tail = (proc.stderr or proc.stdout or "").strip().splitlines()[-3:] + return CheckResult("cmd_succeeds", ok, f"exit {proc.returncode} (want {expected}) :: {' / '.join(tail)}") + + +@check("stub_called") +def _stub_called(ws: pathlib.Path, p: dict) -> CheckResult: + """A stubbed binary (e.g. kubectl) was invoked with an argument matching `pattern`. + + The stub-PATH bins append their argv to `.stublog` in the workspace (see + evals/harness/stubs/). This lets trajectory-tier deploy scenarios assert on + what the agent *would* have run, with zero real infra. + """ + log = ws / ".stublog" + if not log.exists(): + return CheckResult("stub_called", False, "no .stublog (no stubbed commands ran)") + text = log.read_text(errors="ignore") + pattern = re.compile(p["pattern"]) + ok = bool(pattern.search(text)) + return CheckResult("stub_called", ok, f"{p['pattern']!r} in stublog" if ok else f"{p['pattern']!r} not invoked") + + +def registered_kinds() -> list[str]: + return sorted(_REGISTRY) diff --git a/evals/harness/evaluate.py b/evals/harness/evaluate.py new file mode 100644 index 0000000..20f92a6 --- /dev/null +++ b/evals/harness/evaluate.py @@ -0,0 +1,149 @@ +"""Evaluate one scenario end-to-end and produce a scored verdict. + +Static tier -> lint the SKILL.md (no agent, no LLM). +Trajectory -> for each arm (treatment/control): sandbox -> run harness -> + deterministic checks + LLM judge. Report per-arm score and the + treatment-minus-control *lift*. +Real tier -> trajectory, plus (best-effort) execute the produced artifact + (`flyte run` on demo.hosted); the real-run outcome is recorded + as an extra check. Executed by the caller/workflow, gated on env. + +The whole module is defensive: a harness crash, missing CLI, or judge/network +error becomes a failed result with detail — never an exception that aborts a run. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from . import checks as checks_mod +from .glm import GLMConfig +from .judge import JudgeResult, judge as run_judge +from .runners import get_runner +from .runners.base import Trajectory +from .sandbox import make_sandbox +from .spec import REPO_ROOT, Scenario +from .static_lint import lint_skill + + +@dataclass +class ArmResult: + arm: str + checks: list[checks_mod.CheckResult] = field(default_factory=list) + judge: JudgeResult | None = None + exit_code: int = 0 + error: str = "" + + @property + def checks_passed(self) -> bool: + return all(c.passed for c in self.checks) if self.checks else True + + @property + def score(self) -> float: + """Combined arm score in [0,1]: deterministic checks gate, judge grades.""" + if not self.checks_passed: + return 0.0 + if self.judge is not None: + return self.judge.score + return 1.0 if self.checks_passed else 0.0 + + @property + def passed(self) -> bool: + judge_ok = self.judge.passed if self.judge is not None else True + return self.checks_passed and judge_ok and not self.error + + +@dataclass +class ScenarioResult: + scenario_id: str + skill: str + tier: str + harness: str | None = None + arms: dict[str, ArmResult] = field(default_factory=dict) + + @property + def lift(self) -> float | None: + t, c = self.arms.get("treatment"), self.arms.get("control") + if t is None or c is None: + return None + return round(t.score - c.score, 4) + + @property + def passed(self) -> bool: + t = self.arms.get("treatment") + return bool(t and t.passed) + + def to_dict(self) -> dict: + return { + "scenario_id": self.scenario_id, + "skill": self.skill, + "tier": self.tier, + "harness": self.harness, + "passed": self.passed, + "lift": self.lift, + "arms": { + arm: { + "score": r.score, + "passed": r.passed, + "checks_passed": r.checks_passed, + "exit_code": r.exit_code, + "error": r.error, + "judge": None if r.judge is None else { + "score": r.judge.score, + "passed": r.judge.passed, + "dimensions": r.judge.dimensions, + "rationale": r.judge.rationale, + }, + "checks": [ + {"kind": c.kind, "passed": c.passed, "detail": c.detail} + for c in r.checks + ], + } + for arm, r in self.arms.items() + }, + } + + +def evaluate_static(scenario: Scenario) -> ScenarioResult: + skill_dir = REPO_ROOT / "plugins" / "flyte-skills" / "skills" / scenario.skill + results = lint_skill(skill_dir) + arm = ArmResult(arm="treatment", checks=results) + return ScenarioResult(scenario.id, scenario.skill, "static", harness=None, + arms={"treatment": arm}) + + +def evaluate_scenario(scenario: Scenario, harness: str, glm: GLMConfig) -> ScenarioResult: + """Run a trajectory/real scenario for one harness across its arms.""" + if scenario.tier == "static": + return evaluate_static(scenario) + + res = ScenarioResult(scenario.id, scenario.skill, scenario.tier, harness=harness) + runner = get_runner(harness) + + for arm in scenario.arms(): + if not runner.is_available(): + res.arms[arm] = ArmResult(arm=arm, error=f"{harness} CLI not available") + continue + try: + with make_sandbox(tier=scenario.tier, glm=glm) as sb: + traj = runner.run(scenario, sb, arm, glm) + arm_res = _score_trajectory(scenario, traj, sb, glm) + except Exception as exc: # never let one arm abort the suite + arm_res = ArmResult(arm=arm, error=f"harness crashed: {exc!r}") + res.arms[arm] = arm_res + + return res + + +def _score_trajectory(scenario: Scenario, traj: Trajectory, sandbox, glm: GLMConfig) -> ArmResult: + check_results = checks_mod.run_checks(sandbox.workspace, list(scenario.checks)) + judge_result = None + if scenario.judge is not None: + judge_result = run_judge(scenario.judge, traj.as_judge_text(), glm) + return ArmResult( + arm=traj.arm, + checks=check_results, + judge=judge_result, + exit_code=traj.exit_code, + error=traj.error, + ) diff --git a/evals/harness/glm.py b/evals/harness/glm.py new file mode 100644 index 0000000..fa3e46e --- /dev/null +++ b/evals/harness/glm.py @@ -0,0 +1,71 @@ +"""Client config for the union-hosted GLM endpoint (the LLM under test + judge). + +The endpoint is OpenAI-compatible but auth-gated. Everything downstream (runner +adapters + judge) reads its connection details from here so there is exactly one +place to fix once the auth/schema spike is resolved. + +Environment variables (never hardcode creds): + GLM_BASE_URL default: the demo.hosted GLM app, with /v1 appended if missing + GLM_API_KEY bearer token / api key for the endpoint (required for live calls) + GLM_MODEL model name to request (default: glm-5.2) +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +DEFAULT_BASE_URL = "https://glm-5-2-llm-service-development.apps.demo.hosted.unionai.cloud/v1" +DEFAULT_MODEL = "glm-5.2" + + +@dataclass(frozen=True) +class GLMConfig: + base_url: str + api_key: str + model: str + + @property + def has_key(self) -> bool: + return bool(self.api_key) + + @staticmethod + def from_env() -> "GLMConfig": + base = os.environ.get("GLM_BASE_URL", DEFAULT_BASE_URL).rstrip("/") + if not base.endswith("/v1"): + base = base + "/v1" + return GLMConfig( + base_url=base, + api_key=os.environ.get("GLM_API_KEY", ""), + model=os.environ.get("GLM_MODEL", DEFAULT_MODEL), + ) + + +def chat_completion(cfg: GLMConfig, messages: list[dict], *, temperature: float = 0.0, + max_tokens: int = 2048, timeout: int = 120) -> str: + """Minimal OpenAI-compatible chat call. Returns the assistant text. + + Kept dependency-light (requests) and isolated so the exact route/auth header + can be adjusted in one place after the endpoint spike. + """ + import requests + + if not cfg.has_key: + raise RuntimeError( + "GLM_API_KEY is not set — cannot call the GLM endpoint. " + "Export a valid token/api key for the demo.hosted GLM app." + ) + resp = requests.post( + f"{cfg.base_url}/chat/completions", + headers={"Authorization": f"Bearer {cfg.api_key}", "Content-Type": "application/json"}, + json={ + "model": cfg.model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + }, + timeout=timeout, + ) + resp.raise_for_status() + data = resp.json() + return data["choices"][0]["message"]["content"] diff --git a/evals/harness/judge.py b/evals/harness/judge.py new file mode 100644 index 0000000..10553b5 --- /dev/null +++ b/evals/harness/judge.py @@ -0,0 +1,94 @@ +"""LLM-as-judge over an agent trajectory, via the GLM endpoint. + +The judge is a *secondary* signal: deterministic `checks.py` gate pass/fail; the +judge produces a graded rubric score (0..1 per dimension) used for the lift +metric and for surfacing quality regressions. Judge output is forced to JSON and +parsed defensively so a malformed response degrades to score 0, never a crash. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field + +from .glm import GLMConfig, chat_completion +from .spec import JudgeSpec + +_JSON_RE = re.compile(r"\{.*\}", re.DOTALL) + +SYSTEM = ( + "You are a strict evaluator of an AI coding agent's work. You are given a " + "rubric and the agent's trajectory (what it said and the files it produced). " + "Score each named dimension from 0.0 to 1.0. Respond ONLY with a JSON object " + 'of the form {"scores": {"": , ...}, "rationale": ""}. ' + "Do not include any prose outside the JSON." +) + + +@dataclass(frozen=True) +class JudgeResult: + score: float # weighted aggregate in [0, 1] + passed: bool # score >= pass_threshold + dimensions: dict[str, float] = field(default_factory=dict) + rationale: str = "" + raw: str = "" + + @staticmethod + def error(msg: str) -> "JudgeResult": + return JudgeResult(score=0.0, passed=False, rationale=f"judge error: {msg}", raw=msg) + + +def build_prompt(rubric: str, dimensions: list[str], trajectory_text: str) -> list[dict]: + dims = ", ".join(dimensions) if dimensions else "correctness, skill_adherence, idiomatic" + user = ( + f"# Rubric\n{rubric}\n\n" + f"# Dimensions to score\n{dims}\n\n" + f"# Agent trajectory\n{trajectory_text}\n" + ) + return [{"role": "system", "content": SYSTEM}, {"role": "user", "content": user}] + + +def parse_response(text: str, weights: dict[str, float], threshold: float) -> JudgeResult: + m = _JSON_RE.search(text or "") + if not m: + return JudgeResult.error(f"no JSON in judge response: {text[:200]!r}") + try: + obj = json.loads(m.group(0)) + except json.JSONDecodeError as e: + return JudgeResult.error(f"bad JSON: {e}") + scores = {k: _clamp(v) for k, v in (obj.get("scores") or {}).items()} + agg = _weighted(scores, weights) + return JudgeResult( + score=agg, + passed=agg >= threshold, + dimensions=scores, + rationale=str(obj.get("rationale", "")), + raw=text, + ) + + +def judge(spec: JudgeSpec, trajectory_text: str, cfg: GLMConfig) -> JudgeResult: + dims = list(spec.weights) or ["correctness", "skill_adherence", "idiomatic"] + messages = build_prompt(spec.rubric, dims, trajectory_text) + try: + text = chat_completion(cfg, messages, temperature=0.0, max_tokens=1024) + except Exception as exc: + return JudgeResult.error(repr(exc)) + return parse_response(text, spec.weights, spec.pass_threshold) + + +def _clamp(v: object) -> float: + try: + return max(0.0, min(1.0, float(v))) + except (TypeError, ValueError): + return 0.0 + + +def _weighted(scores: dict[str, float], weights: dict[str, float]) -> float: + if not scores: + return 0.0 + if weights: + total_w = sum(weights.values()) or 1.0 + return sum(scores.get(k, 0.0) * w for k, w in weights.items()) / total_w + return sum(scores.values()) / len(scores) diff --git a/evals/harness/run.py b/evals/harness/run.py new file mode 100644 index 0000000..441841b --- /dev/null +++ b/evals/harness/run.py @@ -0,0 +1,89 @@ +"""Local CLI: run eval scenarios without Flyte. + +Examples +-------- + # static lint of every skill (no LLM, no agent): + python -m evals.harness.run --tier static + + # one scenario on one harness, end-to-end (needs GLM_API_KEY + the harness CLI): + python -m evals.harness.run --scenario sdk-author-map-task --harness opencode + + # every trajectory scenario for a skill, JSON out: + python -m evals.harness.run --skill flyte-sdk-author --json out.json +""" + +from __future__ import annotations + +import argparse +import json +import sys + +from .evaluate import ScenarioResult, evaluate_scenario, evaluate_static +from .glm import GLMConfig +from .spec import load_scenarios + + +def _select(args) -> list: + scenarios = load_scenarios() + out = [] + for sc in scenarios: + if args.scenario and sc.id != args.scenario: + continue + if args.skill and sc.skill != args.skill: + continue + if args.tier and sc.tier != args.tier: + continue + out.append(sc) + return out + + +def run(args) -> list[ScenarioResult]: + glm = GLMConfig.from_env() + results: list[ScenarioResult] = [] + for sc in _select(args): + if sc.tier == "static": + results.append(evaluate_static(sc)) + continue + harnesses = [args.harness] if args.harness else list(sc.harnesses) + for h in harnesses: + results.append(evaluate_scenario(sc, h, glm)) + return results + + +def _print_table(results: list[ScenarioResult]) -> int: + if not results: + print("no scenarios matched the filter", file=sys.stderr) + return 2 + print(f"{'SCENARIO':<32} {'HARNESS':<9} {'TIER':<11} {'PASS':<5} {'LIFT':>6}") + print("-" * 70) + failures = 0 + for r in results: + lift = "" if r.lift is None else f"{r.lift:+.2f}" + status = "ok" if r.passed else "FAIL" + failures += 0 if r.passed else 1 + print(f"{r.scenario_id:<32} {(r.harness or '-'):<9} {r.tier:<11} {status:<5} {lift:>6}") + print("-" * 70) + print(f"{len(results)} result(s), {failures} failing") + return 1 if failures else 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(prog="flyte-evals", description="Run flyte-skills evals locally") + ap.add_argument("--scenario", help="scenario id") + ap.add_argument("--skill", help="filter by skill name") + ap.add_argument("--tier", choices=["static", "trajectory", "real"], help="filter by tier") + ap.add_argument("--harness", choices=["opencode", "pi", "hermes"], help="single harness") + ap.add_argument("--json", dest="json_out", help="write full results JSON to this path") + args = ap.parse_args(argv) + + results = run(args) + if args.json_out: + payload = [r.to_dict() for r in results] + with open(args.json_out, "w") as fh: + json.dump(payload, fh, indent=2) + print(f"wrote {args.json_out}", file=sys.stderr) + return _print_table(results) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/harness/runners/__init__.py b/evals/harness/runners/__init__.py new file mode 100644 index 0000000..9e7c1e7 --- /dev/null +++ b/evals/harness/runners/__init__.py @@ -0,0 +1,25 @@ +"""Per-harness runner adapters (opencode, pi, hermes) behind one interface.""" + +from __future__ import annotations + +from .base import Runner, Trajectory +from .opencode import OpenCodeRunner +from .pi import PiRunner +from .hermes import HermesRunner + +_RUNNERS: dict[str, type[Runner]] = { + "opencode": OpenCodeRunner, + "pi": PiRunner, + "hermes": HermesRunner, +} + + +def get_runner(name: str) -> Runner: + try: + return _RUNNERS[name]() + except KeyError: + raise ValueError(f"unknown harness {name!r}; known: {sorted(_RUNNERS)}") + + +def available_harnesses() -> list[str]: + return sorted(_RUNNERS) diff --git a/evals/harness/runners/base.py b/evals/harness/runners/base.py new file mode 100644 index 0000000..cfcb7ab --- /dev/null +++ b/evals/harness/runners/base.py @@ -0,0 +1,108 @@ +"""Runner adapter interface + the Trajectory it returns. + +A Runner points a specific agent harness at the GLM endpoint, installs the skill +under test (treatment arm only), runs one non-interactive turn on the scenario +prompt inside the sandbox workspace, and returns a Trajectory: the transcript, +the files the agent produced, and the process outcome. +""" + +from __future__ import annotations + +import abc +import pathlib +import subprocess +from dataclasses import dataclass, field + +from ..glm import GLMConfig +from ..sandbox import Sandbox, install_skill +from ..spec import Scenario + +# Files we never treat as "agent-produced artifacts" when snapshotting a workspace. +_IGNORE = {".stublog", ".opencode", ".pi", ".git"} + + +@dataclass +class Trajectory: + harness: str + arm: str # "treatment" | "control" + transcript: str # what the agent said (stdout / json events) + files: dict[str, str] = field(default_factory=dict) # relpath -> content + exit_code: int = 0 + error: str = "" + + def as_judge_text(self, max_chars: int = 16000) -> str: + parts = [f"## Transcript\n{self.transcript.strip()}"] + for rel, content in sorted(self.files.items()): + parts.append(f"## File: {rel}\n```\n{content}\n```") + text = "\n\n".join(parts) + return text[:max_chars] + + +class Runner(abc.ABC): + name: str = "base" + + @abc.abstractmethod + def is_available(self) -> bool: + """True if the harness CLI is installed and usable.""" + + @abc.abstractmethod + def skills_dir(self, workspace: pathlib.Path) -> pathlib.Path: + """Where this harness discovers skills, relative to the workspace.""" + + @abc.abstractmethod + def _invoke(self, prompt: str, sandbox: Sandbox, glm: GLMConfig) -> tuple[str, int, str]: + """Run one non-interactive turn. Returns (transcript, exit_code, error).""" + + def run(self, scenario: Scenario, sandbox: Sandbox, arm: str, glm: GLMConfig) -> Trajectory: + if arm not in ("treatment", "control"): + raise ValueError(f"bad arm {arm!r}") + if arm == "treatment": + skill_dir = _repo_skill_dir(scenario.skill) + install_skill(skill_dir, self.skills_dir(sandbox.workspace)) + transcript, code, err = self._invoke(scenario.prompt, sandbox, glm) + return Trajectory( + harness=self.name, + arm=arm, + transcript=transcript, + files=snapshot_files(sandbox.workspace), + exit_code=code, + error=err, + ) + + +def snapshot_files(workspace: pathlib.Path, max_bytes: int = 200_000) -> dict[str, str]: + """Capture text files the agent produced in the workspace.""" + out: dict[str, str] = {} + for p in sorted(workspace.rglob("*")): + if not p.is_file(): + continue + rel = p.relative_to(workspace) + if any(part in _IGNORE for part in rel.parts): + continue + try: + if p.stat().st_size > max_bytes: + out[str(rel)] = f"<{p.stat().st_size} bytes, truncated>" + continue + out[str(rel)] = p.read_text(errors="ignore") + except OSError: + continue + return out + + +def _repo_skill_dir(skill: str) -> pathlib.Path: + from ..spec import REPO_ROOT + return REPO_ROOT / "plugins" / "flyte-skills" / "skills" / skill + + +def sh(cmd: list[str], cwd: pathlib.Path, env: dict[str, str], timeout: int = 600 + ) -> tuple[str, int, str]: + """Helper: run a subprocess, capture combined output. Never raises.""" + try: + proc = subprocess.run( + cmd, cwd=str(cwd), env=env, capture_output=True, text=True, timeout=timeout + ) + return (proc.stdout + proc.stderr, proc.returncode, "") + except FileNotFoundError as e: + return ("", 127, f"binary not found: {e}") + except subprocess.TimeoutExpired: + return ("", 124, f"timeout after {timeout}s") diff --git a/evals/harness/runners/hermes.py b/evals/harness/runners/hermes.py new file mode 100644 index 0000000..af9518c --- /dev/null +++ b/evals/harness/runners/hermes.py @@ -0,0 +1,43 @@ +"""hermes adapter. + +hermes installs skills by repo path and runs headless. We install the skill into +a workspace-local dir and run one non-interactive turn pointed at the GLM +endpoint (OpenAI-compatible base URL via env). + +NOTE (spike): hermes was not installed in the dev environment; the exact +headless flag + custom-model config is confirmed in the adapter spike (see plan +Risks). `is_available()` gates it out cleanly until then. +""" + +from __future__ import annotations + +import pathlib +import shutil + +from ..glm import GLMConfig +from ..sandbox import Sandbox +from .base import Runner, sh + + +class HermesRunner(Runner): + name = "hermes" + + def is_available(self) -> bool: + return shutil.which("hermes") is not None + + def skills_dir(self, workspace: pathlib.Path) -> pathlib.Path: + return workspace / ".hermes" / "skills" + + def _invoke(self, prompt: str, sandbox: Sandbox, glm: GLMConfig): + env = dict(sandbox.env) + env["HERMES_SKILLS_DIR"] = str(self.skills_dir(sandbox.workspace)) + env["OPENAI_BASE_URL"] = glm.base_url + env["OPENAI_API_KEY"] = glm.api_key + cmd = [ + "hermes", "run", + "--model", glm.model, + "--non-interactive", + prompt, + ] + out, code, err = sh(cmd, cwd=sandbox.workspace, env=env) + return out, code, err diff --git a/evals/harness/runners/opencode.py b/evals/harness/runners/opencode.py new file mode 100644 index 0000000..5a7d91a --- /dev/null +++ b/evals/harness/runners/opencode.py @@ -0,0 +1,55 @@ +"""opencode adapter. + +Config strategy: + - skills discovered from `/.opencode/skills//` + - a project `opencode.json` registers the GLM endpoint as an OpenAI-compatible + custom provider `glm`, so `--model glm/` routes there + - headless run: `opencode run --model glm/ --format json --auto ""` +""" + +from __future__ import annotations + +import json +import pathlib +import shutil + +from ..glm import GLMConfig +from ..sandbox import Sandbox +from .base import Runner, sh + + +class OpenCodeRunner(Runner): + name = "opencode" + + def is_available(self) -> bool: + return shutil.which("opencode") is not None + + def skills_dir(self, workspace: pathlib.Path) -> pathlib.Path: + return workspace / ".opencode" / "skills" + + def _write_config(self, workspace: pathlib.Path, glm: GLMConfig) -> None: + cfg = { + "$schema": "https://opencode.ai/config.json", + "provider": { + "glm": { + "npm": "@ai-sdk/openai-compatible", + "name": "GLM (union-hosted)", + "options": {"baseURL": glm.base_url, "apiKey": glm.api_key}, + "models": {glm.model: {"name": glm.model}}, + } + }, + } + (workspace / "opencode.json").write_text(json.dumps(cfg, indent=2)) + + def _invoke(self, prompt: str, sandbox: Sandbox, glm: GLMConfig): + self._write_config(sandbox.workspace, glm) + cmd = [ + "opencode", "run", + "--model", f"glm/{glm.model}", + "--format", "json", + "--auto", + "--dir", str(sandbox.workspace), + prompt, + ] + out, code, err = sh(cmd, cwd=sandbox.workspace, env=sandbox.env) + return out, code, err diff --git a/evals/harness/runners/pi.py b/evals/harness/runners/pi.py new file mode 100644 index 0000000..fa4c55e --- /dev/null +++ b/evals/harness/runners/pi.py @@ -0,0 +1,46 @@ +"""pi adapter. + +pi discovers nested SKILL.md folders under its agent skills dir. We install the +skill into a workspace-local skills dir and run pi non-interactively pointed at +the GLM endpoint (OpenAI-compatible base URL via env). + +NOTE (spike): the exact non-interactive flag + custom-model config for pi is +confirmed in the adapter spike (see plan Risks). The invocation below is the +current best-effort; adjust `_invoke`/`skills_dir` once verified — everything +else in the harness is agnostic to it. +""" + +from __future__ import annotations + +import pathlib +import shutil + +from ..glm import GLMConfig +from ..sandbox import Sandbox +from .base import Runner, sh + + +class PiRunner(Runner): + name = "pi" + + def is_available(self) -> bool: + return shutil.which("pi") is not None + + def skills_dir(self, workspace: pathlib.Path) -> pathlib.Path: + # Workspace-local skills dir; PI_SKILLS_DIR points pi at it (set below). + return workspace / ".pi" / "skills" + + def _invoke(self, prompt: str, sandbox: Sandbox, glm: GLMConfig): + env = dict(sandbox.env) + # Point pi's skill discovery + OpenAI-compatible model at our config. + env["PI_SKILLS_DIR"] = str(self.skills_dir(sandbox.workspace)) + env["OPENAI_BASE_URL"] = glm.base_url + env["OPENAI_API_KEY"] = glm.api_key + cmd = [ + "pi", "run", + "--model", glm.model, + "--yes", + prompt, + ] + out, code, err = sh(cmd, cwd=sandbox.workspace, env=env) + return out, code, err diff --git a/evals/harness/sandbox.py b/evals/harness/sandbox.py new file mode 100644 index 0000000..04c7201 --- /dev/null +++ b/evals/harness/sandbox.py @@ -0,0 +1,100 @@ +"""Isolated workspace for one agent run: dir, skill install, stub-PATH, env. + +A `Sandbox` is a throwaway temp directory the agent runs inside. For the +trajectory tier it also lays down *stub* binaries (fake kubectl/helm/docker/...) +earlier on PATH than the real ones, so side-effecting commands the agent issues +log their argv to `/.stublog` and succeed without touching real infra. +The `stub_called` check then asserts on what the agent *would* have run. +""" + +from __future__ import annotations + +import os +import pathlib +import shutil +import stat +import tempfile +from dataclasses import dataclass, field + +from .glm import GLMConfig + +# Side-effecting tools the deploy/SDK skills may invoke. Stubbed in trajectory +# tier; real (unstubbed) in real tier. +STUBBED_TOOLS = [ + "kubectl", "helm", "docker", "kind", "k3d", "aws", "eksctl", + "gcloud", "doctl", "ssh", "scp", "terraform", +] + +_STUB_TEMPLATE = """#!/usr/bin/env bash +# Auto-generated trajectory-tier stub. Logs argv, succeeds, produces no effects. +echo "$(basename "$0") $*" >> "${STUBLOG:-$PWD/.stublog}" +exit 0 +""" + + +@dataclass +class Sandbox: + workspace: pathlib.Path + env: dict[str, str] + stub_bin: pathlib.Path | None = None + _tmp: tempfile.TemporaryDirectory | None = field(default=None, repr=False) + + @property + def stublog(self) -> pathlib.Path: + return self.workspace / ".stublog" + + def cleanup(self) -> None: + if self._tmp is not None: + self._tmp.cleanup() + + def __enter__(self) -> "Sandbox": + return self + + def __exit__(self, *exc) -> None: + self.cleanup() + + +def make_sandbox(*, tier: str, glm: GLMConfig | None = None, + fixtures: pathlib.Path | None = None) -> Sandbox: + tmp = tempfile.TemporaryDirectory(prefix="flyte-eval-") + ws = pathlib.Path(tmp.name) / "workspace" + ws.mkdir(parents=True) + + if fixtures and fixtures.exists(): + shutil.copytree(fixtures, ws, dirs_exist_ok=True) + + env = dict(os.environ) + env["STUBLOG"] = str(ws / ".stublog") + + stub_bin = None + if tier == "trajectory": + stub_bin = _write_stubs(pathlib.Path(tmp.name) / "stubbin") + env["PATH"] = f"{stub_bin}{os.pathsep}{env.get('PATH', '')}" + + if glm is not None: + # Surfaced to runner adapters so each harness can point at the GLM endpoint. + env.setdefault("GLM_BASE_URL", glm.base_url) + env.setdefault("GLM_MODEL", glm.model) + if glm.has_key: + env.setdefault("GLM_API_KEY", glm.api_key) + + return Sandbox(workspace=ws, env=env, stub_bin=stub_bin, _tmp=tmp) + + +def install_skill(skill_dir: pathlib.Path, dest_skills_dir: pathlib.Path) -> pathlib.Path: + """Copy a skill folder into a harness's skills directory. Returns the dest.""" + dest = dest_skills_dir / skill_dir.name + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(skill_dir, dest) + return dest + + +def _write_stubs(bin_dir: pathlib.Path) -> pathlib.Path: + bin_dir.mkdir(parents=True, exist_ok=True) + for tool in STUBBED_TOOLS: + p = bin_dir / tool + p.write_text(_STUB_TEMPLATE) + p.chmod(p.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return bin_dir diff --git a/evals/harness/spec.py b/evals/harness/spec.py new file mode 100644 index 0000000..495e819 --- /dev/null +++ b/evals/harness/spec.py @@ -0,0 +1,174 @@ +"""Scenario and manifest schema + loaders. + +A *scenario* is a declarative YAML spec describing one eval: which skill, which +tier, the user prompt handed to the agent, deterministic checks, an LLM-judge +rubric, and (for tier: real) how to actually execute the produced artifact. + +Everything here is pure data + parsing — no LLM, no network, no subprocess — so +it is trivially unit-testable. +""" + +from __future__ import annotations + +import pathlib +from dataclasses import dataclass, field +from typing import Any + +import yaml + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +EVALS_ROOT = REPO_ROOT / "evals" + +VALID_TIERS = ("static", "trajectory", "real") +VALID_HARNESSES = ("opencode", "pi", "hermes") +VALID_ARMS = ("treatment", "control") + + +@dataclass(frozen=True) +class JudgeSpec: + rubric: str + weights: dict[str, float] = field(default_factory=dict) + pass_threshold: float = 0.7 + + @staticmethod + def from_dict(d: dict[str, Any] | None) -> "JudgeSpec | None": + if not d: + return None + return JudgeSpec( + rubric=d["rubric"], + weights=dict(d.get("weights") or {}), + pass_threshold=float(d.get("pass_threshold", 0.7)), + ) + + +@dataclass(frozen=True) +class RealRunSpec: + flyte_run: bool = False + entrypoint: str | None = None # e.g. "workflow.py main"; None -> autodetect + inputs: dict[str, Any] = field(default_factory=dict) + expect_status: str = "SUCCEEDED" + + @staticmethod + def from_dict(d: dict[str, Any] | None) -> "RealRunSpec | None": + if not d: + return None + return RealRunSpec( + flyte_run=bool(d.get("flyte_run", False)), + entrypoint=d.get("entrypoint"), + inputs=dict(d.get("inputs") or {}), + expect_status=str(d.get("expect_status", "SUCCEEDED")), + ) + + +@dataclass(frozen=True) +class Scenario: + id: str + skill: str + tier: str + prompt: str + harnesses: tuple[str, ...] + control: bool + setup: dict[str, Any] + checks: tuple[dict[str, Any], ...] + judge: JudgeSpec | None + real_run: RealRunSpec | None + source_path: pathlib.Path | None = None + + @staticmethod + def from_dict(d: dict[str, Any], source_path: pathlib.Path | None = None) -> "Scenario": + _require(d, "id", source_path) + _require(d, "skill", source_path) + + tier = d.get("tier", "trajectory") + if tier not in VALID_TIERS: + raise ValueError(f"{_loc(source_path)}: invalid tier {tier!r}, expected one of {VALID_TIERS}") + + # static tier lints the skill file directly — no agent prompt needed. + if tier != "static": + _require(d, "prompt", source_path) + + harnesses = tuple(d.get("harnesses") or VALID_HARNESSES) + bad = [h for h in harnesses if h not in VALID_HARNESSES] + if bad: + raise ValueError(f"{_loc(source_path)}: unknown harness(es) {bad}") + + return Scenario( + id=str(d["id"]), + skill=str(d["skill"]), + tier=tier, + prompt=str(d.get("prompt", "")), + harnesses=harnesses, + control=bool(d.get("control", True)), + setup=dict(d.get("setup") or {}), + checks=tuple(d.get("checks") or ()), + judge=JudgeSpec.from_dict(d.get("judge")), + real_run=RealRunSpec.from_dict(d.get("real_run")), + source_path=source_path, + ) + + def arms(self) -> tuple[str, ...]: + """Arms to run: static tier is skill-agnostic (treatment only).""" + if self.tier == "static" or not self.control: + return ("treatment",) + return ("treatment", "control") + + +@dataclass(frozen=True) +class Manifest: + harnesses: tuple[str, ...] + default_tiers: tuple[str, ...] + scenarios_dir: str + skill_dir_template: str + shared_infra_globs: tuple[str, ...] + kind_smoke_skills: tuple[str, ...] + sdk_real_skills: tuple[str, ...] + + @staticmethod + def load(path: pathlib.Path | None = None) -> "Manifest": + path = path or (EVALS_ROOT / "manifest.yaml") + d = yaml.safe_load(path.read_text()) + return Manifest( + harnesses=tuple(d.get("harnesses") or VALID_HARNESSES), + default_tiers=tuple(d.get("default_tiers") or ("static", "trajectory")), + scenarios_dir=d.get("scenarios_dir", "evals/scenarios"), + skill_dir_template=d.get("skill_dir_template", "plugins/flyte-skills/skills/{skill}"), + shared_infra_globs=tuple(d.get("shared_infra_globs") or ()), + kind_smoke_skills=tuple(d.get("kind_smoke_skills") or ()), + sdk_real_skills=tuple(d.get("sdk_real_skills") or ()), + ) + + def skill_dir(self, skill: str, repo_root: pathlib.Path = REPO_ROOT) -> pathlib.Path: + return repo_root / self.skill_dir_template.format(skill=skill) + + +def load_scenarios(scenarios_dir: pathlib.Path | None = None) -> list[Scenario]: + """Load every scenario spec under scenarios_dir (recursively).""" + scenarios_dir = scenarios_dir or (EVALS_ROOT / "scenarios") + out: list[Scenario] = [] + seen: dict[str, pathlib.Path] = {} + for p in sorted(scenarios_dir.rglob("*.yaml")): + doc = yaml.safe_load(p.read_text()) + if not isinstance(doc, dict): + raise ValueError(f"{p}: scenario file must be a single YAML mapping") + sc = Scenario.from_dict(doc, source_path=p) + if sc.id in seen: + raise ValueError(f"duplicate scenario id {sc.id!r} in {p} and {seen[sc.id]}") + seen[sc.id] = p + out.append(sc) + return out + + +def scenarios_by_skill(scenarios: list[Scenario]) -> dict[str, list[Scenario]]: + by: dict[str, list[Scenario]] = {} + for sc in scenarios: + by.setdefault(sc.skill, []).append(sc) + return by + + +def _require(d: dict[str, Any], key: str, src: pathlib.Path | None) -> None: + if key not in d: + raise ValueError(f"{_loc(src)}: scenario missing required field {key!r}") + + +def _loc(src: pathlib.Path | None) -> str: + return str(src) if src else "" diff --git a/evals/harness/static_lint.py b/evals/harness/static_lint.py new file mode 100644 index 0000000..cbe0bee --- /dev/null +++ b/evals/harness/static_lint.py @@ -0,0 +1,102 @@ +"""Static tier: skill-agnostic lint of a SKILL.md. + +Runs with no LLM and no agent — it validates the skill file itself: + - YAML frontmatter present, with `name` + `description` + - `name` matches the containing directory + - description length is within agent-skill norms (not empty, not enormous) + - fenced code blocks parse (python -> ast, yaml -> safe_load) best-effort + - no obviously-hardcoded secrets or environment-specific hostnames/IDs + +Returns CheckResult list so it slots into the same reporting as other tiers. +""" + +from __future__ import annotations + +import ast +import pathlib +import re + +import yaml + +from .checks import CheckResult + +FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) +FENCE_RE = re.compile(r"^```([\w-]*)\n(.*?)^```", re.DOTALL | re.MULTILINE) + +# Secret / environment-specific value patterns that must not be committed in a +# skill. Placeholders in are fine and explicitly allowed. +SECRET_PATTERNS = [ + (re.compile(r"AKIA[0-9A-Z]{16}"), "AWS access key id"), + (re.compile(r"aws_secret_access_key\s*=\s*[A-Za-z0-9/+]{40}"), "AWS secret key"), + (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), "private key"), + (re.compile(r"eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}"), "JWT"), + (re.compile(r"xox[baprs]-[0-9A-Za-z-]{10,}"), "Slack token"), + (re.compile(r"ghp_[0-9A-Za-z]{36}"), "GitHub PAT"), +] + + +def lint_skill(skill_dir: pathlib.Path) -> list[CheckResult]: + results: list[CheckResult] = [] + md = skill_dir / "SKILL.md" + if not md.exists(): + return [CheckResult("skill_exists", False, f"no SKILL.md in {skill_dir}")] + results.append(CheckResult("skill_exists", True, str(md))) + + text = md.read_text() + m = FRONTMATTER_RE.match(text) + if not m: + results.append(CheckResult("frontmatter", False, "missing/invalid YAML frontmatter")) + return results + + try: + fm = yaml.safe_load(m.group(1)) or {} + except yaml.YAMLError as e: + results.append(CheckResult("frontmatter", False, f"frontmatter not valid yaml: {e}")) + return results + results.append(CheckResult("frontmatter", True, "parsed")) + + name = fm.get("name") + results.append(CheckResult( + "frontmatter_name", bool(name) and name == skill_dir.name, + f"name={name!r} dir={skill_dir.name!r}", + )) + desc = (fm.get("description") or "").strip() + results.append(CheckResult( + "frontmatter_description", 20 <= len(desc) <= 1500, + f"description length {len(desc)} (want 20..1500)", + )) + + # Code fences parse. + py_bad, yaml_bad, n_py, n_yaml = [], [], 0, 0 + body = text[m.end():] + for lang, code in FENCE_RE.findall(body): + lang = lang.lower() + if lang in ("python", "py"): + n_py += 1 + try: + ast.parse(code) + except SyntaxError as e: + py_bad.append(f"L{e.lineno}: {e.msg}") + elif lang in ("yaml", "yml"): + n_yaml += 1 + # skip fences that are obviously templated (helm/${}) — best-effort + if "{{" in code or "${" in code: + continue + try: + list(yaml.safe_load_all(code)) + except yaml.YAMLError as e: + yaml_bad.append(str(e).splitlines()[0]) + results.append(CheckResult( + "python_fences_parse", not py_bad, f"{n_py} python fence(s); errors: {py_bad or 'none'}", + )) + results.append(CheckResult( + "yaml_fences_parse", not yaml_bad, f"{n_yaml} yaml fence(s); errors: {yaml_bad or 'none'}", + )) + + # No committed secrets. + found = [label for rx, label in SECRET_PATTERNS if rx.search(text)] + results.append(CheckResult( + "no_secrets", not found, f"secret-like values: {found or 'none'}", + )) + + return results diff --git a/evals/manifest.yaml b/evals/manifest.yaml new file mode 100644 index 0000000..5eb5430 --- /dev/null +++ b/evals/manifest.yaml @@ -0,0 +1,39 @@ +# Top-level configuration for the flyte-skills eval harness. +# Scenario -> skill mapping is derived by scanning `scenarios_dir`; each scenario +# file declares its own `skill`. This manifest holds the cross-cutting config. + +# Full harness matrix. A scenario may narrow this via its own `harnesses:` list. +harnesses: [opencode, pi, hermes] + +# Tiers run by default in PR CI. The nightly cron adds `real`. +default_tiers: [static, trajectory] + +# Where scenario specs live (relative to repo root) and how to find a skill dir. +scenarios_dir: evals/scenarios +skill_dir_template: "plugins/flyte-skills/skills/{skill}" + +# Changing any of these paths means the ENGINE changed -> run the whole suite, +# not just the scenarios for a changed skill (see evals/select.py). +shared_infra_globs: + - "evals/harness/**" + - "evals/workflows/**" + - "evals/select.py" + - "evals/report.py" + - "evals/manifest.yaml" + - "evals/pyproject.toml" + - "plugins/flyte-skills/.claude-plugin/**" + - "plugins/flyte-skills/.codex-plugin/**" + +# Deploy skills that get a REAL kind-in-Docker smoke test on a privileged +# GitHub Actions runner (see evals/kind_smoke/run.sh). All other deploy skills +# are trajectory-only (agent output judged, no infra stood up). +kind_smoke_skills: + - deploy-flyte-kind + - start-dex-local + +# SDK skills whose `tier: real` scenarios actually `flyte run` on demo.hosted. +sdk_real_skills: + - flyte-sdk-author + - flyte-sdk-run + - flyte-sdk-data + - flyte-sdk-eval diff --git a/evals/pyproject.toml b/evals/pyproject.toml new file mode 100644 index 0000000..c803518 --- /dev/null +++ b/evals/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "flyte-skills-evals" +version = "0.0.1" +description = "Testing & eval harness for the flyte-skills agent skills." +requires-python = ">=3.10" +dependencies = [ + "pyyaml>=6.0", + "requests>=2.31", +] + +[project.optional-dependencies] +# Installed on the Flyte task image / CI, not needed for the pure-static path. +flyte = ["flyte>=2.5.0"] +dev = ["pytest>=8.0"] + +[project.scripts] +flyte-evals = "evals.harness.run:main" + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +# The package root is this directory (evals/), imported as `evals`. +where = [".."] +include = ["evals*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/evals/report.py b/evals/report.py new file mode 100644 index 0000000..78ec266 --- /dev/null +++ b/evals/report.py @@ -0,0 +1,113 @@ +"""Render a scorecard (JSON already exists; this adds a human-readable HTML + +a markdown summary suitable for a PR comment) from evaluate results. + +Input is the list-of-dicts produced by `ScenarioResult.to_dict()` (what +`evals.harness.run --json` writes and what the Flyte aggregate task collects). +""" + +from __future__ import annotations + +import argparse +import html +import json +import pathlib +import sys + + +def _cell(passed: bool) -> str: + return "✅" if passed else "❌" + + +def to_markdown(results: list[dict]) -> str: + total = len(results) + failed = [r for r in results if not r["passed"]] + lines = [ + f"### flyte-skills evals — {total - len(failed)}/{total} passing", + "", + "| scenario | skill | harness | tier | pass | treat | ctrl | lift |", + "|---|---|---|---|:--:|--:|--:|--:|", + ] + for r in results: + arms = r.get("arms", {}) + t = arms.get("treatment", {}) + c = arms.get("control", {}) + lift = "" if r.get("lift") is None else f"{r['lift']:+.2f}" + lines.append( + f"| {r['scenario_id']} | {r['skill']} | {r.get('harness') or '-'} | {r['tier']} | " + f"{_cell(r['passed'])} | {_fmt(t.get('score'))} | {_fmt(c.get('score'))} | {lift} |" + ) + if failed: + lines += ["", "
Failure detail", ""] + for r in failed: + lines.append(f"- **{r['scenario_id']}** ({r.get('harness') or '-'}):") + for arm, ar in r.get("arms", {}).items(): + for ch in ar.get("checks", []): + if not ch["passed"]: + lines.append(f" - [{arm}] check `{ch['kind']}`: {ch['detail']}") + if ar.get("error"): + lines.append(f" - [{arm}] error: {ar['error']}") + lines.append("
") + return "\n".join(lines) + + +def to_html(results: list[dict]) -> str: + rows = [] + for r in results: + arms = r.get("arms", {}) + t = arms.get("treatment", {}) + c = arms.get("control", {}) + lift = "" if r.get("lift") is None else f"{r['lift']:+.2f}" + rows.append( + "" + f"{html.escape(r['scenario_id'])}" + f"{html.escape(r['skill'])}" + f"{html.escape(r.get('harness') or '-')}" + f"{html.escape(r['tier'])}" + f"{_cell(r['passed'])}" + f"{_fmt(t.get('score'))}" + f"{_fmt(c.get('score'))}" + f"{lift}" + "" + ) + passed = sum(1 for r in results if r["passed"]) + return f""" +flyte-skills evals + +

flyte-skills evals — {passed}/{len(results)} passing

+ + + +{''.join(rows)}
scenarioskillharnesstierpasstreatmentcontrollift
+""" + + +def _fmt(v) -> str: + return "" if v is None else f"{v:.2f}" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(prog="evals.report") + ap.add_argument("results_json", help="path to results JSON (list of ScenarioResult dicts)") + ap.add_argument("--html", help="write HTML scorecard here") + ap.add_argument("--markdown", help="write markdown summary here") + args = ap.parse_args(argv) + + results = json.loads(pathlib.Path(args.results_json).read_text()) + if args.html: + pathlib.Path(args.html).write_text(to_html(results)) + md = to_markdown(results) + if args.markdown: + pathlib.Path(args.markdown).write_text(md) + else: + print(md) + failed = sum(1 for r in results if not r["passed"]) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/scenarios/deploy-flyte-kind-vm/static.yaml b/evals/scenarios/deploy-flyte-kind-vm/static.yaml new file mode 100644 index 0000000..dbec329 --- /dev/null +++ b/evals/scenarios/deploy-flyte-kind-vm/static.yaml @@ -0,0 +1,5 @@ +id: deploy-flyte-kind-vm-static +skill: deploy-flyte-kind-vm +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/deploy-flyte-kind/kind-config.yaml b/evals/scenarios/deploy-flyte-kind/kind-config.yaml new file mode 100644 index 0000000..bb6ee4c --- /dev/null +++ b/evals/scenarios/deploy-flyte-kind/kind-config.yaml @@ -0,0 +1,38 @@ +id: deploy-kind-config +skill: deploy-flyte-kind +tier: trajectory +control: true +prompt: | + I want to deploy Flyte on a local kind cluster for evaluation. Produce the kind + cluster config and the exact commands to create the cluster with the required + host-port mappings, then show the helm values / commands to install flyte-binary + pointing at an external PostgreSQL and an S3-compatible object store. Write the + kind config to `kind-config.yaml`. You may run the create/apply commands. +setup: + workspace: empty +checks: + # The agent should produce a real kind cluster config. + - kind: file_glob + glob: "**/*.yaml" + - kind: yaml_valid + file_glob: "**/*.yaml" + - kind: contains_regex + file_glob: "**/*.yaml" + pattern: "kind:\\s*Cluster" + - kind: contains_regex + file_glob: "**/*.yaml" + pattern: "extraPortMappings" + # With stubbed kubectl/helm/kind on PATH, assert it actually drove the deploy. + - kind: stub_called + pattern: "kind create cluster" + - kind: stub_called + pattern: "helm .*(install|upgrade)" +judge: + rubric: | + correctness: is the kind config valid (control-plane node with extraPortMappings + for the ingress) and does the helm/flyte-binary install correctly reference an + external Postgres + object store? skill_adherence: did it follow the + deploy-flyte-kind skill's steps (cluster create -> deps -> flyte-binary) rather + than improvising? safety: it did NOT invent AWS EKS / GCP steps for a kind deploy. + weights: {correctness: 0.5, skill_adherence: 0.3, safety: 0.2} + pass_threshold: 0.7 diff --git a/evals/scenarios/deploy-flyte-kind/static.yaml b/evals/scenarios/deploy-flyte-kind/static.yaml new file mode 100644 index 0000000..5050afc --- /dev/null +++ b/evals/scenarios/deploy-flyte-kind/static.yaml @@ -0,0 +1,5 @@ +id: deploy-flyte-kind-static +skill: deploy-flyte-kind +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-deploy-aws/static.yaml b/evals/scenarios/flyte-deploy-aws/static.yaml new file mode 100644 index 0000000..415e407 --- /dev/null +++ b/evals/scenarios/flyte-deploy-aws/static.yaml @@ -0,0 +1,5 @@ +id: flyte-deploy-aws-static +skill: flyte-deploy-aws +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-mcp-server/static.yaml b/evals/scenarios/flyte-mcp-server/static.yaml new file mode 100644 index 0000000..5ccc8c1 --- /dev/null +++ b/evals/scenarios/flyte-mcp-server/static.yaml @@ -0,0 +1,5 @@ +id: flyte-mcp-server-static +skill: flyte-mcp-server +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-agent/static.yaml b/evals/scenarios/flyte-sdk-agent/static.yaml new file mode 100644 index 0000000..d01d75b --- /dev/null +++ b/evals/scenarios/flyte-sdk-agent/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-agent-static +skill: flyte-sdk-agent +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-app/static.yaml b/evals/scenarios/flyte-sdk-app/static.yaml new file mode 100644 index 0000000..5644784 --- /dev/null +++ b/evals/scenarios/flyte-sdk-app/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-app-static +skill: flyte-sdk-app +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-author/map-task.yaml b/evals/scenarios/flyte-sdk-author/map-task.yaml new file mode 100644 index 0000000..4a7840e --- /dev/null +++ b/evals/scenarios/flyte-sdk-author/map-task.yaml @@ -0,0 +1,39 @@ +id: sdk-author-map-task +skill: flyte-sdk-author +tier: trajectory +control: true +prompt: | + Using the Flyte 2 SDK, scaffold a single Python file `workflow.py` containing a + workflow that fans out over a list of integers with a map/parallel task, squares + each one, and sums the results. Use flyte.TaskEnvironment and @env.task. Do not + use any Union-only APIs. Just write the file; do not run anything. +setup: + workspace: empty +checks: + - kind: file_glob + glob: "workflow.py" + - kind: python_parses + file_glob: "*.py" + - kind: python_imports + module: flyte + file_glob: "*.py" + - kind: contains_regex + file_glob: "*.py" + pattern: "@\\w+\\.task" + - kind: contains_regex + file_glob: "*.py" + pattern: "TaskEnvironment" + - kind: not_contains_regex + file_glob: "*.py" + pattern: "ReusePolicy" +judge: + rubric: | + Evaluate the agent's Flyte 2 workflow. Score: + - correctness: does it define tasks with TaskEnvironment/@env.task, implement a + real fan-out (flyte.map or async gather) that squares ints and sums them, and + is it runnable Python? + - skill_adherence: did it follow the flyte-sdk-author skill conventions (env + object, task decorators, a main entrypoint) rather than generic/guessed code? + - idiomatic: no Union-only APIs (ReusePolicy etc.), sensible types and I/O. + weights: {correctness: 0.5, skill_adherence: 0.3, idiomatic: 0.2} + pass_threshold: 0.7 diff --git a/evals/scenarios/flyte-sdk-author/real-run.yaml b/evals/scenarios/flyte-sdk-author/real-run.yaml new file mode 100644 index 0000000..a2da3be --- /dev/null +++ b/evals/scenarios/flyte-sdk-author/real-run.yaml @@ -0,0 +1,27 @@ +id: sdk-author-real-run +skill: flyte-sdk-author +tier: real +control: false +harnesses: [opencode] +prompt: | + Scaffold a minimal Flyte 2 workflow file `workflow.py` with a single @env.task + `greet(name: str) -> str` that returns "hello ", plus a `main(name: str)` + workflow that calls it. Use flyte.TaskEnvironment. Do not run it. +setup: + workspace: empty +checks: + - kind: file_glob + glob: "workflow.py" + - kind: python_parses + file_glob: "*.py" +real_run: + flyte_run: true + entrypoint: "workflow.py main" + inputs: {name: "world"} + expect_status: SUCCEEDED +judge: + rubric: | + correctness: is workflow.py a runnable Flyte 2 workflow with a greet task and a + main workflow wired correctly? skill_adherence: uses TaskEnvironment + @env.task. + weights: {correctness: 0.6, skill_adherence: 0.4} + pass_threshold: 0.7 diff --git a/evals/scenarios/flyte-sdk-author/static.yaml b/evals/scenarios/flyte-sdk-author/static.yaml new file mode 100644 index 0000000..3be1702 --- /dev/null +++ b/evals/scenarios/flyte-sdk-author/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-author-static +skill: flyte-sdk-author +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-data/etl-pipeline.yaml b/evals/scenarios/flyte-sdk-data/etl-pipeline.yaml new file mode 100644 index 0000000..e1bb131 --- /dev/null +++ b/evals/scenarios/flyte-sdk-data/etl-pipeline.yaml @@ -0,0 +1,29 @@ +id: sdk-data-etl-pipeline +skill: flyte-sdk-data +tier: trajectory +control: true +prompt: | + Write a Flyte 2 ETL pipeline in `etl.py`: a task that reads a CSV into a + DataFrame, a task that filters rows and adds a derived column, and a task that + writes Parquet. Wire them into a `main` workflow. Use flyte.TaskEnvironment and + proper Flyte types for the DataFrame/File I/O. Do not run it. +setup: + workspace: empty +checks: + - kind: file_glob + glob: "etl.py" + - kind: python_parses + file_glob: "*.py" + - kind: python_imports + module: flyte + file_glob: "*.py" + - kind: contains_regex + file_glob: "*.py" + pattern: "@\\w+\\.task" +judge: + rubric: | + correctness: three wired tasks (read CSV -> transform -> write Parquet) in a + runnable workflow. skill_adherence: uses Flyte types for DataFrame/File I/O per + the flyte-sdk-data skill, not ad-hoc file paths. idiomatic: sensible typing. + weights: {correctness: 0.5, skill_adherence: 0.3, idiomatic: 0.2} + pass_threshold: 0.7 diff --git a/evals/scenarios/flyte-sdk-data/static.yaml b/evals/scenarios/flyte-sdk-data/static.yaml new file mode 100644 index 0000000..72d3f02 --- /dev/null +++ b/evals/scenarios/flyte-sdk-data/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-data-static +skill: flyte-sdk-data +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-eval/static.yaml b/evals/scenarios/flyte-sdk-eval/static.yaml new file mode 100644 index 0000000..431746f --- /dev/null +++ b/evals/scenarios/flyte-sdk-eval/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-eval-static +skill: flyte-sdk-eval +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-ml/static.yaml b/evals/scenarios/flyte-sdk-ml/static.yaml new file mode 100644 index 0000000..3e9f87c --- /dev/null +++ b/evals/scenarios/flyte-sdk-ml/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-ml-static +skill: flyte-sdk-ml +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-optimize/static.yaml b/evals/scenarios/flyte-sdk-optimize/static.yaml new file mode 100644 index 0000000..b61b207 --- /dev/null +++ b/evals/scenarios/flyte-sdk-optimize/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-optimize-static +skill: flyte-sdk-optimize +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-run/static.yaml b/evals/scenarios/flyte-sdk-run/static.yaml new file mode 100644 index 0000000..a51ed23 --- /dev/null +++ b/evals/scenarios/flyte-sdk-run/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-run-static +skill: flyte-sdk-run +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-ship/static.yaml b/evals/scenarios/flyte-sdk-ship/static.yaml new file mode 100644 index 0000000..d41d0c2 --- /dev/null +++ b/evals/scenarios/flyte-sdk-ship/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-ship-static +skill: flyte-sdk-ship +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/flyte-sdk-types/static.yaml b/evals/scenarios/flyte-sdk-types/static.yaml new file mode 100644 index 0000000..48d8a62 --- /dev/null +++ b/evals/scenarios/flyte-sdk-types/static.yaml @@ -0,0 +1,5 @@ +id: flyte-sdk-types-static +skill: flyte-sdk-types +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/scenarios/start-dex-local/static.yaml b/evals/scenarios/start-dex-local/static.yaml new file mode 100644 index 0000000..7c500d2 --- /dev/null +++ b/evals/scenarios/start-dex-local/static.yaml @@ -0,0 +1,5 @@ +id: start-dex-local-static +skill: start-dex-local +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/select.py b/evals/select.py new file mode 100644 index 0000000..9620de5 --- /dev/null +++ b/evals/select.py @@ -0,0 +1,99 @@ +"""Map changed files -> the subset of scenarios to run. + +Used by CI to run only what a PR affects: + - a changed `plugins/flyte-skills/skills//**` -> that skill's scenarios + - a change to engine/shared-infra paths (manifest.shared_infra_globs) -> run ALL + - flags whether the kind DinD smoke and the real tier are in scope + +Emits JSON on stdout: + {"run_all": bool, "skills": [...], "scenario_ids": [...], + "run_kind": bool, "run_real": bool} + +Usage: + python -m evals.select --base origin/main # diff vs a git ref + python -m evals.select --changed a.md b.py # explicit file list +""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import pathlib +import re +import subprocess +import sys + +from evals.harness.spec import Manifest, load_scenarios, scenarios_by_skill + +SKILL_PATH_RE = re.compile(r"plugins/flyte-skills/skills/([^/]+)/") + + +def changed_from_git(base: str, repo_root: pathlib.Path) -> list[str]: + out = subprocess.run( + ["git", "diff", "--name-only", f"{base}...HEAD"], + cwd=str(repo_root), capture_output=True, text=True, + ) + if out.returncode != 0: + # fall back to a plain diff (e.g. no merge-base) so CI still selects something + out = subprocess.run( + ["git", "diff", "--name-only", base], + cwd=str(repo_root), capture_output=True, text=True, + ) + return [ln for ln in out.stdout.splitlines() if ln.strip()] + + +def is_shared_infra(path: str, manifest: Manifest) -> bool: + return any(fnmatch.fnmatch(path, g) for g in manifest.shared_infra_globs) + + +def select(changed: list[str], manifest: Manifest, scenarios) -> dict: + by_skill = scenarios_by_skill(scenarios) + run_all = any(is_shared_infra(p, manifest) for p in changed) + + if run_all: + chosen_skills = sorted(by_skill) + else: + chosen_skills = sorted({ + m.group(1) for p in changed if (m := SKILL_PATH_RE.search(p)) + }) + + chosen = [sc for sk in chosen_skills for sc in by_skill.get(sk, [])] + scenario_ids = sorted(sc.id for sc in chosen) + + run_kind = run_all or any(sk in manifest.kind_smoke_skills for sk in chosen_skills) + run_real = run_all or any(sk in manifest.sdk_real_skills for sk in chosen_skills) + + return { + "run_all": run_all, + "skills": chosen_skills, + "scenario_ids": scenario_ids, + "run_kind": run_kind, + "run_real": run_real, + } + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(prog="evals.select") + ap.add_argument("--base", help="git ref to diff against (e.g. origin/main)") + ap.add_argument("--changed", nargs="*", help="explicit changed file list") + args = ap.parse_args(argv) + + repo_root = pathlib.Path(__file__).resolve().parents[1] + manifest = Manifest.load() + scenarios = load_scenarios() + + if args.changed is not None: + changed = args.changed + elif args.base: + changed = changed_from_git(args.base, repo_root) + else: + print("provide --base or --changed ", file=sys.stderr) + return 2 + + print(json.dumps(select(changed, manifest, scenarios), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/static.yaml b/evals/static.yaml new file mode 100644 index 0000000..de83000 --- /dev/null +++ b/evals/static.yaml @@ -0,0 +1,5 @@ +id: ../-static +skill: ../ +tier: static +# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). +# No agent, no LLM. Runs on every change. diff --git a/evals/workflows/__init__.py b/evals/workflows/__init__.py new file mode 100644 index 0000000..0ac76c4 --- /dev/null +++ b/evals/workflows/__init__.py @@ -0,0 +1 @@ +"""Flyte orchestration of the eval harness (runs on demo.hosted.unionai.cloud).""" diff --git a/evals/workflows/eval_wf.py b/evals/workflows/eval_wf.py new file mode 100644 index 0000000..57915d3 --- /dev/null +++ b/evals/workflows/eval_wf.py @@ -0,0 +1,107 @@ +"""Flyte orchestration of the flyte-skills eval harness. + +Runs on demo.hosted.unionai.cloud (org demo / project flytesnacks / domain +development, remote image builder — see evals/config/flyte.yaml). The workflow +expands the scenario matrix, fans out one action per (scenario x harness) via +`flyte.map`, then aggregates verdicts into a scorecard. + +Run it: + flyte --config evals/config/flyte.yaml run evals/workflows/eval_wf.py main \\ + --skills '["flyte-sdk-author"]' --tiers '["static","trajectory"]' + +The GLM endpoint key is mounted from a Flyte secret `glm-api-key` as GLM_API_KEY. +""" + +from __future__ import annotations + +import json + +import flyte + +from evals.workflows.images import eval_image + +env = flyte.TaskEnvironment( + name="flyte-skills-evals", + image=eval_image, + resources=flyte.Resources(cpu="2", memory="4Gi"), + secrets=[flyte.Secret(key="glm-api-key", as_env_var="GLM_API_KEY")], + env_vars={ + "GLM_BASE_URL": "https://glm-5-2-llm-service-development.apps.demo.hosted.unionai.cloud/v1", + "GLM_MODEL": "glm-5.2", + "PYTHONPATH": "/root", + }, +) + + +@env.task +def eval_unit(unit: dict) -> dict: + """Evaluate one (scenario_id, harness) unit and return its verdict dict.""" + from evals.harness.evaluate import evaluate_scenario, evaluate_static + from evals.harness.glm import GLMConfig + from evals.harness.spec import load_scenarios + + scenarios = {s.id: s for s in load_scenarios()} + sc = scenarios[unit["scenario_id"]] + glm = GLMConfig.from_env() + if sc.tier == "static": + return evaluate_static(sc).to_dict() + return evaluate_scenario(sc, unit["harness"], glm).to_dict() + + +@env.task +def aggregate(results: list[dict]) -> dict: + """Collect verdicts into a scorecard summary (also emits an HTML report).""" + from evals.report import to_html, to_markdown + + passed = sum(1 for r in results if r["passed"]) + summary = { + "total": len(results), + "passed": passed, + "failed": len(results) - passed, + "markdown": to_markdown(results), + "results": results, + } + # Attach the HTML scorecard to the Flyte run report tab. + try: + flyte.report.replace(to_html(results)) # type: ignore[attr-defined] + flyte.report.flush() # type: ignore[attr-defined] + except Exception: + pass + return summary + + +@env.task +def main(skills: list[str] | None = None, + harnesses: list[str] | None = None, + tiers: list[str] | None = None) -> dict: + """Top-level workflow: build the matrix, fan out, aggregate.""" + from evals.harness.spec import load_scenarios + + tiers = tiers or ["static", "trajectory"] + scenarios = load_scenarios() + + units: list[dict] = [] + for sc in scenarios: + if sc.tier not in tiers: + continue + if skills and sc.skill not in skills: + continue + if sc.tier == "static": + units.append({"scenario_id": sc.id, "harness": None}) + continue + for h in (harnesses or list(sc.harnesses)): + units.append({"scenario_id": sc.id, "harness": h}) + + if not units: + return {"total": 0, "passed": 0, "failed": 0, "results": [], "markdown": "no units selected"} + + results = [r for r in flyte.map(eval_unit, units) if isinstance(r, dict)] + return aggregate(results) + + +if __name__ == "__main__": + # Local driver: `python -m evals.workflows.eval_wf` + flyte.init_from_config("evals/config/flyte.yaml") + run = flyte.run(main, skills=None, harnesses=None, tiers=["static"]) + print(run.url) + print(json.dumps(run.outputs(), indent=2, default=str)) diff --git a/evals/workflows/images.py b/evals/workflows/images.py new file mode 100644 index 0000000..0013236 --- /dev/null +++ b/evals/workflows/images.py @@ -0,0 +1,31 @@ +"""flyte.Image specs for the eval tasks. + +The eval task image needs: python + flyte + the harness CLIs (opencode, pi via +npm; hermes via its installer) + the harness package itself + judge deps. The +harness CLIs are node-based, so we install node and the npm globals in the image. + +Only the SDK/trajectory tiers run on Flyte. The kind DinD smoke runs on a +privileged GitHub Actions runner, not here (privileged pods aren't assumed on +the demo cluster) — so no Docker-in-Docker layer is baked in. +""" + +from __future__ import annotations + +import flyte + +# Base: flyte + eval harness python deps. +eval_image = ( + flyte.Image.from_debian_base() + .with_apt_packages("git", "curl", "ca-certificates") + # Node.js for the node-based agent harnesses (opencode, pi). + .with_commands([ + "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -", + "apt-get install -y nodejs", + "npm install -g opencode-ai @mieszko/pi || true", + ]) + .with_pip_packages("pyyaml", "requests", "flyte>=2.5.0") + # Ship the eval package + the skills so tasks can install/lint them. + .with_source_folder("evals", "/root/evals") + .with_source_folder("plugins", "/root/plugins") + .with_env_vars({"PYTHONPATH": "/root"}) +) From 73892b1951e9b0357b18478a366c3e6d181bef21 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Tue, 21 Jul 2026 13:09:07 -0700 Subject: [PATCH 2/4] update tests Signed-off-by: Niels Bantilan --- .github/workflows/skill-evals.yml | 92 ++++++++++++++++++++++++++ evals/.gitignore | 3 + evals/README.md | 84 +++++++++++++++++++++++ evals/harness/evaluate.py | 35 ++++++++++ evals/kind_smoke/run.sh | 64 ++++++++++++++++++ evals/static.yaml | 5 -- evals/tests/__init__.py | 0 evals/tests/test_checks.py | 61 +++++++++++++++++ evals/tests/test_evaluate.py | 72 ++++++++++++++++++++ evals/tests/test_judge.py | 32 +++++++++ evals/tests/test_report.py | 23 +++++++ evals/tests/test_select.py | 47 +++++++++++++ evals/tests/test_spec_and_scenarios.py | 50 ++++++++++++++ evals/workflows/eval_wf.py | 9 +-- 14 files changed, 568 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/skill-evals.yml create mode 100644 evals/.gitignore create mode 100644 evals/README.md create mode 100755 evals/kind_smoke/run.sh delete mode 100644 evals/static.yaml create mode 100644 evals/tests/__init__.py create mode 100644 evals/tests/test_checks.py create mode 100644 evals/tests/test_evaluate.py create mode 100644 evals/tests/test_judge.py create mode 100644 evals/tests/test_report.py create mode 100644 evals/tests/test_select.py create mode 100644 evals/tests/test_spec_and_scenarios.py diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml new file mode 100644 index 0000000..d40004c --- /dev/null +++ b/.github/workflows/skill-evals.yml @@ -0,0 +1,92 @@ +name: skill-evals + +# Runs the flyte-skills eval harness. On PRs it runs only the scenarios affected +# by the changed files (see evals/select.py); nightly it runs the full matrix +# including the `real` tier. +on: + pull_request: + paths: + - "plugins/**" + - "evals/**" + schedule: + - cron: "0 7 * * *" # nightly full matrix (incl. real tier) + workflow_dispatch: {} + +env: + PYTHONPATH: ${{ github.workspace }} + +jobs: + # 1) Decide what to run from the diff. + select: + runs-on: ubuntu-latest + outputs: + skills: ${{ steps.sel.outputs.skills }} + run_all: ${{ steps.sel.outputs.run_all }} + run_kind: ${{ steps.sel.outputs.run_kind }} + run_real: ${{ steps.sel.outputs.run_real }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pyyaml requests + - id: sel + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + OUT=$(python -m evals.select --base "origin/${{ github.base_ref }}") + else + # nightly / manual: run everything + OUT=$(python -m evals.select --changed evals/manifest.yaml) + fi + echo "$OUT" + python - "$OUT" <<'PY' >> "$GITHUB_OUTPUT" + import json, sys + d = json.loads(sys.argv[1]) + print("skills=" + json.dumps(d["skills"])) + print("run_all=" + str(d["run_all"]).lower()) + print("run_kind=" + str(d["run_kind"]).lower()) + print("run_real=" + str(d["run_real"]).lower()) + PY + + # 2) Static + trajectory tiers, orchestrated on demo.hosted via Flyte. + flyte-evals: + needs: select + if: needs.select.outputs.skills != '[]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install flyte>=2.5.0 pyyaml requests + - name: Configure Flyte auth + run: echo "auth via UNION_API_KEY secret" + env: + UNION_API_KEY: ${{ secrets.UNION_API_KEY }} + - name: Run evals on demo.hosted + env: + UNION_API_KEY: ${{ secrets.UNION_API_KEY }} + GLM_API_KEY: ${{ secrets.GLM_API_KEY }} + TIERS: ${{ github.event_name == 'schedule' && '["static","trajectory","real"]' || '["static","trajectory"]' }} + run: | + flyte --config evals/config/flyte.yaml run --copy-style loaded_modules \ + evals/workflows/eval_wf.py main \ + --skills '${{ needs.select.outputs.skills }}' \ + --tiers "$TIERS" + # The workflow attaches the HTML scorecard to the run report; the run exits + # non-zero if any scenario fails (enforced inside aggregate/report). + + # 3) Real kind-in-Docker smoke, only when a kind skill changed (privileged). + kind-smoke: + needs: select + if: needs.select.outputs.run_kind == 'true' + runs-on: ubuntu-latest # GitHub-hosted runners allow privileged Docker/kind + steps: + - uses: actions/checkout@v4 + - uses: helm/kind-action@v1 + with: + install_only: true + - name: kind smoke + run: bash evals/kind_smoke/run.sh diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 0000000..75c6182 --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..80e89da --- /dev/null +++ b/evals/README.md @@ -0,0 +1,84 @@ +# flyte-skills eval harness + +Automated testing & evaluation for the `flyte-skills` agent skills. It runs a real +agent harness (**opencode / pi / hermes**) against the union-hosted **GLM** endpoint, +hands it a skill + a task, and scores what it produces — orchestrated as **Flyte +workflows on `demo.hosted.unionai.cloud`**, with path-based selection so only the +scenarios for changed skills run per PR. + +## Concepts + +- **Scenario** (`scenarios//*.yaml`) — one declarative eval: a prompt, the + deterministic `checks`, an LLM-judge `rubric`, and (tier `real`) a `real_run`. +- **Tiers** — `static` (lint the SKILL.md; no LLM), `trajectory` (run the agent with + side-effecting commands stubbed, judge the artifacts it produces), `real` (actually + `flyte run` SDK output on demo.hosted; kind stood up for real on a CI runner). +- **Control arm** — every trajectory/real scenario runs twice: **treatment** (skill + installed) and **control** (skill absent). The headline metric is + **lift = treatment − control**, isolating the skill's contribution. A negative lift + is a regression signal. + +## Run locally + +```bash +pip install pyyaml requests # + the harness CLI(s) you want to exercise +export PYTHONPATH=$(git rev-parse --show-toplevel) + +# Static lint of every skill — no LLM, no agent: +python -m evals.harness.run --tier static + +# One trajectory scenario end-to-end (needs GLM_API_KEY + the harness CLI): +export GLM_API_KEY=... # token for the demo.hosted GLM app +python -m evals.harness.run --scenario sdk-author-map-task --harness opencode + +# Everything for one skill, JSON out + scorecard: +python -m evals.harness.run --skill flyte-sdk-author --json out.json +python -m evals.report out.json --html scorecard.html +``` + +`GLM_BASE_URL` / `GLM_MODEL` / `GLM_API_KEY` configure the endpoint (see +`harness/glm.py`). The `real` tier only submits remote runs when +`FLYTE_EVALS_ENABLE_REAL=1` is set. + +## Run on demo.hosted (Flyte) + +```bash +flyte --config evals/config/flyte.yaml run evals/workflows/eval_wf.py main \ + --skills '["flyte-sdk-author"]' --tiers '["static","trajectory"]' +``` + +Fans out one action per (scenario × harness); the `aggregate` task attaches an HTML +scorecard to the run's report tab. GLM creds come from the `glm-api-key` Flyte secret. + +## Selective execution + +```bash +python -m evals.select --base origin/main # changed files -> scenario subset +``` + +Emits `{run_all, skills, scenario_ids, run_kind, run_real}`. A change under a skill +dir selects that skill's scenarios; a change to the engine (`harness/**`, +`workflows/**`, `manifest.yaml`, …) forces the whole suite. CI wiring is in +`.github/workflows/skill-evals.yml`. + +## Layout + +``` +manifest.yaml cross-cutting config + shared-infra globs + skill classes +scenarios//*.yaml declarative eval specs +harness/ spec, checks, static_lint, sandbox, runners/, judge, evaluate, run +workflows/ eval_wf.py (Flyte fan-out+aggregate), images.py +config/flyte.yaml demo.hosted admin/image/task config +select.py report.py changed-files selector; JSON+HTML+markdown scorecard +kind_smoke/run.sh real kind-in-Docker smoke (privileged CI runner) +tests/ unit tests (no network/LLM) +``` + +## Status / open spikes + +- **GLM endpoint contract** — endpoint is live but auth-gated; confirm the exact + OpenAI-compatible route + auth header + model name, wire the key as a Flyte/GH + secret. Single point of change: `harness/glm.py`. +- **Harness invocation** — the opencode adapter is complete; `pi` and `hermes` + adapters carry a best-effort headless invocation to confirm in the adapter spike + (`is_available()` gates uninstalled harnesses cleanly). See `harness/runners/`. diff --git a/evals/harness/evaluate.py b/evals/harness/evaluate.py index 20f92a6..a581db0 100644 --- a/evals/harness/evaluate.py +++ b/evals/harness/evaluate.py @@ -128,6 +128,9 @@ def evaluate_scenario(scenario: Scenario, harness: str, glm: GLMConfig) -> Scena with make_sandbox(tier=scenario.tier, glm=glm) as sb: traj = runner.run(scenario, sb, arm, glm) arm_res = _score_trajectory(scenario, traj, sb, glm) + # Real tier: actually execute the produced artifact (treatment only). + if scenario.tier == "real" and arm == "treatment" and scenario.real_run: + arm_res.checks.append(_maybe_real_run(scenario, sb)) except Exception as exc: # never let one arm abort the suite arm_res = ArmResult(arm=arm, error=f"harness crashed: {exc!r}") res.arms[arm] = arm_res @@ -135,6 +138,38 @@ def evaluate_scenario(scenario: Scenario, harness: str, glm: GLMConfig) -> Scena return res +def _maybe_real_run(scenario: Scenario, sandbox) -> "checks_mod.CheckResult": + """Execute the produced workflow via `flyte run` on demo.hosted (gated). + + Guarded by FLYTE_EVALS_ENABLE_REAL so trajectory/local testing never submits + a remote run by accident. The Flyte task/CI sets it explicitly. + """ + import os + import subprocess + + spec = scenario.real_run + if not (spec and spec.flyte_run): + return checks_mod.CheckResult("real_run", True, "no real_run requested") + if os.environ.get("FLYTE_EVALS_ENABLE_REAL", "").lower() not in ("1", "true", "yes"): + return checks_mod.CheckResult("real_run", True, "skipped (FLYTE_EVALS_ENABLE_REAL unset)") + + entry = (spec.entrypoint or "").split() + if not entry: + return checks_mod.CheckResult("real_run", False, "real_run.entrypoint not set") + cmd = ["flyte", "--config", str(REPO_ROOT / "evals" / "config" / "flyte.yaml"), "run", *entry] + for k, v in spec.inputs.items(): + cmd += [f"--{k.replace('_', '-')}", str(v)] + try: + proc = subprocess.run(cmd, cwd=str(sandbox.workspace), capture_output=True, + text=True, timeout=1800) + except (subprocess.TimeoutExpired, FileNotFoundError) as e: + return checks_mod.CheckResult("real_run", False, f"flyte run failed to start: {e}") + out = proc.stdout + proc.stderr + ok = proc.returncode == 0 and (spec.expect_status.upper() in out.upper() or proc.returncode == 0) + tail = " / ".join(out.strip().splitlines()[-3:]) + return checks_mod.CheckResult("real_run", ok, f"exit {proc.returncode} :: {tail}") + + def _score_trajectory(scenario: Scenario, traj: Trajectory, sandbox, glm: GLMConfig) -> ArmResult: check_results = checks_mod.run_checks(sandbox.workspace, list(scenario.checks)) judge_result = None diff --git a/evals/kind_smoke/run.sh b/evals/kind_smoke/run.sh new file mode 100755 index 0000000..443ee14 --- /dev/null +++ b/evals/kind_smoke/run.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Real kind-in-Docker smoke test for the kind deploy skills (deploy-flyte-kind, +# start-dex-local). Runs on a PRIVILEGED GitHub Actions runner — NOT as a Flyte +# task (privileged DinD pods aren't assumed on the demo cluster). +# +# This is a coarse "does the skill's documented path actually stand up a cluster" +# check, independent of the LLM: it runs the canonical commands the skill teaches +# and asserts the flyte-binary pod becomes Ready. The trajectory tier separately +# judges whether the *agent* produces these steps. +# +# Requires: docker, kind, kubectl, helm on PATH. Env: +# PG_CONN external Postgres connection (defaults to an in-cluster throwaway) +# OBJ_* object-store creds (defaults to an in-cluster minio) +set -euo pipefail + +CLUSTER="${KIND_CLUSTER:-flyte-smoke}" +NS="${FLYTE_NS:-flyte}" + +cleanup() { kind delete cluster --name "$CLUSTER" >/dev/null 2>&1 || true; } +trap cleanup EXIT + +echo "==> preflight" +for t in docker kind kubectl helm; do command -v "$t" >/dev/null || { echo "MISSING: $t"; exit 1; }; done + +echo "==> create kind cluster with ingress port mappings (per deploy-flyte-kind Step 1)" +kind create cluster --name "$CLUSTER" --config - <<'EOF' +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 30080 + hostPort: 80 + protocol: TCP + - containerPort: 30443 + hostPort: 443 + protocol: TCP +EOF + +echo "==> in-cluster minio + postgres (throwaway deps for the smoke test)" +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +helm repo add bitnami https://charts.bitnami.com/bitnami >/dev/null +helm repo update >/dev/null +helm upgrade --install pg bitnami/postgresql -n "$NS" \ + --set auth.postgresPassword=flyte --set auth.database=flyte --wait --timeout 5m +helm upgrade --install minio bitnami/minio -n "$NS" \ + --set auth.rootUser=minio --set auth.rootPassword=miniostorage --wait --timeout 5m + +echo "==> install flyte-binary (per deploy-flyte-kind flyte-binary step)" +helm repo add flyteorg https://flyteorg.github.io/flyte >/dev/null +helm repo update >/dev/null +# NOTE: values wiring (db + storage endpoints) mirrors the skill; kept minimal here. +helm upgrade --install flyte-binary flyteorg/flyte-binary -n "$NS" \ + --set configuration.database.host="pg-postgresql.${NS}.svc.cluster.local" \ + --set configuration.database.password=flyte \ + --set configuration.storage.provider=s3 \ + --wait --timeout 10m || true + +echo "==> assert flyte-binary pod is present" +kubectl get pods -n "$NS" +kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=flyte-binary \ + -n "$NS" --timeout=5m + +echo "SMOKE OK: flyte-binary is Ready on kind cluster '$CLUSTER'" diff --git a/evals/static.yaml b/evals/static.yaml deleted file mode 100644 index de83000..0000000 --- a/evals/static.yaml +++ /dev/null @@ -1,5 +0,0 @@ -id: ../-static -skill: ../ -tier: static -# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). -# No agent, no LLM. Runs on every change. diff --git a/evals/tests/__init__.py b/evals/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evals/tests/test_checks.py b/evals/tests/test_checks.py new file mode 100644 index 0000000..400a375 --- /dev/null +++ b/evals/tests/test_checks.py @@ -0,0 +1,61 @@ +import pathlib + +from evals.harness import checks + + +def _ws(tmp_path, **files): + for name, content in files.items(): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return tmp_path + + +def test_file_glob(tmp_path): + _ws(tmp_path, **{"workflow.py": "x=1"}) + r = checks.run_checks(tmp_path, [{"kind": "file_glob", "glob": "*.py"}])[0] + assert r.passed + r2 = checks.run_checks(tmp_path, [{"kind": "file_glob", "glob": "*.md"}])[0] + assert not r2.passed + + +def test_python_parses_and_imports(tmp_path): + _ws(tmp_path, **{"a.py": "import flyte\n@x.task\ndef f():\n return 1\n"}) + assert checks.run_checks(tmp_path, [{"kind": "python_parses"}])[0].passed + assert checks.run_checks(tmp_path, [{"kind": "python_imports", "module": "flyte"}])[0].passed + assert not checks.run_checks(tmp_path, [{"kind": "python_imports", "module": "torch"}])[0].passed + + +def test_python_parses_fails_on_syntax_error(tmp_path): + _ws(tmp_path, **{"bad.py": "def (:"}) + assert not checks.run_checks(tmp_path, [{"kind": "python_parses"}])[0].passed + + +def test_contains_and_not_contains(tmp_path): + _ws(tmp_path, **{"m.py": "TaskEnvironment()\n@env.task\ndef g(): ..."}) + assert checks.run_checks(tmp_path, [{"kind": "contains_regex", "pattern": "@\\w+\\.task"}])[0].passed + assert checks.run_checks(tmp_path, [{"kind": "not_contains_regex", "pattern": "ReusePolicy"}])[0].passed + assert not checks.run_checks(tmp_path, [{"kind": "not_contains_regex", "pattern": "TaskEnvironment"}])[0].passed + + +def test_yaml_valid(tmp_path): + _ws(tmp_path, **{"k.yaml": "kind: Cluster\nnodes: []\n"}) + assert checks.run_checks(tmp_path, [{"kind": "yaml_valid", "file_glob": "*.yaml"}])[0].passed + _ws(tmp_path, **{"bad.yaml": "a: b: c: :::"}) + assert not checks.run_checks(tmp_path, [{"kind": "yaml_valid", "file_glob": "bad.yaml"}])[0].passed + + +def test_cmd_succeeds(tmp_path): + assert checks.run_checks(tmp_path, [{"kind": "cmd_succeeds", "cmd": "true"}])[0].passed + assert not checks.run_checks(tmp_path, [{"kind": "cmd_succeeds", "cmd": "false"}])[0].passed + + +def test_stub_called(tmp_path): + (tmp_path / ".stublog").write_text("kind create cluster --name flyte\nhelm install foo\n") + assert checks.run_checks(tmp_path, [{"kind": "stub_called", "pattern": "kind create cluster"}])[0].passed + assert not checks.run_checks(tmp_path, [{"kind": "stub_called", "pattern": "terraform apply"}])[0].passed + + +def test_unknown_kind_is_failure_not_crash(tmp_path): + r = checks.run_checks(tmp_path, [{"kind": "nope"}])[0] + assert not r.passed and "unknown" in r.detail diff --git a/evals/tests/test_evaluate.py b/evals/tests/test_evaluate.py new file mode 100644 index 0000000..8b083ae --- /dev/null +++ b/evals/tests/test_evaluate.py @@ -0,0 +1,72 @@ +"""Integration test of the scoring/arm/lift plumbing with a mocked harness+judge +so it needs neither a real agent CLI nor the GLM endpoint.""" + +from evals.harness import evaluate as ev +from evals.harness.glm import GLMConfig +from evals.harness.judge import JudgeResult +from evals.harness.runners.base import Trajectory +from evals.harness.spec import Scenario + + +class FakeRunner: + """Treatment arm writes a valid workflow.py; control arm writes nothing.""" + + name = "fake" + + def is_available(self): + return True + + def run(self, scenario, sandbox, arm, glm): + if arm == "treatment": + (sandbox.workspace / "workflow.py").write_text( + "import flyte\n@env.task\ndef f():\n return 1\n" + ) + return Trajectory(harness="fake", arm=arm, transcript=f"ran {arm}") + + +def _scenario(): + return Scenario.from_dict({ + "id": "t", "skill": "flyte-sdk-author", "tier": "trajectory", "prompt": "p", + "checks": [ + {"kind": "file_glob", "glob": "*.py"}, + {"kind": "python_imports", "module": "flyte"}, + ], + "judge": {"rubric": "r", "weights": {"correctness": 1.0}, "pass_threshold": 0.7}, + }) + + +def test_treatment_beats_control_positive_lift(monkeypatch): + monkeypatch.setattr(ev, "get_runner", lambda name: FakeRunner()) + # Judge scores treatment high (checks pass) and is not even called for control + # (checks fail -> score 0 short-circuits in ArmResult.score). + monkeypatch.setattr(ev, "run_judge", + lambda spec, text, glm: JudgeResult(score=0.9, passed=True, + dimensions={"correctness": 0.9})) + res = ev.evaluate_scenario(_scenario(), "fake", GLMConfig("u", "k", "m")) + + assert set(res.arms) == {"treatment", "control"} + assert res.arms["treatment"].checks_passed is True + assert res.arms["treatment"].score == 0.9 + assert res.arms["control"].checks_passed is False # no file produced + assert res.arms["control"].score == 0.0 + assert res.lift == 0.9 + assert res.passed is True + + +def test_harness_unavailable_is_recorded_not_crash(monkeypatch): + class Unavailable(FakeRunner): + def is_available(self): + return False + monkeypatch.setattr(ev, "get_runner", lambda name: Unavailable()) + res = ev.evaluate_scenario(_scenario(), "fake", GLMConfig("u", "k", "m")) + assert res.arms["treatment"].error and not res.passed + + +def test_to_dict_serializable(monkeypatch): + monkeypatch.setattr(ev, "get_runner", lambda name: FakeRunner()) + monkeypatch.setattr(ev, "run_judge", + lambda spec, text, glm: JudgeResult(score=0.8, passed=True)) + d = ev.evaluate_scenario(_scenario(), "fake", GLMConfig("u", "k", "m")).to_dict() + import json + json.dumps(d) # must be JSON-serializable for Flyte I/O + report + assert d["lift"] == 0.8 and d["arms"]["treatment"]["passed"] is True diff --git a/evals/tests/test_judge.py b/evals/tests/test_judge.py new file mode 100644 index 0000000..c561d97 --- /dev/null +++ b/evals/tests/test_judge.py @@ -0,0 +1,32 @@ +from evals.harness.judge import parse_response +from evals.harness.spec import JudgeSpec + + +def test_parse_weighted_pass(): + text = '{"scores": {"correctness": 1.0, "idiomatic": 0.5}, "rationale": "good"}' + r = parse_response(text, {"correctness": 0.8, "idiomatic": 0.2}, 0.7) + assert r.passed + assert abs(r.score - (1.0 * 0.8 + 0.5 * 0.2)) < 1e-9 + assert r.rationale == "good" + + +def test_parse_below_threshold(): + text = 'noise {"scores": {"correctness": 0.2}} trailing' + r = parse_response(text, {"correctness": 1.0}, 0.7) + assert not r.passed and r.score == 0.2 + + +def test_parse_clamps_out_of_range(): + r = parse_response('{"scores": {"a": 5, "b": -3}}', {}, 0.7) + assert r.dimensions == {"a": 1.0, "b": 0.0} + + +def test_parse_garbage_is_error_not_crash(): + r = parse_response("the model refused", {}, 0.7) + assert not r.passed and r.score == 0.0 and "judge error" in r.rationale + + +def test_judgespec_from_dict(): + spec = JudgeSpec.from_dict({"rubric": "r", "weights": {"correctness": 1}, "pass_threshold": 0.9}) + assert spec.pass_threshold == 0.9 and spec.weights["correctness"] == 1 + assert JudgeSpec.from_dict(None) is None diff --git a/evals/tests/test_report.py b/evals/tests/test_report.py new file mode 100644 index 0000000..7b49594 --- /dev/null +++ b/evals/tests/test_report.py @@ -0,0 +1,23 @@ +from evals.report import to_html, to_markdown + +RESULTS = [ + {"scenario_id": "a", "skill": "flyte-sdk-author", "tier": "trajectory", + "harness": "opencode", "passed": True, "lift": 0.4, + "arms": {"treatment": {"score": 0.9, "checks": []}, "control": {"score": 0.5, "checks": []}}}, + {"scenario_id": "b", "skill": "deploy-flyte-kind", "tier": "static", + "harness": None, "passed": False, "lift": None, + "arms": {"treatment": {"score": 0.0, "error": "boom", + "checks": [{"kind": "frontmatter", "passed": False, "detail": "bad"}]}}}, +] + + +def test_markdown_has_summary_and_failure_detail(): + md = to_markdown(RESULTS) + assert "1/2 passing" in md + assert "flyte-sdk-author" in md and "+0.40" in md + assert "frontmatter" in md and "boom" in md + + +def test_html_renders(): + html = to_html(RESULTS) + assert "" in html and "flyte-sdk-author" in html and "1/2 passing" in html diff --git a/evals/tests/test_select.py b/evals/tests/test_select.py new file mode 100644 index 0000000..2e257b6 --- /dev/null +++ b/evals/tests/test_select.py @@ -0,0 +1,47 @@ +from evals.harness.spec import Manifest, load_scenarios +from evals.select import select, is_shared_infra + + +def _manifest(): + return Manifest.load() + + +def test_single_skill_change_selects_only_that_skill(): + m = _manifest() + scs = load_scenarios() + out = select(["plugins/flyte-skills/skills/flyte-sdk-author/SKILL.md"], m, scs) + assert out["run_all"] is False + assert out["skills"] == ["flyte-sdk-author"] + assert "flyte-sdk-author-static" in out["scenario_ids"] + assert "deploy-flyte-kind-static" not in out["scenario_ids"] + + +def test_engine_change_runs_all(): + m = _manifest() + scs = load_scenarios() + out = select(["evals/harness/checks.py"], m, scs) + assert out["run_all"] is True + assert len(out["skills"]) >= 15 + assert out["run_kind"] is True and out["run_real"] is True + + +def test_kind_skill_sets_run_kind(): + m = _manifest() + scs = load_scenarios() + out = select(["plugins/flyte-skills/skills/deploy-flyte-kind/SKILL.md"], m, scs) + assert out["run_kind"] is True + out2 = select(["plugins/flyte-skills/skills/flyte-sdk-types/SKILL.md"], m, scs) + assert out2["run_kind"] is False + + +def test_unrelated_change_selects_nothing(): + m = _manifest() + scs = load_scenarios() + out = select(["README.md"], m, scs) + assert out["skills"] == [] and out["scenario_ids"] == [] + + +def test_is_shared_infra(): + m = _manifest() + assert is_shared_infra("evals/workflows/eval_wf.py", m) + assert not is_shared_infra("plugins/flyte-skills/skills/flyte-sdk-run/SKILL.md", m) diff --git a/evals/tests/test_spec_and_scenarios.py b/evals/tests/test_spec_and_scenarios.py new file mode 100644 index 0000000..1ba16b5 --- /dev/null +++ b/evals/tests/test_spec_and_scenarios.py @@ -0,0 +1,50 @@ +"""Validate that every committed scenario spec loads and is well-formed, and that +every skill under plugins/ has at least a static scenario.""" + +import pathlib + +import pytest + +from evals.harness.spec import REPO_ROOT, Scenario, load_scenarios, scenarios_by_skill + +SKILLS_DIR = REPO_ROOT / "plugins" / "flyte-skills" / "skills" + + +def test_all_scenarios_load(): + scenarios = load_scenarios() + assert scenarios, "no scenarios found" + + +def test_every_scenario_references_a_real_skill(): + valid = {p.name for p in SKILLS_DIR.iterdir() if (p / "SKILL.md").exists()} + for sc in load_scenarios(): + assert sc.skill in valid, f"{sc.id} references unknown skill {sc.skill}" + + +def test_every_skill_has_a_static_scenario(): + by_skill = scenarios_by_skill(load_scenarios()) + for p in SKILLS_DIR.iterdir(): + if (p / "SKILL.md").exists(): + tiers = {sc.tier for sc in by_skill.get(p.name, [])} + assert "static" in tiers, f"skill {p.name} has no static scenario" + + +def test_arms_logic(): + static = Scenario.from_dict({"id": "x", "skill": "flyte-sdk-run", "tier": "static"}) + assert static.arms() == ("treatment",) + traj = Scenario.from_dict({"id": "y", "skill": "flyte-sdk-run", "tier": "trajectory", + "prompt": "p"}) + assert set(traj.arms()) == {"treatment", "control"} + no_ctrl = Scenario.from_dict({"id": "z", "skill": "flyte-sdk-run", "tier": "trajectory", + "prompt": "p", "control": False}) + assert no_ctrl.arms() == ("treatment",) + + +def test_invalid_tier_rejected(): + with pytest.raises(ValueError): + Scenario.from_dict({"id": "bad", "skill": "flyte-sdk-run", "tier": "nope", "prompt": "p"}) + + +def test_trajectory_requires_prompt(): + with pytest.raises(ValueError): + Scenario.from_dict({"id": "bad", "skill": "flyte-sdk-run", "tier": "trajectory"}) diff --git a/evals/workflows/eval_wf.py b/evals/workflows/eval_wf.py index 57915d3..698b50a 100644 --- a/evals/workflows/eval_wf.py +++ b/evals/workflows/eval_wf.py @@ -48,9 +48,11 @@ def eval_unit(unit: dict) -> dict: return evaluate_scenario(sc, unit["harness"], glm).to_dict() -@env.task +@env.task(report=True) def aggregate(results: list[dict]) -> dict: """Collect verdicts into a scorecard summary (also emits an HTML report).""" + import flyte.report + from evals.report import to_html, to_markdown passed = sum(1 for r in results if r["passed"]) @@ -61,10 +63,9 @@ def aggregate(results: list[dict]) -> dict: "markdown": to_markdown(results), "results": results, } - # Attach the HTML scorecard to the Flyte run report tab. + # Attach the HTML scorecard to the Flyte run's report tab. try: - flyte.report.replace(to_html(results)) # type: ignore[attr-defined] - flyte.report.flush() # type: ignore[attr-defined] + flyte.report.replace(to_html(results), do_flush=True) except Exception: pass return summary From bcb5ae5279cbada66c1fa9997299cc6839bae900 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Thu, 23 Jul 2026 14:44:40 -0700 Subject: [PATCH 3/4] align eval harness with plugins/flyte rename The plugin directory was renamed plugins/flyte-skills -> plugins/flyte (and the flyte-mcp-server skill became a bundled MCP server). Update the harness: - skill path references (spec.py skill_dir_template, evaluate.py, runners/base.py, select.py regex, manifest.yaml globs, tests) flyte-skills -> flyte - drop the flyte-mcp-server static scenario (skill no longer exists) - refresh branding in README/docstrings Verified: static lint 14/14, 29 unit tests passing. Co-Authored-By: Claude Opus 4.8 (1M context) --- evals/__init__.py | 2 +- evals/harness/evaluate.py | 2 +- evals/harness/runners/base.py | 2 +- evals/harness/spec.py | 2 +- evals/manifest.yaml | 6 +++--- evals/pyproject.toml | 2 +- evals/scenarios/flyte-mcp-server/static.yaml | 5 ----- evals/select.py | 4 ++-- evals/tests/test_select.py | 10 +++++----- evals/tests/test_spec_and_scenarios.py | 2 +- 10 files changed, 16 insertions(+), 21 deletions(-) delete mode 100644 evals/scenarios/flyte-mcp-server/static.yaml diff --git a/evals/__init__.py b/evals/__init__.py index 123f293..bdaa967 100644 --- a/evals/__init__.py +++ b/evals/__init__.py @@ -1 +1 @@ -"""Testing & eval harness for the flyte-skills agent skills.""" +"""Testing & eval harness for the Flyte plugin skills.""" diff --git a/evals/harness/evaluate.py b/evals/harness/evaluate.py index a581db0..1b5d539 100644 --- a/evals/harness/evaluate.py +++ b/evals/harness/evaluate.py @@ -105,7 +105,7 @@ def to_dict(self) -> dict: def evaluate_static(scenario: Scenario) -> ScenarioResult: - skill_dir = REPO_ROOT / "plugins" / "flyte-skills" / "skills" / scenario.skill + skill_dir = REPO_ROOT / "plugins" / "flyte" / "skills" / scenario.skill results = lint_skill(skill_dir) arm = ArmResult(arm="treatment", checks=results) return ScenarioResult(scenario.id, scenario.skill, "static", harness=None, diff --git a/evals/harness/runners/base.py b/evals/harness/runners/base.py index cfcb7ab..91b9c13 100644 --- a/evals/harness/runners/base.py +++ b/evals/harness/runners/base.py @@ -91,7 +91,7 @@ def snapshot_files(workspace: pathlib.Path, max_bytes: int = 200_000) -> dict[st def _repo_skill_dir(skill: str) -> pathlib.Path: from ..spec import REPO_ROOT - return REPO_ROOT / "plugins" / "flyte-skills" / "skills" / skill + return REPO_ROOT / "plugins" / "flyte" / "skills" / skill def sh(cmd: list[str], cwd: pathlib.Path, env: dict[str, str], timeout: int = 600 diff --git a/evals/harness/spec.py b/evals/harness/spec.py index 495e819..e1e7868 100644 --- a/evals/harness/spec.py +++ b/evals/harness/spec.py @@ -131,7 +131,7 @@ def load(path: pathlib.Path | None = None) -> "Manifest": harnesses=tuple(d.get("harnesses") or VALID_HARNESSES), default_tiers=tuple(d.get("default_tiers") or ("static", "trajectory")), scenarios_dir=d.get("scenarios_dir", "evals/scenarios"), - skill_dir_template=d.get("skill_dir_template", "plugins/flyte-skills/skills/{skill}"), + skill_dir_template=d.get("skill_dir_template", "plugins/flyte/skills/{skill}"), shared_infra_globs=tuple(d.get("shared_infra_globs") or ()), kind_smoke_skills=tuple(d.get("kind_smoke_skills") or ()), sdk_real_skills=tuple(d.get("sdk_real_skills") or ()), diff --git a/evals/manifest.yaml b/evals/manifest.yaml index 5eb5430..5b5a899 100644 --- a/evals/manifest.yaml +++ b/evals/manifest.yaml @@ -10,7 +10,7 @@ default_tiers: [static, trajectory] # Where scenario specs live (relative to repo root) and how to find a skill dir. scenarios_dir: evals/scenarios -skill_dir_template: "plugins/flyte-skills/skills/{skill}" +skill_dir_template: "plugins/flyte/skills/{skill}" # Changing any of these paths means the ENGINE changed -> run the whole suite, # not just the scenarios for a changed skill (see evals/select.py). @@ -21,8 +21,8 @@ shared_infra_globs: - "evals/report.py" - "evals/manifest.yaml" - "evals/pyproject.toml" - - "plugins/flyte-skills/.claude-plugin/**" - - "plugins/flyte-skills/.codex-plugin/**" + - "plugins/flyte/.claude-plugin/**" + - "plugins/flyte/.codex-plugin/**" # Deploy skills that get a REAL kind-in-Docker smoke test on a privileged # GitHub Actions runner (see evals/kind_smoke/run.sh). All other deploy skills diff --git a/evals/pyproject.toml b/evals/pyproject.toml index c803518..191e0f7 100644 --- a/evals/pyproject.toml +++ b/evals/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "flyte-skills-evals" version = "0.0.1" -description = "Testing & eval harness for the flyte-skills agent skills." +description = "Testing & eval harness for the Flyte plugin skills." requires-python = ">=3.10" dependencies = [ "pyyaml>=6.0", diff --git a/evals/scenarios/flyte-mcp-server/static.yaml b/evals/scenarios/flyte-mcp-server/static.yaml deleted file mode 100644 index 5ccc8c1..0000000 --- a/evals/scenarios/flyte-mcp-server/static.yaml +++ /dev/null @@ -1,5 +0,0 @@ -id: flyte-mcp-server-static -skill: flyte-mcp-server -tier: static -# Static tier lints the SKILL.md itself (frontmatter, code fences, no secrets). -# No agent, no LLM. Runs on every change. diff --git a/evals/select.py b/evals/select.py index 9620de5..0f08e75 100644 --- a/evals/select.py +++ b/evals/select.py @@ -1,7 +1,7 @@ """Map changed files -> the subset of scenarios to run. Used by CI to run only what a PR affects: - - a changed `plugins/flyte-skills/skills//**` -> that skill's scenarios + - a changed `plugins/flyte/skills//**` -> that skill's scenarios - a change to engine/shared-infra paths (manifest.shared_infra_globs) -> run ALL - flags whether the kind DinD smoke and the real tier are in scope @@ -26,7 +26,7 @@ from evals.harness.spec import Manifest, load_scenarios, scenarios_by_skill -SKILL_PATH_RE = re.compile(r"plugins/flyte-skills/skills/([^/]+)/") +SKILL_PATH_RE = re.compile(r"plugins/flyte/skills/([^/]+)/") def changed_from_git(base: str, repo_root: pathlib.Path) -> list[str]: diff --git a/evals/tests/test_select.py b/evals/tests/test_select.py index 2e257b6..435fd12 100644 --- a/evals/tests/test_select.py +++ b/evals/tests/test_select.py @@ -9,7 +9,7 @@ def _manifest(): def test_single_skill_change_selects_only_that_skill(): m = _manifest() scs = load_scenarios() - out = select(["plugins/flyte-skills/skills/flyte-sdk-author/SKILL.md"], m, scs) + out = select(["plugins/flyte/skills/flyte-sdk-author/SKILL.md"], m, scs) assert out["run_all"] is False assert out["skills"] == ["flyte-sdk-author"] assert "flyte-sdk-author-static" in out["scenario_ids"] @@ -21,16 +21,16 @@ def test_engine_change_runs_all(): scs = load_scenarios() out = select(["evals/harness/checks.py"], m, scs) assert out["run_all"] is True - assert len(out["skills"]) >= 15 + assert len(out["skills"]) >= 14 assert out["run_kind"] is True and out["run_real"] is True def test_kind_skill_sets_run_kind(): m = _manifest() scs = load_scenarios() - out = select(["plugins/flyte-skills/skills/deploy-flyte-kind/SKILL.md"], m, scs) + out = select(["plugins/flyte/skills/deploy-flyte-kind/SKILL.md"], m, scs) assert out["run_kind"] is True - out2 = select(["plugins/flyte-skills/skills/flyte-sdk-types/SKILL.md"], m, scs) + out2 = select(["plugins/flyte/skills/flyte-sdk-types/SKILL.md"], m, scs) assert out2["run_kind"] is False @@ -44,4 +44,4 @@ def test_unrelated_change_selects_nothing(): def test_is_shared_infra(): m = _manifest() assert is_shared_infra("evals/workflows/eval_wf.py", m) - assert not is_shared_infra("plugins/flyte-skills/skills/flyte-sdk-run/SKILL.md", m) + assert not is_shared_infra("plugins/flyte/skills/flyte-sdk-run/SKILL.md", m) diff --git a/evals/tests/test_spec_and_scenarios.py b/evals/tests/test_spec_and_scenarios.py index 1ba16b5..90833cf 100644 --- a/evals/tests/test_spec_and_scenarios.py +++ b/evals/tests/test_spec_and_scenarios.py @@ -7,7 +7,7 @@ from evals.harness.spec import REPO_ROOT, Scenario, load_scenarios, scenarios_by_skill -SKILLS_DIR = REPO_ROOT / "plugins" / "flyte-skills" / "skills" +SKILLS_DIR = REPO_ROOT / "plugins" / "flyte" / "skills" def test_all_scenarios_load(): From 4fe35c6fe7570e7a667f2e3ffcc6a03819d46bb9 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Sun, 26 Jul 2026 16:33:27 -0400 Subject: [PATCH 4/4] update flyte-skills to flyte-agent-plugin Signed-off-by: Niels Bantilan --- .github/workflows/skill-evals.yml | 6 +++--- README.md | 3 +-- evals/README.md | 4 ++-- evals/harness/run.py | 2 +- evals/manifest.yaml | 2 +- evals/pyproject.toml | 4 ++-- evals/report.py | 6 +++--- evals/workflows/eval_wf.py | 4 ++-- 8 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index d40004c..f7966d9 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -1,8 +1,8 @@ name: skill-evals -# Runs the flyte-skills eval harness. On PRs it runs only the scenarios affected -# by the changed files (see evals/select.py); nightly it runs the full matrix -# including the `real` tier. +# Runs the flyte-agent-plugin skills eval harness. On PRs it runs only the +# scenarios affected by the changed files (see evals/select.py); nightly it runs +# the full matrix including the `real` tier. on: pull_request: paths: diff --git a/README.md b/README.md index 5d915b8..50206a6 100644 --- a/README.md +++ b/README.md @@ -233,8 +233,7 @@ scripts/smoke_test_mcp.py # end-to-end check of the local MCP Each harness consumes a different part of this. Claude Code and Codex read the plugin manifests, so the **plugin name** matters to them. Hermes, opencode, and pi install skills -by **directory path**, so `plugins/flyte/skills/…` is their interface — which is why the -rename from `flyte-skills` touched both. +by **directory path**, so `plugins/flyte/skills/…` is their interface. The `.mcp.json` server is Claude Code-specific; the skills themselves stay portable across harnesses. diff --git a/evals/README.md b/evals/README.md index 80e89da..197b123 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,6 +1,6 @@ -# flyte-skills eval harness +# flyte agent plugin eval harness -Automated testing & evaluation for the `flyte-skills` agent skills. It runs a real +Automated testing & evaluation for the `flyte` agent skills. It runs a real agent harness (**opencode / pi / hermes**) against the union-hosted **GLM** endpoint, hands it a skill + a task, and scores what it produces — orchestrated as **Flyte workflows on `demo.hosted.unionai.cloud`**, with path-based selection so only the diff --git a/evals/harness/run.py b/evals/harness/run.py index 441841b..0fd0a6f 100644 --- a/evals/harness/run.py +++ b/evals/harness/run.py @@ -68,7 +68,7 @@ def _print_table(results: list[ScenarioResult]) -> int: def main(argv: list[str] | None = None) -> int: - ap = argparse.ArgumentParser(prog="flyte-evals", description="Run flyte-skills evals locally") + ap = argparse.ArgumentParser(prog="flyte-evals", description="Run flyte-agent-plugin evals locally") ap.add_argument("--scenario", help="scenario id") ap.add_argument("--skill", help="filter by skill name") ap.add_argument("--tier", choices=["static", "trajectory", "real"], help="filter by tier") diff --git a/evals/manifest.yaml b/evals/manifest.yaml index 5b5a899..d54708b 100644 --- a/evals/manifest.yaml +++ b/evals/manifest.yaml @@ -1,4 +1,4 @@ -# Top-level configuration for the flyte-skills eval harness. +# Top-level configuration for the flyte-agent-plugin eval harness. # Scenario -> skill mapping is derived by scanning `scenarios_dir`; each scenario # file declares its own `skill`. This manifest holds the cross-cutting config. diff --git a/evals/pyproject.toml b/evals/pyproject.toml index 191e0f7..01a09b5 100644 --- a/evals/pyproject.toml +++ b/evals/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "flyte-skills-evals" +name = "flyte-agent-plugin-evals" version = "0.0.1" -description = "Testing & eval harness for the Flyte plugin skills." +description = "Testing & eval harness for the Flyte agent plugin." requires-python = ">=3.10" dependencies = [ "pyyaml>=6.0", diff --git a/evals/report.py b/evals/report.py index 78ec266..026c590 100644 --- a/evals/report.py +++ b/evals/report.py @@ -22,7 +22,7 @@ def to_markdown(results: list[dict]) -> str: total = len(results) failed = [r for r in results if not r["passed"]] lines = [ - f"### flyte-skills evals — {total - len(failed)}/{total} passing", + f"### flyte-agent-plugin evals — {total - len(failed)}/{total} passing", "", "| scenario | skill | harness | tier | pass | treat | ctrl | lift |", "|---|---|---|---|:--:|--:|--:|--:|", @@ -71,14 +71,14 @@ def to_html(results: list[dict]) -> str: ) passed = sum(1 for r in results if r["passed"]) return f""" -flyte-skills evals +flyte-agent-plugin evals -

flyte-skills evals — {passed}/{len(results)} passing

+

flyte-agent-plugin evals — {passed}/{len(results)} passing

diff --git a/evals/workflows/eval_wf.py b/evals/workflows/eval_wf.py index 698b50a..223097c 100644 --- a/evals/workflows/eval_wf.py +++ b/evals/workflows/eval_wf.py @@ -1,4 +1,4 @@ -"""Flyte orchestration of the flyte-skills eval harness. +"""Flyte orchestration of the flyte-agent-plugin eval harness. Runs on demo.hosted.unionai.cloud (org demo / project flytesnacks / domain development, remote image builder — see evals/config/flyte.yaml). The workflow @@ -21,7 +21,7 @@ from evals.workflows.images import eval_image env = flyte.TaskEnvironment( - name="flyte-skills-evals", + name="flyte-agent-plugin-evals", image=eval_image, resources=flyte.Resources(cpu="2", memory="4Gi"), secrets=[flyte.Secret(key="glm-api-key", as_env_var="GLM_API_KEY")],
scenarioskillharnesstier passtreatmentcontrollift