"""Seam 2 skill eval: run the skills on a fixture project with `claude -p`, then grade. Usage (from the repo root): uv run --with pytest --with playwright python tests/evals/run_eval.py a-empty Copies tests/fixtures/ to a temp folder, installs every rpa-dashboard-* skill there as a project skill, and runs `claude -p` with the user's answers in the prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1 page check on the page it built, and writes tests/evals/results/.json. Exit code 0 means every check passed. """ import importlib.util import json import os import re import shutil import subprocess import sys import tempfile from pathlib import Path ROOT = Path(__file__).resolve().parents[2] REPLY_FORMAT = """ When you're done, end your reply with exactly these two lines: FRONTEND: QUESTIONS: """ # Per fixture: the request, what the reply must (not) mention, and checks on the output. FIXTURES = { "a-empty": { "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you " "work, so here are my answers to anything the skill asks. Colours: custom, my " "brand colour is #7a3cff. Desktop or web: desktop. Optional sections: none.", "frontend": r"^\s*(none|no\b)", "asks": [r"colou?r", r"desktop|web", r"section"], "never_asks": [r"\bkeep\b|restyle"], "brand": "#7a3cff", }, "b-pyqt-bot": { "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you " "work, so here are my answers to anything the skill asks. Colours: Tebra Sky. " "Optional sections: none.", "frontend": r"^\s*(none|no\b)", "asks": [r"colou?r", r"section"], "never_asks": [r"\bkeep\b|restyle", r"desktop|web"], "bot": True, }, } def bot_checks(project, html, checks, replay): """Bot fixtures: the page is wired, and a dry run of the generated wiring gives real steps and logs. Returns the env that points the page check at that run.""" checks["page wired to the bot (sample data off)"] = bool(re.search(r"SAMPLE_DATA\s*=\s*false", html)) runners = [p for p in project.rglob("bot_runner.py") if ".claude" not in p.parts] checks["bot_runner.py built"] = len(runners) == 1 if not runners: return {} host = [runners[0], runners[0].with_name("bridge.py")] checks["no placeholders left in the host"] = all(p.exists() and "{{" not in p.read_text(encoding="utf-8") for p in host) dry = subprocess.run([sys.executable, str(runners[0])], cwd=project, capture_output=True, text=True, encoding="utf-8", timeout=300) replay.write_text(dry.stdout, encoding="utf-8") events = [json.loads(x) for x in dry.stdout.splitlines() if x.startswith("{")] steps = [e["data"] for e in events if e["event"] == "step"] checks["bot dry run ends done"] = bool(events) and events[-1]["data"].get("state") == "done" checks["real steps reach the page"] = len({s["label"] for s in steps}) >= 3 and steps[-1]["pct"] == 100 checks["bot error line reaches the log"] = any(e["event"] == "log" and e["data"]["type"] == "error" for e in events) return {"PAGE_CHECK_WIRED": "1", "PAGE_CHECK_REPLAY": str(replay)} def main(fixture): cfg = FIXTURES[fixture] work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-")) project = work / "project" shutil.copytree(ROOT / "tests" / "fixtures" / fixture, project) for skill in ROOT.glob("rpa-dashboard-*"): shutil.copytree(skill, project / ".claude" / "skills" / skill.name) # The prompt goes in on stdin: on Windows a multi-line argv prompt is cut at the first newline. run = subprocess.run( [shutil.which("claude") or "claude", "-p", "--output-format", "json", "--permission-mode", "acceptEdits", "--allowedTools", "Skill", "Read", "Write", "Edit", "Glob", "Grep", "Bash(python *)", "PowerShell(python *)"], input=cfg["prompt"] + REPLY_FORMAT, cwd=project, capture_output=True, text=True, encoding="utf-8", timeout=1800) reply = json.loads(run.stdout).get("result", "") if run.stdout.strip().startswith("{") else run.stdout def line(key): m = re.search(rf"^\W*{key}\W*:(.*)$", reply, re.M | re.I) return m[1] if m else "" checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))} for pat in cfg["asks"]: checks[f"asks /{pat}/"] = bool(re.search(pat, line("QUESTIONS"), re.I)) for pat in cfg["never_asks"]: checks[f"never asks /{pat}/"] = not re.search(pat, line("QUESTIONS"), re.I) pages = [p for p in project.rglob("*.html") if ".claude" not in p.parts] checks["exactly one page built"] = len(pages) == 1 results = ROOT / "tests" / "evals" / "results" results.mkdir(exist_ok=True) if pages: html = pages[0].read_text(encoding="utf-8") shutil.copy(pages[0], results / f"{fixture}.dashboard.html") checks["no placeholders left"] = "{{" not in html if cfg.get("brand"): spec = importlib.util.spec_from_file_location("palette", ROOT / "rpa-dashboard-no-ai/templates/palette.py") palette = importlib.util.module_from_spec(spec) spec.loader.exec_module(palette) t = palette.derive(cfg["brand"]) checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t) env = {**os.environ, "PAGE_CHECK_HTML": str(pages[0])} if cfg.get("bot"): env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl")) pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"], env=env, capture_output=True, text=True) checks["page check passes"] = pc.returncode == 0 page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]] else: page_check = ["no page"] out = {"fixture": fixture, "passed": all(checks.values()), "checks": checks, "page_check": page_check, "files": sorted(str(p.relative_to(project)) for p in project.rglob("*") if p.is_file() and ".claude" not in p.parts), "reply": reply} (results / f"{fixture}.json").write_text(json.dumps(out, indent=2), encoding="utf-8") for name, ok in checks.items(): print(f"{'PASS' if ok else 'FAIL'} {name}") print("page check:", *page_check) print("work dir:", project) return 0 if out["passed"] else 1 if __name__ == "__main__": sys.exit(main(sys.argv[1]))