VasanthandClaude Opus 5.5
73adc74637
Brings in tickets 05-07 (AI skill, cost page, AI desktop sections). tests/README.md keeps both the optional-section notes and the AI page-check notes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
250 lines
13 KiB
Python
250 lines
13 KiB
Python
"""Seam 2 skill eval: run the skills on a fixture project with `claude -p`, then grade.
|
|
|
|
Usage (from the repo root):
|
|
uv run --with pytest --with playwright python tests/evals/run_eval.py a-empty
|
|
|
|
Copies tests/fixtures/<fixture> to a temp folder, installs every rpa-dashboard-*
|
|
skill there as a project skill, and runs `claude -p` with the user's answers in the
|
|
prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1 page
|
|
check on the page it built, and writes tests/evals/results/<fixture>.json.
|
|
Exit code 0 means every check passed.
|
|
"""
|
|
import difflib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
REPLY_FORMAT = """
|
|
When you're done, end your reply with exactly these two lines:
|
|
FRONTEND: <what your scan found, or none>
|
|
QUESTIONS: <each question the skill told you to ask, in order, separated by " | ">"""
|
|
|
|
# Per fixture: the request, what the reply must (not) mention, and checks on the output.
|
|
FIXTURES = {
|
|
"a-empty": {
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Colours: custom, my "
|
|
"brand colour is #7a3cff. Desktop or web: desktop. Optional sections: none.",
|
|
"frontend": r"^\s*(none|no\b)",
|
|
"asks": [r"colou?r", r"desktop|web", r"section"],
|
|
"never_asks": [r"\bkeep\b|restyle"],
|
|
"brand": "#7a3cff",
|
|
"sections": [],
|
|
},
|
|
"a-empty-sections": {
|
|
"fixture": "a-empty",
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Colours: Superbill Ember. "
|
|
"Desktop or web: web. Optional sections: all three (checkpoint tracker, admin/roles "
|
|
"screen, login page). The project has no auth yet.",
|
|
"frontend": r"^\s*(none|no\b)",
|
|
"asks": [r"colou?r", r"desktop|web", r"section"],
|
|
"never_asks": [r"\bkeep\b|restyle"],
|
|
"sections": ["checkpoints", "admin", "login"],
|
|
},
|
|
"b-pyqt-bot": {
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Colours: Tebra Sky. "
|
|
"Optional sections: none.",
|
|
"frontend": r"^\s*(none|no\b)",
|
|
"asks": [r"colou?r", r"section"],
|
|
"never_asks": [r"\bkeep\b|restyle", r"desktop|web"],
|
|
"bot": True,
|
|
},
|
|
# (c) existing teal frontend in web/, run once per answer to keep-or-restyle.
|
|
"c-keep": {
|
|
"fixture": "c-web-existing-frontend",
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Keep or restyle the "
|
|
"existing frontend: keep its look. Optional sections: none.",
|
|
"frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))",
|
|
"asks": [r"keep|restyle", r"section"],
|
|
"never_asks": [r"^(?!.*(keep|restyle)).*(colou?r|Tebra Sky|Ember)", r"desktop"],
|
|
"page_has": ["--accent:#0f766e", "Trebuchet MS"],
|
|
"existing": "unchanged",
|
|
},
|
|
"c-restyle": {
|
|
"fixture": "c-web-existing-frontend",
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Keep or restyle the "
|
|
"existing frontend: restyle it. Colours: Superbill Ember. Optional sections: none.",
|
|
"frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))",
|
|
"asks": [r"keep|restyle", r"Tebra Sky|Ember", r"section"],
|
|
"never_asks": [r"desktop"],
|
|
"page_has": ["--accent:#ff3049"],
|
|
"existing": "#ff3049", # only token lines may change, and the theme now uses this colour
|
|
},
|
|
# (d) web service with an LLM extraction call and no UI: the AI skill's web path.
|
|
"d-ai-extraction": {
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Colours: Superbill Ember. "
|
|
"Desktop or web: web. Optional sections: none.",
|
|
"frontend": r"^\s*(none|no\b)",
|
|
"asks": [r"colou?r", r"desktop|web", r"section"],
|
|
"never_asks": [r"\bkeep\b|restyle"],
|
|
"page_has": ["--accent:#ff3049"],
|
|
"ai": True,
|
|
},
|
|
# (e) PyQt bot with an LLM extraction call: the AI skill's desktop path, no web server.
|
|
"e-pyqt-ai": {
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Colours: Tebra Sky. "
|
|
"Optional sections: none.",
|
|
"frontend": r"^\s*(none|no\b)",
|
|
"asks": [r"colou?r", r"section"],
|
|
"never_asks": [r"\bkeep\b|restyle", r"desktop|web"],
|
|
"bot": True,
|
|
"ai_desktop": True,
|
|
},
|
|
}
|
|
# A line a restyle may change: a custom property, a comment, a blank, or a web-font link.
|
|
TOKEN_LINE = r"^\s*(--[\w-]+\s*:.*|/\*.*\*/|<link[^>]*fonts\.googleapis\.com.*|@import.*fonts\.googleapis\.com.*)?\s*$"
|
|
|
|
|
|
def squash(s):
|
|
"""Ignore spacing and case when looking for a token in the built page."""
|
|
return re.sub(r"\s+", "", s).lower()
|
|
|
|
|
|
def changed_lines(src, project):
|
|
"""Lines removed or added in the fixture's own files."""
|
|
out = []
|
|
for f in src.rglob("*"):
|
|
if f.is_file():
|
|
new = project / f.relative_to(src)
|
|
after = new.read_text(encoding="utf-8").splitlines() if new.exists() else []
|
|
out += [x[1:] for x in difflib.unified_diff(f.read_text(encoding="utf-8").splitlines(), after, lineterm="", n=0)
|
|
if x[:1] in "+-" and x[:3] not in ("+++", "---")]
|
|
return out
|
|
|
|
|
|
def bot_checks(project, html, checks, replay):
|
|
"""Bot fixtures: the page is wired, and a dry run of the generated wiring gives real steps and logs.
|
|
Returns the env that points the page check at that run."""
|
|
checks["page wired to the bot (sample data off)"] = bool(re.search(r"SAMPLE_DATA\s*=\s*false", html))
|
|
runners = [p for p in project.rglob("bot_runner.py") if ".claude" not in p.parts]
|
|
checks["bot_runner.py built"] = len(runners) == 1
|
|
if not runners:
|
|
return {}
|
|
host = [runners[0], runners[0].with_name("bridge.py")]
|
|
checks["no placeholders left in the host"] = all(p.exists() and "{{" not in p.read_text(encoding="utf-8") for p in host)
|
|
dry = subprocess.run([sys.executable, str(runners[0])], cwd=project, capture_output=True, text=True,
|
|
encoding="utf-8", timeout=300)
|
|
replay.write_text(dry.stdout, encoding="utf-8")
|
|
events = [json.loads(x) for x in dry.stdout.splitlines() if x.startswith("{")]
|
|
steps = [e["data"] for e in events if e["event"] == "step"]
|
|
checks["bot dry run ends done"] = bool(events) and events[-1]["data"].get("state") == "done"
|
|
checks["real steps reach the page"] = len({s["label"] for s in steps}) >= 3 and steps[-1]["pct"] == 100
|
|
checks["bot error line reaches the log"] = any(e["event"] == "log" and e["data"]["type"] == "error" for e in events)
|
|
return {"PAGE_CHECK_WIRED": "1", "PAGE_CHECK_REPLAY": str(replay)}
|
|
|
|
|
|
def main(fixture):
|
|
cfg = FIXTURES[fixture]
|
|
work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-"))
|
|
project = work / "project"
|
|
src = ROOT / "tests" / "fixtures" / cfg.get("fixture", fixture)
|
|
shutil.copytree(src, project)
|
|
for skill in ROOT.glob("rpa-dashboard-*"):
|
|
shutil.copytree(skill, project / ".claude" / "skills" / skill.name)
|
|
|
|
# The prompt goes in on stdin: on Windows a multi-line argv prompt is cut at the first newline.
|
|
run = subprocess.run(
|
|
[shutil.which("claude") or "claude", "-p", "--output-format", "json",
|
|
"--permission-mode", "acceptEdits", "--allowedTools", "Skill", "Read", "Write", "Edit",
|
|
"Glob", "Grep", "Bash(python *)", "PowerShell(python *)"],
|
|
input=cfg["prompt"] + REPLY_FORMAT, cwd=project, capture_output=True, text=True, encoding="utf-8",
|
|
timeout=1800)
|
|
reply = json.loads(run.stdout).get("result", "") if run.stdout.strip().startswith("{") else run.stdout
|
|
|
|
def line(key):
|
|
m = re.search(rf"^\W*{key}\W*:(.*)$", reply, re.M | re.I)
|
|
return m[1] if m else ""
|
|
|
|
checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))}
|
|
questions = line("QUESTIONS").split("|")
|
|
for pat in cfg["asks"]:
|
|
checks[f"asks /{pat}/"] = any(re.search(pat, q, re.I) for q in questions)
|
|
for pat in cfg["never_asks"]:
|
|
checks[f"never asks /{pat}/"] = not any(re.search(pat, q, re.I) for q in questions)
|
|
if "existing" in cfg:
|
|
changed = changed_lines(src, project)
|
|
if cfg["existing"] == "unchanged":
|
|
checks["existing files unchanged"] = not changed
|
|
else:
|
|
checks["existing files: only token lines changed"] = bool(changed) and all(re.match(TOKEN_LINE, x) for x in changed)
|
|
checks[f"existing tokens now use {cfg['existing']}"] = any(cfg["existing"] in x.lower() for x in changed)
|
|
|
|
pages = [p for p in project.rglob("*.html")
|
|
if ".claude" not in p.parts and not (src / p.relative_to(project)).exists()] # new pages only
|
|
checks["exactly one page built"] = len(pages) == 1
|
|
results = ROOT / "tests" / "evals" / "results"
|
|
results.mkdir(exist_ok=True)
|
|
if pages:
|
|
html = pages[0].read_text(encoding="utf-8")
|
|
if cfg.get("ai"): # the React starter: its theme lives in src/index.css
|
|
css = pages[0].parent / "src" / "index.css"
|
|
html += css.read_text(encoding="utf-8") if css.exists() else ""
|
|
checks["AI path: review starter and server stub built"] = all(
|
|
(pages[0].parent / f).exists() for f in ["server_stub.py", "bands.py", "src/components/Review.tsx"])
|
|
if cfg.get("ai_desktop"): # the no-AI page with the plain-HTML AI sections, hosted by ai_bridge.py
|
|
host = pages[0].with_name("ai_bridge.py")
|
|
checks["AI desktop path: sections, ai_bridge and review backend built"] = 'id="ai-review-tab"' in html and all(
|
|
pages[0].with_name(f).exists() for f in ["ai_bridge.py", "server_stub.py", "bands.py", "pricing.py"])
|
|
checks["no placeholders left in ai_bridge.py"] = host.exists() and "{{" not in host.read_text(encoding="utf-8")
|
|
shutil.copy(pages[0], results / f"{fixture}.dashboard.html")
|
|
checks["no placeholders left"] = "{{" not in html
|
|
if cfg.get("brand"):
|
|
spec = importlib.util.spec_from_file_location("palette", ROOT / "rpa-dashboard-no-ai/templates/palette.py")
|
|
palette = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(palette)
|
|
t = palette.derive(cfg["brand"])
|
|
checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t)
|
|
if "sections" in cfg: # chosen sections built, the others leave no trace
|
|
spec = importlib.util.spec_from_file_location("test_sections", ROOT / "tests" / "test_sections.py")
|
|
traces = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(traces)
|
|
for name, trace in traces.TRACES.items():
|
|
want = name in cfg["sections"]
|
|
checks[f"{name} {'built' if want else 'left no trace'}"] = bool(trace.search(html)) == want
|
|
for needle in cfg.get("page_has", []):
|
|
checks[f"page has {needle}"] = squash(needle) in squash(html)
|
|
env = {**os.environ, "PAGE_CHECK_HTML": str(pages[0])}
|
|
if cfg.get("bot"):
|
|
env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl"))
|
|
if cfg.get("ai"):
|
|
env["PAGE_CHECK_AI_DIR"] = str(pages[0].parent)
|
|
check_dirs = [ROOT / "tests" / ("page_check_ai" if cfg.get("ai") else "page_check")]
|
|
if cfg.get("ai_desktop"):
|
|
env["PAGE_CHECK_AI_DESKTOP_DIR"] = str(pages[0].parent)
|
|
check_dirs.append(ROOT / "tests" / "page_check_ai_desktop")
|
|
pc = subprocess.run([sys.executable, "-m", "pytest", *map(str, check_dirs), "-q"],
|
|
env=env, capture_output=True, text=True)
|
|
checks["page check passes"] = pc.returncode == 0
|
|
page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]
|
|
if cfg.get("sections"): # every chosen section's page check ran
|
|
checks["no section checks skipped"] = "skipped" not in page_check[0]
|
|
else:
|
|
page_check = ["no page"]
|
|
|
|
out = {"fixture": fixture, "passed": all(checks.values()), "checks": checks, "page_check": page_check,
|
|
"files": sorted(str(p.relative_to(project)) for p in project.rglob("*") if p.is_file() and ".claude" not in p.parts),
|
|
"reply": reply}
|
|
(results / f"{fixture}.json").write_text(json.dumps(out, indent=2), encoding="utf-8")
|
|
for name, ok in checks.items():
|
|
print(f"{'PASS' if ok else 'FAIL'} {name}")
|
|
print("page check:", *page_check)
|
|
print("work dir:", project)
|
|
return 0 if out["passed"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1]))
|