VasanthandClaude Opus 5.5
7f2ddecffb
- rpa-dashboard-no-ai/SKILL.md: trigger description and the workflow in spec order (scan, report/keep-or-restyle, colours, stack, sections, build, wire, verify). - templates/themes.md: Tebra Sky and Superbill Ember, light and dark blocks. - templates/palette.py: derives a custom palette from one brand hex and checks text contrast in light and dark. - dashboard.html: built-in sample data so a page with no bot works on first open; cleared batches in the donut; component tokens so Ember is a token-only swap; accessible names for KPI cards, nav, steps, log, search and the dark toggle. - bridge.py: placeholders inside string literals, so it passes the lint hook unfilled. - tests/page_check: Seam 1 harness (fake Qt bridge, light/dark/narrow). - tests/evals: Seam 2 runner and fixture a-empty. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
101 lines
4.6 KiB
Python
101 lines
4.6 KiB
Python
"""Seam 2 skill eval: run the skills on a fixture project with `claude -p`, then grade.
|
|
|
|
Usage (from the repo root):
|
|
uv run --with pytest --with playwright python tests/evals/run_eval.py a-empty
|
|
|
|
Copies tests/fixtures/<fixture> to a temp folder, installs every rpa-dashboard-*
|
|
skill there as a project skill, and runs `claude -p` with the user's answers in the
|
|
prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1 page
|
|
check on the page it built, and writes tests/evals/results/<fixture>.json.
|
|
Exit code 0 means every check passed.
|
|
"""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
REPLY_FORMAT = """
|
|
When you're done, end your reply with exactly these two lines:
|
|
FRONTEND: <what your scan found, or none>
|
|
QUESTIONS: <each question the skill told you to ask, in order, separated by " | ">"""
|
|
|
|
# Per fixture: the request, what the reply must (not) mention, and checks on the output.
|
|
FIXTURES = {
|
|
"a-empty": {
|
|
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
|
"work, so here are my answers to anything the skill asks. Colours: custom, my "
|
|
"brand colour is #7a3cff. Desktop or web: desktop. Optional sections: none.",
|
|
"frontend": r"^\s*(none|no\b)",
|
|
"asks": [r"colou?r", r"desktop|web", r"section"],
|
|
"never_asks": [r"\bkeep\b|restyle"],
|
|
"brand": "#7a3cff",
|
|
},
|
|
}
|
|
|
|
|
|
def main(fixture):
|
|
cfg = FIXTURES[fixture]
|
|
work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-"))
|
|
project = work / "project"
|
|
shutil.copytree(ROOT / "tests" / "fixtures" / fixture, project)
|
|
for skill in ROOT.glob("rpa-dashboard-*"):
|
|
shutil.copytree(skill, project / ".claude" / "skills" / skill.name)
|
|
|
|
run = subprocess.run(
|
|
[shutil.which("claude") or "claude", "-p", cfg["prompt"] + REPLY_FORMAT, "--output-format", "json",
|
|
"--permission-mode", "acceptEdits", "--allowedTools", "Skill", "Read", "Write", "Edit",
|
|
"Glob", "Grep", "Bash(python *)", "PowerShell(python *)"],
|
|
cwd=project, capture_output=True, text=True, encoding="utf-8", timeout=1800)
|
|
reply = json.loads(run.stdout).get("result", "") if run.stdout.strip().startswith("{") else run.stdout
|
|
|
|
def line(key):
|
|
m = re.search(rf"^\W*{key}\W*:(.*)$", reply, re.M | re.I)
|
|
return m[1] if m else ""
|
|
|
|
checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))}
|
|
for pat in cfg["asks"]:
|
|
checks[f"asks /{pat}/"] = bool(re.search(pat, line("QUESTIONS"), re.I))
|
|
for pat in cfg["never_asks"]:
|
|
checks[f"never asks /{pat}/"] = not re.search(pat, line("QUESTIONS"), re.I)
|
|
|
|
pages = [p for p in project.rglob("*.html") if ".claude" not in p.parts]
|
|
checks["exactly one page built"] = len(pages) == 1
|
|
results = ROOT / "tests" / "evals" / "results"
|
|
results.mkdir(exist_ok=True)
|
|
if pages:
|
|
html = pages[0].read_text(encoding="utf-8")
|
|
shutil.copy(pages[0], results / f"{fixture}.dashboard.html")
|
|
checks["no placeholders left"] = "{{" not in html
|
|
if cfg.get("brand"):
|
|
spec = importlib.util.spec_from_file_location("palette", ROOT / "rpa-dashboard-no-ai/templates/palette.py")
|
|
palette = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(palette)
|
|
t = palette.derive(cfg["brand"])
|
|
checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t)
|
|
pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"],
|
|
env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True)
|
|
checks["page check passes"] = pc.returncode == 0
|
|
page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]
|
|
else:
|
|
page_check = ["no page"]
|
|
|
|
out = {"fixture": fixture, "passed": all(checks.values()), "checks": checks, "page_check": page_check,
|
|
"files": sorted(str(p.relative_to(project)) for p in project.rglob("*") if p.is_file() and ".claude" not in p.parts),
|
|
"reply": reply}
|
|
(results / f"{fixture}.json").write_text(json.dumps(out, indent=2), encoding="utf-8")
|
|
for name, ok in checks.items():
|
|
print(f"{'PASS' if ok else 'FAIL'} {name}")
|
|
print("page check:", *page_check)
|
|
print("work dir:", project)
|
|
return 0 if out["passed"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1]))
|