Rpa_Skills/tests/evals/run_eval.py

"""Seam 2 skill eval: run the skills on a fixture project with `claude -p`, then grade.

Usage (from the repo root):
    uv run --with pytest --with playwright python tests/evals/run_eval.py a-empty

Copies tests/fixtures/<fixture> to a temp folder, installs every rpa-dashboard-*
skill there as a project skill, and runs `claude -p` with the user's answers in the
prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1 page
check on the page it built, and writes tests/evals/results/<fixture>.json.
Exit code 0 means every check passed.
"""
import importlib.util
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
REPLY_FORMAT = """
When you're done, end your reply with exactly these two lines:
FRONTEND: <what your scan found, or none>
QUESTIONS: <each question the skill told you to ask, in order, separated by " | ">"""

# Per fixture: the request, what the reply must (not) mention, and checks on the output.
FIXTURES = {
    "a-empty": {
        "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
                  "work, so here are my answers to anything the skill asks. Colours: custom, my "
                  "brand colour is #7a3cff. Desktop or web: desktop. Optional sections: none.",
        "frontend": r"^\s*(none|no\b)",
        "asks": [r"colou?r", r"desktop|web", r"section"],
        "never_asks": [r"\bkeep\b|restyle"],
        "brand": "#7a3cff",
        "sections": [],
    },
    "a-empty-sections": {
        "dir": "a-empty",
        "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
                  "work, so here are my answers to anything the skill asks. Colours: Superbill Ember. "
                  "Desktop or web: web. Optional sections: all three (checkpoint tracker, admin/roles "
                  "screen, login page). The project has no auth yet.",
        "frontend": r"^\s*(none|no\b)",
        "asks": [r"colou?r", r"desktop|web", r"section"],
        "never_asks": [r"\bkeep\b|restyle"],
        "sections": ["checkpoints", "admin", "login"],
    },
}


def main(fixture):
    cfg = FIXTURES[fixture]
    work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-"))
    project = work / "project"
    shutil.copytree(ROOT / "tests" / "fixtures" / cfg.get("dir", fixture), project)
    for skill in ROOT.glob("rpa-dashboard-*"):
        shutil.copytree(skill, project / ".claude" / "skills" / skill.name)

    # The prompt goes in on stdin: on Windows a multi-line argv prompt is cut at the first newline.
    run = subprocess.run(
        [shutil.which("claude") or "claude", "-p", "--output-format", "json",
         "--permission-mode", "acceptEdits", "--allowedTools", "Skill", "Read", "Write", "Edit",
         "Glob", "Grep", "Bash(python *)", "PowerShell(python *)"],
        input=cfg["prompt"] + REPLY_FORMAT, cwd=project, capture_output=True, text=True, encoding="utf-8",
        timeout=1800)
    reply = json.loads(run.stdout).get("result", "") if run.stdout.strip().startswith("{") else run.stdout

    def line(key):
        m = re.search(rf"^\W*{key}\W*:(.*)$", reply, re.M | re.I)
        return m[1] if m else ""

    checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))}
    for pat in cfg["asks"]:
        checks[f"asks /{pat}/"] = bool(re.search(pat, line("QUESTIONS"), re.I))
    for pat in cfg["never_asks"]:
        checks[f"never asks /{pat}/"] = not re.search(pat, line("QUESTIONS"), re.I)

    pages = [p for p in project.rglob("*.html") if ".claude" not in p.parts]
    checks["exactly one page built"] = len(pages) == 1
    results = ROOT / "tests" / "evals" / "results"
    results.mkdir(exist_ok=True)
    if pages:
        html = pages[0].read_text(encoding="utf-8")
        shutil.copy(pages[0], results / f"{fixture}.dashboard.html")
        checks["no placeholders left"] = "{{" not in html
        if cfg.get("brand"):
            spec = importlib.util.spec_from_file_location("palette", ROOT / "rpa-dashboard-no-ai/templates/palette.py")
            palette = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(palette)
            t = palette.derive(cfg["brand"])
            checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t)
        if "sections" in cfg:  # chosen sections built, the others leave no trace
            spec = importlib.util.spec_from_file_location("test_sections", ROOT / "tests" / "test_sections.py")
            traces = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(traces)
            for name, trace in traces.TRACES.items():
                want = name in cfg["sections"]
                checks[f"{name} {'built' if want else 'left no trace'}"] = bool(trace.search(html)) == want
        pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"],
                            env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True)
        checks["page check passes"] = pc.returncode == 0
        page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]
        if cfg.get("sections"):  # every chosen section's page check ran
            checks["no section checks skipped"] = "skipped" not in page_check[0]
    else:
        page_check = ["no page"]

    out = {"fixture": fixture, "passed": all(checks.values()), "checks": checks, "page_check": page_check,
           "files": sorted(str(p.relative_to(project)) for p in project.rglob("*") if p.is_file() and ".claude" not in p.parts),
           "reply": reply}
    (results / f"{fixture}.json").write_text(json.dumps(out, indent=2), encoding="utf-8")
    for name, ok in checks.items():
        print(f"{'PASS' if ok else 'FAIL'}  {name}")
    print("page check:", *page_check)
    print("work dir:", project)
    return 0 if out["passed"] else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1]))