No-AI skill tracer: SKILL.md, theme presets, sample data, Seam 1 page check

- rpa-dashboard-no-ai/SKILL.md: trigger description and the workflow in spec
  order (scan, report/keep-or-restyle, colours, stack, sections, build, wire,
  verify).
- templates/themes.md: Tebra Sky and Superbill Ember, light and dark blocks.
- templates/palette.py: derives a custom palette from one brand hex and
  checks text contrast in light and dark.
- dashboard.html: built-in sample data so a page with no bot works on first
  open; cleared batches in the donut; component tokens so Ember is a
  token-only swap; accessible names for KPI cards, nav, steps, log, search
  and the dark toggle.
- bridge.py: placeholders inside string literals, so it passes the lint hook
  unfilled.
- tests/page_check: Seam 1 harness (fake Qt bridge, light/dark/narrow).
- tests/evals: Seam 2 runner and fixture a-empty.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
VasanthandClaude Opus 5.5 committed 2026-10-16 13:59:26 +05:30
1 parent 49f8f49b65
commit 7f2ddecffb
13 files changed
+865 -57

No files matched your search

+100
View File
@@ -0,0 +1,100 @@
"""Seam 2 skill eval: run the skills on a fixture project with `claude -p`, then grade.
Usage (from the repo root):
uv run --with pytest --with playwright python tests/evals/run_eval.py a-empty
Copies tests/fixtures/<fixture> to a temp folder, installs every rpa-dashboard-*
skill there as a project skill, and runs `claude -p` with the user's answers in the
prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1 page
check on the page it built, and writes tests/evals/results/<fixture>.json.
Exit code 0 means every check passed.
"""
import importlib.util
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
REPLY_FORMAT = """
When you're done, end your reply with exactly these two lines:
FRONTEND: <what your scan found, or none>
QUESTIONS: <each question the skill told you to ask, in order, separated by " | ">"""
# Per fixture: the request, what the reply must (not) mention, and checks on the output.
FIXTURES = {
"a-empty": {
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
"work, so here are my answers to anything the skill asks. Colours: custom, my "
"brand colour is #7a3cff. Desktop or web: desktop. Optional sections: none.",
"frontend": r"^\s*(none|no\b)",
"asks": [r"colou?r", r"desktop|web", r"section"],
"never_asks": [r"\bkeep\b|restyle"],
"brand": "#7a3cff",
},
}
def main(fixture):
cfg = FIXTURES[fixture]
work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-"))
project = work / "project"
shutil.copytree(ROOT / "tests" / "fixtures" / fixture, project)
for skill in ROOT.glob("rpa-dashboard-*"):
shutil.copytree(skill, project / ".claude" / "skills" / skill.name)
run = subprocess.run(
[shutil.which("claude") or "claude", "-p", cfg["prompt"] + REPLY_FORMAT, "--output-format", "json",
"--permission-mode", "acceptEdits", "--allowedTools", "Skill", "Read", "Write", "Edit",
"Glob", "Grep", "Bash(python *)", "PowerShell(python *)"],
cwd=project, capture_output=True, text=True, encoding="utf-8", timeout=1800)
reply = json.loads(run.stdout).get("result", "") if run.stdout.strip().startswith("{") else run.stdout
def line(key):
m = re.search(rf"^\W*{key}\W*:(.*)$", reply, re.M | re.I)
return m[1] if m else ""
checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))}
for pat in cfg["asks"]:
checks[f"asks /{pat}/"] = bool(re.search(pat, line("QUESTIONS"), re.I))
for pat in cfg["never_asks"]:
checks[f"never asks /{pat}/"] = not re.search(pat, line("QUESTIONS"), re.I)
pages = [p for p in project.rglob("*.html") if ".claude" not in p.parts]
checks["exactly one page built"] = len(pages) == 1
results = ROOT / "tests" / "evals" / "results"
results.mkdir(exist_ok=True)
if pages:
html = pages[0].read_text(encoding="utf-8")
shutil.copy(pages[0], results / f"{fixture}.dashboard.html")
checks["no placeholders left"] = "{{" not in html
if cfg.get("brand"):
spec = importlib.util.spec_from_file_location("palette", ROOT / "rpa-dashboard-no-ai/templates/palette.py")
palette = importlib.util.module_from_spec(spec)
spec.loader.exec_module(palette)
t = palette.derive(cfg["brand"])
checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t)
pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"],
env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True)
checks["page check passes"] = pc.returncode == 0
page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]
else:
page_check = ["no page"]
out = {"fixture": fixture, "passed": all(checks.values()), "checks": checks, "page_check": page_check,
"files": sorted(str(p.relative_to(project)) for p in project.rglob("*") if p.is_file() and ".claude" not in p.parts),
"reply": reply}
(results / f"{fixture}.json").write_text(json.dumps(out, indent=2), encoding="utf-8")
for name, ok in checks.items():
print(f"{'PASS' if ok else 'FAIL'} {name}")
print("page check:", *page_check)
print("work dir:", project)
return 0 if out["passed"] else 1
if __name__ == "__main__":
sys.exit(main(sys.argv[1]))