Merge integration/rpa-dashboard-skills into ticket 04

Keeps both sides: frontend detection and bot wiring from tickets 02/03, optional
sections from ticket 04. The a-empty-sections eval now uses the shared "fixture" key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
VasanthandClaude Opus 5.5 committed 2026-10-16 15:04:34 +05:30
commit 3a11e55b58
29 files changed
+1964 -59

No files matched your search

+94 -6
View File
@@ -9,6 +9,7 @@ prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1
check on the page it built, and writes tests/evals/results/<fixture>.json.
Exit code 0 means every check passed.
"""
import difflib
import importlib.util
import json
import os
@@ -38,7 +39,7 @@ FIXTURES = {
"sections": [],
},
"a-empty-sections": {
"dir": "a-empty",
"fixture": "a-empty",
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
"work, so here are my answers to anything the skill asks. Colours: Superbill Ember. "
"Desktop or web: web. Optional sections: all three (checkpoint tracker, admin/roles "
@@ -48,14 +49,87 @@ FIXTURES = {
"never_asks": [r"\bkeep\b|restyle"],
"sections": ["checkpoints", "admin", "login"],
},
"b-pyqt-bot": {
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
"work, so here are my answers to anything the skill asks. Colours: Tebra Sky. "
"Optional sections: none.",
"frontend": r"^\s*(none|no\b)",
"asks": [r"colou?r", r"section"],
"never_asks": [r"\bkeep\b|restyle", r"desktop|web"],
"bot": True,
},
# (c) existing teal frontend in web/, run once per answer to keep-or-restyle.
"c-keep": {
"fixture": "c-web-existing-frontend",
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
"work, so here are my answers to anything the skill asks. Keep or restyle the "
"existing frontend: keep its look. Optional sections: none.",
"frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))",
"asks": [r"keep|restyle", r"section"],
"never_asks": [r"^(?!.*(keep|restyle)).*(colou?r|Tebra Sky|Ember)", r"desktop"],
"page_has": ["--accent:#0f766e", "Trebuchet MS"],
"existing": "unchanged",
},
"c-restyle": {
"fixture": "c-web-existing-frontend",
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
"work, so here are my answers to anything the skill asks. Keep or restyle the "
"existing frontend: restyle it. Colours: Superbill Ember. Optional sections: none.",
"frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))",
"asks": [r"keep|restyle", r"Tebra Sky|Ember", r"section"],
"never_asks": [r"desktop"],
"page_has": ["--accent:#ff3049"],
"existing": "#ff3049", # only token lines may change, and the theme now uses this colour
},
}
# A line a restyle may change: a custom property, a comment, a blank, or a web-font link.
TOKEN_LINE = r"^\s*(--[\w-]+\s*:.*|/\*.*\*/|<link[^>]*fonts\.googleapis\.com.*|@import.*fonts\.googleapis\.com.*)?\s*$"
def squash(s):
"""Ignore spacing and case when looking for a token in the built page."""
return re.sub(r"\s+", "", s).lower()
def changed_lines(src, project):
"""Lines removed or added in the fixture's own files."""
out = []
for f in src.rglob("*"):
if f.is_file():
new = project / f.relative_to(src)
after = new.read_text(encoding="utf-8").splitlines() if new.exists() else []
out += [x[1:] for x in difflib.unified_diff(f.read_text(encoding="utf-8").splitlines(), after, lineterm="", n=0)
if x[:1] in "+-" and x[:3] not in ("+++", "---")]
return out
def bot_checks(project, html, checks, replay):
"""Bot fixtures: the page is wired, and a dry run of the generated wiring gives real steps and logs.
Returns the env that points the page check at that run."""
checks["page wired to the bot (sample data off)"] = bool(re.search(r"SAMPLE_DATA\s*=\s*false", html))
runners = [p for p in project.rglob("bot_runner.py") if ".claude" not in p.parts]
checks["bot_runner.py built"] = len(runners) == 1
if not runners:
return {}
host = [runners[0], runners[0].with_name("bridge.py")]
checks["no placeholders left in the host"] = all(p.exists() and "{{" not in p.read_text(encoding="utf-8") for p in host)
dry = subprocess.run([sys.executable, str(runners[0])], cwd=project, capture_output=True, text=True,
encoding="utf-8", timeout=300)
replay.write_text(dry.stdout, encoding="utf-8")
events = [json.loads(x) for x in dry.stdout.splitlines() if x.startswith("{")]
steps = [e["data"] for e in events if e["event"] == "step"]
checks["bot dry run ends done"] = bool(events) and events[-1]["data"].get("state") == "done"
checks["real steps reach the page"] = len({s["label"] for s in steps}) >= 3 and steps[-1]["pct"] == 100
checks["bot error line reaches the log"] = any(e["event"] == "log" and e["data"]["type"] == "error" for e in events)
return {"PAGE_CHECK_WIRED": "1", "PAGE_CHECK_REPLAY": str(replay)}
def main(fixture):
cfg = FIXTURES[fixture]
work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-"))
project = work / "project"
shutil.copytree(ROOT / "tests" / "fixtures" / cfg.get("dir", fixture), project)
src = ROOT / "tests" / "fixtures" / cfg.get("fixture", fixture)
shutil.copytree(src, project)
for skill in ROOT.glob("rpa-dashboard-*"):
shutil.copytree(skill, project / ".claude" / "skills" / skill.name)
@@ -73,12 +147,21 @@ def main(fixture):
return m[1] if m else ""
checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))}
questions = line("QUESTIONS").split("|")
for pat in cfg["asks"]:
checks[f"asks /{pat}/"] = bool(re.search(pat, line("QUESTIONS"), re.I))
checks[f"asks /{pat}/"] = any(re.search(pat, q, re.I) for q in questions)
for pat in cfg["never_asks"]:
checks[f"never asks /{pat}/"] = not re.search(pat, line("QUESTIONS"), re.I)
checks[f"never asks /{pat}/"] = not any(re.search(pat, q, re.I) for q in questions)
if "existing" in cfg:
changed = changed_lines(src, project)
if cfg["existing"] == "unchanged":
checks["existing files unchanged"] = not changed
else:
checks["existing files: only token lines changed"] = bool(changed) and all(re.match(TOKEN_LINE, x) for x in changed)
checks[f"existing tokens now use {cfg['existing']}"] = any(cfg["existing"] in x.lower() for x in changed)
pages = [p for p in project.rglob("*.html") if ".claude" not in p.parts]
pages = [p for p in project.rglob("*.html")
if ".claude" not in p.parts and not (src / p.relative_to(project)).exists()] # new pages only
checks["exactly one page built"] = len(pages) == 1
results = ROOT / "tests" / "evals" / "results"
results.mkdir(exist_ok=True)
@@ -99,8 +182,13 @@ def main(fixture):
for name, trace in traces.TRACES.items():
want = name in cfg["sections"]
checks[f"{name} {'built' if want else 'left no trace'}"] = bool(trace.search(html)) == want
for needle in cfg.get("page_has", []):
checks[f"page has {needle}"] = squash(needle) in squash(html)
env = {**os.environ, "PAGE_CHECK_HTML": str(pages[0])}
if cfg.get("bot"):
env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl"))
pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"],
env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True)
env=env, capture_output=True, text=True)
checks["page check passes"] = pc.returncode == 0
page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]
if cfg.get("sections"): # every chosen section's page check ran