Merge integration/rpa-dashboard-skills into ticket 04
Keeps both sides: frontend detection and bot wiring from tickets 02/03, optional sections from ticket 04. The a-empty-sections eval now uses the shared "fixture" key. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
3a11e55b58
29 files changed
+1964
-59
No files matched your search
+94
-6
@@ -9,6 +9,7 @@ prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1
|
||||
check on the page it built, and writes tests/evals/results/<fixture>.json.
|
||||
Exit code 0 means every check passed.
|
||||
"""
|
||||
import difflib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
@@ -38,7 +39,7 @@ FIXTURES = {
|
||||
"sections": [],
|
||||
},
|
||||
"a-empty-sections": {
|
||||
"dir": "a-empty",
|
||||
"fixture": "a-empty",
|
||||
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
||||
"work, so here are my answers to anything the skill asks. Colours: Superbill Ember. "
|
||||
"Desktop or web: web. Optional sections: all three (checkpoint tracker, admin/roles "
|
||||
@@ -48,14 +49,87 @@ FIXTURES = {
|
||||
"never_asks": [r"\bkeep\b|restyle"],
|
||||
"sections": ["checkpoints", "admin", "login"],
|
||||
},
|
||||
"b-pyqt-bot": {
|
||||
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
||||
"work, so here are my answers to anything the skill asks. Colours: Tebra Sky. "
|
||||
"Optional sections: none.",
|
||||
"frontend": r"^\s*(none|no\b)",
|
||||
"asks": [r"colou?r", r"section"],
|
||||
"never_asks": [r"\bkeep\b|restyle", r"desktop|web"],
|
||||
"bot": True,
|
||||
},
|
||||
# (c) existing teal frontend in web/, run once per answer to keep-or-restyle.
|
||||
"c-keep": {
|
||||
"fixture": "c-web-existing-frontend",
|
||||
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
||||
"work, so here are my answers to anything the skill asks. Keep or restyle the "
|
||||
"existing frontend: keep its look. Optional sections: none.",
|
||||
"frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))",
|
||||
"asks": [r"keep|restyle", r"section"],
|
||||
"never_asks": [r"^(?!.*(keep|restyle)).*(colou?r|Tebra Sky|Ember)", r"desktop"],
|
||||
"page_has": ["--accent:#0f766e", "Trebuchet MS"],
|
||||
"existing": "unchanged",
|
||||
},
|
||||
"c-restyle": {
|
||||
"fixture": "c-web-existing-frontend",
|
||||
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
||||
"work, so here are my answers to anything the skill asks. Keep or restyle the "
|
||||
"existing frontend: restyle it. Colours: Superbill Ember. Optional sections: none.",
|
||||
"frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))",
|
||||
"asks": [r"keep|restyle", r"Tebra Sky|Ember", r"section"],
|
||||
"never_asks": [r"desktop"],
|
||||
"page_has": ["--accent:#ff3049"],
|
||||
"existing": "#ff3049", # only token lines may change, and the theme now uses this colour
|
||||
},
|
||||
}
|
||||
# A line a restyle may change: a custom property, a comment, a blank, or a web-font link.
|
||||
TOKEN_LINE = r"^\s*(--[\w-]+\s*:.*|/\*.*\*/|<link[^>]*fonts\.googleapis\.com.*|@import.*fonts\.googleapis\.com.*)?\s*$"
|
||||
|
||||
|
||||
def squash(s):
|
||||
"""Ignore spacing and case when looking for a token in the built page."""
|
||||
return re.sub(r"\s+", "", s).lower()
|
||||
|
||||
|
||||
def changed_lines(src, project):
|
||||
"""Lines removed or added in the fixture's own files."""
|
||||
out = []
|
||||
for f in src.rglob("*"):
|
||||
if f.is_file():
|
||||
new = project / f.relative_to(src)
|
||||
after = new.read_text(encoding="utf-8").splitlines() if new.exists() else []
|
||||
out += [x[1:] for x in difflib.unified_diff(f.read_text(encoding="utf-8").splitlines(), after, lineterm="", n=0)
|
||||
if x[:1] in "+-" and x[:3] not in ("+++", "---")]
|
||||
return out
|
||||
|
||||
|
||||
def bot_checks(project, html, checks, replay):
|
||||
"""Bot fixtures: the page is wired, and a dry run of the generated wiring gives real steps and logs.
|
||||
Returns the env that points the page check at that run."""
|
||||
checks["page wired to the bot (sample data off)"] = bool(re.search(r"SAMPLE_DATA\s*=\s*false", html))
|
||||
runners = [p for p in project.rglob("bot_runner.py") if ".claude" not in p.parts]
|
||||
checks["bot_runner.py built"] = len(runners) == 1
|
||||
if not runners:
|
||||
return {}
|
||||
host = [runners[0], runners[0].with_name("bridge.py")]
|
||||
checks["no placeholders left in the host"] = all(p.exists() and "{{" not in p.read_text(encoding="utf-8") for p in host)
|
||||
dry = subprocess.run([sys.executable, str(runners[0])], cwd=project, capture_output=True, text=True,
|
||||
encoding="utf-8", timeout=300)
|
||||
replay.write_text(dry.stdout, encoding="utf-8")
|
||||
events = [json.loads(x) for x in dry.stdout.splitlines() if x.startswith("{")]
|
||||
steps = [e["data"] for e in events if e["event"] == "step"]
|
||||
checks["bot dry run ends done"] = bool(events) and events[-1]["data"].get("state") == "done"
|
||||
checks["real steps reach the page"] = len({s["label"] for s in steps}) >= 3 and steps[-1]["pct"] == 100
|
||||
checks["bot error line reaches the log"] = any(e["event"] == "log" and e["data"]["type"] == "error" for e in events)
|
||||
return {"PAGE_CHECK_WIRED": "1", "PAGE_CHECK_REPLAY": str(replay)}
|
||||
|
||||
|
||||
def main(fixture):
|
||||
cfg = FIXTURES[fixture]
|
||||
work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-"))
|
||||
project = work / "project"
|
||||
shutil.copytree(ROOT / "tests" / "fixtures" / cfg.get("dir", fixture), project)
|
||||
src = ROOT / "tests" / "fixtures" / cfg.get("fixture", fixture)
|
||||
shutil.copytree(src, project)
|
||||
for skill in ROOT.glob("rpa-dashboard-*"):
|
||||
shutil.copytree(skill, project / ".claude" / "skills" / skill.name)
|
||||
|
||||
@@ -73,12 +147,21 @@ def main(fixture):
|
||||
return m[1] if m else ""
|
||||
|
||||
checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))}
|
||||
questions = line("QUESTIONS").split("|")
|
||||
for pat in cfg["asks"]:
|
||||
checks[f"asks /{pat}/"] = bool(re.search(pat, line("QUESTIONS"), re.I))
|
||||
checks[f"asks /{pat}/"] = any(re.search(pat, q, re.I) for q in questions)
|
||||
for pat in cfg["never_asks"]:
|
||||
checks[f"never asks /{pat}/"] = not re.search(pat, line("QUESTIONS"), re.I)
|
||||
checks[f"never asks /{pat}/"] = not any(re.search(pat, q, re.I) for q in questions)
|
||||
if "existing" in cfg:
|
||||
changed = changed_lines(src, project)
|
||||
if cfg["existing"] == "unchanged":
|
||||
checks["existing files unchanged"] = not changed
|
||||
else:
|
||||
checks["existing files: only token lines changed"] = bool(changed) and all(re.match(TOKEN_LINE, x) for x in changed)
|
||||
checks[f"existing tokens now use {cfg['existing']}"] = any(cfg["existing"] in x.lower() for x in changed)
|
||||
|
||||
pages = [p for p in project.rglob("*.html") if ".claude" not in p.parts]
|
||||
pages = [p for p in project.rglob("*.html")
|
||||
if ".claude" not in p.parts and not (src / p.relative_to(project)).exists()] # new pages only
|
||||
checks["exactly one page built"] = len(pages) == 1
|
||||
results = ROOT / "tests" / "evals" / "results"
|
||||
results.mkdir(exist_ok=True)
|
||||
@@ -99,8 +182,13 @@ def main(fixture):
|
||||
for name, trace in traces.TRACES.items():
|
||||
want = name in cfg["sections"]
|
||||
checks[f"{name} {'built' if want else 'left no trace'}"] = bool(trace.search(html)) == want
|
||||
for needle in cfg.get("page_has", []):
|
||||
checks[f"page has {needle}"] = squash(needle) in squash(html)
|
||||
env = {**os.environ, "PAGE_CHECK_HTML": str(pages[0])}
|
||||
if cfg.get("bot"):
|
||||
env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl"))
|
||||
pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"],
|
||||
env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True)
|
||||
env=env, capture_output=True, text=True)
|
||||
checks["page check passes"] = pc.returncode == 0
|
||||
page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]
|
||||
if cfg.get("sections"): # every chosen section's page check ran
|
||||
|
||||
Reference in new issue
Block a user