Optional sections: checkpoint tracker, admin/roles, login page
- dashboard.html: each section sits in OPTIONAL:<name> BEGIN/END blocks in the style, markup and script, with sample answers so a page with no bot shows it working. Core change: setUser records the role on <body>, so CSS hides admin-only parts from operators. - templates/sections.py keeps the chosen sections and deletes the rest, markers included; it stops on a broken marker. - templates/sections.md: what each section needs from the host (events, actions, desktop slot snippets, the role re-check, the login hand-off). - SKILL.md step 5 asks one multi-select question and runs sections.py. - Page check: checkpoint states, admin visibility by role and assign/remove, login theme and hand-off to auth or the stub. Cases skip on a generated page without the section; the template must carry all three. The harness signs in when the page has a login screen. - tests/test_sections.py: no sections leave no trace, and that page still passes the page check. - Eval runner: a-empty-sections variant (all three) and per-section checks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3eb3fdf00e
commit
9a5b95448d
10 files changed
+536
-10
No files matched your search
+22
-1
@@ -35,6 +35,18 @@ FIXTURES = {
|
||||
"asks": [r"colou?r", r"desktop|web", r"section"],
|
||||
"never_asks": [r"\bkeep\b|restyle"],
|
||||
"brand": "#7a3cff",
|
||||
"sections": [],
|
||||
},
|
||||
"a-empty-sections": {
|
||||
"dir": "a-empty",
|
||||
"prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
|
||||
"work, so here are my answers to anything the skill asks. Colours: Superbill Ember. "
|
||||
"Desktop or web: web. Optional sections: all three (checkpoint tracker, admin/roles "
|
||||
"screen, login page). The project has no auth yet.",
|
||||
"frontend": r"^\s*(none|no\b)",
|
||||
"asks": [r"colou?r", r"desktop|web", r"section"],
|
||||
"never_asks": [r"\bkeep\b|restyle"],
|
||||
"sections": ["checkpoints", "admin", "login"],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -43,7 +55,7 @@ def main(fixture):
|
||||
cfg = FIXTURES[fixture]
|
||||
work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-"))
|
||||
project = work / "project"
|
||||
shutil.copytree(ROOT / "tests" / "fixtures" / fixture, project)
|
||||
shutil.copytree(ROOT / "tests" / "fixtures" / cfg.get("dir", fixture), project)
|
||||
for skill in ROOT.glob("rpa-dashboard-*"):
|
||||
shutil.copytree(skill, project / ".claude" / "skills" / skill.name)
|
||||
|
||||
@@ -80,10 +92,19 @@ def main(fixture):
|
||||
spec.loader.exec_module(palette)
|
||||
t = palette.derive(cfg["brand"])
|
||||
checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t)
|
||||
if "sections" in cfg: # chosen sections built, the others leave no trace
|
||||
spec = importlib.util.spec_from_file_location("test_sections", ROOT / "tests" / "test_sections.py")
|
||||
traces = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(traces)
|
||||
for name, trace in traces.TRACES.items():
|
||||
want = name in cfg["sections"]
|
||||
checks[f"{name} {'built' if want else 'left no trace'}"] = bool(trace.search(html)) == want
|
||||
pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"],
|
||||
env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True)
|
||||
checks["page check passes"] = pc.returncode == 0
|
||||
page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]
|
||||
if cfg.get("sections"): # every chosen section's page check ran
|
||||
checks["no section checks skipped"] = "skipped" not in page_check[0]
|
||||
else:
|
||||
page_check = ["no page"]
|
||||
|
||||
|
||||
Reference in new issue
Block a user