From f11bddd95832cd3553d136d37f2f72aa92907c93 Mon Sep 17 00:00:00 2001 From: Vasanth Date: Fri, 16 Oct 2026 14:27:50 +0530 Subject: [PATCH 1/2] Existing frontend: detect, report, then keep or restyle (ticket 03) - templates/detect_frontend.py: finds plain HTML, React/Vue/Angular/Svelte apps and PyQt web views; prints location, stack, main colours, font and every token block (CSS custom properties, ", text, re.DOTALL | re.IGNORECASE)) + + +def token_blocks(css): + """[(selector path, {token: value})] for each innermost rule that declares custom properties.""" + css = re.sub(r"/\*.*?\*/", "", css, flags=re.DOTALL) + out, stack, last = [], [], 0 + for m in re.finditer(r"[{}]", css): + if m[0] == "{": + if stack: + stack[-1][2] = True # has a nested rule + stack.append([css[last:m.start()].split(";")[-1].strip(), m.end(), False]) + elif stack: + sel, start, nested = stack.pop() + # ponytail: declarations next to a nested rule (CSS nesting) are skipped; token blocks rarely nest + tokens = {} if nested else {k: v.strip() for k, v in re.findall(r"(--[\w-]+)\s*:\s*([^;]+)", css[start:m.start()])} + if tokens: + out.append((" ".join([s for s, *_ in stack] + [sel]), tokens)) + last = m.end() + return out + + +def rel(root, p): + r = p.relative_to(root).as_posix() + return "./" if r == "." else r + "/" if (root / r).is_dir() else r + + +def tailwind_colours(path): + """Quoted hex colours in a Tailwind v3 config, by key.""" + # ponytail: regex, not a JS parser; nested shades keep only their own key (e.g. "500") + return dict(re.findall(r"""['"]?([\w-]+)['"]?\s*:\s*['"](#[0-9a-fA-F]{3,8})['"]""", path.read_text(encoding="utf-8"))) + + +def styled(p): + return p.suffix.lower() in MARKUP | STYLE or p.name.startswith("tailwind.config.") + + +def frontend(root, path, stack, files): + """files: the frontend's style-bearing files (see styled) plus any PyQt host.""" + blocks = [] + for f in files: + found = [("theme", tailwind_colours(f))] if f.name.startswith("tailwind.config.") else token_blocks(css_of(f)) + blocks += [{"file": rel(root, f), "selector": sel, "dark": "dark" in sel.lower(), "tokens": t} for sel, t in found if t] + return {"path": rel(root, path), "stack": stack, "blocks": blocks} + + +def detect(root): + root = Path(root).resolve() + files = list(walk(root)) + found = [] + for p in files: + if p.name != "package.json": + continue + try: + pkg = json.loads(p.read_text(encoding="utf-8")) + except ValueError: + continue + deps = {**pkg.get("dependencies", {}), **pkg.get("devDependencies", {})} + name = next((n for k, n in FRAMEWORKS if k in deps), None) + if name: + stack = f"{name} ({', '.join([n for k, n in TOOLS if k in deps] + ['web'])})" + found.append(frontend(root, p.parent, stack, [f for f in files if p.parent in f.parents and styled(f)])) + apps = [root / f["path"] for f in found] + rest = [p for p in files if not any(a in p.parents for a in apps)] + pages = [p for p in rest if p.suffix.lower() in PAGE] + hosts = [p for p in rest if p.suffix == ".py" and "QWebEngineView" in p.read_text(encoding="utf-8", errors="replace")] + if pages or hosts: + styles = [p for p in rest if styled(p) and p not in pages] + # ponytail: one plain-HTML/PyQt frontend per project, at the common folder of its pages and styles + home = Path(os.path.commonpath([p.parent for p in pages + styles])) if pages else hosts[0] + stack = f"PyQt web view (desktop, host `{rel(root, hosts[0])}`)" if hosts else "plain HTML (web)" + found.append(frontend(root, home, stack, pages + styles + hosts)) + return found + + +def report(found): + if not found: + return "No existing frontend found." + lines = [] + for f in found: + light = [b["tokens"] for b in f["blocks"] if not b["dark"]] + colours = sorted(((k, v) for t in light for k, v in t.items() if COLOUR.search(v)), key=lambda kv: not BRAND.search(kv[0])) + fonts = [v for t in light for k, v in t.items() if FONT_TOKEN.search(k)] + shown = ", ".join(f"{k} {v}" for k, v in colours[:6]) + (f" (+{len(colours) - 6} more)" if len(colours) > 6 else "") + lines.append(f"Frontend found at `{f['path']}`, stack `{f['stack']}`, colours `{shown or 'none found'}`" + + (f", fonts `{fonts[0]}`" if fonts else "") + ".") + for b in f["blocks"]: + lines.append(f" {b['file']} {b['selector']}{' (dark)' if b['dark'] else ''}") + lines.append(" " + "; ".join(f"{k}:{v}" for k, v in b["tokens"].items())) + return "\n".join(lines) + + +if __name__ == "__main__": + print(report(detect(sys.argv[1] if len(sys.argv) > 1 else "."))) diff --git a/rpa-dashboard-no-ai/templates/frontend.md b/rpa-dashboard-no-ai/templates/frontend.md new file mode 100644 index 0000000..694adab --- /dev/null +++ b/rpa-dashboard-no-ai/templates/frontend.md @@ -0,0 +1,114 @@ +# Existing frontend: detect, report, keep or restyle + +Shared by `rpa-dashboard-no-ai` and `rpa-dashboard-ai`. The files named here sit +next to this one, in `rpa-dashboard-no-ai/templates/`. From the AI skill, reach them +through `../rpa-dashboard-no-ai/templates/`. + +## 1. Detect + +Run this before asking anything: + +``` +python /templates/detect_frontend.py +``` + +It finds: +- React, Vue, Angular and Svelte apps (a `package.json` that lists one); +- a PyQt `QWebEngineView` host and the pages it shows; +- plain HTML pages and their stylesheets. + +It skips `node_modules`, virtualenvs, dot folders (including `.claude`), build +output, coverage and `docs`. For each frontend it prints: +- one line: location, stack (with `web` or `desktop`), main colours and font; +- then every token block: the file, the selector (dark blocks are marked) and + the tokens. + +Tokens come from: +- CSS custom properties, including Tailwind v4 `@theme`; +- ` + + +
+ + +
+
+
Dashboard
Overview of every batch
+
+ + +
System ready
+
+
+ +
+ +
+
+
0
Total batches
+
0
Completed
+
0
In progress
+
0
Not completed
+
+
+
Batch status split
+
    +
    +
    Recent batches
    No batches yet
    +
    +
    + + + + + + +
    +
    +
    + + + + diff --git a/tests/evals/results/c-keep.json b/tests/evals/results/c-keep.json new file mode 100644 index 0000000..aba94f2 --- /dev/null +++ b/tests/evals/results/c-keep.json @@ -0,0 +1,29 @@ +{ + "fixture": "c-keep", + "passed": true, + "checks": { + "frontend reported correctly": true, + "asks /keep|restyle/": true, + "asks /section/": true, + "never asks /^(?!.*(keep|restyle)).*(colou?r|Tebra Sky|Ember)/": true, + "never asks /desktop/": true, + "existing files unchanged": true, + "exactly one page built": true, + "no placeholders left": true, + "page has --accent:#0f766e": true, + "page has Trebuchet MS": true, + "page check passes": true + }, + "page_check": [ + "22 passed in 18.02s" + ], + "files": [ + "README.md", + "web\\css\\app.css", + "web\\css\\theme.css", + "web\\dashboard.html", + "web\\index.html", + "web\\js\\app.js" + ], + "reply": "Dashboard ready \u2014 **`web/dashboard.html`** (web path, plain HTML, sample data).\n\n| Item | Status |\n|---|---|\n| Look | **Kept** \u2014 the colours and fonts come from `web/css/theme.css` (teal brand, Trebuchet/Georgia), for both light and dark |\n| Existing files | **Not touched** (index.html, theme.css, app.css, app.js are unchanged) |\n| Sections | Sidebar, top bar (dark mode toggle + status dot), KPI cards, status donut, Run panel (steps + log), Report table (search + Excel). No optional sections. |\n| Data | `SAMPLE_DATA = true` \u2014 there is no bot yet. Steps are `Initializing \u2192 Load jobs \u2192 Process jobs \u2192 Completed`. Report columns are Job / Site / Status. |\n\n**Checks (headless Playwright):** 6 KPI batches, 4 legend items, 4/4 steps done, 5 log lines, 4/4 report rows with pills. No `{{` left and no console errors. I checked light, dark and 800px wide.\n\n- **Small tweak:** the Run button was the same teal as the top bar, so it was hard to see. I set `--primary-bg` to the project's own `--color-brand-strong` (#115e59).\n- **Palette fill-ins:** the project has no token for `--accent-2`, `--accent-dark` or `--accent-soft-2`, so those come from `palette.py`. All contrast checks pass.\n- **When the bot exists:** it must send the `status`, `step`, `log`, `dashboard_data` and `report_data` events. Set `SAMPLE_DATA=false` and serve `/api/dashboard` and `/api/run/status`.\n- **Navigation:** add a link to `dashboard.html` from your nav in `index.html`. I didn't edit that file.\n\nFRONTEND: web/ \u2014 plain HTML (web), tokens in web/css/theme.css (:root + [data-theme=\"dark\"]), brand #0f766e, fonts \"Trebuchet MS\"/Georgia\nQUESTIONS: Keep its look, or restyle it? | Which optional sections should I add: checkpoint tracker, admin/roles screen, login page, or none?" +} \ No newline at end of file diff --git a/tests/evals/results/c-restyle.dashboard.html b/tests/evals/results/c-restyle.dashboard.html new file mode 100644 index 0000000..b700e20 --- /dev/null +++ b/tests/evals/results/c-restyle.dashboard.html @@ -0,0 +1,309 @@ + + + + + + +Field Ops Portal + + + + + +
    + + +
    +
    +
    Dashboard
    Overview of every batch
    +
    + + +
    System ready
    +
    +
    + +
    + +
    +
    +
    0
    Total batches
    +
    0
    Completed
    +
    0
    In progress
    +
    0
    Not completed
    +
    +
    +
    Batch status split
    +
      +
      +
      Recent batches
      No batches yet
      +
      +
      + + + + + + +
      +
      +
      + + + + diff --git a/tests/evals/results/c-restyle.json b/tests/evals/results/c-restyle.json new file mode 100644 index 0000000..59a8cb7 --- /dev/null +++ b/tests/evals/results/c-restyle.json @@ -0,0 +1,29 @@ +{ + "fixture": "c-restyle", + "passed": true, + "checks": { + "frontend reported correctly": true, + "asks /keep|restyle/": true, + "asks /Tebra Sky|Ember/": true, + "asks /section/": true, + "never asks /desktop/": true, + "existing files: only token lines changed": true, + "existing tokens now use #ff3049": true, + "exactly one page built": true, + "no placeholders left": true, + "page has --accent:#ff3049": true, + "page check passes": true + }, + "page_check": [ + "22 passed in 23.29s" + ], + "files": [ + "README.md", + "web\\css\\app.css", + "web\\css\\theme.css", + "web\\dashboard.html", + "web\\index.html", + "web\\js\\app.js" + ], + "reply": "Dashboard ready, Superbill Ember theme-la restyle pannirukken. Frontend `web/`-la irundhudhu (plain HTML, web). Andha site-la token lines and oru font link mattum dhaan maathinen.\n\n| Item | Result |\n|---|---|\n| Pudhu page | `web/dashboard.html` (sample data, `SAMPLE_DATA = true`) |\n| Sections | Shell, KPI, donut, run panel, report. Optional sections edhuvum illa; `OPTIONAL` marker-um illa |\n| Restyle | `web/css/theme.css` token values + `web/index.html`-la font link |\n| Check | `{{` 0 left \u00b7 KPI total 6 \u00b7 legend 4 \u00b7 steps 4/4, log 5 lines, status \"Completed\" \u00b7 report 4 rows / 4 pills \u00b7 light, dark, 800px screenshots paathen, ellam correct-ah irukku |\n\n**Existing files-la changed lines:**\n```\ntheme.css :root --color-brand #0f766e\u2192#ff3049 \u00b7 -strong #115e59\u2192#18151f \u00b7 -tint #ccfbf1\u2192#f7f6fa\n --color-bg \u2192#f4f3f7 \u00b7 -surface-alt \u2192#f7f6fa \u00b7 -border \u2192#e2dfe9 \u00b7 -text \u2192#18151f \u00b7 -text-muted \u2192#66607a\n -success \u2192#1f9d57 \u00b7 -warning \u2192#b7791f \u00b7 -danger \u2192#c0392b\n --font-sans \u2192'IBM Plex Sans',\u2026 \u00b7 --font-heading \u2192'Sora',\u2026 (--color-surface, --radius maarala)\ntheme.css [dark] brand \u2192#ff3049 \u00b7 -strong \u2192#100e1a \u00b7 -tint \u2192#1b1727 \u00b7 bg \u2192#0a0912 \u00b7 surface \u2192#151220\n surface-alt \u2192#1b1727 \u00b7 border \u2192#2a2438 \u00b7 text \u2192#f3f1f8 \u00b7 muted \u2192#8d87a0 \u00b7 ok/warn/bad \u2192#2fbf72/#d9a13c/#e05a4b\nindex.html + Google Fonts (IBM Plex Sans + Sora), theme.css link-ku munnaadi\n```\n\nBot innum illa. Bot ready aana, backend-ah wire pannanum: `GET /api/dashboard`, `GET /api/run/status` (idhu `{running, events[]}` return pannanum), and `POST /api/`. Bot anuppa vendiya events: `status`, `step`, `log`, `dashboard_data`, `report_data`. `index.html`-oda nav-la `dashboard.html`-ku link nee dhaan podanum; adha naan touch pannala.\n\n**Also noticed (fix pannala):**\n- `app.css`-la hard-coded `#ffffff` irukku: topbar text, links, `.button`, `.tag`. Token-only restyle-nala adhu maarala.\n- Portal dark mode-la \"Dark mode\" button text theriyala. Text-um background-um ore colour family (`brand-strong` on `brand-tint`). Idhu munnaadiye irundha issue; teal theme-la rendum `#134e4a` dhaan.\n- Dashboard-la file:// vazhiya open pannum bodhu dark toggle `set-dark`-ku POST anuppudhu, adhu console-la oru error kaattudhu. Page-ku edhuvum aagala; server vazhiya serve pannina indha error varaadhu.\n\nFRONTEND: web/ \u2014 plain HTML (web), tokens in web/css/theme.css (:root + [data-theme=\"dark\"]), brand #0f766e teal, fonts \"Trebuchet MS\" / Georgia\nQUESTIONS: Keep its look, or restyle it? | Which colours: Tebra Sky, Superbill Ember, or Custom (one brand hex)? | Which optional sections to add: checkpoint tracker, admin/roles screen, login page, or none?" +} \ No newline at end of file diff --git a/tests/evals/run_eval.py b/tests/evals/run_eval.py index 5d9b532..e0de65c 100644 --- a/tests/evals/run_eval.py +++ b/tests/evals/run_eval.py @@ -9,6 +9,7 @@ prompt (`-p` can't answer questions). It then grades the reply, runs the Seam 1 check on the page it built, and writes tests/evals/results/.json. Exit code 0 means every check passed. """ +import difflib import importlib.util import json import os @@ -36,14 +37,57 @@ FIXTURES = { "never_asks": [r"\bkeep\b|restyle"], "brand": "#7a3cff", }, + # (c) existing teal frontend in web/, run once per answer to keep-or-restyle. + "c-keep": { + "fixture": "c-web-existing-frontend", + "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you " + "work, so here are my answers to anything the skill asks. Keep or restyle the " + "existing frontend: keep its look. Optional sections: none.", + "frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))", + "asks": [r"keep|restyle", r"section"], + "never_asks": [r"^(?!.*(keep|restyle)).*(colou?r|Tebra Sky|Ember)", r"desktop"], + "page_has": ["--accent:#0f766e", "Trebuchet MS"], + "existing": "unchanged", + }, + "c-restyle": { + "fixture": "c-web-existing-frontend", + "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you " + "work, so here are my answers to anything the skill asks. Keep or restyle the " + "existing frontend: restyle it. Colours: Superbill Ember. Optional sections: none.", + "frontend": r"(?=.*\bweb\b)(?=.*html)(?=.*(#0f766e|--color-brand))", + "asks": [r"keep|restyle", r"Tebra Sky|Ember", r"section"], + "never_asks": [r"desktop"], + "page_has": ["--accent:#ff3049"], + "existing": "#ff3049", # only token lines may change, and the theme now uses this colour + }, } +# A line a restyle may change: a custom property, a comment, a blank, or a web-font link. +TOKEN_LINE = r"^\s*(--[\w-]+\s*:.*|/\*.*\*/|]*fonts\.googleapis\.com.*|@import.*fonts\.googleapis\.com.*)?\s*$" + + +def squash(s): + """Ignore spacing and case when looking for a token in the built page.""" + return re.sub(r"\s+", "", s).lower() + + +def changed_lines(src, project): + """Lines removed or added in the fixture's own files.""" + out = [] + for f in src.rglob("*"): + if f.is_file(): + new = project / f.relative_to(src) + after = new.read_text(encoding="utf-8").splitlines() if new.exists() else [] + out += [x[1:] for x in difflib.unified_diff(f.read_text(encoding="utf-8").splitlines(), after, lineterm="", n=0) + if x[:1] in "+-" and x[:3] not in ("+++", "---")] + return out def main(fixture): cfg = FIXTURES[fixture] work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-")) project = work / "project" - shutil.copytree(ROOT / "tests" / "fixtures" / fixture, project) + src = ROOT / "tests" / "fixtures" / cfg.get("fixture", fixture) + shutil.copytree(src, project) for skill in ROOT.glob("rpa-dashboard-*"): shutil.copytree(skill, project / ".claude" / "skills" / skill.name) @@ -61,12 +105,21 @@ def main(fixture): return m[1] if m else "" checks = {"frontend reported correctly": bool(re.search(cfg["frontend"], line("FRONTEND"), re.I))} + questions = line("QUESTIONS").split("|") for pat in cfg["asks"]: - checks[f"asks /{pat}/"] = bool(re.search(pat, line("QUESTIONS"), re.I)) + checks[f"asks /{pat}/"] = any(re.search(pat, q, re.I) for q in questions) for pat in cfg["never_asks"]: - checks[f"never asks /{pat}/"] = not re.search(pat, line("QUESTIONS"), re.I) + checks[f"never asks /{pat}/"] = not any(re.search(pat, q, re.I) for q in questions) + if "existing" in cfg: + changed = changed_lines(src, project) + if cfg["existing"] == "unchanged": + checks["existing files unchanged"] = not changed + else: + checks["existing files: only token lines changed"] = bool(changed) and all(re.match(TOKEN_LINE, x) for x in changed) + checks[f"existing tokens now use {cfg['existing']}"] = any(cfg["existing"] in x.lower() for x in changed) - pages = [p for p in project.rglob("*.html") if ".claude" not in p.parts] + pages = [p for p in project.rglob("*.html") + if ".claude" not in p.parts and not (src / p.relative_to(project)).exists()] # new pages only checks["exactly one page built"] = len(pages) == 1 results = ROOT / "tests" / "evals" / "results" results.mkdir(exist_ok=True) @@ -80,6 +133,8 @@ def main(fixture): spec.loader.exec_module(palette) t = palette.derive(cfg["brand"]) checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t) + for needle in cfg.get("page_has", []): + checks[f"page has {needle}"] = squash(needle) in squash(html) pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"], env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True) checks["page check passes"] = pc.returncode == 0 diff --git a/tests/fixtures/c-web-existing-frontend/README.md b/tests/fixtures/c-web-existing-frontend/README.md new file mode 100644 index 0000000..aba6eab --- /dev/null +++ b/tests/fixtures/c-web-existing-frontend/README.md @@ -0,0 +1,10 @@ +# Fixture (c): web project with an existing frontend + +A plain HTML site in `web/` with its own teal theme: light and dark tokens in +`web/css/theme.css`, layout in `web/css/app.css`, logic in `web/js/app.js`. No bot. + +The no-AI skill should report this frontend (location, stack, colours), ask keep or +restyle, never ask desktop or web, and build a dashboard page on sample data. +- Keep: no colour question; the page uses the theme's tokens and fonts; no existing + file changes. +- Restyle: asks for colours; only token lines in the existing files change. diff --git a/tests/fixtures/c-web-existing-frontend/web/css/app.css b/tests/fixtures/c-web-existing-frontend/web/css/app.css new file mode 100644 index 0000000..5182090 --- /dev/null +++ b/tests/fixtures/c-web-existing-frontend/web/css/app.css @@ -0,0 +1,18 @@ +/* Layout and components. Colours and fonts come from theme.css. */ +* { box-sizing: border-box; margin: 0; } +body { font-family: var(--font-sans); background: var(--color-bg); color: var(--color-text); } +h1, h2 { font-family: var(--font-heading); } +.topbar { display: flex; align-items: center; gap: 16px; padding: 14px 24px; background: var(--color-brand); color: #ffffff; } +.topbar nav { margin-left: auto; display: flex; gap: 12px; } +.topbar a { color: #ffffff; text-decoration: none; } +.page { max-width: 960px; margin: 24px auto; padding: 0 16px; } +.card { background: var(--color-surface); border: 1px solid var(--color-border); border-radius: var(--radius); padding: 20px; } +.button { background: var(--color-brand); color: #ffffff; border: 0; border-radius: var(--radius); padding: 8px 14px; cursor: pointer; } +.button.secondary { background: var(--color-brand-tint); color: var(--color-brand-strong); } +table { width: 100%; border-collapse: collapse; margin-top: 12px; } +th { text-align: left; background: var(--color-surface-alt); color: var(--color-text-muted); } +th, td { padding: 8px 10px; border-bottom: 1px solid var(--color-border); } +.tag { padding: 2px 8px; border-radius: 999px; color: #ffffff; font-size: 12px; } +.tag.ok { background: var(--color-success); } +.tag.late { background: var(--color-warning); } +.tag.failed { background: var(--color-danger); } diff --git a/tests/fixtures/c-web-existing-frontend/web/css/theme.css b/tests/fixtures/c-web-existing-frontend/web/css/theme.css new file mode 100644 index 0000000..3cf2438 --- /dev/null +++ b/tests/fixtures/c-web-existing-frontend/web/css/theme.css @@ -0,0 +1,33 @@ +/* Field Ops Portal theme. Every colour and font the site uses comes from here. */ +:root { + --color-brand: #0f766e; + --color-brand-strong: #115e59; + --color-brand-tint: #ccfbf1; + --color-bg: #f8fafc; + --color-surface: #ffffff; + --color-surface-alt: #f1f5f9; + --color-border: #e2e8f0; + --color-text: #0f172a; + --color-text-muted: #64748b; + --color-success: #15803d; + --color-warning: #b45309; + --color-danger: #b91c1c; + --font-sans: "Trebuchet MS", "Segoe UI", sans-serif; + --font-heading: Georgia, "Times New Roman", serif; + --radius: 6px; +} + +[data-theme="dark"] { + --color-brand: #0d9488; + --color-brand-strong: #134e4a; + --color-brand-tint: #134e4a; + --color-bg: #0b1120; + --color-surface: #111827; + --color-surface-alt: #1f2937; + --color-border: #334155; + --color-text: #e2e8f0; + --color-text-muted: #94a3b8; + --color-success: #22c55e; + --color-warning: #f59e0b; + --color-danger: #ef4444; +} diff --git a/tests/fixtures/c-web-existing-frontend/web/index.html b/tests/fixtures/c-web-existing-frontend/web/index.html new file mode 100644 index 0000000..543512a --- /dev/null +++ b/tests/fixtures/c-web-existing-frontend/web/index.html @@ -0,0 +1,26 @@ + + + + + +Field Ops Portal + + + + +
      +

      Field Ops Portal

      + +
      +
      +
      +

      Today's jobs

      + + + +
      JobSiteStatus
      +
      +
      + + + diff --git a/tests/fixtures/c-web-existing-frontend/web/js/app.js b/tests/fixtures/c-web-existing-frontend/web/js/app.js new file mode 100644 index 0000000..5741688 --- /dev/null +++ b/tests/fixtures/c-web-existing-frontend/web/js/app.js @@ -0,0 +1,15 @@ +// Field Ops Portal: job list and dark toggle. +const JOBS = [ + { job: "J-101", site: "North depot", status: "Done", tag: "ok" }, + { job: "J-102", site: "Harbour road", status: "Late", tag: "late" }, + { job: "J-103", site: "West yard", status: "Failed", tag: "failed" }, +]; + +document.getElementById("jobs").innerHTML = JOBS.map( + (j) => `${j.job}${j.site}${j.status}` +).join(""); + +document.getElementById("theme-btn").onclick = () => { + const root = document.documentElement; + root.dataset.theme = root.dataset.theme === "dark" ? "light" : "dark"; +}; diff --git a/tests/test_detect_frontend.py b/tests/test_detect_frontend.py new file mode 100644 index 0000000..37fb4fa --- /dev/null +++ b/tests/test_detect_frontend.py @@ -0,0 +1,105 @@ +"""Frontend detection: where an existing frontend is, its stack and its colour tokens.""" +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +DETECT = Path(__file__).resolve().parents[1] / "rpa-dashboard-no-ai" / "templates" / "detect_frontend.py" +spec = importlib.util.spec_from_file_location("detect_frontend", DETECT) +detect_frontend = importlib.util.module_from_spec(spec) +spec.loader.exec_module(detect_frontend) + + +def project(tmp_path, files): + for name, text in files.items(): + (tmp_path / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / name).write_text(text, encoding="utf-8") + return tmp_path + + +def cli(root): + return subprocess.run([sys.executable, str(DETECT), str(root)], capture_output=True, text=True, check=True).stdout + + +def test_no_frontend_when_only_tooling_and_installed_skills_have_html(tmp_path): + root = project(tmp_path, { + "main.py": "print('[step 1] Login')\n", + "node_modules/pkg/index.html": "", + ".venv/lib/site.html": "

      ", + ".claude/skills/rpa-dashboard-no-ai/templates/dashboard.html": "", + "dist/index.html": "

      ", + }) + assert detect_frontend.detect(root) == [] + assert cli(root).strip() == "No existing frontend found." + + +FIXTURE_C = Path(__file__).resolve().parent / "fixtures" / "c-web-existing-frontend" + + +def test_plain_html_site_reports_location_stack_light_and_dark_tokens(): + [f] = detect_frontend.detect(FIXTURE_C) + assert f["path"] == "web/" + assert f["stack"] == "plain HTML (web)" + light, dark = f["blocks"] + assert (light["file"], light["selector"], light["dark"]) == ("web/css/theme.css", ":root", False) + assert light["tokens"]["--color-brand"] == "#0f766e" + assert light["tokens"]["--font-sans"] == '"Trebuchet MS", "Segoe UI", sans-serif' + assert (dark["selector"], dark["dark"]) == ('[data-theme="dark"]', True) + assert dark["tokens"]["--color-brand"] == "#0d9488" + + out = cli(FIXTURE_C) + first = out.splitlines()[0] + assert first.startswith("Frontend found at `web/`, stack `plain HTML (web)`, colours `--color-brand #0f766e,") + assert 'fonts `"Trebuchet MS", "Segoe UI", sans-serif`' in first + assert "web/css/theme.css" in out and '[data-theme="dark"]' in out + + +def test_react_app_is_one_frontend_with_its_media_query_dark_tokens(tmp_path): + root = project(tmp_path, { + "server.py": "from fastapi import FastAPI\n", + "frontend/package.json": json.dumps({"dependencies": {"react": "^19.0.0", "react-dom": "^19.0.0"}, + "devDependencies": {"vite": "^6.0.0"}}), + "frontend/index.html": '

      ', + "frontend/src/index.css": ":root { --primary: 222.2 47.4% 11.2%; --radius: .5rem }\n" + "@media (prefers-color-scheme: dark) { :root { --primary: 210 40% 98% } }", + "frontend/node_modules/react/index.html": "

      ", + }) + [f] = detect_frontend.detect(root) + assert (f["path"], f["stack"]) == ("frontend/", "React (Vite, web)") + light, dark = f["blocks"] + assert (light["file"], light["tokens"]["--primary"], light["dark"]) == ("frontend/src/index.css", "222.2 47.4% 11.2%", False) + assert (dark["selector"], dark["tokens"], dark["dark"]) == ( + "@media (prefers-color-scheme: dark) :root", {"--primary": "210 40% 98%"}, True) + assert "colours `--primary 222.2 47.4% 11.2%`" in cli(root) + + +def test_vue_app_reads_component_styles_and_tailwind_theme(tmp_path): + root = project(tmp_path, { + "package.json": json.dumps({"dependencies": {"vue": "^3.5.0"}, "devDependencies": {"tailwindcss": "^3.4.0"}}), + "tailwind.config.js": "module.exports = { theme: { extend: { colors: { brand: '#e11d48', 'brand-dark': \"#9f1239\" } } } }", + "src/App.vue": "\n", + }) + [f] = detect_frontend.detect(root) + assert (f["path"], f["stack"]) == ("./", "Vue (Tailwind, web)") + blocks = {b["file"]: b for b in f["blocks"]} + assert blocks["src/App.vue"]["tokens"] == {"--surface": "#ffffff"} + assert blocks["tailwind.config.js"]["tokens"] == {"brand": "#e11d48", "brand-dark": "#9f1239"} + assert "brand #e11d48" in cli(root).splitlines()[0] + + +def test_pyqt_web_view_is_a_desktop_frontend(tmp_path): + host = "from PyQt6.QtWebEngineWidgets import QWebEngineView\nview = QWebEngineView()\n" + root = project(tmp_path / "file", { + "app.py": host + "view.load(QUrl.fromLocalFile('ui/page.html'))\n", + "bot.py": "print('[step 1] Login')\n", + "ui/page.html": "", + }) + [f] = detect_frontend.detect(root) + assert (f["path"], f["stack"]) == ("ui/", "PyQt web view (desktop, host `app.py`)") + assert f["blocks"][0]["tokens"] == {"--accent": "#2563eb"} + + inline = project(tmp_path / "inline", {"main.py": host + "view.setHtml('

      Hi

      ')\n"}) + [f] = detect_frontend.detect(inline) + assert (f["path"], f["stack"]) == ("main.py", "PyQt web view (desktop, host `main.py`)") + assert f["blocks"][0]["tokens"] == {"--accent": "#16a34a"} From 162edd41df8c4986ab0f17756c7925227ab48620 Mon Sep 17 00:00:00 2001 From: Vasanth Date: Fri, 16 Oct 2026 14:28:51 +0530 Subject: [PATCH 2/2] No-AI desktop path: PyQt bot wired to the page - templates/bot_runner.py (new, no Qt): runs the bot as a child process (unbuffered, UTF-8), tails its output, keeps raw lines in bot_raw.log, and turns lines into step/log/status events. Step markers and friendly rules use specific substrings, first match wins; unmatched error lines are always shown, other unmatched lines hidden. Dry run: python dashboard/bot_runner.py prints the events the page would get. - bridge.py: delegates to bot_runner, one run at a time, restores the saved dark mode on every page load (fixes sample-data mode in PyQt), adds load_latest_report / download_report slots the page calls. - SKILL.md: skip the stack question for a PyQt project; desktop wiring steps from the bot's real output; verify steps for the host. - tests: assert-based self-check for the line logic and run loop; Seam 1 run replay (status dot, Run button, steps, log) and wired-page mode; fixture b-pyqt-bot and its Seam 2 eval (all 14 checks pass). Co-Authored-By: Claude Opus 5.5 --- rpa-dashboard-no-ai/SKILL.md | 36 ++- rpa-dashboard-no-ai/templates/bot_runner.py | 64 ++++ rpa-dashboard-no-ai/templates/bridge.py | 61 ++-- rpa-dashboard-no-ai/templates/contract.md | 1 + tests/README.md | 9 +- tests/evals/results/b-pyqt-bot.dashboard.html | 287 ++++++++++++++++++ tests/evals/results/b-pyqt-bot.json | 33 ++ tests/evals/results/b-pyqt-bot.run.jsonl | 19 ++ tests/evals/run_eval.py | 35 ++- tests/fixtures/b-pyqt-bot/README.md | 10 + tests/fixtures/b-pyqt-bot/bot.py | 29 ++ tests/fixtures/b-pyqt-bot/requirements.txt | 2 + tests/page_check/conftest.py | 12 + tests/page_check/test_bot_run.py | 84 +++++ tests/page_check/test_contract.py | 3 + tests/test_bot_runner.py | 112 +++++++ tests/test_templates.py | 3 +- 17 files changed, 753 insertions(+), 47 deletions(-) create mode 100644 rpa-dashboard-no-ai/templates/bot_runner.py create mode 100644 tests/evals/results/b-pyqt-bot.dashboard.html create mode 100644 tests/evals/results/b-pyqt-bot.json create mode 100644 tests/evals/results/b-pyqt-bot.run.jsonl create mode 100644 tests/fixtures/b-pyqt-bot/README.md create mode 100644 tests/fixtures/b-pyqt-bot/bot.py create mode 100644 tests/fixtures/b-pyqt-bot/requirements.txt create mode 100644 tests/page_check/test_bot_run.py create mode 100644 tests/test_bot_runner.py diff --git a/rpa-dashboard-no-ai/SKILL.md b/rpa-dashboard-no-ai/SKILL.md index dcfdcd6..021a2f3 100644 --- a/rpa-dashboard-no-ai/SKILL.md +++ b/rpa-dashboard-no-ai/SKILL.md @@ -11,7 +11,8 @@ source projects. | File | What it is | |---|---| | `templates/dashboard.html` | The single-file page: theme tokens, shell, KPI, donut, run panel, report, sample data. Works in PyQt (QWebChannel) or on the web (polling). | -| `templates/bridge.py` | Minimal PyQt6 host. It runs the bot as a child process and tails its output. | +| `templates/bridge.py` | Minimal PyQt6 host: the window, the bridge slots, and the saved dark-mode choice. | +| `templates/bot_runner.py` | The bot wiring, without Qt. It runs the bot as a child process, tails its output, and turns lines into step and log events. | | `templates/contract.md` | The data contract: events the bot sends, actions the page sends. | | `templates/themes.md` | Theme presets (Tebra Sky, Superbill Ember) and the custom-colour steps. | | `templates/palette.py` | Derives a custom palette from one brand hex and checks contrast. | @@ -50,6 +51,8 @@ Skip this if the user chose to keep the existing look. Offer: ## 4. Ask desktop or web Ask only if there is no frontend. If there is one, use its stack. +Don't ask if the project already uses PyQt (`PyQt6` or `PyQt5` in its requirements, +`pyproject.toml` or imports): choose desktop and say so. - **Desktop:** a PyQt6 window with the page embedded. The page talks to Python through a bridge, so no web server is needed. - **Web:** the same page, served by a backend that it polls. @@ -84,9 +87,9 @@ sections must leave no markup, script or style behind, so delete their with the preset from `templates/themes.md`. For Ember, also add its font link. For a custom colour, run `python templates/palette.py ""`, paste the tokens it prints, and show the user its contrast table. -4. Desktop: also copy `templates/bridge.py` next to the page, and fill - `{{BOT_MODULE}}`, the `STEPS` list (the same steps as the page), `{{ORG}}` and - `{{APP_NAME}}`. +4. Desktop: also copy `templates/bridge.py` and `templates/bot_runner.py` next to + the page. Fill `{{ORG}}` and `{{APP_NAME}}` in `bridge.py`, and `{{BOT_MODULE}}` in + `bot_runner.py` (with no bot yet, use the module the bot will have, for example `bot`). 5. Search the output for `{{`. None may be left. ## 7. Wire the data @@ -97,10 +100,22 @@ both directions. answers its own data actions, so it opens with sample batches and report rows, and Run plays a demo run. Tell the user which events their bot must send later. - **Bot exists:** set `SAMPLE_DATA = false` and delete the `SAMPLE DATA` block. Then: - - Desktop: in `bridge.py`, set `BOT_CMD` to start the bot as a child process. - Fill `STEP_MARKERS` and `FRIENDLY_RULES` from lines the bot really prints. Use - specific substrings (for example `[step 3]`, not a bare word); the first match - wins. Unmatched error lines must always reach the log. + - Desktop: all the wiring is in `bot_runner.py`. + 1. Get the bot's real output: run it once (for example `python -m bot`), or read + its `print` and logging calls. Never invent lines it doesn't print. + 2. Set `BOT_CMD` to start the bot as a child process, and `BOT_CWD` to the folder + it runs from. + 3. Set `STEPS_JSON` in the page to the bot's real stages. Fill `STEP_MARKERS` with + `(substring, step label, pct)`: one specific substring per stage (for example + `[step 3]`, never a bare word like `login`), each label exactly one of the + page's STEPS, and pct rising to 100 on the bot's last line. The first match wins. + 4. Fill `FRIENDLY_RULES` with `(substring, plain-English message, type)` for the + lines an operator should see. Unmatched lines are hidden, except error lines + (any line with a word from `ERROR_WORDS`), which are always shown as they are. + Add the bot's own failure words to `ERROR_WORDS` if it uses others. + 5. Dry run without the window: `python dashboard/bot_runner.py`. It prints each + event the page would get. Check that the steps arrive in order and end at + 100, and that the bot's error lines appear as `error` log events. - Web: the backend serves `GET {API_BASE}/dashboard`, `GET {API_BASE}/run/status` (returning `{running, events[]}`) and `POST {API_BASE}/`. - Apply the contract's derived rules: batch state from Pending plus date, and @@ -108,7 +123,10 @@ both directions. ## 8. Verify -Open the page. On desktop run the host; otherwise open the file or the served URL. +Open the page. On desktop run the host (`python dashboard/bridge.py`), click Run, +and check that the Run button stays disabled until the bot finishes, and that the +dark-mode choice is still set after you close and reopen the window. Otherwise +open the file or the served URL. With sample data, opening the file in a browser is enough. Check: - the light theme, the dark theme (use the toggle), and a window 820px wide or less; - KPI cards, donut and legend, the Run tab's steps and log, and the Report tab's diff --git a/rpa-dashboard-no-ai/templates/bot_runner.py b/rpa-dashboard-no-ai/templates/bot_runner.py new file mode 100644 index 0000000..545a301 --- /dev/null +++ b/rpa-dashboard-no-ai/templates/bot_runner.py @@ -0,0 +1,64 @@ +"""Bot wiring for the desktop host: runs the bot and turns its output into page events. + +No Qt here, so it can be tested and dry-run on its own. Fill BOT_CMD, STEP_MARKERS and +FRIENDLY_RULES from lines the bot really prints. + +Dry run, no window: python bot_runner.py (prints each event the page would get) +""" +import json, os, subprocess, sys +from pathlib import Path + +BOT_CMD = [sys.executable, "-m", "{{BOT_MODULE}}"] # the bot, started as a child process +BOT_CWD = Path(__file__).resolve().parents[1] # folder BOT_CMD runs in (the project root) +RAW_LOG = Path(__file__).with_name("bot_raw.log") # every raw bot line, kept for audit +# (specific substring, step label from STEPS in dashboard.html, pct). First match wins. +# e.g. ("[step 2]", "Login", 40). Never a bare word like "login": it matches unrelated lines. +STEP_MARKERS = [] +# (specific substring, plain-English message, type: info|success|warn|error). First match wins. +# e.g. ("[login] done", "Logged in.", "success"). Unmatched lines are hidden, except error lines. +FRIENDLY_RULES = [] +# A line holding any of these is an error line. Unmatched error lines are shown raw, never hidden. +ERROR_WORDS = ("error", "exception", "traceback", "fatal", "failed") + + +def _first(rules, line): + low = line.lower() + return next((r for r in rules if r[0].lower() in low), None) + + +def events_for(line): + """The page events, as (event, data) pairs, for one raw bot line.""" + out = [] + if marker := _first(STEP_MARKERS, line): + out.append(("step", {"label": marker[1], "pct": marker[2]})) + if rule := _first(FRIENDLY_RULES, line): + out.append(("log", {"msg": rule[1], "type": rule[2]})) + elif any(w in line.lower() for w in ERROR_WORDS): # never hide a real failure + out.append(("log", {"msg": line, "type": "error"})) + return out + + +def run(send): + """Run the bot in its own process and send(event, data) for each line it prints, as it prints it.""" + send("status", {"state": "running", "msg": "Running…"}) + env = {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8"} # live lines; no cp1252 crash + try: + raw = open(RAW_LOG, "a", encoding="utf-8") + p = subprocess.Popen(BOT_CMD, cwd=BOT_CWD, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding="utf-8", errors="replace") + except OSError as e: + send("log", {"msg": f"Could not start the bot: {e}", "type": "error"}) + send("status", {"state": "error", "msg": "Could not start the bot"}) + return + with p, raw: + for line in p.stdout: + raw.write(line) + for event, data in events_for(line.rstrip()): + send(event, data) + ok = p.returncode == 0 + send("status", {"state": "done", "msg": "Completed"} if ok else + {"state": "error", "msg": f"Failed (exit code {p.returncode})"}) + + +if __name__ == "__main__": + run(lambda event, data: print(json.dumps({"event": event, "data": data}), flush=True)) diff --git a/rpa-dashboard-no-ai/templates/bridge.py b/rpa-dashboard-no-ai/templates/bridge.py index 179501a..fa7d83a 100644 --- a/rpa-dashboard-no-ai/templates/bridge.py +++ b/rpa-dashboard-no-ai/templates/bridge.py @@ -1,25 +1,23 @@ """Minimal PyQt6 host for dashboard.html (desktop mode). Pattern from Tebra main_window.py. -The bot runs as a child process that prints lines, and this window tails them. -Fill in STEP_MARKERS, FRIENDLY_RULES and BOT_CMD for the project. +The bot runs as a child process and this window shows its progress. The bot wiring +(command, step markers, friendly rules) lives in bot_runner.py, next to this file. +Run it with: python dashboard/bridge.py """ -import json, subprocess, sys, threading +import json, sys, threading from pathlib import Path from PyQt6.QtCore import QObject, pyqtSlot, pyqtSignal, QUrl, QSettings from PyQt6.QtWidgets import QApplication, QMainWindow from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWebChannel import QWebChannel +import bot_runner + HTML_PATH = Path(__file__).with_name("dashboard.html") -BOT_CMD = [sys.executable, "-m", "{{BOT_MODULE}}"] # child process -STEPS = ["{{STEP_1}}", "{{STEP_2}}"] # same list as STEPS in dashboard.html -STEP_MARKERS = [("[step 1]", 0, 10), ("[all done]", len(STEPS) - 1, 100)] # (substring, step idx, pct) -FRIENDLY_RULES = [("[login] done", "Logged in.", "success")] # (substring, friendly msg | None, type) -def classify(line): - s = line.lower() - return "error" if ("error" in s or "traceback" in s) else "warn" if "warn" in s else "info" +def settings(): + return QSettings("{{ORG}}", "{{APP_NAME}}") class Bridge(QObject): @@ -27,8 +25,10 @@ class Bridge(QObject): def __init__(self, view): super().__init__() - self.view = view + self.view, self._one_run = view, threading.Lock() self._emit.connect(lambda js: self.view.page().runJavaScript(f"window.__pyMsg({js})")) + # Restore the saved dark-mode choice on every page load, sample-data mode included. + view.loadFinished.connect(lambda _ok: self.send_to_js("apply_dark", {"on": settings().value("dark", False, bool)})) def send_to_js(self, event, data): self._emit.emit(json.dumps(json.dumps({"event": event, "data": data}))) @@ -36,38 +36,29 @@ class Bridge(QObject): @pyqtSlot() def load_dashboard(self): self.send_to_js("dashboard_data", {"batches": []}) # TODO: read from DB on a worker thread - self.send_to_js("apply_dark", {"on": QSettings("{{ORG}}", "{{APP_NAME}}").value("dark", False, bool)}) + + @pyqtSlot() + def load_latest_report(self): + self.send_to_js("report_data", {"rows": []}) # TODO: read the latest report rows + + @pyqtSlot() + def download_report(self): + pass # TODO: export the report rows to .xlsx @pyqtSlot(bool) def set_dark(self, on): - QSettings("{{ORG}}", "{{APP_NAME}}").setValue("dark", on) + settings().setValue("dark", on) @pyqtSlot() def start_automation(self): - threading.Thread(target=self._run, daemon=True).start() + if self._one_run.acquire(blocking=False): # a second click can't start a second bot + threading.Thread(target=self._run, daemon=True).start() def _run(self): - self.send_to_js("status", {"state": "running", "msg": "Running…"}) - p = subprocess.Popen(BOT_CMD, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) - for raw in p.stdout: - self._on_line(raw.rstrip()) - ok = p.wait() == 0 - self.send_to_js("status", {"state": "done" if ok else "error", "msg": "Completed" if ok else "Failed"}) - - def _on_line(self, line): - # ponytail: raw line only printed here; persist to file/DB if audit is needed - low, kind = line.lower(), classify(line) - for marker, idx, pct in STEP_MARKERS: - if marker in low: - self.send_to_js("step", {"label": STEPS[idx], "pct": pct}) - break - for marker, msg, t in FRIENDLY_RULES: - if marker in low: - if msg: - self.send_to_js("log", {"msg": msg, "type": t}) - return - if kind == "error": # never hide a real failure - self.send_to_js("log", {"msg": line, "type": "error"}) + try: + bot_runner.run(self.send_to_js) + finally: + self._one_run.release() if __name__ == "__main__": diff --git a/rpa-dashboard-no-ai/templates/contract.md b/rpa-dashboard-no-ai/templates/contract.md index 9e33309..2f63400 100644 --- a/rpa-dashboard-no-ai/templates/contract.md +++ b/rpa-dashboard-no-ai/templates/contract.md @@ -2,6 +2,7 @@ All updates use one envelope: `{"event": "", "data": {...}}`. - **Desktop (PyQt):** Python calls `window.__pyMsg(json)` via `runJavaScript` (see `bridge.py`). + `bot_runner.py` turns the bot's output lines into `status`, `step` and `log` events, and keeps every raw line in `bot_raw.log`. - **Web:** the page polls `GET {API_BASE}/dashboard` (returns `dashboard_data`) and `GET {API_BASE}/run/status` → `{"running": bool, "events": [envelope, ...]}`. Poll fast (2s) only while running. - **No bot yet:** with `SAMPLE_DATA = true`, the page answers its own data actions with sample envelopes. diff --git a/tests/README.md b/tests/README.md index 2209443..1023436 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,6 +15,11 @@ uv run --with pytest --with playwright pytest tests generated page instead, set `PAGE_CHECK_HTML=path/to/dashboard.html`. - The page sees a fake Qt bridge. Every action the page sends is recorded in `window.__calls` as `[slot, ...args]`. + - `PAGE_CHECK_WIRED=1`: the page is wired to a bot (sample data off). Sample-data + cases skip, and a case checks that the page's actions go to the bot. With no + `PAGE_CHECK_HTML`, the template is checked with `SAMPLE_DATA` switched off. + - `PAGE_CHECK_REPLAY=run.jsonl`: replay a real bot run (the JSON lines that + `python dashboard/bot_runner.py` prints) instead of the built-in run. - To add a case, write a `test_*.py` file in `page_check/` that uses the `page` and `push(event, data)` fixtures from `conftest.py`. - `evals/`: **Seam 2.** Runs the skills on a fixture project with `claude -p`, @@ -25,5 +30,7 @@ uv run --with pytest --with playwright pytest tests ``` Results go to `evals/results/.json`. To add a fixture, add an entry to - `FIXTURES` in `run_eval.py`. + `FIXTURES` in `run_eval.py`. For a bot fixture (`b-pyqt-bot`), the eval also + dry-runs the generated `bot_runner.py`, saves its events to + `evals/results/.run.jsonl`, and replays them in the page check. - `fixtures/-/`: small test projects for the evals. diff --git a/tests/evals/results/b-pyqt-bot.dashboard.html b/tests/evals/results/b-pyqt-bot.dashboard.html new file mode 100644 index 0000000..e819205 --- /dev/null +++ b/tests/evals/results/b-pyqt-bot.dashboard.html @@ -0,0 +1,287 @@ + + + + + + +Work-item follow-up bot + + + + +
      + + +
      +
      +
      Dashboard
      Overview of every batch
      +
      + + +
      System ready
      +
      +
      + +
      + +
      +
      +
      0
      Total batches
      +
      0
      Completed
      +
      0
      In progress
      +
      0
      Not completed
      +
      +
      +
      Batch status split
      +
        +
        +
        Recent batches
        No batches yet
        +
        +
        + + + + + + +
        +
        +
        + + + + diff --git a/tests/evals/results/b-pyqt-bot.json b/tests/evals/results/b-pyqt-bot.json new file mode 100644 index 0000000..61d2e9c --- /dev/null +++ b/tests/evals/results/b-pyqt-bot.json @@ -0,0 +1,33 @@ +{ + "fixture": "b-pyqt-bot", + "passed": true, + "checks": { + "frontend reported correctly": true, + "asks /colou?r/": true, + "asks /section/": true, + "never asks /\\bkeep\\b|restyle/": true, + "never asks /desktop|web/": true, + "exactly one page built": true, + "no placeholders left": true, + "page wired to the bot (sample data off)": true, + "bot_runner.py built": true, + "no placeholders left in the host": true, + "bot dry run ends done": true, + "real steps reach the page": true, + "bot error line reaches the log": true, + "page check passes": true + }, + "page_check": [ + "27 passed, 4 skipped in 13.24s" + ], + "files": [ + "README.md", + "bot.py", + "dashboard\\bot_raw.log", + "dashboard\\bot_runner.py", + "dashboard\\bridge.py", + "dashboard\\dashboard.html", + "requirements.txt" + ], + "reply": "Dashboard ready aachu, PyQt bot-oda wire pannirukken. Ithu desktop app (PyQt6), local project mattum; cloud/deploy edhuvum illa. Neenga work pannumbothu answer panna mudiyaathu-nu sonneenga, adhaan 3-file change-ku \"go\" wait pannala.\n\n| Item | Result |\n|---|---|\n| Files | `dashboard/dashboard.html`, `dashboard/bridge.py`, `dashboard/bot_runner.py` |\n| Theme / extras | Tebra Sky / onnum illa (optional markers ellam remove pannitten) |\n| Data | Sample data illa. `bot_runner.py` `python -m bot`-a child process-aa run pannum |\n| Steps | Initializing \u2192 Login \u2192 Process work items \u2192 Write summary \u2192 Completed (bot-oda `[step 1-4]`, `[all done]` lines-la irundhu) |\n| Dry run | 5/5 steps order-la vandhuchu, 100%-la end aachu; `ERROR: could not attach\u2026` line `error` log-aa vandhuchu |\n| Qt host (offscreen) | Run click pannadhum button disable aachu, bot mudinjadhum enable; status \"Completed\", 100%; dark mode reopen pannappuram-um on-la irundhuchu |\n| Screenshots | Light, dark, 800px width: KPI, donut, steps, log, report pills ellam correct-aa render aachu; horizontal scroll illa; `{{` onnum meedhi illa |\n\nRun panna: `python dashboard/bridge.py`\n\nAlso noticed (fix pannala):\n- `load_dashboard` / `load_latest_report` empty list-dhaan anuppudhu, bot DB-yo report-o ezhudhala. Adhanaala live data-vula KPI-yum Report tab-um empty-aa irukkum. Test pannumbothu sample rows inject panni dhaan check panninen.\n- `bot.py` `ERROR` print pannaalum exit code 0-dhaan, so status \"Completed\"-nu dhaan kaattum.\n- \"Work item updated.\" log line-la WI number varaadhu; raw line `dashboard/bot_raw.log`-la save aagum.\n- Browser-la file-aa direct-aa open panna fetch errors varum, aana Qt-kulla avai varaadhu.\n\nFRONTEND: none\nQUESTIONS: Which colours: Tebra Sky, Superbill Ember, or Custom (one brand hex)? | Which optional sections to build: checkpoint tracker, admin/roles screen, login page, or none?" +} \ No newline at end of file diff --git a/tests/evals/results/b-pyqt-bot.run.jsonl b/tests/evals/results/b-pyqt-bot.run.jsonl new file mode 100644 index 0000000..c5bd675 --- /dev/null +++ b/tests/evals/results/b-pyqt-bot.run.jsonl @@ -0,0 +1,19 @@ +{"event": "status", "data": {"state": "running", "msg": "Running\u2026"}} +{"event": "step", "data": {"label": "Initializing", "pct": 10}} +{"event": "log", "data": {"msg": "Starting up.", "type": "info"}} +{"event": "log", "data": {"msg": "Work items loaded.", "type": "info"}} +{"event": "step", "data": {"label": "Login", "pct": 30}} +{"event": "log", "data": {"msg": "Logging in to the portal.", "type": "info"}} +{"event": "log", "data": {"msg": "Logged in.", "type": "success"}} +{"event": "step", "data": {"label": "Process work items", "pct": 50}} +{"event": "log", "data": {"msg": "Processing work items.", "type": "info"}} +{"event": "log", "data": {"msg": "Work item updated.", "type": "success"}} +{"event": "log", "data": {"msg": "Work item updated.", "type": "success"}} +{"event": "log", "data": {"msg": "Work item updated.", "type": "success"}} +{"event": "log", "data": {"msg": "The portal is slow to respond.", "type": "warn"}} +{"event": "log", "data": {"msg": "ERROR: could not attach the note to WI-1003", "type": "error"}} +{"event": "step", "data": {"label": "Write summary", "pct": 85}} +{"event": "log", "data": {"msg": "Writing the summary.", "type": "info"}} +{"event": "step", "data": {"label": "Completed", "pct": 100}} +{"event": "log", "data": {"msg": "All work items processed.", "type": "success"}} +{"event": "status", "data": {"state": "done", "msg": "Completed"}} diff --git a/tests/evals/run_eval.py b/tests/evals/run_eval.py index 5d9b532..9e7366f 100644 --- a/tests/evals/run_eval.py +++ b/tests/evals/run_eval.py @@ -36,9 +36,39 @@ FIXTURES = { "never_asks": [r"\bkeep\b|restyle"], "brand": "#7a3cff", }, + "b-pyqt-bot": { + "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you " + "work, so here are my answers to anything the skill asks. Colours: Tebra Sky. " + "Optional sections: none.", + "frontend": r"^\s*(none|no\b)", + "asks": [r"colou?r", r"section"], + "never_asks": [r"\bkeep\b|restyle", r"desktop|web"], + "bot": True, + }, } +def bot_checks(project, html, checks, replay): + """Bot fixtures: the page is wired, and a dry run of the generated wiring gives real steps and logs. + Returns the env that points the page check at that run.""" + checks["page wired to the bot (sample data off)"] = bool(re.search(r"SAMPLE_DATA\s*=\s*false", html)) + runners = [p for p in project.rglob("bot_runner.py") if ".claude" not in p.parts] + checks["bot_runner.py built"] = len(runners) == 1 + if not runners: + return {} + host = [runners[0], runners[0].with_name("bridge.py")] + checks["no placeholders left in the host"] = all(p.exists() and "{{" not in p.read_text(encoding="utf-8") for p in host) + dry = subprocess.run([sys.executable, str(runners[0])], cwd=project, capture_output=True, text=True, + encoding="utf-8", timeout=300) + replay.write_text(dry.stdout, encoding="utf-8") + events = [json.loads(x) for x in dry.stdout.splitlines() if x.startswith("{")] + steps = [e["data"] for e in events if e["event"] == "step"] + checks["bot dry run ends done"] = bool(events) and events[-1]["data"].get("state") == "done" + checks["real steps reach the page"] = len({s["label"] for s in steps}) >= 3 and steps[-1]["pct"] == 100 + checks["bot error line reaches the log"] = any(e["event"] == "log" and e["data"]["type"] == "error" for e in events) + return {"PAGE_CHECK_WIRED": "1", "PAGE_CHECK_REPLAY": str(replay)} + + def main(fixture): cfg = FIXTURES[fixture] work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-")) @@ -80,8 +110,11 @@ def main(fixture): spec.loader.exec_module(palette) t = palette.derive(cfg["brand"]) checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t) + env = {**os.environ, "PAGE_CHECK_HTML": str(pages[0])} + if cfg.get("bot"): + env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl")) pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"], - env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True) + env=env, capture_output=True, text=True) checks["page check passes"] = pc.returncode == 0 page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]] else: diff --git a/tests/fixtures/b-pyqt-bot/README.md b/tests/fixtures/b-pyqt-bot/README.md new file mode 100644 index 0000000..668787c --- /dev/null +++ b/tests/fixtures/b-pyqt-bot/README.md @@ -0,0 +1,10 @@ +# Work-item follow-up bot + +A desktop RPA bot. It logs in to the portal and updates each work item. +The operator window will be a PyQt6 app. + +Run the bot on its own: + +``` +python -m bot +``` diff --git a/tests/fixtures/b-pyqt-bot/bot.py b/tests/fixtures/b-pyqt-bot/bot.py new file mode 100644 index 0000000..d38dc9c --- /dev/null +++ b/tests/fixtures/b-pyqt-bot/bot.py @@ -0,0 +1,29 @@ +"""Work-item follow-up bot. Start it with: python -m bot""" +import time + +ITEMS = ["WI-1001", "WI-1002", "WI-1003"] + + +def say(line): + print(line) + time.sleep(0.2) + + +def main(): + say("[step 1] Initializing") + say("Loading settings from config.ini") + say("Completed loading 3 work items") + say("[step 2] Logging in to the portal") + say("Login page loaded") + say("[login] ok") + say("[step 3] Processing work items") + for i, item in enumerate(ITEMS, 1): + say(f"[item] {item} updated ({i}/{len(ITEMS)})") + say("WARNING: portal slow to respond, waited 5s") + say("ERROR: could not attach the note to WI-1003") + say("[step 4] Writing the summary") + say(f"[all done] {len(ITEMS)} items processed") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/b-pyqt-bot/requirements.txt b/tests/fixtures/b-pyqt-bot/requirements.txt new file mode 100644 index 0000000..8da5982 --- /dev/null +++ b/tests/fixtures/b-pyqt-bot/requirements.txt @@ -0,0 +1,2 @@ +PyQt6 +PyQt6-WebEngine diff --git a/tests/page_check/conftest.py b/tests/page_check/conftest.py index fe34b0e..c34ce33 100644 --- a/tests/page_check/conftest.py +++ b/tests/page_check/conftest.py @@ -4,6 +4,8 @@ through window.__pyMsg, and assert on what is visible. Page under test: set PAGE_CHECK_HTML to a generated dashboard.html. Without it, the no-AI template is filled with TEST_FILL. Every test runs in light, dark and narrow mode. The page sees a fake Qt bridge: each slot call lands in window.__calls as [slot, ...args]. +PAGE_CHECK_WIRED=1: the page is wired to a bot (SAMPLE_DATA off), so sample-data cases skip. +PAGE_CHECK_REPLAY: a JSON-lines file of envelopes a real bot run produced (bot_runner.py's dry run). """ import json import os @@ -32,6 +34,14 @@ window.QWebChannel = function (_transport, ready) { }; """ MODES = {"light": (1280, 820, False), "dark": (1280, 820, True), "narrow": (800, 900, False)} +WIRED = os.environ.get("PAGE_CHECK_WIRED") == "1" + + +@pytest.fixture +def sample_data(): + """Use on cases that need the built-in sample data; they skip on a bot-wired page.""" + if WIRED: + pytest.skip("page is wired to a bot, so it has no sample data") @pytest.fixture(scope="session") @@ -39,6 +49,8 @@ def page_url(tmp_path_factory): if os.environ.get("PAGE_CHECK_HTML"): return Path(os.environ["PAGE_CHECK_HTML"]).resolve().as_uri() html = re.sub(r"\{\{(\w+)\}\}", lambda m: TEST_FILL[m[1]], TEMPLATE.read_text(encoding="utf-8")) + if WIRED: + html = html.replace("const SAMPLE_DATA = true", "const SAMPLE_DATA = false") out = tmp_path_factory.mktemp("page") / "dashboard.html" out.write_text(html, encoding="utf-8") return out.as_uri() diff --git a/tests/page_check/test_bot_run.py b/tests/page_check/test_bot_run.py new file mode 100644 index 0000000..72552bc --- /dev/null +++ b/tests/page_check/test_bot_run.py @@ -0,0 +1,84 @@ +"""A bot run as the page sees it: status, step and log envelopes in order, from a real bot +run (PAGE_CHECK_REPLAY, the JSON lines bot_runner.py's dry run prints) or a built-in one.""" +import json +import os +import re +from pathlib import Path + +import pytest +from conftest import WIRED +from playwright.sync_api import expect +from test_contract import colour_of, kpi, open_tab + +LOG_COLOURS = {"info": "blue", "success": "green", "warn": "amber", "error": "red"} +DOT_COLOURS = {"ready": "green", "running": "amber", "done": "green", "error": "red"} + + +def built_in_run(steps): + run = [{"event": "status", "data": {"state": "running", "msg": "Running…"}}] + for i, label in enumerate(steps): + run.append({"event": "step", "data": {"label": label, "pct": round((i + 1) / len(steps) * 100)}}) + run.append({"event": "log", "data": {"msg": f"Finished {label}.", "type": "success"}}) + run.insert(4, {"event": "log", "data": {"msg": "ERROR: portal timed out, retrying", "type": "error"}}) + return run + [{"event": "status", "data": {"state": "done", "msg": "Completed"}}] + + +def recorded_run(): + path = os.environ.get("PAGE_CHECK_REPLAY") + return [json.loads(x) for x in Path(path).read_text(encoding="utf-8-sig").splitlines() if x.strip()] if path else None + + +def expect_status(page, state, msg): + # The status indicator has no role of its own, so it's found by its documented ids. + expect(page.locator("#st-text")).to_have_text(msg or state) + assert colour_of(page.locator("#st-dot"), "backgroundColor") == DOT_COLOURS[state], state + run = page.get_by_role("button", name="▶ Run", exact=True) + expect(run).to_be_disabled() if state == "running" else expect(run).to_be_enabled() + + +def test_bot_run_shows_status_steps_and_log(page, push): + steps = page.evaluate("STEPS") + run = recorded_run() or built_in_run(steps) + open_tab(page, "Run") + expect_status(page, "ready", "System ready") + items = page.get_by_role("list", name="Steps").get_by_role("listitem") + log, last_step = page.get_by_role("log"), -1 + for envelope in run: + event, d = envelope["event"], envelope["data"] + push(event, d) + if event == "status": + expect_status(page, d["state"], d.get("msg")) + elif event == "step": + assert d["label"] in steps, f"step label {d['label']!r} is not one of the page's STEPS {steps}" + idx = steps.index(d["label"]) + assert idx >= last_step, f"progress went back from {steps[last_step]!r} to {d['label']!r}" + last_step = idx + expect(page.get_by_text(f"{d['pct']}%", exact=True)).to_be_visible() + if d["pct"] >= 100: + for i in range(len(steps)): + expect(items.nth(i)).to_have_text(re.compile(r"^\s*✓")) + else: + expect(items.nth(idx)).to_have_attribute("aria-current", "step") + elif event == "log": + line = log.get_by_text(d["msg"], exact=True).last + expect(line).to_be_visible() + assert colour_of(line) == LOG_COLOURS[d["type"]], d["msg"] + assert any(e["event"] == "step" for e in run), "the run never moved the progress bar" + + +def test_failed_run_shows_error_status(page, push): + push("status", {"state": "running", "msg": "Running…"}) + expect_status(page, "running", "Running…") + push("status", {"state": "error", "msg": "Failed (exit code 1)"}) + expect_status(page, "error", "Failed (exit code 1)") + + +@pytest.mark.skipif(not WIRED, reason="only for a page wired to a bot (PAGE_CHECK_WIRED=1)") +def test_wired_page_sends_its_actions_to_the_bot(page): + calls = lambda: page.evaluate("window.__calls") # noqa: E731 + assert ["load_dashboard"] in calls() + kpi(page, "Total batches", 0) # no sample batches on a wired page + page.get_by_role("button", name="▶ Run", exact=True).click() + assert ["start_automation"] in calls() + open_tab(page, "Report") + assert ["load_latest_report"] in calls() diff --git a/tests/page_check/test_contract.py b/tests/page_check/test_contract.py index 218cbd9..299b8f8 100644 --- a/tests/page_check/test_contract.py +++ b/tests/page_check/test_contract.py @@ -1,6 +1,7 @@ """No-AI contract cases: events in, visible result out (see templates/contract.md).""" import re +import pytest from playwright.sync_api import expect @@ -146,6 +147,7 @@ def expect_sample_dashboard(page): kpi(page, "Total batches", n) +@pytest.mark.usefixtures("sample_data") def test_no_bot_page_runs_on_sample_data(page): """No bot yet: the page answers its own data actions, so it works on day one.""" expect_sample_dashboard(page) @@ -162,6 +164,7 @@ def test_no_bot_page_runs_on_sample_data(page): expect(run).to_be_enabled() +@pytest.mark.usefixtures("sample_data") def test_plain_browser_first_open_shows_sample_data(browser, page_url): pg = browser.new_page() pg.goto(page_url) diff --git a/tests/test_bot_runner.py b/tests/test_bot_runner.py new file mode 100644 index 0000000..788e8e7 --- /dev/null +++ b/tests/test_bot_runner.py @@ -0,0 +1,112 @@ +"""Bot wiring self-check (templates/bot_runner.py): which page events each raw bot line makes.""" +import importlib.util +import os +import sys +import time +from pathlib import Path + +import pytest + +RUNNER = Path(__file__).resolve().parents[1] / "rpa-dashboard-no-ai" / "templates" / "bot_runner.py" + + +@pytest.fixture +def runner(): + spec = importlib.util.spec_from_file_location("bot_runner", RUNNER) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.STEP_MARKERS = [("[step 1]", "Initializing", 10), ("[step 2]", "Login", 40), + ("[step 3]", "Process items", 70), ("[all done]", "Completed", 100)] + mod.FRIENDLY_RULES = [("[login] done", "Logged in.", "success"), ("[login]", "Logging in...", "info"), + ("could not save receipt", "Could not save the receipt.", "error")] + return mod + + +def steps(runner, line): + return [d for e, d in runner.events_for(line) if e == "step"] + + +def test_step_marker_moves_progress(runner): + assert steps(runner, "[step 2] Opening the login page") == [{"label": "Login", "pct": 40}] + assert steps(runner, "[STEP 3] Processing 12 items") == [{"label": "Process items", "pct": 70}] + + +def test_common_word_does_not_move_progress(runner): + assert steps(runner, "Login page loaded slowly") == [] + assert steps(runner, "Completed loading the config") == [] + + +def test_first_step_marker_wins(runner): + assert steps(runner, "[step 1] retry after [step 3] timed out") == [{"label": "Initializing", "pct": 10}] + + +def logs(runner, line): + return [d for e, d in runner.events_for(line) if e == "log"] + + +def test_friendly_rule_rewrites_the_line_and_first_rule_wins(runner): + assert logs(runner, "[login] done in 2.1s") == [{"msg": "Logged in.", "type": "success"}] + assert logs(runner, "[LOGIN] sending credentials") == [{"msg": "Logging in...", "type": "info"}] + + +def test_unmatched_non_error_lines_are_hidden(runner): + for line in ["Loading config from settings.ini", "[step 2] Opening the login page", "warning: slow page"]: + assert logs(runner, line) == [], line + + +@pytest.mark.parametrize("line", [ + "ERROR: portal timed out", "Traceback (most recent call last):", "ValueError: bad date", + "Exception: boom", "FATAL disk full", "Login failed for user 3"]) +def test_unmatched_error_line_is_always_shown_raw(runner, line): + assert logs(runner, line) == [{"msg": line, "type": "error"}] + + +def test_matched_error_line_shows_its_friendly_message(runner): + assert logs(runner, "ERROR could not save receipt 42") == [{"msg": "Could not save the receipt.", "type": "error"}] + + +def test_error_on_a_step_line_moves_progress_and_is_shown(runner): + line = "[step 3] ERROR item 2 rejected" + assert runner.events_for(line) == [("step", {"label": "Process items", "pct": 70}), + ("log", {"msg": line, "type": "error"})] + + +def run_bot(runner, tmp_path, script): + """Run a fake bot through runner.run(); return [(seconds since start, event, data)].""" + bot = tmp_path / "fake_bot.py" + bot.write_text(script, encoding="utf-8") + runner.BOT_CMD, runner.BOT_CWD, runner.RAW_LOG = [sys.executable, str(bot)], tmp_path, tmp_path / "raw.log" + t0, sent = time.monotonic(), [] + runner.run(lambda event, data: sent.append((time.monotonic() - t0, event, data))) + return sent + + +def test_run_streams_bot_output_live_from_a_child_process(runner, tmp_path): + sent = run_bot(runner, tmp_path, "import os, time\n" + "print('[step 1] pid', os.getpid())\n" + "print('[login] done')\n" + "time.sleep(1)\n" + "print('[all done] ✓ 3 items')\n") + events = [(e, d) for _, e, d in sent] + assert events == [("status", {"state": "running", "msg": "Running…"}), + ("step", {"label": "Initializing", "pct": 10}), + ("log", {"msg": "Logged in.", "type": "success"}), + ("step", {"label": "Completed", "pct": 100}), + ("status", {"state": "done", "msg": "Completed"})] + assert sent[3][0] - sent[1][0] > 0.5, "lines must reach the page while the bot runs, not at exit" + raw = (tmp_path / "raw.log").read_text(encoding="utf-8") + assert "[login] done" in raw and "✓ 3 items" in raw # raw lines kept for audit + assert f"pid {os.getpid()}" not in raw # the bot ran in its own process + + +def test_run_ends_in_error_when_the_bot_fails(runner, tmp_path): + sent = run_bot(runner, tmp_path, "print('[step 1] start')\nraise SystemExit(3)\n") + assert sent[-1][1:] == ("status", {"state": "error", "msg": "Failed (exit code 3)"}) + + +def test_run_ends_in_error_when_the_bot_cannot_start(runner, tmp_path): + runner.BOT_CMD, runner.BOT_CWD, runner.RAW_LOG = ["no-such-bot-xyz"], tmp_path, tmp_path / "raw.log" + sent = [] + runner.run(lambda event, data: sent.append((event, data))) + assert sent[-2][0] == "log" and sent[-2][1]["type"] == "error" + assert sent[-1] == ("status", {"state": "error", "msg": "Could not start the bot"}) diff --git a/tests/test_templates.py b/tests/test_templates.py index d1a9d28..89174ea 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -9,7 +9,8 @@ ROOT = Path(__file__).resolve().parents[1] HOOK_RULES = "E9,F63,F7,F82,F811" # same rule set as the PostToolUse lint hook -@pytest.mark.parametrize("template", ["rpa-dashboard-no-ai/templates/bridge.py"]) +@pytest.mark.parametrize("template", ["rpa-dashboard-no-ai/templates/bridge.py", + "rpa-dashboard-no-ai/templates/bot_runner.py"]) def test_template_passes_lint_hook_unfilled(template): ruff = shutil.which("ruff") or pytest.skip("ruff not on PATH") r = subprocess.run([ruff, "check", "--select", HOOK_RULES, "--no-cache", str(ROOT / template)],