diff --git a/rpa-dashboard-no-ai/SKILL.md b/rpa-dashboard-no-ai/SKILL.md index dcfdcd6..021a2f3 100644 --- a/rpa-dashboard-no-ai/SKILL.md +++ b/rpa-dashboard-no-ai/SKILL.md @@ -11,7 +11,8 @@ source projects. | File | What it is | |---|---| | `templates/dashboard.html` | The single-file page: theme tokens, shell, KPI, donut, run panel, report, sample data. Works in PyQt (QWebChannel) or on the web (polling). | -| `templates/bridge.py` | Minimal PyQt6 host. It runs the bot as a child process and tails its output. | +| `templates/bridge.py` | Minimal PyQt6 host: the window, the bridge slots, and the saved dark-mode choice. | +| `templates/bot_runner.py` | The bot wiring, without Qt. It runs the bot as a child process, tails its output, and turns lines into step and log events. | | `templates/contract.md` | The data contract: events the bot sends, actions the page sends. | | `templates/themes.md` | Theme presets (Tebra Sky, Superbill Ember) and the custom-colour steps. | | `templates/palette.py` | Derives a custom palette from one brand hex and checks contrast. | @@ -50,6 +51,8 @@ Skip this if the user chose to keep the existing look. Offer: ## 4. Ask desktop or web Ask only if there is no frontend. If there is one, use its stack. +Don't ask if the project already uses PyQt (`PyQt6` or `PyQt5` in its requirements, +`pyproject.toml` or imports): choose desktop and say so. - **Desktop:** a PyQt6 window with the page embedded. The page talks to Python through a bridge, so no web server is needed. - **Web:** the same page, served by a backend that it polls. @@ -84,9 +87,9 @@ sections must leave no markup, script or style behind, so delete their with the preset from `templates/themes.md`. For Ember, also add its font link. For a custom colour, run `python templates/palette.py ""`, paste the tokens it prints, and show the user its contrast table. -4. Desktop: also copy `templates/bridge.py` next to the page, and fill - `{{BOT_MODULE}}`, the `STEPS` list (the same steps as the page), `{{ORG}}` and - `{{APP_NAME}}`. +4. Desktop: also copy `templates/bridge.py` and `templates/bot_runner.py` next to + the page. Fill `{{ORG}}` and `{{APP_NAME}}` in `bridge.py`, and `{{BOT_MODULE}}` in + `bot_runner.py` (with no bot yet, use the module the bot will have, for example `bot`). 5. Search the output for `{{`. None may be left. ## 7. Wire the data @@ -97,10 +100,22 @@ both directions. answers its own data actions, so it opens with sample batches and report rows, and Run plays a demo run. Tell the user which events their bot must send later. - **Bot exists:** set `SAMPLE_DATA = false` and delete the `SAMPLE DATA` block. Then: - - Desktop: in `bridge.py`, set `BOT_CMD` to start the bot as a child process. - Fill `STEP_MARKERS` and `FRIENDLY_RULES` from lines the bot really prints. Use - specific substrings (for example `[step 3]`, not a bare word); the first match - wins. Unmatched error lines must always reach the log. + - Desktop: all the wiring is in `bot_runner.py`. + 1. Get the bot's real output: run it once (for example `python -m bot`), or read + its `print` and logging calls. Never invent lines it doesn't print. + 2. Set `BOT_CMD` to start the bot as a child process, and `BOT_CWD` to the folder + it runs from. + 3. Set `STEPS_JSON` in the page to the bot's real stages. Fill `STEP_MARKERS` with + `(substring, step label, pct)`: one specific substring per stage (for example + `[step 3]`, never a bare word like `login`), each label exactly one of the + page's STEPS, and pct rising to 100 on the bot's last line. The first match wins. + 4. Fill `FRIENDLY_RULES` with `(substring, plain-English message, type)` for the + lines an operator should see. Unmatched lines are hidden, except error lines + (any line with a word from `ERROR_WORDS`), which are always shown as they are. + Add the bot's own failure words to `ERROR_WORDS` if it uses others. + 5. Dry run without the window: `python dashboard/bot_runner.py`. It prints each + event the page would get. Check that the steps arrive in order and end at + 100, and that the bot's error lines appear as `error` log events. - Web: the backend serves `GET {API_BASE}/dashboard`, `GET {API_BASE}/run/status` (returning `{running, events[]}`) and `POST {API_BASE}/`. - Apply the contract's derived rules: batch state from Pending plus date, and @@ -108,7 +123,10 @@ both directions. ## 8. Verify -Open the page. On desktop run the host; otherwise open the file or the served URL. +Open the page. On desktop run the host (`python dashboard/bridge.py`), click Run, +and check that the Run button stays disabled until the bot finishes, and that the +dark-mode choice is still set after you close and reopen the window. Otherwise +open the file or the served URL. With sample data, opening the file in a browser is enough. Check: - the light theme, the dark theme (use the toggle), and a window 820px wide or less; - KPI cards, donut and legend, the Run tab's steps and log, and the Report tab's diff --git a/rpa-dashboard-no-ai/templates/bot_runner.py b/rpa-dashboard-no-ai/templates/bot_runner.py new file mode 100644 index 0000000..545a301 --- /dev/null +++ b/rpa-dashboard-no-ai/templates/bot_runner.py @@ -0,0 +1,64 @@ +"""Bot wiring for the desktop host: runs the bot and turns its output into page events. + +No Qt here, so it can be tested and dry-run on its own. Fill BOT_CMD, STEP_MARKERS and +FRIENDLY_RULES from lines the bot really prints. + +Dry run, no window: python bot_runner.py (prints each event the page would get) +""" +import json, os, subprocess, sys +from pathlib import Path + +BOT_CMD = [sys.executable, "-m", "{{BOT_MODULE}}"] # the bot, started as a child process +BOT_CWD = Path(__file__).resolve().parents[1] # folder BOT_CMD runs in (the project root) +RAW_LOG = Path(__file__).with_name("bot_raw.log") # every raw bot line, kept for audit +# (specific substring, step label from STEPS in dashboard.html, pct). First match wins. +# e.g. ("[step 2]", "Login", 40). Never a bare word like "login": it matches unrelated lines. +STEP_MARKERS = [] +# (specific substring, plain-English message, type: info|success|warn|error). First match wins. +# e.g. ("[login] done", "Logged in.", "success"). Unmatched lines are hidden, except error lines. +FRIENDLY_RULES = [] +# A line holding any of these is an error line. Unmatched error lines are shown raw, never hidden. +ERROR_WORDS = ("error", "exception", "traceback", "fatal", "failed") + + +def _first(rules, line): + low = line.lower() + return next((r for r in rules if r[0].lower() in low), None) + + +def events_for(line): + """The page events, as (event, data) pairs, for one raw bot line.""" + out = [] + if marker := _first(STEP_MARKERS, line): + out.append(("step", {"label": marker[1], "pct": marker[2]})) + if rule := _first(FRIENDLY_RULES, line): + out.append(("log", {"msg": rule[1], "type": rule[2]})) + elif any(w in line.lower() for w in ERROR_WORDS): # never hide a real failure + out.append(("log", {"msg": line, "type": "error"})) + return out + + +def run(send): + """Run the bot in its own process and send(event, data) for each line it prints, as it prints it.""" + send("status", {"state": "running", "msg": "Running…"}) + env = {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8"} # live lines; no cp1252 crash + try: + raw = open(RAW_LOG, "a", encoding="utf-8") + p = subprocess.Popen(BOT_CMD, cwd=BOT_CWD, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding="utf-8", errors="replace") + except OSError as e: + send("log", {"msg": f"Could not start the bot: {e}", "type": "error"}) + send("status", {"state": "error", "msg": "Could not start the bot"}) + return + with p, raw: + for line in p.stdout: + raw.write(line) + for event, data in events_for(line.rstrip()): + send(event, data) + ok = p.returncode == 0 + send("status", {"state": "done", "msg": "Completed"} if ok else + {"state": "error", "msg": f"Failed (exit code {p.returncode})"}) + + +if __name__ == "__main__": + run(lambda event, data: print(json.dumps({"event": event, "data": data}), flush=True)) diff --git a/rpa-dashboard-no-ai/templates/bridge.py b/rpa-dashboard-no-ai/templates/bridge.py index 179501a..fa7d83a 100644 --- a/rpa-dashboard-no-ai/templates/bridge.py +++ b/rpa-dashboard-no-ai/templates/bridge.py @@ -1,25 +1,23 @@ """Minimal PyQt6 host for dashboard.html (desktop mode). Pattern from Tebra main_window.py. -The bot runs as a child process that prints lines, and this window tails them. -Fill in STEP_MARKERS, FRIENDLY_RULES and BOT_CMD for the project. +The bot runs as a child process and this window shows its progress. The bot wiring +(command, step markers, friendly rules) lives in bot_runner.py, next to this file. +Run it with: python dashboard/bridge.py """ -import json, subprocess, sys, threading +import json, sys, threading from pathlib import Path from PyQt6.QtCore import QObject, pyqtSlot, pyqtSignal, QUrl, QSettings from PyQt6.QtWidgets import QApplication, QMainWindow from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWebChannel import QWebChannel +import bot_runner + HTML_PATH = Path(__file__).with_name("dashboard.html") -BOT_CMD = [sys.executable, "-m", "{{BOT_MODULE}}"] # child process -STEPS = ["{{STEP_1}}", "{{STEP_2}}"] # same list as STEPS in dashboard.html -STEP_MARKERS = [("[step 1]", 0, 10), ("[all done]", len(STEPS) - 1, 100)] # (substring, step idx, pct) -FRIENDLY_RULES = [("[login] done", "Logged in.", "success")] # (substring, friendly msg | None, type) -def classify(line): - s = line.lower() - return "error" if ("error" in s or "traceback" in s) else "warn" if "warn" in s else "info" +def settings(): + return QSettings("{{ORG}}", "{{APP_NAME}}") class Bridge(QObject): @@ -27,8 +25,10 @@ class Bridge(QObject): def __init__(self, view): super().__init__() - self.view = view + self.view, self._one_run = view, threading.Lock() self._emit.connect(lambda js: self.view.page().runJavaScript(f"window.__pyMsg({js})")) + # Restore the saved dark-mode choice on every page load, sample-data mode included. + view.loadFinished.connect(lambda _ok: self.send_to_js("apply_dark", {"on": settings().value("dark", False, bool)})) def send_to_js(self, event, data): self._emit.emit(json.dumps(json.dumps({"event": event, "data": data}))) @@ -36,38 +36,29 @@ class Bridge(QObject): @pyqtSlot() def load_dashboard(self): self.send_to_js("dashboard_data", {"batches": []}) # TODO: read from DB on a worker thread - self.send_to_js("apply_dark", {"on": QSettings("{{ORG}}", "{{APP_NAME}}").value("dark", False, bool)}) + + @pyqtSlot() + def load_latest_report(self): + self.send_to_js("report_data", {"rows": []}) # TODO: read the latest report rows + + @pyqtSlot() + def download_report(self): + pass # TODO: export the report rows to .xlsx @pyqtSlot(bool) def set_dark(self, on): - QSettings("{{ORG}}", "{{APP_NAME}}").setValue("dark", on) + settings().setValue("dark", on) @pyqtSlot() def start_automation(self): - threading.Thread(target=self._run, daemon=True).start() + if self._one_run.acquire(blocking=False): # a second click can't start a second bot + threading.Thread(target=self._run, daemon=True).start() def _run(self): - self.send_to_js("status", {"state": "running", "msg": "Running…"}) - p = subprocess.Popen(BOT_CMD, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) - for raw in p.stdout: - self._on_line(raw.rstrip()) - ok = p.wait() == 0 - self.send_to_js("status", {"state": "done" if ok else "error", "msg": "Completed" if ok else "Failed"}) - - def _on_line(self, line): - # ponytail: raw line only printed here; persist to file/DB if audit is needed - low, kind = line.lower(), classify(line) - for marker, idx, pct in STEP_MARKERS: - if marker in low: - self.send_to_js("step", {"label": STEPS[idx], "pct": pct}) - break - for marker, msg, t in FRIENDLY_RULES: - if marker in low: - if msg: - self.send_to_js("log", {"msg": msg, "type": t}) - return - if kind == "error": # never hide a real failure - self.send_to_js("log", {"msg": line, "type": "error"}) + try: + bot_runner.run(self.send_to_js) + finally: + self._one_run.release() if __name__ == "__main__": diff --git a/rpa-dashboard-no-ai/templates/contract.md b/rpa-dashboard-no-ai/templates/contract.md index 9e33309..2f63400 100644 --- a/rpa-dashboard-no-ai/templates/contract.md +++ b/rpa-dashboard-no-ai/templates/contract.md @@ -2,6 +2,7 @@ All updates use one envelope: `{"event": "", "data": {...}}`. - **Desktop (PyQt):** Python calls `window.__pyMsg(json)` via `runJavaScript` (see `bridge.py`). + `bot_runner.py` turns the bot's output lines into `status`, `step` and `log` events, and keeps every raw line in `bot_raw.log`. - **Web:** the page polls `GET {API_BASE}/dashboard` (returns `dashboard_data`) and `GET {API_BASE}/run/status` → `{"running": bool, "events": [envelope, ...]}`. Poll fast (2s) only while running. - **No bot yet:** with `SAMPLE_DATA = true`, the page answers its own data actions with sample envelopes. diff --git a/tests/README.md b/tests/README.md index 2209443..1023436 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,6 +15,11 @@ uv run --with pytest --with playwright pytest tests generated page instead, set `PAGE_CHECK_HTML=path/to/dashboard.html`. - The page sees a fake Qt bridge. Every action the page sends is recorded in `window.__calls` as `[slot, ...args]`. + - `PAGE_CHECK_WIRED=1`: the page is wired to a bot (sample data off). Sample-data + cases skip, and a case checks that the page's actions go to the bot. With no + `PAGE_CHECK_HTML`, the template is checked with `SAMPLE_DATA` switched off. + - `PAGE_CHECK_REPLAY=run.jsonl`: replay a real bot run (the JSON lines that + `python dashboard/bot_runner.py` prints) instead of the built-in run. - To add a case, write a `test_*.py` file in `page_check/` that uses the `page` and `push(event, data)` fixtures from `conftest.py`. - `evals/`: **Seam 2.** Runs the skills on a fixture project with `claude -p`, @@ -25,5 +30,7 @@ uv run --with pytest --with playwright pytest tests ``` Results go to `evals/results/.json`. To add a fixture, add an entry to - `FIXTURES` in `run_eval.py`. + `FIXTURES` in `run_eval.py`. For a bot fixture (`b-pyqt-bot`), the eval also + dry-runs the generated `bot_runner.py`, saves its events to + `evals/results/.run.jsonl`, and replays them in the page check. - `fixtures/-/`: small test projects for the evals. diff --git a/tests/evals/results/b-pyqt-bot.dashboard.html b/tests/evals/results/b-pyqt-bot.dashboard.html new file mode 100644 index 0000000..e819205 --- /dev/null +++ b/tests/evals/results/b-pyqt-bot.dashboard.html @@ -0,0 +1,287 @@ + + + + + + +Work-item follow-up bot + + + + +
+ + +
+
+
Dashboard
Overview of every batch
+
+ + +
System ready
+
+
+ +
+ +
+
+
0
Total batches
+
0
Completed
+
0
In progress
+
0
Not completed
+
+
+
Batch status split
+
    +
    +
    Recent batches
    No batches yet
    +
    +
    + + + + + + +
    +
    +
    + + + + diff --git a/tests/evals/results/b-pyqt-bot.json b/tests/evals/results/b-pyqt-bot.json new file mode 100644 index 0000000..61d2e9c --- /dev/null +++ b/tests/evals/results/b-pyqt-bot.json @@ -0,0 +1,33 @@ +{ + "fixture": "b-pyqt-bot", + "passed": true, + "checks": { + "frontend reported correctly": true, + "asks /colou?r/": true, + "asks /section/": true, + "never asks /\\bkeep\\b|restyle/": true, + "never asks /desktop|web/": true, + "exactly one page built": true, + "no placeholders left": true, + "page wired to the bot (sample data off)": true, + "bot_runner.py built": true, + "no placeholders left in the host": true, + "bot dry run ends done": true, + "real steps reach the page": true, + "bot error line reaches the log": true, + "page check passes": true + }, + "page_check": [ + "27 passed, 4 skipped in 13.24s" + ], + "files": [ + "README.md", + "bot.py", + "dashboard\\bot_raw.log", + "dashboard\\bot_runner.py", + "dashboard\\bridge.py", + "dashboard\\dashboard.html", + "requirements.txt" + ], + "reply": "Dashboard ready aachu, PyQt bot-oda wire pannirukken. Ithu desktop app (PyQt6), local project mattum; cloud/deploy edhuvum illa. Neenga work pannumbothu answer panna mudiyaathu-nu sonneenga, adhaan 3-file change-ku \"go\" wait pannala.\n\n| Item | Result |\n|---|---|\n| Files | `dashboard/dashboard.html`, `dashboard/bridge.py`, `dashboard/bot_runner.py` |\n| Theme / extras | Tebra Sky / onnum illa (optional markers ellam remove pannitten) |\n| Data | Sample data illa. `bot_runner.py` `python -m bot`-a child process-aa run pannum |\n| Steps | Initializing \u2192 Login \u2192 Process work items \u2192 Write summary \u2192 Completed (bot-oda `[step 1-4]`, `[all done]` lines-la irundhu) |\n| Dry run | 5/5 steps order-la vandhuchu, 100%-la end aachu; `ERROR: could not attach\u2026` line `error` log-aa vandhuchu |\n| Qt host (offscreen) | Run click pannadhum button disable aachu, bot mudinjadhum enable; status \"Completed\", 100%; dark mode reopen pannappuram-um on-la irundhuchu |\n| Screenshots | Light, dark, 800px width: KPI, donut, steps, log, report pills ellam correct-aa render aachu; horizontal scroll illa; `{{` onnum meedhi illa |\n\nRun panna: `python dashboard/bridge.py`\n\nAlso noticed (fix pannala):\n- `load_dashboard` / `load_latest_report` empty list-dhaan anuppudhu, bot DB-yo report-o ezhudhala. Adhanaala live data-vula KPI-yum Report tab-um empty-aa irukkum. Test pannumbothu sample rows inject panni dhaan check panninen.\n- `bot.py` `ERROR` print pannaalum exit code 0-dhaan, so status \"Completed\"-nu dhaan kaattum.\n- \"Work item updated.\" log line-la WI number varaadhu; raw line `dashboard/bot_raw.log`-la save aagum.\n- Browser-la file-aa direct-aa open panna fetch errors varum, aana Qt-kulla avai varaadhu.\n\nFRONTEND: none\nQUESTIONS: Which colours: Tebra Sky, Superbill Ember, or Custom (one brand hex)? | Which optional sections to build: checkpoint tracker, admin/roles screen, login page, or none?" +} \ No newline at end of file diff --git a/tests/evals/results/b-pyqt-bot.run.jsonl b/tests/evals/results/b-pyqt-bot.run.jsonl new file mode 100644 index 0000000..c5bd675 --- /dev/null +++ b/tests/evals/results/b-pyqt-bot.run.jsonl @@ -0,0 +1,19 @@ +{"event": "status", "data": {"state": "running", "msg": "Running\u2026"}} +{"event": "step", "data": {"label": "Initializing", "pct": 10}} +{"event": "log", "data": {"msg": "Starting up.", "type": "info"}} +{"event": "log", "data": {"msg": "Work items loaded.", "type": "info"}} +{"event": "step", "data": {"label": "Login", "pct": 30}} +{"event": "log", "data": {"msg": "Logging in to the portal.", "type": "info"}} +{"event": "log", "data": {"msg": "Logged in.", "type": "success"}} +{"event": "step", "data": {"label": "Process work items", "pct": 50}} +{"event": "log", "data": {"msg": "Processing work items.", "type": "info"}} +{"event": "log", "data": {"msg": "Work item updated.", "type": "success"}} +{"event": "log", "data": {"msg": "Work item updated.", "type": "success"}} +{"event": "log", "data": {"msg": "Work item updated.", "type": "success"}} +{"event": "log", "data": {"msg": "The portal is slow to respond.", "type": "warn"}} +{"event": "log", "data": {"msg": "ERROR: could not attach the note to WI-1003", "type": "error"}} +{"event": "step", "data": {"label": "Write summary", "pct": 85}} +{"event": "log", "data": {"msg": "Writing the summary.", "type": "info"}} +{"event": "step", "data": {"label": "Completed", "pct": 100}} +{"event": "log", "data": {"msg": "All work items processed.", "type": "success"}} +{"event": "status", "data": {"state": "done", "msg": "Completed"}} diff --git a/tests/evals/run_eval.py b/tests/evals/run_eval.py index 5d9b532..9e7366f 100644 --- a/tests/evals/run_eval.py +++ b/tests/evals/run_eval.py @@ -36,9 +36,39 @@ FIXTURES = { "never_asks": [r"\bkeep\b|restyle"], "brand": "#7a3cff", }, + "b-pyqt-bot": { + "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you " + "work, so here are my answers to anything the skill asks. Colours: Tebra Sky. " + "Optional sections: none.", + "frontend": r"^\s*(none|no\b)", + "asks": [r"colou?r", r"section"], + "never_asks": [r"\bkeep\b|restyle", r"desktop|web"], + "bot": True, + }, } +def bot_checks(project, html, checks, replay): + """Bot fixtures: the page is wired, and a dry run of the generated wiring gives real steps and logs. + Returns the env that points the page check at that run.""" + checks["page wired to the bot (sample data off)"] = bool(re.search(r"SAMPLE_DATA\s*=\s*false", html)) + runners = [p for p in project.rglob("bot_runner.py") if ".claude" not in p.parts] + checks["bot_runner.py built"] = len(runners) == 1 + if not runners: + return {} + host = [runners[0], runners[0].with_name("bridge.py")] + checks["no placeholders left in the host"] = all(p.exists() and "{{" not in p.read_text(encoding="utf-8") for p in host) + dry = subprocess.run([sys.executable, str(runners[0])], cwd=project, capture_output=True, text=True, + encoding="utf-8", timeout=300) + replay.write_text(dry.stdout, encoding="utf-8") + events = [json.loads(x) for x in dry.stdout.splitlines() if x.startswith("{")] + steps = [e["data"] for e in events if e["event"] == "step"] + checks["bot dry run ends done"] = bool(events) and events[-1]["data"].get("state") == "done" + checks["real steps reach the page"] = len({s["label"] for s in steps}) >= 3 and steps[-1]["pct"] == 100 + checks["bot error line reaches the log"] = any(e["event"] == "log" and e["data"]["type"] == "error" for e in events) + return {"PAGE_CHECK_WIRED": "1", "PAGE_CHECK_REPLAY": str(replay)} + + def main(fixture): cfg = FIXTURES[fixture] work = Path(tempfile.mkdtemp(prefix=f"rpa-eval-{fixture}-")) @@ -80,8 +110,11 @@ def main(fixture): spec.loader.exec_module(palette) t = palette.derive(cfg["brand"]) checks["custom palette applied"] = all(f"--accent:{t[m]['--accent']}" in html for m in t) + env = {**os.environ, "PAGE_CHECK_HTML": str(pages[0])} + if cfg.get("bot"): + env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl")) pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"], - env={**os.environ, "PAGE_CHECK_HTML": str(pages[0])}, capture_output=True, text=True) + env=env, capture_output=True, text=True) checks["page check passes"] = pc.returncode == 0 page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]] else: diff --git a/tests/fixtures/b-pyqt-bot/README.md b/tests/fixtures/b-pyqt-bot/README.md new file mode 100644 index 0000000..668787c --- /dev/null +++ b/tests/fixtures/b-pyqt-bot/README.md @@ -0,0 +1,10 @@ +# Work-item follow-up bot + +A desktop RPA bot. It logs in to the portal and updates each work item. +The operator window will be a PyQt6 app. + +Run the bot on its own: + +``` +python -m bot +``` diff --git a/tests/fixtures/b-pyqt-bot/bot.py b/tests/fixtures/b-pyqt-bot/bot.py new file mode 100644 index 0000000..d38dc9c --- /dev/null +++ b/tests/fixtures/b-pyqt-bot/bot.py @@ -0,0 +1,29 @@ +"""Work-item follow-up bot. Start it with: python -m bot""" +import time + +ITEMS = ["WI-1001", "WI-1002", "WI-1003"] + + +def say(line): + print(line) + time.sleep(0.2) + + +def main(): + say("[step 1] Initializing") + say("Loading settings from config.ini") + say("Completed loading 3 work items") + say("[step 2] Logging in to the portal") + say("Login page loaded") + say("[login] ok") + say("[step 3] Processing work items") + for i, item in enumerate(ITEMS, 1): + say(f"[item] {item} updated ({i}/{len(ITEMS)})") + say("WARNING: portal slow to respond, waited 5s") + say("ERROR: could not attach the note to WI-1003") + say("[step 4] Writing the summary") + say(f"[all done] {len(ITEMS)} items processed") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/b-pyqt-bot/requirements.txt b/tests/fixtures/b-pyqt-bot/requirements.txt new file mode 100644 index 0000000..8da5982 --- /dev/null +++ b/tests/fixtures/b-pyqt-bot/requirements.txt @@ -0,0 +1,2 @@ +PyQt6 +PyQt6-WebEngine diff --git a/tests/page_check/conftest.py b/tests/page_check/conftest.py index fe34b0e..c34ce33 100644 --- a/tests/page_check/conftest.py +++ b/tests/page_check/conftest.py @@ -4,6 +4,8 @@ through window.__pyMsg, and assert on what is visible. Page under test: set PAGE_CHECK_HTML to a generated dashboard.html. Without it, the no-AI template is filled with TEST_FILL. Every test runs in light, dark and narrow mode. The page sees a fake Qt bridge: each slot call lands in window.__calls as [slot, ...args]. +PAGE_CHECK_WIRED=1: the page is wired to a bot (SAMPLE_DATA off), so sample-data cases skip. +PAGE_CHECK_REPLAY: a JSON-lines file of envelopes a real bot run produced (bot_runner.py's dry run). """ import json import os @@ -32,6 +34,14 @@ window.QWebChannel = function (_transport, ready) { }; """ MODES = {"light": (1280, 820, False), "dark": (1280, 820, True), "narrow": (800, 900, False)} +WIRED = os.environ.get("PAGE_CHECK_WIRED") == "1" + + +@pytest.fixture +def sample_data(): + """Use on cases that need the built-in sample data; they skip on a bot-wired page.""" + if WIRED: + pytest.skip("page is wired to a bot, so it has no sample data") @pytest.fixture(scope="session") @@ -39,6 +49,8 @@ def page_url(tmp_path_factory): if os.environ.get("PAGE_CHECK_HTML"): return Path(os.environ["PAGE_CHECK_HTML"]).resolve().as_uri() html = re.sub(r"\{\{(\w+)\}\}", lambda m: TEST_FILL[m[1]], TEMPLATE.read_text(encoding="utf-8")) + if WIRED: + html = html.replace("const SAMPLE_DATA = true", "const SAMPLE_DATA = false") out = tmp_path_factory.mktemp("page") / "dashboard.html" out.write_text(html, encoding="utf-8") return out.as_uri() diff --git a/tests/page_check/test_bot_run.py b/tests/page_check/test_bot_run.py new file mode 100644 index 0000000..72552bc --- /dev/null +++ b/tests/page_check/test_bot_run.py @@ -0,0 +1,84 @@ +"""A bot run as the page sees it: status, step and log envelopes in order, from a real bot +run (PAGE_CHECK_REPLAY, the JSON lines bot_runner.py's dry run prints) or a built-in one.""" +import json +import os +import re +from pathlib import Path + +import pytest +from conftest import WIRED +from playwright.sync_api import expect +from test_contract import colour_of, kpi, open_tab + +LOG_COLOURS = {"info": "blue", "success": "green", "warn": "amber", "error": "red"} +DOT_COLOURS = {"ready": "green", "running": "amber", "done": "green", "error": "red"} + + +def built_in_run(steps): + run = [{"event": "status", "data": {"state": "running", "msg": "Running…"}}] + for i, label in enumerate(steps): + run.append({"event": "step", "data": {"label": label, "pct": round((i + 1) / len(steps) * 100)}}) + run.append({"event": "log", "data": {"msg": f"Finished {label}.", "type": "success"}}) + run.insert(4, {"event": "log", "data": {"msg": "ERROR: portal timed out, retrying", "type": "error"}}) + return run + [{"event": "status", "data": {"state": "done", "msg": "Completed"}}] + + +def recorded_run(): + path = os.environ.get("PAGE_CHECK_REPLAY") + return [json.loads(x) for x in Path(path).read_text(encoding="utf-8-sig").splitlines() if x.strip()] if path else None + + +def expect_status(page, state, msg): + # The status indicator has no role of its own, so it's found by its documented ids. + expect(page.locator("#st-text")).to_have_text(msg or state) + assert colour_of(page.locator("#st-dot"), "backgroundColor") == DOT_COLOURS[state], state + run = page.get_by_role("button", name="▶ Run", exact=True) + expect(run).to_be_disabled() if state == "running" else expect(run).to_be_enabled() + + +def test_bot_run_shows_status_steps_and_log(page, push): + steps = page.evaluate("STEPS") + run = recorded_run() or built_in_run(steps) + open_tab(page, "Run") + expect_status(page, "ready", "System ready") + items = page.get_by_role("list", name="Steps").get_by_role("listitem") + log, last_step = page.get_by_role("log"), -1 + for envelope in run: + event, d = envelope["event"], envelope["data"] + push(event, d) + if event == "status": + expect_status(page, d["state"], d.get("msg")) + elif event == "step": + assert d["label"] in steps, f"step label {d['label']!r} is not one of the page's STEPS {steps}" + idx = steps.index(d["label"]) + assert idx >= last_step, f"progress went back from {steps[last_step]!r} to {d['label']!r}" + last_step = idx + expect(page.get_by_text(f"{d['pct']}%", exact=True)).to_be_visible() + if d["pct"] >= 100: + for i in range(len(steps)): + expect(items.nth(i)).to_have_text(re.compile(r"^\s*✓")) + else: + expect(items.nth(idx)).to_have_attribute("aria-current", "step") + elif event == "log": + line = log.get_by_text(d["msg"], exact=True).last + expect(line).to_be_visible() + assert colour_of(line) == LOG_COLOURS[d["type"]], d["msg"] + assert any(e["event"] == "step" for e in run), "the run never moved the progress bar" + + +def test_failed_run_shows_error_status(page, push): + push("status", {"state": "running", "msg": "Running…"}) + expect_status(page, "running", "Running…") + push("status", {"state": "error", "msg": "Failed (exit code 1)"}) + expect_status(page, "error", "Failed (exit code 1)") + + +@pytest.mark.skipif(not WIRED, reason="only for a page wired to a bot (PAGE_CHECK_WIRED=1)") +def test_wired_page_sends_its_actions_to_the_bot(page): + calls = lambda: page.evaluate("window.__calls") # noqa: E731 + assert ["load_dashboard"] in calls() + kpi(page, "Total batches", 0) # no sample batches on a wired page + page.get_by_role("button", name="▶ Run", exact=True).click() + assert ["start_automation"] in calls() + open_tab(page, "Report") + assert ["load_latest_report"] in calls() diff --git a/tests/page_check/test_contract.py b/tests/page_check/test_contract.py index 218cbd9..299b8f8 100644 --- a/tests/page_check/test_contract.py +++ b/tests/page_check/test_contract.py @@ -1,6 +1,7 @@ """No-AI contract cases: events in, visible result out (see templates/contract.md).""" import re +import pytest from playwright.sync_api import expect @@ -146,6 +147,7 @@ def expect_sample_dashboard(page): kpi(page, "Total batches", n) +@pytest.mark.usefixtures("sample_data") def test_no_bot_page_runs_on_sample_data(page): """No bot yet: the page answers its own data actions, so it works on day one.""" expect_sample_dashboard(page) @@ -162,6 +164,7 @@ def test_no_bot_page_runs_on_sample_data(page): expect(run).to_be_enabled() +@pytest.mark.usefixtures("sample_data") def test_plain_browser_first_open_shows_sample_data(browser, page_url): pg = browser.new_page() pg.goto(page_url) diff --git a/tests/test_bot_runner.py b/tests/test_bot_runner.py new file mode 100644 index 0000000..788e8e7 --- /dev/null +++ b/tests/test_bot_runner.py @@ -0,0 +1,112 @@ +"""Bot wiring self-check (templates/bot_runner.py): which page events each raw bot line makes.""" +import importlib.util +import os +import sys +import time +from pathlib import Path + +import pytest + +RUNNER = Path(__file__).resolve().parents[1] / "rpa-dashboard-no-ai" / "templates" / "bot_runner.py" + + +@pytest.fixture +def runner(): + spec = importlib.util.spec_from_file_location("bot_runner", RUNNER) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.STEP_MARKERS = [("[step 1]", "Initializing", 10), ("[step 2]", "Login", 40), + ("[step 3]", "Process items", 70), ("[all done]", "Completed", 100)] + mod.FRIENDLY_RULES = [("[login] done", "Logged in.", "success"), ("[login]", "Logging in...", "info"), + ("could not save receipt", "Could not save the receipt.", "error")] + return mod + + +def steps(runner, line): + return [d for e, d in runner.events_for(line) if e == "step"] + + +def test_step_marker_moves_progress(runner): + assert steps(runner, "[step 2] Opening the login page") == [{"label": "Login", "pct": 40}] + assert steps(runner, "[STEP 3] Processing 12 items") == [{"label": "Process items", "pct": 70}] + + +def test_common_word_does_not_move_progress(runner): + assert steps(runner, "Login page loaded slowly") == [] + assert steps(runner, "Completed loading the config") == [] + + +def test_first_step_marker_wins(runner): + assert steps(runner, "[step 1] retry after [step 3] timed out") == [{"label": "Initializing", "pct": 10}] + + +def logs(runner, line): + return [d for e, d in runner.events_for(line) if e == "log"] + + +def test_friendly_rule_rewrites_the_line_and_first_rule_wins(runner): + assert logs(runner, "[login] done in 2.1s") == [{"msg": "Logged in.", "type": "success"}] + assert logs(runner, "[LOGIN] sending credentials") == [{"msg": "Logging in...", "type": "info"}] + + +def test_unmatched_non_error_lines_are_hidden(runner): + for line in ["Loading config from settings.ini", "[step 2] Opening the login page", "warning: slow page"]: + assert logs(runner, line) == [], line + + +@pytest.mark.parametrize("line", [ + "ERROR: portal timed out", "Traceback (most recent call last):", "ValueError: bad date", + "Exception: boom", "FATAL disk full", "Login failed for user 3"]) +def test_unmatched_error_line_is_always_shown_raw(runner, line): + assert logs(runner, line) == [{"msg": line, "type": "error"}] + + +def test_matched_error_line_shows_its_friendly_message(runner): + assert logs(runner, "ERROR could not save receipt 42") == [{"msg": "Could not save the receipt.", "type": "error"}] + + +def test_error_on_a_step_line_moves_progress_and_is_shown(runner): + line = "[step 3] ERROR item 2 rejected" + assert runner.events_for(line) == [("step", {"label": "Process items", "pct": 70}), + ("log", {"msg": line, "type": "error"})] + + +def run_bot(runner, tmp_path, script): + """Run a fake bot through runner.run(); return [(seconds since start, event, data)].""" + bot = tmp_path / "fake_bot.py" + bot.write_text(script, encoding="utf-8") + runner.BOT_CMD, runner.BOT_CWD, runner.RAW_LOG = [sys.executable, str(bot)], tmp_path, tmp_path / "raw.log" + t0, sent = time.monotonic(), [] + runner.run(lambda event, data: sent.append((time.monotonic() - t0, event, data))) + return sent + + +def test_run_streams_bot_output_live_from_a_child_process(runner, tmp_path): + sent = run_bot(runner, tmp_path, "import os, time\n" + "print('[step 1] pid', os.getpid())\n" + "print('[login] done')\n" + "time.sleep(1)\n" + "print('[all done] ✓ 3 items')\n") + events = [(e, d) for _, e, d in sent] + assert events == [("status", {"state": "running", "msg": "Running…"}), + ("step", {"label": "Initializing", "pct": 10}), + ("log", {"msg": "Logged in.", "type": "success"}), + ("step", {"label": "Completed", "pct": 100}), + ("status", {"state": "done", "msg": "Completed"})] + assert sent[3][0] - sent[1][0] > 0.5, "lines must reach the page while the bot runs, not at exit" + raw = (tmp_path / "raw.log").read_text(encoding="utf-8") + assert "[login] done" in raw and "✓ 3 items" in raw # raw lines kept for audit + assert f"pid {os.getpid()}" not in raw # the bot ran in its own process + + +def test_run_ends_in_error_when_the_bot_fails(runner, tmp_path): + sent = run_bot(runner, tmp_path, "print('[step 1] start')\nraise SystemExit(3)\n") + assert sent[-1][1:] == ("status", {"state": "error", "msg": "Failed (exit code 3)"}) + + +def test_run_ends_in_error_when_the_bot_cannot_start(runner, tmp_path): + runner.BOT_CMD, runner.BOT_CWD, runner.RAW_LOG = ["no-such-bot-xyz"], tmp_path, tmp_path / "raw.log" + sent = [] + runner.run(lambda event, data: sent.append((event, data))) + assert sent[-2][0] == "log" and sent[-2][1]["type"] == "error" + assert sent[-1] == ("status", {"state": "error", "msg": "Could not start the bot"}) diff --git a/tests/test_templates.py b/tests/test_templates.py index d1a9d28..89174ea 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -9,7 +9,8 @@ ROOT = Path(__file__).resolve().parents[1] HOOK_RULES = "E9,F63,F7,F82,F811" # same rule set as the PostToolUse lint hook -@pytest.mark.parametrize("template", ["rpa-dashboard-no-ai/templates/bridge.py"]) +@pytest.mark.parametrize("template", ["rpa-dashboard-no-ai/templates/bridge.py", + "rpa-dashboard-no-ai/templates/bot_runner.py"]) def test_template_passes_lint_hook_unfilled(template): ruff = shutil.which("ruff") or pytest.skip("ruff not on PATH") r = subprocess.run([ruff, "check", "--select", HOOK_RULES, "--no-cache", str(ROOT / template)],