From d89dce3c3328f1e112340f0f0efa995e016f692f Mon Sep 17 00:00:00 2001 From: Vasanth Date: Fri, 16 Oct 2026 15:32:45 +0530 Subject: [PATCH] AI desktop path: plain-HTML Review and Cost tabs over the Qt bridge, gate in Python ai_sections.html pastes into the no-AI dashboard.html; ai_bridge.py adds one ai_request slot that hands every AI contract request (incl. GET /roi) to the review backend's dispatch(). Desktop page check reruns the web review and cost cases plus a tampered-page gate case. Eval fixture e-pyqt-ai added. Co-Authored-By: Claude Opus 5.5 --- rpa-dashboard-ai/SKILL.md | 12 +-- rpa-dashboard-ai/templates/contract.md | 3 + .../templates/desktop/ai_sections.html | 76 ++++++++++++++++++- tests/README.md | 10 ++- tests/evals/run_eval.py | 23 +++++- tests/fixtures/e-pyqt-ai/README.md | 11 +++ tests/fixtures/e-pyqt-ai/bot.py | 31 ++++++++ tests/fixtures/e-pyqt-ai/extractor.py | 34 +++++++++ .../fixtures/e-pyqt-ai/inbox/invoice-2201.pdf | 2 + .../fixtures/e-pyqt-ai/inbox/invoice-2202.pdf | 2 + tests/fixtures/e-pyqt-ai/requirements.txt | 3 + tests/page_check_ai_desktop/conftest.py | 2 +- .../page_check_ai_desktop/test_ai_desktop.py | 8 ++ 13 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 tests/fixtures/e-pyqt-ai/README.md create mode 100644 tests/fixtures/e-pyqt-ai/bot.py create mode 100644 tests/fixtures/e-pyqt-ai/extractor.py create mode 100644 tests/fixtures/e-pyqt-ai/inbox/invoice-2201.pdf create mode 100644 tests/fixtures/e-pyqt-ai/inbox/invoice-2202.pdf create mode 100644 tests/fixtures/e-pyqt-ai/requirements.txt diff --git a/rpa-dashboard-ai/SKILL.md b/rpa-dashboard-ai/SKILL.md index 421bdd2..084f5ed 100644 --- a/rpa-dashboard-ai/SKILL.md +++ b/rpa-dashboard-ai/SKILL.md @@ -15,7 +15,7 @@ means `../rpa-dashboard-no-ai/templates/`. | `templates/web/pricing.py` | The pricing table (with its source and as-of date) and `cost_for_call`. The backend imports it. | | `templates/web/server_stub.py` | Sample-data backend for runs, review rows and the approve gate. Standard library only. | | `templates/web/bands.py` | The band cut points and rules. The one source of truth: the backend imports it. | -| `templates/desktop/ai_sections.html` | Desktop: the Review tab (band legend, review split pane, drafts, Verify, approve gate) in plain HTML, pasted into `dashboard.html`. | +| `templates/desktop/ai_sections.html` | Desktop: the Review tab (band legend, review split pane, drafts, Verify, approve gate) and the Cost tab in plain HTML, pasted into `dashboard.html`. | | `templates/desktop/ai_bridge.py` | Desktop: the no-AI `Bridge` plus the `ai_request` slot that hands every AI request to the Python review backend. | | `templates/contract.md` | The AI contract: runs, review rows, endpoints and the gate, over HTTP or the Qt bridge. Extends `contract.md`. | | `templates/reference/` | Superbill Review extracts, marked EXAMPLE ONLY. Use them to port extra polish, never copy their project names. | @@ -71,9 +71,9 @@ pane, the approve gate and the cost page. Ask which extras to add (choose any, o the model per batch), **retry on another model** (for failed runs). Build only the chosen ones at their `OPTIONAL:` markers, and delete the markers. - **Model picker:** a `` (from `GET /models`, defaulting to the run's own model) and a Retry button that sends `POST /runs/{job_id}/retry {model}`, then polls the run as usual. Both need the endpoints in `templates/contract.md`; add them to the backend (and the stub). @@ -106,11 +106,11 @@ Both need the endpoints in `templates/contract.md`; add them to the backend (and 1. Build the no-AI desktop page and host first, exactly as steps 6 and 7 of `../rpa-dashboard-no-ai/SKILL.md` say: `dashboard/dashboard.html` (theme, placeholders, `SAMPLE_DATA = false` once the bot is wired), `bridge.py` and `bot_runner.py`. -2. Paste the whole of `templates/desktop/ai_sections.html` into `dashboard/dashboard.html`, +2. Paste the whole of `templates/desktop/ai_sections.html` (Review and Cost tabs) into `dashboard/dashboard.html`, just before ``. Change nothing in it: it uses the page's theme tokens, and its band colours are fixed. -3. Copy `templates/desktop/ai_bridge.py`, `templates/web/server_stub.py` and - `templates/web/bands.py` into `dashboard/`. Fill `{{APP_NAME}}` in `ai_bridge.py`. +3. Copy `templates/desktop/ai_bridge.py`, `templates/web/server_stub.py`, + `templates/web/bands.py` and `templates/web/pricing.py` into `dashboard/`. Fill `{{APP_NAME}}` in `ai_bridge.py`. 4. The app now starts with `python dashboard/ai_bridge.py` (not `bridge.py`). 5. Search the output for `{{`. None may be left. diff --git a/rpa-dashboard-ai/templates/contract.md b/rpa-dashboard-ai/templates/contract.md index b2c679f..88bd978 100644 --- a/rpa-dashboard-ai/templates/contract.md +++ b/rpa-dashboard-ai/templates/contract.md @@ -78,6 +78,9 @@ The same endpoints and rules travel over the Qt bridge instead of HTTP: path above without `{API_BASE}`, `body` is a JSON string (`"{}"` when there is none). - Python to page: one envelope, `{event: "ai_response", data: {id, status, body}}`, with the same `status` and JSON `body` the web endpoint would answer. +- Every endpoint above uses this pair, the cost page's `GET /roi` included (the page sends + `ai_request(id, "GET", "/roi", "{}")` and gets `{runs: [RoiRun], pricing: {source, as_of}}` + back as the `body`). No other AI events or slots exist. - `GET /runs/{job_id}/document` answers an SVG string or a URL the page can load (for example a `file:///` path to the PDF). diff --git a/rpa-dashboard-ai/templates/desktop/ai_sections.html b/rpa-dashboard-ai/templates/desktop/ai_sections.html index cbc4944..7cda176 100644 --- a/rpa-dashboard-ai/templates/desktop/ai_sections.html +++ b/rpa-dashboard-ai/templates/desktop/ai_sections.html @@ -34,6 +34,9 @@ body{--clean:#51a68f;--verify:#e39b34;--review:#de5243;--absent:#6b6580} .rv-fields .btn{width:92px} .rv-save{display:flex;align-items:center;gap:10px;border-top:1px solid var(--border);padding:10px 16px} .rv-save .rv-unsaved{font-size:12px;font-weight:700;color:var(--warn)} +.co-tile{padding:16px 18px;border-radius:16px;background:var(--glass-strong);border:1px solid var(--border)} +.co-tile .kpi-val{margin-top:4px} +.co-r{text-align:right} @media (max-width:820px){.rv-split{flex-direction:column}.rv-filters{margin-left:0}} @@ -43,6 +46,8 @@ body{--clean:#51a68f;--verify:#e39b34;--review:#de5243;--absent:#6b6580}
+ +
@@ -61,6 +66,29 @@ body{--clean:#51a68f;--verify:#e39b34;--review:#de5243;--absent:#6b6580}
+ + diff --git a/tests/README.md b/tests/README.md index 2ebe9c5..dec0cc4 100644 --- a/tests/README.md +++ b/tests/README.md @@ -29,6 +29,12 @@ uv run --with pytest --with playwright pytest tests by-hand assumptions), plus every no-AI case. Tests set run and review state on the stub directly (the `stub` fixture). To check a generated dashboard folder instead, set `PAGE_CHECK_AI_DIR=path/to/dashboard`. Needs npm on PATH, else it skips. +- `page_check_ai_desktop/`: **Seam 1, AI desktop page.** The no-AI `dashboard.html` with + `ai_sections.html` pasted in, loaded from `file://` with a fake Qt bridge whose + `ai_request` slot goes to `server_stub.dispatch()` in this process (what `ai_bridge.py` + does). Runs the AI web page's review and cost cases and the no-AI cases unchanged, plus + a tampered page calling the slot directly (still refused). To check a generated folder, + set `PAGE_CHECK_AI_DESKTOP_DIR=path/to/dashboard`; `PAGE_CHECK_WIRED=1` as above. - `test_pricing.py`: the AI cost function (normal, long-context, thinking tokens, unknown model). - `test_reference_scan.py`: no patient data, internal IPs, credentials or keys in any @@ -45,7 +51,9 @@ uv run --with pytest --with playwright pytest tests fixture folder with different answers: `c-keep` and `c-restyle` both run on `c-web-existing-frontend`, and also check what changed in the files it already had. An `"ai": True` entry (`d-ai-extraction`) expects the AI skill's React starter and - runs `page_check_ai/` on the folder it built. + runs `page_check_ai/` on the folder it built. An `"ai_desktop": True` entry + (`e-pyqt-ai`) expects the desktop AI page and runs `page_check/` (with the bot replay) + and `page_check_ai_desktop/` on it. For a bot fixture (`b-pyqt-bot`), the eval also dry-runs the generated `bot_runner.py`, saves its events to `evals/results/.run.jsonl`, and replays them in the page check. diff --git a/tests/evals/run_eval.py b/tests/evals/run_eval.py index 6510c18..65b780a 100644 --- a/tests/evals/run_eval.py +++ b/tests/evals/run_eval.py @@ -80,6 +80,17 @@ FIXTURES = { "page_has": ["--accent:#ff3049"], "ai": True, }, + # (e) PyQt bot with an LLM extraction call: the AI skill's desktop path, no web server. + "e-pyqt-ai": { + "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you " + "work, so here are my answers to anything the skill asks. Colours: Tebra Sky. " + "Optional sections: none.", + "frontend": r"^\s*(none|no\b)", + "asks": [r"colou?r", r"section"], + "never_asks": [r"\bkeep\b|restyle", r"desktop|web"], + "bot": True, + "ai_desktop": True, + }, } # A line a restyle may change: a custom property, a comment, a blank, or a web-font link. TOKEN_LINE = r"^\s*(--[\w-]+\s*:.*|/\*.*\*/|]*fonts\.googleapis\.com.*|@import.*fonts\.googleapis\.com.*)?\s*$" @@ -171,6 +182,11 @@ def main(fixture): html += css.read_text(encoding="utf-8") if css.exists() else "" checks["AI path: review starter and server stub built"] = all( (pages[0].parent / f).exists() for f in ["server_stub.py", "bands.py", "src/components/Review.tsx"]) + if cfg.get("ai_desktop"): # the no-AI page with the plain-HTML AI sections, hosted by ai_bridge.py + host = pages[0].with_name("ai_bridge.py") + checks["AI desktop path: sections, ai_bridge and review backend built"] = 'id="ai-review-tab"' in html and all( + pages[0].with_name(f).exists() for f in ["ai_bridge.py", "server_stub.py", "bands.py", "pricing.py"]) + checks["no placeholders left in ai_bridge.py"] = host.exists() and "{{" not in host.read_text(encoding="utf-8") shutil.copy(pages[0], results / f"{fixture}.dashboard.html") checks["no placeholders left"] = "{{" not in html if cfg.get("brand"): @@ -186,8 +202,11 @@ def main(fixture): env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl")) if cfg.get("ai"): env["PAGE_CHECK_AI_DIR"] = str(pages[0].parent) - check_dir = ROOT / "tests" / ("page_check_ai" if cfg.get("ai") else "page_check") - pc = subprocess.run([sys.executable, "-m", "pytest", str(check_dir), "-q"], + check_dirs = [ROOT / "tests" / ("page_check_ai" if cfg.get("ai") else "page_check")] + if cfg.get("ai_desktop"): + env["PAGE_CHECK_AI_DESKTOP_DIR"] = str(pages[0].parent) + check_dirs.append(ROOT / "tests" / "page_check_ai_desktop") + pc = subprocess.run([sys.executable, "-m", "pytest", *map(str, check_dirs), "-q"], env=env, capture_output=True, text=True) checks["page check passes"] = pc.returncode == 0 page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]] diff --git a/tests/fixtures/e-pyqt-ai/README.md b/tests/fixtures/e-pyqt-ai/README.md new file mode 100644 index 0000000..88c0c63 --- /dev/null +++ b/tests/fixtures/e-pyqt-ai/README.md @@ -0,0 +1,11 @@ +# Invoice intake bot + +A desktop RPA bot. It reads each scanned invoice in `inbox/` with an LLM +(`extractor.py`) and queues the fields for a person to review. +The operator window will be a PyQt6 app. + +Run the bot on its own: + +``` +python -m bot +``` diff --git a/tests/fixtures/e-pyqt-ai/bot.py b/tests/fixtures/e-pyqt-ai/bot.py new file mode 100644 index 0000000..9e961bb --- /dev/null +++ b/tests/fixtures/e-pyqt-ai/bot.py @@ -0,0 +1,31 @@ +"""Invoice intake bot: picks up scanned invoices and extracts their fields with an LLM. +Start it with: python -m bot""" +import time +from pathlib import Path + +INBOX = Path(__file__).with_name("inbox") + + +def say(line): + print(line, flush=True) + time.sleep(0.2) + + +def main(): + say("[step 1] Initializing") + docs = sorted(INBOX.glob("*.pdf")) + say(f"[inbox] {len(docs)} invoices waiting") + say("[step 2] Extracting fields") + for i, doc in enumerate(docs, 1): + try: + from extractor import extract_invoice # the LLM call + rows = extract_invoice(doc.read_bytes(), "application/pdf") + say(f"[extract] {doc.name}: {len(rows)} fields ({i}/{len(docs)})") + except Exception as e: # one bad document must not stop the batch + say(f"ERROR: extraction failed for {doc.name}: {type(e).__name__}") + say("[step 3] Queueing for review") + say(f"[all done] {len(docs)} invoices processed") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/e-pyqt-ai/extractor.py b/tests/fixtures/e-pyqt-ai/extractor.py new file mode 100644 index 0000000..19e4d09 --- /dev/null +++ b/tests/fixtures/e-pyqt-ai/extractor.py @@ -0,0 +1,34 @@ +"""The LLM extraction call: one scanned invoice in, its fields out, each with word scores.""" +import json + +from google import genai +from google.genai import types + +MODEL = "gemini-2.5-flash" +FIELDS = { # field name -> mandatory + "Invoice number": True, + "Invoice date": True, + "Vendor": True, + "PO number": False, + "Due date": True, + "Total": True, +} +PROMPT = ( + "Read this invoice. Return JSON: a list of {field, value, word_scores}, one per field in " + + ", ".join(FIELDS) + + ". word_scores is your 0-1 legibility score for each word of the value. Use an empty " + "value and no scores when a field isn't on the page." +) + + +def extract_invoice(document: bytes, mime_type: str) -> list[dict]: + client = genai.Client() # credentials come from the environment + response = client.models.generate_content( + model=MODEL, + contents=[types.Part.from_bytes(data=document, mime_type=mime_type), PROMPT], + config=types.GenerateContentConfig(response_mime_type="application/json"), + ) + rows = json.loads(response.text) + for row in rows: + row["mandatory"] = FIELDS.get(row["field"], False) + return rows diff --git a/tests/fixtures/e-pyqt-ai/inbox/invoice-2201.pdf b/tests/fixtures/e-pyqt-ai/inbox/invoice-2201.pdf new file mode 100644 index 0000000..42a32b3 --- /dev/null +++ b/tests/fixtures/e-pyqt-ai/inbox/invoice-2201.pdf @@ -0,0 +1,2 @@ +%PDF-1.4 +% sample invoice placeholder diff --git a/tests/fixtures/e-pyqt-ai/inbox/invoice-2202.pdf b/tests/fixtures/e-pyqt-ai/inbox/invoice-2202.pdf new file mode 100644 index 0000000..42a32b3 --- /dev/null +++ b/tests/fixtures/e-pyqt-ai/inbox/invoice-2202.pdf @@ -0,0 +1,2 @@ +%PDF-1.4 +% sample invoice placeholder diff --git a/tests/fixtures/e-pyqt-ai/requirements.txt b/tests/fixtures/e-pyqt-ai/requirements.txt new file mode 100644 index 0000000..b7a8fae --- /dev/null +++ b/tests/fixtures/e-pyqt-ai/requirements.txt @@ -0,0 +1,3 @@ +PyQt6 +PyQt6-WebEngine +google-genai diff --git a/tests/page_check_ai_desktop/conftest.py b/tests/page_check_ai_desktop/conftest.py index 73d10b1..c145441 100644 --- a/tests/page_check_ai_desktop/conftest.py +++ b/tests/page_check_ai_desktop/conftest.py @@ -63,7 +63,7 @@ def site(tmp_path_factory): sections = (AI / "desktop" / "ai_sections.html").read_text(encoding="utf-8") out = tmp_path_factory.mktemp("desktop") (out / "dashboard.html").write_text(html.replace("", sections + "\n"), encoding="utf-8") - for f in ["server_stub.py", "bands.py"]: + for f in ["server_stub.py", "bands.py", "pricing.py"]: (out / f).write_bytes((AI / "web" / f).read_bytes()) return out diff --git a/tests/page_check_ai_desktop/test_ai_desktop.py b/tests/page_check_ai_desktop/test_ai_desktop.py index e122efa..8ffef30 100644 --- a/tests/page_check_ai_desktop/test_ai_desktop.py +++ b/tests/page_check_ai_desktop/test_ai_desktop.py @@ -16,6 +16,14 @@ from test_ai_review import ( # noqa: E402,F401 test_page_x_of_y_updates_while_running, ) from test_ai_review import FIELDS, FLAGGED, open_review # noqa: E402 +from test_ai_cost import ( # noqa: E402,F401 + cost_page, + test_by_hand_panel_is_labelled_assumptions_and_recomputes, + test_coverage_pill_counts_priced_of_all_documents, + test_month_breakdown_groups_runs_by_month, + test_runs_show_tokens_and_cost_and_unpriced_is_not_zero, + test_spend_tiles_sum_the_priced_runs, +) sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "page_check")) from test_contract import ( # noqa: E402,F401