"""AI contract cases: confidence bands, review drafts, the approve gate and run progress. Run and review state is set on the server stub (the page's backend); what the page shows is asserted through the browser. See rpa-dashboard-ai/templates/contract.md. """ import json import re import urllib.error import urllib.request from playwright.sync_api import expect # (field, value, word scores, mandatory) -> the band each one must land in FIELDS = [ ("Invoice number", "INV-2207", [1.0], True), # green ("Vendor", "Northwind Traders", [0.99, 0.97], True), # green ("Invoice date", "2026-10-02", [0.93], True), # amber ("Total", "1,284.50", [0.98, 0.71], True), # red: worst word wins ("Due date", "", [1.0], True), # blank mandatory ("PO number", "PO-118", [], False), # unscored ] EXPECTED = {"green": ["Invoice number", "Vendor"], "amber": ["Invoice date"], "red": ["Total"], "blank": ["Due date"], "unscored": ["PO number"]} FLAGGED = 4 # everything but green def api(server, method, path, body=None): req = urllib.request.Request(server + "/api" + path, method=method, data=None if body is None else json.dumps(body).encode(), headers={"Content-Type": "application/json"}) try: with urllib.request.urlopen(req) as r: return r.status, json.loads(r.read()) except urllib.error.HTTPError as e: return e.code, json.loads(e.read()) def test_server_refuses_approval_while_flagged_fields_are_unverified(stub, server): """The gate lives on the server: no request from any page can skip it.""" stub.put_run("job-gate", "gate.pdf", FIELDS) status, body = api(server, "POST", "/runs/job-gate/approve") assert status == 409 and str(FLAGGED) in body["detail"] _, rows = api(server, "GET", "/runs/job-gate/review") first_flagged = next(r for r in rows["rows"] if r["band"] != "green") api(server, "PATCH", f"/review/{first_flagged['row_id']}", {"verified": True}) status, body = api(server, "POST", "/runs/job-gate/approve") assert status == 409 and str(FLAGGED - 1) in body["detail"] api(server, "POST", "/runs/job-gate/verify-remaining") status, _ = api(server, "POST", "/runs/job-gate/approve") assert status == 200 assert api(server, "GET", "/runs/job-gate")[1]["approved"] is True def open_review(page, job_id): page.get_by_role("navigation").get_by_role("button", name="Review", exact=True).click() page.get_by_label("Document", exact=True).select_option(job_id) def fields_list(page): return page.get_by_role("list", name="Extracted fields").get_by_role("listitem") LEGEND = {"green": "Clean", "amber": "Check", "red": "Review", "blank": "Missing", "unscored": "Unscored"} def test_band_counts_and_legend_match(page, stub): stub.put_run("job-t", "bands.pdf", FIELDS) open_review(page, "job-t") legend = page.get_by_role("list", name="Confidence bands").get_by_role("listitem") expect(legend).to_have_count(len(LEGEND)) for i, (band, word) in enumerate(LEGEND.items()): expect(legend.nth(i)).to_have_text(re.compile(rf"^\s*{word}\b.*· {len(EXPECTED[band])}\s*$")) expect(page.get_by_text(re.compile("print clarity, not correctness", re.I))).to_be_visible() def test_band_filters_show_the_right_rows(page, stub): stub.put_run("job-t", "bands.pdf", FIELDS) open_review(page, "job-t") expect(fields_list(page)).to_have_count(len(FIELDS)) filters = page.get_by_role("group", name="Filter by band") for name, band in [("Red", "red"), ("Amber", "amber"), ("Missing", "blank"), ("Unscored", "unscored")]: filters.get_by_role("button", name=name, exact=True).click() expect(fields_list(page)).to_have_count(len(EXPECTED[band])) for field in EXPECTED[band]: expect(page.get_by_role("textbox", name=field, exact=True)).to_be_visible() filters.get_by_role("button", name="All", exact=True).click() expect(fields_list(page)).to_have_count(len(FIELDS)) def row(stub, job_id, field): return next(r for r in stub.ROWS[job_id] if r["field"] == field) def test_draft_edit_is_not_saved_until_save(page, stub): stub.put_run("job-t", "bands.pdf", FIELDS) open_review(page, "job-t") box = page.get_by_role("textbox", name="Vendor", exact=True) box.fill("Northwind Traders Ltd") expect(page.get_by_text("1 unsaved change", exact=True)).to_be_visible() assert row(stub, "job-t", "Vendor")["corrected_value"] is None asked = [] page.once("dialog", lambda d: (asked.append(d.message), d.dismiss())) page.get_by_role("navigation").get_by_role("button", name="Report", exact=True).click() expect(box).to_have_value("Northwind Traders Ltd") # leaving was refused, the draft is still there assert asked and "unsaved" in asked[0].lower() page.get_by_role("button", name="Save", exact=True).click() expect(page.get_by_text("1 unsaved change", exact=True)).to_be_hidden() saved = row(stub, "job-t", "Vendor") assert saved["corrected_value"] == "Northwind Traders Ltd" and saved["verified"] is True def test_approve_stays_blocked_while_rows_are_unverified(page, stub): stub.put_run("job-t", "bands.pdf", FIELDS) open_review(page, "job-t") page.get_by_role("button", name="Verify Total", exact=True).click() expect(page.get_by_role("button", name="Verify Total", exact=True)).to_have_attribute("aria-pressed", "true") page.get_by_role("button", name="Approve", exact=True).click() expect(page.get_by_role("alert")).to_contain_text(f"{FLAGGED - 1} flagged") assert stub.RUNS["job-t"]["approved"] is False page.get_by_role("button", name="Verify remaining", exact=True).click() page.get_by_role("button", name="Approve", exact=True).click() expect(page.get_by_text("✓ Approved", exact=True)).to_be_visible() expect(page.get_by_role("alert")).to_be_hidden() assert stub.RUNS["job-t"]["approved"] is True def test_page_x_of_y_updates_while_running(page, stub): stub.put_run("job-busy", "busy.pdf", FIELDS, status="running", pages_done=1, pages_total=4) open_review(page, "job-busy") expect(page.get_by_text("page 1 of 4", exact=True)).to_be_visible() stub.RUNS["job-busy"]["pages_done"] = 3 expect(page.get_by_text("page 3 of 4", exact=True)).to_be_visible() stub.RUNS["job-busy"].update(status="done", pages_done=4) expect(fields_list(page)).to_have_count(len(FIELDS)) expect(page.get_by_text(re.compile(r"^page \d+ of \d+$"))).to_be_hidden()