AI skill: SKILL.md, AI contract, scrubbed references, eval fixture (d)

- rpa-dashboard-ai/SKILL.md: trigger description and the shared question flow;
  points at the no-AI frontend.md, themes and palette instead of copying them.
- templates/contract.md: runs, review rows, endpoints and the server approve gate.
- Superbill extracts: patient names and the client name removed, every file
  carries an EXAMPLE ONLY banner. tests/test_reference_scan.py checks both skills
  for patient data, internal IPs, credentials and keys.
- tests/fixtures/d-ai-extraction and its eval entry, which runs page_check_ai
  on the built folder.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/README.md

@@ -22,6 +22,14 @@ uv run --with pytest --with playwright pytest tests
     `python dashboard/bot_runner.py` prints) instead of the built-in run.
   - To add a case, write a `test_*.py` file in `page_check/` that uses the `page`
     and `push(event, data)` fixtures from `conftest.py`.
+- `page_check_ai/`: **Seam 1, AI web page.** Builds the React starter with npm
+  (filled with test values, in a temp folder that keeps `node_modules`), serves it with
+  its own `server_stub.py`, and checks bands, band filters, drafts, the approve gate
+  and "page X of Y", plus every no-AI case. Tests set run and review state on the stub
+  directly (the `stub` fixture). To check a generated dashboard folder instead, set
+  `PAGE_CHECK_AI_DIR=path/to/dashboard`. Needs npm on PATH, else it skips.
+- `test_reference_scan.py`: no patient data, internal IPs, credentials or keys in any
+  bundled file, and every Superbill extract carries an `EXAMPLE ONLY` banner.
 - `evals/`: **Seam 2.** Runs the skills on a fixture project with `claude -p`,
   then grades the reply and runs the page check on the page it built:
 
@@ -33,6 +41,8 @@ uv run --with pytest --with playwright pytest tests
   `FIXTURES` in `run_eval.py`. An entry with a `"fixture"` key reuses another
   fixture folder with different answers: `c-keep` and `c-restyle` both run on
   `c-web-existing-frontend`, and also check what changed in the files it already had.
+  An `"ai": True` entry (`d-ai-extraction`) expects the AI skill's React starter and
+  runs `page_check_ai/` on the folder it built.
   For a bot fixture (`b-pyqt-bot`), the eval also dry-runs the generated
   `bot_runner.py`, saves its events to `evals/results/<fixture>.run.jsonl`, and
   replays them in the page check.

tests/evals/run_eval.py

@@ -69,6 +69,17 @@ FIXTURES = {
         "page_has": ["--accent:#ff3049"],
         "existing": "#ff3049",  # only token lines may change, and the theme now uses this colour
     },
+    # (d) web service with an LLM extraction call and no UI: the AI skill's web path.
+    "d-ai-extraction": {
+        "prompt": "Build me an RPA dashboard for this project. I can't answer questions while you "
+                  "work, so here are my answers to anything the skill asks. Colours: Superbill Ember. "
+                  "Desktop or web: web. Optional sections: none.",
+        "frontend": r"^\s*(none|no\b)",
+        "asks": [r"colou?r", r"desktop|web", r"section"],
+        "never_asks": [r"\bkeep\b|restyle"],
+        "page_has": ["--accent:#ff3049"],
+        "ai": True,
+    },
 }
 # A line a restyle may change: a custom property, a comment, a blank, or a web-font link.
 TOKEN_LINE = r"^\s*(--[\w-]+\s*:.*|/\*.*\*/|<link[^>]*fonts\.googleapis\.com.*|@import.*fonts\.googleapis\.com.*)?\s*$"
@@ -155,6 +166,11 @@ def main(fixture):
     results.mkdir(exist_ok=True)
     if pages:
         html = pages[0].read_text(encoding="utf-8")
+        if cfg.get("ai"):  # the React starter: its theme lives in src/index.css
+            css = pages[0].parent / "src" / "index.css"
+            html += css.read_text(encoding="utf-8") if css.exists() else ""
+            checks["AI path: review starter and server stub built"] = all(
+                (pages[0].parent / f).exists() for f in ["server_stub.py", "bands.py", "src/components/Review.tsx"])
         shutil.copy(pages[0], results / f"{fixture}.dashboard.html")
         checks["no placeholders left"] = "{{" not in html
         if cfg.get("brand"):
@@ -168,7 +184,10 @@ def main(fixture):
         env = {**os.environ, "PAGE_CHECK_HTML": str(pages[0])}
         if cfg.get("bot"):
             env.update(bot_checks(project, html, checks, results / f"{fixture}.run.jsonl"))
-        pc = subprocess.run([sys.executable, "-m", "pytest", str(ROOT / "tests" / "page_check"), "-q"],
+        if cfg.get("ai"):
+            env["PAGE_CHECK_AI_DIR"] = str(pages[0].parent)
+        check_dir = ROOT / "tests" / ("page_check_ai" if cfg.get("ai") else "page_check")
+        pc = subprocess.run([sys.executable, "-m", "pytest", str(check_dir), "-q"],
                             env=env, capture_output=True, text=True)
         checks["page check passes"] = pc.returncode == 0
         page_check = pc.stdout.strip().splitlines()[-1:] if pc.stdout else [pc.stderr[-500:]]

tests/fixtures/d-ai-extraction/README.md

@@ -0,0 +1,8 @@
+# Invoice extraction service
+
+A small web service. It sends a scanned invoice to an LLM and gets back the
+invoice fields, each with a confidence score per word. There is no UI yet.
+
+```
+uvicorn app:app --port 8000
+```

tests/fixtures/d-ai-extraction/app.py

@@ -0,0 +1,11 @@
+"""Upload a scanned invoice, get its fields back."""
+from fastapi import FastAPI, UploadFile
+
+from extractor import extract_invoice
+
+app = FastAPI()
+
+
+@app.post("/extract")
+async def extract(file: UploadFile):
+    return extract_invoice(await file.read(), file.content_type or "application/pdf")

tests/fixtures/d-ai-extraction/extractor.py

@@ -0,0 +1,34 @@
+"""The LLM extraction call: one scanned invoice in, its fields out, each with word scores."""
+import json
+
+from google import genai
+from google.genai import types
+
+MODEL = "gemini-2.5-flash"
+FIELDS = {  # field name -> mandatory
+    "Invoice number": True,
+    "Invoice date": True,
+    "Vendor": True,
+    "PO number": False,
+    "Due date": True,
+    "Total": True,
+}
+PROMPT = (
+    "Read this invoice. Return JSON: a list of {field, value, word_scores}, one per field in "
+    + ", ".join(FIELDS)
+    + ". word_scores is your 0-1 legibility score for each word of the value. Use an empty "
+    "value and no scores when a field isn't on the page."
+)
+
+
+def extract_invoice(document: bytes, mime_type: str) -> list[dict]:
+    client = genai.Client()  # credentials come from the environment
+    response = client.models.generate_content(
+        model=MODEL,
+        contents=[types.Part.from_bytes(data=document, mime_type=mime_type), PROMPT],
+        config=types.GenerateContentConfig(response_mime_type="application/json"),
+    )
+    rows = json.loads(response.text)
+    for row in rows:
+        row["mandatory"] = FIELDS.get(row["field"], False)
+    return rows

tests/fixtures/d-ai-extraction/requirements.txt

@@ -0,0 +1,4 @@
+fastapi
+uvicorn
+google-genai
+python-multipart

tests/page_check_ai/conftest.py

@@ -47,6 +47,13 @@ MODES = {"light": (1280, 820, False), "dark": (1280, 820, True), "narrow": (800,
 SKIP_COPY = shutil.ignore_patterns("node_modules", "dist", "__pycache__")
 
 
+@pytest.fixture
+def sample_data():
+    """Shared no-AI cases that need the page's own sample data skip on a bot-wired page."""
+    if os.environ.get("PAGE_CHECK_WIRED") == "1":
+        pytest.skip("page is wired to a bot, so it has no sample data")
+
+
 @pytest.fixture(scope="session")
 def site():
     """The dashboard folder under test: server_stub.py, index.html and the sources."""

tests/test_reference_scan.py

@@ -0,0 +1,45 @@
+"""The skills are shared outside the source projects: no patient data, internal IPs,
+credentials or keys in any bundled file, and the Superbill extracts are marked as examples."""
+import re
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+SKILLS = [ROOT / "rpa-dashboard-ai", ROOT / "rpa-dashboard-no-ai"]
+AI_REFERENCE = ROOT / "rpa-dashboard-ai" / "templates" / "reference"
+FORBIDDEN = {
+    "internal IP": r"\b(10\.\d{1,3}|192\.168|172\.(1[6-9]|2\d|3[01]))\.\d{1,3}\.\d{1,3}\b",
+    "Google API key": r"AIza[0-9A-Za-z_-]{30,}",
+    "OpenAI/Anthropic key": r"\bsk-[A-Za-z0-9_-]{20,}",
+    "AWS key": r"\bAKIA[0-9A-Z]{16}\b",
+    "private key": r"-----BEGIN [A-Z ]*PRIVATE KEY-----",
+    "hard-coded secret": r"""(?i)\b(password|passwd|api_?key|secret|client_secret)\b\s*[:=]\s*['"][^'"{}\s]{6,}['"]""",
+    "SSN": r"\b\d{3}-\d{2}-\d{4}\b",
+    "quoted roster name ('SURNAME, FIRST')": r"""["'][A-Z]{2,}, [A-Z]""",
+    "client name": r"Gulf Coast",
+}
+MARKER = "EXAMPLE ONLY"
+
+
+def text_files():
+    for skill in SKILLS:
+        for f in skill.rglob("*"):
+            if f.is_file() and "node_modules" not in f.parts and f.name != "package-lock.json":
+                yield f
+
+
+@pytest.mark.parametrize("what", FORBIDDEN)
+def test_no_sensitive_data_in_bundled_files(what):
+    hits = [f"{f.relative_to(ROOT)}:{n}" for f in text_files()
+            for n, line in enumerate(f.read_text(encoding="utf-8", errors="ignore").splitlines(), 1)
+            if re.search(FORBIDDEN[what], line)]
+    assert not hits, f"{what} found: {hits}"
+
+
+def test_superbill_extracts_are_marked_as_examples():
+    files = [f for f in AI_REFERENCE.rglob("*") if f.is_file()]
+    assert files
+    unmarked = [str(f.relative_to(ROOT)) for f in files
+                if MARKER not in "".join(f.read_text(encoding="utf-8").splitlines(keepends=True)[:5])]
+    assert not unmarked, f"no '{MARKER}' banner in the first lines: {unmarked}"

tests/test_templates.py

@@ -1,4 +1,5 @@
 """Python templates must stay valid Python before their placeholders are filled."""
+import re
 import shutil
 import subprocess
 from pathlib import Path
@@ -23,3 +24,13 @@ def test_tebra_sky_preset_is_the_template_default():
     themes = (templates / "themes.md").read_text(encoding="utf-8")
     sky = themes.split("## 1. Tebra Sky")[1].split("```css\n")[1].split("```")[0]
     assert sky in (templates / "dashboard.html").read_text(encoding="utf-8")
+
+
+def test_ai_skill_references_files_that_exist():
+    skill = ROOT / "rpa-dashboard-ai"
+    text = (skill / "SKILL.md").read_text(encoding="utf-8")
+    paths = re.findall(r"`((?:templates/|<no-ai>)[\w./-]+)`", text)
+    assert len(paths) > 5
+    missing = [p for p in paths
+               if not (skill / p.replace("<no-ai>", "../rpa-dashboard-no-ai/templates/")).exists()]
+    assert not missing