From 58baf7f778dc2576279ac719923028737efad4eb Mon Sep 17 00:00:00 2001 From: Vasanth Date: Fri, 16 Oct 2026 14:09:50 +0530 Subject: [PATCH] Eval runner: send the prompt on stdin; add a-empty results On Windows a multi-line argv prompt is cut at its first newline, which also dropped the permission flags. The a-empty eval now passes every check, including the page check on the page the skill built. Co-Authored-By: Claude Opus 5.5 --- tests/evals/results/a-empty.dashboard.html | 308 +++++++++++++++++++++ tests/evals/results/a-empty.json | 24 ++ tests/evals/run_eval.py | 6 +- 3 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 tests/evals/results/a-empty.dashboard.html create mode 100644 tests/evals/results/a-empty.json diff --git a/tests/evals/results/a-empty.dashboard.html b/tests/evals/results/a-empty.dashboard.html new file mode 100644 index 0000000..ee8cecf --- /dev/null +++ b/tests/evals/results/a-empty.dashboard.html @@ -0,0 +1,308 @@ + + + + + + +RPA Dashboard + + + + +
+ + +
+
+
Dashboard
Overview of every batch
+
+ + +
System ready
+
+
+ +
+ +
+
+
0
Total batches
+
0
Completed
+
0
In progress
+
0
Not completed
+
+
+
Batch status split
+
    +
    +
    Recent batches
    No batches yet
    +
    +
    + + + + + + +
    +
    +
    + + + + diff --git a/tests/evals/results/a-empty.json b/tests/evals/results/a-empty.json new file mode 100644 index 0000000..a1f90b8 --- /dev/null +++ b/tests/evals/results/a-empty.json @@ -0,0 +1,24 @@ +{ + "fixture": "a-empty", + "passed": true, + "checks": { + "frontend reported correctly": true, + "asks /colou?r/": true, + "asks /desktop|web/": true, + "asks /section/": true, + "never asks /\\bkeep\\b|restyle/": true, + "exactly one page built": true, + "no placeholders left": true, + "custom palette applied": true, + "page check passes": true + }, + "page_check": [ + "22 passed in 17.68s" + ], + "files": [ + "README.md", + "dashboard\\bridge.py", + "dashboard\\dashboard.html" + ], + "reply": "Dashboard ready ah iruku. Sample data la run aagudhu, headless browser la check panniten.\n\n| Item | Details |\n|---|---|\n| Files | `dashboard/dashboard.html`, `dashboard/bridge.py` (PyQt6 desktop host) |\n| Theme | Custom `#7a3cff`. Contrast check 10/10 PASS, colour-ah change panna thevai illa. Dark mode la button colour konjam light ah `#8248ff` nu irukum. Background innum template oda sky-blue gradient dhaan, purple accents mattum dhaan maarinadhu. |\n| Sections | Core mattum: sidebar, top bar, dark toggle, status dot, KPI cards, donut, run panel, report table. Optional edhuvum add pannala. |\n| Data | `SAMPLE_DATA = true`. Project la bot illa. |\n\n**Check results (Playwright, browser la open panni):**\n- KPI: 6 total / 3 completed / 1 in progress / 1 not completed. Donut legend la 4 rows.\n- Run: 4/4 steps done, 5 log lines, status \"Completed\".\n- Report: 4/4 rows la status pill varudhu.\n- Light, dark, 800px width moonum sariya varudhu. Script errors 0, `{{` placeholders 0.\n- **UNVERIFIED:** `bridge.py` la irukura PyQt window-ah naan open pannala. Adhu real ah run aagudhanu check pannala.\n\n**Bot wire pannumbodhu:**\n- Bot indha events anuppanum: `status`, `step`, `log`, `dashboard_data`, `report_data`.\n- `bridge.py` la `BOT_CMD` (ippo `bot` nu placeholder), `STEP_MARKERS`, `FRIENDLY_RULES` set pannunga.\n- Apram `SAMPLE_DATA = false` panni, SAMPLE block-ah delete pannunga.\n\n**Also noticed:** steps (`Initializing \u2192 Login \u2192 Process items \u2192 Completed`) and report columns (Item/Batch/Status) placeholder dhaan, ungaloda real bot stages vandha maathanum.\n\nFRONTEND: none\nQUESTIONS: Which colours: Tebra Sky, Superbill Ember, or Custom (one brand hex)? | Desktop or web? | Which optional sections: checkpoint tracker, admin/roles screen, login page, or none?" +} \ No newline at end of file diff --git a/tests/evals/run_eval.py b/tests/evals/run_eval.py index 3f34f8f..5d9b532 100644 --- a/tests/evals/run_eval.py +++ b/tests/evals/run_eval.py @@ -47,11 +47,13 @@ def main(fixture): for skill in ROOT.glob("rpa-dashboard-*"): shutil.copytree(skill, project / ".claude" / "skills" / skill.name) + # The prompt goes in on stdin: on Windows a multi-line argv prompt is cut at the first newline. run = subprocess.run( - [shutil.which("claude") or "claude", "-p", cfg["prompt"] + REPLY_FORMAT, "--output-format", "json", + [shutil.which("claude") or "claude", "-p", "--output-format", "json", "--permission-mode", "acceptEdits", "--allowedTools", "Skill", "Read", "Write", "Edit", "Glob", "Grep", "Bash(python *)", "PowerShell(python *)"], - cwd=project, capture_output=True, text=True, encoding="utf-8", timeout=1800) + input=cfg["prompt"] + REPLY_FORMAT, cwd=project, capture_output=True, text=True, encoding="utf-8", + timeout=1800) reply = json.loads(run.stdout).get("result", "") if run.stdout.strip().startswith("{") else run.stdout def line(key):