From 9dcdd8f572c2bd534a57f76afcaac004b63128da Mon Sep 17 00:00:00 2001 From: Vasanth Date: Fri, 16 Oct 2026 15:23:52 +0530 Subject: [PATCH] AI cost/ROI page: pricing table, cost per call, Cost tab, ROI endpoint - templates/web/pricing.py: pricing table with source and as-of date; cost_for_call (input rate + (output + thinking) at output rate, long-context above threshold; unknown model raises naming the known models). - Cost tab: coverage pill, total/this-month/tokens tiles, month-by-month table, per-run tokens and cost ("not priced", never $0), by-hand panel with editable assumptions labelled as such. - Contract: RoiRun and GET /roi; optional GET /models and POST /runs/{id}/retry. - SKILL.md: cost page wiring, model picker and retry-on-other-model as optional sections. - Tests: test_pricing.py self-check, page_check_ai/test_ai_cost.py. Co-Authored-By: Claude Opus 5.5 --- rpa-dashboard-ai/SKILL.md | 31 +++- rpa-dashboard-ai/templates/contract.md | 19 +++ rpa-dashboard-ai/templates/web/pricing.py | 32 +++++ rpa-dashboard-ai/templates/web/server_stub.py | 25 ++++ rpa-dashboard-ai/templates/web/src/App.tsx | 3 + rpa-dashboard-ai/templates/web/src/ai.ts | 14 ++ .../templates/web/src/components/Cost.tsx | 133 ++++++++++++++++++ .../templates/web/src/components/Review.tsx | 1 + tests/README.md | 5 +- tests/page_check_ai/test_ai_cost.py | 85 +++++++++++ tests/test_pricing.py | 50 +++++++ 11 files changed, 392 insertions(+), 6 deletions(-) create mode 100644 rpa-dashboard-ai/templates/web/pricing.py create mode 100644 rpa-dashboard-ai/templates/web/src/components/Cost.tsx create mode 100644 tests/page_check_ai/test_ai_cost.py create mode 100644 tests/test_pricing.py diff --git a/rpa-dashboard-ai/SKILL.md b/rpa-dashboard-ai/SKILL.md index 1de3601..dfec692 100644 --- a/rpa-dashboard-ai/SKILL.md +++ b/rpa-dashboard-ai/SKILL.md @@ -1,6 +1,6 @@ --- name: rpa-dashboard-ai -description: Builds a working dashboard for an RPA bot or service that uses AI (an LLM or OCR extraction call). It has everything from rpa-dashboard-no-ai (sidebar, top bar with dark toggle and status dot, KPI cards, batch donut, run panel with steps and live log, report table with search and export) plus confidence bands, a review split pane (source document beside the extracted fields, band filters, draft edits, Verify) and an approve gate the server enforces. Web path is React + Vite + Tailwind with a sample-data server stub. Use when the user asks for an "AI RPA dashboard", an "extraction dashboard", an "LLM/OCR review dashboard", or any RPA/bot dashboard in a project that calls an LLM or OCR to extract fields. Needs rpa-dashboard-no-ai installed next to it. +description: Builds a working dashboard for an RPA bot or service that uses AI (an LLM or OCR extraction call). It has everything from rpa-dashboard-no-ai (sidebar, top bar with dark toggle and status dot, KPI cards, batch donut, run panel with steps and live log, report table with search and export) plus confidence bands, a review split pane (source document beside the extracted fields, band filters, draft edits, Verify), an approve gate the server enforces, and a cost/ROI page (tokens and cost per run, total and monthly spend, priced coverage, an against-doing-it-by-hand panel). Web path is React + Vite + Tailwind with a sample-data server stub. Use when the user asks for an "AI RPA dashboard", an "extraction dashboard", an "LLM/OCR review dashboard", or any RPA/bot dashboard in a project that calls an LLM or OCR to extract fields. Needs rpa-dashboard-no-ai installed next to it. --- # RPA dashboard (AI) @@ -11,7 +11,8 @@ means `../rpa-dashboard-no-ai/templates/`. | File | What it is | |---|---| -| `templates/web/` | The React + Vite + Tailwind starter: shell, KPI, donut, run panel, report, **band legend, review split pane, approve gate**, sample data. | +| `templates/web/` | The React + Vite + Tailwind starter: shell, KPI, donut, run panel, report, **band legend, review split pane, approve gate, cost page**, sample data. | +| `templates/web/pricing.py` | The pricing table (with its source and as-of date) and `cost_for_call`. The backend imports it. | | `templates/web/server_stub.py` | Sample-data backend for runs, review rows and the approve gate. Standard library only. | | `templates/web/bands.py` | The band cut points and rules. The one source of truth: the backend imports it. | | `templates/contract.md` | The AI contract: runs, review rows, endpoints and the gate. Extends `contract.md`. | @@ -31,7 +32,8 @@ Before asking anything: `frontend.md` section 1. - **AI call:** find the LLM or OCR extraction (for example `genai`, `openai`, `anthropic`, Textract or Tesseract calls): which fields it returns, which are - mandatory, and whether it returns per-word scores. + mandatory, and whether it returns per-word scores. Note the provider, the model + names, and whether the response's token usage (input, output, thinking) is kept. - **Bot or backend:** the entry point, what it prints, and any web framework (FastAPI, Flask, Django). @@ -63,9 +65,17 @@ uses PyQt, choose desktop and say so. ## 5. Ask which optional sections to build Always built: the no-AI shell and sections, plus the band legend, the review split -pane and the approve gate. Ask which extras to add (choose any, or none): -**checkpoint tracker**, **admin/roles screen**, **login page**. Build only the +pane, the approve gate and the cost page. Ask which extras to add (choose any, or none): +**checkpoint tracker**, **admin/roles screen**, **login page**, **model picker** (choose +the model per batch), **retry on another model** (for failed runs). Build only the chosen ones at their `OPTIONAL:` markers, and delete the markers. +- **Model picker:** a `` (from `GET /models`, defaulting to the run's own model) and a Retry button + that sends `POST /runs/{job_id}/retry {model}`, then polls the run as usual. +Both need the endpoints in `templates/contract.md`; add them to the backend (and the stub). ## 6. Build (web) @@ -103,6 +113,14 @@ review rows and the gate. mandatory)`. The approve endpoint must refuse (409 with `detail`) while any flagged field is unverified: the page never decides that. For the shell, set `SAMPLE_DATA = false` and serve the no-AI endpoints too. +- **Cost page:** copy `pricing.py` next to the backend. Fill `PRICING` with the rates of + the models the project really calls, from the provider's pricing page, and set + `PRICING_SOURCE` and `PRICING_AS_OF`. If you can't read the rates, ask the user for + them. The backend stores each run's token usage and serves `GET /roi`, pricing each run + with `cost_for_call`. A run with no recorded usage gets `null`, never 0. If the + extraction doesn't keep token usage yet, wire it to (for example Gemini's + `usage_metadata`, OpenAI's and Anthropic's `usage`). In sample mode, replace the stub's + `ROI_RUNS` models with the project's models. - Never invent fields, model names or prices the project doesn't have. ## 8. Verify @@ -119,6 +137,9 @@ Open the page and check, in light, dark (the toggle) and a window 820px wide or - on Review: the band legend counts and the caveat, band filters, a typed edit marked unsaved until Save, a refused Approve while flagged fields are unverified, and "page X of Y" on a running document; +- on Cost: the coverage pill, total and this month's spend, the month table, a run + marked "not priced", the pricing source and as-of date, and the by-hand figure + changing when an assumption is edited; - no `{{` is left and the console shows no script errors. If you can't run npm here, say so and give the user these commands. Finish by diff --git a/rpa-dashboard-ai/templates/contract.md b/rpa-dashboard-ai/templates/contract.md index 2674ca1..6fa6af3 100644 --- a/rpa-dashboard-ai/templates/contract.md +++ b/rpa-dashboard-ai/templates/contract.md @@ -13,11 +13,16 @@ Run { job_id, document, status: pending|queued|running|done|failed, model, approved, error } ReviewRow { row_id, field, value, band: green|amber|red|blank|unscored, score, reason, verified, corrected_value } +RoiRun { job_id, document, created_at, model, input_tokens, output_tokens, + thinking_tokens, total_cost } ``` - `summary` counts the run's rows per band. It is `null` until the run is `done`. - `score` is the field's worst word score, or `null` when it has none. - `corrected_value` is `null` until a reviewer saves a value different from `value`. +- `created_at` is ISO 8601 in UTC. Months on the cost page are UTC months. +- `RoiRun` token counts and `total_cost` (USD) are `null` when the run recorded no token + usage. The page shows it as "not priced" and counts it in the coverage pill, never as $0. ## Endpoints (web: `{API_BASE}` + path, JSON in and out) @@ -29,8 +34,16 @@ ReviewRow { row_id, field, value, band: green|amber|red|blank|unscored, | `GET /runs/{job_id}/document` | The source document (PDF or image) shown beside the fields. | | `PATCH /review/{row_id}` | Body `{corrected_value?, verified}`. Patches one row, returns it. | | `POST /runs/{job_id}/verify-remaining` | Verifies every flagged row not yet verified. Returns `{verified: n}`. | +| `GET /roi` | **Fetch ROI.** `{runs: [RoiRun], pricing: {source, as_of}}`, every run, newest first. The page sums spend, groups months and computes the by-hand panel itself. | | `POST /runs/{job_id}/approve` | **The approve gate.** `409 {detail}` while the run isn't done or any flagged row is unverified, else `200 {ok: true}`. | +Optional, only when the user chose those sections: + +| Method and path | What it does | +|---|---| +| `GET /models` | `{models: [name]}`: the keys of the pricing table, so only priced models can be picked. | +| `POST /runs/{job_id}/retry` | Body `{model}`. Re-queues a `failed` run on that model, returns the `Run`. | + Errors answer with a non-2xx status and `{detail: ""}`. The page shows `detail`. ## Rules @@ -45,6 +58,12 @@ Errors answer with a non-2xx status and `{detail: ""}`. The page shows `det edited row (with `verified: true`). Leaving with unsaved edits asks first. - **Polling:** while a run is `queued` or `running`, the page polls `GET /runs/{job_id}` every 1.5s and shows "page X of Y". It stops when the run settles. +- **Cost** comes from `templates/web/pricing.py` and nowhere else: per million tokens, + input × input rate + (output + thinking) × output rate, long-context rates above the + model's threshold. An unknown model is an error naming the known models, never $0. + The table records its source URL and as-of date; `GET /roi` returns both. +- **Assumptions** (minutes per document, hourly rate) live only on the page, typed by the + reader and labelled as assumptions. The backend never stores or guesses them. - **Caveat:** the legend always shows that colour reflects print clarity, not correctness. diff --git a/rpa-dashboard-ai/templates/web/pricing.py b/rpa-dashboard-ai/templates/web/pricing.py new file mode 100644 index 0000000..320741f --- /dev/null +++ b/rpa-dashboard-ai/templates/web/pricing.py @@ -0,0 +1,32 @@ +"""Pricing table and cost per call for the cost page. The backend imports it. + +Rates are USD per million tokens. Thinking tokens bill at the output rate. A model +with "long_threshold" switches both rates to its *_long rates when a call's input +tokens go above that threshold. + +Replace the table with the rates of the provider and models the project really uses, +copied from the provider's pricing page, and update PRICING_SOURCE and PRICING_AS_OF. +Never guess a rate: a model missing here is an error, never a cost of zero. +""" + +PRICING_SOURCE = "https://ai.google.dev/gemini-api/docs/pricing" +PRICING_AS_OF = "2026-08-11" + +PRICING = { + "gemini-2.5-pro": {"input": 1.25, "output": 10.00, + "long_threshold": 200_000, "input_long": 2.50, "output_long": 15.00}, + "gemini-2.5-flash": {"input": 0.30, "output": 2.50}, + "gemini-2.5-flash-lite": {"input": 0.10, "output": 0.40}, +} + + +def cost_for_call(model, input_tokens, output_tokens, thinking_tokens=0): + """USD for one call. Raises ValueError, naming the known models, for a model not in PRICING.""" + if model not in PRICING: + raise ValueError(f"No pricing for model '{model}'. Add it to PRICING from {PRICING_SOURCE}. " + f"Known models: {', '.join(PRICING)}") + p = PRICING[model] + in_rate, out_rate = p["input"], p["output"] + if "long_threshold" in p and input_tokens > p["long_threshold"]: + in_rate, out_rate = p["input_long"], p["output_long"] + return (input_tokens * in_rate + (output_tokens + thinking_tokens) * out_rate) / 1_000_000 diff --git a/rpa-dashboard-ai/templates/web/server_stub.py b/rpa-dashboard-ai/templates/web/server_stub.py index b1775e1..32b6191 100644 --- a/rpa-dashboard-ai/templates/web/server_stub.py +++ b/rpa-dashboard-ai/templates/web/server_stub.py @@ -18,6 +18,7 @@ from pathlib import Path from urllib.parse import urlparse from bands import BANDS, field_band, needs_review +from pricing import PRICING_AS_OF, PRICING_SOURCE, cost_for_call HERE = Path(__file__).resolve().parent API = "/api" # same as API_BASE in index.html @@ -34,6 +35,7 @@ SAMPLE_FIELDS = [ ("Tax", "196.00", [], False), ] RUNS, ROWS = {}, {} +ROI_RUNS = [] # [{job_id, document, created_at, model, input_tokens, output_tokens, thinking_tokens}] _next_row = [1] @@ -56,6 +58,16 @@ def reset(): ROWS.clear() put_run("job-1002", "invoice-1002.pdf", SAMPLE_FIELDS, status="running", pages_done=1, pages_total=3) put_run("job-1001", "invoice-1001.pdf", SAMPLE_FIELDS, pages_total=2, pages_done=2) + ROI_RUNS.clear() + now = time.time() + for i, (model, tokens) in enumerate([("gemini-2.5-flash", (21_000, 1_800, 900)), + ("gemini-2.5-flash", (18_500, 1_600, 0)), + ("gemini-2.5-pro", (240_000, 3_000, 2_500)), + ("gemini-2.5-flash-lite", (22_000, 1_700, 0)), + ("gemini-2.5-flash", (None, None, None))]): # usage not recorded + ROI_RUNS.append({"job_id": f"job-{900 + i}", "document": f"invoice-{900 + i}.pdf", + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now - i * 12 * 864e2)), + "model": model, **dict(zip(("input_tokens", "output_tokens", "thinking_tokens"), tokens))}) def run_view(job_id): @@ -104,6 +116,18 @@ def approve(job_id, body): return 200, {"ok": True} +def roi(body): + """Cost per run. total_cost is null when the run recorded no token usage: unpriced, never zero. + An unknown model is an error (500 naming the known models), never a cost of zero.""" + try: + runs = [{**r, "total_cost": None if r["input_tokens"] is None else + cost_for_call(r["model"], r["input_tokens"], r["output_tokens"], r["thinking_tokens"] or 0)} + for r in ROI_RUNS] + except ValueError as e: + return 500, {"detail": str(e)} + return 200, {"runs": runs, "pricing": {"source": PRICING_SOURCE, "as_of": PRICING_AS_OF}} + + def document(job_id, body): """A stand-in for the source document. A real backend returns the PDF or page image.""" lines = "".join(f'{escape(f)}: {escape(v)}' @@ -122,6 +146,7 @@ ROUTES = [ ("PATCH", r"/review/(\d+)", patch_row), ("POST", r"/runs/([\w-]+)/verify-remaining", verify_remaining), ("POST", r"/runs/([\w-]+)/approve", approve), + ("GET", r"/roi", roi), ("POST", r"/set-dark", lambda body: (200, {})), ] diff --git a/rpa-dashboard-ai/templates/web/src/App.tsx b/rpa-dashboard-ai/templates/web/src/App.tsx index d6c8134..64f4946 100644 --- a/rpa-dashboard-ai/templates/web/src/App.tsx +++ b/rpa-dashboard-ai/templates/web/src/App.tsx @@ -4,12 +4,14 @@ import { Overview } from './components/Overview' import { RunPanel } from './components/RunPanel' import { Report } from './components/Report' import { Review } from './components/Review' +import { Cost } from './components/Cost' const TABS = { dashboard: ['Dashboard', 'Overview of every batch', <>], run: ['Run', 'Live progress and log', ], review: ['Review', 'Check extracted fields against the document', <>], report: ['Report', 'Result of every processed item', <>], + cost: ['Cost', 'Tokens, spend and the case against doing it by hand', <>], // OPTIONAL: admin/roles tab goes here } as const type Tab = keyof typeof TABS @@ -98,6 +100,7 @@ export function App() { {tab === 'run' && } {tab === 'review' && } {tab === 'report' && } + {tab === 'cost' && } diff --git a/rpa-dashboard-ai/templates/web/src/ai.ts b/rpa-dashboard-ai/templates/web/src/ai.ts index 1f29c4b..cfbdd9f 100644 --- a/rpa-dashboard-ai/templates/web/src/ai.ts +++ b/rpa-dashboard-ai/templates/web/src/ai.ts @@ -23,6 +23,19 @@ export type ReviewRow = { corrected_value: string | null } +/** One run on the cost page. total_cost is null when the run recorded no token usage (unpriced). */ +export type RoiRun = { + job_id: string + document: string + created_at: string // ISO, UTC + model: string + input_tokens: number | null + output_tokens: number | null + thinking_tokens: number | null + total_cost: number | null // USD +} +export type Pricing = { source: string; as_of: string } + export const BUSY: RunStatus[] = ['queued', 'running'] // polled every 1.5s while in one of these /** Legend order, label, and the colour token. Band colours are semantic: never on buttons or headings. */ @@ -53,5 +66,6 @@ export const ai = { req(`/review/${rowId}`, { method: 'PATCH', body: JSON.stringify(body) }), verifyRemaining: (id: string) => req<{ verified: number }>(`/runs/${id}/verify-remaining`, { method: 'POST' }), approve: (id: string) => req<{ ok: boolean }>(`/runs/${id}/approve`, { method: 'POST' }), + roi: () => req<{ runs: RoiRun[]; pricing: Pricing }>('/roi'), documentUrl: (id: string) => `${API_BASE}/runs/${id}/document`, } diff --git a/rpa-dashboard-ai/templates/web/src/components/Cost.tsx b/rpa-dashboard-ai/templates/web/src/components/Cost.tsx new file mode 100644 index 0000000..036bcb2 --- /dev/null +++ b/rpa-dashboard-ai/templates/web/src/components/Cost.tsx @@ -0,0 +1,133 @@ +import { useEffect, useState } from 'react' +import { ai, type Pricing, type RoiRun } from '../ai' + +const money = (n: number) => `$${n.toFixed(n > 0 && n < 0.01 ? 4 : 2)}` +const num = (n: number | null) => (n == null ? '—' : n.toLocaleString('en-US')) +const monthOf = (iso: string) => iso.slice(0, 7) // 'YYYY-MM', UTC +const monthLabel = (ym: string) => + new Date(`${ym}-01T00:00:00Z`).toLocaleString('en-US', { month: 'short', year: 'numeric', timeZone: 'UTC' }) + +/** + * The cost page. Two kinds of number, kept apart: + * measured: tokens and cost per run, priced by the backend's pricing table; + * assumed: minutes per document and an hourly rate, typed by the reader and labelled so. + * A run with no recorded token usage is "not priced", never $0, and the coverage pill says how many. + */ +export function Cost() { + const [runs, setRuns] = useState([]) + const [pricing, setPricing] = useState() + const [error, setError] = useState() + const [minutes, setMinutes] = useState(5) // assumption: minutes to key one document by hand + const [rate, setRate] = useState(20) // assumption: USD per hour of that person's time + + useEffect(() => { + ai.roi().then((d) => { setRuns(d.runs); setPricing(d.pricing) }).catch((e) => setError(e.message)) + }, []) + + const priced = runs.filter((r) => r.total_cost != null) + const sum = (rs: RoiRun[]) => rs.reduce((n, r) => n + (r.total_cost ?? 0), 0) + const thisMonth = new Date().toISOString().slice(0, 7) + const months = [...new Set(runs.map((r) => monthOf(r.created_at)))].sort().reverse() + .map((m) => { + const rs = runs.filter((r) => monthOf(r.created_at) === m) + return { m, docs: rs.length, priced: rs.filter((r) => r.total_cost != null).length, cost: sum(rs) } + }) + const tokens = (k: 'input_tokens' | 'output_tokens' | 'thinking_tokens') => priced.reduce((n, r) => n + (r[k] ?? 0), 0) + const byHand = (priced.length * minutes / 60) * rate + + if (error) return
Can't load costs from {API_BASE}/roi: {error}
+ + return ( + <> +
+ + {priced.length} of {runs.length} documents priced + + {pricing && ( + + Rates from {pricing.source}, as of {pricing.as_of} + + )} +
+ +
+ + monthOf(r.created_at) === thisMonth)))} note={monthLabel(thisMonth)} /> + + +
+ +
+
Month by month
+
+ + + + {months.map((x) => ( + + + + + + ))} + +
MonthDocuments pricedSpend
{monthLabel(x.m)}{x.m === thisMonth && this month}{x.priced} of {x.docs}{x.priced ? money(x.cost) : '—'}
+
+
+ +
+
Against doing it by hand
+

These are your assumptions, not a measurement. Change them to match the team.

+
+ + +
+

+ By hand, {priced.length} documents would take {num(Math.round(priced.length * minutes / 6) / 10)} h, about {money(byHand)}. + The AI spent {money(sum(priced))} on them. +

+
+ + {/* OPTIONAL: model picker (per batch) goes here */} + +
+
Cost per run
+
+ + + + + + {runs.map((r) => ( + + + + + + ))} + +
DocumentDateModelInputOutputThinkingCost
{r.document}{r.created_at.slice(0, 10)}{r.model}{num(r.input_tokens)}{num(r.output_tokens)}{num(r.thinking_tokens)}{r.total_cost == null ? not priced : money(r.total_cost)}
+
+
+ + ) +} + +function Tile({ name, value, note }: { name: string; value: string; note: string }) { + return ( +
+
{name}
+
{value}
+
{note}
+
+ ) +} +const Th = ({ children, right }: { children: React.ReactNode; right?: boolean }) => + {children} +const Td = ({ children, right }: { children: React.ReactNode; right?: boolean }) => + {children} diff --git a/rpa-dashboard-ai/templates/web/src/components/Review.tsx b/rpa-dashboard-ai/templates/web/src/components/Review.tsx index 022ab1c..c1cbd53 100644 --- a/rpa-dashboard-ai/templates/web/src/components/Review.tsx +++ b/rpa-dashboard-ai/templates/web/src/components/Review.tsx @@ -105,6 +105,7 @@ export function Review({ onUnsaved }: { onUnsaved: (n: number) => void }) { {run && {run.status}} {run && BUSY.includes(run.status) && page {run.pages_done} of {run.pages_total || '?'}} {run?.error && {run.error}} + {/* OPTIONAL: retry on another model (failed runs) goes here */} {run?.status === 'done' && (
{left} to verify diff --git a/tests/README.md b/tests/README.md index 68aaf0d..2ebe9c5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -25,9 +25,12 @@ uv run --with pytest --with playwright pytest tests - `page_check_ai/`: **Seam 1, AI web page.** Builds the React starter with npm (filled with test values, in a temp folder that keeps `node_modules`), serves it with its own `server_stub.py`, and checks bands, band filters, drafts, the approve gate - and "page X of Y", plus every no-AI case. Tests set run and review state on the stub + "page X of Y" and the cost page (spend tiles, coverage pill, month breakdown, + by-hand assumptions), plus every no-AI case. Tests set run and review state on the stub directly (the `stub` fixture). To check a generated dashboard folder instead, set `PAGE_CHECK_AI_DIR=path/to/dashboard`. Needs npm on PATH, else it skips. +- `test_pricing.py`: the AI cost function (normal, long-context, thinking tokens, + unknown model). - `test_reference_scan.py`: no patient data, internal IPs, credentials or keys in any bundled file, and every Superbill extract carries an `EXAMPLE ONLY` banner. - `evals/`: **Seam 2.** Runs the skills on a fixture project with `claude -p`, diff --git a/tests/page_check_ai/test_ai_cost.py b/tests/page_check_ai/test_ai_cost.py new file mode 100644 index 0000000..f7021bd --- /dev/null +++ b/tests/page_check_ai/test_ai_cost.py @@ -0,0 +1,85 @@ +"""AI cost page: spend tiles, coverage pill, month breakdown and the by-hand assumptions. + +ROI runs are set on the server stub (stub.ROI_RUNS); the stub prices them with pricing.py. +The test pins its own pricing table, so it holds on a generated dashboard whose table +lists only the project's models. See rpa-dashboard-ai/templates/contract.md. +""" +import sys +from datetime import datetime, timedelta, timezone + +import pytest +from playwright.sync_api import expect + +NOW = datetime.now(timezone.utc) +LAST = NOW.replace(day=1) - timedelta(days=1) # a day in last month + + +def roi_run(job_id, when, model, inp=None, out=None, thinking=0): + return {"job_id": job_id, "document": f"{job_id}.pdf", "created_at": when.strftime("%Y-%m-%dT%H:%M:%SZ"), + "model": model, "input_tokens": inp, "output_tokens": out, "thinking_tokens": thinking} + + +TABLE = { + "test-flash": {"input": 0.30, "output": 2.50}, + "test-pro": {"input": 1.25, "output": 10.00, "long_threshold": 200_000, "input_long": 2.50, "output_long": 15.00}, + "test-lite": {"input": 0.10, "output": 0.40}, +} +AS_OF = "2030-01-31" +# Costs worked by hand from TABLE: +RUNS = [ + roi_run("r1", NOW, "test-flash", 1_000_000, 200_000), # 0.30 + 0.50 = 0.80 + roi_run("r2", NOW, "test-pro", 400_000, 60_000, 40_000), # long: 1.00 + 1.50 = 2.50 + roi_run("r3", LAST, "test-lite", 1_000_000, 1_000_000), # 0.10 + 0.40 = 0.50 + roi_run("r4", LAST, "test-flash"), # no token usage recorded +] + + +@pytest.fixture +def cost_page(page, stub, monkeypatch): + monkeypatch.setattr(sys.modules["pricing"], "PRICING", TABLE) # the stub's own pricing module + monkeypatch.setattr(stub, "PRICING_AS_OF", AS_OF) + stub.ROI_RUNS[:] = RUNS + page.get_by_role("navigation").get_by_role("button", name="Cost", exact=True).click() + return page + + +def tile(page, name): + return page.get_by_role("group", name=name, exact=True) + + +def test_spend_tiles_sum_the_priced_runs(cost_page): + expect(tile(cost_page, "Total spend")).to_contain_text("$3.80") + expect(tile(cost_page, "This month")).to_contain_text("$3.30") + + +def test_coverage_pill_counts_priced_of_all_documents(cost_page): + expect(cost_page.get_by_text("3 of 4 documents priced")).to_be_visible() + + +def test_month_breakdown_groups_runs_by_month(cost_page): + rows = cost_page.get_by_role("table", name="Spend by month").get_by_role("row") + expect(rows).to_have_count(3) # header + two months, newest first + expect(rows.nth(1)).to_contain_text(NOW.strftime("%b %Y")) + expect(rows.nth(1)).to_contain_text("2 of 2") + expect(rows.nth(1)).to_contain_text("$3.30") + expect(rows.nth(2)).to_contain_text(LAST.strftime("%b %Y")) + expect(rows.nth(2)).to_contain_text("1 of 2") + expect(rows.nth(2)).to_contain_text("$0.50") + + +def test_runs_show_tokens_and_cost_and_unpriced_is_not_zero(cost_page): + runs = cost_page.get_by_role("table", name="Cost per run") + r2 = runs.get_by_role("row").filter(has_text="r2.pdf") + for text in ("test-pro", "400,000", "60,000", "40,000", "$2.50"): + expect(r2).to_contain_text(text) + expect(runs.get_by_role("row").filter(has_text="r4.pdf")).to_contain_text("not priced") + + +def test_by_hand_panel_is_labelled_assumptions_and_recomputes(cost_page): + panel = cost_page.get_by_role("region", name="Against doing it by hand") + expect(panel).to_contain_text("assumptions") + # 3 priced documents x 10 min = 0.5 h at $30/h = $15.00 + panel.get_by_label("Minutes per document").fill("10") + panel.get_by_label("Hourly rate").fill("30") + expect(panel).to_contain_text("$15.00") + expect(cost_page.get_by_text(AS_OF)).to_be_visible() # pricing as-of date shown diff --git a/tests/test_pricing.py b/tests/test_pricing.py new file mode 100644 index 0000000..76e4f13 --- /dev/null +++ b/tests/test_pricing.py @@ -0,0 +1,50 @@ +"""Self-check for the AI cost function: rates per million tokens, thinking bills as output, +long-context rates above the threshold, and an unknown model is an error, never zero.""" +import importlib.util +from pathlib import Path + +import pytest + +SRC = Path(__file__).resolve().parents[1] / "rpa-dashboard-ai" / "templates" / "web" / "pricing.py" +spec = importlib.util.spec_from_file_location("pricing", SRC) +pricing = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pricing) + +TABLE = { + "flat-model": {"input": 0.30, "output": 2.50}, + "long-model": {"input": 1.25, "output": 10.00, "long_threshold": 200_000, "input_long": 2.50, "output_long": 15.00}, +} + + +@pytest.fixture(autouse=True) +def table(monkeypatch): + monkeypatch.setattr(pricing, "PRICING", TABLE) + + +def test_normal_call_prices_input_and_output_per_million(): + # 1M in at 0.30 + 0.2M out at 2.50 = 0.30 + 0.50 + assert pricing.cost_for_call("flat-model", 1_000_000, 200_000) == pytest.approx(0.80) + + +def test_thinking_tokens_bill_at_the_output_rate(): + # 0.1M in at 0.30 + (0.1M out + 0.3M thinking) at 2.50 = 0.03 + 1.00 + assert pricing.cost_for_call("flat-model", 100_000, 100_000, 300_000) == pytest.approx(1.03) + + +def test_long_context_rates_apply_above_the_threshold_only(): + # at the threshold: 0.2M at 1.25 + 0.1M at 10 = 0.25 + 1.00 + assert pricing.cost_for_call("long-model", 200_000, 100_000) == pytest.approx(1.25) + # above it: 0.4M at 2.50 + 0.1M at 15 = 1.00 + 1.50 + assert pricing.cost_for_call("long-model", 400_000, 100_000) == pytest.approx(2.50) + + +def test_unknown_model_raises_and_names_the_known_models(): + with pytest.raises(ValueError) as e: + pricing.cost_for_call("mystery-model", 1000, 1000) + assert "mystery-model" in str(e.value) + assert "flat-model" in str(e.value) and "long-model" in str(e.value) + + +def test_bundled_table_records_its_source_and_as_of_date(): + assert pricing.PRICING_SOURCE.startswith("https://") + assert len(pricing.PRICING_AS_OF) == 10 and pricing.PRICING_AS_OF[4] == "-"