AI cost/ROI page: pricing table, cost per call, Cost tab, ROI endpoint

- templates/web/pricing.py: pricing table with source and as-of date; cost_for_call (input rate + (output + thinking) at output rate, long-context above threshold; unknown model raises naming the known models). - Cost tab: coverage pill, total/this-month/tokens tiles, month-by-month table, per-run tokens and cost ("not priced", never $0), by-hand panel with editable assumptions labelled as such. - Contract: RoiRun and GET /roi; optional GET /models and POST /runs/{id}/retry. - SKILL.md: cost page wiring, model picker and retry-on-other-model as optional sections. - Tests: test_pricing.py self-check, page_check_ai/test_ai_cost.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 15:23:52 +05:30
1 parent 8951ac05a5
commit 9dcdd8f572
@@ -13,11 +13,16 @@ Run       { job_id, document, status: pending|queued|running|done|failed, model,
            approved, error }
 ReviewRow { row_id, field, value, band: green|amber|red|blank|unscored,
            score, reason, verified, corrected_value }
+RoiRun    { job_id, document, created_at, model, input_tokens, output_tokens,
+            thinking_tokens, total_cost }
 ```

 - `summary` counts the run's rows per band. It is `null` until the run is `done`.
 - `score` is the field's worst word score, or `null` when it has none.
 - `corrected_value` is `null` until a reviewer saves a value different from `value`.
+- `created_at` is ISO 8601 in UTC. Months on the cost page are UTC months.
+- `RoiRun` token counts and `total_cost` (USD) are `null` when the run recorded no token
+  usage. The page shows it as "not priced" and counts it in the coverage pill, never as $0.

 ## Endpoints (web: `{API_BASE}` + path, JSON in and out)

@@ -29,8 +34,16 @@ ReviewRow { row_id, field, value, band: green|amber|red|blank|unscored,
 | `GET /runs/{job_id}/document` | The source document (PDF or image) shown beside the fields. |
 | `PATCH /review/{row_id}` | Body `{corrected_value?, verified}`. Patches one row, returns it. |
 | `POST /runs/{job_id}/verify-remaining` | Verifies every flagged row not yet verified. Returns `{verified: n}`. |
+| `GET /roi` | **Fetch ROI.** `{runs: [RoiRun], pricing: {source, as_of}}`, every run, newest first. The page sums spend, groups months and computes the by-hand panel itself. |
 | `POST /runs/{job_id}/approve` | **The approve gate.** `409 {detail}` while the run isn't done or any flagged row is unverified, else `200 {ok: true}`. |

+Optional, only when the user chose those sections:
+
+| Method and path | What it does |
+|---|---|
+| `GET /models` | `{models: [name]}`: the keys of the pricing table, so only priced models can be picked. |
+| `POST /runs/{job_id}/retry` | Body `{model}`. Re-queues a `failed` run on that model, returns the `Run`. |
+
 Errors answer with a non-2xx status and `{detail: "<why>"}`. The page shows `detail`.

 ## Rules
@@ -45,6 +58,12 @@ Errors answer with a non-2xx status and `{detail: "<why>"}`. The page shows `det
  edited row (with `verified: true`). Leaving with unsaved edits asks first.
 - **Polling:** while a run is `queued` or `running`, the page polls `GET /runs/{job_id}`
  every 1.5s and shows "page X of Y". It stops when the run settles.
+- **Cost** comes from `templates/web/pricing.py` and nowhere else: per million tokens,
+  input × input rate + (output + thinking) × output rate, long-context rates above the
+  model's threshold. An unknown model is an error naming the known models, never $0.
+  The table records its source URL and as-of date; `GET /roi` returns both.
+- **Assumptions** (minutes per document, hourly rate) live only on the page, typed by the
+  reader and labelled as assumptions. The backend never stores or guesses them.
 - **Caveat:** the legend always shows that colour reflects print clarity, not
  correctness.

@@ -0,0 +1,32 @@
+"""Pricing table and cost per call for the cost page. The backend imports it.
+
+Rates are USD per million tokens. Thinking tokens bill at the output rate. A model
+with "long_threshold" switches both rates to its *_long rates when a call's input
+tokens go above that threshold.
+
+Replace the table with the rates of the provider and models the project really uses,
+copied from the provider's pricing page, and update PRICING_SOURCE and PRICING_AS_OF.
+Never guess a rate: a model missing here is an error, never a cost of zero.
+"""
+
+PRICING_SOURCE = "https://ai.google.dev/gemini-api/docs/pricing"
+PRICING_AS_OF = "2026-08-11"
+
+PRICING = {
+    "gemini-2.5-pro": {"input": 1.25, "output": 10.00,
+                       "long_threshold": 200_000, "input_long": 2.50, "output_long": 15.00},
+    "gemini-2.5-flash": {"input": 0.30, "output": 2.50},
+    "gemini-2.5-flash-lite": {"input": 0.10, "output": 0.40},
+}
+
+
+def cost_for_call(model, input_tokens, output_tokens, thinking_tokens=0):
+    """USD for one call. Raises ValueError, naming the known models, for a model not in PRICING."""
+    if model not in PRICING:
+        raise ValueError(f"No pricing for model '{model}'. Add it to PRICING from {PRICING_SOURCE}. "
+                         f"Known models: {', '.join(PRICING)}")
+    p = PRICING[model]
+    in_rate, out_rate = p["input"], p["output"]
+    if "long_threshold" in p and input_tokens > p["long_threshold"]:
+        in_rate, out_rate = p["input_long"], p["output_long"]
+    return (input_tokens * in_rate + (output_tokens + thinking_tokens) * out_rate) / 1_000_000
@@ -18,6 +18,7 @@ from pathlib import Path
 from urllib.parse import urlparse

 from bands import BANDS, field_band, needs_review
+from pricing import PRICING_AS_OF, PRICING_SOURCE, cost_for_call

 HERE = Path(__file__).resolve().parent
 API = "/api"  # same as API_BASE in index.html
@@ -34,6 +35,7 @@ SAMPLE_FIELDS = [
    ("Tax", "196.00", [], False),
 ]
 RUNS, ROWS = {}, {}
+ROI_RUNS = []  # [{job_id, document, created_at, model, input_tokens, output_tokens, thinking_tokens}]
 _next_row = [1]


@@ -56,6 +58,16 @@ def reset():
    ROWS.clear()
    put_run("job-1002", "invoice-1002.pdf", SAMPLE_FIELDS, status="running", pages_done=1, pages_total=3)
    put_run("job-1001", "invoice-1001.pdf", SAMPLE_FIELDS, pages_total=2, pages_done=2)
+    ROI_RUNS.clear()
+    now = time.time()
+    for i, (model, tokens) in enumerate([("gemini-2.5-flash", (21_000, 1_800, 900)),
+                                         ("gemini-2.5-flash", (18_500, 1_600, 0)),
+                                         ("gemini-2.5-pro", (240_000, 3_000, 2_500)),
+                                         ("gemini-2.5-flash-lite", (22_000, 1_700, 0)),
+                                         ("gemini-2.5-flash", (None, None, None))]):  # usage not recorded
+        ROI_RUNS.append({"job_id": f"job-{900 + i}", "document": f"invoice-{900 + i}.pdf",
+                         "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now - i * 12 * 864e2)),
+                         "model": model, **dict(zip(("input_tokens", "output_tokens", "thinking_tokens"), tokens))})


 def run_view(job_id):
@@ -104,6 +116,18 @@ def approve(job_id, body):
    return 200, {"ok": True}


+def roi(body):
+    """Cost per run. total_cost is null when the run recorded no token usage: unpriced, never zero.
+    An unknown model is an error (500 naming the known models), never a cost of zero."""
+    try:
+        runs = [{**r, "total_cost": None if r["input_tokens"] is None else
+                 cost_for_call(r["model"], r["input_tokens"], r["output_tokens"], r["thinking_tokens"] or 0)}
+                for r in ROI_RUNS]
+    except ValueError as e:
+        return 500, {"detail": str(e)}
+    return 200, {"runs": runs, "pricing": {"source": PRICING_SOURCE, "as_of": PRICING_AS_OF}}
+
+
 def document(job_id, body):
    """A stand-in for the source document. A real backend returns the PDF or page image."""
    lines = "".join(f'<text x="48" y="{150 + i * 46}" font-size="20">{escape(f)}: {escape(v)}</text>'
@@ -122,6 +146,7 @@ ROUTES = [
    ("PATCH", r"/review/(\d+)", patch_row),
    ("POST", r"/runs/([\w-]+)/verify-remaining", verify_remaining),
    ("POST", r"/runs/([\w-]+)/approve", approve),
+    ("GET", r"/roi", roi),
    ("POST", r"/set-dark", lambda body: (200, {})),
 ]

@@ -4,12 +4,14 @@ import { Overview } from './components/Overview'
 import { RunPanel } from './components/RunPanel'
 import { Report } from './components/Report'
 import { Review } from './components/Review'
+import { Cost } from './components/Cost'

 const TABS = {
  dashboard: ['Dashboard', 'Overview of every batch', <><rect x="3" y="3" width="7" height="7" /><rect x="14" y="3" width="7" height="7" /><rect x="3" y="14" width="7" height="7" /><rect x="14" y="14" width="7" height="7" /></>],
  run: ['Run', 'Live progress and log', <polygon points="6 4 20 12 6 20 6 4" />],
  review: ['Review', 'Check extracted fields against the document', <><path d="M4 4h10l6 6v10H4z" /><path d="m8 14 3 3 5-6" /></>],
  report: ['Report', 'Result of every processed item', <><path d="M4 4h16v16H4z" /><path d="M8 9h8M8 13h8M8 17h5" /></>],
+  cost: ['Cost', 'Tokens, spend and the case against doing it by hand', <><path d="M12 2v20" /><path d="M17 6H9.5a3.5 3.5 0 0 0 0 7h5a3.5 3.5 0 0 1 0 7H6" /></>],
  // OPTIONAL: admin/roles tab goes here
 } as const
 type Tab = keyof typeof TABS
@@ -98,6 +100,7 @@ export function App() {
          {tab === 'run' && <RunPanel />}
          {tab === 'review' && <Review onUnsaved={setUnsaved} />}
          {tab === 'report' && <Report />}
+          {tab === 'cost' && <Cost />}
        </section>
      </main>
    </div>
@@ -23,6 +23,19 @@ export type ReviewRow = {
  corrected_value: string | null
 }

+/** One run on the cost page. total_cost is null when the run recorded no token usage (unpriced). */
+export type RoiRun = {
+  job_id: string
+  document: string
+  created_at: string // ISO, UTC
+  model: string
+  input_tokens: number | null
+  output_tokens: number | null
+  thinking_tokens: number | null
+  total_cost: number | null // USD
+}
+export type Pricing = { source: string; as_of: string }
+
 export const BUSY: RunStatus[] = ['queued', 'running'] // polled every 1.5s while in one of these

 /** Legend order, label, and the colour token. Band colours are semantic: never on buttons or headings. */
@@ -53,5 +66,6 @@ export const ai = {
    req<ReviewRow>(`/review/${rowId}`, { method: 'PATCH', body: JSON.stringify(body) }),
  verifyRemaining: (id: string) => req<{ verified: number }>(`/runs/${id}/verify-remaining`, { method: 'POST' }),
  approve: (id: string) => req<{ ok: boolean }>(`/runs/${id}/approve`, { method: 'POST' }),
+  roi: () => req<{ runs: RoiRun[]; pricing: Pricing }>('/roi'),
  documentUrl: (id: string) => `${API_BASE}/runs/${id}/document`,
 }
@@ -0,0 +1,133 @@
+import { useEffect, useState } from 'react'
+import { ai, type Pricing, type RoiRun } from '../ai'
+
+const money = (n: number) => `$${n.toFixed(n > 0 && n < 0.01 ? 4 : 2)}`
+const num = (n: number | null) => (n == null ? '—' : n.toLocaleString('en-US'))
+const monthOf = (iso: string) => iso.slice(0, 7) // 'YYYY-MM', UTC
+const monthLabel = (ym: string) =>
+  new Date(`${ym}-01T00:00:00Z`).toLocaleString('en-US', { month: 'short', year: 'numeric', timeZone: 'UTC' })
+
+/**
+ * The cost page. Two kinds of number, kept apart:
+ *   measured: tokens and cost per run, priced by the backend's pricing table;
+ *   assumed:  minutes per document and an hourly rate, typed by the reader and labelled so.
+ * A run with no recorded token usage is "not priced", never $0, and the coverage pill says how many.
+ */
+export function Cost() {
+  const [runs, setRuns] = useState<RoiRun[]>([])
+  const [pricing, setPricing] = useState<Pricing>()
+  const [error, setError] = useState<string>()
+  const [minutes, setMinutes] = useState(5) // assumption: minutes to key one document by hand
+  const [rate, setRate] = useState(20) // assumption: USD per hour of that person's time
+
+  useEffect(() => {
+    ai.roi().then((d) => { setRuns(d.runs); setPricing(d.pricing) }).catch((e) => setError(e.message))
+  }, [])
+
+  const priced = runs.filter((r) => r.total_cost != null)
+  const sum = (rs: RoiRun[]) => rs.reduce((n, r) => n + (r.total_cost ?? 0), 0)
+  const thisMonth = new Date().toISOString().slice(0, 7)
+  const months = [...new Set(runs.map((r) => monthOf(r.created_at)))].sort().reverse()
+    .map((m) => {
+      const rs = runs.filter((r) => monthOf(r.created_at) === m)
+      return { m, docs: rs.length, priced: rs.filter((r) => r.total_cost != null).length, cost: sum(rs) }
+    })
+  const tokens = (k: 'input_tokens' | 'output_tokens' | 'thinking_tokens') => priced.reduce((n, r) => n + (r[k] ?? 0), 0)
+  const byHand = (priced.length * minutes / 60) * rate
+
+  if (error) return <div className="card text-bad">Can't load costs from {API_BASE}/roi: {error}</div>
+
+  return (
+    <>
+      <div className="mb-4 flex flex-wrap items-center gap-3">
+        <span className={`pill ${priced.length === runs.length ? 'bg-ok' : 'bg-warn'}`}>
+          {priced.length} of {runs.length} documents priced
+        </span>
+        {pricing && (
+          <span className="text-[11.5px] text-muted">
+            Rates from <a className="underline" href={pricing.source} target="_blank" rel="noreferrer">{pricing.source}</a>, as of {pricing.as_of}
+          </span>
+        )}
+      </div>
+
+      <div className="mb-4 grid grid-cols-[repeat(auto-fit,minmax(170px,1fr))] gap-4">
+        <Tile name="Total spend" value={money(sum(priced))} note={`${priced.length} priced runs`} />
+        <Tile name="This month" value={money(sum(runs.filter((r) => monthOf(r.created_at) === thisMonth)))} note={monthLabel(thisMonth)} />
+        <Tile name="Tokens" value={num(tokens('input_tokens') + tokens('output_tokens') + tokens('thinking_tokens'))}
+          note={`in ${num(tokens('input_tokens'))} · out ${num(tokens('output_tokens'))} · thinking ${num(tokens('thinking_tokens'))}`} />
+        <Tile name="Per document" value={priced.length ? money(sum(priced) / priced.length) : '—'} note="average, priced runs" />
+      </div>
+
+      <div className="card">
+        <div className="card-title">Month by month</div>
+        <div className="overflow-auto">
+          <table aria-label="Spend by month" className="w-full border-collapse whitespace-nowrap text-xs">
+            <thead><tr><Th>Month</Th><Th>Documents priced</Th><Th right>Spend</Th></tr></thead>
+            <tbody>
+              {months.map((x) => (
+                <tr key={x.m} className="border-t border-line">
+                  <Td>{monthLabel(x.m)}{x.m === thisMonth && <span className="ml-2 text-muted">this month</span>}</Td>
+                  <Td>{x.priced} of {x.docs}</Td>
+                  <Td right>{x.priced ? money(x.cost) : '—'}</Td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+      </div>
+
+      <section className="card" aria-labelledby="by-hand">
+        <div id="by-hand" className="card-title">Against doing it by hand</div>
+        <p className="mb-3 text-xs text-muted">These are your assumptions, not a measurement. Change them to match the team.</p>
+        <div className="mb-3 flex flex-wrap gap-4 text-xs">
+          <label className="flex items-center gap-2">Minutes per document
+            <input className="input w-20" type="number" min="0" value={minutes} onChange={(e) => setMinutes(+e.target.value)} />
+          </label>
+          <label className="flex items-center gap-2">Hourly rate
+            <input className="input w-20" type="number" min="0" value={rate} onChange={(e) => setRate(+e.target.value)} />
+          </label>
+        </div>
+        <p className="text-[13px]">
+          By hand, {priced.length} documents would take {num(Math.round(priced.length * minutes / 6) / 10)} h, about <b>{money(byHand)}</b>.
+          The AI spent <b>{money(sum(priced))}</b> on them.
+        </p>
+      </section>
+
+      {/* OPTIONAL: model picker (per batch) goes here */}
+
+      <div className="card">
+        <div className="card-title">Cost per run</div>
+        <div className="max-h-[50vh] overflow-auto">
+          <table aria-label="Cost per run" className="w-full border-collapse whitespace-nowrap text-xs">
+            <thead>
+              <tr><Th>Document</Th><Th>Date</Th><Th>Model</Th><Th right>Input</Th><Th right>Output</Th><Th right>Thinking</Th><Th right>Cost</Th></tr>
+            </thead>
+            <tbody>
+              {runs.map((r) => (
+                <tr key={r.job_id} className="border-t border-line">
+                  <Td>{r.document}</Td><Td>{r.created_at.slice(0, 10)}</Td><Td>{r.model}</Td>
+                  <Td right>{num(r.input_tokens)}</Td><Td right>{num(r.output_tokens)}</Td><Td right>{num(r.thinking_tokens)}</Td>
+                  <Td right>{r.total_cost == null ? <span className="text-muted">not priced</span> : money(r.total_cost)}</Td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </>
+  )
+}
+
+function Tile({ name, value, note }: { name: string; value: string; note: string }) {
+  return (
+    <div role="group" aria-label={name} className="card mb-0">
+      <div className="text-[11px] font-bold uppercase tracking-wide text-muted">{name}</div>
+      <div className="mt-1 font-head text-[26px] font-extrabold">{value}</div>
+      <div className="mt-1 text-[11px] text-muted">{note}</div>
+    </div>
+  )
+}
+const Th = ({ children, right }: { children: React.ReactNode; right?: boolean }) =>
+  <th className={`px-3 py-2 font-bold ${right ? 'text-right' : 'text-left'}`}>{children}</th>
+const Td = ({ children, right }: { children: React.ReactNode; right?: boolean }) =>
+  <td className={`px-3 py-2 ${right ? 'text-right' : ''}`}>{children}</td>
@@ -105,6 +105,7 @@ export function Review({ onUnsaved }: { onUnsaved: (n: number) => void }) {
        {run && <span className={`pill ${run.status === 'done' ? 'bg-ok' : run.status === 'failed' ? 'bg-bad' : 'bg-warn'}`}>{run.status}</span>}
        {run && BUSY.includes(run.status) && <span className="text-[12.5px] text-muted tabular-nums">page {run.pages_done} of {run.pages_total || '?'}</span>}
        {run?.error && <span className="text-[12px] text-bad">{run.error}</span>}
+        {/* OPTIONAL: retry on another model (failed runs) goes here */}
        {run?.status === 'done' && (
          <div className="ml-auto flex items-center gap-2">
            <span className="text-[12px] text-muted">{left} to verify</span>