Check page band legends against bands.py; amber is verified, not sampled
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4ad9d96fb9
commit
7ad73c1fda
3 files changed
+19
-5
No files matched your search
@@ -5,14 +5,17 @@ an exported sheet, the approve gate) imports from here, so they never disagree
|
|||||||
about what counts as red.
|
about what counts as red.
|
||||||
|
|
||||||
GREEN >= 0.95 clean print, straight through
|
GREEN >= 0.95 clean print, straight through
|
||||||
AMBER 0.90-0.95 slightly unclear, sampled
|
AMBER 0.90-0.95 slightly unclear, check it
|
||||||
RED < 0.90 handwriting or a messy scan, always reviewed
|
RED < 0.90 handwriting or a messy scan
|
||||||
BLANK a mandatory field with no value, always flagged
|
BLANK a mandatory field with no value
|
||||||
UNSCORED no word scores came back for the value
|
UNSCORED no word scores came back for the value
|
||||||
|
|
||||||
|
The approve gate: every field that isn't green must be verified by a person before the
|
||||||
|
run can be approved (needs_review). Amber is not sampled; it is checked like red.
|
||||||
|
|
||||||
Calibration (from the prototype these rules came from): crisp print scored 0.90 and
|
Calibration (from the prototype these rules came from): crisp print scored 0.90 and
|
||||||
up, handwriting clustered in 0.80-0.89, and 0.70-0.79 was empty. 0.90 sits in that
|
up, handwriting clustered in 0.80-0.89, and 0.70-0.79 was empty. 0.90 sits in that
|
||||||
gap. 0.95 splits off the 0.90-0.99 band that reviewers sample rather than trust.
|
gap. 0.95 splits off the slightly unclear 0.90-0.95 reads from clean print.
|
||||||
|
|
||||||
What a band claims: the score is the model's own legibility rating. Green means the
|
What a band claims: the score is the model's own legibility rating. Green means the
|
||||||
text was read cleanly, NOT that the value is correct; a confident misread scores
|
text was read cleanly, NOT that the value is correct; a confident misread scores
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from pricing import PRICING_AS_OF, PRICING_SOURCE, cost_for_call
|
|||||||
|
|
||||||
HERE = Path(__file__).resolve().parent
|
HERE = Path(__file__).resolve().parent
|
||||||
API = "/api" # same as API_BASE in index.html
|
API = "/api" # same as API_BASE in index.html
|
||||||
REASON = {"green": None, "amber": "Slightly unclear, sample check", "red": "Unclear print, check every word",
|
REASON = {"green": None, "amber": "Slightly unclear, verify it", "red": "Unclear print, check every word",
|
||||||
"blank": "Required field is empty", "unscored": "Not found in the transcription"}
|
"blank": "Required field is empty", "unscored": "Not found in the transcription"}
|
||||||
# (field, value, word scores, mandatory). Swap in the fields the project's extraction returns.
|
# (field, value, word scores, mandatory). Swap in the fields the project's extraction returns.
|
||||||
SAMPLE_FIELDS = [
|
SAMPLE_FIELDS = [
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
"""Confidence bands: thresholds, the worst-word rule and the blank-mandatory flag (spec: Banding)."""
|
"""Confidence bands: thresholds, the worst-word rule and the blank-mandatory flag (spec: Banding)."""
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
BANDS = Path(__file__).resolve().parents[1] / "rpa-dashboard-ai" / "templates" / "web" / "bands.py"
|
BANDS = Path(__file__).resolve().parents[1] / "rpa-dashboard-ai" / "templates" / "web" / "bands.py"
|
||||||
@@ -39,3 +40,13 @@ def test_no_scores_means_unscored():
|
|||||||
def test_only_green_skips_review():
|
def test_only_green_skips_review():
|
||||||
assert not bands.needs_review("green")
|
assert not bands.needs_review("green")
|
||||||
assert all(bands.needs_review(b) for b in ["amber", "red", "blank", "unscored"])
|
assert all(bands.needs_review(b) for b in ["amber", "red", "blank", "unscored"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_page_legends_use_the_bands_py_cut_points():
|
||||||
|
"""The legend labels are hard-coded in the pages; they must say what bands.py decides."""
|
||||||
|
green, floor = bands.GREEN_THRESHOLD, bands.CONFIDENCE_THRESHOLD
|
||||||
|
for page in [BANDS.parents[1] / "web" / "src" / "ai.ts", BANDS.parents[1] / "desktop" / "ai_sections.html"]:
|
||||||
|
m = re.search(r"Clean ≥([\d.]+).*?Check ([\d.]+)–([\d.]+).*?Review <([\d.]+)",
|
||||||
|
page.read_text(encoding="utf-8"), re.S)
|
||||||
|
assert m, f"{page.name}: band legend labels not found"
|
||||||
|
assert tuple(map(float, m.groups())) == (green, floor, green, floor), page.name
|
||||||
Reference in new issue
Block a user