ManikandanandClaude Opus 5.5
ba15a574ab
Snapshot of the working project as of 2026-10-23 (local 2d4d4fc), one commit with no earlier history: the tracked files were de-identified first (synthetic claim ids and patient data only; see Context.md section 7). Credentials, run logs, screenshots, claim reports and client workbooks are gitignored and not included. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
319 lines
12 KiB
Python
319 lines
12 KiB
Python
r"""
|
|
The Charge Summary rows, also saved to SQL Server for the results web page
|
|
(`web/`).
|
|
|
|
Operator, 2026-10-08: server MECARE331\MECARE331, database
|
|
`Charge_Validation`, table `dbo.CV_ChargeSummary` (every table of this
|
|
project starts with `CV_`, operator 2026-10-12) — the team's Charge Summary
|
|
File Format's 24 columns (`charge_summary.HEADERS`), plus `run_at`,
|
|
`run_mode` and `details_json` (what the conditions found and changed). The
|
|
table is created by the operator from `tools/sql/create_tables.sql`; the bot
|
|
only inserts.
|
|
|
|
The rows are the ones the Excel summary gets (`charge_summary.build`), so the
|
|
two always agree. Only rows the run filled in (a Category) are saved: a
|
|
claim not processed or not found is blank in the Excel too.
|
|
|
|
Off unless `RESULTS_DB=1`. Never fatal: any database error is one
|
|
`[RESULTS-DB]` log line, and the run and its Excel summary carry on.
|
|
|
|
Values are converted for their columns — amounts without `$` and commas,
|
|
dates from mm/dd/yyyy (a trailing time or an Excel serial day number
|
|
included) — and anything that does not convert is stored as NULL (the Excel
|
|
file still has the text as read). Text longer than its column is cut to the
|
|
column's width, so one long value cannot fail the insert. SQL is always
|
|
parameterized, and a database error is logged by its SQLSTATE only, never
|
|
the driver's message, which can quote a value (PHI) back.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from datetime import date, datetime, timedelta
|
|
from decimal import Decimal, InvalidOperation
|
|
|
|
from . import charge_summary
|
|
|
|
TABLE = "dbo.CV_ChargeSummary"
|
|
|
|
# Charge Summary header -> DB column. `COLUMNS` is built from HEADERS, so a
|
|
# header added to the format without a column here fails at import, not
|
|
# silently at insert time.
|
|
_COLUMN_FOR = {
|
|
"S.No": "s_no", "CLAIMS#": "claim_no", "SERVICE DATE": "service_date",
|
|
"PVDR": "pvdr", "PATIENT": "patient", "PAYER": "payer",
|
|
"Group Name": "group_name", "STATUS": "status", "CHARGES": "charges",
|
|
"PMTS/ADJS": "pmts_adjs", "Pmts/Refunds": "pmts_refunds",
|
|
"Adjustment": "adjustment", "WithHeld": "withheld", "Balance": "balance",
|
|
"Visit Type": "visit_type", "Account Number": "account_number",
|
|
"Provider Name": "provider_name", "Guarantor Name": "guarantor_name",
|
|
"Recevied date": "received_date", "Assigned To ": "assigned_to",
|
|
"Completed On": "completed_on", "Comments": "comments",
|
|
"Current Status": "current_status", "Category": "category",
|
|
}
|
|
COLUMNS = [(h, _COLUMN_FOR[h]) for h in charge_summary.HEADERS]
|
|
|
|
DB_COLUMNS = (["run_at", "run_mode"] + [c for _, c in COLUMNS]
|
|
+ ["details_json"])
|
|
|
|
# NVARCHAR widths from tools/sql/create_tables.sql. MAX columns are not
|
|
# listed.
|
|
_MAX_LEN = {
|
|
"run_mode": 20, "claim_no": 20, "pvdr": 50, "patient": 200,
|
|
"payer": 200, "group_name": 100, "status": 100, "visit_type": 100,
|
|
"account_number": 50, "provider_name": 200, "guarantor_name": 200,
|
|
"assigned_to": 100, "current_status": 200, "category": 20,
|
|
}
|
|
|
|
_MONEY = {"charges", "pmts_adjs", "pmts_refunds", "adjustment", "withheld",
|
|
"balance"}
|
|
_DATES = {"service_date", "received_date", "completed_on"}
|
|
_INTS = {"s_no"}
|
|
|
|
# DECIMAL(12,2): ten digits before the point.
|
|
_MONEY_LIMIT = Decimal(10) ** 10
|
|
_CENT = Decimal("0.01")
|
|
|
|
_US_DATE = re.compile(r"^(\d{1,2})/(\d{1,2})/(\d{4})(?:\s|$)")
|
|
_SERIAL = re.compile(r"^\d{5}(?:\.0+)?$")
|
|
# Excel's day 0, and the serial range that is a plausible date
|
|
# (1954-10-03 .. 2119-01-08) — not some other five-digit number.
|
|
_EXCEL_EPOCH = date(1899, 12, 30)
|
|
_SERIAL_RANGE = (20_000, 80_000)
|
|
_ISO_DATE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})")
|
|
|
|
# What details_for() copies from the claim's `_conditions`.
|
|
_CONDITION_KEYS = ("fired", "detected", "changed", "saved",
|
|
"save_unconfirmed", "failed", "review", "comments",
|
|
"changes")
|
|
|
|
# Keys dropped wherever they appear in a condition's changes: they carry
|
|
# patient text, not what the bot did — vaccine_admin_codes' age_source
|
|
# names the DOB it read; the Dental Billing Notes' match quotes the note.
|
|
_PHI_KEYS = {"age_source", "match", "dob"}
|
|
|
|
# SQLSTATE -> what it means for the operator.
|
|
_SQLSTATE_HINT = {
|
|
"08001": "server not reachable", "08S01": "connection lost",
|
|
"HYT00": "timed out", "HYT01": "timed out", "28000": "login failed",
|
|
"42S02": "table missing — run tools/sql/create_tables.sql",
|
|
"42000": "no permission, or a SQL error",
|
|
"22001": "a value is too long for its column",
|
|
"22003": "a number is too large for its column",
|
|
}
|
|
|
|
|
|
def to_money(value):
|
|
"""An amount as Decimal (2 places), or None when blank or not a number."""
|
|
if value is None or isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, (int, float, Decimal)):
|
|
text = str(value)
|
|
else:
|
|
text = str(value).replace("$", "").replace(",", "").strip()
|
|
negative = text.startswith("(") and text.endswith(")")
|
|
if negative:
|
|
text = text[1:-1].strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
amount = Decimal(text)
|
|
except InvalidOperation:
|
|
return None
|
|
if not amount.is_finite() or abs(amount) >= _MONEY_LIMIT:
|
|
return None
|
|
amount = amount.quantize(_CENT)
|
|
# 9999999999.995 rounds up to 10^10, past DECIMAL(12,2).
|
|
if abs(amount) >= _MONEY_LIMIT:
|
|
return None
|
|
return -amount if negative else amount
|
|
|
|
|
|
def to_date(value):
|
|
"""A date from mm/dd/yyyy, yyyy-mm-dd or a date/datetime; else None."""
|
|
if isinstance(value, datetime):
|
|
return value.date()
|
|
if isinstance(value, date):
|
|
return value
|
|
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
value = str(value)
|
|
text = str(value or "").strip()
|
|
try:
|
|
m = _US_DATE.match(text)
|
|
if m:
|
|
return date(int(m[3]), int(m[1]), int(m[2]))
|
|
m = _ISO_DATE.match(text)
|
|
if m:
|
|
return date(int(m[1]), int(m[2]), int(m[3]))
|
|
if _SERIAL.match(text):
|
|
days = int(float(text))
|
|
if _SERIAL_RANGE[0] <= days <= _SERIAL_RANGE[1]:
|
|
return _EXCEL_EPOCH + timedelta(days=days)
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _to_int(value):
|
|
try:
|
|
return int(str(value).strip())
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _to_text(value):
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
return text or None
|
|
|
|
|
|
def _fit(col, text):
|
|
"""Text cut to its column's width (NVARCHAR(n)); MAX columns as is."""
|
|
limit = _MAX_LEN.get(col)
|
|
if text is None or limit is None or len(text) <= limit:
|
|
return text
|
|
return text[:limit]
|
|
|
|
|
|
def to_record(row: dict, *, run_at, run_mode, details) -> dict:
|
|
"""One Charge Summary row ({header: value}) as {db column: value}."""
|
|
rec = {"run_at": run_at, "run_mode": run_mode}
|
|
for header, col in COLUMNS:
|
|
value = row.get(header)
|
|
if col in _MONEY:
|
|
rec[col] = to_money(value)
|
|
elif col in _DATES:
|
|
rec[col] = to_date(value)
|
|
elif col in _INTS:
|
|
rec[col] = _to_int(value)
|
|
else:
|
|
rec[col] = _fit(col, _to_text(value))
|
|
rec["run_mode"] = _fit("run_mode", run_mode)
|
|
rec["details_json"] = (json.dumps(details, default=str, ensure_ascii=False)
|
|
if details else None)
|
|
return rec
|
|
|
|
|
|
def details_for(data: dict) -> dict:
|
|
"""
|
|
What the run found on one claim, for `details_json`: the conditions'
|
|
account (fired / changes / review / failed / comments / saved), the
|
|
Dental checks, and why a claim could not be read. Never the claim's own
|
|
fields (patient, CPT grid...) — those are in the row's columns or not
|
|
stored at all.
|
|
"""
|
|
data = data or {}
|
|
out = {}
|
|
cond = data.get("_conditions")
|
|
if isinstance(cond, dict):
|
|
out.update({k: _without_phi(cond[k])
|
|
for k in _CONDITION_KEYS if k in cond})
|
|
# Dental: the verdicts only — not the Billing Notes text, not the
|
|
# sliding-fee dates.
|
|
hr = data.get("high_risk")
|
|
if isinstance(hr, dict):
|
|
out.setdefault("dental", {})["high_risk"] = hr.get("high_risk")
|
|
sf = data.get("sliding_fee")
|
|
if isinstance(sf, dict):
|
|
out.setdefault("dental", {})["sliding_fee"] = {
|
|
k: sf[k] for k in ("verdict", "reason", "error") if k in sf}
|
|
for key in ("_error", "_not_evaluated", "_search_error"):
|
|
if data.get(key):
|
|
out[key.lstrip("_")] = data[key]
|
|
return out
|
|
|
|
|
|
def _without_phi(value):
|
|
"""`value` with every _PHI_KEYS key dropped, at any depth."""
|
|
if isinstance(value, dict):
|
|
return {k: _without_phi(v) for k, v in value.items()
|
|
if k not in _PHI_KEYS}
|
|
if isinstance(value, (list, tuple)):
|
|
return [_without_phi(v) for v in value]
|
|
return value
|
|
|
|
|
|
def safe_error(exc) -> str:
|
|
"""
|
|
An error for the log without the driver's message: SQL Server's
|
|
"String or binary data would be truncated" (Msg 2628) quotes the value,
|
|
which can be a patient's name. pyodbc puts the SQLSTATE in args[0].
|
|
"""
|
|
state = exc.args[0] if exc.args and isinstance(exc.args[0], str) else ""
|
|
if re.fullmatch(r"[0-9A-Z]{5}", state):
|
|
hint = _SQLSTATE_HINT.get(state, "database error")
|
|
return f"{type(exc).__name__} SQLSTATE {state} ({hint})"
|
|
return type(exc).__name__
|
|
|
|
|
|
def has_column(conn, column: str) -> bool:
|
|
"""True if the results table has `column`. False when it does not — or
|
|
the check itself fails: a column the table lacks must never cost the
|
|
run its rows."""
|
|
try:
|
|
row = conn.cursor().execute("SELECT COL_LENGTH(?, ?)",
|
|
[TABLE, column]).fetchone()
|
|
return bool(row) and row[0] is not None
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def insert_rows(conn, records: list, started_by: str = "") -> int:
|
|
"""One parameterized INSERT for every record, one commit. `started_by`
|
|
(EEWA-33) goes in only once tools/sql/create_tables.sql has added its
|
|
column: until then the rows are saved without it."""
|
|
if not records:
|
|
return 0
|
|
cols = list(DB_COLUMNS)
|
|
extra = []
|
|
if started_by and has_column(conn, "started_by"):
|
|
cols.append("started_by")
|
|
extra = [started_by[:50]]
|
|
sql = (f"INSERT INTO {TABLE} ({', '.join(cols)}) "
|
|
f"VALUES ({', '.join('?' for _ in cols)})")
|
|
cur = conn.cursor()
|
|
cur.executemany(sql, [[rec[c] for c in DB_COLUMNS] + extra
|
|
for rec in records])
|
|
conn.commit()
|
|
return len(records)
|
|
|
|
|
|
def save_summary(sheet: list, details: dict, *, run_at, run_mode,
|
|
conn_str: str, connect=None, started_by: str = ""):
|
|
"""
|
|
Save the rows of `sheet` (header row + rows, as `charge_summary.build`
|
|
returns it) that the run filled in. Returns the number saved, or None
|
|
when the database could not be written — never raises.
|
|
"""
|
|
try:
|
|
header = list(sheet[0])
|
|
records = []
|
|
for values in sheet[1:]:
|
|
row = dict(zip(header, values))
|
|
if not str(row.get("Category") or "").strip():
|
|
continue
|
|
claim = str(row.get("CLAIMS#") or "").strip()
|
|
records.append(to_record(row, run_at=run_at, run_mode=run_mode,
|
|
details=(details or {}).get(claim)))
|
|
if not records:
|
|
print("[RESULTS-DB] No claim rows to save")
|
|
return 0
|
|
if connect is None:
|
|
import pyodbc
|
|
connect = pyodbc.connect
|
|
conn = connect(conn_str, timeout=15)
|
|
try:
|
|
# Query timeout: a blocked insert raises (and is logged)
|
|
# instead of holding the run up.
|
|
conn.timeout = 30
|
|
saved = insert_rows(conn, records, started_by)
|
|
finally:
|
|
conn.close()
|
|
print(f"[RESULTS-DB] {saved} row(s) saved to {TABLE}")
|
|
return saved
|
|
except Exception as exc:
|
|
print(f"[RESULTS-DB] Could not save the charge summary to the "
|
|
f"database: {safe_error(exc)} — the Excel summary is not "
|
|
f"affected")
|
|
return None
|