Initial import: eCW Charge Validation bot and results Portal
Snapshot of the working project as of 2026-10-23 (local 2d4d4fc), one commit with no earlier history: the tracked files were de-identified first (synthetic claim ids and patient data only; see Context.md section 7). Credentials, run logs, screenshots, claim reports and client workbooks are gitignored and not included. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ManikandanandClaude Opus 5.5 committed
commit
ba15a574ab
205 files changed
+56772
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Portal-agnostic plumbing: browser, page helpers, logging, prompts, guard."""
|
||||
@@ -0,0 +1,351 @@
|
||||
"""
|
||||
Browser launch.
|
||||
|
||||
One line in here is the whole reason this bot can work at all:
|
||||
|
||||
kwargs["proxy"] = proxy # chromium.launch(proxy=…)
|
||||
|
||||
The proxy is set **per browser**, not on Windows. That dissolves a deadlock
|
||||
that makes logging into this account by hand impossible. Measured 2026-09-01:
|
||||
|
||||
| target | direct | through the proxy |
|
||||
|----------------------------|-----------|-------------------|
|
||||
| eCW portal | HTTP 400 | HTTP 200 |
|
||||
| emailed approval link | HTTP 400 | HTTP 200 |
|
||||
| webmail site (cPanel :2096)| HTTP 200 | 403, proxy refuses|
|
||||
| mailbox over IMAP :993 | connects | not proxied at all|
|
||||
|
||||
The portal needs the proxy; the webmail site is blocked *by* it. A human would
|
||||
have to toggle the Windows proxy mid-login. The bot never opens the webmail
|
||||
site: this browser gets the proxy, and `auth/mailbox.py` reads the mail with
|
||||
`imaplib` over a raw socket that ignores WinINET and HTTP(S)_PROXY entirely.
|
||||
|
||||
So the Windows system proxy must stay **off**
|
||||
(`HKCU:\\...\\Internet Settings\\ProxyEnable = 0`). `tools\\check_net.py`
|
||||
asserts that.
|
||||
|
||||
With `BROWSER_CDP_URL` set the bot does not launch Playwright's browser: it
|
||||
attaches to Google Chrome on that debug port (profile on D:, 127.0.0.1
|
||||
only), so the run can be watched with the Claude extension in that Chrome.
|
||||
No Chrome there when the run starts → the bot opens it (`start_chrome`,
|
||||
operator 2026-10-22: Start on the Portal is all it should take), and closes
|
||||
it again at the end (`CDP_CLOSE_AFTER_RUN`) — never a Chrome it did not
|
||||
open. The bot opens its own tab, clears eCW's cookies and storage when it
|
||||
attaches and again when it is done, and closes only its own tabs.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .. import settings as cfg
|
||||
from .page_utils import auto_dialog
|
||||
|
||||
|
||||
# Chrome pauses a window it thinks nobody can see. The bot's review and
|
||||
# per-claim boxes are Windows message boxes in front of it, and while one is
|
||||
# up Chrome stopped the claim window's fade-out half-way (live run
|
||||
# 2026-10-13: an HCFA window left at opacity 0.222, pointer-events none, in
|
||||
# the DOM after its Cancel) — the next claim found that "form still open",
|
||||
# and four Cancel clicks on it timed out (~45 s). These keep the page
|
||||
# running behind a box.
|
||||
_KEEP_RUNNING_BEHIND_A_BOX = [
|
||||
"--disable-backgrounding-occluded-windows",
|
||||
"--disable-renderer-backgrounding",
|
||||
"--disable-background-timer-throttling",
|
||||
]
|
||||
|
||||
|
||||
CHROME_ARGS = ["--disable-blink-features=AutomationControlled",
|
||||
"--start-maximized", *_KEEP_RUNNING_BEHIND_A_BOX]
|
||||
|
||||
|
||||
def launch(pw, incognito: bool = False):
|
||||
"""Launch a browser + context wired up with the dialog handler.
|
||||
|
||||
With BROWSER_CDP_URL set, attach to that Chrome instead. The incognito
|
||||
retry after a failed captcha still launches a fresh Chromium of its own:
|
||||
a clean, cookie-less browser is the point of that retry.
|
||||
"""
|
||||
if cfg.BROWSER_CDP_URL and not incognito:
|
||||
return _attach(pw, cfg.BROWSER_CDP_URL)
|
||||
|
||||
args = list(CHROME_ARGS)
|
||||
if incognito:
|
||||
args.append("--incognito")
|
||||
|
||||
kwargs = dict(headless=cfg.HEADLESS, args=args)
|
||||
proxy = cfg.proxy_settings()
|
||||
if proxy:
|
||||
kwargs["proxy"] = proxy
|
||||
print(f"[BROWSER] Proxy {proxy['server']}")
|
||||
if cfg.CHROME_PATH:
|
||||
kwargs["executable_path"] = cfg.CHROME_PATH
|
||||
print(f"[BROWSER] Chrome {cfg.CHROME_PATH}")
|
||||
try:
|
||||
cfg.DOWNLOADS_TMP.mkdir(parents=True, exist_ok=True)
|
||||
kwargs["downloads_path"] = str(cfg.DOWNLOADS_TMP)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
browser = pw.chromium.launch(**kwargs)
|
||||
context = browser.new_context(
|
||||
no_viewport=True, # use the real maximized window
|
||||
ignore_https_errors=True,
|
||||
accept_downloads=True,
|
||||
)
|
||||
context.on("page", lambda p: p.on("dialog", auto_dialog))
|
||||
page = context.new_page()
|
||||
page.on("dialog", auto_dialog)
|
||||
return browser, context, page
|
||||
|
||||
|
||||
# ── Attached Chrome (BROWSER_CDP_URL) ─────────────────────────────────────────
|
||||
|
||||
# The extension signs in and talks to these; the eCW proxy has no business
|
||||
# with them (the Windows proxy stays off, so direct works).
|
||||
PROXY_BYPASS = "claude.ai;*.claude.ai;*.anthropic.com"
|
||||
|
||||
# Debug-port URLs of the Chromes this process opened — the only ones it
|
||||
# closes when it is done.
|
||||
_STARTED = set()
|
||||
|
||||
# Started through a Python that exits at once: Chrome's parent is then
|
||||
# gone, so a Stop on the Portal (taskkill /T on the bot) never takes Chrome
|
||||
# with it.
|
||||
_SPAWN = ("import subprocess, sys; subprocess.Popen(sys.argv[1:], "
|
||||
"creationflags=0x00000008 | 0x00000200, close_fds=True, "
|
||||
"stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, "
|
||||
"stderr=subprocess.DEVNULL)")
|
||||
|
||||
|
||||
def find_chrome():
|
||||
"""Google Chrome's chrome.exe: CHROME_PATH if it is set, else the usual
|
||||
per-user and per-machine install folders."""
|
||||
candidates = [cfg.CHROME_PATH] if cfg.CHROME_PATH else []
|
||||
for base in ("LOCALAPPDATA", "PROGRAMFILES", "PROGRAMFILES(X86)"):
|
||||
if os.environ.get(base):
|
||||
candidates.append(os.path.join(os.environ[base], "Google", "Chrome",
|
||||
"Application", "chrome.exe"))
|
||||
for path in candidates:
|
||||
if path and Path(path).is_file():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def chrome_command(exe: str, port: int, profile: str, proxy) -> list:
|
||||
cmd = [exe, f"--remote-debugging-port={port}",
|
||||
"--remote-debugging-address=127.0.0.1",
|
||||
f"--user-data-dir={profile}",
|
||||
"--no-first-run", "--no-default-browser-check", *CHROME_ARGS]
|
||||
if proxy:
|
||||
cmd += [f"--proxy-server={proxy['server']}",
|
||||
f"--proxy-bypass-list={PROXY_BYPASS}"]
|
||||
return cmd
|
||||
|
||||
|
||||
def _cmdline_on_port(port: int):
|
||||
"""The command line of the process listening on `port`, or None."""
|
||||
ps = (f"$c = Get-NetTCPConnection -LocalPort {int(port)} -State Listen "
|
||||
f"-ErrorAction SilentlyContinue | Select-Object -First 1; "
|
||||
f"if ($c) {{ (Get-CimInstance Win32_Process -Filter "
|
||||
f"\"ProcessId=$($c.OwningProcess)\").CommandLine }}")
|
||||
try:
|
||||
out = subprocess.run(["powershell", "-NoProfile", "-Command", ps],
|
||||
capture_output=True, text=True,
|
||||
timeout=30).stdout.strip()
|
||||
except Exception:
|
||||
return None
|
||||
return out or None
|
||||
|
||||
|
||||
def chrome_proxy_problem(url: str):
|
||||
"""Why the Chrome on `url` must not be used for the eCW login, or None.
|
||||
|
||||
eCW answers only through the proxy. A Chrome on the port without it
|
||||
sends the login straight to eCW, which refuses it as "restricted" — and
|
||||
the guard then locks the bot out (2026-10-22: a test opened such a
|
||||
Chrome, and a Portal run attached to it). Checked on the Chrome's own
|
||||
command line; when that cannot be read, the run goes on."""
|
||||
proxy = cfg.proxy_settings()
|
||||
if not proxy:
|
||||
return None
|
||||
cmdline = _cmdline_on_port(urlsplit(url).port)
|
||||
if cmdline is None:
|
||||
print(f"[BROWSER] Could not read the command line of the Chrome at "
|
||||
f"{url} — its proxy is not checked")
|
||||
return None
|
||||
if f"--proxy-server={proxy['server']}" in cmdline:
|
||||
return None
|
||||
return (f"The Chrome at {url} was started without the eCW proxy "
|
||||
f"({proxy['server']}) — eCW would refuse the login. Close that "
|
||||
f"Chrome; the run opens its own")
|
||||
|
||||
|
||||
def start_chrome(url: str, extra_args=()) -> bool:
|
||||
"""Open Google Chrome on `url`'s debug port with CDP_PROFILE_DIR and the
|
||||
eCW proxy, outside this process's tree. True once it answers."""
|
||||
if cdp_ready(url):
|
||||
return True
|
||||
exe = find_chrome()
|
||||
if not exe:
|
||||
print("[BROWSER] Google Chrome not found — set CHROME_PATH in .env")
|
||||
return False
|
||||
Path(cfg.CDP_PROFILE_DIR).mkdir(parents=True, exist_ok=True)
|
||||
cmd = chrome_command(exe, urlsplit(url).port, cfg.CDP_PROFILE_DIR,
|
||||
cfg.proxy_settings()) + list(extra_args)
|
||||
subprocess.Popen([sys.executable, "-c", _SPAWN, *cmd],
|
||||
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
for _ in range(60):
|
||||
if cdp_ready(url):
|
||||
_STARTED.add(url)
|
||||
print(f"[BROWSER] Opened Chrome at {url} "
|
||||
f"(profile {cfg.CDP_PROFILE_DIR})")
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
print(f"[BROWSER] Started {exe}, but nothing answers at {url} — a "
|
||||
f"Chrome already open with that profile and no debug port? "
|
||||
f"Close it and start again")
|
||||
return False
|
||||
|
||||
|
||||
def cdp_ready(url: str) -> bool:
|
||||
"""True if a Chrome answers on its debug port at `url`."""
|
||||
try:
|
||||
with urllib.request.urlopen(url.rstrip("/") + "/json/version",
|
||||
timeout=3) as resp:
|
||||
return resp.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def ecw_sites(urls) -> set:
|
||||
"""eCW's registrable domains ("ecwcloud.com"), from its configured URLs.
|
||||
|
||||
eCW sets cookies on the parent domain as well as on the app host, so the
|
||||
clean-up goes by the domain, not the host.
|
||||
"""
|
||||
sites = set()
|
||||
for url in urls:
|
||||
host = urlsplit(url).hostname if url else None
|
||||
if host:
|
||||
sites.add(".".join(host.split(".")[-2:]))
|
||||
return sites
|
||||
|
||||
|
||||
def is_ecw_host(host: str, sites) -> bool:
|
||||
host = host.lstrip(".").lower()
|
||||
return any(host == s or host.endswith("." + s) for s in sites)
|
||||
|
||||
|
||||
def clear_ecw_data(context, page) -> None:
|
||||
"""Remove eCW's cookies and site storage from `context`, nothing else.
|
||||
|
||||
Only eCW's: the same profile holds claude.ai's sign-in, which is what
|
||||
keeps the extension connected between runs.
|
||||
"""
|
||||
sites = ecw_sites([cfg.ECW_URL, cfg.ECW_LANDING_URL])
|
||||
if not sites:
|
||||
return
|
||||
hosts = {urlsplit(u).hostname for u in (cfg.ECW_URL, cfg.ECW_LANDING_URL) if u}
|
||||
for p in context.pages:
|
||||
host = urlsplit(p.url).hostname
|
||||
if host and is_ecw_host(host, sites):
|
||||
hosts.add(host)
|
||||
for c in context.cookies():
|
||||
if is_ecw_host(c["domain"], sites):
|
||||
hosts.add(c["domain"].lstrip("."))
|
||||
context.clear_cookies(name=c["name"], domain=c["domain"],
|
||||
path=c["path"])
|
||||
# localStorage, IndexedDB, cache, service workers — per origin.
|
||||
cdp = context.new_cdp_session(page)
|
||||
try:
|
||||
for host in sorted(h for h in hosts if h):
|
||||
cdp.send("Storage.clearDataForOrigin",
|
||||
{"origin": f"https://{host}", "storageTypes": "all"})
|
||||
finally:
|
||||
cdp.detach()
|
||||
|
||||
|
||||
class _AttachedBrowser:
|
||||
"""What `launch()` returns as "browser" when attached to Chrome. Callers
|
||||
only ever `close()` it; here that means clear eCW's data, close the
|
||||
bot's tabs, and disconnect — and close Chrome too if this run opened it
|
||||
(CDP_CLOSE_AFTER_RUN), so its debug port is not left open."""
|
||||
|
||||
def __init__(self, browser, context, page, theirs, url=""):
|
||||
self._browser, self._context, self._page = browser, context, page
|
||||
self._theirs = theirs # tabs that were open before the bot
|
||||
self._url = url
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
clear_ecw_data(self._context, self._page)
|
||||
print("[BROWSER] eCW cookies and storage cleared")
|
||||
except Exception as exc:
|
||||
# Cleared again when the next run attaches.
|
||||
print(f"[BROWSER] Could not clear eCW data: {exc}")
|
||||
if self._url in _STARTED and cfg.CDP_CLOSE_AFTER_RUN:
|
||||
try:
|
||||
self._browser.new_browser_cdp_session().send("Browser.close")
|
||||
print("[BROWSER] Closed the Chrome this run opened")
|
||||
except Exception:
|
||||
pass # the connection goes with it
|
||||
_STARTED.discard(self._url)
|
||||
try:
|
||||
self._browser.close()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
try:
|
||||
mine = [p for p in self._context.pages if p not in self._theirs]
|
||||
if len(mine) == len(self._context.pages) and mine:
|
||||
# Closing Chrome's last tab closes Chrome. Leave one, blank.
|
||||
mine[0].goto("about:blank")
|
||||
mine = mine[1:]
|
||||
for p in mine:
|
||||
p.close()
|
||||
finally:
|
||||
self._browser.close() # connected, not launched: disconnects
|
||||
|
||||
|
||||
def _is_blank(url: str) -> bool:
|
||||
"""A tab showing nothing: about:blank or Chrome's New Tab page."""
|
||||
return (url or "about:blank").split("?")[0].rstrip("/") in (
|
||||
"about:blank", "chrome://newtab", "chrome://new-tab-page",
|
||||
"chrome-search://local-ntp/local-ntp.html")
|
||||
|
||||
|
||||
def _attach(pw, url: str):
|
||||
"""Attach to the operator's Chrome; same (browser, context, page) shape."""
|
||||
browser = pw.chromium.connect_over_cdp(url)
|
||||
print(f"[BROWSER] Attached to Chrome at {url} (its own proxy and profile)")
|
||||
# Chrome's own context: the one the extension can see. It cannot take
|
||||
# new_context()'s options; a real window already has no viewport.
|
||||
context = browser.contexts[0]
|
||||
theirs = list(context.pages)
|
||||
context.on("page", lambda p: p.on("dialog", auto_dialog))
|
||||
page = context.new_page()
|
||||
page.on("dialog", auto_dialog)
|
||||
if url in _STARTED:
|
||||
# The Chrome this run opened shows eCW only (operator, 2026-10-23):
|
||||
# its empty first tab goes, now that the bot's tab keeps Chrome open.
|
||||
# The operator's own Chrome keeps every tab it had.
|
||||
for p in [p for p in theirs if _is_blank(p.url)]:
|
||||
try:
|
||||
p.close()
|
||||
theirs.remove(p)
|
||||
except Exception as exc:
|
||||
print(f"[BROWSER] Could not close the empty first tab: {exc}")
|
||||
# A run killed half-way (Stop on the results page, a crash) never got to
|
||||
# close(), so its session may still be here.
|
||||
clear_ecw_data(context, page)
|
||||
page.bring_to_front()
|
||||
return (_AttachedBrowser(browser, context, page, theirs, url), context,
|
||||
page)
|
||||
@@ -0,0 +1,462 @@
|
||||
r"""
|
||||
The Charge Summary sheet: one row per claim in the run's report, in the
|
||||
operator's own 24-column format ("Charge Summary File Format.xlsx",
|
||||
2026-10-01), written to Output\Charge Summary_<date>.xlsx after every
|
||||
claim of a claims run, so the sheet grows claim by claim — or Output\Charge Summary_TEST_<date>.xlsx when the claim
|
||||
ids came from a TEST_CLAIM_IDS_FILE sheet (see file_name()).
|
||||
|
||||
Columns A-R come from the downloaded claim report (15 of them by the same
|
||||
header), plus three the report does not carry:
|
||||
|
||||
S.No 1, 2, 3 ... in the report's order
|
||||
Group Name payer_groups.json, blank when the payer is unmapped
|
||||
Pmts/Refunds PMTS/ADJS - Adjustment (the format's own sample rows all
|
||||
have PMTS/ADJS = Pmts/Refunds + Adjustment)
|
||||
|
||||
The report's CLAIMTYPE column is dropped. Columns S-X are what this run
|
||||
did with the claim, and stay blank for a claim it never opened (not found,
|
||||
past CLAIM_PROCESS_LIMIT, or OPEN_CLAIM_FORM off). A Dental claim is read
|
||||
and run through the conditions like any other since 2026-10-12, so its row
|
||||
carries the condition comments and its high risk / sliding fee findings:
|
||||
|
||||
Recevied date / Completed On the run date
|
||||
Assigned To SUMMARY_ASSIGNED_TO (default "eCW Bot")
|
||||
Comments one line per finding, then one per review
|
||||
reason, "; "-joined
|
||||
Current Status / Category see claim_outcome() — "Pass", "Fail", or
|
||||
"Review" / "Needs manual review"
|
||||
|
||||
"Review" is the third Category (review items D2/St1, 2026-10-05). Before it,
|
||||
anything the bot did not actually check — a condition that raised or
|
||||
declined, APPLY_CORRECTIONS=0, a progress note never read, a form that
|
||||
would not read — came out "Clean claims / Ready to Submit / Pass": the
|
||||
sheet claimed a check that never happened.
|
||||
|
||||
Header spellings ("Recevied date", "Assigned To ") are the format's own,
|
||||
kept exactly so the sheet lines up with the team's existing ones.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from . import payer_groups, spreadsheet
|
||||
from .text import first_line
|
||||
|
||||
HEADERS = [
|
||||
"S.No", "CLAIMS#", "SERVICE DATE", "PVDR", "PATIENT", "PAYER",
|
||||
"Group Name", "STATUS", "CHARGES", "PMTS/ADJS", "Pmts/Refunds",
|
||||
"Adjustment", "WithHeld", "Balance", "Visit Type", "Account Number",
|
||||
"Provider Name", "Guarantor Name", "Recevied date", "Assigned To ",
|
||||
"Completed On", "Comments", "Current Status", "Category",
|
||||
]
|
||||
|
||||
# Copied straight from the downloaded report, by header name.
|
||||
_FROM_REPORT = [
|
||||
"CLAIMS#", "SERVICE DATE", "PVDR", "PATIENT", "PAYER", "STATUS",
|
||||
"CHARGES", "PMTS/ADJS", "Adjustment", "WithHeld", "Balance",
|
||||
"Visit Type", "Account Number", "Provider Name", "Guarantor Name",
|
||||
]
|
||||
|
||||
# Operator, 2026-10-06: the team's wording for a claim with no findings.
|
||||
CLEAN = ("Clean claims (claims that have no errors, placed in Pending with "
|
||||
"Errors)")
|
||||
|
||||
# Comment per fired condition, worded like the format's own samples.
|
||||
COMMENTS = {
|
||||
"depression_dx_pointer": (
|
||||
"ICD pointer issue (Z13.31 - Depression Diagnosis tagged under E/M "
|
||||
"Procedure, Need to Tag under G8510 only )"),
|
||||
# The two telehealth conditions word their own comment from what
|
||||
# actually happened (`summary_comment`); these fixed texts are only the
|
||||
# fallback when that hook raises, so they claim nothing that may not
|
||||
# have happened — not "G2025 added" (it can fail), not "93 or 95"
|
||||
# (Medicaid is GT), not "unchecked" (the UB form only zeroes fees)
|
||||
# (final review, 2026-10-05).
|
||||
"telehealth": "Telehealth (POS / modifier updated — see the log)",
|
||||
"medicare_telehealth": (
|
||||
"Medicare Telehealth (POS / modifier and G2025 — see the log for "
|
||||
"what changed)"),
|
||||
"j2930_j2001_update": (
|
||||
"Invalid CPT codes (J2930/J2001 - checked that should be unchecked)"),
|
||||
"dummy_codes": (
|
||||
"Invalid CPT codes (DCONS/Dscprit - checked that should be "
|
||||
"unchecked)"),
|
||||
"dental_dummy_codes": (
|
||||
"Invalid CPT codes (DPIMP/DFIMP dental dummy code - checked that should "
|
||||
"be unchecked)"),
|
||||
# Fallback only: the condition words its own comment (codes, payers,
|
||||
# whether the Claim Status was set to Patient — EEWA-27, 2026-10-21).
|
||||
"dental_payer_check": (
|
||||
"Dental codes on a payer other than DentaQuest / Delta Dental / "
|
||||
"CHP+ Dental - Claim Status to Patient"),
|
||||
"medicaid_uncheck_cpts": (
|
||||
"CPT not billable to Medicaid (15851/15852/73521 - unchecked, fee 0)"),
|
||||
"medicare_billing_screening": (
|
||||
"Screening codes not billed to Medicare (G8510/G8431)"),
|
||||
# Fallback only: the condition words its own comment (which codes).
|
||||
"medicare_vaccine_a6": (
|
||||
"Condition code A6 added (G0008/G0009 on Medicare)"),
|
||||
# Not read today: the condition never changes the claim; its Review
|
||||
# reason (which code, billed when, on which claim) reaches the sheet.
|
||||
"medicare_wellness_history": (
|
||||
"G0402/G0438/G0439 billed too often - check the history"),
|
||||
# Fallback only: the condition words its own comment (line, ICD).
|
||||
"medicaid_pregnancy_th": (
|
||||
"TH modifier added (pregnancy visit on Medicaid)"),
|
||||
"vaccine_admin_codes": "CPT Code Missing ( vaccine admin code added )",
|
||||
# Fallback only: the condition words its own comment (96372 added or not).
|
||||
"injection_admin_code": "CPT Code Missing (injection admin code 96372)",
|
||||
# Fallback only: the condition words its own comment by the path it
|
||||
# took (BH payer: modifier removed; any other payer: line unchecked).
|
||||
"bh_h_code_modifier": "H code corrected (#12) — see the log",
|
||||
"zero_fee_uncheck": "Zero billed fee CPT unchecked",
|
||||
"priced_line_check": "Billed CPT checked",
|
||||
}
|
||||
|
||||
HIGH_RISK = "High risk claim (submit through DentaQuest web portal)"
|
||||
|
||||
|
||||
# A progress note in one of these states was never read (pages/claim_form
|
||||
# .open_progress_notes): no button, a disabled one, or a click that opened
|
||||
# nothing — or, "skipped", not opened at all because the form never finished
|
||||
# loading (the flow, final review 2026-10-05). Telehealth and Medicare
|
||||
# telehealth decide from the note's text, so on such a claim they were not
|
||||
# checked at all — not checked, not clean.
|
||||
_NOTE_NOT_READ = ("nothing", "absent", "disabled", "skipped")
|
||||
|
||||
# Leads a Review row's Comments when the claim WAS written to eCW: "Needs
|
||||
# manual review" alone would let the reviewer treat it as untouched and redo
|
||||
# or undo edits already on the live claim (final review, 2026-10-05).
|
||||
SAVED_NOTE = "Corrections SAVED to eCW"
|
||||
|
||||
# The operator approved the save, but eCW never confirmed it: save_claim()
|
||||
# answers False both when OK could not be clicked and when it WAS clicked
|
||||
# and the form did not hide in time — so whether the claim was written is
|
||||
# unknown, and "Corrections pending (not saved)" could be untrue (final
|
||||
# review, 2026-10-05). The flow sets `_conditions["save_unconfirmed"]`.
|
||||
SAVE_UNCONFIRMED = ("Save approved but not confirmed by eCW — check whether "
|
||||
"the correction was written")
|
||||
|
||||
|
||||
def is_dental(data: dict) -> bool:
|
||||
"""
|
||||
A Dental Claim screen read: the flow's `_dental_checks` results, or
|
||||
(since 2026-10-12, Dental claims run the conditions) a read whose
|
||||
`screen` is "dental". Public because the flow's per-claim box asks the
|
||||
same question (review item D9) — one answer, not two that can drift
|
||||
apart.
|
||||
"""
|
||||
return ("high_risk" in data or "sliding_fee" in data
|
||||
or data.get("screen") == "dental")
|
||||
|
||||
|
||||
def review_reasons(data: dict) -> list:
|
||||
"""
|
||||
Every reason a human must look at this claim — empty when the bot
|
||||
really checked everything it was meant to. Order: why the claim could
|
||||
not be evaluated first, then what the conditions reported, then Dental.
|
||||
|
||||
Public because the flow's per-claim message box lists these same
|
||||
reasons as its "Needs review" lines (review item D9, 2026-10-05): the
|
||||
box and the sheet must give the same account of one claim.
|
||||
"""
|
||||
if data.get("_search_error"):
|
||||
# The search raised (browser gone, a timeout): the claim was never
|
||||
# found or opened, which is not "not found" (final review,
|
||||
# 2026-10-05).
|
||||
return [f"Bot could not search the claim ({data['_search_error']})"]
|
||||
if data.get("_error"):
|
||||
if data.get("_conditions") is not None:
|
||||
# Raised AFTER the conditions ran (the flow keeps their account
|
||||
# on the error dict): the claim was read and may have been
|
||||
# changed, so "could not read" would misstate it. What the
|
||||
# conditions themselves reported still follows.
|
||||
cond = data["_conditions"] or {}
|
||||
return ([f"Bot failed after the conditions ran "
|
||||
f"({data['_error']})"]
|
||||
+ [f"{f.get('name')} failed: {f.get('error')}"
|
||||
for f in cond.get("failed") or []]
|
||||
+ [f"{r.get('name')}: {r.get('reason')}"
|
||||
for r in cond.get("review") or []])
|
||||
# The read itself failed: `data` is only {"claim_no", "_error"}, so
|
||||
# every trigger below would just restate "nothing was read".
|
||||
return [f"Bot could not read the claim ({data['_error']})"]
|
||||
|
||||
reasons = []
|
||||
if data.get("_not_evaluated"):
|
||||
reasons.append(data["_not_evaluated"])
|
||||
|
||||
cond = data.get("_conditions")
|
||||
if not is_dental(data) or data.get("screen") == "dental":
|
||||
# Every claim that was read for the conditions: UB/HCFA, and since
|
||||
# 2026-10-12 a Dental claim too (`screen` "dental"). Only a bare
|
||||
# Dental-checks result (high risk / sliding fee alone, no read)
|
||||
# skips these — nothing here applies to it.
|
||||
if cond is None and not data.get("_not_evaluated"):
|
||||
# No `_conditions` on a claim that loaded = run_all never ran.
|
||||
# A claim that did NOT load has none either — the flow skips the
|
||||
# conditions on purpose and `_not_evaluated` already says why —
|
||||
# so calling that "APPLY_CORRECTIONS=0" would be untrue.
|
||||
reasons.append("Conditions not run (APPLY_CORRECTIONS=0)")
|
||||
note = data.get("progress_notes")
|
||||
state = None
|
||||
if not isinstance(note, dict):
|
||||
state = "not opened" # OPEN_PROGRESS_NOTES off, or never got there
|
||||
elif note.get("error"):
|
||||
state = f"error: {first_line(note['error'])}"
|
||||
elif note.get("opened") in _NOTE_NOT_READ:
|
||||
state = note["opened"]
|
||||
if state:
|
||||
reasons.append(f"Progress note not read ({state}) — telehealth "
|
||||
f"not checked")
|
||||
elif note.get("partial"):
|
||||
# Read, but its sections never settled before the deadline
|
||||
# (pages/claim_form._wait_for_note, unit C). With 0 sections only
|
||||
# the header arrived — no Procedure Codes, so telehealth saw
|
||||
# nothing: the D8 false Clean. With some sections the Procedure
|
||||
# Codes may still be missing. Either way "clean" is unproven
|
||||
# (review of unit D, 2026-10-05, finding 1).
|
||||
n = note.get("partial_sections")
|
||||
if n == 0:
|
||||
reasons.append("Progress note partial (0 sections) — "
|
||||
"telehealth not checked")
|
||||
else:
|
||||
# No "?" for an unknown count: these reasons are also the
|
||||
# client's on-screen "Needs review" lines (review item D9).
|
||||
count = ("section count unknown" if n is None
|
||||
else f"{n} section(s)")
|
||||
reasons.append(f"Progress note may be partial ({count}) — "
|
||||
f"telehealth may not be fully checked")
|
||||
if data.get("cpt") == []:
|
||||
# Every CPT condition reads these rows; none of them saw one
|
||||
# (review item D7: a half-loaded HCFA form read 0 and went Pass).
|
||||
reasons.append("No CPT rows were read")
|
||||
elif data.get("cpt") and data.get("icd") == []:
|
||||
# CPT rows but not one diagnosis row: depression_dx_pointer
|
||||
# (the Z13.31 pointer) reads the ICD list and saw nothing, so it
|
||||
# passed without really running. A billed claim always carries
|
||||
# a diagnosis; an empty list is a read that did not land — e.g.
|
||||
# a partly rendered UB form (final review, 2026-10-05).
|
||||
reasons.append("No ICD rows were read")
|
||||
|
||||
cond = cond or {}
|
||||
for f in cond.get("failed") or []:
|
||||
reasons.append(f"{f.get('name')} failed: {f.get('error')}")
|
||||
for r in cond.get("review") or []:
|
||||
reasons.append(f"{r.get('name')}: {r.get('reason')}")
|
||||
if cond.get("save_unconfirmed"):
|
||||
reasons.append(SAVE_UNCONFIRMED)
|
||||
|
||||
if "high_risk" in data and \
|
||||
(data["high_risk"] or {}).get("high_risk") is None:
|
||||
reasons.append("High risk: Billing Notes could not be read")
|
||||
sf = data.get("sliding_fee") or {}
|
||||
if (sf.get("verdict") or "").lower() == "unknown":
|
||||
reasons.append(f"Sliding fee UNKNOWN ({sf.get('reason', '')})")
|
||||
return reasons
|
||||
|
||||
|
||||
def claim_outcome(found: bool, data: dict) -> dict:
|
||||
"""
|
||||
Columns V-X for one claim, from what `_read_claim_form` returned.
|
||||
|
||||
* not found / never opened -> {} (S-X left blank)
|
||||
* Review -> Current Status "Needs manual review": the bot did not, or
|
||||
could not, check everything (see `review_reasons`) — the search
|
||||
raised (`_search_error`), the form would not read (`_error`) or
|
||||
never finished loading (`_not_evaluated`);
|
||||
on UB/HCFA the conditions not run, the progress note not read (or
|
||||
read but marked partial), or zero CPT rows read; a condition that raised or asked for a human
|
||||
(`_conditions` "failed" / "review"), or an approved save eCW did
|
||||
not confirm (SAVE_UNCONFIRMED); Dental Billing Notes unreadable
|
||||
or a sliding fee UNKNOWN. Comments = every finding, then every
|
||||
reason, led by SAVED_NOTE when the correction was saved to eCW.
|
||||
Review wins over Fail: a fired condition on a claim whose note was
|
||||
never read is still a claim a human has to look at.
|
||||
* Fail -> a condition changed something, HIGH RISK, or a sliding fee
|
||||
EXPIRED / NOT_STARTED. A condition's comment is its own
|
||||
`summary_comment()` wording when it has one, else COMMENTS[name].
|
||||
Current Status "Ready to Submit" if the correction was saved, else
|
||||
"Corrections pending (not saved)" — with SAVE_CORRECTIONS=0
|
||||
nothing reaches eCW, and the sheet must not say otherwise
|
||||
* Pass -> CLEAN, "Ready to Submit" (a Dental sliding fee NONE is
|
||||
noted after it, but is not a finding)
|
||||
"""
|
||||
if data and data.get("_search_error"):
|
||||
# Searched, and the search itself raised: Review, not the blank row
|
||||
# of a claim that is really not in the queue.
|
||||
return {"Comments": "; ".join(review_reasons(data)),
|
||||
"Current Status": "Needs manual review", "Category": "Review"}
|
||||
if not found or not data:
|
||||
return {}
|
||||
|
||||
comments, failed = [], False
|
||||
cond = data.get("_conditions") or {}
|
||||
worded = cond.get("comments") or {}
|
||||
for name in cond.get("fired") or []:
|
||||
comments.append(worded.get(name) or COMMENTS.get(name, name))
|
||||
failed = True
|
||||
saved = bool(cond.get("saved"))
|
||||
|
||||
hr = data.get("high_risk")
|
||||
if hr and hr.get("high_risk"):
|
||||
comments.append(HIGH_RISK)
|
||||
failed = True
|
||||
sf = data.get("sliding_fee")
|
||||
if sf:
|
||||
verdict = (sf.get("verdict") or "").lower()
|
||||
if verdict in ("expired", "not_started"):
|
||||
comments.append(f"Sliding fee {verdict.upper().replace('_', ' ')}"
|
||||
f" ({sf.get('reason', '')})")
|
||||
failed = True
|
||||
elif verdict == "none":
|
||||
# Noted, not a finding. "unknown" is a review reason instead.
|
||||
comments.append(f"Sliding fee NONE ({sf.get('reason', '')})")
|
||||
|
||||
reasons = review_reasons(data)
|
||||
if reasons:
|
||||
# A save that happened is never hidden behind "Needs manual review"
|
||||
# (final review, 2026-10-05): it leads the Comments.
|
||||
lead = [SAVED_NOTE] if saved else []
|
||||
return {"Comments": "; ".join(lead + comments + reasons),
|
||||
"Current Status": "Needs manual review", "Category": "Review"}
|
||||
if not failed:
|
||||
return {"Comments": "; ".join([CLEAN] + comments),
|
||||
"Current Status": "Ready to Submit", "Category": "Pass"}
|
||||
return {"Comments": "; ".join(comments),
|
||||
"Current Status": ("Ready to Submit" if saved
|
||||
else "Corrections pending (not saved)"),
|
||||
"Category": "Fail"}
|
||||
|
||||
|
||||
def _money(text):
|
||||
try:
|
||||
return float(str(text).replace(",", "").replace("$", "").strip() or 0)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def build_rows(report_rows: list, header_row: int, claim_col: int,
|
||||
outcomes: dict, run_date: str, assigned_to: str,
|
||||
mapping: dict) -> list:
|
||||
"""
|
||||
Header + one row per claim. `report_rows` is the report as read;
|
||||
`header_row`/`claim_col` are where claim_file found the claim-id
|
||||
header and column (header_row -1: the file has no header row).
|
||||
`outcomes` maps claim id -> claim_outcome(). A claim listed twice in
|
||||
the report (eCW does, for a multi-line claim) gets one row. A report
|
||||
missing a column (a hand-made test sheet with only claim ids) gets it
|
||||
blank.
|
||||
"""
|
||||
header = ([str(h or "").strip() for h in report_rows[header_row]]
|
||||
if header_row >= 0 else [])
|
||||
ix = {h: i for i, h in enumerate(header)}
|
||||
|
||||
out, seen = [HEADERS], set()
|
||||
for row in report_rows[header_row + 1:]:
|
||||
get = lambda h: (str(row[ix[h]]).strip()
|
||||
if h in ix and ix[h] < len(row) else "")
|
||||
claim = str(row[claim_col]).strip() if claim_col < len(row) else ""
|
||||
if not claim or claim in seen:
|
||||
continue
|
||||
seen.add(claim)
|
||||
|
||||
vals = {h: get(h) for h in _FROM_REPORT}
|
||||
vals["CLAIMS#"] = claim
|
||||
vals["S.No"] = len(seen)
|
||||
vals["Group Name"] = payer_groups.classify(vals["PAYER"], mapping) or ""
|
||||
pa, adj = _money(vals["PMTS/ADJS"]), _money(vals["Adjustment"])
|
||||
vals["Pmts/Refunds"] = (f"{pa - adj:,.2f}" if "PMTS/ADJS" in ix
|
||||
and pa is not None and adj is not None else "")
|
||||
done = outcomes.get(claim) or {}
|
||||
if done:
|
||||
vals.update(done)
|
||||
vals["Recevied date"] = run_date
|
||||
vals["Completed On"] = run_date
|
||||
vals["Assigned To "] = assigned_to
|
||||
out.append([vals.get(h, "") for h in HEADERS])
|
||||
return out
|
||||
|
||||
|
||||
def file_name(today: str, test_mode: bool = False) -> str:
|
||||
"""
|
||||
`Charge Summary_<today>.xlsx`, or `Charge Summary_TEST_<today>.xlsx` for
|
||||
a run whose claim ids came from TEST_CLAIM_IDS_FILE.
|
||||
|
||||
Test runs get their own name (review item D1): a test sheet is a hand
|
||||
list of a few claim ids, its summary is mostly blank columns, and under
|
||||
the live name it pushed that day's real summary aside to `__HHMMSS` —
|
||||
so the file the team opens by its usual name was the test one. One
|
||||
function, so the flow's "[TEST MODE] the summary goes to ..." line names
|
||||
exactly the file write() saves.
|
||||
"""
|
||||
return (f"Charge Summary_TEST_{today}.xlsx" if test_mode
|
||||
else f"Charge Summary_{today}.xlsx")
|
||||
|
||||
|
||||
# The last report read: (path, mtime_ns, size) -> (rows, header_row,
|
||||
# claim_col). The flow builds the sheet after every claim, for the Excel
|
||||
# and again for the database, from the same unchanged report — reading it
|
||||
# once is enough (review, 2026-10-12). A changed file is read again.
|
||||
_REPORT_CACHE = {}
|
||||
|
||||
|
||||
def _read_report(report_path):
|
||||
# claim_file's own header/column finder, so the summary reads the
|
||||
# report exactly the way the claim ids were read from it.
|
||||
from ..claim_file import _find_column
|
||||
path = Path(report_path)
|
||||
stat = path.stat()
|
||||
key = (str(path.resolve()), stat.st_mtime_ns, stat.st_size)
|
||||
if key not in _REPORT_CACHE:
|
||||
rows = spreadsheet.read_rows(path)
|
||||
_REPORT_CACHE.clear()
|
||||
_REPORT_CACHE[key] = (rows, *_find_column(rows))
|
||||
return _REPORT_CACHE[key]
|
||||
|
||||
|
||||
def build(report_path, outcomes: dict, assigned_to: str) -> list:
|
||||
"""
|
||||
The summary sheet (header row + one row per claim) for this report and
|
||||
these outcomes — what `write()` saves to Excel, and what
|
||||
`core/results_db.py` saves to SQL Server.
|
||||
"""
|
||||
rows, header_row, claim_col = _read_report(report_path)
|
||||
run_date = datetime.now().strftime("%m/%d/%Y")
|
||||
return build_rows(rows, header_row, claim_col, outcomes, run_date,
|
||||
assigned_to, payer_groups.load_mapping())
|
||||
|
||||
|
||||
def write(report_path, outcomes: dict, out_dir, today: str,
|
||||
assigned_to: str, *, test_mode: bool = False, into=None) -> Path:
|
||||
"""
|
||||
Read the report, build the sheet, save it in `out_dir` as file_name()
|
||||
— `Charge Summary_<today>.xlsx`, or `Charge Summary_TEST_<today>.xlsx`
|
||||
with `test_mode` (claim ids from TEST_CLAIM_IDS_FILE, not the live queue).
|
||||
|
||||
A same-day file is kept aside, never overwritten (operators edit it); if
|
||||
it is open in Excel, this run's sheet is saved under a `__HHMMSS` name
|
||||
next to it instead of being lost (spreadsheet.write_xlsx_keep_old).
|
||||
Returns the path actually written — report that one, not the usual name.
|
||||
|
||||
`into`: the file this run's earlier claims were written to (the flow
|
||||
writes after every claim, operator 2026-10-12). It is this run's own
|
||||
file, so it is rewritten in place — no `__HHMMSS` copy per claim. If it
|
||||
is open in Excel by now, the sheet goes to a new `__HHMMSS` name, which
|
||||
the caller then writes into for the rest of the run.
|
||||
"""
|
||||
sheet = build(report_path, outcomes, assigned_to)
|
||||
out_path = Path(out_dir) / file_name(today, test_mode)
|
||||
if into is None:
|
||||
return spreadsheet.write_xlsx_keep_old(out_path, sheet, "[SUMMARY]")
|
||||
into = Path(into)
|
||||
try:
|
||||
spreadsheet.write_xlsx(into, sheet)
|
||||
return into
|
||||
except OSError as exc:
|
||||
fallback = spreadsheet._first_free_name(
|
||||
out_path, datetime.now().strftime("%H%M%S"))
|
||||
spreadsheet.write_xlsx(fallback, sheet)
|
||||
print(f"[SUMMARY] {into.name} is open or locked ({first_line(exc)}) "
|
||||
f"— this run's summary continues in {fallback.name}")
|
||||
return fallback
|
||||
@@ -0,0 +1,67 @@
|
||||
r"""
|
||||
Fallback selectors (2026-10-21): the selectors a run cannot do without carry
|
||||
one or two alternatives, so a renamed id does not stop the run.
|
||||
|
||||
Each one is a CHAIN — the primary first, then alternatives taken from
|
||||
attributes seen in the live DOM (ng-click, ng-model, type / value), never
|
||||
guessed. The selector constant the code uses is the chain joined as a CSS
|
||||
selector list ("primary, alternative, ..."), so every existing call site —
|
||||
`page.click`, `locator(...)`, `:has(...)`, the JS `querySelectorAll` ones —
|
||||
gets the fallback with no change. The alternatives are plain CSS for that
|
||||
reason (no Playwright-only `:has-text` / `text=`).
|
||||
|
||||
What a list cannot say is WHICH one matched. `report()` does: where the
|
||||
primary is gone and an alternative stands in, it logs
|
||||
|
||||
[SELECTOR] fallback: Lookup button — #btnclaimlookup not found, matched
|
||||
[ng-click="getClaimsListFromLookup('lookup')"]; update selectors/...
|
||||
|
||||
so the primary is fixed before the alternative breaks too. The pre-flight
|
||||
check (pages/preflight.py) and the login call it, passing the chains
|
||||
registered in selectors/fallbacks.py.
|
||||
"""
|
||||
|
||||
from . import run_events
|
||||
|
||||
# Names already reported this run: one log line each, not one per claim.
|
||||
_reported = set()
|
||||
|
||||
|
||||
def reset() -> None:
|
||||
_reported.clear()
|
||||
|
||||
|
||||
def _shown(page, selector: str) -> bool:
|
||||
return page.locator(selector).filter(visible=True).count() > 0
|
||||
|
||||
|
||||
def which(page, chain):
|
||||
"""Index of the first entry in `chain` shown on the page; None if none."""
|
||||
for i, selector in enumerate(chain):
|
||||
if _shown(page, selector):
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
def report(page, names, chains) -> list:
|
||||
"""Log each chain in `names` whose primary is gone but an alternative
|
||||
is shown; the names so reported (each once a run). Never raises.
|
||||
`chains` is `selectors/fallbacks.CHAINS` (core imports no selectors)."""
|
||||
out = []
|
||||
for name in names:
|
||||
if name in _reported:
|
||||
continue
|
||||
module, chain = chains[name]
|
||||
try:
|
||||
i = which(page, chain)
|
||||
except Exception:
|
||||
continue
|
||||
if i:
|
||||
_reported.add(name)
|
||||
out.append(name)
|
||||
print(f"[SELECTOR] fallback: {name} — {chain[0]} not found, "
|
||||
f"matched {chain[i]}; update "
|
||||
f"src/ecw_bot/selectors/{module}.py")
|
||||
run_events.warning("fallback", f"{name}: {chain[0]} not found, "
|
||||
f"matched {chain[i]}")
|
||||
return out
|
||||
@@ -0,0 +1,201 @@
|
||||
"""
|
||||
Login attempt guard.
|
||||
|
||||
A courtesy to the account, not a security boundary: attempts are recorded to
|
||||
`Logs\\attempts.json` and refused when they come too close together or in a
|
||||
burst. eCW does stop mailing approval links after a run of failed logins, and
|
||||
nothing in the code otherwise knew how often it had been run. `ECW_FORCE=1`
|
||||
skips every check here.
|
||||
|
||||
Note on the restriction page — the cause was initially misread as rate
|
||||
limiting. It is an IP check (see `config.TXT_RESTRICTED`): the same link answers
|
||||
400 unproxied and 200 through the proxy. So a restriction the bot walks into
|
||||
means "the browser lost the proxy" far more often than "the account is locked",
|
||||
which is why `record_restricted()` does not impose a long cool-off.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
from .. import settings as cfg
|
||||
|
||||
STATE_PATH = cfg.LOGS / "attempts.json"
|
||||
|
||||
# A restriction seen on the page pauses the bot for this long. Deliberately
|
||||
# short: the usual cause is a proxy problem, which no amount of waiting fixes,
|
||||
# and a long lockout would just block the run that has the proxy working again.
|
||||
# Long enough to stop a retry loop, short enough to stay out of the way.
|
||||
RESTRICTED_LOCKOUT = timedelta(minutes=5)
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _parse(raw):
|
||||
try:
|
||||
return datetime.fromisoformat(raw) if raw else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _read() -> dict:
|
||||
try:
|
||||
return json.loads(STATE_PATH.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _write(state: dict) -> None:
|
||||
try:
|
||||
cfg.LOGS.mkdir(parents=True, exist_ok=True)
|
||||
STATE_PATH.write_text(json.dumps(state, indent=2), encoding="utf-8")
|
||||
except Exception as exc:
|
||||
print(f"[GUARD] Could not write {STATE_PATH}: {exc}")
|
||||
|
||||
|
||||
def _forced() -> bool:
|
||||
return os.environ.get("ECW_FORCE", "").strip().lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def check() -> str:
|
||||
"""Return a reason to refuse this attempt, or "" to go ahead."""
|
||||
if _forced():
|
||||
print("[GUARD] ECW_FORCE set — skipping the cooldown check")
|
||||
return ""
|
||||
|
||||
state = _read()
|
||||
now = _now()
|
||||
|
||||
locked = _parse(state.get("locked_until"))
|
||||
if locked and now < locked:
|
||||
mins = (locked - now).total_seconds() / 60
|
||||
return (f"eCW restricted this account at "
|
||||
f"{state.get('restricted_at', '?')[:19]} UTC. "
|
||||
f"Waiting another {mins:.0f} min before trying again — "
|
||||
f"more attempts extend the restriction. "
|
||||
f"Set ECW_FORCE=1 to override.")
|
||||
|
||||
last = _parse(state.get("last_attempt"))
|
||||
if last:
|
||||
elapsed = (now - last).total_seconds()
|
||||
if elapsed < cfg.LOGIN_COOLDOWN_SECONDS:
|
||||
wait = cfg.LOGIN_COOLDOWN_SECONDS - elapsed
|
||||
return (f"Last login attempt was {elapsed:.0f}s ago. "
|
||||
f"Waiting {wait:.0f}s more — rapid retries are what get the "
|
||||
f"account restricted. Set ECW_FORCE=1 to override.")
|
||||
|
||||
# Recent-attempt burst check: eCW tolerates the occasional retry, not a run
|
||||
# of them.
|
||||
window = now - timedelta(hours=1)
|
||||
recent = [d for d in (_parse(x) for x in state.get("recent", [])) if d and d > window]
|
||||
if len(recent) >= cfg.LOGIN_MAX_PER_HOUR:
|
||||
# Say WHEN, not just "give it a rest". The window is a rolling hour, so
|
||||
# it clears when the oldest attempt ages out — which is not obvious from
|
||||
# the outside, and guessing at it means re-running to find out.
|
||||
clears = min(recent) + timedelta(hours=1)
|
||||
mins = (clears - now).total_seconds() / 60
|
||||
return (f"{len(recent)} attempts in the last hour "
|
||||
f"(limit {cfg.LOGIN_MAX_PER_HOUR}). The oldest was at "
|
||||
f"{min(recent).astimezone():%H:%M:%S}, so this clears at "
|
||||
f"{clears.astimezone():%H:%M:%S} — {mins:.0f} min from now. "
|
||||
f"Waiting is the fix: seven attempts in forty-five minutes is "
|
||||
f"what got this account restricted. "
|
||||
f"Run tools\\check_guard.py to see the clock without starting "
|
||||
f"a run.")
|
||||
return ""
|
||||
|
||||
|
||||
def wait_if_cooling() -> None:
|
||||
"""
|
||||
Sleep out a plain cooldown instead of refusing the run over it.
|
||||
|
||||
`check()` used to be the only gate, so a run started 297s into a 300s
|
||||
cooldown died with "Waiting 3s more" — five of six consecutive runs ended
|
||||
that way, and re-running by hand until the clock cleared is exactly the
|
||||
pattern the cooldown was meant to discourage. Sleeping honours the cooldown
|
||||
just as well as exiting does and actually gets the work done.
|
||||
|
||||
Only a plain cooldown is waited out. A restriction lockout and the
|
||||
per-hour burst limit still refuse through `check()`: those mean "stop",
|
||||
not "not yet".
|
||||
"""
|
||||
if _forced() or cfg.COOLDOWN_MAX_WAIT_SECONDS <= 0:
|
||||
return
|
||||
|
||||
state = _read()
|
||||
now = _now()
|
||||
|
||||
locked = _parse(state.get("locked_until"))
|
||||
if locked and now < locked:
|
||||
return # a restriction — check() refuses it
|
||||
|
||||
window = now - timedelta(hours=1)
|
||||
recent = [d for d in (_parse(x) for x in state.get("recent", []))
|
||||
if d and d > window]
|
||||
if len(recent) >= cfg.LOGIN_MAX_PER_HOUR:
|
||||
return # a burst — check() refuses it
|
||||
|
||||
last = _parse(state.get("last_attempt"))
|
||||
if not last:
|
||||
return
|
||||
remaining = cfg.LOGIN_COOLDOWN_SECONDS - (now - last).total_seconds()
|
||||
if remaining <= 0:
|
||||
return
|
||||
if remaining > cfg.COOLDOWN_MAX_WAIT_SECONDS:
|
||||
# Too long to sit on. Let check() say so and exit, so the operator can
|
||||
# decide rather than watch a silent process.
|
||||
return
|
||||
|
||||
print(f"[GUARD] Cooldown has {remaining:.0f}s left — waiting it out "
|
||||
f"instead of quitting (COOLDOWN_MAX_WAIT_SECONDS="
|
||||
f"{cfg.COOLDOWN_MAX_WAIT_SECONDS})")
|
||||
|
||||
# Measured against `_now()`, the same clock `check()` uses, plus a second of
|
||||
# margin. An earlier version counted with `time.time()` while `check()`
|
||||
# counted with `datetime.now()`; two clocks for one deadline is how you get
|
||||
# a run that waits out its cooldown and is then refused for being a
|
||||
# fraction of a second early.
|
||||
target = last + timedelta(seconds=cfg.LOGIN_COOLDOWN_SECONDS + 1)
|
||||
while True:
|
||||
left = (target - _now()).total_seconds()
|
||||
if left <= 0:
|
||||
break
|
||||
# Say something periodically: a silent process looks hung, which is what
|
||||
# got earlier runs killed by hand.
|
||||
print(f"[GUARD] {left:.0f}s to go")
|
||||
time.sleep(min(30, left))
|
||||
print("[GUARD] Cooldown cleared — starting")
|
||||
|
||||
|
||||
def record_attempt() -> None:
|
||||
state = _read()
|
||||
now = _now()
|
||||
window = now - timedelta(hours=1)
|
||||
recent = [x for x in state.get("recent", []) if (_parse(x) or now) > window]
|
||||
recent.append(now.isoformat())
|
||||
state["last_attempt"] = now.isoformat()
|
||||
state["recent"] = recent[-50:]
|
||||
_write(state)
|
||||
|
||||
|
||||
def record_restricted() -> None:
|
||||
"""Called when the portal shows its restriction page."""
|
||||
state = _read()
|
||||
now = _now()
|
||||
state["restricted_at"] = now.isoformat()
|
||||
state["locked_until"] = (now + RESTRICTED_LOCKOUT).isoformat()
|
||||
_write(state)
|
||||
print(f"[GUARD] Restriction recorded — locked out until "
|
||||
f"{(now + RESTRICTED_LOCKOUT):%H:%M:%S} UTC")
|
||||
|
||||
|
||||
def record_success() -> None:
|
||||
"""A clean login clears the burst history — the account is evidently fine."""
|
||||
state = _read()
|
||||
state["recent"] = []
|
||||
state["locked_until"] = None
|
||||
state["last_success"] = _now().isoformat()
|
||||
_write(state)
|
||||
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
Operator prompts.
|
||||
|
||||
Everything here can time out. An earlier run wedged for forty minutes on a
|
||||
plain MessageBoxW: the mailbox poll found nothing, the code fell back to asking
|
||||
a human, and nobody was at the desk — so the process sat holding a browser open
|
||||
with no log output while eCW went on rate-limiting the account. A prompt that
|
||||
can block forever is a prompt that will, so `MessageBoxTimeoutW` is used
|
||||
instead and callers are handed an explicit TIMEOUT.
|
||||
|
||||
`MessageBoxTimeoutW` is undocumented but has shipped in user32 since XP. If it
|
||||
is ever missing, the plain call is used and the timeout silently does not
|
||||
apply — better a hang than a crash at the moment a human is being asked for
|
||||
help.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
from . import watchdog
|
||||
|
||||
# Return values
|
||||
OK = "ok"
|
||||
CANCEL = "cancel"
|
||||
TIMEOUT = "timeout"
|
||||
|
||||
_MB_OK = 0x00000000
|
||||
_MB_OKCANCEL = 0x00000001
|
||||
_MB_ICONINFO = 0x00000040
|
||||
_MB_ICONWARN = 0x00000030
|
||||
_MB_TOPMOST = 0x00040000 # otherwise it can open behind the browser window
|
||||
# Second button (Cancel, on an OK/Cancel box) is the default one, so Enter or
|
||||
# Space picks Cancel instead of OK. Without it, Enter on the per-claim save
|
||||
# box meant OK — save the claim to live eCW (review item P1).
|
||||
_MB_DEFBUTTON2 = 0x00000100
|
||||
|
||||
_IDOK = 1
|
||||
_IDTIMEOUT = 32000
|
||||
|
||||
|
||||
def _raw_call(text: str, caption: str, flags: int, timeout_s: int) -> int:
|
||||
# A box waits on a person, not a frozen page: the claim watchdog's
|
||||
# clock starts again when it opens and when it closes.
|
||||
watchdog.beat("message box")
|
||||
try:
|
||||
return _call_box(text, caption, flags, timeout_s)
|
||||
finally:
|
||||
watchdog.beat("message box closed")
|
||||
|
||||
|
||||
def _call_box(text: str, caption: str, flags: int, timeout_s: int) -> int:
|
||||
user32 = ctypes.windll.user32
|
||||
try:
|
||||
fn = user32.MessageBoxTimeoutW
|
||||
fn.argtypes = [wintypes.HWND, wintypes.LPCWSTR, wintypes.LPCWSTR,
|
||||
wintypes.UINT, wintypes.WORD, wintypes.DWORD]
|
||||
fn.restype = ctypes.c_int
|
||||
return fn(None, text, caption, flags, 0, int(timeout_s * 1000))
|
||||
except AttributeError:
|
||||
return user32.MessageBoxW(None, text, caption, flags)
|
||||
|
||||
|
||||
def info(title: str, msg: str, timeout_s: int,
|
||||
needs_person: bool = False) -> str:
|
||||
"""Informational box. Returns OK or TIMEOUT.
|
||||
|
||||
With QUIET_BOXES on (a run the results page started), a box that does
|
||||
not need a person is one log line instead, and answers OK."""
|
||||
if cfg.QUIET_BOXES and not needs_person:
|
||||
first = (msg or "").strip().splitlines()[0] if (msg or "").strip() \
|
||||
else ""
|
||||
print(f"[BOX] {titled(title)}: {first}")
|
||||
return OK
|
||||
try:
|
||||
rc = _raw_call(msg, titled(title), _MB_OK | _MB_ICONINFO | _MB_TOPMOST, timeout_s)
|
||||
except Exception:
|
||||
return TIMEOUT
|
||||
return TIMEOUT if rc == _IDTIMEOUT else OK
|
||||
|
||||
|
||||
def confirm(title: str, msg: str, timeout_s: int,
|
||||
default_cancel: bool = False) -> str:
|
||||
"""
|
||||
OK/Cancel box. Returns OK, CANCEL or TIMEOUT.
|
||||
|
||||
`default_cancel=True` makes Cancel the default button, so Enter/Space
|
||||
answers Cancel. For boxes where OK writes to live eCW: there, a key
|
||||
pressed out of habit must not count as a decision. Off by default so the
|
||||
captcha and security-image prompts, where OK only lets a read continue,
|
||||
behave as they always have.
|
||||
"""
|
||||
footer = (f"\n\n(If there's no reply, this will cancel automatically "
|
||||
f"after {timeout_s}s.)")
|
||||
flags = _MB_OKCANCEL | _MB_ICONWARN | _MB_TOPMOST
|
||||
if default_cancel:
|
||||
flags |= _MB_DEFBUTTON2
|
||||
try:
|
||||
rc = _raw_call(msg + footer, titled(title), flags, timeout_s)
|
||||
except Exception:
|
||||
# No user32 (not Windows): nothing can be asked, so do not pretend a
|
||||
# human agreed.
|
||||
return TIMEOUT
|
||||
if rc == _IDTIMEOUT:
|
||||
return TIMEOUT
|
||||
return OK if rc == _IDOK else CANCEL
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Convenience wrappers that apply the configured timeout and log the timeout
|
||||
# =============================================================================
|
||||
# These were private helpers in auth.py. They are here because every caller
|
||||
# wants the same two things — the configured timeout, and a log line when a
|
||||
# prompt goes unanswered — and duplicating that per module is how one of them
|
||||
# ends up silently treating a timeout as consent.
|
||||
|
||||
from .. import settings as cfg # noqa: E402 (kept below the ctypes core)
|
||||
|
||||
TEST_PREFIX = "[TEST] "
|
||||
|
||||
|
||||
def titled(title: str) -> str:
|
||||
"""
|
||||
`title` as the box shows it: with `[TEST] ` in front when the claim ids
|
||||
come from a TEST_CLAIM_IDS_FILE sheet (review item D9, 2026-10-05).
|
||||
|
||||
A test run drives the same live eCW as a real one, so a screen with a
|
||||
box on it looks exactly like production to anyone watching — the client
|
||||
in a demo included. The [MODE] banner says "TEST MODE" in the log only;
|
||||
this puts it on every box. Applied in the raw `info()` / `confirm()`,
|
||||
the two functions every box reaches `_raw_call` through, not only in
|
||||
the `info_box` / `ok_cancel` wrappers: three callers use the raw ones
|
||||
directly (the 2FA manual box in auth/mailbox.py, which a test run does
|
||||
reach at login; "Run the Lookup?" in pages/claim_results.py; the export
|
||||
report box in pages/claim_report.py). Done here rather than at each call
|
||||
site so a box added later cannot forget it. The wrappers apply it too,
|
||||
so their "timed out" log line names the title the box showed; that is
|
||||
safe because the prefix is never doubled. Read at call time (this
|
||||
module imports settings below the ctypes core, and settings can change
|
||||
under a test).
|
||||
"""
|
||||
if cfg.TEST_CLAIM_IDS_FILE and not title.startswith(TEST_PREFIX):
|
||||
return TEST_PREFIX + title
|
||||
return title
|
||||
|
||||
|
||||
def info_box(title: str, msg: str, needs_person: bool = False) -> None:
|
||||
"""Informational box on the configured timeout. Never blocks the run.
|
||||
`needs_person=True` for a box a person must act on (it is never
|
||||
quieted by QUIET_BOXES)."""
|
||||
title = titled(title)
|
||||
# The keyword is passed only when set, so a test that replaces info()
|
||||
# with a three-argument stub keeps working.
|
||||
extra = {"needs_person": True} if needs_person else {}
|
||||
if info(title, msg, cfg.MSGBOX_TIMEOUT_SECONDS, **extra) == TIMEOUT:
|
||||
print(f"[UI] '{title}' timed out with nobody at the desk — continuing")
|
||||
|
||||
|
||||
def ok_cancel(title: str, msg: str, default_cancel: bool = False) -> bool:
|
||||
"""
|
||||
True only on an explicit OK. A timeout counts as 'no'.
|
||||
|
||||
Note the asymmetry with `claim_results.run_lookup`, which treats a timeout
|
||||
as "carry on": there, proceeding only reads. Here the prompts guard captcha
|
||||
and security-image steps where proceeding without a human is meaningless,
|
||||
so silence must not be read as agreement.
|
||||
|
||||
`default_cancel=True` (see `confirm`) for any box whose OK saves to eCW:
|
||||
the per-claim save box and the LIVE-WRITE start box.
|
||||
"""
|
||||
title = titled(title)
|
||||
result = confirm(title, msg, cfg.MSGBOX_TIMEOUT_SECONDS,
|
||||
default_cancel=default_cancel)
|
||||
if result == TIMEOUT:
|
||||
print(f"[UI] '{title}' timed out with nobody at the desk — treating as cancel")
|
||||
return result == OK
|
||||
@@ -0,0 +1,330 @@
|
||||
"""
|
||||
Page helpers shared by every screen. No eCW knowledge lives here.
|
||||
|
||||
These earned their edge cases the hard way, which is exactly why they are
|
||||
in one place instead of being reimplemented per module:
|
||||
|
||||
* `shot` writes into a **dated** screenshot folder and never raises — a
|
||||
screenshot failing must not lose the run it was documenting.
|
||||
A FAILURE shot (`is_failure_tag`) also saves the page's DOM beside
|
||||
it as `<HHMMSS>_<tag>.html` (`SAVE_DOM_ON_FAILURE`, 2026-10-21):
|
||||
when eCW changes a selector, the HTML to fix it from is already on
|
||||
disk — no re-run, no hand copy from DevTools into Dom.txt. PHI, like
|
||||
the screenshots: it stays in the gitignored Screenshots folder.
|
||||
* `visible` is an INSTANT "is it on screen right now?" and `appears` is
|
||||
"give it up to N ms to show up". They are two functions because
|
||||
Playwright 1.62's `is_visible(timeout=...)` silently ignores its timeout
|
||||
(review item P8, 2026-10-01): every caller that passed `timeout=10_000`
|
||||
expecting a grace period was getting an instant check. Both look at
|
||||
VISIBLE matches only rather than `.first` — eCW renders some ids twice
|
||||
(`#jellybean-panelLink4` among them), and a hidden leftover copy first in
|
||||
the DOM used to make `.first` answer False while the real one was on
|
||||
screen.
|
||||
* `goto_with_retry` retries `net::ERR_ABORTED` — the corporate proxy
|
||||
intermittently kills the first navigation, and one try used to abort a whole
|
||||
run.
|
||||
* `reset_scroll` does NOT trust `window.scrollTo(0, 0)`. On a single-page app
|
||||
the document itself is usually not the scroller — an inner container is —
|
||||
so scrolling the window is a silent no-op on an element that was never
|
||||
offset. It sweeps for whatever actually moved and names it in the log.
|
||||
* `auto_dialog` answers every native dialog: alerts OK, confirms / prompts
|
||||
OK only in LIVE-WRITE (Cancel in VERIFY: live eCW is read-only), and
|
||||
beforeunload **dismissed**. An unhandled native dialog freezes the
|
||||
renderer's JS thread, after which every Playwright call hangs with no
|
||||
exception; accepting beforeunload means "yes, close this page", which has
|
||||
been observed closing the working tab mid-flow.
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from playwright.sync_api import Page
|
||||
|
||||
from .. import settings as cfg
|
||||
from . import run_events
|
||||
|
||||
|
||||
# A shot taken because something went wrong — by its tag, as every caller
|
||||
# names it: `claims_status_failed`, `claim_form_no_modal_<id>`,
|
||||
# `login_unconfirmed`, `search_not_ready_<id>` ... Not `logged_in`,
|
||||
# `landing`, `claims_status`, `progress_note`, `claims_balance_cleared`.
|
||||
_FAILURE_TAG = re.compile(
|
||||
r"fail|missing|not_|notfound|nodownload|(^|_)no_|unconfirmed|wrong|"
|
||||
r"leftover|unreadable|timed_out|locked|disabled|lost|restricted|"
|
||||
r"still_open|expired|error")
|
||||
|
||||
# DOM files written by this process (SAVE_DOM_LIMIT caps them per run).
|
||||
_dom_saved = 0
|
||||
|
||||
|
||||
def is_failure_tag(tag: str) -> bool:
|
||||
return bool(_FAILURE_TAG.search(tag or ""))
|
||||
|
||||
|
||||
def save_dom(page: Page, path) -> bool:
|
||||
"""The page's HTML and every frame's, in one file. Never raises."""
|
||||
try:
|
||||
parts = [page.content()]
|
||||
for n, frame in enumerate(page.frames[1:], start=1):
|
||||
try:
|
||||
parts.append(f"\n<!-- ===== frame {n}: {frame.name or ''} "
|
||||
f"===== -->\n{frame.content()}")
|
||||
except Exception:
|
||||
pass
|
||||
path.write_text("".join(parts), encoding="utf-8")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def shot(page: Page, tag: str) -> None:
|
||||
"""Screenshot into Screenshots\\<MM-DD-YYYY>\\<HHMMSS>_<tag>.png — and,
|
||||
for a failure (`is_failure_tag`), the DOM beside it as `.html`."""
|
||||
global _dom_saved
|
||||
ts = datetime.now().strftime("%H%M%S")
|
||||
png = cfg.SCREENSHOTS / f"{ts}_{tag}.png"
|
||||
try:
|
||||
cfg.SCREENSHOTS.mkdir(parents=True, exist_ok=True)
|
||||
page.screenshot(path=str(png))
|
||||
if is_failure_tag(tag):
|
||||
# The Runs page links a Run's failure shots.
|
||||
run_events.emit("screenshot", path=str(png), tag=tag)
|
||||
except Exception:
|
||||
pass
|
||||
if not (cfg.SAVE_DOM_ON_FAILURE and is_failure_tag(tag)
|
||||
and _dom_saved < cfg.SAVE_DOM_LIMIT):
|
||||
return
|
||||
try:
|
||||
cfg.SCREENSHOTS.mkdir(parents=True, exist_ok=True)
|
||||
except Exception:
|
||||
return
|
||||
path = cfg.SCREENSHOTS / f"{ts}_{tag}.html"
|
||||
if save_dom(page, path):
|
||||
_dom_saved += 1
|
||||
print(f"[DOM] Saved the page's HTML for the failure: {path}")
|
||||
|
||||
|
||||
def visible(page: Page, selector: str, timeout: int = 500) -> bool:
|
||||
"""
|
||||
Whether any element matching `selector` is visible RIGHT NOW. Instant.
|
||||
|
||||
`timeout` is accepted for call compatibility and **ignored** — exactly as
|
||||
it always was underneath: Playwright 1.62's `locator.is_visible(timeout=)`
|
||||
does not wait (verified in this project; the review's item P8). Keeping
|
||||
the parameter means no caller breaks, but nothing here pretends to wait.
|
||||
A caller that means "give it time to show up" must use `appears()`.
|
||||
|
||||
Counts visible matches instead of asking `.first`: when eCW leaves a
|
||||
hidden copy of an element ahead of the live one in the DOM, `.first` is
|
||||
the hidden copy and the answer came back False with the real element on
|
||||
screen. Never raises — "is it there?" is a question, not an assertion.
|
||||
"""
|
||||
try:
|
||||
return page.locator(selector).filter(visible=True).count() > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def appears(page: Page, selector: str, timeout: int = 5_000) -> bool:
|
||||
"""
|
||||
Wait up to `timeout` ms for an element matching `selector` to be visible.
|
||||
|
||||
Built on `wait_for(state="visible")`, the call that actually waits —
|
||||
`is_visible(timeout=...)` looks like it should and does not (Playwright
|
||||
1.62 ignores the timeout and answers instantly). That gotcha is why a slow
|
||||
CPT tab was read short and a slow Prog. Notes button read as "absent"
|
||||
(review items P8/D11).
|
||||
|
||||
Filtered to visible matches before `.first`, for the same reason as
|
||||
`visible()`: a hidden leftover copy earlier in the DOM must not be the
|
||||
one that gets waited on. Returns at once when it is already showing, so
|
||||
the cost on the happy path is nothing. Never raises.
|
||||
"""
|
||||
try:
|
||||
page.locator(selector).filter(visible=True).first.wait_for(
|
||||
state="visible", timeout=timeout)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Marks the one match a click would really land on: the shown element whose
|
||||
# own centre hit-tests to itself (nothing covering it), or, when none does —
|
||||
# off screen, or covered by something else entirely — the last one in the
|
||||
# document (eCW's dialogs are appended, so later is higher). Returns how
|
||||
# many shown matches there were. The mark is an attribute, removed from any
|
||||
# earlier match first, so the locator below finds exactly one element.
|
||||
#
|
||||
# A match that computes `pointer-events: none` is not counted at all: no
|
||||
# click can land on it (a window eCW is still opening, or one whose fade-out
|
||||
# Chrome paused — live run 2026-10-13, ~45 s of Cancel clicks timing out).
|
||||
_MARK_TOPMOST_JS = r"""
|
||||
(s) => {
|
||||
for (const el of document.querySelectorAll('[data-ecwbot-top]'))
|
||||
el.removeAttribute('data-ecwbot-top');
|
||||
const shown = el => {
|
||||
const r = el.getBoundingClientRect();
|
||||
const cs = getComputedStyle(el);
|
||||
return r.width > 0 && r.height > 0
|
||||
&& cs.visibility !== 'hidden' && cs.pointerEvents !== 'none';
|
||||
};
|
||||
const all = [...document.querySelectorAll(s)].filter(shown);
|
||||
if (!all.length) return 0;
|
||||
const onTop = el => {
|
||||
const r = el.getBoundingClientRect();
|
||||
const hit = document.elementFromPoint(r.left + r.width / 2,
|
||||
r.top + r.height / 2);
|
||||
return !!hit && (hit === el || el.contains(hit));
|
||||
};
|
||||
const top = all.filter(onTop).pop() || all[all.length - 1];
|
||||
top.setAttribute('data-ecwbot-top', '1');
|
||||
return all.length;
|
||||
}
|
||||
"""
|
||||
_TOPMOST = '[data-ecwbot-top="1"]'
|
||||
|
||||
|
||||
def count_shown(page: Page, selector: str) -> int:
|
||||
"""How many matches are on screen right now (0 on any page error)."""
|
||||
try:
|
||||
return int(page.evaluate(_MARK_TOPMOST_JS, selector) or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def click_topmost(page: Page, selector: str, timeout: int = 15_000) -> bool:
|
||||
"""
|
||||
Click the match that is on TOP — not `.first`, which is the earliest in
|
||||
the document and, with two dialogs stacked, the one underneath. Live run
|
||||
2026-10-12: a second Progress Note viewer and a second copy of an HCFA
|
||||
claim window sat over the first; every `.first` click (Close, Cancel)
|
||||
was intercepted for its full timeout, and the dialog that stayed blocked
|
||||
the next claims too.
|
||||
|
||||
Returns False when nothing matching is on screen. A click that fails
|
||||
raises, as `.click()` does — callers already handle that.
|
||||
"""
|
||||
if not count_shown(page, selector):
|
||||
return False
|
||||
page.locator(_TOPMOST).first.click(timeout=timeout)
|
||||
return True
|
||||
|
||||
|
||||
def text_of(page: Page, selector: str) -> str:
|
||||
"""
|
||||
An input's current value, or "?" when it cannot be read.
|
||||
|
||||
"?" is deliberately not "" — an unreadable control is *unknown*, and code
|
||||
that treats unknown as empty will happily skip a filter that is still live.
|
||||
"""
|
||||
try:
|
||||
return page.locator(selector).first.input_value(timeout=5_000).strip()
|
||||
except Exception:
|
||||
return "?"
|
||||
|
||||
|
||||
def is_disabled(page: Page, selector: str) -> bool:
|
||||
"""
|
||||
Whether a control is disabled — and **True when it cannot be read**.
|
||||
|
||||
That default is deliberate and load-bearing. Its one caller decides between
|
||||
typing into the Balance row and clearing it through Angular's scope; the
|
||||
scope path works either way, so an unreadable control has to fall to the
|
||||
safe side. Returning False here would make an unreadable control look
|
||||
editable and the clear would silently do nothing.
|
||||
"""
|
||||
try:
|
||||
return not page.locator(selector).first.is_enabled(timeout=5_000)
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
|
||||
# Every element is walked rather than a known list of containers: which div
|
||||
# scrolls is a property of the page, not something this codebase can hardcode,
|
||||
# and the whole point of the sweep is that it reports the answer instead of
|
||||
# requiring someone to already know it.
|
||||
_RESET_SCROLL_JS = r"""() => {
|
||||
const out = [];
|
||||
window.scrollTo(0, 0);
|
||||
document.querySelectorAll('*').forEach(el => {
|
||||
if (el.scrollTop > 0) {
|
||||
out.push((el.tagName || '?').toLowerCase()
|
||||
+ (el.id ? '#' + el.id : '')
|
||||
+ ' @' + el.scrollTop);
|
||||
el.scrollTop = 0;
|
||||
}
|
||||
});
|
||||
return out;
|
||||
}"""
|
||||
|
||||
|
||||
def reset_scroll(page: Page) -> list:
|
||||
"""
|
||||
Scroll everything back to the top, and return what actually moved.
|
||||
|
||||
`window.scrollTo(0, 0)` was tried on its own first and did nothing at all:
|
||||
the page came back still scrolled, which is what proved the document is not
|
||||
the scroller here. An inner container is, so the window call is a no-op on
|
||||
an element that was never offset in the first place.
|
||||
|
||||
The return value is the point as much as the scrolling is — each entry names
|
||||
a container that was genuinely offset (`div#someGrid @240`), so the next
|
||||
person reading a log does not have to rediscover which element scrolls.
|
||||
|
||||
Never raises: this is tidying up after a step, not the step itself.
|
||||
"""
|
||||
try:
|
||||
return page.evaluate(_RESET_SCROLL_JS) or []
|
||||
except Exception as exc:
|
||||
print(f"[PAGE] Could not reset the scroll: "
|
||||
f"{str(exc).splitlines()[0]}")
|
||||
return []
|
||||
|
||||
|
||||
def goto_with_retry(page: Page, url: str) -> None:
|
||||
"""
|
||||
The first navigation through the corporate proxy intermittently dies with
|
||||
net::ERR_ABORTED — a transient proxy/TLS hiccup, not a bad URL.
|
||||
"""
|
||||
for attempt in range(1, cfg.NAV_RETRIES + 1):
|
||||
try:
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=cfg.NAV_TIMEOUT)
|
||||
return
|
||||
except Exception as exc:
|
||||
if attempt == cfg.NAV_RETRIES:
|
||||
raise
|
||||
print(f"[NAV] Navigation {attempt}/{cfg.NAV_RETRIES} failed "
|
||||
f"({str(exc).splitlines()[0]}) — retrying in 5s")
|
||||
page.wait_for_timeout(5_000)
|
||||
|
||||
|
||||
def auto_dialog(dialog) -> None:
|
||||
"""
|
||||
Answer every native dialog: one left open wedges the renderer, and in
|
||||
tests crashed the tab outright ("Target crashed"). Until 2026-10-22 none
|
||||
was answered — `dialog.message()` called a property, and the TypeError
|
||||
was swallowed.
|
||||
|
||||
* alert → OK: it only informs.
|
||||
* confirm / prompt → OK only in LIVE-WRITE. In VERIFY, Cancel: live
|
||||
eCW is read-only, and a confirm's OK may be a save.
|
||||
* beforeunload → dismissed: accepting means "yes, close this page".
|
||||
|
||||
Each one is logged and is a Warning on the Runs page, with the answer.
|
||||
"""
|
||||
kind = getattr(dialog, "type", "?")
|
||||
message = str(getattr(dialog, "message", "") or "")
|
||||
accept = kind == "alert" or (kind in ("confirm", "prompt")
|
||||
and bool(cfg.SAVE_CORRECTIONS))
|
||||
answer = "OK" if accept else "Cancel"
|
||||
print(f"[DIALOG] {kind}: {message!r} -> {answer}")
|
||||
run_events.warning("dialog", f"{kind}: {message} -> {answer}"[:300])
|
||||
try:
|
||||
if accept:
|
||||
dialog.accept()
|
||||
else:
|
||||
dialog.dismiss()
|
||||
except Exception:
|
||||
pass # answered already by the other listener
|
||||
# (core/browser.py wires context and page)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
Payer -> Group Name lookup for the categorization step.
|
||||
|
||||
Exact-string match against a small hand-edited table (`payer_groups.json`,
|
||||
next to this package) — no fuzzy or substring matching. eCW's own PAYER
|
||||
values already differ by case and spacing for what is really the same
|
||||
carrier (see `tests/test_payer_groups.py`'s docstring), so "close enough"
|
||||
matching has no safe way to pick which known string a new one is closest to.
|
||||
An unrecognized payer must come back as None and get a human's attention —
|
||||
never a guess, on billing data.
|
||||
|
||||
Since 2026-10-06 the table is the team's own template: the Payer Name ->
|
||||
Payer Group Name pairs of their Claim Status Dashboard, imported with
|
||||
`tools/import_payer_groups.py` (re-run it when the dashboard changes). Adding
|
||||
a payer by hand is still a one-line edit to `payer_groups.json`.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
_CONFIG_PATH = Path(__file__).resolve().parent.parent / "payer_groups.json"
|
||||
|
||||
|
||||
def load_mapping(path=None) -> dict:
|
||||
"""The payer -> group table. `path` defaults to the shipped config."""
|
||||
p = Path(path) if path else _CONFIG_PATH
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(
|
||||
f"{p} not found — the payer group table is missing")
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(
|
||||
f"{p} does not contain a JSON object (payer -> group name) — "
|
||||
f"got {type(data).__name__}")
|
||||
return data
|
||||
|
||||
|
||||
def classify(payer: str, mapping: dict) -> str:
|
||||
"""The Group Name for this exact PAYER string, or None if unmapped."""
|
||||
return mapping.get((payer or "").strip())
|
||||
|
||||
|
||||
# ── Group names the CONDITIONS decide on ─────────────────────────────────────
|
||||
# Since 2026-10-06 `payer_groups.json` is the team's own template — the
|
||||
# "Payer Name" -> "Payer Group Name" pairs of their Claim Status Dashboard
|
||||
# (tools/import_payer_groups.py) — so its group names are the dashboard's
|
||||
# ("Commercial Ins", "RAE (Behavioral Health)", "Dental - CHP+/DentaQuest"...)
|
||||
# and that is what the Group Name columns show. The condition modules were
|
||||
# written against the earlier, shorter names, which stay the vocabulary of
|
||||
# their rules; this table translates. A name not listed (including every
|
||||
# earlier short name) passes through unchanged.
|
||||
RULE_GROUP = {
|
||||
"Commercial Ins": "Commercial",
|
||||
"RAE (Behavioral Health)": "RAE payer",
|
||||
"Regional Accountable Entity-RAE (Reporting Group)": "RAE payer",
|
||||
"Dental - CHP+/DentaQuest": "Dentaquest",
|
||||
"Self Pay": "Program", # the clinic's programs (Outreach, slides…)
|
||||
"Payer Not In Group": None, # the dashboard's own "unclassified"
|
||||
}
|
||||
|
||||
# A payer whose condition rule differs from its dashboard group. Empty since
|
||||
# 2026-10-21: the Medicare Advantage plans (AARP Medicare Advantage, Aetna
|
||||
# Medicare ADV) were kept on Medicare here until the operator ruled; the
|
||||
# ruling is "follow the dashboard", so they are Commercial for every
|
||||
# condition (#21 95/93 not #22 G2025; no #14, #19, #27-#29; #18 / #20 add).
|
||||
RULE_OVERRIDES = {}
|
||||
|
||||
|
||||
def rule_group(payer: str, mapping: dict) -> str:
|
||||
"""
|
||||
The group a condition's rule should use for this PAYER, or None if
|
||||
unmapped: the payer's own override if it has one, else its Group Name
|
||||
translated through RULE_GROUP.
|
||||
"""
|
||||
key = (payer or "").strip()
|
||||
if key in RULE_OVERRIDES:
|
||||
return RULE_OVERRIDES[key]
|
||||
group = classify(key, mapping)
|
||||
if group is None:
|
||||
return None
|
||||
return RULE_GROUP.get(group, group)
|
||||
@@ -0,0 +1,318 @@
|
||||
r"""
|
||||
The Charge Summary rows, also saved to SQL Server for the results web page
|
||||
(`web/`).
|
||||
|
||||
Operator, 2026-10-08: server MECARE331\MECARE331, database
|
||||
`Charge_Validation`, table `dbo.CV_ChargeSummary` (every table of this
|
||||
project starts with `CV_`, operator 2026-10-12) — the team's Charge Summary
|
||||
File Format's 24 columns (`charge_summary.HEADERS`), plus `run_at`,
|
||||
`run_mode` and `details_json` (what the conditions found and changed). The
|
||||
table is created by the operator from `tools/sql/create_tables.sql`; the bot
|
||||
only inserts.
|
||||
|
||||
The rows are the ones the Excel summary gets (`charge_summary.build`), so the
|
||||
two always agree. Only rows the run filled in (a Category) are saved: a
|
||||
claim not processed or not found is blank in the Excel too.
|
||||
|
||||
Off unless `RESULTS_DB=1`. Never fatal: any database error is one
|
||||
`[RESULTS-DB]` log line, and the run and its Excel summary carry on.
|
||||
|
||||
Values are converted for their columns — amounts without `$` and commas,
|
||||
dates from mm/dd/yyyy (a trailing time or an Excel serial day number
|
||||
included) — and anything that does not convert is stored as NULL (the Excel
|
||||
file still has the text as read). Text longer than its column is cut to the
|
||||
column's width, so one long value cannot fail the insert. SQL is always
|
||||
parameterized, and a database error is logged by its SQLSTATE only, never
|
||||
the driver's message, which can quote a value (PHI) back.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
from . import charge_summary
|
||||
|
||||
TABLE = "dbo.CV_ChargeSummary"
|
||||
|
||||
# Charge Summary header -> DB column. `COLUMNS` is built from HEADERS, so a
|
||||
# header added to the format without a column here fails at import, not
|
||||
# silently at insert time.
|
||||
_COLUMN_FOR = {
|
||||
"S.No": "s_no", "CLAIMS#": "claim_no", "SERVICE DATE": "service_date",
|
||||
"PVDR": "pvdr", "PATIENT": "patient", "PAYER": "payer",
|
||||
"Group Name": "group_name", "STATUS": "status", "CHARGES": "charges",
|
||||
"PMTS/ADJS": "pmts_adjs", "Pmts/Refunds": "pmts_refunds",
|
||||
"Adjustment": "adjustment", "WithHeld": "withheld", "Balance": "balance",
|
||||
"Visit Type": "visit_type", "Account Number": "account_number",
|
||||
"Provider Name": "provider_name", "Guarantor Name": "guarantor_name",
|
||||
"Recevied date": "received_date", "Assigned To ": "assigned_to",
|
||||
"Completed On": "completed_on", "Comments": "comments",
|
||||
"Current Status": "current_status", "Category": "category",
|
||||
}
|
||||
COLUMNS = [(h, _COLUMN_FOR[h]) for h in charge_summary.HEADERS]
|
||||
|
||||
DB_COLUMNS = (["run_at", "run_mode"] + [c for _, c in COLUMNS]
|
||||
+ ["details_json"])
|
||||
|
||||
# NVARCHAR widths from tools/sql/create_tables.sql. MAX columns are not
|
||||
# listed.
|
||||
_MAX_LEN = {
|
||||
"run_mode": 20, "claim_no": 20, "pvdr": 50, "patient": 200,
|
||||
"payer": 200, "group_name": 100, "status": 100, "visit_type": 100,
|
||||
"account_number": 50, "provider_name": 200, "guarantor_name": 200,
|
||||
"assigned_to": 100, "current_status": 200, "category": 20,
|
||||
}
|
||||
|
||||
_MONEY = {"charges", "pmts_adjs", "pmts_refunds", "adjustment", "withheld",
|
||||
"balance"}
|
||||
_DATES = {"service_date", "received_date", "completed_on"}
|
||||
_INTS = {"s_no"}
|
||||
|
||||
# DECIMAL(12,2): ten digits before the point.
|
||||
_MONEY_LIMIT = Decimal(10) ** 10
|
||||
_CENT = Decimal("0.01")
|
||||
|
||||
_US_DATE = re.compile(r"^(\d{1,2})/(\d{1,2})/(\d{4})(?:\s|$)")
|
||||
_SERIAL = re.compile(r"^\d{5}(?:\.0+)?$")
|
||||
# Excel's day 0, and the serial range that is a plausible date
|
||||
# (1954-10-03 .. 2119-01-08) — not some other five-digit number.
|
||||
_EXCEL_EPOCH = date(1899, 12, 30)
|
||||
_SERIAL_RANGE = (20_000, 80_000)
|
||||
_ISO_DATE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})")
|
||||
|
||||
# What details_for() copies from the claim's `_conditions`.
|
||||
_CONDITION_KEYS = ("fired", "detected", "changed", "saved",
|
||||
"save_unconfirmed", "failed", "review", "comments",
|
||||
"changes")
|
||||
|
||||
# Keys dropped wherever they appear in a condition's changes: they carry
|
||||
# patient text, not what the bot did — vaccine_admin_codes' age_source
|
||||
# names the DOB it read; the Dental Billing Notes' match quotes the note.
|
||||
_PHI_KEYS = {"age_source", "match", "dob"}
|
||||
|
||||
# SQLSTATE -> what it means for the operator.
|
||||
_SQLSTATE_HINT = {
|
||||
"08001": "server not reachable", "08S01": "connection lost",
|
||||
"HYT00": "timed out", "HYT01": "timed out", "28000": "login failed",
|
||||
"42S02": "table missing — run tools/sql/create_tables.sql",
|
||||
"42000": "no permission, or a SQL error",
|
||||
"22001": "a value is too long for its column",
|
||||
"22003": "a number is too large for its column",
|
||||
}
|
||||
|
||||
|
||||
def to_money(value):
|
||||
"""An amount as Decimal (2 places), or None when blank or not a number."""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
text = str(value)
|
||||
else:
|
||||
text = str(value).replace("$", "").replace(",", "").strip()
|
||||
negative = text.startswith("(") and text.endswith(")")
|
||||
if negative:
|
||||
text = text[1:-1].strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
amount = Decimal(text)
|
||||
except InvalidOperation:
|
||||
return None
|
||||
if not amount.is_finite() or abs(amount) >= _MONEY_LIMIT:
|
||||
return None
|
||||
amount = amount.quantize(_CENT)
|
||||
# 9999999999.995 rounds up to 10^10, past DECIMAL(12,2).
|
||||
if abs(amount) >= _MONEY_LIMIT:
|
||||
return None
|
||||
return -amount if negative else amount
|
||||
|
||||
|
||||
def to_date(value):
|
||||
"""A date from mm/dd/yyyy, yyyy-mm-dd or a date/datetime; else None."""
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
value = str(value)
|
||||
text = str(value or "").strip()
|
||||
try:
|
||||
m = _US_DATE.match(text)
|
||||
if m:
|
||||
return date(int(m[3]), int(m[1]), int(m[2]))
|
||||
m = _ISO_DATE.match(text)
|
||||
if m:
|
||||
return date(int(m[1]), int(m[2]), int(m[3]))
|
||||
if _SERIAL.match(text):
|
||||
days = int(float(text))
|
||||
if _SERIAL_RANGE[0] <= days <= _SERIAL_RANGE[1]:
|
||||
return _EXCEL_EPOCH + timedelta(days=days)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _to_int(value):
|
||||
try:
|
||||
return int(str(value).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _to_text(value):
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _fit(col, text):
|
||||
"""Text cut to its column's width (NVARCHAR(n)); MAX columns as is."""
|
||||
limit = _MAX_LEN.get(col)
|
||||
if text is None or limit is None or len(text) <= limit:
|
||||
return text
|
||||
return text[:limit]
|
||||
|
||||
|
||||
def to_record(row: dict, *, run_at, run_mode, details) -> dict:
|
||||
"""One Charge Summary row ({header: value}) as {db column: value}."""
|
||||
rec = {"run_at": run_at, "run_mode": run_mode}
|
||||
for header, col in COLUMNS:
|
||||
value = row.get(header)
|
||||
if col in _MONEY:
|
||||
rec[col] = to_money(value)
|
||||
elif col in _DATES:
|
||||
rec[col] = to_date(value)
|
||||
elif col in _INTS:
|
||||
rec[col] = _to_int(value)
|
||||
else:
|
||||
rec[col] = _fit(col, _to_text(value))
|
||||
rec["run_mode"] = _fit("run_mode", run_mode)
|
||||
rec["details_json"] = (json.dumps(details, default=str, ensure_ascii=False)
|
||||
if details else None)
|
||||
return rec
|
||||
|
||||
|
||||
def details_for(data: dict) -> dict:
|
||||
"""
|
||||
What the run found on one claim, for `details_json`: the conditions'
|
||||
account (fired / changes / review / failed / comments / saved), the
|
||||
Dental checks, and why a claim could not be read. Never the claim's own
|
||||
fields (patient, CPT grid...) — those are in the row's columns or not
|
||||
stored at all.
|
||||
"""
|
||||
data = data or {}
|
||||
out = {}
|
||||
cond = data.get("_conditions")
|
||||
if isinstance(cond, dict):
|
||||
out.update({k: _without_phi(cond[k])
|
||||
for k in _CONDITION_KEYS if k in cond})
|
||||
# Dental: the verdicts only — not the Billing Notes text, not the
|
||||
# sliding-fee dates.
|
||||
hr = data.get("high_risk")
|
||||
if isinstance(hr, dict):
|
||||
out.setdefault("dental", {})["high_risk"] = hr.get("high_risk")
|
||||
sf = data.get("sliding_fee")
|
||||
if isinstance(sf, dict):
|
||||
out.setdefault("dental", {})["sliding_fee"] = {
|
||||
k: sf[k] for k in ("verdict", "reason", "error") if k in sf}
|
||||
for key in ("_error", "_not_evaluated", "_search_error"):
|
||||
if data.get(key):
|
||||
out[key.lstrip("_")] = data[key]
|
||||
return out
|
||||
|
||||
|
||||
def _without_phi(value):
|
||||
"""`value` with every _PHI_KEYS key dropped, at any depth."""
|
||||
if isinstance(value, dict):
|
||||
return {k: _without_phi(v) for k, v in value.items()
|
||||
if k not in _PHI_KEYS}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_without_phi(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def safe_error(exc) -> str:
|
||||
"""
|
||||
An error for the log without the driver's message: SQL Server's
|
||||
"String or binary data would be truncated" (Msg 2628) quotes the value,
|
||||
which can be a patient's name. pyodbc puts the SQLSTATE in args[0].
|
||||
"""
|
||||
state = exc.args[0] if exc.args and isinstance(exc.args[0], str) else ""
|
||||
if re.fullmatch(r"[0-9A-Z]{5}", state):
|
||||
hint = _SQLSTATE_HINT.get(state, "database error")
|
||||
return f"{type(exc).__name__} SQLSTATE {state} ({hint})"
|
||||
return type(exc).__name__
|
||||
|
||||
|
||||
def has_column(conn, column: str) -> bool:
|
||||
"""True if the results table has `column`. False when it does not — or
|
||||
the check itself fails: a column the table lacks must never cost the
|
||||
run its rows."""
|
||||
try:
|
||||
row = conn.cursor().execute("SELECT COL_LENGTH(?, ?)",
|
||||
[TABLE, column]).fetchone()
|
||||
return bool(row) and row[0] is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def insert_rows(conn, records: list, started_by: str = "") -> int:
|
||||
"""One parameterized INSERT for every record, one commit. `started_by`
|
||||
(EEWA-33) goes in only once tools/sql/create_tables.sql has added its
|
||||
column: until then the rows are saved without it."""
|
||||
if not records:
|
||||
return 0
|
||||
cols = list(DB_COLUMNS)
|
||||
extra = []
|
||||
if started_by and has_column(conn, "started_by"):
|
||||
cols.append("started_by")
|
||||
extra = [started_by[:50]]
|
||||
sql = (f"INSERT INTO {TABLE} ({', '.join(cols)}) "
|
||||
f"VALUES ({', '.join('?' for _ in cols)})")
|
||||
cur = conn.cursor()
|
||||
cur.executemany(sql, [[rec[c] for c in DB_COLUMNS] + extra
|
||||
for rec in records])
|
||||
conn.commit()
|
||||
return len(records)
|
||||
|
||||
|
||||
def save_summary(sheet: list, details: dict, *, run_at, run_mode,
|
||||
conn_str: str, connect=None, started_by: str = ""):
|
||||
"""
|
||||
Save the rows of `sheet` (header row + rows, as `charge_summary.build`
|
||||
returns it) that the run filled in. Returns the number saved, or None
|
||||
when the database could not be written — never raises.
|
||||
"""
|
||||
try:
|
||||
header = list(sheet[0])
|
||||
records = []
|
||||
for values in sheet[1:]:
|
||||
row = dict(zip(header, values))
|
||||
if not str(row.get("Category") or "").strip():
|
||||
continue
|
||||
claim = str(row.get("CLAIMS#") or "").strip()
|
||||
records.append(to_record(row, run_at=run_at, run_mode=run_mode,
|
||||
details=(details or {}).get(claim)))
|
||||
if not records:
|
||||
print("[RESULTS-DB] No claim rows to save")
|
||||
return 0
|
||||
if connect is None:
|
||||
import pyodbc
|
||||
connect = pyodbc.connect
|
||||
conn = connect(conn_str, timeout=15)
|
||||
try:
|
||||
# Query timeout: a blocked insert raises (and is logged)
|
||||
# instead of holding the run up.
|
||||
conn.timeout = 30
|
||||
saved = insert_rows(conn, records, started_by)
|
||||
finally:
|
||||
conn.close()
|
||||
print(f"[RESULTS-DB] {saved} row(s) saved to {TABLE}")
|
||||
return saved
|
||||
except Exception as exc:
|
||||
print(f"[RESULTS-DB] Could not save the charge summary to the "
|
||||
f"database: {safe_error(exc)} — the Excel summary is not "
|
||||
f"affected")
|
||||
return None
|
||||
@@ -0,0 +1,146 @@
|
||||
"""
|
||||
Run events (spec docs/superpowers/specs/2026-10-22-run-tracking-and-portal-
|
||||
sign-in-design.md, Stage 1; ADR docs/adr/0001-run-events-in-files.md).
|
||||
|
||||
The run log has no times on its lines, so how long the eCW login took, or
|
||||
when a claim went wrong, could not be told afterwards. Every Run now also
|
||||
writes `Logs\\events_<run>.jsonl` beside `run_<run>.log`: one JSON object a
|
||||
line, each with its time to the millisecond, flushed at once so a killed Run
|
||||
keeps what it wrote. The Portal's Runs page reads these files.
|
||||
|
||||
An event never stops a Run: anything that goes wrong writing one drops that
|
||||
event and nothing else. Nothing secret goes in — no OTP code, no password,
|
||||
no eCW username: the callers pass fixed fields, never a whole log line.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import getpass
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
_handle = None
|
||||
_attempt = 0
|
||||
|
||||
|
||||
def close() -> None:
|
||||
"""End of the Run: let go of the file (Windows keeps an open one locked)."""
|
||||
_reset()
|
||||
|
||||
|
||||
def _reset() -> None:
|
||||
"""Close the file and forget the attempt count (tests; a new Run)."""
|
||||
global _handle, _attempt
|
||||
try:
|
||||
if _handle:
|
||||
_handle.close()
|
||||
except Exception:
|
||||
pass
|
||||
_handle, _attempt = None, 0
|
||||
|
||||
|
||||
def run_id_of(log_path) -> str:
|
||||
"""`run_20261022_141530.log` → `20261022_141530`."""
|
||||
stem = Path(log_path).stem
|
||||
return stem[len("run_"):] if stem.startswith("run_") else stem
|
||||
|
||||
|
||||
def path_for(log_path) -> Path:
|
||||
"""The events file of the Run whose log is `log_path`."""
|
||||
log_path = Path(log_path)
|
||||
return log_path.with_name(f"events_{run_id_of(log_path)}.jsonl")
|
||||
|
||||
|
||||
def _windows_user() -> str:
|
||||
try:
|
||||
return getpass.getuser()
|
||||
except Exception:
|
||||
return os.environ.get("USERNAME", "")
|
||||
|
||||
|
||||
def started_by() -> str:
|
||||
"""Started By: the Portal User the Portal passed (ECW_STARTED_BY), else
|
||||
the Windows user of a terminal run."""
|
||||
return os.environ.get("ECW_STARTED_BY") or _windows_user()
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now().isoformat(timespec="milliseconds")
|
||||
|
||||
|
||||
def _line(kind: str, fields: dict) -> str:
|
||||
# The time and kind last, so no field can overwrite them.
|
||||
return json.dumps({**fields, "t": _now(), "kind": kind},
|
||||
ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def start(log_path, mode: str, claim_limit=None, test_sheet: str = "") -> None:
|
||||
"""Open this Run's events file and write `run_start`."""
|
||||
global _handle
|
||||
_reset()
|
||||
try:
|
||||
_handle = open(path_for(log_path), "a", encoding="utf-8",
|
||||
buffering=1)
|
||||
except Exception:
|
||||
_handle = None
|
||||
return
|
||||
source = os.environ.get("ECW_RUN_SOURCE", "terminal")
|
||||
emit("run_start",
|
||||
source="portal" if source == "web" else source,
|
||||
started_by=started_by(),
|
||||
mode=mode, claim_limit=claim_limit,
|
||||
test_sheet=Path(test_sheet).name if test_sheet else "",
|
||||
pid=os.getpid(), log=str(log_path))
|
||||
|
||||
|
||||
def emit(kind: str, **fields) -> None:
|
||||
"""Write one event. Never raises."""
|
||||
if _handle is None:
|
||||
return
|
||||
try:
|
||||
_handle.write(_line(kind, fields) + "\n")
|
||||
_handle.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def append_to(log_path, kind: str, **fields) -> None:
|
||||
"""Add one event to another Run's file — the Portal, after a Stop."""
|
||||
try:
|
||||
with open(path_for(log_path), "a", encoding="utf-8") as fh:
|
||||
fh.write(_line(kind, fields) + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def warning(code: str, message: str, **fields) -> None:
|
||||
emit("warning", code=code, message=message, **fields)
|
||||
|
||||
|
||||
def claim_error(claim: str, step: str, message: str) -> None:
|
||||
emit("claim_error", claim=claim, step=step, message=message)
|
||||
|
||||
|
||||
def run_error(step: str, message: str, exit_code: int, exc=None) -> None:
|
||||
tb = ""
|
||||
if exc is not None:
|
||||
tb = "".join(traceback.format_exception(type(exc), exc,
|
||||
exc.__traceback__))
|
||||
emit("run_error", step=step, exit_code=exit_code, message=message,
|
||||
traceback=tb)
|
||||
|
||||
|
||||
def login_attempt_start() -> None:
|
||||
global _attempt
|
||||
_attempt += 1
|
||||
emit("login_attempt_start", attempt=_attempt)
|
||||
|
||||
|
||||
def login_step(step: str, **fields) -> None:
|
||||
emit("login_step", attempt=_attempt, step=step, **fields)
|
||||
|
||||
|
||||
def login_attempt_end(result: str, message: str = "") -> None:
|
||||
emit("login_attempt_end", attempt=_attempt, result=result,
|
||||
message=message)
|
||||
@@ -0,0 +1,229 @@
|
||||
r"""
|
||||
The run record — `Logs\bot_run.json`, the latest bot run (spec
|
||||
docs/superpowers/specs/2026-10-14-start-bot-from-web-page-design.md).
|
||||
|
||||
Written by the bot itself at the start of `main()` and finished when
|
||||
`main()` returns, so the results page sees a run started in the terminal
|
||||
as well as one it started. While a run is live a second `main()` refuses to
|
||||
start: two runs would share one eCW login guard and one Charge Summary.
|
||||
|
||||
{"pid", "started_at", "log_path", "source": "web" | "terminal",
|
||||
"ended_at", "exit_code", "stopped_by_operator"}
|
||||
|
||||
Live = no `ended_at`, the pid is a running python process, and that process
|
||||
was created no later than the record's `started_at` (plus a few seconds) —
|
||||
so a record left unended by a killed run, whose pid Windows later hands to
|
||||
another python process, does not read as live. A killed run cannot finish
|
||||
its own record; its pid is gone or reused, so it reads as ended.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from .. import settings as cfg
|
||||
|
||||
FILE_NAME = "bot_run.json"
|
||||
|
||||
_PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
|
||||
_STILL_ACTIVE = 259
|
||||
# started_at is written after the process exists, truncated to the second.
|
||||
_CREATED_SLACK = timedelta(seconds=5)
|
||||
_FILETIME_UNIX_EPOCH = 116444736000000000 # 1601-01-01 -> 1970-01-01, 100 ns
|
||||
|
||||
|
||||
class Busy(Exception):
|
||||
"""Another bot run is live."""
|
||||
|
||||
|
||||
def path():
|
||||
"""Read at call time: tests point cfg.LOGS elsewhere."""
|
||||
return cfg.LOGS / FILE_NAME
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def pid_alive(pid) -> bool:
|
||||
"""
|
||||
A running process with this pid whose program is python.
|
||||
|
||||
Never `os.kill(pid, 0)`: on Windows that TERMINATES the process.
|
||||
The program-name check keeps a stale record whose pid Windows gave to
|
||||
some other program from reading as a live bot run.
|
||||
"""
|
||||
try:
|
||||
pid = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if pid <= 0:
|
||||
return False
|
||||
if os.name != "nt":
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
k32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
k32.OpenProcess.restype = wintypes.HANDLE
|
||||
k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
||||
k32.GetExitCodeProcess.argtypes = [wintypes.HANDLE,
|
||||
ctypes.POINTER(wintypes.DWORD)]
|
||||
k32.QueryFullProcessImageNameW.argtypes = [
|
||||
wintypes.HANDLE, wintypes.DWORD, wintypes.LPWSTR,
|
||||
ctypes.POINTER(wintypes.DWORD)]
|
||||
k32.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
handle = k32.OpenProcess(_PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
||||
if not handle:
|
||||
return False
|
||||
try:
|
||||
code = wintypes.DWORD()
|
||||
if not k32.GetExitCodeProcess(handle, ctypes.byref(code)):
|
||||
return False
|
||||
if code.value != _STILL_ACTIVE:
|
||||
return False
|
||||
buf = ctypes.create_unicode_buffer(1024)
|
||||
size = wintypes.DWORD(len(buf))
|
||||
if not k32.QueryFullProcessImageNameW(handle, 0, buf,
|
||||
ctypes.byref(size)):
|
||||
return True
|
||||
return "python" in os.path.basename(buf.value).lower()
|
||||
finally:
|
||||
k32.CloseHandle(handle)
|
||||
|
||||
|
||||
def process_created(pid):
|
||||
"""
|
||||
When process `pid` was created, as a local naive datetime (the same
|
||||
clock as the record's `started_at`); None when it cannot be read or
|
||||
the process is no longer running.
|
||||
"""
|
||||
if os.name != "nt":
|
||||
return None
|
||||
try:
|
||||
pid = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if pid <= 0:
|
||||
return None
|
||||
k32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
k32.OpenProcess.restype = wintypes.HANDLE
|
||||
k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
||||
k32.GetExitCodeProcess.argtypes = [wintypes.HANDLE,
|
||||
ctypes.POINTER(wintypes.DWORD)]
|
||||
k32.GetProcessTimes.argtypes = (
|
||||
[wintypes.HANDLE] + [ctypes.POINTER(wintypes.FILETIME)] * 4)
|
||||
k32.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
handle = k32.OpenProcess(_PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
||||
if not handle:
|
||||
return None
|
||||
try:
|
||||
# An ended process can still be opened while some handle to it is
|
||||
# held (its parent's, say) and still has times: not a running one.
|
||||
code = wintypes.DWORD()
|
||||
if (not k32.GetExitCodeProcess(handle, ctypes.byref(code))
|
||||
or code.value != _STILL_ACTIVE):
|
||||
return None
|
||||
created, exited, kernel, user = (wintypes.FILETIME() for _ in range(4))
|
||||
if not k32.GetProcessTimes(handle, ctypes.byref(created),
|
||||
ctypes.byref(exited), ctypes.byref(kernel),
|
||||
ctypes.byref(user)):
|
||||
return None
|
||||
ticks = (created.dwHighDateTime << 32) | created.dwLowDateTime
|
||||
if ticks <= _FILETIME_UNIX_EPOCH:
|
||||
return None
|
||||
return datetime.fromtimestamp(
|
||||
(ticks - _FILETIME_UNIX_EPOCH) / 10_000_000)
|
||||
except (OSError, OverflowError, ValueError):
|
||||
return None
|
||||
finally:
|
||||
k32.CloseHandle(handle)
|
||||
|
||||
|
||||
def read():
|
||||
"""The record, or None when there is none or it cannot be read."""
|
||||
try:
|
||||
data = json.loads(path().read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def is_live(rec) -> bool:
|
||||
if not rec or rec.get("ended_at") or not pid_alive(rec.get("pid")):
|
||||
return False
|
||||
# Same pid, but a younger process: the pid was reused after the run
|
||||
# died without finishing its record. If either time cannot be read,
|
||||
# trust the pid.
|
||||
try:
|
||||
started = datetime.fromisoformat(rec["started_at"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return True
|
||||
created = process_created(rec.get("pid"))
|
||||
try:
|
||||
if created is not None and created > started + _CREATED_SLACK:
|
||||
return False
|
||||
except TypeError:
|
||||
pass # started_at carries a UTC offset: keep the pid verdict
|
||||
return True
|
||||
|
||||
|
||||
def _write(rec: dict) -> None:
|
||||
target = path()
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = target.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(rec, indent=2), encoding="utf-8")
|
||||
# On Windows the replace is refused for a moment while another process
|
||||
# (the page polling, a scanner) holds the file open.
|
||||
for attempt in range(10):
|
||||
try:
|
||||
os.replace(tmp, target)
|
||||
return
|
||||
except PermissionError:
|
||||
if attempt == 9:
|
||||
raise
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def acquire(log_path, source=None) -> dict:
|
||||
"""This process's record, written; raises Busy over a live one."""
|
||||
rec = read()
|
||||
if is_live(rec):
|
||||
raise Busy(f"A bot run is already going (pid {rec.get('pid')}, "
|
||||
f"started {rec.get('started_at') or '?'}) — not starting "
|
||||
f"a second one")
|
||||
rec = {"pid": os.getpid(), "started_at": _now(),
|
||||
"log_path": str(log_path),
|
||||
"source": source or os.environ.get("ECW_RUN_SOURCE") or "terminal",
|
||||
# The Portal User for a Portal run (EEWA-32), else Windows.
|
||||
"started_by": (os.environ.get("ECW_STARTED_BY")
|
||||
or os.environ.get("USERNAME") or ""),
|
||||
"ended_at": None, "exit_code": None,
|
||||
"stopped_by_operator": False}
|
||||
_write(rec)
|
||||
return rec
|
||||
|
||||
|
||||
def finish(exit_code) -> None:
|
||||
"""End this process's own record. Never raises."""
|
||||
try:
|
||||
rec = read()
|
||||
if not rec or rec.get("pid") != os.getpid():
|
||||
return
|
||||
rec.update(ended_at=_now(), exit_code=exit_code)
|
||||
_write(rec)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def mark_stopped(pid) -> None:
|
||||
"""The page killed run `pid`: say so on its record."""
|
||||
rec = read()
|
||||
if not rec or rec.get("pid") != pid or rec.get("ended_at"):
|
||||
return
|
||||
rec.update(ended_at=_now(), exit_code=None, stopped_by_operator=True)
|
||||
_write(rec)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""
|
||||
Run log.
|
||||
|
||||
Python buffers stdout when it is not a terminal, so a background run that hung
|
||||
produced a zero-byte log and there was no way to see how far it had got — the
|
||||
only evidence left was a screenshot. Every run now tees to
|
||||
`Logs\\run_<timestamp>.log`, flushed per line, so a stuck or killed run still
|
||||
leaves a readable trail.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
from .. import settings as cfg
|
||||
|
||||
|
||||
class _Tee:
|
||||
def __init__(self, stream, handle):
|
||||
self._stream = stream
|
||||
self._handle = handle
|
||||
|
||||
def write(self, text):
|
||||
try:
|
||||
self._stream.write(text)
|
||||
self._stream.flush()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self._handle.write(text)
|
||||
self._handle.flush() # a killed run must still have its log
|
||||
except Exception:
|
||||
pass
|
||||
return len(text)
|
||||
|
||||
def flush(self):
|
||||
for target in (self._stream, self._handle):
|
||||
try:
|
||||
target.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def isatty(self):
|
||||
return getattr(self._stream, "isatty", lambda: False)()
|
||||
|
||||
|
||||
def _force_utf8_console() -> None:
|
||||
"""
|
||||
Make the console accept the characters this bot actually prints.
|
||||
|
||||
Windows hands Python a cp1252 console, which cannot encode an em dash or an
|
||||
arrow — and nearly every log line here has one. The tee below swallows the
|
||||
resulting UnicodeEncodeError so a run never dies of it, which is right but
|
||||
means those lines were being **silently dropped from the console** while
|
||||
landing correctly in the log file. Watching a run and reading its log
|
||||
afterwards showed different things, which is a nasty way to debug.
|
||||
|
||||
`errors="replace"` as well as the encoding: a console that still cannot
|
||||
represent a character should print a box, not raise.
|
||||
"""
|
||||
for stream in (sys.__stdout__, sys.__stderr__):
|
||||
try:
|
||||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass # not a real console, or too old to care
|
||||
|
||||
def start() -> "object":
|
||||
"""Begin teeing stdout/stderr. Returns the log path."""
|
||||
_force_utf8_console()
|
||||
cfg.LOGS.mkdir(parents=True, exist_ok=True)
|
||||
path = cfg.LOGS / f"run_{datetime.now():%Y%m%d_%H%M%S}.log"
|
||||
handle = open(path, "a", encoding="utf-8", buffering=1)
|
||||
handle.write(f"\n{'=' * 70}\n{datetime.now():%Y-%m-%d %H:%M:%S} — run start\n"
|
||||
f"{'=' * 70}\n")
|
||||
sys.stdout = _Tee(sys.__stdout__, handle)
|
||||
sys.stderr = _Tee(sys.__stderr__, handle)
|
||||
return path
|
||||
@@ -0,0 +1,396 @@
|
||||
"""
|
||||
Read rows out of a downloaded report. No eCW knowledge, no dependencies.
|
||||
|
||||
**Why not openpyxl.** Same reason `settings.py` parses `.env` by hand and the
|
||||
tests run on plain asserts: these machines are locked down and every added
|
||||
package is one more approval to chase. An `.xlsx` is a zip of XML, and the one
|
||||
eCW produces is about as simple as the format gets, so `zipfile` plus
|
||||
`xml.etree` from the stdlib is enough. If eCW ever ships something genuinely
|
||||
complicated, that is the moment to argue for openpyxl — not now.
|
||||
|
||||
**Three formats, because eCW has sent more than one.** Its "EXCEL" export is a
|
||||
real xlsx today (verified against the 391 KB file it produced on 2026-09-07),
|
||||
but the same menu on other builds serves an HTML table with an `.xls` name, and
|
||||
the CSV item next to it is one keystroke away. Sniffing the file's own bytes
|
||||
rather than trusting its extension is what keeps a format change from reading as
|
||||
a parsing bug.
|
||||
|
||||
What the eCW xlsx actually looks like, since it shapes the parser:
|
||||
|
||||
<row r="1">
|
||||
<c r="A1" t="str"><v></v></c>
|
||||
<c r="C1" t="str"><v>CLAIMS#</v></c>
|
||||
...
|
||||
|
||||
Every value is an inline `t="str"`, and there is **no `sharedStrings.xml`** — so
|
||||
no string table to resolve. The shared-string path below is still implemented,
|
||||
because "this build does not use it" is not the same as "no build will".
|
||||
"""
|
||||
|
||||
import csv
|
||||
import re
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from .text import first_line
|
||||
|
||||
|
||||
def _tag(name: str) -> str:
|
||||
"""Local tag name, with the OOXML namespace stripped."""
|
||||
return name.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def _col_index(ref: str) -> int:
|
||||
"""`"D2"` → 3. Column letters are base-26 with no zero digit."""
|
||||
letters = re.match(r"[A-Z]+", ref or "")
|
||||
if not letters:
|
||||
return 0
|
||||
n = 0
|
||||
for ch in letters.group(0):
|
||||
n = n * 26 + (ord(ch) - 64)
|
||||
return n - 1
|
||||
|
||||
|
||||
def _shared_strings(zf: zipfile.ZipFile) -> list:
|
||||
if "xl/sharedStrings.xml" not in zf.namelist():
|
||||
return []
|
||||
out = []
|
||||
root = ElementTree.fromstring(zf.read("xl/sharedStrings.xml"))
|
||||
for si in root:
|
||||
# A shared string can be split across several <r><t> runs.
|
||||
out.append("".join(t.text or "" for t in si.iter()
|
||||
if _tag(t.tag) == "t"))
|
||||
return out
|
||||
|
||||
|
||||
def _read_xlsx(path: Path) -> list:
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
sheets = [n for n in zf.namelist()
|
||||
if n.startswith("xl/worksheets/sheet")]
|
||||
if not sheets:
|
||||
raise ValueError(f"{path.name} is a zip but has no worksheet")
|
||||
shared = _shared_strings(zf)
|
||||
root = ElementTree.fromstring(zf.read(sorted(sheets)[0]))
|
||||
|
||||
rows = []
|
||||
for row in root.iter():
|
||||
if _tag(row.tag) != "row":
|
||||
continue
|
||||
cells = {}
|
||||
for c in row:
|
||||
if _tag(c.tag) != "c":
|
||||
continue
|
||||
# Addressed by column letter, never by position: a cell with no
|
||||
# value can be omitted entirely, and reading positionally would
|
||||
# then shift every column after it.
|
||||
idx = _col_index(c.get("r", ""))
|
||||
kind = c.get("t")
|
||||
text = ""
|
||||
if kind == "s": # shared-string index
|
||||
v = c.find("./{*}v")
|
||||
if v is not None and (v.text or "").isdigit():
|
||||
i = int(v.text)
|
||||
text = shared[i] if i < len(shared) else ""
|
||||
elif kind == "inlineStr":
|
||||
text = "".join(t.text or "" for t in c.iter()
|
||||
if _tag(t.tag) == "t")
|
||||
else: # str, n, b, or untyped
|
||||
v = c.find("./{*}v")
|
||||
text = (v.text or "") if v is not None else ""
|
||||
cells[idx] = text.strip()
|
||||
width = max(cells) + 1 if cells else 0
|
||||
rows.append([cells.get(i, "") for i in range(width)])
|
||||
return rows
|
||||
|
||||
|
||||
class _TableParser(HTMLParser):
|
||||
"""Pull the first table's rows out of an HTML-as-.xls export."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.rows, self._row, self._cell, self._in_cell = [], None, [], False
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "tr":
|
||||
self._row = []
|
||||
elif tag in ("td", "th"):
|
||||
self._in_cell, self._cell = True, []
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in ("td", "th") and self._row is not None:
|
||||
self._row.append("".join(self._cell).strip())
|
||||
self._in_cell = False
|
||||
elif tag == "tr" and self._row is not None:
|
||||
if any(self._row):
|
||||
self.rows.append(self._row)
|
||||
self._row = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._in_cell:
|
||||
self._cell.append(data)
|
||||
|
||||
|
||||
def _read_html(path: Path) -> list:
|
||||
p = _TableParser()
|
||||
p.feed(path.read_text(encoding="utf-8", errors="replace"))
|
||||
return p.rows
|
||||
|
||||
|
||||
def _read_csv(path: Path) -> list:
|
||||
text = path.read_text(encoding="utf-8-sig", errors="replace")
|
||||
# Sniff on the header line only: a patient name with a comma in it would
|
||||
# otherwise outvote the real delimiter.
|
||||
first = text.splitlines()[0] if text else ""
|
||||
delim = "\t" if first.count("\t") > first.count(",") else ","
|
||||
return [[c.strip() for c in row]
|
||||
for row in csv.reader(text.splitlines(), delimiter=delim)
|
||||
if any(c.strip() for c in row)]
|
||||
|
||||
|
||||
def read_rows(path) -> list:
|
||||
"""
|
||||
Every row of the report as a list of strings, blank rows dropped.
|
||||
|
||||
The format is decided by the file's first bytes, not its extension.
|
||||
"""
|
||||
path = Path(path)
|
||||
head = path.open("rb").read(8)
|
||||
|
||||
if head.startswith(b"PK"):
|
||||
return _read_xlsx(path)
|
||||
if head.lstrip()[:1] == b"<":
|
||||
return _read_html(path)
|
||||
if head.startswith(b"\xd0\xcf\x11\xe0"):
|
||||
# Legacy OLE2 .xls. Not a zip and not XML — a real binary format that
|
||||
# would need a parser of its own. eCW has not sent one, so this says so
|
||||
# instead of silently returning nothing.
|
||||
raise ValueError(
|
||||
f"{path.name} is a legacy binary .xls (OLE2). Nothing here can "
|
||||
f"read that. Set CLAIM_REPORT_FORMAT=CSV, which eCW offers in the "
|
||||
f"same menu, or add a parser for it.")
|
||||
return _read_csv(path)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Writing (the categorization step's output)
|
||||
# =============================================================================
|
||||
# The read side above has no dependency on openpyxl; the write side keeps that
|
||||
# constraint too. What follows is the minimal set of OOXML parts real Excel
|
||||
# actually requires to open a file without a "repair" prompt — more than the
|
||||
# reader above needs (it only ever looks at xl/worksheets/sheet*.xml), because
|
||||
# this file is meant to be opened by a person, not just this codebase.
|
||||
|
||||
_CONTENT_TYPES = (
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||||
'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
|
||||
'<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
|
||||
'<Default Extension="xml" ContentType="application/xml"/>'
|
||||
'<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'
|
||||
'<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>'
|
||||
'</Types>'
|
||||
)
|
||||
|
||||
_ROOT_RELS = (
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||||
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
|
||||
'<Relationship Id="rId1" '
|
||||
'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
|
||||
'Target="xl/workbook.xml"/>'
|
||||
'</Relationships>'
|
||||
)
|
||||
|
||||
_WORKBOOK = (
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||||
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
|
||||
'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
|
||||
'<sheets><sheet name="Sheet1" sheetId="1" r:id="rId1"/></sheets>'
|
||||
'</workbook>'
|
||||
)
|
||||
|
||||
_WORKBOOK_RELS = (
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||||
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
|
||||
'<Relationship Id="rId1" '
|
||||
'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '
|
||||
'Target="worksheets/sheet1.xml"/>'
|
||||
'</Relationships>'
|
||||
)
|
||||
|
||||
|
||||
# XML 1.0 does not allow these control characters anywhere in a document,
|
||||
# escaped or not. `escape()` only handles &, <, > — a cell value carrying one
|
||||
# of these would still produce a file that "wrote successfully" but is
|
||||
# malformed XML, silently corrupt rather than caught. Stripped, not escaped:
|
||||
# there is no valid XML representation for them at all.
|
||||
_XML_ILLEGAL = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f]")
|
||||
|
||||
|
||||
def _col_letter(idx: int) -> str:
|
||||
"""0 -> 'A', 25 -> 'Z', 26 -> 'AA'. Inverse of `_col_index` above."""
|
||||
letters = ""
|
||||
idx += 1
|
||||
while idx > 0:
|
||||
idx, rem = divmod(idx - 1, 26)
|
||||
letters = chr(65 + rem) + letters
|
||||
return letters
|
||||
|
||||
|
||||
def write_xlsx(path, rows: list) -> None:
|
||||
"""
|
||||
Write `rows` (a header row plus data rows, each a list of values) as a
|
||||
minimal single-sheet xlsx.
|
||||
|
||||
Inline `t="str"` cells, no sharedStrings — the same shape `_read_xlsx`
|
||||
above already parses, and what eCW's own export already sends (see the
|
||||
module docstring). Every value is written through `str()`, so callers can
|
||||
pass numbers or None without pre-formatting.
|
||||
"""
|
||||
path = Path(path)
|
||||
body = []
|
||||
for r, row in enumerate(rows, start=1):
|
||||
cells = []
|
||||
for c, value in enumerate(row):
|
||||
text = escape(_XML_ILLEGAL.sub("", str(value)))
|
||||
cells.append(f'<c r="{_col_letter(c)}{r}" t="str"><v>{text}</v></c>')
|
||||
body.append(f'<row r="{r}">{"".join(cells)}</row>')
|
||||
sheet = (
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||||
'<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'
|
||||
f'<sheetData>{"".join(body)}</sheetData></worksheet>'
|
||||
)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr("[Content_Types].xml", _CONTENT_TYPES)
|
||||
zf.writestr("_rels/.rels", _ROOT_RELS)
|
||||
zf.writestr("xl/workbook.xml", _WORKBOOK)
|
||||
zf.writestr("xl/_rels/workbook.xml.rels", _WORKBOOK_RELS)
|
||||
zf.writestr("xl/worksheets/sheet1.xml", sheet)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Writing next to a file the operator may have open
|
||||
# =============================================================================
|
||||
# Both dated outputs (Output\Medcor Ready_<date>.xlsx and Output\Charge
|
||||
# Summary_<date>.xlsx) are files people work in: the operator hand-fills blank
|
||||
# Group Names in one and reviews claims in the other, and a same-day re-run
|
||||
# must neither overwrite that work nor lose its own rows over it.
|
||||
#
|
||||
# Until 2026-10-05 each caller had its own copy of "rename the old file aside,
|
||||
# then write the new one". On Windows that rename FAILS while the old file is
|
||||
# open in Excel ([WinError 32] "being used by another process"), the caller
|
||||
# caught the error, and the whole run's sheet was thrown away (review item
|
||||
# D10) — on exactly the second run of a day, when the first run's sheet is the
|
||||
# one most likely to be open. This one helper replaces both copies, and its
|
||||
# answer to a locked file is to write the new rows under a free name next to
|
||||
# it instead.
|
||||
|
||||
def _first_free_name(out_path: Path, stamp: str) -> Path:
|
||||
"""
|
||||
`<stem>__<HHMMSS>.xlsx`, else `<stem>__<HHMMSS>_2.xlsx`, `_3` ... — the
|
||||
first one not already on disk. The `_<n>` suffix exists because two
|
||||
writes inside the same wall-clock second do happen (back-to-back in this
|
||||
project's own tests, and a fast manual re-run), and Windows refuses to
|
||||
rename or write onto a path that is already taken.
|
||||
"""
|
||||
candidate = out_path.with_stem(f"{out_path.stem}__{stamp}")
|
||||
n = 2
|
||||
while candidate.exists():
|
||||
candidate = out_path.with_stem(f"{out_path.stem}__{stamp}_{n}")
|
||||
n += 1
|
||||
return candidate
|
||||
|
||||
|
||||
def _move_back(out_path: Path, backup: Path, log_tag: str) -> None:
|
||||
"""
|
||||
The old file was renamed aside but the new one could not be written in
|
||||
its place: put the old one back under the name the operator knows it by,
|
||||
so that — when the move back succeeds — a failed write leaves the folder
|
||||
as it was found.
|
||||
|
||||
Whatever is at `out_path` now is the failed write's own half-file — the
|
||||
old file was moved off that name a moment ago — so it is removed first;
|
||||
a half-written zip under the real name would open as a "corrupt file"
|
||||
error instead of the operator's sheet. If the removal or the move back
|
||||
fails (a virus scanner holding the half-file, say), the old file is
|
||||
still safe under `backup`, but the usual name may now hold that
|
||||
incomplete file — so the log says both: which name not to trust, and
|
||||
where the operator's sheet really is.
|
||||
"""
|
||||
try:
|
||||
out_path.unlink(missing_ok=True)
|
||||
backup.rename(out_path)
|
||||
except OSError as exc:
|
||||
print(f"{log_tag} {out_path.name} may be an incomplete file — your "
|
||||
f"earlier sheet is {backup.name} (could not move it back: "
|
||||
f"{first_line(exc)})")
|
||||
|
||||
|
||||
def write_xlsx_keep_old(out_path, rows: list, log_tag: str) -> Path:
|
||||
"""
|
||||
`write_xlsx(out_path, rows)`, without ever overwriting a file already at
|
||||
`out_path` and without ever losing `rows` because of one. Returns the
|
||||
path actually written, which callers must report — it is not always
|
||||
`out_path`.
|
||||
|
||||
* nothing at `out_path` -> written there.
|
||||
* a file already there -> renamed aside to the first free
|
||||
`<stem>__<HHMMSS>[_<n>].xlsx` (kept, not deleted: it may hold the
|
||||
operator's own edits), then `rows` written to `out_path`. Logged as
|
||||
"<tag> <name> already existed — kept as <backup>".
|
||||
* that rename, or the write after it, fails with OSError (the file is
|
||||
open in Excel, or locked by a sync/virus scanner) -> `rows` go to the
|
||||
first free `<stem>__<HHMMSS>[_<n>].xlsx` instead. Logged as "<tag>
|
||||
<name> is open or locked (<error>) — saved this run as <fallback>
|
||||
instead". The old file: when the rename itself failed, it never
|
||||
moved; when the rename went through and the write after it failed,
|
||||
it is put back under its usual name (`_move_back`) when possible —
|
||||
and if that fails too, it stays at the `__HHMMSS` backup and the log
|
||||
names both files, because the usual name may then hold the
|
||||
incomplete write (final review, 2026-10-05: this bullet used to say
|
||||
the old file was always "left where and as it was").
|
||||
|
||||
If the fallback write fails as well, that error propagates: both callers
|
||||
already catch it and tell the operator, and there is nowhere left to put
|
||||
the rows that would not be a guess.
|
||||
|
||||
The fallback writes directly under its final name rather than to a temp
|
||||
file moved into place (as the D10 review suggested): virus scanners
|
||||
commonly open a freshly closed file for a moment, so an extra rename
|
||||
would be one more step that can hit the same WinError 32 this exists to
|
||||
get around.
|
||||
"""
|
||||
out_path = Path(out_path)
|
||||
if not out_path.exists():
|
||||
write_xlsx(out_path, rows)
|
||||
return out_path
|
||||
|
||||
stamp = datetime.now().strftime("%H%M%S")
|
||||
backup = _first_free_name(out_path, stamp)
|
||||
moved = False
|
||||
try:
|
||||
out_path.rename(backup)
|
||||
moved = True
|
||||
write_xlsx(out_path, rows)
|
||||
except Exception as exc:
|
||||
# Any failure after the old file moved puts it back first — even one
|
||||
# that is a bug rather than a lock — so nothing is left renamed
|
||||
# behind a half-written file.
|
||||
if moved:
|
||||
_move_back(out_path, backup, log_tag)
|
||||
if not isinstance(exc, OSError):
|
||||
raise
|
||||
fallback = _first_free_name(out_path, stamp)
|
||||
write_xlsx(fallback, rows)
|
||||
print(f"{log_tag} {out_path.name} is open or locked "
|
||||
f"({first_line(exc)}) — saved this run as {fallback.name} "
|
||||
f"instead")
|
||||
return fallback
|
||||
|
||||
print(f"{log_tag} {out_path.name} already existed — kept as "
|
||||
f"{backup.name}")
|
||||
return out_path
|
||||
@@ -0,0 +1,23 @@
|
||||
"""
|
||||
core/text.py — small text helpers shared by core/, conditions/ and flows/.
|
||||
|
||||
Lives in core/ because core/ must not import from conditions/, pages/ or
|
||||
flows/, and all three need it (review of unit D, 2026-10-05: the condition
|
||||
runner, the Charge Summary and the flow each had their own copy).
|
||||
"""
|
||||
|
||||
|
||||
def first_line(value) -> str:
|
||||
"""
|
||||
The first non-blank line of `value`'s text, stripped.
|
||||
|
||||
For an exception: Playwright's messages carry a whole call log after the
|
||||
first line, which is all a log line or a summary cell wants. An exception
|
||||
with an empty message gives its class name instead — a bare
|
||||
`str(exc).splitlines()[0]` raises IndexError on one. Any other empty
|
||||
value (an error string that came back blank) gives "".
|
||||
"""
|
||||
lines = str(value).strip().splitlines()
|
||||
if lines:
|
||||
return lines[0].strip()
|
||||
return type(value).__name__ if isinstance(value, BaseException) else ""
|
||||
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
Claim watchdog (2026-10-22).
|
||||
|
||||
A frozen page hangs the bot for good: Playwright's `evaluate` has no
|
||||
timeout, and on 2026-10-21 / 22 a run sat 28 minutes on one claim after
|
||||
its form opened — no error, no end, the Runs page frozen on its last event.
|
||||
|
||||
While the claims are checked (`watching()`), the flow beats at every step
|
||||
of a claim, and every message box beats as it opens and closes. When no
|
||||
beat comes for the limit (`WATCHDOG_MINUTES`, never under a box's timeout
|
||||
plus two minutes), a thread here:
|
||||
|
||||
1. closes this process's Playwright driver (`node.exe`, a child of this
|
||||
process). The frozen call then returns with an error, so the claim
|
||||
fails like any other and the flow carries on to its next step. With
|
||||
BROWSER_CDP_URL only the driver goes: the operator's Chrome stays.
|
||||
2. marks the watchdog fired. The flow's next `check()` — at the top of
|
||||
the next claim — raises `Stalled`, and the Run ends with exit 8, a Run
|
||||
Error naming where it stopped, and the rest of the queue Not Reached.
|
||||
3. if the main thread still does not come back (GRACE_S), writes the Run
|
||||
Error and the end itself and exits the process with code 8 — a hung
|
||||
bot never again runs without an end.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
import subprocess
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .. import settings as cfg
|
||||
from . import run_events, run_lock
|
||||
|
||||
EXIT_CODE = 8
|
||||
POLL_S = 5.0 # how often the thread looks
|
||||
GRACE_S = 120.0 # after the kill, how long the main thread gets
|
||||
|
||||
_lock = threading.Lock()
|
||||
_state = {"armed": False, "last": 0.0, "where": "", "limit": 0.0,
|
||||
"fired": False, "answered": False, "message": ""}
|
||||
_thread = None
|
||||
|
||||
|
||||
class Stalled(RuntimeError):
|
||||
"""No progress for the limit: the Run stops (exit 8)."""
|
||||
|
||||
|
||||
def limit_seconds() -> float:
|
||||
"""WATCHDOG_MINUTES in seconds, never under a message box's timeout
|
||||
plus two minutes (a box left open is not a hang); 0 = off."""
|
||||
if not cfg.WATCHDOG_MINUTES:
|
||||
return 0
|
||||
return max(cfg.WATCHDOG_MINUTES * 60, cfg.MSGBOX_TIMEOUT_SECONDS + 120)
|
||||
|
||||
|
||||
def beat(where: str = "") -> None:
|
||||
"""Progress: the clock starts again, and `where` names the step."""
|
||||
with _lock:
|
||||
_state["last"] = time.monotonic()
|
||||
if where:
|
||||
_state["where"] = where
|
||||
|
||||
|
||||
def has_fired() -> bool:
|
||||
return _state["fired"]
|
||||
|
||||
|
||||
def message() -> str:
|
||||
return _state["message"]
|
||||
|
||||
|
||||
def check() -> None:
|
||||
"""Raise `Stalled` once the watchdog has fired."""
|
||||
if _state["fired"]:
|
||||
_state["answered"] = True
|
||||
raise Stalled(_state["message"])
|
||||
|
||||
|
||||
@contextmanager
|
||||
def watching(limit_s: float):
|
||||
"""Watch the block; `limit_s` 0 or less = not at all."""
|
||||
global _thread
|
||||
with _lock:
|
||||
_state.update(armed=limit_s > 0, last=time.monotonic(), where="",
|
||||
limit=float(limit_s), fired=False, answered=False,
|
||||
message="")
|
||||
if limit_s > 0 and (_thread is None or not _thread.is_alive()):
|
||||
_thread = threading.Thread(target=_watch, name="claim-watchdog",
|
||||
daemon=True)
|
||||
_thread.start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with _lock:
|
||||
_state["armed"] = False
|
||||
|
||||
|
||||
def _watch() -> None:
|
||||
while True:
|
||||
time.sleep(POLL_S)
|
||||
with _lock:
|
||||
due = (_state["armed"] and not _state["fired"]
|
||||
and time.monotonic() - _state["last"] >= _state["limit"])
|
||||
if due:
|
||||
minutes = _state["limit"] / 60
|
||||
_state["fired"] = True
|
||||
_state["message"] = (
|
||||
f"No progress for {minutes:g} min at "
|
||||
f"{_state['where'] or 'the claims'} — the page froze; "
|
||||
f"the browser was closed and the run stopped")
|
||||
if not due:
|
||||
continue
|
||||
print(f"[WATCHDOG] {_state['message']}")
|
||||
run_events.warning("watchdog", _state["message"])
|
||||
try:
|
||||
_kill()
|
||||
except Exception as exc:
|
||||
print(f"[WATCHDOG] Could not close the driver: {exc}")
|
||||
end = time.monotonic() + GRACE_S
|
||||
while time.monotonic() < end and not _state["answered"]:
|
||||
if not _state["armed"]: # the claims ended another way
|
||||
break
|
||||
time.sleep(POLL_S / 5)
|
||||
if not _state["answered"] and _state["armed"]:
|
||||
print("[WATCHDOG] The run did not come back — ending it")
|
||||
run_events.run_error("stalled", _state["message"], EXIT_CODE)
|
||||
run_events.emit("run_end", exit_code=EXIT_CODE)
|
||||
run_events.close()
|
||||
run_lock.finish(EXIT_CODE)
|
||||
_exit(EXIT_CODE)
|
||||
|
||||
|
||||
def _kill() -> None:
|
||||
"""Close this process's Playwright driver (node.exe) and what it
|
||||
started. Nothing else of this process is touched."""
|
||||
ps = (f"Get-CimInstance Win32_Process -Filter "
|
||||
f"'ParentProcessId={os.getpid()} and Name=\"node.exe\"' | "
|
||||
f"ForEach-Object {{ $_.ProcessId }}")
|
||||
out = subprocess.run(["powershell", "-NoProfile", "-Command", ps],
|
||||
capture_output=True, text=True, timeout=30).stdout
|
||||
for pid in out.split():
|
||||
if pid.isdigit():
|
||||
subprocess.run(["taskkill", "/PID", pid, "/T", "/F"],
|
||||
capture_output=True, timeout=30)
|
||||
|
||||
|
||||
def _exit(code: int) -> None:
|
||||
os._exit(code)
|
||||
Reference in new issue
Block a user