Self-healing agent (supervisor, recovery, login fixes, operator mail) and Portal sage redesign
Snapshot of D:\Ecw Process a11c9ff (2026-10-30). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fafa5285eb
commit
d500f750da
26 files changed
+1704
-357
No files matched your search
@@ -12,6 +12,7 @@ spreadsheet, one client instead of a loop, and the second factor is a mailed
|
||||
approval link rather than an authenticator code — see `twofactor.py`.
|
||||
"""
|
||||
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -32,6 +33,25 @@ from .twofactor import handle_2fa
|
||||
|
||||
|
||||
SECURITY_IMAGE = "text=VERIFY YOUR SECURITY IMAGE"
|
||||
# The page's own go-on control, by its words: its markup has never been
|
||||
# saved (the first page the bot clicks through is, as security_image_page).
|
||||
_CONTINUE = re.compile(r"^\s*(continue|ok)\s*$", re.I)
|
||||
|
||||
|
||||
def continue_security_image(page: Page) -> bool:
|
||||
"""
|
||||
Click Continue on eCW's "VERIFY YOUR SECURITY IMAGE" page (operator,
|
||||
2026-10-30: the run must not wait for a person). True once clicked;
|
||||
False when no visible Continue/OK control is there, so the caller falls
|
||||
back to waiting for a person.
|
||||
"""
|
||||
shot(page, "security_image_page")
|
||||
for role in ("button", "link"):
|
||||
btn = page.get_by_role(role, name=_CONTINUE).filter(visible=True)
|
||||
if btn.count():
|
||||
btn.first.click(timeout=10_000)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def wait_after_username(page: Page, timeout_ms: int, image: bool = True) -> str:
|
||||
@@ -125,15 +145,20 @@ def _do_login(page: Page, context, client_name: str):
|
||||
|
||||
# Some logins interpose a security-image confirmation page — sometimes
|
||||
# seconds after Next (2026-10-28 17:59: missed by a one-look check).
|
||||
# No box with QUIET_BOXES (2026-10-29, bulk runs): the wait below is
|
||||
# what holds the run, the box only ever said so.
|
||||
if wait_after_username(page, cfg.LOGIN_TIMEOUT) == "image":
|
||||
print("[LOGIN] Security image page")
|
||||
msgbox.info_box(
|
||||
f"eCW Bot — Security Image ({client_name})",
|
||||
"The 'VERIFY YOUR SECURITY IMAGE' page came up.\n"
|
||||
"Click Continue/OK in the browser.\n\n"
|
||||
"Click OK once the password page appears.",
|
||||
needs_person=True,
|
||||
)
|
||||
if continue_security_image(page):
|
||||
print("[LOGIN] Security image page — clicked Continue")
|
||||
else:
|
||||
print(f"[LOGIN] Security image page — no Continue found; click it "
|
||||
f"in the browser (waiting up to {cfg.LOGIN_TIMEOUT // 1000}s)")
|
||||
msgbox.info_box(
|
||||
f"eCW Bot — Security Image ({client_name})",
|
||||
"The 'VERIFY YOUR SECURITY IMAGE' page came up.\n"
|
||||
"Click Continue/OK in the browser.\n\n"
|
||||
"Click OK once the password page appears.",
|
||||
)
|
||||
wait_after_username(page, cfg.LOGIN_TIMEOUT, image=False)
|
||||
# A renamed password field or Log In button still works through its
|
||||
# fallback (selectors/login.py); this says so in the log.
|
||||
|
||||
@@ -185,6 +185,61 @@ def _challenge_with_resend(page: Page, client_name: str,
|
||||
f"No eCW 2FA mail arrived (tried {cfg.OTP_RESEND_ATTEMPTS} resend(s)).")
|
||||
|
||||
|
||||
def _picked_up(page: Page, seconds: int) -> bool:
|
||||
"""The login page left the push screen (home, or the notice gone)."""
|
||||
for _ in range(max(1, seconds)):
|
||||
if visible(page, sel.SEL_HOME, timeout=500) or \
|
||||
not visible(page, sel.SEL_PUSH_NOTICE, timeout=500):
|
||||
return True
|
||||
page.wait_for_timeout(500)
|
||||
return False
|
||||
|
||||
|
||||
def _approve_until_picked_up(page: Page, context, client_name: str,
|
||||
ch) -> None:
|
||||
"""
|
||||
Approve the link; if the login page does not pick it up within
|
||||
OTP_PICKUP_SECONDS, press Resend and approve the new mail, up to
|
||||
OTP_PICKUP_RESENDS times (live 2026-10-29: three logins lost to late
|
||||
mail with Resend still on screen). Returns once picked up, or when no
|
||||
resend is left — the caller's wait for home then reports it.
|
||||
"""
|
||||
for resend in range(cfg.OTP_PICKUP_RESENDS + 1):
|
||||
if not _approve_in_new_tab(context, ch.value):
|
||||
raise RuntimeError("Approval link opened but could not be confirmed")
|
||||
# eCW's login page polls for the approval; it moves on by itself. But
|
||||
# it may already have stopped polling while the mail was in flight —
|
||||
# the link is then spent and confirming it changed nothing, so say that
|
||||
# rather than letting the caller wait out LOGIN_TIMEOUT for a home
|
||||
# screen that is never coming.
|
||||
page.wait_for_timeout(2_000)
|
||||
why = dead_end_reason(page)
|
||||
if why:
|
||||
shot(page, "2fa_timed_out")
|
||||
raise LoginTimedOut(
|
||||
f"The approval was confirmed, but the login page had already "
|
||||
f"given up: {why}.\n"
|
||||
f" The mail arrived too late to be useful. Starting over.")
|
||||
print("[2FA] Waiting for the login page to pick up the approval")
|
||||
if _picked_up(page, cfg.OTP_PICKUP_SECONDS):
|
||||
return
|
||||
if resend == cfg.OTP_PICKUP_RESENDS:
|
||||
return
|
||||
print(f"[2FA] Not picked up in {cfg.OTP_PICKUP_SECONDS}s — resending")
|
||||
shot(page, "2fa_not_picked_up")
|
||||
if not _resend_push(page):
|
||||
return
|
||||
since = datetime.now(timezone.utc)
|
||||
run_events.login_step("twofa_resent")
|
||||
ch = mailbox.get_challenge(since=since, client_name=client_name,
|
||||
allow_manual=False)
|
||||
if ch.kind != "link":
|
||||
print("[2FA] No new approval mail after the resend")
|
||||
return
|
||||
print(f"[2FA] Resend {resend + 1}/{cfg.OTP_PICKUP_RESENDS}: "
|
||||
f"approving the new mail")
|
||||
|
||||
|
||||
def handle_2fa(page: Page, context, client_name: str,
|
||||
clicked_at: datetime) -> None:
|
||||
"""Deal with whatever second factor eCW shows. No-op when there isn't one."""
|
||||
@@ -206,22 +261,7 @@ def handle_2fa(page: Page, context, client_name: str,
|
||||
ch = _challenge_with_resend(page, client_name, clicked_at)
|
||||
|
||||
if ch.kind == "link":
|
||||
if not _approve_in_new_tab(context, ch.value):
|
||||
raise RuntimeError("Approval link opened but could not be confirmed")
|
||||
# eCW's login page polls for the approval; it moves on by itself. But
|
||||
# it may already have stopped polling while the mail was in flight —
|
||||
# the link is then spent and confirming it changed nothing, so say that
|
||||
# rather than letting the caller wait out LOGIN_TIMEOUT for a home
|
||||
# screen that is never coming.
|
||||
page.wait_for_timeout(2_000)
|
||||
why = dead_end_reason(page)
|
||||
if why:
|
||||
shot(page, "2fa_timed_out")
|
||||
raise LoginTimedOut(
|
||||
f"The approval was confirmed, but the login page had already "
|
||||
f"given up: {why}.\n"
|
||||
f" The mail arrived too late to be useful. Starting over.")
|
||||
print("[2FA] Waiting for the login page to pick up the approval")
|
||||
_approve_until_picked_up(page, context, client_name, ch)
|
||||
return
|
||||
|
||||
if ch.kind in ("code", "totp"):
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
The agent's email to the operator (operator, 2026-10-30): when something
|
||||
technical goes wrong — a login that fails, a screen that sticks, a run that
|
||||
stops — say what happened, what the agent did about it and whether that
|
||||
worked, instead of leaving it for someone to find in the morning.
|
||||
|
||||
Sent from the bot's own mailbox (the 2FA mailbox's account, over SMTP). What
|
||||
it may carry is fixed here, not by the caller:
|
||||
* screenshots of login and error pages only (`_EMAILABLE`), PNG only — a
|
||||
claim screen or a page's HTML carries PHI and never leaves the PC;
|
||||
* no claim numbers: any run of 6+ digits in the subject or body is masked.
|
||||
|
||||
Never fatal: a mail that cannot be sent is a log line, and the run goes on.
|
||||
"""
|
||||
|
||||
import re
|
||||
import smtplib
|
||||
import ssl
|
||||
from email.message import EmailMessage
|
||||
from pathlib import Path
|
||||
|
||||
from .. import settings as cfg
|
||||
|
||||
# `<HHMMSS>_<step>.png` as page_utils.shot() names them. Login and error
|
||||
# pages only: not "logged_in" or "landing" (eCW's home can list patients).
|
||||
_EMAILABLE = re.compile(
|
||||
r"^\d{6}_(security_image_page|login_unconfirmed|username_error|"
|
||||
r"username_step_disabled|no_login_form|captcha|restricted|"
|
||||
r"password_expired|password_expiry_no_cancel|2fa_[a-z_]+|"
|
||||
r"approval_(page|done|no_button))\.png$")
|
||||
_DIGITS = re.compile(r"\d{6,}")
|
||||
|
||||
|
||||
def _mask(text: str) -> str:
|
||||
return _DIGITS.sub("<claim>", text or "")
|
||||
|
||||
|
||||
def emailable(path) -> bool:
|
||||
return bool(_EMAILABLE.match(Path(path).name))
|
||||
|
||||
|
||||
def _transport(msg: EmailMessage) -> None:
|
||||
with smtplib.SMTP_SSL(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30,
|
||||
context=ssl.create_default_context()) as smtp:
|
||||
smtp.login(cfg.SMTP_USERNAME, cfg.SMTP_PASSWORD)
|
||||
smtp.send_message(msg)
|
||||
|
||||
|
||||
def _send(subject: str, body: str, shots=()) -> bool:
|
||||
if not cfg.NOTIFY_EMAIL_TO:
|
||||
return False
|
||||
msg = EmailMessage()
|
||||
msg["From"] = cfg.SMTP_USERNAME
|
||||
msg["To"] = cfg.NOTIFY_EMAIL_TO
|
||||
msg["Subject"] = f"[eCW Agent] {_mask(subject)}"
|
||||
msg.set_content(_mask(body))
|
||||
for p in shots or ():
|
||||
p = Path(p)
|
||||
if emailable(p) and p.exists():
|
||||
msg.add_attachment(p.read_bytes(), maintype="image",
|
||||
subtype="png", filename=p.name)
|
||||
try:
|
||||
_transport(msg)
|
||||
print(f"[NOTIFY] Mailed: {msg['Subject']}")
|
||||
return True
|
||||
except Exception as exc:
|
||||
print(f"[NOTIFY] Could not mail '{msg['Subject']}': "
|
||||
f"{str(exc).splitlines()[0] if str(exc) else type(exc).__name__}")
|
||||
return False
|
||||
|
||||
|
||||
def incident(title: str, what_happened: str, what_agent_did: str,
|
||||
recovered: bool, shots=()) -> bool:
|
||||
"""One technical problem: what, what the agent did, and the outcome."""
|
||||
outcome = "Recovered - the run goes on." if recovered else \
|
||||
"Not recovered - the agent stopped or is waiting to retry."
|
||||
body = (f"What happened:\n {what_happened}\n\n"
|
||||
f"What the agent did:\n {what_agent_did}\n\n"
|
||||
f"Outcome:\n {outcome}\n\n"
|
||||
f"Run log and screenshots are on the bot PC and in the Portal.")
|
||||
return _send(title, body, shots)
|
||||
|
||||
|
||||
def summary(title: str, lines) -> bool:
|
||||
"""The end of a run: counts and anything left for a person."""
|
||||
return _send(title, "\n".join(lines))
|
||||
@@ -56,6 +56,8 @@ from ..pages import preflight
|
||||
from ..core.text import first_line
|
||||
from ..core.browser import cdp_ready, chrome_proxy_problem, start_chrome
|
||||
from ..core import run_events, watchdog
|
||||
from ..core.page_utils import shot
|
||||
from . import recovery
|
||||
from ..auth.login import login
|
||||
from ..auth.dead_end import LoginTimedOut
|
||||
from ..auth.restricted import Restricted
|
||||
@@ -983,6 +985,7 @@ def _check_claims(page, info) -> None:
|
||||
keep_pages = list(page.context.pages)
|
||||
except Exception:
|
||||
keep_pages = []
|
||||
stuck = {} # recovery.run_claim's streak of unrecovered claims
|
||||
for n, claim_id in enumerate(todo, start=1):
|
||||
# A claim that froze the page stops the run here, before the next
|
||||
# one: the rest of the queue is Not Reached, not 40 failed searches.
|
||||
@@ -998,29 +1001,43 @@ def _check_claims(page, info) -> None:
|
||||
# 2 and 3 were never searched. A page that will not come clean
|
||||
# makes this claim a failed search — Review — and the next claim
|
||||
# clears and checks again.
|
||||
claim_form.clear_screen(page, claim_id, keep_pages)
|
||||
try:
|
||||
r = claim_results.search_claim(page, claim_id)
|
||||
except Exception as exc:
|
||||
# One bad claim must not cost the other 540.
|
||||
print(f"[CLAIM {claim_id}] Failed: {exc}")
|
||||
# Not "not found" (final review, 2026-10-05): that is a fact
|
||||
# about the queue, and a search that raised (browser gone, a
|
||||
# timeout) says nothing about the queue. It is counted apart
|
||||
# and its row is Review, not left blank like a missing claim.
|
||||
r = {"id": claim_id, "count": "?", "found": False, "seconds": 0,
|
||||
"search_error": first_line(exc)}
|
||||
# Built before the read, not after: the review-before-close box is
|
||||
# shown while the form is still open and leads with it too.
|
||||
head = _claim_head(n, len(todo), claim_id, r)
|
||||
if r["found"]:
|
||||
watchdog.beat(f"claim {n}/{len(todo)} {claim_id}: open the form")
|
||||
data = _read_claim_form(page, claim_id, head)
|
||||
elif r.get("search_error"):
|
||||
data = {"claim_no": claim_id,
|
||||
"_search_error": r["search_error"]}
|
||||
else:
|
||||
data = {}
|
||||
def attempt():
|
||||
claim_form.clear_screen(page, claim_id, keep_pages)
|
||||
try:
|
||||
r = claim_results.search_claim(page, claim_id)
|
||||
except Exception as exc:
|
||||
# One bad claim must not cost the other 540.
|
||||
print(f"[CLAIM {claim_id}] Failed: {exc}")
|
||||
# Not "not found" (final review, 2026-10-05): that is a fact
|
||||
# about the queue, and a search that raised (browser gone, a
|
||||
# timeout) says nothing about the queue. It is counted apart
|
||||
# and its row is Review, not left blank like a missing claim.
|
||||
r = {"id": claim_id, "count": "?", "found": False,
|
||||
"seconds": 0, "search_error": first_line(exc)}
|
||||
# Built before the read, not after: the review-before-close box
|
||||
# is shown while the form is still open and leads with it too.
|
||||
r["_head"] = _claim_head(n, len(todo), claim_id, r)
|
||||
if r["found"]:
|
||||
watchdog.beat(f"claim {n}/{len(todo)} {claim_id}: "
|
||||
f"open the form")
|
||||
data = _read_claim_form(page, claim_id, r["_head"])
|
||||
elif r.get("search_error"):
|
||||
data = {"claim_no": claim_id,
|
||||
"_search_error": r["search_error"]}
|
||||
else:
|
||||
data = {}
|
||||
return r, data
|
||||
|
||||
def recover_now():
|
||||
# Operator, 2026-10-30: a stuck screen is the agent's to clear.
|
||||
watchdog.beat(f"claim {n}/{len(todo)} {claim_id}: recover")
|
||||
how = recovery.recover(page, claim_id, keep_pages, _open_search)
|
||||
print(f"[RECOVER {claim_id}] Search page back ({how}) — "
|
||||
f"trying the claim again")
|
||||
run_events.emit("recovered", claim=str(claim_id), how=how)
|
||||
|
||||
r, data = recovery.run_claim(attempt, recover_now, stuck)
|
||||
head = r.pop("_head")
|
||||
# `_error` = the form was opened (or tried) and could not be read:
|
||||
# not "read", whatever the dict holds (review item D2).
|
||||
error = data.get("_error") if data else None
|
||||
@@ -1206,8 +1223,9 @@ def _write_charge_summary(info, outcomes: dict) -> None:
|
||||
f"This run's charge summary is:\n {path.name}")
|
||||
|
||||
|
||||
def _claims_phase(page) -> None:
|
||||
"""Everything on claimLookup.jsp, in the one order that works."""
|
||||
def _open_search(page) -> None:
|
||||
"""The claim search with the run's filters — at the start, and again
|
||||
when flows/recovery.py reopens it after a stuck screen."""
|
||||
leftnav.open_claims(page)
|
||||
print(f"[OK] On: {page.url}")
|
||||
claim_filters.set_claim_status(page) # FIRST — see the module docstring
|
||||
@@ -1215,6 +1233,20 @@ def _claims_phase(page) -> None:
|
||||
claim_filters.set_date_range(page)
|
||||
claim_filters.clear_balance(page)
|
||||
|
||||
|
||||
def _claims_phase(page) -> None:
|
||||
"""Everything on claimLookup.jsp, in the one order that works."""
|
||||
_open_search(page)
|
||||
|
||||
if cfg.RESUME_CLAIM_IDS_FILE and not cfg.TEST_CLAIM_IDS_FILE:
|
||||
# The agent's later pass (flows/supervisor.py): today's live queue,
|
||||
# only the claims still pending, no second export.
|
||||
info = claim_file.claim_ids(Path(cfg.RESUME_CLAIM_IDS_FILE))
|
||||
print(f"[RESUME] {len(info['ids'])} claim(s) of today's live queue "
|
||||
f"still pending")
|
||||
_process_claims(page, info)
|
||||
return
|
||||
|
||||
if cfg.TEST_CLAIM_IDS_FILE:
|
||||
_process_test_claim_ids_file(page)
|
||||
return
|
||||
@@ -1520,12 +1552,12 @@ def _main(log_path) -> int:
|
||||
return _stop("config", problem, 2)
|
||||
if opened_chrome:
|
||||
# Operator, 2026-10-23: a moment to add the Claude extension before
|
||||
# eCW opens. A person's box (shown on a Portal run too); after the
|
||||
# usual timeout the run goes on.
|
||||
# eCW opens; after the usual timeout the run goes on. 2026-10-29:
|
||||
# bulk runs show no box at all, so QUIET_BOXES makes it a log line.
|
||||
msgbox.info_box("eCW Bot — Chrome open",
|
||||
"Chrome is open. Add the Claude extension if you "
|
||||
"need it, then click OK — the bot logs in to eCW "
|
||||
"after this box.", needs_person=True)
|
||||
"after this box.")
|
||||
if not _disk_check():
|
||||
return 2
|
||||
|
||||
@@ -1551,6 +1583,12 @@ def _main(log_path) -> int:
|
||||
except watchdog.Stalled as exc:
|
||||
print(f"[WATCHDOG] Run stopped: {exc}")
|
||||
code = _stop("stalled", str(exc), watchdog.EXIT_CODE)
|
||||
except recovery.PageStuck as exc:
|
||||
# The supervisor restarts Chrome and logs in again for the
|
||||
# claims still pending (flows/recovery.py).
|
||||
print(f"[RECOVER] Run stopped: {exc}")
|
||||
shot(page, "page_stuck")
|
||||
code = _stop("stuck", first_line(exc), recovery.EXIT_CODE)
|
||||
except Exception as exc:
|
||||
if watchdog.has_fired():
|
||||
# The frozen call came back with an error somewhere
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
Getting the run out of a stuck screen without a person (operator,
|
||||
2026-10-30: close or refresh, re-login, and continue).
|
||||
|
||||
Live 2026-10-29: a Dental claim's sliding fee check left Patient Information
|
||||
open; its overlay took every click, and the next 18 claims were each marked
|
||||
"Bot could not search the claim" without being looked at.
|
||||
|
||||
A claim the bot could not search or read (`bot_failed`) sets off `recover`:
|
||||
1. the login page on screen = eCW ended the session → PageStuck;
|
||||
2. Escape, close Patient Information, clear the screen → search ready?
|
||||
3. reopen the claim search page (`reopen`) → search ready?
|
||||
4. still blocked → PageStuck.
|
||||
`run_claim` then tries that claim once more. Two claims in a row still
|
||||
failing after their recovery → PageStuck as well. PageStuck ends the run
|
||||
with EXIT_CODE; the supervisor restarts Chrome, logs in again and carries
|
||||
on with the claims still pending.
|
||||
"""
|
||||
|
||||
from ..core.page_utils import visible
|
||||
from ..pages import claim_form, claim_results, sliding_fee
|
||||
from ..selectors import login as login_sel
|
||||
|
||||
EXIT_CODE = 9
|
||||
STREAK_LIMIT = 2
|
||||
|
||||
|
||||
class PageStuck(RuntimeError):
|
||||
"""The page cannot be brought back without a new browser and login."""
|
||||
|
||||
|
||||
def bot_failed(r: dict, data: dict) -> bool:
|
||||
"""The bot could not search or read the claim — a fault of the screen,
|
||||
not a fact about the claim."""
|
||||
return bool((r or {}).get("search_error") or (data or {}).get("_error"))
|
||||
|
||||
|
||||
def _logged_out(page) -> bool:
|
||||
return visible(page, login_sel.SEL_USERNAME, timeout=1_000)
|
||||
|
||||
|
||||
def _escape_dialogs(page) -> None:
|
||||
for _ in range(2):
|
||||
try:
|
||||
page.keyboard.press("Escape")
|
||||
page.wait_for_timeout(300)
|
||||
except Exception:
|
||||
pass
|
||||
sliding_fee.close_patient_info(page)
|
||||
|
||||
|
||||
def _clear(page, claim_id, keep_pages) -> None:
|
||||
claim_form.clear_screen(page, claim_id, keep_pages)
|
||||
|
||||
|
||||
def _blocker(page):
|
||||
return claim_results.search_blocker(page)
|
||||
|
||||
|
||||
def recover(page, claim_id: str, keep_pages, reopen) -> str:
|
||||
"""'cleared' or 'reopened' once the claim search is free; else raises."""
|
||||
if _logged_out(page):
|
||||
raise PageStuck("eCW showed the login page - the session ended")
|
||||
_escape_dialogs(page)
|
||||
_clear(page, claim_id, keep_pages)
|
||||
if not _blocker(page):
|
||||
return "cleared"
|
||||
reopen(page)
|
||||
blocker = _blocker(page)
|
||||
if not blocker:
|
||||
return "reopened"
|
||||
raise PageStuck(f"the claim search stayed blocked after a reopen: "
|
||||
f"{blocker}")
|
||||
|
||||
|
||||
def run_claim(attempt, recover_now, state: dict):
|
||||
"""
|
||||
`attempt()` → (r, data). A bot failure: `recover_now()`, then one more
|
||||
attempt. `state["streak"]` counts claims in a row that still failed
|
||||
after that; at STREAK_LIMIT, PageStuck.
|
||||
"""
|
||||
state.setdefault("streak", 0)
|
||||
r, data = attempt()
|
||||
if not bot_failed(r, data):
|
||||
state["streak"] = 0
|
||||
return r, data
|
||||
recover_now()
|
||||
r, data = attempt()
|
||||
if not bot_failed(r, data):
|
||||
state["streak"] = 0
|
||||
return r, data
|
||||
state["streak"] += 1
|
||||
if state["streak"] >= STREAK_LIMIT:
|
||||
raise PageStuck(f"{state['streak']} claims in a row could not be "
|
||||
f"searched or read, even after a recovery")
|
||||
return r, data
|
||||
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
The agent: runs the bot over the day's Medcor Ready queue until every claim
|
||||
has a result, without a person (operator, 2026-10-30: "we are an agent, not
|
||||
a bot — observe, fix, continue, and mail me what happened").
|
||||
|
||||
python agent.py
|
||||
|
||||
Each pass runs the bot in a child process (a fresh Chrome and login every
|
||||
time). The first pass downloads the live queue; a later pass takes only the
|
||||
claims still pending — the queue minus every claim that got a result since
|
||||
the agent started, a "Bot could not ..." Review not counting as a result.
|
||||
|
||||
What a pass's exit code leads to is `next_action`, the operator's rules of
|
||||
2026-10-30:
|
||||
* login failures (1, 6): retry after 10, 20 minutes; the 3rd in a row
|
||||
stops the agent — the client's eCW account must not be restricted;
|
||||
* restricted (4), eCW screen changed (7), config (2): stop at once;
|
||||
* stuck page (9), stall (8), claims error (5): restart in a minute;
|
||||
* the attempt guard said wait (3): wait, not a failure;
|
||||
* nothing pending: done; a clean pass that left the same claims: stop,
|
||||
those are for a person.
|
||||
Every failure is mailed (core/notify.py), and so is the end.
|
||||
|
||||
VERIFY only: the agent refuses to start with SAVE_CORRECTIONS=1, since
|
||||
LIVE-WRITE asks a person before every save.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from .. import settings as cfg
|
||||
from .. import claim_file
|
||||
from ..core import notify, spreadsheet
|
||||
|
||||
LOGIN_WAITS = (600, 1200, 1800)
|
||||
LOGIN_FAIL_LIMIT = 3
|
||||
CRASH_LIMIT = 6
|
||||
RUN_LIMIT = 15
|
||||
|
||||
_STOP_AT_ONCE = {
|
||||
2: "the bot's settings are incomplete",
|
||||
4: "eCW restricted the account",
|
||||
7: "an eCW screen changed and the bot no longer knows it",
|
||||
}
|
||||
_LOGIN = {1: "the login failed", 6: "the 2FA approval timed out"}
|
||||
_RESTART = {5: "a claim step failed", 8: "a claim stopped making progress",
|
||||
9: "the eCW page got stuck and could not be cleared"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class State:
|
||||
login_fails: int = 0
|
||||
crashes: int = 0
|
||||
runs: int = 0
|
||||
last_pending: list | None = None
|
||||
outcome: str = ""
|
||||
history: list = field(default_factory=list)
|
||||
|
||||
|
||||
def next_action(code: int, pending, state: State):
|
||||
"""(action, wait_seconds, why) — action is done, retry or stop."""
|
||||
state.runs += 1
|
||||
if code not in _LOGIN and code != 3:
|
||||
state.login_fails = 0
|
||||
if pending == []:
|
||||
return "done", 0, "every claim in the queue has a result"
|
||||
if code in _STOP_AT_ONCE:
|
||||
return "stop", 0, _STOP_AT_ONCE[code]
|
||||
if state.runs >= RUN_LIMIT:
|
||||
return "stop", 0, f"{RUN_LIMIT} passes today without finishing"
|
||||
if code in _LOGIN:
|
||||
state.login_fails += 1
|
||||
if state.login_fails >= LOGIN_FAIL_LIMIT:
|
||||
return "stop", 0, (f"{LOGIN_FAIL_LIMIT} login failures in a row "
|
||||
f"- stopping so eCW does not restrict the "
|
||||
f"account")
|
||||
return "retry", LOGIN_WAITS[state.login_fails - 1], _LOGIN[code]
|
||||
if code == 3:
|
||||
return "retry", 300, "the login guard asked to wait"
|
||||
if code in _RESTART or code != 0:
|
||||
state.crashes += 1
|
||||
if state.crashes >= CRASH_LIMIT:
|
||||
return "stop", 0, f"{CRASH_LIMIT} restarts today"
|
||||
return "retry", 60, _RESTART.get(code, f"the bot ended with {code}")
|
||||
if pending is None:
|
||||
return "stop", 0, "the run ended but no queue file was found"
|
||||
if state.last_pending is not None and \
|
||||
sorted(pending) == sorted(state.last_pending):
|
||||
return "stop", 0, (f"{len(pending)} claim(s) still could not be "
|
||||
f"checked after a retry - left for a person")
|
||||
state.last_pending = list(pending)
|
||||
return "retry", 60, f"{len(pending)} claim(s) to retry"
|
||||
|
||||
|
||||
def pending(queue_ids, rows) -> list:
|
||||
"""Queue order kept; rows = (claim_no, comments) since the agent began."""
|
||||
done = {str(c) for c, comments in rows
|
||||
if not (comments or "").startswith("Bot could not")}
|
||||
return [c for c in queue_ids if str(c) not in done]
|
||||
|
||||
|
||||
def run_loop(run_bot, pending_now, mail, sleep) -> State:
|
||||
st = State()
|
||||
resume = None
|
||||
while True:
|
||||
code = run_bot(resume)
|
||||
left = pending_now()
|
||||
action, wait, why = next_action(code, left, st)
|
||||
st.history.append((code, action, why))
|
||||
print(f"[AGENT] Pass {st.runs}: exit {code} - {action} ({why})")
|
||||
if action == "done":
|
||||
st.outcome = "done"
|
||||
mail("summary", "Run finished", st)
|
||||
return st
|
||||
if action == "stop":
|
||||
st.outcome = "stopped"
|
||||
mail("incident", "Agent stopped", why,
|
||||
"Stopped; nothing more will run today without a person.",
|
||||
False)
|
||||
return st
|
||||
mail("incident", "Recovering", why,
|
||||
f"Waiting {wait // 60} min, then a fresh Chrome and login for "
|
||||
f"the {len(left) if left else 'queued'} claim(s) still pending.",
|
||||
True)
|
||||
resume = left or None
|
||||
sleep(wait)
|
||||
|
||||
|
||||
# ── the real world ───────────────────────────────────────────────────────────
|
||||
|
||||
_BOOT = ("import sys; sys.path.insert(0, 'src'); "
|
||||
"from ecw_bot.flows.export_medcor_ready import main; sys.exit(main())")
|
||||
|
||||
|
||||
def _queue_file() -> Path:
|
||||
return cfg.ROOT / "Output" / f"Medcor Ready_{cfg.TODAY}.xlsx"
|
||||
|
||||
|
||||
def _run_bot_child(resume, started_at: datetime):
|
||||
env = dict(os.environ, SAVE_CORRECTIONS="0", QUIET_BOXES="1",
|
||||
SHOW_EACH_CLAIM="0", SHOW_CLAIM_IDS="0", CLAIM_PROCESS_LIMIT="0",
|
||||
ECW_KEEP_OPEN="0", TEST_CLAIM_IDS_FILE="",
|
||||
RESUME_CLAIM_IDS_FILE="")
|
||||
if resume:
|
||||
path = cfg.ROOT / "Output" / f"Agent pending_{cfg.TODAY}.xlsx"
|
||||
spreadsheet.write_xlsx(path, [["CLAIMS#"]] + [[c] for c in resume])
|
||||
env["RESUME_CLAIM_IDS_FILE"] = str(path)
|
||||
print(f"[AGENT] Resuming {len(resume)} pending claim(s)")
|
||||
else:
|
||||
print("[AGENT] Running the live Medcor Ready queue")
|
||||
return subprocess.call([sys.executable, "-c", _BOOT], cwd=cfg.ROOT,
|
||||
env=env)
|
||||
|
||||
|
||||
def _pending_now(started_at: datetime):
|
||||
if not _queue_file().exists():
|
||||
return None
|
||||
queue = claim_file.claim_ids(_queue_file())["ids"]
|
||||
import pyodbc
|
||||
with pyodbc.connect(cfg.RESULTS_DB_CONN, timeout=15) as conn:
|
||||
rows = conn.cursor().execute(
|
||||
"SELECT claim_no, comments FROM dbo.CV_ChargeSummary "
|
||||
"WHERE run_at >= ?", started_at).fetchall()
|
||||
return pending(queue, [(r[0], r[1]) for r in rows])
|
||||
|
||||
|
||||
def _shots_since(when: datetime) -> list:
|
||||
folder = Path(cfg.SCREENSHOTS)
|
||||
if not folder.exists():
|
||||
return []
|
||||
t = when.timestamp()
|
||||
return [p for p in folder.glob("*.png") if p.stat().st_mtime >= t]
|
||||
|
||||
|
||||
def _mail(started_at):
|
||||
def mail(kind, title, *args):
|
||||
if kind == "summary":
|
||||
st = args[0]
|
||||
lines = [f"The agent finished the Medcor Ready queue for "
|
||||
f"{cfg.TODAY} in {st.runs} pass(es).",
|
||||
"", "Passes:"]
|
||||
lines += [f" exit {c}: {a} - {w}" for c, a, w in st.history]
|
||||
lines += ["", "Results are in the Portal under today's date."]
|
||||
notify.summary(title, lines)
|
||||
return
|
||||
what, did, recovered = args
|
||||
notify.incident(title, what, did, recovered,
|
||||
shots=_shots_since(started_at))
|
||||
return mail
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if cfg.SAVE_CORRECTIONS:
|
||||
print("[AGENT] SAVE_CORRECTIONS=1 - the agent runs VERIFY only; "
|
||||
"LIVE-WRITE needs a person at every save.")
|
||||
return 2
|
||||
started_at = datetime.now().replace(microsecond=0)
|
||||
print(f"[AGENT] Started {started_at:%Y-%m-%d %H:%M:%S}")
|
||||
st = run_loop(
|
||||
run_bot=lambda resume: _run_bot_child(resume, started_at),
|
||||
pending_now=lambda: _pending_now(started_at),
|
||||
mail=_mail(started_at),
|
||||
sleep=time.sleep)
|
||||
return 0 if st.outcome == "done" else 1
|
||||
@@ -192,6 +192,10 @@ def _close_patient_info(page: Page) -> None:
|
||||
"sliding_fee_patient_info_still_open.png")
|
||||
|
||||
|
||||
# For flows/recovery.py: the same close, after a claim left the dialog up.
|
||||
close_patient_info = _close_patient_info
|
||||
|
||||
|
||||
def read(page: Page) -> dict:
|
||||
"""Open Info -> Sliding Fee Schedule, read it, close both. Never raises."""
|
||||
if not appears(page, sel.SEL_CLAIM_PATIENT_INFO, 5_000):
|
||||
|
||||
@@ -235,6 +235,15 @@ IMAP_PASSWORD = _env("IMAP_PASSWORD")
|
||||
IMAP_FOLDER = _env("IMAP_FOLDER", "INBOX")
|
||||
IMAP_SSL = _env_bool("IMAP_SSL", True)
|
||||
|
||||
# The agent's email to the operator (core/notify.py, operator 2026-10-30):
|
||||
# sent from the 2FA mailbox's own account over SMTP, so no new credential.
|
||||
# Blank NOTIFY_EMAIL_TO = no mail at all.
|
||||
NOTIFY_EMAIL_TO = _env("NOTIFY_EMAIL_TO")
|
||||
SMTP_HOST = _env("SMTP_HOST") or IMAP_HOST
|
||||
SMTP_PORT = _env_int("SMTP_PORT", 465)
|
||||
SMTP_USERNAME = _env("SMTP_USERNAME") or IMAP_USERNAME
|
||||
SMTP_PASSWORD = _env("SMTP_PASSWORD") or IMAP_PASSWORD
|
||||
|
||||
# How long to keep polling the mailbox for the OTP mail, and how often.
|
||||
#
|
||||
# Ten minutes. eCW's push mail is DELAYED, not dropped — proven by delivery
|
||||
@@ -272,6 +281,12 @@ OTP_LINK_AGE_WARN_SECONDS = _env_int("OTP_LINK_AGE_WARN_SECONDS", 540)
|
||||
# for that and stops with it as the stated reason.
|
||||
OTP_RESEND_ATTEMPTS = _env_int("OTP_RESEND_ATTEMPTS", 1)
|
||||
OTP_RESEND_WAIT_SECONDS = _env_int("OTP_RESEND_WAIT_SECONDS", 600)
|
||||
# A link approved but never picked up by the login page (live 2026-10-29: mail
|
||||
# 6 to 8.5 minutes late, page still on "sent a notification", Resend still
|
||||
# there): wait this long for the page to move on, then Resend and approve the
|
||||
# new mail, up to OTP_PICKUP_RESENDS times. eCW allows three resends a login.
|
||||
OTP_PICKUP_SECONDS = _env_int("OTP_PICKUP_SECONDS", 60)
|
||||
OTP_PICKUP_RESENDS = _env_int("OTP_PICKUP_RESENDS", 2)
|
||||
# Ignore anything already sitting in the inbox from before the login attempt.
|
||||
OTP_MAX_AGE_SECONDS = _env_int("OTP_MAX_AGE_SECONDS", 300)
|
||||
|
||||
@@ -516,6 +531,11 @@ NOTIFY_UNMAPPED_PAYERS = _env_bool("NOTIFY_UNMAPPED_PAYERS", True)
|
||||
# Accepted on purpose — a path in .env that points at nothing is a config
|
||||
# mistake whichever program reads the file (unit B fix round 1, 2026-10-05).
|
||||
TEST_CLAIM_IDS_FILE = _env("TEST_CLAIM_IDS_FILE", "")
|
||||
# Set by the agent (flows/supervisor.py), not by hand: the claims of today's
|
||||
# live queue still pending after an earlier pass stopped. Run like the live
|
||||
# queue — no TEST label, the live Charge Summary — but from this list, with
|
||||
# no fresh export.
|
||||
RESUME_CLAIM_IDS_FILE = _env("RESUME_CLAIM_IDS_FILE", "")
|
||||
|
||||
|
||||
# ── Per-claim processing ──────────────────────────────────────────────────────
|
||||
|
||||
Reference in new issue
Block a user