Self-healing agent (supervisor, recovery, login fixes, operator mail) and Portal sage redesign

Snapshot of D:\Ecw Process a11c9ff (2026-10-30).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ManikandanandClaude Opus 5.5 committed 2026-10-30 10:58:35 +05:30
1 parent fafa5285eb
commit d500f750da
26 files changed
+1704 -357

No files matched your search

+8 -1
View File
@@ -194,7 +194,14 @@ def _run(*, fires=False, review=True, show_each=True, dental=False,
out = io.StringIO()
try:
with contextlib.redirect_stdout(out):
flow._process_claims(None, {"ids": ["9103000"], "file": None})
# No page to recover on: flows/recovery.py's steps are
# its own tests' business (test_recovery.py).
real_recover = flow.recovery.recover
flow.recovery.recover = lambda *a: "cleared"
try:
flow._process_claims(None, {"ids": ["9103000"], "file": None})
finally:
flow.recovery.recover = real_recover
finally:
_restore()
return events, out.getvalue(), written[0]
+8 -1
View File
@@ -321,7 +321,14 @@ def _run(ids, search, events=None):
lambda page, cid, keep=(): events.append(("clear", cid)) or [])
try:
with contextlib.redirect_stdout(io.StringIO()):
flow._process_claims(None, {"ids": list(ids), "file": None})
# No page to recover on: flows/recovery.py's steps are
# its own tests' business (test_recovery.py).
real_recover = flow.recovery.recover
flow.recovery.recover = lambda *a: "cleared"
try:
flow._process_claims(None, {"ids": list(ids), "file": None})
finally:
flow.recovery.recover = real_recover
finally:
(cfg.SHOW_EACH_CLAIM, cfg.SHOW_CLAIM_IDS, cfg.CLAIM_PROCESS_LIMIT,
flow.claim_results.search_claim, flow._read_claim_form,
+28
View File
@@ -79,6 +79,34 @@ def a_password_box_that_stays_disabled_is_an_error_with_a_screenshot(page):
assert {p.suffix for p in shots} == {".png", ".html"}, shots
# The security image page with its own Continue: clicking it is what lets
# eCW enable the password box (operator, 2026-10-30: no manual click).
_IMAGE_PAGE = """
<input id="doctorID" value="someone">
<input disabled type="password" id="passwordField">
<div id="img">VERIFY YOUR SECURITY IMAGE
<input type="button" value="%s" onclick="document.getElementById('img')
.style.display = 'none'; document.getElementById('passwordField')
.disabled = false;">
</div>
"""
@case
def continue_on_the_security_image_page_is_clicked_by_the_bot(page):
page.set_content(_IMAGE_PAGE % "Continue")
assert login.wait_after_username(page, 5_000) == "image"
assert login.continue_security_image(page) is True
assert login.wait_after_username(page, 5_000, image=False) == "password"
@case
def no_continue_button_is_left_to_a_person(page):
page.set_content(_IMAGE_PAGE % "Not me")
assert login.continue_security_image(page) is False
assert page.locator("#passwordField").is_disabled()
def main() -> int:
from playwright.sync_api import sync_playwright
+3
View File
@@ -237,6 +237,9 @@ def without_quiet_boxes_an_info_box_is_shown():
def main() -> int:
failures = 0
# This PC's .env runs bulk with QUIET_BOXES=1 (2026-10-29); the cases
# that want it on set it themselves.
msgbox.cfg.QUIET_BOXES = False
for fn in CASES:
name = fn.__name__.replace("_", " ")
try:
+130
View File
@@ -0,0 +1,130 @@
r"""
core/notify.py — the agent's email to the operator (operator, 2026-10-30):
what happened, what the agent did about it, and whether it recovered.
Rules the operator set:
* sent from the bot's own mailbox (SMTP, the 2FA mailbox's account);
* screenshots of login and error pages only — never a claim screen, never
page HTML: those carry PHI;
* no claim numbers in the mail.
python tests\test_notify.py
No mail is sent: the SMTP transport is a stand-in.
"""
import sys
import pathlib
import tempfile
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src"))
from ecw_bot import settings as cfg # noqa: E402
from ecw_bot.core import notify # noqa: E402
CASES = []
def case(fn):
CASES.append(fn)
return fn
def _capture():
sent = []
notify._transport = lambda msg: sent.append(msg)
return sent
def _files(tmp, *names):
out = []
for n in names:
p = pathlib.Path(tmp) / n
p.write_bytes(b"x")
out.append(p)
return out
@case
def an_incident_says_what_happened_and_what_the_agent_did(tmp):
sent = _capture()
assert notify.incident("Login failed", "2FA mail came 7 minutes late",
"Resent the OTP, will retry in 10 minutes",
recovered=False) is True
msg = sent[0]
assert msg["To"] == cfg.NOTIFY_EMAIL_TO
assert "Login failed" in msg["Subject"]
body = msg.get_body(("plain",)).get_content()
assert "2FA mail came 7 minutes late" in body
assert "Resent the OTP" in body
assert "Not recovered" in body
@case
def claim_numbers_never_reach_the_mail(tmp):
sent = _capture()
notify.incident("Stuck dialog on claim 9107000",
"Claim 9107000 form would not close", "Reloaded eCW",
recovered=True)
msg = sent[0]
text = msg["Subject"] + msg.get_body(("plain",)).get_content()
assert "9107000" not in text, text
@case
def only_login_and_error_screenshots_are_attached(tmp):
sent = _capture()
shots = _files(tmp, "211944_login_unconfirmed.png",
"9107000_claim_form_leftover_204500.png",
"211944_login_unconfirmed.html",
"101500_security_image_page.png")
notify.incident("Login failed", "x", "y", recovered=False, shots=shots)
names = sorted(a.get_filename() for a in sent[0].iter_attachments())
assert names == ["101500_security_image_page.png",
"211944_login_unconfirmed.png"], names
@case
def nothing_is_sent_when_no_recipient_is_set(tmp):
sent = _capture()
old = cfg.NOTIFY_EMAIL_TO
cfg.NOTIFY_EMAIL_TO = ""
try:
assert notify.incident("x", "y", "z", recovered=True) is False
finally:
cfg.NOTIFY_EMAIL_TO = old
assert sent == []
@case
def a_mail_server_error_never_stops_the_run(tmp):
def boom(msg):
raise OSError("connection refused")
notify._transport = boom
assert notify.incident("x", "y", "z", recovered=True) is False
def main() -> int:
failures = 0
real = notify._transport
cfg.NOTIFY_EMAIL_TO = "operator@example.test"
for fn in CASES:
name = fn.__name__.replace("_", " ")
with tempfile.TemporaryDirectory() as tmp:
try:
fn(tmp)
print(f" ok {name}")
except AssertionError as exc:
print(f" FAIL {name}: {exc}")
failures += 1
except Exception as exc:
print(f" ERROR {name}: {type(exc).__name__}: {exc}")
failures += 1
finally:
notify._transport = real
print(f"\n{len(CASES) - failures}/{len(CASES)} passed")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())
+2 -1
View File
@@ -220,7 +220,8 @@ def the_header_names_the_user_and_admin_links_are_admin_only():
for name, role in (("sam", "Super Admin"), ("asha", "Admin")):
a, _ = _sign_in(app, name)
page = a.get("/runs").text
assert f"<b>{name.title()}</b> · {role}" in page, name
# The side menu (2026-10-30): name over role.
assert f"<b>{name.title()}</b><small>{role}</small>" in page, name
assert 'href="/admin/users"' in page and 'href="/admin/signins"' in page
v, _ = _sign_in(app, "vee")
page = v.get("/runs").text
+168
View File
@@ -0,0 +1,168 @@
r"""
flows/recovery.py — the run gets itself out of a stuck screen (operator,
2026-10-30: "you can close or refresh and re-login and continue the agent").
Live 2026-10-29: a Dental claim's sliding fee check left Patient Information
open; its overlay took every click, and the next 18 claims were each marked
"Bot could not search the claim" without being looked at.
Now a claim the bot could not search or read sets off a recovery:
1. the login page on screen = the session ended → PageStuck (re-login);
2. Escape, close Patient Information, clear the screen → search ready?
3. reopen the claim search page → search ready?
4. still blocked → PageStuck.
After a recovery the same claim is tried once more. Two claims in a row
that still fail after their recovery → PageStuck, so the supervisor
restarts Chrome and logs in again.
python tests\test_recovery.py
No browser: the page steps are stand-ins.
"""
import sys
import pathlib
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src"))
from ecw_bot.flows import recovery # noqa: E402
CASES = []
def case(fn):
CASES.append(fn)
return fn
class Steps:
"""Stand-ins for the page steps; `blocked` = what search_blocker says,
in order, one per look."""
def __init__(self, blocked, logged_out=False):
self.blocked = list(blocked)
self.logged_out = logged_out
self.done = []
def install(self):
recovery._logged_out = lambda page: self.logged_out
recovery._escape_dialogs = lambda page: self.done.append("escape")
recovery._clear = lambda page, cid, keep: self.done.append("clear")
recovery._blocker = lambda page: self.blocked.pop(0)
def reopen(self, page):
self.done.append("reopen")
@case
def a_dialog_cleared_by_escape_needs_no_reopen():
s = Steps([None])
s.install()
assert recovery.recover(None, "1", (), s.reopen) == "cleared"
assert s.done == ["escape", "clear"], s.done
@case
def a_page_still_blocked_is_reopened():
s = Steps(["covered by div.modalOverlay", None])
s.install()
assert recovery.recover(None, "1", (), s.reopen) == "reopened"
assert s.done == ["escape", "clear", "reopen"], s.done
@case
def a_page_blocked_after_reopen_is_stuck():
s = Steps(["covered", "covered"])
s.install()
try:
recovery.recover(None, "1", (), s.reopen)
except recovery.PageStuck as exc:
assert "covered" in str(exc), exc
else:
raise AssertionError("no PageStuck")
@case
def the_login_page_means_the_session_ended():
s = Steps([], logged_out=True)
s.install()
try:
recovery.recover(None, "1", (), s.reopen)
except recovery.PageStuck as exc:
assert "login page" in str(exc), exc
else:
raise AssertionError("no PageStuck")
assert s.done == [], s.done
def _attempts(*outcomes):
"""attempt() returning each outcome in turn: 'ok' or 'bot'."""
seq = list(outcomes)
def attempt():
o = seq.pop(0)
return ({"search_error": "x"}, {}) if o == "bot" else ({}, {"cpt": []})
return attempt
@case
def a_claim_that_fails_is_recovered_and_tried_again():
tried = []
state = {}
r, data = recovery.run_claim(_attempts("bot", "ok"),
lambda: tried.append(1), state)
assert tried == [1] and not r.get("search_error"), (tried, r)
assert state["streak"] == 0
@case
def a_claim_that_fails_again_is_kept_as_review():
state = {}
r, _ = recovery.run_claim(_attempts("bot", "bot"), lambda: None, state)
assert r.get("search_error") and state["streak"] == 1
@case
def two_claims_in_a_row_failing_after_recovery_stop_the_run():
state = {}
recovery.run_claim(_attempts("bot", "bot"), lambda: None, state)
try:
recovery.run_claim(_attempts("bot", "bot"), lambda: None, state)
except recovery.PageStuck:
pass
else:
raise AssertionError("no PageStuck after two in a row")
@case
def a_good_claim_resets_the_streak():
state = {}
recovery.run_claim(_attempts("bot", "bot"), lambda: None, state)
recovery.run_claim(_attempts("ok"), lambda: None, state)
assert state["streak"] == 0
def main() -> int:
failures = 0
real = {k: getattr(recovery, k, None) for k in
("_logged_out", "_escape_dialogs", "_clear", "_blocker")}
for fn in CASES:
name = fn.__name__.replace("_", " ")
try:
fn()
print(f" ok {name}")
except AssertionError as exc:
print(f" FAIL {name}: {exc}")
failures += 1
except Exception as exc:
print(f" ERROR {name}: {type(exc).__name__}: {exc}")
failures += 1
finally:
for k, v in real.items():
setattr(recovery, k, v)
print(f"\n{len(CASES) - failures}/{len(CASES)} passed")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())
+8 -1
View File
@@ -739,7 +739,14 @@ def process_claims_counts_an_unreadable_form_as_not_read_and_reviews_it():
out = io.StringIO()
try:
with contextlib.redirect_stdout(out):
flow._process_claims(None, {"ids": ["9103000"], "file": None})
# No page to recover on: flows/recovery.py's steps are
# its own tests' business (test_recovery.py).
real_recover = flow.recovery.recover
flow.recovery.recover = lambda *a: "cleared"
try:
flow._process_claims(None, {"ids": ["9103000"], "file": None})
finally:
flow.recovery.recover = real_recover
finally:
(cfg.SHOW_EACH_CLAIM, cfg.SHOW_CLAIM_IDS, cfg.CLAIM_PROCESS_LIMIT,
cfg.OPEN_CLAIM_FORM, msgbox.info_box, claim_form.open_claim,
+140
View File
@@ -0,0 +1,140 @@
r"""
flows/supervisor.py — the agent loop around the bot (operator, 2026-10-30):
run the day's queue to the end without a person, restarting on technical
failures and mailing what happened.
Rules (operator's choices that day):
* login failures (exit 1, 6): retry after 10, 20, then 30 minutes; the
3rd failure in a row stops the agent and mails it — the client's eCW
account must not be restricted;
* restricted (4), eCW screen changed (7), config (2): stop at once — no
retry can fix them;
* a stuck page (9), a stall (8), a claims error (5): restart with a fresh
Chrome and login for the claims still pending;
* the attempt guard said wait (3): wait, it is not a failure;
* nothing pending: done. A clean run that left the same claims pending
as before: stop, those are for a person.
python tests\test_supervisor.py
No bot, no browser, no mail, no database: all stand-ins.
"""
import sys
import pathlib
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src"))
from ecw_bot.flows import supervisor as sv # noqa: E402
CASES = []
def case(fn):
CASES.append(fn)
return fn
def _decide(code, pending, state=None):
state = state if state is not None else sv.State()
return sv.next_action(code, pending, state), state
@case
def nothing_pending_is_done():
(action, _, _), _ = _decide(0, [])
assert action == "done"
@case
def login_failures_back_off_10_20_then_stop_on_the_third():
st = sv.State()
waits = []
for _ in range(2):
(action, wait, _), st = _decide(1, ["a"], st)
assert action == "retry"
waits.append(wait)
assert waits == [600, 1200], waits
(action, _, why), st = _decide(6, ["a"], st)
assert action == "stop" and "3 login" in why, why
@case
def a_good_login_resets_the_login_failures():
st = sv.State()
_decide(1, ["a"], st)
_decide(9, ["a"], st)
assert st.login_fails == 0
@case
def restricted_screen_changed_and_config_stop_at_once():
for code in (2, 4, 7):
(action, _, _), _ = _decide(code, ["a"])
assert action == "stop", code
@case
def a_stuck_page_restarts_soon():
(action, wait, _), _ = _decide(9, ["a", "b"])
assert action == "retry" and wait <= 120
@case
def the_guard_saying_wait_is_not_a_failure():
(action, wait, _), st = _decide(3, ["a"])
assert action == "retry" and wait >= 300 and st.login_fails == 0
@case
def a_clean_run_that_left_the_same_claims_stops():
st = sv.State()
(action, _, _), st = _decide(0, ["a", "b"], st)
assert action == "retry"
(action, _, why), st = _decide(0, ["a", "b"], st)
assert action == "stop" and "person" in why, why
@case
def pending_is_the_queue_minus_claims_done_since_the_agent_started():
queue = ["1", "2", "3", "4"]
rows = [("1", "Clean claims"), ("2", "Bot could not search the claim"),
("3", "Telehealth (POS 02 on T1017)")]
assert sv.pending(queue, rows) == ["2", "4"]
@case
def the_loop_restarts_mails_and_ends_when_nothing_is_left():
codes = [6, 9, 0]
left = [["1", "2"], ["2"], []]
runs, mails, slept = [], [], []
st = sv.run_loop(
run_bot=lambda resume: runs.append(resume) or codes.pop(0),
pending_now=lambda: left.pop(0),
mail=lambda kind, *a: mails.append(kind),
sleep=slept.append)
assert runs == [None, ["1", "2"], ["2"]], runs
assert slept == [600, 60], slept
assert mails == ["incident", "incident", "summary"], mails
assert st.outcome == "done"
def main() -> int:
failures = 0
for fn in CASES:
name = fn.__name__.replace("_", " ")
try:
fn()
print(f" ok {name}")
except AssertionError as exc:
print(f" FAIL {name}: {exc}")
failures += 1
except Exception as exc:
print(f" ERROR {name}: {type(exc).__name__}: {exc}")
failures += 1
print(f"\n{len(CASES) - failures}/{len(CASES)} passed")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())
+134
View File
@@ -0,0 +1,134 @@
r"""
auth/twofactor.py `handle_2fa()` — an approval the login page never picks up.
Live 2026-10-29, three logins in a row: the push mail came 6 to 8.5 minutes
late, the bot opened its link and clicked "Yes, it's me", and the login page
stayed on "eClinicalWorks sent a notification to your email" with Resend OTP
still there. The bot waited out LOGIN_TIMEOUT and gave up ("Login not
confirmed"). Now it presses Resend, waits for the new mail and approves that
one, up to OTP_PICKUP_RESENDS times (operator, 2026-10-30: no manual step).
python tests\test_twofa_late_approval.py
Local headless chromium, `page.set_content()` only. The mailbox and the
approval tab are stand-ins: no eCW, no mail.
"""
import sys
import pathlib
import tempfile
ROOT = pathlib.Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "src"))
from ecw_bot import settings as cfg # noqa: E402
from ecw_bot.auth import twofactor # noqa: E402
from ecw_bot.auth.mailbox import Challenge # noqa: E402
CASES = []
def case(fn):
CASES.append(fn)
return fn
# The push screen as seen live: the notice in #dispMessage, Resend enabled.
_PAGE = """
<div id="dispMessage">eClinicalWorks sent a notification to your email.
Please confirm to login.</div>
<input type="button" id="resendOtpBtn" value="Resend OTP"
onclick="window.resends = (window.resends || 0) + 1">
<a id="jellybean-panelLink4" style="display:none">Home</a>
<script>window.resends = 0</script>
"""
def _go_home(page):
page.evaluate("document.getElementById('jellybean-panelLink4')"
".style.display = ''; document.getElementById('dispMessage')"
".style.display = 'none'")
def _fakes(page, picked_up_on):
"""Mailbox hands out links in order; approval number `picked_up_on`
(1-based) is the one the page takes."""
seen = {"approvals": 0, "mails": 0}
def get_challenge(**kw):
seen["mails"] += 1
return Challenge("link", f"https://example.test/approve/{seen['mails']}")
def approve(context, url):
seen["approvals"] += 1
if seen["approvals"] == picked_up_on:
_go_home(page)
return True
twofactor.mailbox.get_challenge = get_challenge
twofactor._approve_in_new_tab = approve
return seen
@case
def an_approval_not_picked_up_is_resent_and_approved_again(page):
page.set_content(_PAGE)
seen = _fakes(page, picked_up_on=2)
twofactor.handle_2fa(page, None, "Test", None)
assert seen["approvals"] == 2, seen
assert page.evaluate("window.resends") == 1
assert page.locator("#jellybean-panelLink4").is_visible()
@case
def an_approval_picked_up_at_once_needs_no_resend(page):
page.set_content(_PAGE)
seen = _fakes(page, picked_up_on=1)
twofactor.handle_2fa(page, None, "Test", None)
assert seen["approvals"] == 1, seen
assert page.evaluate("window.resends || 0") == 0
@case
def resends_stop_at_the_limit(page):
page.set_content(_PAGE)
seen = _fakes(page, picked_up_on=99)
twofactor.handle_2fa(page, None, "Test", None)
assert page.evaluate("window.resends") == cfg.OTP_PICKUP_RESENDS
assert seen["approvals"] == cfg.OTP_PICKUP_RESENDS + 1, seen
def main() -> int:
from playwright.sync_api import sync_playwright
failures = 0
real = (twofactor.mailbox.get_challenge, twofactor._approve_in_new_tab)
with tempfile.TemporaryDirectory() as tmp:
cfg.SCREENSHOTS = pathlib.Path(tmp) # never the real folder
cfg.ECW_SECRET_KEY = ""
cfg.OTP_PICKUP_SECONDS = 2
with sync_playwright() as pw:
browser = pw.chromium.launch(headless=True)
page = browser.new_page()
for fn in CASES:
name = fn.__name__.replace("_", " ")
try:
fn(page)
print(f" ok {name}")
except AssertionError as exc:
print(f" FAIL {name}: {exc}")
failures += 1
except Exception as exc:
print(f" ERROR {name}: {type(exc).__name__}: {exc}")
failures += 1
finally:
(twofactor.mailbox.get_challenge,
twofactor._approve_in_new_tab) = real
browser.close()
print(f"\n{len(CASES) - failures}/{len(CASES)} passed")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())
+3 -1
View File
@@ -232,7 +232,9 @@ def a_box_says_chrome_is_open_before_the_login():
assert code == 0, code
assert seen[-1] == "login" and len(seen) == 2, seen
_, title, needs_person = seen[0]
assert "Chrome" in title and needs_person is True, seen
# Operator, 2026-10-29: bulk runs show no box at all, so this one is
# quieted by QUIET_BOXES like any other (a log line instead).
assert "Chrome" in title and needs_person is False, seen
seen.clear()
code, _ = _main_with(lambda page: None, box,
+31 -3
View File
@@ -283,6 +283,33 @@ def the_dashboard_lists_todays_top_fail_and_review_reasons():
assert "No Fail or Review today" in html
@case
def the_dashboard_lists_todays_runs_newest_first():
"""Operator, 2026-10-30 (the approved mockup): a Runs today card —
each run's start, its mode and its claims by result, from the same
rows the reasons card reads, so no second query."""
today = datetime.combine(date.today(), datetime.min.time())
first, second = today + timedelta(hours=16, minutes=39), \
today + timedelta(hours=17, minutes=18)
rows = ([_row("9000001", "Pass", 1, first),
_row("9000002", "Fail", 2, first)]
+ [_row(f"900001{i}", "Pass", i, second) for i in range(3)])
client, repo = _client(FakeRepo(days=[_day(date.today(), 5, 2, 4, 1, 0)],
rows=rows))
html = client.get("/").text
box = html.split('id="runs-today"', 1)[1].split("</section>", 1)[0]
assert box.index("5:18 PM") < box.index("4:39 PM"), box
assert "<b>3</b> claims" in box and "<b>2</b> claims" in box, box
assert repo.calls == [("days",), ("day_results", date.today())], \
repo.calls
@case
def the_dashboard_greets_the_signed_in_user():
html = _client()[0].get("/").text
assert re.search(r"Good (morning|afternoon|evening), ", html), html[:400]
@case
def every_page_has_the_footer_and_marks_where_you_are():
client, _ = _client()
@@ -440,14 +467,15 @@ def find_a_claim_with_no_number_shows_the_box_only():
# ── every page ───────────────────────────────────────────────────────────────
@case
def every_page_has_the_navy_header_breadcrumb_and_mode():
def every_page_has_the_side_menu_breadcrumb_and_mode():
client, _ = _client()
for path in _PAGES:
html = client.get(path).text
assert '<header class="top">' in html, path
# Sage side menu since 2026-10-30 (the approved mockup).
assert '<aside class="side">' in html, path
assert 'class="pill VERIFY"' in html, path
assert '<nav class="crumbs">' in html, path
assert "--navy: #12355b" in html, path
assert "--navy: #2e3f36" in html, path
html = client.get("/day/2026-10-16").text
assert '<a href="/days">Claim status</a>' in html
assert '<span class="here">Fri, 16 Oct 2026</span>' in html