Snapshot of the working project as of 2026-10-23 (local 2d4d4fc), one commit with no earlier history: the tracked files were de-identified first (synthetic claim ids and patient data only; see Context.md section 7). Credentials, run logs, screenshots, claim reports and client workbooks are gitignored and not included. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
234 lines
7.0 KiB
Python
234 lines
7.0 KiB
Python
r"""
|
|
`core/watchdog.py` — a claim that stops making progress ends the Run with a
|
|
record instead of hanging it for good (2026-10-22: the 18:11 run sat 28
|
|
minutes on claim 1 after its form opened, no error, no end).
|
|
|
|
python tests\test_watchdog.py
|
|
|
|
* **No heartbeat for the limit → fired**: the Playwright driver is closed
|
|
(so the frozen call returns with an error) and `check()` raises
|
|
`Stalled`, naming where it stopped.
|
|
* **Heartbeats keep it quiet**, and outside `watching()` nothing fires.
|
|
* **The limit is never under a message box's timeout** plus two
|
|
minutes: a box left open is not a hang.
|
|
* **The driver is found and closed**: only this process's own
|
|
`node.exe` (Playwright's driver), never anything else.
|
|
* **A real frozen page** (`evaluate` on a promise that never settles)
|
|
comes back within seconds of the limit.
|
|
"""
|
|
|
|
import io
|
|
import sys
|
|
import time
|
|
import pathlib
|
|
import contextlib
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
|
|
from ecw_bot import settings as cfg # noqa: E402
|
|
from ecw_bot.core import watchdog # noqa: E402
|
|
|
|
CASES = []
|
|
|
|
|
|
def case(fn):
|
|
CASES.append(fn)
|
|
return fn
|
|
|
|
|
|
class Fake:
|
|
"""Stand-ins for the driver kill and the last-resort exit."""
|
|
|
|
def __init__(self):
|
|
self.killed, self.exited = 0, []
|
|
|
|
def kill(self):
|
|
self.killed += 1
|
|
|
|
def exit(self, code):
|
|
self.exited.append(code)
|
|
|
|
|
|
def _quiet(fn, *a, **k):
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
return fn(*a, **k)
|
|
|
|
|
|
def _patched(fake, poll=0.05, grace=0.5):
|
|
return [(watchdog, "_kill", fake.kill), (watchdog, "_exit", fake.exit),
|
|
(watchdog, "POLL_S", poll), (watchdog, "GRACE_S", grace)]
|
|
|
|
|
|
class Patch:
|
|
def __init__(self, triples):
|
|
self.triples = triples
|
|
|
|
def __enter__(self):
|
|
self.saved = [(o, n, getattr(o, n)) for o, n, _ in self.triples]
|
|
for o, n, v in self.triples:
|
|
setattr(o, n, v)
|
|
|
|
def __exit__(self, *e):
|
|
for o, n, v in self.saved:
|
|
setattr(o, n, v)
|
|
|
|
|
|
@case
|
|
def no_heartbeat_fires_closes_the_driver_and_check_raises():
|
|
fake = Fake()
|
|
with Patch(_patched(fake)):
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
with watchdog.watching(0.3):
|
|
watchdog.beat("claim 1/3 X: read the form")
|
|
time.sleep(0.8)
|
|
assert watchdog.has_fired()
|
|
try:
|
|
watchdog.check()
|
|
raise AssertionError("check() did not raise")
|
|
except watchdog.Stalled as exc:
|
|
assert "claim 1/3 X: read the form" in str(exc), exc
|
|
assert fake.killed == 1, fake.killed
|
|
assert fake.exited == [], "the main thread answered: no forced exit"
|
|
|
|
|
|
@case
|
|
def heartbeats_keep_it_quiet():
|
|
fake = Fake()
|
|
with Patch(_patched(fake)):
|
|
with watchdog.watching(0.4):
|
|
for _ in range(10):
|
|
watchdog.beat("busy")
|
|
time.sleep(0.1)
|
|
watchdog.check()
|
|
assert fake.killed == 0 and not watchdog.has_fired()
|
|
|
|
|
|
@case
|
|
def outside_watching_nothing_fires():
|
|
fake = Fake()
|
|
with Patch(_patched(fake)):
|
|
with watchdog.watching(0.2):
|
|
pass
|
|
time.sleep(0.5)
|
|
watchdog.beat("after")
|
|
watchdog.check()
|
|
assert fake.killed == 0
|
|
|
|
|
|
@case
|
|
def zero_minutes_turns_it_off():
|
|
fake = Fake()
|
|
with Patch(_patched(fake)):
|
|
with watchdog.watching(0):
|
|
time.sleep(0.3)
|
|
watchdog.check()
|
|
assert fake.killed == 0
|
|
|
|
|
|
@case
|
|
def a_main_thread_that_never_answers_is_ended_with_its_code():
|
|
fake = Fake()
|
|
with Patch(_patched(fake, grace=0.3)):
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
with watchdog.watching(0.2):
|
|
time.sleep(1.2) # never calls check(): still frozen
|
|
assert fake.killed == 1 and fake.exited == [watchdog.EXIT_CODE], \
|
|
(fake.killed, fake.exited)
|
|
|
|
|
|
@case
|
|
def the_limit_is_never_under_a_message_box():
|
|
saved = (cfg.WATCHDOG_MINUTES, cfg.MSGBOX_TIMEOUT_SECONDS)
|
|
try:
|
|
cfg.WATCHDOG_MINUTES, cfg.MSGBOX_TIMEOUT_SECONDS = 10, 180
|
|
assert watchdog.limit_seconds() == 600
|
|
cfg.WATCHDOG_MINUTES, cfg.MSGBOX_TIMEOUT_SECONDS = 2, 600
|
|
assert watchdog.limit_seconds() == 720, "box 600 s + 120 s"
|
|
cfg.WATCHDOG_MINUTES = 0
|
|
assert watchdog.limit_seconds() == 0, "0 = off"
|
|
finally:
|
|
cfg.WATCHDOG_MINUTES, cfg.MSGBOX_TIMEOUT_SECONDS = saved
|
|
|
|
|
|
@case
|
|
def a_message_box_beats_before_and_after():
|
|
from ecw_bot.core import msgbox
|
|
seen = []
|
|
saved = (watchdog.beat, msgbox._call_box)
|
|
watchdog.beat = lambda where="": seen.append(where)
|
|
msgbox._call_box = lambda *a: 1
|
|
try:
|
|
msgbox._raw_call("text", "title", 0, 1)
|
|
finally:
|
|
watchdog.beat, msgbox._call_box = saved
|
|
assert seen == ["message box", "message box closed"], seen
|
|
|
|
|
|
def _real_frozen_page() -> int:
|
|
"""The real thing: a page call that never returns, in a child Python
|
|
with its own Playwright driver, so the kill touches nothing here."""
|
|
import subprocess
|
|
script = r'''
|
|
import sys, time, io, contextlib
|
|
sys.path.insert(0, sys.argv[1])
|
|
from ecw_bot.core import watchdog
|
|
from playwright.sync_api import sync_playwright
|
|
watchdog.POLL_S = 0.2
|
|
start = time.monotonic()
|
|
with sync_playwright() as pw:
|
|
b = pw.chromium.launch(headless=True)
|
|
page = b.new_page()
|
|
page.set_content("<p>x</p>")
|
|
with watchdog.watching(3):
|
|
watchdog.beat("claim 1/1 TEST: read the form")
|
|
try:
|
|
page.evaluate("new Promise(() => {})") # never settles
|
|
print("RESULT returned")
|
|
except Exception as exc:
|
|
print("RESULT raised", type(exc).__name__)
|
|
try:
|
|
watchdog.check()
|
|
print("CHECK no")
|
|
except watchdog.Stalled as exc:
|
|
print("CHECK stalled", "TEST" in str(exc))
|
|
print("ELAPSED", round(time.monotonic() - start))
|
|
'''
|
|
r = subprocess.run([sys.executable, "-c", script, str(ROOT / "src")],
|
|
capture_output=True, text=True, timeout=120)
|
|
out = r.stdout
|
|
name = "a real frozen page call comes back after the limit"
|
|
try:
|
|
assert "RESULT raised" in out, out + r.stderr[-500:]
|
|
assert "CHECK stalled True" in out, out
|
|
elapsed = int(out.split("ELAPSED")[1].split()[0])
|
|
assert elapsed < 30, elapsed
|
|
print(f" ok {name}")
|
|
return 0
|
|
except Exception as exc:
|
|
print(f" FAIL {name}: {exc}")
|
|
return 1
|
|
|
|
|
|
def main() -> int:
|
|
failures = 0
|
|
for fn in CASES:
|
|
name = fn.__name__.replace("_", " ")
|
|
try:
|
|
fn()
|
|
print(f" ok {name}")
|
|
except AssertionError as exc:
|
|
print(f" FAIL {name}: {exc}")
|
|
failures += 1
|
|
except Exception as exc:
|
|
print(f" ERROR {name}: {type(exc).__name__}: {exc}")
|
|
failures += 1
|
|
failures += _real_frozen_page()
|
|
total = len(CASES) + 1
|
|
print(f"\n{total - failures}/{total} passed")
|
|
return 1 if failures else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|