Initial import: eCW Charge Validation bot and results Portal
Snapshot of the working project as of 2026-10-23 (local 2d4d4fc), one commit with no earlier history: the tracked files were de-identified first (synthetic claim ids and patient data only; see Context.md section 7). Credentials, run logs, screenshots, claim reports and client workbooks are gitignored and not included. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ManikandanandClaude Opus 5.5 committed
commit
ba15a574ab
205 files changed
+56772
No files matched your search
+292
@@ -0,0 +1,292 @@
|
||||
r"""
|
||||
The Runs page's data: every Run's `Logs\events_<run>.jsonl` (written by
|
||||
`ecw_bot.core.run_events`) read into a summary for the list and a detail for
|
||||
one Run (spec docs/superpowers/specs/2026-10-22-run-tracking-and-portal-
|
||||
sign-in-design.md, Stage 1; terms in GLOSSARY.md).
|
||||
|
||||
Only Runs with an events file are listed: the run logs from before
|
||||
2026-10-22 have no times on their lines and are not imported.
|
||||
|
||||
A Run's result:
|
||||
* **Done** — `run_end` with exit 0;
|
||||
* **Run Error** — `run_end` with any other exit (its Run Error's step and
|
||||
message alongside);
|
||||
* **Stopped** — the Portal's `stopped` event, written after its Stop;
|
||||
* **Running** — no end yet and its process is the live one;
|
||||
* **Ended without a record** — no end and not live: killed some other way.
|
||||
|
||||
Everything here is read-only and never raises on a bad file: a line a
|
||||
killed Run left half-written is skipped.
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
from datetime import datetime, date
|
||||
from pathlib import Path
|
||||
|
||||
RUN_ID = re.compile(r"^\d{8}_\d{6}$")
|
||||
STATUSES = ("Running", "Done", "Run Error", "Stopped",
|
||||
"Ended without a record")
|
||||
|
||||
LOGIN_STEPS = ("page_open", "password", "captcha_shown", "captcha_solved",
|
||||
"captcha_failed", "twofa_requested", "mail_received",
|
||||
"approval_clicked", "home")
|
||||
STEP_LABELS = {
|
||||
"page_open": "Login page opened", "password": "Password typed",
|
||||
"captcha_shown": "Captcha shown", "captcha_solved": "Captcha solved",
|
||||
"captcha_failed": "Captcha failed", "twofa_requested":
|
||||
"Log In clicked (2FA mail requested)", "mail_received":
|
||||
"2FA mail received", "approval_clicked": "Approval clicked",
|
||||
"home": "Home page reached",
|
||||
}
|
||||
LOGIN_RESULTS = {"ok": "OK", "twofa_timeout": "2FA timeout",
|
||||
"captcha_failed": "Captcha failed",
|
||||
"restricted": "Restricted", "failed": "Failed"}
|
||||
|
||||
|
||||
def valid_run_id(run_id: str) -> bool:
|
||||
return bool(RUN_ID.match(run_id or ""))
|
||||
|
||||
|
||||
def _time(text):
|
||||
try:
|
||||
return datetime.fromisoformat(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _secs(a, b):
|
||||
return int((b - a).total_seconds()) if a and b else None
|
||||
|
||||
|
||||
def read_events(logs_dir, run_id) -> list:
|
||||
"""A Run's events in file order; [] for no file. Never raises."""
|
||||
if not valid_run_id(run_id):
|
||||
return []
|
||||
path = Path(logs_dir) / f"events_{run_id}.jsonl"
|
||||
out = []
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return []
|
||||
for line in text.splitlines():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except ValueError:
|
||||
continue # half-written by a killed Run
|
||||
if isinstance(e, dict) and _time(e.get("t")):
|
||||
e["_t"] = _time(e["t"])
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
|
||||
def _text(e) -> str:
|
||||
"""One timeline line."""
|
||||
k = e.get("kind")
|
||||
if k == "run_start":
|
||||
return (f"Run started by {e.get('started_by') or '?'} "
|
||||
f"({e.get('source')}, {e.get('mode')})")
|
||||
if k == "disk":
|
||||
return f"Disk {e.get('drive')}: {e.get('free_mb')} MB free"
|
||||
if k == "login_attempt_start":
|
||||
return f"eCW Login Attempt {e.get('attempt')} started"
|
||||
if k == "login_step":
|
||||
return (f"Attempt {e.get('attempt')}: "
|
||||
f"{STEP_LABELS.get(e.get('step'), e.get('step'))}")
|
||||
if k == "login_attempt_end":
|
||||
msg = f" — {e['message']}" if e.get("message") else ""
|
||||
return (f"eCW Login Attempt {e.get('attempt')}: "
|
||||
f"{LOGIN_RESULTS.get(e.get('result'), e.get('result'))}{msg}")
|
||||
if k == "queue":
|
||||
return f"{len(e.get('claims') or [])} claim(s) queued"
|
||||
if k == "claim_start":
|
||||
return f"Claim {e.get('n')}/{e.get('total')} {e.get('claim')} started"
|
||||
if k == "claim_result":
|
||||
return f"Claim {e.get('claim')}: {e.get('category')}"
|
||||
if k == "claim_error":
|
||||
return (f"Claim Error {e.get('claim')} ({e.get('step')}): "
|
||||
f"{e.get('message')}")
|
||||
if k == "warning":
|
||||
who = f" {e['claim']}" if e.get("claim") else ""
|
||||
return f"Warning{who} ({e.get('code')}): {e.get('message')}"
|
||||
if k == "screenshot":
|
||||
return f"Screenshot: {Path(e.get('path') or '').name}"
|
||||
if k == "run_error":
|
||||
return (f"Run Error ({e.get('step')}, exit {e.get('exit_code')}): "
|
||||
f"{e.get('message')}")
|
||||
if k == "run_end":
|
||||
return f"Run ended, exit {e.get('exit_code')}"
|
||||
if k == "stopped":
|
||||
return f"Stopped from the Portal by {e.get('stopped_by') or '?'}"
|
||||
return k or "?"
|
||||
|
||||
|
||||
def _attempts(events) -> list:
|
||||
by_n = {}
|
||||
for e in events:
|
||||
k = e.get("kind")
|
||||
if k not in ("login_attempt_start", "login_step", "login_attempt_end"):
|
||||
continue
|
||||
a = by_n.setdefault(e.get("attempt"), {
|
||||
"attempt": e.get("attempt"), "start": None, "end": None,
|
||||
"result": None, "result_label": "Did not finish", "message": "",
|
||||
"steps": {}, "_steps": {}})
|
||||
if k == "login_attempt_start":
|
||||
a["start"] = e["_t"]
|
||||
elif k == "login_step":
|
||||
a["_steps"].setdefault(e.get("step"), e["_t"])
|
||||
a["steps"].setdefault(e.get("step"), e["t"])
|
||||
else:
|
||||
a["end"] = e["_t"]
|
||||
a["result"] = e.get("result")
|
||||
a["result_label"] = LOGIN_RESULTS.get(e.get("result"),
|
||||
e.get("result"))
|
||||
a["message"] = e.get("message") or ""
|
||||
out = []
|
||||
for a in sorted(by_n.values(), key=lambda a: a["attempt"] or 0):
|
||||
a["duration_s"] = _secs(a["start"], a["end"])
|
||||
a["twofa_wait_s"] = _secs(a["_steps"].get("twofa_requested"),
|
||||
a["_steps"].get("mail_received"))
|
||||
a["step_rows"] = [(STEP_LABELS[s], a["steps"][s][11:19])
|
||||
for s in LOGIN_STEPS if s in a["steps"]]
|
||||
del a["_steps"]
|
||||
out.append(a)
|
||||
return out
|
||||
|
||||
|
||||
def summarize(run_id, events, live_run_id=None, now=None) -> dict:
|
||||
now = now or datetime.now()
|
||||
first = events[0] if events else {}
|
||||
start_ev = next((e for e in events if e["kind"] == "run_start"), {})
|
||||
end_ev = next((e for e in events if e["kind"] == "run_end"), None)
|
||||
stop_ev = next((e for e in events if e["kind"] == "stopped"), None)
|
||||
run_errors = [e for e in events if e["kind"] == "run_error"]
|
||||
started = start_ev.get("_t") or first.get("_t") or _time_of_id(run_id)
|
||||
|
||||
if end_ev is not None:
|
||||
code = end_ev.get("exit_code")
|
||||
status = "Done" if code == 0 else "Run Error"
|
||||
ended = end_ev["_t"]
|
||||
elif stop_ev is not None:
|
||||
code, status, ended = None, "Stopped", stop_ev["_t"]
|
||||
elif run_id == live_run_id:
|
||||
code, status, ended = None, "Running", None
|
||||
else:
|
||||
code, status = None, "Ended without a record"
|
||||
ended = events[-1]["_t"] if events else None
|
||||
|
||||
results = [e for e in events if e["kind"] == "claim_result"]
|
||||
queue = next((e.get("claims") or [] for e in events
|
||||
if e["kind"] == "queue"), [])
|
||||
started_claims = {e.get("claim") for e in events
|
||||
if e["kind"] == "claim_start"}
|
||||
not_reached = ([c for c in queue if c not in started_claims]
|
||||
if status != "Running" else [])
|
||||
attempts = _attempts(events)
|
||||
err = run_errors[0] if run_errors else {}
|
||||
|
||||
return {
|
||||
"run_id": run_id,
|
||||
"started": started,
|
||||
"ended": ended,
|
||||
"duration_s": _secs(started, ended or now),
|
||||
"started_by": start_ev.get("started_by") or "",
|
||||
"source": start_ev.get("source") or "",
|
||||
"mode": start_ev.get("mode") or "",
|
||||
"test_sheet": start_ev.get("test_sheet") or "",
|
||||
"status": status,
|
||||
"exit_code": code,
|
||||
"error_step": err.get("step") or "",
|
||||
"error_message": err.get("message") or "",
|
||||
"stopped_by": (stop_ev or {}).get("stopped_by") or "",
|
||||
"login_attempts": attempts,
|
||||
"login_total_s": sum(a["duration_s"] or 0 for a in attempts),
|
||||
"pass": sum(e.get("category") == "Pass" for e in results),
|
||||
"fail": sum(e.get("category") == "Fail" for e in results),
|
||||
"review": sum(e.get("category") == "Review" for e in results),
|
||||
"not_found": sum(e.get("category") == "Not found" for e in results),
|
||||
"claims_total": len(queue),
|
||||
"claims_done": len(results),
|
||||
"run_errors": run_errors,
|
||||
"claim_errors": [e for e in events if e["kind"] == "claim_error"],
|
||||
"warnings": [e for e in events if e["kind"] == "warning"],
|
||||
"screenshots": [e for e in events if e["kind"] == "screenshot"],
|
||||
"not_reached": not_reached,
|
||||
"timeline": [{"t": e["t"], "time": e["t"][11:19],
|
||||
"kind": e["kind"], "text": _text(e)} for e in events],
|
||||
}
|
||||
|
||||
|
||||
def _time_of_id(run_id):
|
||||
try:
|
||||
return datetime.strptime(run_id, "%Y%m%d_%H%M%S")
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def load_run(logs_dir, run_id, live_run_id=None, now=None):
|
||||
"""One Run's detail, or None (bad id, no events file)."""
|
||||
if not valid_run_id(run_id):
|
||||
return None
|
||||
if not (Path(logs_dir) / f"events_{run_id}.jsonl").is_file():
|
||||
return None
|
||||
return summarize(run_id, read_events(logs_dir, run_id), live_run_id, now)
|
||||
|
||||
|
||||
def list_runs(logs_dir, live_run_id=None, now=None, day_from=None,
|
||||
day_to=None, status=None) -> list:
|
||||
"""Every Run with an events file, newest first, filtered. `day_from`
|
||||
/ `day_to` are ISO dates (inclusive), `status` one of STATUSES."""
|
||||
d_from = date.fromisoformat(day_from) if day_from else None
|
||||
d_to = date.fromisoformat(day_to) if day_to else None
|
||||
out = []
|
||||
try:
|
||||
files = sorted(Path(logs_dir).glob("events_*.jsonl"), reverse=True)
|
||||
except OSError:
|
||||
files = []
|
||||
for path in files:
|
||||
run_id = path.stem[len("events_"):]
|
||||
if not valid_run_id(run_id):
|
||||
continue
|
||||
day = _time_of_id(run_id).date()
|
||||
if (d_from and day < d_from) or (d_to and day > d_to):
|
||||
continue
|
||||
r = summarize(run_id, read_events(logs_dir, run_id), live_run_id, now)
|
||||
if status and r["status"] != status:
|
||||
continue
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def latest_run(logs_dir, live_run_id=None, now=None):
|
||||
"""The newest Run's summary, or None — reads one file, not all."""
|
||||
try:
|
||||
files = sorted(Path(logs_dir).glob("events_*.jsonl"), reverse=True)
|
||||
except OSError:
|
||||
return None
|
||||
for path in files:
|
||||
run_id = path.stem[len("events_"):]
|
||||
if valid_run_id(run_id):
|
||||
return summarize(run_id, read_events(logs_dir, run_id),
|
||||
live_run_id, now)
|
||||
return None
|
||||
|
||||
|
||||
def screenshot_path(logs_dir, run_id, n, screenshots_root):
|
||||
"""The n-th screenshot event's .png, if it is inside `screenshots_root`
|
||||
and exists; else None. Only paths a Run itself recorded are served."""
|
||||
shots = [e for e in read_events(logs_dir, run_id)
|
||||
if e["kind"] == "screenshot"]
|
||||
if not 0 <= n < len(shots):
|
||||
return None
|
||||
try:
|
||||
root = Path(screenshots_root).resolve()
|
||||
path = Path(shots[n].get("path") or "").resolve()
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
# PNG only: a saved page's HTML served from the Portal's own origin
|
||||
# would run eCW's scripts there.
|
||||
if (root not in path.parents or path.suffix.lower() != ".png"
|
||||
or not path.is_file()):
|
||||
return None
|
||||
return path
|
||||
Reference in new issue
Block a user