ManikandanandClaude Opus 5.5
840272032a
Snapshot of D:\Ecw Process 65c92c5 (2026-10-30). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
388 lines
15 KiB
Python
388 lines
15 KiB
Python
r"""
|
|
The Runs page's data: every Run's `Logs\events_<run>.jsonl` (written by
|
|
`ecw_bot.core.run_events`) read into a summary for the list and a detail for
|
|
one Run (spec docs/superpowers/specs/2026-10-22-run-tracking-and-portal-
|
|
sign-in-design.md, Stage 1; terms in GLOSSARY.md).
|
|
|
|
Only Runs with an events file are listed: the run logs from before
|
|
2026-10-22 have no times on their lines and are not imported.
|
|
|
|
A Run's result:
|
|
* **Done** — `run_end` with exit 0;
|
|
* **Run Error** — `run_end` with any other exit (its Run Error's step and
|
|
message alongside);
|
|
* **Stopped** — the Portal's `stopped` event, written after its Stop;
|
|
* **Running** — no end yet and its process is the live one;
|
|
* **Ended without a record** — no end and not live: killed some other way.
|
|
|
|
Everything here is read-only and never raises on a bad file: a line a
|
|
killed Run left half-written is skipped.
|
|
"""
|
|
|
|
import re
|
|
import json
|
|
from datetime import datetime, date
|
|
from pathlib import Path
|
|
|
|
RUN_ID = re.compile(r"^\d{8}_\d{6}$")
|
|
STATUSES = ("Running", "Done", "Run Error", "Stopped",
|
|
"Ended without a record")
|
|
|
|
LOGIN_STEPS = ("page_open", "password", "captcha_shown", "captcha_solved",
|
|
"captcha_failed", "twofa_requested", "mail_received",
|
|
"approval_clicked", "home")
|
|
STEP_LABELS = {
|
|
"page_open": "Login page opened", "password": "Password typed",
|
|
"captcha_shown": "Captcha shown", "captcha_solved": "Captcha solved",
|
|
"captcha_failed": "Captcha failed", "twofa_requested":
|
|
"Log In clicked (2FA mail requested)", "mail_received":
|
|
"2FA mail received", "approval_clicked": "Approval clicked",
|
|
"home": "Home page reached",
|
|
}
|
|
LOGIN_RESULTS = {"ok": "OK", "twofa_timeout": "2FA timeout",
|
|
"captcha_failed": "Captcha failed",
|
|
"restricted": "Restricted", "failed": "Failed"}
|
|
|
|
|
|
def valid_run_id(run_id: str) -> bool:
|
|
return bool(RUN_ID.match(run_id or ""))
|
|
|
|
|
|
def _time(text):
|
|
try:
|
|
return datetime.fromisoformat(text)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _secs(a, b):
|
|
return int((b - a).total_seconds()) if a and b else None
|
|
|
|
|
|
def read_events(logs_dir, run_id) -> list:
|
|
"""A Run's events in file order; [] for no file. Never raises."""
|
|
if not valid_run_id(run_id):
|
|
return []
|
|
path = Path(logs_dir) / f"events_{run_id}.jsonl"
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return []
|
|
return _parse(text.splitlines())
|
|
|
|
|
|
def _parse(lines) -> list:
|
|
"""Event dicts from JSON lines, a bad or untimed line skipped."""
|
|
out = []
|
|
for line in lines:
|
|
try:
|
|
e = json.loads(line)
|
|
except ValueError:
|
|
continue # half-written by a killed Run
|
|
if isinstance(e, dict) and _time(e.get("t")):
|
|
e["_t"] = _time(e["t"])
|
|
out.append(e)
|
|
return out
|
|
|
|
|
|
def _text(e) -> str:
|
|
"""One timeline line."""
|
|
k = e.get("kind")
|
|
if k == "run_start":
|
|
return (f"Run started by {e.get('started_by') or '?'} "
|
|
f"({e.get('source')}, {e.get('mode')})")
|
|
if k == "disk":
|
|
return f"Disk {e.get('drive')}: {e.get('free_mb')} MB free"
|
|
if k == "login_attempt_start":
|
|
return f"eCW Login Attempt {e.get('attempt')} started"
|
|
if k == "login_step":
|
|
return (f"Attempt {e.get('attempt')}: "
|
|
f"{STEP_LABELS.get(e.get('step'), e.get('step'))}")
|
|
if k == "login_attempt_end":
|
|
msg = f" — {e['message']}" if e.get("message") else ""
|
|
return (f"eCW Login Attempt {e.get('attempt')}: "
|
|
f"{LOGIN_RESULTS.get(e.get('result'), e.get('result'))}{msg}")
|
|
if k == "queue":
|
|
return f"{len(e.get('claims') or [])} claim(s) queued"
|
|
if k == "claim_start":
|
|
return f"Claim {e.get('n')}/{e.get('total')} {e.get('claim')} started"
|
|
if k == "claim_result":
|
|
return f"Claim {e.get('claim')}: {e.get('category')}"
|
|
if k == "claim_error":
|
|
return (f"Claim Error {e.get('claim')} ({e.get('step')}): "
|
|
f"{e.get('message')}")
|
|
if k == "warning":
|
|
who = f" {e['claim']}" if e.get("claim") else ""
|
|
return f"Warning{who} ({e.get('code')}): {e.get('message')}"
|
|
if k == "screenshot":
|
|
return f"Screenshot: {Path(e.get('path') or '').name}"
|
|
if k == "run_error":
|
|
return (f"Run Error ({e.get('step')}, exit {e.get('exit_code')}): "
|
|
f"{e.get('message')}")
|
|
if k == "run_end":
|
|
return f"Run ended, exit {e.get('exit_code')}"
|
|
if k == "stopped":
|
|
return f"Stopped from the Portal by {e.get('stopped_by') or '?'}"
|
|
return k or "?"
|
|
|
|
|
|
def _attempts(events) -> list:
|
|
by_n = {}
|
|
for e in events:
|
|
k = e.get("kind")
|
|
if k not in ("login_attempt_start", "login_step", "login_attempt_end"):
|
|
continue
|
|
a = by_n.setdefault(e.get("attempt"), {
|
|
"attempt": e.get("attempt"), "start": None, "end": None,
|
|
"result": None, "result_label": "Did not finish", "message": "",
|
|
"steps": {}, "_steps": {}})
|
|
if k == "login_attempt_start":
|
|
a["start"] = e["_t"]
|
|
elif k == "login_step":
|
|
a["_steps"].setdefault(e.get("step"), e["_t"])
|
|
a["steps"].setdefault(e.get("step"), e["t"])
|
|
else:
|
|
a["end"] = e["_t"]
|
|
a["result"] = e.get("result")
|
|
a["result_label"] = LOGIN_RESULTS.get(e.get("result"),
|
|
e.get("result"))
|
|
a["message"] = e.get("message") or ""
|
|
out = []
|
|
for a in sorted(by_n.values(), key=lambda a: a["attempt"] or 0):
|
|
a["duration_s"] = _secs(a["start"], a["end"])
|
|
a["twofa_wait_s"] = _secs(a["_steps"].get("twofa_requested"),
|
|
a["_steps"].get("mail_received"))
|
|
a["step_rows"] = [(STEP_LABELS[s], a["steps"][s][11:19])
|
|
for s in LOGIN_STEPS if s in a["steps"]]
|
|
del a["_steps"]
|
|
out.append(a)
|
|
return out
|
|
|
|
|
|
def summarize(run_id, events, live_run_id=None, now=None) -> dict:
|
|
now = now or datetime.now()
|
|
first = events[0] if events else {}
|
|
start_ev = next((e for e in events if e["kind"] == "run_start"), {})
|
|
end_ev = next((e for e in events if e["kind"] == "run_end"), None)
|
|
stop_ev = next((e for e in events if e["kind"] == "stopped"), None)
|
|
run_errors = [e for e in events if e["kind"] == "run_error"]
|
|
started = start_ev.get("_t") or first.get("_t") or _time_of_id(run_id)
|
|
|
|
if end_ev is not None:
|
|
code = end_ev.get("exit_code")
|
|
status = "Done" if code == 0 else "Run Error"
|
|
ended = end_ev["_t"]
|
|
elif stop_ev is not None:
|
|
code, status, ended = None, "Stopped", stop_ev["_t"]
|
|
elif run_id == live_run_id:
|
|
code, status, ended = None, "Running", None
|
|
else:
|
|
code, status = None, "Ended without a record"
|
|
ended = events[-1]["_t"] if events else None
|
|
|
|
results = [e for e in events if e["kind"] == "claim_result"]
|
|
queue = next((e.get("claims") or [] for e in events
|
|
if e["kind"] == "queue"), [])
|
|
started_claims = {e.get("claim") for e in events
|
|
if e["kind"] == "claim_start"}
|
|
not_reached = ([c for c in queue if c not in started_claims]
|
|
if status != "Running" else [])
|
|
attempts = _attempts(events)
|
|
err = run_errors[0] if run_errors else {}
|
|
|
|
return {
|
|
"run_id": run_id,
|
|
"started": started,
|
|
"ended": ended,
|
|
"duration_s": _secs(started, ended or now),
|
|
"started_by": start_ev.get("started_by") or "",
|
|
"source": start_ev.get("source") or "",
|
|
"mode": start_ev.get("mode") or "",
|
|
"test_sheet": start_ev.get("test_sheet") or "",
|
|
"status": status,
|
|
"exit_code": code,
|
|
"error_step": err.get("step") or "",
|
|
"error_message": err.get("message") or "",
|
|
"stopped_by": (stop_ev or {}).get("stopped_by") or "",
|
|
"login_attempts": attempts,
|
|
"login_total_s": sum(a["duration_s"] or 0 for a in attempts),
|
|
"pass": sum(e.get("category") == "Pass" for e in results),
|
|
"fail": sum(e.get("category") == "Fail" for e in results),
|
|
"review": sum(e.get("category") == "Review" for e in results),
|
|
"not_found": sum(e.get("category") == "Not found" for e in results),
|
|
"claims_total": len(queue),
|
|
"claims_done": len(results),
|
|
"run_errors": run_errors,
|
|
"claim_errors": [e for e in events if e["kind"] == "claim_error"],
|
|
"warnings": [e for e in events if e["kind"] == "warning"],
|
|
"screenshots": [e for e in events if e["kind"] == "screenshot"],
|
|
"not_reached": not_reached,
|
|
"timeline": [{"t": e["t"], "time": e["t"][11:19],
|
|
"kind": e["kind"], "text": _text(e)} for e in events],
|
|
}
|
|
|
|
|
|
def _time_of_id(run_id):
|
|
try:
|
|
return datetime.strptime(run_id, "%Y%m%d_%H%M%S")
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
class FileEvents:
|
|
"""Runs from this PC's `Logs/events_<run>.jsonl` files."""
|
|
|
|
def __init__(self, logs_dir):
|
|
self.dir = Path(logs_dir)
|
|
|
|
def run_ids(self) -> list:
|
|
try:
|
|
files = sorted(self.dir.glob("events_*.jsonl"), reverse=True)
|
|
except OSError:
|
|
return []
|
|
return [p.stem[len("events_"):] for p in files
|
|
if valid_run_id(p.stem[len("events_"):])]
|
|
|
|
def events(self, run_ids) -> dict:
|
|
return {r: read_events(self.dir, r) for r in run_ids
|
|
if (self.dir / f"events_{r}.jsonl").is_file()}
|
|
|
|
|
|
class DbEvents:
|
|
"""Runs from `dbo.CV_RunEvent` (2026-10-30): the Portal on Dokploy,
|
|
which cannot read the bot PC's Logs folder. One query per page."""
|
|
|
|
def __init__(self, repo):
|
|
self.repo = repo
|
|
|
|
def run_ids(self) -> list:
|
|
return [r for r in self.repo.run_ids() if valid_run_id(r)]
|
|
|
|
def events(self, run_ids) -> dict:
|
|
run_ids = list(run_ids)
|
|
if not run_ids:
|
|
return {}
|
|
lines = {}
|
|
for run_id, body in self.repo.run_event_lines(run_ids):
|
|
lines.setdefault(run_id, []).append(body)
|
|
return {r: _parse(lines[r]) for r in run_ids if r in lines}
|
|
|
|
|
|
def _source(src):
|
|
"""A path (the old way, and the tests) or an events source."""
|
|
return src if hasattr(src, "run_ids") else FileEvents(src)
|
|
|
|
|
|
def load_run(logs_dir, run_id, live_run_id=None, now=None):
|
|
"""One Run's detail, or None (bad id, no events)."""
|
|
if not valid_run_id(run_id):
|
|
return None
|
|
found = _source(logs_dir).events([run_id])
|
|
if run_id not in found:
|
|
return None
|
|
return summarize(run_id, found[run_id], live_run_id, now)
|
|
|
|
|
|
def list_runs(logs_dir, live_run_id=None, now=None, day_from=None,
|
|
day_to=None, status=None) -> list:
|
|
"""Every Run with events, newest first, filtered. `day_from` /
|
|
`day_to` are ISO dates (inclusive), `status` one of STATUSES."""
|
|
d_from = date.fromisoformat(day_from) if day_from else None
|
|
d_to = date.fromisoformat(day_to) if day_to else None
|
|
src = _source(logs_dir)
|
|
ids = []
|
|
for run_id in src.run_ids():
|
|
day = _time_of_id(run_id).date()
|
|
if (d_from and day < d_from) or (d_to and day > d_to):
|
|
continue
|
|
ids.append(run_id)
|
|
found = src.events(ids)
|
|
out = []
|
|
for run_id in ids:
|
|
if run_id not in found:
|
|
continue
|
|
r = summarize(run_id, found[run_id], live_run_id, now)
|
|
if status and r["status"] != status:
|
|
continue
|
|
out.append(r)
|
|
return out
|
|
|
|
|
|
def latest_run(logs_dir, live_run_id=None, now=None):
|
|
"""The newest Run's summary, or None — reads one Run, not all."""
|
|
src = _source(logs_dir)
|
|
for run_id in src.run_ids():
|
|
found = src.events([run_id])
|
|
if run_id in found:
|
|
return summarize(run_id, found[run_id], live_run_id, now)
|
|
return None
|
|
|
|
|
|
def screenshot_path(logs_dir, run_id, n, screenshots_root):
|
|
"""The n-th screenshot event's .png, if it is inside `screenshots_root`
|
|
and exists; else None. Only paths a Run itself recorded are served."""
|
|
if not valid_run_id(run_id):
|
|
return None
|
|
events = _source(logs_dir).events([run_id]).get(run_id, [])
|
|
shots = [e for e in events if e["kind"] == "screenshot"]
|
|
if not 0 <= n < len(shots):
|
|
return None
|
|
try:
|
|
root = Path(screenshots_root).resolve()
|
|
path = Path(shots[n].get("path") or "").resolve()
|
|
except (OSError, ValueError):
|
|
return None
|
|
# PNG only: a saved page's HTML served from the Portal's own origin
|
|
# would run eCW's scripts there.
|
|
if (root not in path.parents or path.suffix.lower() != ".png"
|
|
or not path.is_file()):
|
|
return None
|
|
return path
|
|
|
|
|
|
LIVE_STALE_S = 600
|
|
|
|
|
|
def live_status(source, now=None) -> dict:
|
|
"""The Dashboard's Agent run card on a server Portal (2026-10-30): the
|
|
newest Run in the events source, in `/api/bot/status`'s shape. Running
|
|
= no end yet and an event in the last LIVE_STALE_S seconds (the server
|
|
cannot see the bot PC's process). No log tail: that is the PC's log."""
|
|
from .run_status import exit_text
|
|
now = now or datetime.now()
|
|
out = {"running": False, "starting": False, "started_at": None,
|
|
"ended_at": None, "exit_code": None, "stopped_by_operator": False,
|
|
"source": None, "started_by": None, "phase": "", "progress": None,
|
|
"current": None, "counts": {"Pass": 0, "Fail": 0, "Review": 0},
|
|
"tail": [], "claims": [], "results_run_at": None,
|
|
"elapsed_s": None, "message": "No run yet"}
|
|
try:
|
|
ids = source.run_ids()
|
|
except Exception:
|
|
return out
|
|
for run_id in ids:
|
|
found = source.events([run_id])
|
|
if run_id not in found or not found[run_id]:
|
|
continue
|
|
events = found[run_id]
|
|
r = summarize(run_id, events, None, now)
|
|
ended = any(e["kind"] in ("run_end", "stopped") for e in events)
|
|
fresh = (now - events[-1]["_t"]).total_seconds() < LIVE_STALE_S
|
|
running = not ended and fresh
|
|
stopped = r["status"] == "Stopped"
|
|
out.update(
|
|
running=running, source=r["source"], started_by=r["started_by"],
|
|
started_at=r["started"].isoformat() if r["started"] else None,
|
|
ended_at=r["ended"].isoformat() if r["ended"] else None,
|
|
exit_code=r["exit_code"], stopped_by_operator=stopped,
|
|
progress=([r["claims_done"], r["claims_total"]]
|
|
if r["claims_total"] else None),
|
|
counts={"Pass": r["pass"], "Fail": r["fail"],
|
|
"Review": r["review"]},
|
|
elapsed_s=r["duration_s"],
|
|
message=(f"Running on the bot PC · {r['claims_done']} of "
|
|
f"{r['claims_total']} claims" if running
|
|
else exit_text(r["exit_code"], stopped)))
|
|
return out
|
|
return out
|