automation-tools/MISF-Planner-Download
8
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ECW/HTTP planner scripts, browser crash recovery, and Planner error-overlay handling

Browse Source 
- Add ecw_download.py and onedrive_planner_http.py for ECW and HTTP-based Planner access
- Add build_exe.bat for packaging
- Restart Chrome automatically on crash mid-run in planner_download.py
- Detect and recover from Planner "something went wrong" error overlays
- Improve board-load waiting, sort-descending menu matching, and screenshot timestamp rendering in chrome.py/onedrive_planner.py
This commit is contained in:
Lubna Fathima committed 2026-07-09 08:41:28 -07:00
1 parent c2dec96b1f
commit 291507c2f7
6 files changed
+2131 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

build_exe.bat
+57
View File
@@ -0,0 +1,57 @@
@echo off
setlocal EnableDelayedExpansion
cd /d "%~dp0"
echo ============================================================
echo MISF Planner Download -- Build EXE
echo ============================================================
echo.
REM ── 1. Install / upgrade PyInstaller inside the venv ─────────────────────
echo [1/4] Ensuring PyInstaller is installed...
.venv\Scripts\pip install --quiet --upgrade pyinstaller
if %ERRORLEVEL% NEQ 0 (
echo ERROR: pip install pyinstaller failed. Is the venv set up?
pause & exit /b 1
)
REM ── 2. Clean previous build artefacts ────────────────────────────────────
echo [2/4] Cleaning previous build...
if exist build rmdir /s /q build
if exist dist\MISF_Planner_Download rmdir /s /q dist\MISF_Planner_Download
REM ── 3. Build ──────────────────────────────────────────────────────────────
echo [3/4] Building (this takes 2-5 minutes)...
echo.
.venv\Scripts\pyinstaller onedrive_planner_http.spec --clean --noconfirm
if %ERRORLEVEL% NEQ 0 (
echo.
echo ERROR: PyInstaller build failed. Check output above.
pause & exit /b 1
)
REM ── 4. Copy Input folder so the exe knows where to read config ────────────
echo.
echo [4/4] Copying Input\ folder into dist...
if exist Input (
xcopy /E /I /Y "Input" "dist\MISF_Planner_Download\Input" >nul
echo Input\ copied.
) else (
echo WARNING: No Input\ folder found -- copy it manually before shipping.
)
echo.
echo ============================================================
echo BUILD COMPLETE
echo.
echo Folder : dist\MISF_Planner_Download\
echo Exe : dist\MISF_Planner_Download\MISF_Planner_Download.exe
echo.
echo TO DEPLOY ON ANOTHER MACHINE:
echo 1. Copy the entire dist\MISF_Planner_Download\ folder.
echo 2. Make sure Input\input.xlsx is inside that folder.
echo 3. Double-click MISF_Planner_Download.exe.
echo (First run downloads the browser automatically.)
echo ============================================================
echo.
pause
ecw_download.py
+679
View File
@@ -0,0 +1,679 @@
#!/usr/bin/env python3
"""
ECW-server VOB / face-sheet downloader.
Auth flow
---------
Playwright opens the ECW login page, fills credentials + OTP if required,
waits for a successful session, then harvests all cookies and closes the
browser. An httpx.Client is built from those cookies and used for every
subsequent request — no browser involved after login.
Document / download flow (pure HTTP after login)
--------------------------------------------------
1. GET the ECW document-list endpoint → HTML response
2. Parse the HTML to find per-facility <a href> download links
3. Download each file directly into Output/<date>/Original/<Facility>/
4. Copy to Workable/ and Workable_Medicare/ with the standard naming
DD-MM-YYYY_<Facility Name>[_N].pdf
5. Write Download_Summary_ECW_<date>.xlsx
Config keys in Input/input.xlsx Sheet1
----------------------------------------
ECW URL https://ecw.yourdomain.com (required)
ECW Username user@domain.com (optional — falls back to OneDrive Username)
ECW Password secret (optional — falls back to OneDrive Password)
ECW TOTP Secret BASE32SECRET (optional — for ECW-specific 2FA)
Proxy IP / Proxy Port / Proxy Username / Proxy Password (reused from existing config)
Usage
-----
python ecw_download.py
python ecw_download.py --limit 5
python ecw_download.py --facility "Arbor Hills"
python ecw_download.py --date 2026-06-19
python ecw_download.py --resume
"""
from __future__ import annotations
import argparse
import logging
import re
import shutil
import sys
import time
from datetime import datetime
from pathlib import Path
from urllib.parse import urljoin, urlparse
ROOT = Path(__file__).resolve().parent
SRC = ROOT / "src"
if str(SRC) not in sys.path:
sys.path.insert(0, str(SRC))
import httpx
from bs4 import BeautifulSoup
from checkpoint import clear_checkpoint, load_checkpoint, save_checkpoint
from config import (
PROJECT_ROOT,
TODAY_ISO_DATE,
AppConfig,
read_app_config,
read_facility_list,
_read_config_key_value_sheet,
)
from logging_setup import setup_logging
from run_log import log_step
_log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# ECW-specific config (on top of the existing AppConfig)
# ---------------------------------------------------------------------------
def _read_ecw_config() -> dict[str, str]:
"""Read ECW-specific keys from the same input.xlsx Sheet1."""
try:
kv = _read_config_key_value_sheet()
except Exception:
kv = {}
cfg = read_app_config()
return {
# ECW server base URL e.g. "https://ecw.yourdomain.com"
"ecw_url": kv.get("ECW URL", "").strip(),
# ECW credentials — fall back to the OneDrive credentials if not set
"ecw_username": kv.get("ECW Username", "").strip() or cfg.onedrive_username,
"ecw_password": kv.get("ECW Password", "").strip() or cfg.onedrive_password,
# Optional separate TOTP for ECW; falls back to the Microsoft TOTP secret
"ecw_totp_secret": kv.get("ECW TOTP Secret", "").strip() or cfg.microsoft_totp_secret,
# Proxy (reused from existing config)
"proxy_ip": cfg.proxy_ip,
"proxy_port": cfg.proxy_port,
"proxy_username": cfg.proxy_username,
"proxy_password": str(cfg.proxy_password) if cfg.proxy_password else "",
}
# ---------------------------------------------------------------------------
# Step 1 — Playwright login (UI only, closes browser when done)
# ---------------------------------------------------------------------------
def _generate_totp(secret: str) -> str:
import pyotp
return pyotp.TOTP(secret).now()
def login_and_harvest_cookies(ecw_cfg: dict[str, str]) -> dict[str, str]:
"""
Open the ECW login page in a headless Playwright browser, authenticate,
then return all cookies as {name: value} and close the browser.
Only this function uses a browser — everything after it is pure HTTP.
TODO (fill in once the login page HTML is shared):
- Update the CSS selectors for the username / password / submit fields
- Update the success condition (URL prefix or element that appears after login)
- Add OTP step if ECW shows a 2-FA prompt
"""
from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
base_url = ecw_cfg["ecw_url"]
username = ecw_cfg["ecw_username"]
password = ecw_cfg["ecw_password"]
totp_secret = ecw_cfg["ecw_totp_secret"]
# Build proxy dict for Playwright if configured
pw_proxy = None
if ecw_cfg.get("proxy_ip") and ecw_cfg.get("proxy_port"):
proxy_server = f"http://{ecw_cfg['proxy_ip']}:{ecw_cfg['proxy_port']}"
pw_proxy = {"server": proxy_server}
if ecw_cfg.get("proxy_username"):
pw_proxy["username"] = ecw_cfg["proxy_username"]
pw_proxy["password"] = ecw_cfg.get("proxy_password", "")
log_step("ECW login via Playwright — opening browser")
cookies: dict[str, str] = {}
with sync_playwright() as pw:
browser = pw.chromium.launch(
headless=True,
args=["--disable-blink-features=AutomationControlled"],
proxy=pw_proxy,
)
ctx = browser.new_context(
ignore_https_errors=True,
viewport={"width": 1280, "height": 900},
)
page = ctx.new_page()
# ── Navigate to login page ────────────────────────────────────────
login_url = base_url # TODO: update if login page is at a sub-path
log_step("ECW navigate to login", url=login_url[:80])
page.goto(login_url, wait_until="domcontentloaded", timeout=30_000)
# ── Fill credentials ──────────────────────────────────────────────
# TODO: update selectors to match the actual ECW login form
#
# Common patterns:
# input[name="username"] / input[name="j_username"] / #userId
# input[name="password"] / input[name="j_password"] / #userPass
# button[type="submit"] / input[type="submit"] / #login-btn
#
USERNAME_SELECTOR = "input[name='username']" # TODO
PASSWORD_SELECTOR = "input[name='password']" # TODO
SUBMIT_SELECTOR = "button[type='submit']" # TODO
page.wait_for_selector(USERNAME_SELECTOR, timeout=15_000)
page.fill(USERNAME_SELECTOR, username)
page.fill(PASSWORD_SELECTOR, password)
log_step("ECW credentials filled")
page.click(SUBMIT_SELECTOR)
# ── Handle OTP / 2-FA prompt (if ECW requires it) ────────────────
# TODO: update selector and logic based on the actual 2-FA screen
OTP_SELECTOR = "input[name='otp']" # TODO — set to "" to skip
if OTP_SELECTOR:
try:
page.wait_for_selector(OTP_SELECTOR, timeout=8_000)
otp = _generate_totp(totp_secret) if totp_secret else ""
if otp:
page.fill(OTP_SELECTOR, otp)
page.click(SUBMIT_SELECTOR)
log_step("ECW OTP submitted")
except PWTimeout:
log_step("No OTP prompt detected — continuing")
# ── Wait for successful login ─────────────────────────────────────
# TODO: update the success condition.
# Option A — wait for URL to change away from the login page:
# page.wait_for_url("**/dashboard**", timeout=30_000)
# Option B — wait for a specific element that only appears after login:
# page.wait_for_selector("#mainContent", timeout=30_000)
#
try:
# Placeholder: just wait for network idle
page.wait_for_load_state("networkidle", timeout=30_000)
log_step("ECW login appears successful", url=page.url[:80])
except PWTimeout:
_log.warning("ECW login wait timed out — harvesting cookies anyway")
# ── Harvest cookies ───────────────────────────────────────────────
for c in ctx.cookies():
cookies[c["name"]] = c["value"]
log_step("ECW cookies harvested", count=len(cookies))
browser.close()
if not cookies:
raise RuntimeError("ECW login produced no cookies — check credentials and selectors.")
return cookies
# ---------------------------------------------------------------------------
# Step 2 — Build httpx client from harvested cookies
# ---------------------------------------------------------------------------
def build_http_client(ecw_cfg: dict[str, str], cookies: dict[str, str]) -> httpx.Client:
"""
Create a persistent httpx.Client with the ECW session cookies baked in.
All subsequent requests use this client — no browser involved.
"""
base_url = ecw_cfg["ecw_url"]
proxies = None
if ecw_cfg.get("proxy_ip") and ecw_cfg.get("proxy_port"):
proxy_url = f"http://{ecw_cfg['proxy_ip']}:{ecw_cfg['proxy_port']}"
if ecw_cfg.get("proxy_username"):
proxy_url = (
f"http://{ecw_cfg['proxy_username']}:{ecw_cfg['proxy_password']}"
f"@{ecw_cfg['proxy_ip']}:{ecw_cfg['proxy_port']}"
)
proxies = {"http://": proxy_url, "https://": proxy_url}
client = httpx.Client(
base_url=base_url,
follow_redirects=True,
timeout=60,
verify=False, # set to True in production if ECW has a valid cert
**({"proxies": proxies} if proxies else {}),
)
for name, value in cookies.items():
client.cookies.set(name, value)
log_step("httpx client ready", base_url=base_url[:60], cookies=len(cookies))
return client
# ---------------------------------------------------------------------------
# Step 3 — Fetch document list (pure HTTP)
# ---------------------------------------------------------------------------
def fetch_document_list(client: httpx.Client, run_date: str) -> str:
"""
GET (or POST) the ECW document-list endpoint for run_date and return raw HTML.
TODO: fill in once the HTML body / network tab is shared:
- ECW_DOC_LIST_PATH — the path relative to ECW_BASE_URL
- query params or POST body the server expects
- any additional headers (Referer, Accept, X-Requested-With, etc.)
"""
ECW_DOC_LIST_PATH = "" # TODO e.g. "/ecw/rd/Documents/ViewDocuments.jsp"
if not ECW_DOC_LIST_PATH:
raise NotImplementedError(
"ECW document list URL not configured.\n"
"Share the HTML body / browser Network tab and this will be filled in."
)
# TODO: adjust params / method once the request format is known
# Example GET:
# resp = client.get(ECW_DOC_LIST_PATH, params={"fromDate": run_date, "toDate": run_date})
#
# Example POST:
# resp = client.post(ECW_DOC_LIST_PATH, data={"date": run_date, "type": "VOB"})
#
resp = client.get(ECW_DOC_LIST_PATH, params={"date": run_date})
resp.raise_for_status()
log_step("Document list fetched", status=resp.status_code, bytes=len(resp.text))
return resp.text
# ---------------------------------------------------------------------------
# Step 4 — Parse HTML response (pure Python / BeautifulSoup)
# ---------------------------------------------------------------------------
def parse_facility_files(html: str, facilities: list[str]) -> dict[str, list[str]]:
"""
Parse the ECW document-list HTML and return:
{ facility_name: [absolute_download_url, ...] }
Only facilities in the configured list are returned.
TODO: update selectors once the HTML body is shared.
The two commented-out patterns below cover the most common ECW layouts.
"""
soup = BeautifulSoup(html, "html.parser")
facility_lower = {f.strip().lower(): f for f in facilities}
result: dict[str, list[str]] = {f: [] for f in facilities}
# ── Pattern A — one section / card per facility ───────────────────────
# for section in soup.select(".facility-section"): # TODO selector
# name_el = section.select_one(".facility-name") # TODO selector
# if not name_el:
# continue
# ecw_name = name_el.get_text(strip=True)
# matched = _match_facility(ecw_name, facility_lower)
# if not matched:
# continue
# for a in section.select("a[href]"):
# href = a.get("href", "")
# if _is_file_link(href):
# result[matched].append(_abs_url(client.base_url, href))
# ── Pattern B — flat table, each row = one file ───────────────────────
# for row in soup.select("table tbody tr"): # TODO selector
# cells = row.find_all("td")
# if len(cells) < 2:
# continue
# ecw_name = cells[0].get_text(strip=True) # TODO column index
# a = cells[-1].find("a", href=True) # TODO column index
# if not a:
# continue
# matched = _match_facility(ecw_name, facility_lower)
# if not matched:
# continue
# href = a["href"]
# if _is_file_link(href):
# result[matched].append(_abs_url(client.base_url, href))
_log.warning(
"parse_facility_files: selectors not yet configured — "
"share the HTML body so this can be filled in."
)
return result
def _match_facility(ecw_name: str, facility_lower: dict[str, str]) -> str | None:
"""Fuzzy match an ECW display name to a known facility name."""
key = ecw_name.strip().lower()
if key in facility_lower:
return facility_lower[key]
# Longest prefix overlap wins
candidates = [
(k, v) for k, v in facility_lower.items()
if key.startswith(k) or k.startswith(key)
]
return max(candidates, key=lambda t: len(t[0]))[1] if candidates else None
def _is_file_link(href: str) -> bool:
low = href.lower()
return (
any(low.endswith(ext) for ext in (".pdf", ".xlsx", ".xls", ".doc", ".docx"))
or any(kw in low for kw in ("download", "getfile", "getdoc", "document", "attachment"))
)
def _abs_url(base_url: str, href: str) -> str:
"""Return an absolute URL, joining with base_url if href is relative."""
if href.startswith(("http://", "https://")):
return href
return urljoin(base_url, href)
# ---------------------------------------------------------------------------
# Step 5 — Download a file (pure HTTP)
# ---------------------------------------------------------------------------
def download_file(client: httpx.Client, url: str, dest_folder: Path) -> Path | None:
"""Download one file from url into dest_folder. Returns saved Path or None."""
try:
resp = client.get(url, follow_redirects=True)
resp.raise_for_status()
filename = _filename_from_response(resp, url)
dest_folder.mkdir(parents=True, exist_ok=True)
dest = _unique_path(dest_folder, filename)
dest.write_bytes(resp.content)
log_step("Downloaded", url=url[-60:], file=dest.name, bytes=len(resp.content))
return dest
except Exception as exc:
_log.warning("Download failed (%s): %s", url[-60:], exc)
return None
def _filename_from_response(resp: httpx.Response, fallback_url: str) -> str:
cd = resp.headers.get("content-disposition", "")
m = re.search(r'filename\*?=["\']?(?:UTF-8\'\')?([^"\';\r\n]+)', cd, re.IGNORECASE)
if m:
return m.group(1).strip().strip('"\'')
path = urlparse(fallback_url).path
name = path.split("/")[-1]
return name or "file.pdf"
def _unique_path(folder: Path, filename: str) -> Path:
candidate = folder / filename
if not candidate.exists():
return candidate
stem, suffix = Path(filename).stem, Path(filename).suffix
for i in range(1, 9999):
candidate = folder / f"{stem}_{i}{suffix}"
if not candidate.exists():
return candidate
return folder / filename
# ---------------------------------------------------------------------------
# Workable copy (identical logic to planner_download._build_workable_files)
# ---------------------------------------------------------------------------
def _sanitize_folder_name(name: str) -> str:
return re.sub(r'[<>:"/\\|?*]', "_", name.strip()) or "unknown_facility"
def _unique_workable_path(folder: Path, base: str, ext: str) -> Path:
candidate = folder / f"{base}{ext}"
if not candidate.exists():
return candidate
for i in range(1, 9999):
candidate = folder / f"{base}_{i}{ext}"
if not candidate.exists():
return candidate
return folder / f"{base}{ext}"
def build_workable_files(
*,
facility_name: str,
downloaded: list[Path],
output_root: Path,
today_iso: str,
) -> tuple[list[Path], Path]:
"""Copy originals to Workable/ (DD-MM-YYYY_FacilityName[_N].pdf) and mirror to Workable_Medicare/."""
ddmmyyyy = datetime.strptime(today_iso, "%Y-%m-%d").strftime("%d-%m-%Y")
fac_safe = _sanitize_folder_name(facility_name)
workable_root = output_root / today_iso / "Workable"
medicare_root = output_root / today_iso / "Workable_Medicare"
workable_paths: list[Path] = []
if not downloaded:
return workable_paths, workable_root
workable_root.mkdir(parents=True, exist_ok=True)
medicare_root.mkdir(parents=True, exist_ok=True)
for i, src in enumerate(downloaded, start=1):
ext = src.suffix or ".pdf"
base = (
f"{ddmmyyyy}_{fac_safe}"
if len(downloaded) == 1
else f"{ddmmyyyy}_{fac_safe}_{i}"
)
dst = _unique_workable_path(workable_root, base, ext)
shutil.copy2(src, dst)
workable_paths.append(dst)
med_dst = medicare_root / dst.name
if med_dst.exists():
med_dst = _unique_workable_path(medicare_root, dst.stem, ext)
shutil.copy2(dst, med_dst)
_log.info("Workable (%d files) -> %s | Medicare -> %s",
len(workable_paths), workable_root, medicare_root)
return workable_paths, workable_root
# ---------------------------------------------------------------------------
# Summary Excel (same columns as the Planner bot)
# ---------------------------------------------------------------------------
def write_summary(
results: dict[str, dict],
failures: list[tuple[str, str]],
summary_path: Path,
run_timestamp: str,
) -> None:
try:
import pandas as pd
except ImportError:
_log.warning("pandas not available — skipping summary Excel")
return
rows = []
failure_names = {f for f, _ in failures}
for facility, payload in results.items():
files = len(payload.get("downloaded", []))
rows.append({
"Run Timestamp": run_timestamp,
"Facility Name": facility,
"Files Downloaded": files,
"Status": "Success" if files else "No files",
"Status Changed": "N/A (ECW)",
})
for facility, err in failures:
rows.append({
"Run Timestamp": run_timestamp,
"Facility Name": facility,
"Files Downloaded": 0,
"Status": "FAILED",
"Status Changed": "N/A",
})
summary_path.parent.mkdir(parents=True, exist_ok=True)
import pandas as pd
pd.DataFrame(rows).to_excel(summary_path, index=False, sheet_name="Download")
_log.info("Summary -> %s", summary_path)
# ---------------------------------------------------------------------------
# Main workflow
# ---------------------------------------------------------------------------
def run_ecw_workflow(
*,
facility_limit: int = 0,
facility_name: str | None = None,
run_date: str | None = None,
resume: bool = False,
) -> None:
run_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
today_iso = run_date or TODAY_ISO_DATE
log_path = setup_logging(PROJECT_ROOT)
_log.info("=" * 66)
_log.info(" MISF ECW Download — Start")
_log.info(" Run date : %s", today_iso)
_log.info(" Log : %s", log_path)
_log.info("=" * 66)
log_step("ECW run started", date=today_iso)
ecw_cfg = _read_ecw_config()
if not ecw_cfg["ecw_url"]:
raise ValueError(
"ECW URL not configured. "
"Add 'ECW URL' to Input/input.xlsx Sheet1."
)
output_root = PROJECT_ROOT / "Output"
original_root = output_root / today_iso / "Original"
checkpoint_path = output_root / today_iso / "checkpoint_ecw.json"
summary_path = (
output_root / today_iso / "Summary"
/ f"Download_Summary_ECW_{today_iso}.xlsx"
)
# ── Facility list ─────────────────────────────────────────────────────
facilities: list[str] = read_facility_list()
if facility_name:
facilities = [f for f in facilities if f.lower() == facility_name.lower()]
if not facilities:
_log.error("Facility '%s' not found in list.", facility_name)
return
if facility_limit > 0:
facilities = facilities[:facility_limit]
# ── Resume / checkpoint ───────────────────────────────────────────────
start_index = 0
completed: set[str] = set()
if resume:
ckpt = load_checkpoint(checkpoint_path)
start_index = int(ckpt.get("last_completed_index", 0) or 0)
completed = {str(x) for x in (ckpt.get("completed_facilities", []) or [])}
_log.info("Resuming from index %d; %d already completed", start_index, len(completed))
else:
clear_checkpoint(checkpoint_path)
# ── Step 1 — Login via Playwright (UI only) ───────────────────────────
cookies = login_and_harvest_cookies(ecw_cfg)
# ── Step 2 — Build httpx client (all remaining requests are HTTP) ─────
client = build_http_client(ecw_cfg, cookies)
# ── Step 3 — Fetch document list ──────────────────────────────────────
log_step("Fetching ECW document list", date=today_iso)
html = fetch_document_list(client, today_iso)
# ── Step 4 — Parse HTML for per-facility download links ───────────────
facility_files = parse_facility_files(html, facilities)
total_found = sum(len(v) for v in facility_files.values())
log_step("Files parsed from document list", total=total_found)
results: dict[str, dict] = {}
failures: list[tuple[str, str]] = []
total_dl = 0
for idx, fac in enumerate(facilities, start=1):
if idx <= start_index:
continue
if fac in completed:
_log.info("Skipping already completed: %s", fac)
continue
log_step("Facility", index=f"{idx}/{len(facilities)}", facility=fac)
urls = facility_files.get(fac, [])
if not urls:
_log.info("%s: no files found in ECW document list", fac)
results[fac] = {"downloaded": [], "workable": []}
completed.add(fac)
save_checkpoint(checkpoint_path, last_completed_index=idx,
facility_name=fac, completed_facilities=sorted(completed))
continue
dest_folder = original_root / _sanitize_folder_name(fac)
downloaded: list[Path] = []
try:
for url in urls:
path = download_file(client, url, dest_folder)
if path:
downloaded.append(path)
workable_paths, workable_root = build_workable_files(
facility_name=fac,
downloaded=downloaded,
output_root=output_root,
today_iso=today_iso,
)
total_dl += len(downloaded)
results[fac] = {"downloaded": downloaded, "workable": workable_paths}
_log.info("%s: %d downloaded, %d workable", fac, len(downloaded), len(workable_paths))
completed.add(fac)
save_checkpoint(checkpoint_path, last_completed_index=idx,
facility_name=fac, completed_facilities=sorted(completed))
except Exception as exc:
_log.error("%s: %s", fac, exc)
failures.append((fac, str(exc)))
results[fac] = {"downloaded": [], "workable": []}
save_checkpoint(checkpoint_path, last_completed_index=idx,
facility_name=fac, completed_facilities=sorted(completed))
write_summary(results, failures, summary_path, run_timestamp)
clear_checkpoint(checkpoint_path)
_log.info("=" * 66)
_log.info(" ECW DOWNLOAD COMPLETE — %d file(s), %d failure(s)", total_dl, len(failures))
_log.info("=" * 66)
log_step("ECW run complete", files=total_dl, failures=len(failures))
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def _build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description="MISF ECW face-sheet downloader")
p.add_argument("--limit", type=int, default=0, metavar="N",
help="Process only first N facilities")
p.add_argument("--facility", default="", metavar="NAME",
help="Process a single facility by exact name")
p.add_argument("--date", default="", metavar="YYYY-MM-DD",
help="Override run date (default: today)")
p.add_argument("--resume", action="store_true",
help="Resume from last checkpoint")
return p
def main() -> int:
args = _build_parser().parse_args()
try:
run_ecw_workflow(
facility_limit=max(0, args.limit),
facility_name=args.facility.strip() or None,
run_date=args.date.strip() or None,
resume=args.resume,
)
return 0
except KeyboardInterrupt:
print("\nCancelled.")
return 130
except Exception:
import traceback
traceback.print_exc()
return 1
if __name__ == "__main__":
raise SystemExit(main())
onedrive_planner_http.py
+1088
View File
File diff suppressed because it is too large. Load diff
src/chrome.py
+48 -7
View File
@@ -397,10 +397,33 @@ def _is_image_mostly_black(path: Path, *, threshold: int = 18) -> bool:
return False
def _load_timestamp_font(size: int = 22):
"""Return a PIL ImageFont at the requested size, falling back to the built-in bitmap font."""
try:
from PIL import ImageFont
candidates = [
"C:/Windows/Fonts/segoeui.ttf",
"C:/Windows/Fonts/arial.ttf",
"C:/Windows/Fonts/calibri.ttf",
"C:/Windows/Fonts/verdana.ttf",
]
for path in candidates:
try:
return ImageFont.truetype(path, size)
except Exception:
continue
except Exception:
pass
try:
from PIL import ImageFont
return ImageFont.load_default()
except Exception:
return None
def _capture_browser_with_timestamp_bar(driver: webdriver.Chrome, path: Path) -> bool:
"""
Fallback when desktop capture is black: maximized browser PNG plus a taskbar-style clock strip.
"""
"""Browser viewport PNG with a timestamp bar showing machine time."""
try:
import io
@@ -409,18 +432,36 @@ def _capture_browser_with_timestamp_bar(driver: webdriver.Chrome, path: Path) ->
prepare_browser_for_screenshot(driver)
png = driver.get_screenshot_as_png()
browser = Image.open(io.BytesIO(png)).convert("RGB")
bar_h = 48
canvas = Image.new("RGB", (browser.width, browser.height + bar_h), (30, 30, 30))
bar_h = 36
canvas = Image.new("RGB", (browser.width, browser.height + bar_h), (24, 24, 24))
canvas.paste(browser, (0, 0))
draw = ImageDraw.Draw(canvas)
stamp = datetime.now().strftime("%I:%M %p %m/%d/%Y").lstrip("0")
draw.text((max(browser.width - 260, 8), browser.height + 14), stamp, fill=(255, 255, 255))
stamp = datetime.now().strftime("%I:%M:%S %p %m/%d/%Y").lstrip("0")
font = _load_timestamp_font(22)
y = browser.height + (bar_h - 22) // 2
if font:
draw.text((12, y), stamp, fill=(220, 220, 220), font=font)
else:
draw.text((12, y), stamp, fill=(220, 220, 220))
canvas.save(str(path), "PNG")
return path.is_file() and path.stat().st_size > 0
except Exception:
return False
def capture_browser_screenshot(driver: webdriver.Chrome, path: Path) -> bool:
"""Capture only the Chromium browser viewport — no dependency on the host desktop."""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
log_step("Browser screenshot", path=str(path))
if _capture_browser_with_timestamp_bar(driver, path):
log_step("Browser screenshot saved", path=str(path))
return True
driver.save_screenshot(str(path))
log_step("Viewport screenshot saved", path=str(path))
return path.is_file()
def capture_fullscreen_screenshot(driver: webdriver.Chrome, path: Path) -> bool:
"""
Full-screen capture with system taskbar visible when an interactive desktop exists.
src/onedrive_planner.py
+183 -39
View File
@@ -19,6 +19,7 @@ from pathlib import Path
import pyotp
from chrome import DOWNLOADS_DIR as _CHROME_DOWNLOADS_DIR
from chrome import capture_browser_screenshot
from chrome import capture_fullscreen_screenshot
from config import DEFAULT_ONEDRIVE_URL, DEFAULT_PLANNER_URL, PROJECT_ROOT, TODAY_ISO_DATE
from debug_log import write_agent_session_debug as _agent_session_log
@@ -367,11 +368,15 @@ def _on_planner_plan_board(driver: WebDriver) -> bool:
def _plan_load_error_present(driver: WebDriver) -> bool:
"""True when Planner shows 'Couldn't open your plan' (URL may still be /plan/)."""
"""True when Planner shows any error overlay that blocks normal operation."""
phrases = (
"couldn't open your plan",
"could not open your plan",
"try again in a few minutes",
"something went wrong",
"something's not right",
"we couldn't load",
"please try again",
)
try:
blob = (driver.find_element(By.TAG_NAME, "body").text or "").lower()
@@ -380,6 +385,33 @@ def _plan_load_error_present(driver: WebDriver) -> bool:
return False
def _recover_planner_error(driver: WebDriver) -> bool:
"""Detect and dismiss a Planner error overlay ('Something went wrong', etc).
Tries clicking the 'Try again' button first; falls back to a full page refresh.
Returns True if an error was detected (regardless of whether recovery succeeded).
"""
if not _plan_load_error_present(driver):
return False
log_step("Planner error overlay detected — attempting recovery")
if _click_plan_try_again(driver):
time.sleep(4)
if not _plan_load_error_present(driver):
log_step("Planner error overlay cleared via Try again")
return True
# Fallback: full page refresh
try:
log_step("Refreshing page to clear Planner error overlay")
driver.refresh()
time.sleep(6)
driver.switch_to.default_content()
_switch_to_planner_context(driver)
log_step("Page refreshed — error overlay cleared")
except Exception as exc:
log_step("Refresh failed during error recovery", error=str(exc)[:120])
return True
def _plan_page_ready(driver: WebDriver) -> bool:
"""Plan UI is usable — not the error page."""
if _plan_load_error_present(driver):
@@ -1157,6 +1189,7 @@ def open_my_plans_shared(driver: WebDriver) -> None:
This matches your current UI flow and avoids search-based plan opening.
"""
_recover_planner_error(driver)
log_step("Navigate: My plans -> Shared")
log_url(driver, "before")
driver.switch_to.default_content()
@@ -1374,6 +1407,25 @@ def _open_facility_from_shared_list(driver: WebDriver, facility_name: str) -> No
_type_facility_filter(driver, term)
if _filter_shows_no_results(driver):
log_step("Filter returned no results", query=term[:60], facility=facility_name)
# A Planner error overlay may be blocking the search UI — try to recover
if _recover_planner_error(driver):
log_step("Recovered from error overlay during filter search", facility=facility_name)
# Re-open My plans -> Shared after the page refreshed
_click_any(driver, _MY_PLANS_NAV_LOCATORS, timeout=WAIT_LONG)
time.sleep(2)
shared_locators = [
(By.XPATH, "//*[self::button or self::div or self::span][normalize-space()='Shared']"),
(By.XPATH, "//*[@role='tab' and normalize-space()='Shared']"),
]
try:
_click_any(driver, shared_locators, timeout=WAIT_LONG)
time.sleep(2)
except TimeoutException:
pass
_type_facility_filter(driver, term)
if _filter_shows_no_results(driver):
continue
else:
continue
rows = _shared_list_data_rows(driver)
log_step(
@@ -1611,6 +1663,53 @@ def _ensure_board_view(driver: WebDriver) -> None:
log_step("Board tab not found (continuing on current view)")
def _wait_for_board_loaded(driver: WebDriver, timeout: int = 20) -> None:
"""Switch to Board tab and wait until board content is visible before screenshotting."""
board_locators = [
(By.XPATH, "//*[@role='tab' and normalize-space()='Board']"),
(By.XPATH, "//*[self::button or self::div][normalize-space()='Board']"),
]
# Click the Board tab (ignore if already on board or tab not found)
try:
tab = _first_clickable(driver, board_locators, timeout=8)
# Only click if the tab isn't already selected
selected = (tab.get_attribute("aria-selected") or "").lower()
if selected != "true":
log_step("Switch to Board view")
_js_click(driver, tab)
else:
log_step("Board tab already active")
except TimeoutException:
log_step("Board tab not found (continuing on current view)")
# Poll until board content is visible (task cards, bucket columns, or Add task)
content_xpaths = [
"//button[@data-testid='attachment-indicator']",
"//*[@role='checkbox']",
"//label[@role='checkbox']",
"//*[contains(@data-testid,'TaskCard') or contains(@class,'TaskCard')]",
"//*[@role='list' and ancestor::*[@role='group']]",
"//*[normalize-space()='Add task' and ancestor::*[@role='group']]",
"//*[normalize-space()='Add task']",
]
log_step("Waiting for board content to load")
deadline = time.time() + timeout
while time.time() < deadline:
for xpath in content_xpaths:
try:
for el in driver.find_elements(By.XPATH, xpath)[:6]:
if el.is_displayed():
log_step("Board content visible — ready for screenshot")
return
except Exception:
continue
time.sleep(0.5)
log_step("Board content wait timed out — proceeding with screenshot anyway")
def _ensure_grid_view(driver: WebDriver) -> None:
grid_locators = [
(By.CSS_SELECTOR, "button#gridViewPivot"),
@@ -1642,13 +1741,13 @@ def _is_status_sorted_descending(header) -> bool:
def _sort_descending_menu_selected(driver: WebDriver) -> bool:
"""True when the column header menu shows Sort Descending as active."""
"""True when the column header menu shows Sort descending as active."""
probes = [
"//button[@name='Sort Descending' and @aria-checked='true']",
"//button[@name='Sort Descending' and contains(@class,'is-checked')]",
"//button[@name='Sort Descending']//*[@data-icon-name='CheckMark']",
"//button[@name='Sort descending' and @aria-checked='true']",
"//button[@name='Sort descending' and contains(@class,'is-checked')]",
"//button[@name='Sort descending']//*[@data-icon-name='CheckMark']",
(
"//button[@name='Sort Descending']"
"//button[@name='Sort descending']"
"/ancestor::li[contains(@class,'is-checked') or contains(@class,'is-selected')]"
),
]
@@ -1692,13 +1791,18 @@ def _ensure_status_sort_descending(driver: WebDriver) -> None:
return
sort_locators = [
# Exact match — actual Planner DOM uses lowercase 'd'
(By.XPATH, "//button[@name='Sort descending' and @role='menuitem']"),
(By.XPATH, "//ul[contains(@class,'ContextualMenu')]//button[normalize-space()='Sort descending']"),
(By.XPATH, "//button[@role='menuitem' and normalize-space()='Sort descending']"),
(By.XPATH, "//span[normalize-space()='Sort descending']/ancestor::button[@role='menuitem']"),
# Uppercase fallback (in case Planner localises differently)
(By.XPATH, "//button[@name='Sort Descending' and @role='menuitem']"),
(
By.XPATH,
"//ul[contains(@class,'ContextualMenu')]"
"//button[normalize-space()='Sort Descending']",
),
(By.XPATH, "//button[@role='menuitem' and normalize-space()='Sort Descending']"),
# Broad contains — catches any capitalisation
(By.XPATH, "//*[@role='menuitem' and contains(normalize-space(),'Sort descending')]"),
(By.XPATH, "//*[@role='menuitem' and contains(normalize-space(),'Sort Descending')]"),
(By.XPATH, "//li[contains(@class,'ms-ContextualMenu-item')]//button[contains(normalize-space(),'escending')]"),
]
try:
_click_any(driver, sort_locators, timeout=5)
@@ -1712,7 +1816,29 @@ def _ensure_status_sort_descending(driver: WebDriver) -> None:
except TimeoutException:
log_step("Status column sort descending clicked")
except TimeoutException:
log_step("Sort Descending menu item not found (continuing on current sort)")
log_step("Sort descending menu item not found via XPath — trying JS click")
clicked = False
try:
clicked = driver.execute_script(
"""
var items = Array.from(document.querySelectorAll(
'[role="menuitem"], [role="menuitem"] button, button[role="menuitem"]'
));
var target = items.find(function(el) {
var t = (el.innerText || el.textContent || '').trim().toLowerCase();
return t === 'sort descending';
});
if (target) { target.click(); return true; }
return false;
"""
)
except Exception:
pass
if clicked:
time.sleep(1)
log_step("Sort descending clicked via JS")
else:
log_step("Sort descending not found (continuing on current sort)")
_dismiss_open_menu(driver)
@@ -1902,11 +2028,14 @@ def _scroll_grid_down(driver: WebDriver, pixels: int = 450) -> None:
time.sleep(0.4)
def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100) -> list[str]:
"""Scroll the virtualized grid and collect Not started task titles."""
def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100) -> list[tuple[str, int]]:
"""Scroll the virtualized grid and collect (title, aria-rowindex) for Not started tasks.
Uses aria-rowindex as the unique key so duplicate task names are all captured.
"""
_scroll_grid_to_top(driver)
seen: set[str] = set()
titles: list[str] = []
seen_indices: set[int] = set()
tasks: list[tuple[str, int]] = []
stagnant = 0
for _ in range(max_scrolls):
new_this_pass = 0
@@ -1914,14 +2043,15 @@ def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100)
status = _grid_row_status(row)
if status != "not started":
continue
idx_raw = row.get_attribute("aria-rowindex") or ""
row_idx = int(idx_raw) if idx_raw.isdigit() else -1
if row_idx < 1 or row_idx in seen_indices:
continue
title = _grid_row_task_name(row).strip()
if not title or _is_weak_task_title_key(title.lower()):
continue
key = title.lower()
if key in seen:
continue
seen.add(key)
titles.append(title)
seen_indices.add(row_idx)
tasks.append((title, row_idx))
new_this_pass += 1
if new_this_pass == 0:
stagnant += 1
@@ -1931,8 +2061,22 @@ def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100)
stagnant = 0
_scroll_grid_down(driver)
_scroll_grid_to_top(driver)
log_step("Grid not-started tasks scanned", count=len(titles))
return titles
log_step("Grid not-started tasks scanned", count=len(tasks))
return tasks
def _find_grid_row_by_index(driver: WebDriver, row_index: int, *, max_scrolls: int = 100):
"""Find a grid row by its aria-rowindex — stable even when multiple rows share the same title."""
if row_index < 1:
return None
_scroll_grid_to_top(driver)
for _ in range(max_scrolls):
for row in _grid_visible_data_rows(driver):
idx_raw = row.get_attribute("aria-rowindex") or ""
if idx_raw.isdigit() and int(idx_raw) == row_index:
return row
_scroll_grid_down(driver)
return None
def _find_grid_row_by_title(driver: WebDriver, title: str, *, max_scrolls: int = 100):
@@ -6047,13 +6191,13 @@ def save_board_screenshot(
safe = _sanitize_folder_name(facility_name)
taken_at = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
path = shots_dir / f"{safe}_{taken_at}.png"
capture_fullscreen_screenshot(driver, path)
capture_browser_screenshot(driver, path)
log_step(
"Board screenshot saved",
facility=facility_name,
tasks=task_count,
path=str(path),
fullscreen="true",
fullscreen="false",
view="board",
)
if task_count:
@@ -6114,6 +6258,7 @@ def process_facility(
try:
open_my_plans_shared(driver)
_open_facility_from_shared_list(driver, facility_name)
_wait_for_board_loaded(driver)
save_board_screenshot(driver, facility_name)
_ensure_grid_view(driver)
_ensure_status_sort_descending(driver)
@@ -6135,8 +6280,8 @@ def process_facility(
log_step("Grid not visible — waiting for grid to settle", facility=facility_name)
time.sleep(6)
task_titles = _scan_grid_not_started_titles(driver)
task_count = len(task_titles)
task_entries = _scan_grid_not_started_titles(driver)
task_count = len(task_entries)
log_step(
"Scan tasks on grid",
task_count=task_count,
@@ -6147,19 +6292,21 @@ def process_facility(
return FacilityRunResult()
download_dir_ready = False
processed_titles: set[str] = set()
processed_indices: set[int] = set()
for task_num, grid_title in enumerate(task_titles, start=1):
for task_num, (grid_title, row_index) in enumerate(task_entries, start=1):
title_key = grid_title.strip().lower()
if title_key in processed_titles:
if row_index in processed_indices:
continue
_close_task_panel(driver)
task_title = ""
try:
row = _find_grid_row_by_title(driver, grid_title)
# Locate by row index first (handles duplicate task names correctly),
# fall back to title search if the row has no aria-rowindex.
row = _find_grid_row_by_index(driver, row_index) or _find_grid_row_by_title(driver, grid_title)
if row is None:
log_step("Grid row not found", title=grid_title[:60], facility=facility_name)
processed_titles.add(title_key)
processed_indices.add(row_index)
continue
row_status = _grid_row_status(row)
@@ -6170,7 +6317,7 @@ def process_facility(
status=row_status or "",
facility=facility_name,
)
processed_titles.add(title_key)
processed_indices.add(row_index)
continue
log_step(
@@ -6180,7 +6327,7 @@ def process_facility(
facility=facility_name,
)
# Re-find row right before click — virtualized grid can shift rows.
row = _find_grid_row_by_title(driver, grid_title)
row = _find_grid_row_by_index(driver, row_index) or _find_grid_row_by_title(driver, grid_title)
if row is None or not _open_grid_row_details(driver, row):
log_step("Grid task open failed", title=grid_title[:60], facility=facility_name)
continue
@@ -6188,7 +6335,7 @@ def process_facility(
# Grid Status column already filtered to Not started — go straight
# to Attachments tab (no panel status gate).
task_title = _clean_grid_task_title(grid_title)
processed_titles.add(task_title.strip().lower() or title_key)
processed_indices.add(row_index)
log_step("Task opened from grid", title=task_title[:60])
time.sleep(PANEL_TAB_LOAD_DELAY)
@@ -6280,10 +6427,7 @@ def process_facility(
)
_log.warning("Task %d failed: %s", task_num, exc)
_close_task_panel(driver)
if task_title:
processed_titles.add(task_title.strip().lower())
elif title_key:
processed_titles.add(title_key)
processed_indices.add(row_index)
_remove_empty_directory(dest_folder)
log_step(
src/planner_download.py
+76
View File
@@ -58,6 +58,64 @@ from debug_log import write_debug as _agent_log
_log = logging.getLogger(__name__)
def _is_browser_dead(exc: Exception) -> bool:
"""True when Chrome itself has crashed and the WebDriver session is gone."""
err = str(exc).lower()
return any(phrase in err for phrase in (
"invalid session id",
"session deleted",
"browser has closed the connection",
"chrome not reachable",
"cannot connect to chrome",
"no such window",
))
def _restart_browser(cfg: "AppConfig") -> "webdriver.Chrome | None":
"""Kill Chrome, relaunch it, and re-login to Planner. Returns new driver on success."""
log_step("Browser crash detected — restarting Chrome")
_log.warning("Restarting Chrome after crash...")
try:
stop_chrome_browser()
except Exception:
pass
time.sleep(4)
try:
if not start_chrome_debug(
cfg.proxy_ip, cfg.proxy_port, cfg.proxy_username, cfg.proxy_password
):
log_step("Chrome restart failed — start_chrome_debug returned False")
return None
new_driver = attach_browser(create_download_dir=False)
if new_driver is None:
log_step("Chrome restart failed — attach_browser returned None")
return None
consolidate_browser_to_single_tab(new_driver)
if cfg.proxy_username and cfg.proxy_password:
dismiss_proxy_auth_popup(
new_driver,
cfg.proxy_username.strip(),
str(cfg.proxy_password),
wait_secs=8.0,
max_attempts=3,
)
time.sleep(2)
login_and_open_planner(
new_driver,
cfg.onedrive_username,
cfg.onedrive_password,
cfg.microsoft_totp_secret,
planner_url=cfg.planner_url,
)
log_step("Chrome restarted and Planner re-opened after crash")
_log.info("Chrome restarted successfully after crash.")
return new_driver
except Exception as exc:
log_step("Browser restart failed", error=str(exc)[:200])
_log.error("Browser restart failed: %s", exc)
return None
def _show_error_dialog(title: str, message: str) -> None:
try:
root = tk.Tk()
@@ -624,7 +682,10 @@ def run_download_workflow(
log_step("Facility failed", facility=fac, error=err_msg[:200], status=status)
_log.error("%s: %s", fac, err_msg)
if screenshot_on_error and driver is not None:
try:
save_failure_screenshot(driver, fac, failure_log_dir=failure_log_dir)
except Exception:
pass
failures.append((fac, err_msg, status))
results[fac] = {
"original": [],
@@ -644,6 +705,21 @@ def run_download_workflow(
completed_facilities=sorted(completed),
)
# Chrome crashed — restart it so remaining facilities can still run
if _is_browser_dead(exc):
_log.warning(
"Browser crash detected after '%s' — restarting Chrome to continue run",
fac,
)
new_driver = _restart_browser(cfg)
if new_driver is not None:
driver = new_driver
else:
_log.error(
"Could not restart browser after crash — aborting remaining facilities"
)
break
summary_path = (
PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "Summary"
/ f"Download_Summary_{TODAY_ISO_DATE}.xlsx"