diff --git a/build_exe.bat b/build_exe.bat new file mode 100644 index 0000000..ceae374 --- /dev/null +++ b/build_exe.bat @@ -0,0 +1,57 @@ +@echo off +setlocal EnableDelayedExpansion +cd /d "%~dp0" + +echo ============================================================ +echo MISF Planner Download -- Build EXE +echo ============================================================ +echo. + +REM ── 1. Install / upgrade PyInstaller inside the venv ───────────────────── +echo [1/4] Ensuring PyInstaller is installed... +.venv\Scripts\pip install --quiet --upgrade pyinstaller +if %ERRORLEVEL% NEQ 0 ( + echo ERROR: pip install pyinstaller failed. Is the venv set up? + pause & exit /b 1 +) + +REM ── 2. Clean previous build artefacts ──────────────────────────────────── +echo [2/4] Cleaning previous build... +if exist build rmdir /s /q build +if exist dist\MISF_Planner_Download rmdir /s /q dist\MISF_Planner_Download + +REM ── 3. Build ────────────────────────────────────────────────────────────── +echo [3/4] Building (this takes 2-5 minutes)... +echo. +.venv\Scripts\pyinstaller onedrive_planner_http.spec --clean --noconfirm +if %ERRORLEVEL% NEQ 0 ( + echo. + echo ERROR: PyInstaller build failed. Check output above. + pause & exit /b 1 +) + +REM ── 4. Copy Input folder so the exe knows where to read config ──────────── +echo. +echo [4/4] Copying Input\ folder into dist... +if exist Input ( + xcopy /E /I /Y "Input" "dist\MISF_Planner_Download\Input" >nul + echo Input\ copied. +) else ( + echo WARNING: No Input\ folder found -- copy it manually before shipping. +) + +echo. +echo ============================================================ +echo BUILD COMPLETE +echo. +echo Folder : dist\MISF_Planner_Download\ +echo Exe : dist\MISF_Planner_Download\MISF_Planner_Download.exe +echo. +echo TO DEPLOY ON ANOTHER MACHINE: +echo 1. Copy the entire dist\MISF_Planner_Download\ folder. +echo 2. Make sure Input\input.xlsx is inside that folder. +echo 3. Double-click MISF_Planner_Download.exe. +echo (First run downloads the browser automatically.) +echo ============================================================ +echo. +pause diff --git a/ecw_download.py b/ecw_download.py new file mode 100644 index 0000000..f47cac7 --- /dev/null +++ b/ecw_download.py @@ -0,0 +1,679 @@ +#!/usr/bin/env python3 +""" +ECW-server VOB / face-sheet downloader. + +Auth flow +--------- +Playwright opens the ECW login page, fills credentials + OTP if required, +waits for a successful session, then harvests all cookies and closes the +browser. An httpx.Client is built from those cookies and used for every +subsequent request — no browser involved after login. + +Document / download flow (pure HTTP after login) +-------------------------------------------------- +1. GET the ECW document-list endpoint → HTML response +2. Parse the HTML to find per-facility download links +3. Download each file directly into Output//Original// +4. Copy to Workable/ and Workable_Medicare/ with the standard naming + DD-MM-YYYY_[_N].pdf +5. Write Download_Summary_ECW_.xlsx + +Config keys in Input/input.xlsx Sheet1 +---------------------------------------- + ECW URL https://ecw.yourdomain.com (required) + ECW Username user@domain.com (optional — falls back to OneDrive Username) + ECW Password secret (optional — falls back to OneDrive Password) + ECW TOTP Secret BASE32SECRET (optional — for ECW-specific 2FA) + Proxy IP / Proxy Port / Proxy Username / Proxy Password (reused from existing config) + +Usage +----- + python ecw_download.py + python ecw_download.py --limit 5 + python ecw_download.py --facility "Arbor Hills" + python ecw_download.py --date 2026-06-19 + python ecw_download.py --resume +""" +from __future__ import annotations + +import argparse +import logging +import re +import shutil +import sys +import time +from datetime import datetime +from pathlib import Path +from urllib.parse import urljoin, urlparse + +ROOT = Path(__file__).resolve().parent +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +import httpx +from bs4 import BeautifulSoup + +from checkpoint import clear_checkpoint, load_checkpoint, save_checkpoint +from config import ( + PROJECT_ROOT, + TODAY_ISO_DATE, + AppConfig, + read_app_config, + read_facility_list, + _read_config_key_value_sheet, +) +from logging_setup import setup_logging +from run_log import log_step + +_log = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# ECW-specific config (on top of the existing AppConfig) +# --------------------------------------------------------------------------- + +def _read_ecw_config() -> dict[str, str]: + """Read ECW-specific keys from the same input.xlsx Sheet1.""" + try: + kv = _read_config_key_value_sheet() + except Exception: + kv = {} + cfg = read_app_config() + return { + # ECW server base URL e.g. "https://ecw.yourdomain.com" + "ecw_url": kv.get("ECW URL", "").strip(), + # ECW credentials — fall back to the OneDrive credentials if not set + "ecw_username": kv.get("ECW Username", "").strip() or cfg.onedrive_username, + "ecw_password": kv.get("ECW Password", "").strip() or cfg.onedrive_password, + # Optional separate TOTP for ECW; falls back to the Microsoft TOTP secret + "ecw_totp_secret": kv.get("ECW TOTP Secret", "").strip() or cfg.microsoft_totp_secret, + # Proxy (reused from existing config) + "proxy_ip": cfg.proxy_ip, + "proxy_port": cfg.proxy_port, + "proxy_username": cfg.proxy_username, + "proxy_password": str(cfg.proxy_password) if cfg.proxy_password else "", + } + + +# --------------------------------------------------------------------------- +# Step 1 — Playwright login (UI only, closes browser when done) +# --------------------------------------------------------------------------- + +def _generate_totp(secret: str) -> str: + import pyotp + return pyotp.TOTP(secret).now() + + +def login_and_harvest_cookies(ecw_cfg: dict[str, str]) -> dict[str, str]: + """ + Open the ECW login page in a headless Playwright browser, authenticate, + then return all cookies as {name: value} and close the browser. + + Only this function uses a browser — everything after it is pure HTTP. + + TODO (fill in once the login page HTML is shared): + - Update the CSS selectors for the username / password / submit fields + - Update the success condition (URL prefix or element that appears after login) + - Add OTP step if ECW shows a 2-FA prompt + """ + from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout + + base_url = ecw_cfg["ecw_url"] + username = ecw_cfg["ecw_username"] + password = ecw_cfg["ecw_password"] + totp_secret = ecw_cfg["ecw_totp_secret"] + + # Build proxy dict for Playwright if configured + pw_proxy = None + if ecw_cfg.get("proxy_ip") and ecw_cfg.get("proxy_port"): + proxy_server = f"http://{ecw_cfg['proxy_ip']}:{ecw_cfg['proxy_port']}" + pw_proxy = {"server": proxy_server} + if ecw_cfg.get("proxy_username"): + pw_proxy["username"] = ecw_cfg["proxy_username"] + pw_proxy["password"] = ecw_cfg.get("proxy_password", "") + + log_step("ECW login via Playwright — opening browser") + cookies: dict[str, str] = {} + + with sync_playwright() as pw: + browser = pw.chromium.launch( + headless=True, + args=["--disable-blink-features=AutomationControlled"], + proxy=pw_proxy, + ) + ctx = browser.new_context( + ignore_https_errors=True, + viewport={"width": 1280, "height": 900}, + ) + page = ctx.new_page() + + # ── Navigate to login page ──────────────────────────────────────── + login_url = base_url # TODO: update if login page is at a sub-path + log_step("ECW navigate to login", url=login_url[:80]) + page.goto(login_url, wait_until="domcontentloaded", timeout=30_000) + + # ── Fill credentials ────────────────────────────────────────────── + # TODO: update selectors to match the actual ECW login form + # + # Common patterns: + # input[name="username"] / input[name="j_username"] / #userId + # input[name="password"] / input[name="j_password"] / #userPass + # button[type="submit"] / input[type="submit"] / #login-btn + # + USERNAME_SELECTOR = "input[name='username']" # TODO + PASSWORD_SELECTOR = "input[name='password']" # TODO + SUBMIT_SELECTOR = "button[type='submit']" # TODO + + page.wait_for_selector(USERNAME_SELECTOR, timeout=15_000) + page.fill(USERNAME_SELECTOR, username) + page.fill(PASSWORD_SELECTOR, password) + log_step("ECW credentials filled") + page.click(SUBMIT_SELECTOR) + + # ── Handle OTP / 2-FA prompt (if ECW requires it) ──────────────── + # TODO: update selector and logic based on the actual 2-FA screen + OTP_SELECTOR = "input[name='otp']" # TODO — set to "" to skip + if OTP_SELECTOR: + try: + page.wait_for_selector(OTP_SELECTOR, timeout=8_000) + otp = _generate_totp(totp_secret) if totp_secret else "" + if otp: + page.fill(OTP_SELECTOR, otp) + page.click(SUBMIT_SELECTOR) + log_step("ECW OTP submitted") + except PWTimeout: + log_step("No OTP prompt detected — continuing") + + # ── Wait for successful login ───────────────────────────────────── + # TODO: update the success condition. + # Option A — wait for URL to change away from the login page: + # page.wait_for_url("**/dashboard**", timeout=30_000) + # Option B — wait for a specific element that only appears after login: + # page.wait_for_selector("#mainContent", timeout=30_000) + # + try: + # Placeholder: just wait for network idle + page.wait_for_load_state("networkidle", timeout=30_000) + log_step("ECW login appears successful", url=page.url[:80]) + except PWTimeout: + _log.warning("ECW login wait timed out — harvesting cookies anyway") + + # ── Harvest cookies ─────────────────────────────────────────────── + for c in ctx.cookies(): + cookies[c["name"]] = c["value"] + log_step("ECW cookies harvested", count=len(cookies)) + + browser.close() + + if not cookies: + raise RuntimeError("ECW login produced no cookies — check credentials and selectors.") + + return cookies + + +# --------------------------------------------------------------------------- +# Step 2 — Build httpx client from harvested cookies +# --------------------------------------------------------------------------- + +def build_http_client(ecw_cfg: dict[str, str], cookies: dict[str, str]) -> httpx.Client: + """ + Create a persistent httpx.Client with the ECW session cookies baked in. + All subsequent requests use this client — no browser involved. + """ + base_url = ecw_cfg["ecw_url"] + proxies = None + if ecw_cfg.get("proxy_ip") and ecw_cfg.get("proxy_port"): + proxy_url = f"http://{ecw_cfg['proxy_ip']}:{ecw_cfg['proxy_port']}" + if ecw_cfg.get("proxy_username"): + proxy_url = ( + f"http://{ecw_cfg['proxy_username']}:{ecw_cfg['proxy_password']}" + f"@{ecw_cfg['proxy_ip']}:{ecw_cfg['proxy_port']}" + ) + proxies = {"http://": proxy_url, "https://": proxy_url} + + client = httpx.Client( + base_url=base_url, + follow_redirects=True, + timeout=60, + verify=False, # set to True in production if ECW has a valid cert + **({"proxies": proxies} if proxies else {}), + ) + for name, value in cookies.items(): + client.cookies.set(name, value) + + log_step("httpx client ready", base_url=base_url[:60], cookies=len(cookies)) + return client + + +# --------------------------------------------------------------------------- +# Step 3 — Fetch document list (pure HTTP) +# --------------------------------------------------------------------------- + +def fetch_document_list(client: httpx.Client, run_date: str) -> str: + """ + GET (or POST) the ECW document-list endpoint for run_date and return raw HTML. + + TODO: fill in once the HTML body / network tab is shared: + - ECW_DOC_LIST_PATH — the path relative to ECW_BASE_URL + - query params or POST body the server expects + - any additional headers (Referer, Accept, X-Requested-With, etc.) + """ + ECW_DOC_LIST_PATH = "" # TODO e.g. "/ecw/rd/Documents/ViewDocuments.jsp" + + if not ECW_DOC_LIST_PATH: + raise NotImplementedError( + "ECW document list URL not configured.\n" + "Share the HTML body / browser Network tab and this will be filled in." + ) + + # TODO: adjust params / method once the request format is known + # Example GET: + # resp = client.get(ECW_DOC_LIST_PATH, params={"fromDate": run_date, "toDate": run_date}) + # + # Example POST: + # resp = client.post(ECW_DOC_LIST_PATH, data={"date": run_date, "type": "VOB"}) + # + resp = client.get(ECW_DOC_LIST_PATH, params={"date": run_date}) + resp.raise_for_status() + log_step("Document list fetched", status=resp.status_code, bytes=len(resp.text)) + return resp.text + + +# --------------------------------------------------------------------------- +# Step 4 — Parse HTML response (pure Python / BeautifulSoup) +# --------------------------------------------------------------------------- + +def parse_facility_files(html: str, facilities: list[str]) -> dict[str, list[str]]: + """ + Parse the ECW document-list HTML and return: + { facility_name: [absolute_download_url, ...] } + + Only facilities in the configured list are returned. + + TODO: update selectors once the HTML body is shared. + The two commented-out patterns below cover the most common ECW layouts. + """ + soup = BeautifulSoup(html, "html.parser") + facility_lower = {f.strip().lower(): f for f in facilities} + result: dict[str, list[str]] = {f: [] for f in facilities} + + # ── Pattern A — one section / card per facility ─────────────────────── + # for section in soup.select(".facility-section"): # TODO selector + # name_el = section.select_one(".facility-name") # TODO selector + # if not name_el: + # continue + # ecw_name = name_el.get_text(strip=True) + # matched = _match_facility(ecw_name, facility_lower) + # if not matched: + # continue + # for a in section.select("a[href]"): + # href = a.get("href", "") + # if _is_file_link(href): + # result[matched].append(_abs_url(client.base_url, href)) + + # ── Pattern B — flat table, each row = one file ─────────────────────── + # for row in soup.select("table tbody tr"): # TODO selector + # cells = row.find_all("td") + # if len(cells) < 2: + # continue + # ecw_name = cells[0].get_text(strip=True) # TODO column index + # a = cells[-1].find("a", href=True) # TODO column index + # if not a: + # continue + # matched = _match_facility(ecw_name, facility_lower) + # if not matched: + # continue + # href = a["href"] + # if _is_file_link(href): + # result[matched].append(_abs_url(client.base_url, href)) + + _log.warning( + "parse_facility_files: selectors not yet configured — " + "share the HTML body so this can be filled in." + ) + return result + + +def _match_facility(ecw_name: str, facility_lower: dict[str, str]) -> str | None: + """Fuzzy match an ECW display name to a known facility name.""" + key = ecw_name.strip().lower() + if key in facility_lower: + return facility_lower[key] + # Longest prefix overlap wins + candidates = [ + (k, v) for k, v in facility_lower.items() + if key.startswith(k) or k.startswith(key) + ] + return max(candidates, key=lambda t: len(t[0]))[1] if candidates else None + + +def _is_file_link(href: str) -> bool: + low = href.lower() + return ( + any(low.endswith(ext) for ext in (".pdf", ".xlsx", ".xls", ".doc", ".docx")) + or any(kw in low for kw in ("download", "getfile", "getdoc", "document", "attachment")) + ) + + +def _abs_url(base_url: str, href: str) -> str: + """Return an absolute URL, joining with base_url if href is relative.""" + if href.startswith(("http://", "https://")): + return href + return urljoin(base_url, href) + + +# --------------------------------------------------------------------------- +# Step 5 — Download a file (pure HTTP) +# --------------------------------------------------------------------------- + +def download_file(client: httpx.Client, url: str, dest_folder: Path) -> Path | None: + """Download one file from url into dest_folder. Returns saved Path or None.""" + try: + resp = client.get(url, follow_redirects=True) + resp.raise_for_status() + filename = _filename_from_response(resp, url) + dest_folder.mkdir(parents=True, exist_ok=True) + dest = _unique_path(dest_folder, filename) + dest.write_bytes(resp.content) + log_step("Downloaded", url=url[-60:], file=dest.name, bytes=len(resp.content)) + return dest + except Exception as exc: + _log.warning("Download failed (%s): %s", url[-60:], exc) + return None + + +def _filename_from_response(resp: httpx.Response, fallback_url: str) -> str: + cd = resp.headers.get("content-disposition", "") + m = re.search(r'filename\*?=["\']?(?:UTF-8\'\')?([^"\';\r\n]+)', cd, re.IGNORECASE) + if m: + return m.group(1).strip().strip('"\'') + path = urlparse(fallback_url).path + name = path.split("/")[-1] + return name or "file.pdf" + + +def _unique_path(folder: Path, filename: str) -> Path: + candidate = folder / filename + if not candidate.exists(): + return candidate + stem, suffix = Path(filename).stem, Path(filename).suffix + for i in range(1, 9999): + candidate = folder / f"{stem}_{i}{suffix}" + if not candidate.exists(): + return candidate + return folder / filename + + +# --------------------------------------------------------------------------- +# Workable copy (identical logic to planner_download._build_workable_files) +# --------------------------------------------------------------------------- + +def _sanitize_folder_name(name: str) -> str: + return re.sub(r'[<>:"/\\|?*]', "_", name.strip()) or "unknown_facility" + + +def _unique_workable_path(folder: Path, base: str, ext: str) -> Path: + candidate = folder / f"{base}{ext}" + if not candidate.exists(): + return candidate + for i in range(1, 9999): + candidate = folder / f"{base}_{i}{ext}" + if not candidate.exists(): + return candidate + return folder / f"{base}{ext}" + + +def build_workable_files( + *, + facility_name: str, + downloaded: list[Path], + output_root: Path, + today_iso: str, +) -> tuple[list[Path], Path]: + """Copy originals to Workable/ (DD-MM-YYYY_FacilityName[_N].pdf) and mirror to Workable_Medicare/.""" + ddmmyyyy = datetime.strptime(today_iso, "%Y-%m-%d").strftime("%d-%m-%Y") + fac_safe = _sanitize_folder_name(facility_name) + workable_root = output_root / today_iso / "Workable" + medicare_root = output_root / today_iso / "Workable_Medicare" + workable_paths: list[Path] = [] + + if not downloaded: + return workable_paths, workable_root + + workable_root.mkdir(parents=True, exist_ok=True) + medicare_root.mkdir(parents=True, exist_ok=True) + + for i, src in enumerate(downloaded, start=1): + ext = src.suffix or ".pdf" + base = ( + f"{ddmmyyyy}_{fac_safe}" + if len(downloaded) == 1 + else f"{ddmmyyyy}_{fac_safe}_{i}" + ) + dst = _unique_workable_path(workable_root, base, ext) + shutil.copy2(src, dst) + workable_paths.append(dst) + + med_dst = medicare_root / dst.name + if med_dst.exists(): + med_dst = _unique_workable_path(medicare_root, dst.stem, ext) + shutil.copy2(dst, med_dst) + + _log.info("Workable (%d files) -> %s | Medicare -> %s", + len(workable_paths), workable_root, medicare_root) + return workable_paths, workable_root + + +# --------------------------------------------------------------------------- +# Summary Excel (same columns as the Planner bot) +# --------------------------------------------------------------------------- + +def write_summary( + results: dict[str, dict], + failures: list[tuple[str, str]], + summary_path: Path, + run_timestamp: str, +) -> None: + try: + import pandas as pd + except ImportError: + _log.warning("pandas not available — skipping summary Excel") + return + + rows = [] + failure_names = {f for f, _ in failures} + for facility, payload in results.items(): + files = len(payload.get("downloaded", [])) + rows.append({ + "Run Timestamp": run_timestamp, + "Facility Name": facility, + "Files Downloaded": files, + "Status": "Success" if files else "No files", + "Status Changed": "N/A (ECW)", + }) + for facility, err in failures: + rows.append({ + "Run Timestamp": run_timestamp, + "Facility Name": facility, + "Files Downloaded": 0, + "Status": "FAILED", + "Status Changed": "N/A", + }) + + summary_path.parent.mkdir(parents=True, exist_ok=True) + import pandas as pd + pd.DataFrame(rows).to_excel(summary_path, index=False, sheet_name="Download") + _log.info("Summary -> %s", summary_path) + + +# --------------------------------------------------------------------------- +# Main workflow +# --------------------------------------------------------------------------- + +def run_ecw_workflow( + *, + facility_limit: int = 0, + facility_name: str | None = None, + run_date: str | None = None, + resume: bool = False, +) -> None: + run_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + today_iso = run_date or TODAY_ISO_DATE + log_path = setup_logging(PROJECT_ROOT) + + _log.info("=" * 66) + _log.info(" MISF ECW Download — Start") + _log.info(" Run date : %s", today_iso) + _log.info(" Log : %s", log_path) + _log.info("=" * 66) + log_step("ECW run started", date=today_iso) + + ecw_cfg = _read_ecw_config() + if not ecw_cfg["ecw_url"]: + raise ValueError( + "ECW URL not configured. " + "Add 'ECW URL' to Input/input.xlsx Sheet1." + ) + + output_root = PROJECT_ROOT / "Output" + original_root = output_root / today_iso / "Original" + checkpoint_path = output_root / today_iso / "checkpoint_ecw.json" + summary_path = ( + output_root / today_iso / "Summary" + / f"Download_Summary_ECW_{today_iso}.xlsx" + ) + + # ── Facility list ───────────────────────────────────────────────────── + facilities: list[str] = read_facility_list() + if facility_name: + facilities = [f for f in facilities if f.lower() == facility_name.lower()] + if not facilities: + _log.error("Facility '%s' not found in list.", facility_name) + return + if facility_limit > 0: + facilities = facilities[:facility_limit] + + # ── Resume / checkpoint ─────────────────────────────────────────────── + start_index = 0 + completed: set[str] = set() + if resume: + ckpt = load_checkpoint(checkpoint_path) + start_index = int(ckpt.get("last_completed_index", 0) or 0) + completed = {str(x) for x in (ckpt.get("completed_facilities", []) or [])} + _log.info("Resuming from index %d; %d already completed", start_index, len(completed)) + else: + clear_checkpoint(checkpoint_path) + + # ── Step 1 — Login via Playwright (UI only) ─────────────────────────── + cookies = login_and_harvest_cookies(ecw_cfg) + + # ── Step 2 — Build httpx client (all remaining requests are HTTP) ───── + client = build_http_client(ecw_cfg, cookies) + + # ── Step 3 — Fetch document list ────────────────────────────────────── + log_step("Fetching ECW document list", date=today_iso) + html = fetch_document_list(client, today_iso) + + # ── Step 4 — Parse HTML for per-facility download links ─────────────── + facility_files = parse_facility_files(html, facilities) + total_found = sum(len(v) for v in facility_files.values()) + log_step("Files parsed from document list", total=total_found) + + results: dict[str, dict] = {} + failures: list[tuple[str, str]] = [] + total_dl = 0 + + for idx, fac in enumerate(facilities, start=1): + if idx <= start_index: + continue + if fac in completed: + _log.info("Skipping already completed: %s", fac) + continue + + log_step("Facility", index=f"{idx}/{len(facilities)}", facility=fac) + urls = facility_files.get(fac, []) + + if not urls: + _log.info("%s: no files found in ECW document list", fac) + results[fac] = {"downloaded": [], "workable": []} + completed.add(fac) + save_checkpoint(checkpoint_path, last_completed_index=idx, + facility_name=fac, completed_facilities=sorted(completed)) + continue + + dest_folder = original_root / _sanitize_folder_name(fac) + downloaded: list[Path] = [] + try: + for url in urls: + path = download_file(client, url, dest_folder) + if path: + downloaded.append(path) + + workable_paths, workable_root = build_workable_files( + facility_name=fac, + downloaded=downloaded, + output_root=output_root, + today_iso=today_iso, + ) + total_dl += len(downloaded) + results[fac] = {"downloaded": downloaded, "workable": workable_paths} + _log.info("%s: %d downloaded, %d workable", fac, len(downloaded), len(workable_paths)) + completed.add(fac) + save_checkpoint(checkpoint_path, last_completed_index=idx, + facility_name=fac, completed_facilities=sorted(completed)) + + except Exception as exc: + _log.error("%s: %s", fac, exc) + failures.append((fac, str(exc))) + results[fac] = {"downloaded": [], "workable": []} + save_checkpoint(checkpoint_path, last_completed_index=idx, + facility_name=fac, completed_facilities=sorted(completed)) + + write_summary(results, failures, summary_path, run_timestamp) + clear_checkpoint(checkpoint_path) + + _log.info("=" * 66) + _log.info(" ECW DOWNLOAD COMPLETE — %d file(s), %d failure(s)", total_dl, len(failures)) + _log.info("=" * 66) + log_step("ECW run complete", files=total_dl, failures=len(failures)) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description="MISF ECW face-sheet downloader") + p.add_argument("--limit", type=int, default=0, metavar="N", + help="Process only first N facilities") + p.add_argument("--facility", default="", metavar="NAME", + help="Process a single facility by exact name") + p.add_argument("--date", default="", metavar="YYYY-MM-DD", + help="Override run date (default: today)") + p.add_argument("--resume", action="store_true", + help="Resume from last checkpoint") + return p + + +def main() -> int: + args = _build_parser().parse_args() + try: + run_ecw_workflow( + facility_limit=max(0, args.limit), + facility_name=args.facility.strip() or None, + run_date=args.date.strip() or None, + resume=args.resume, + ) + return 0 + except KeyboardInterrupt: + print("\nCancelled.") + return 130 + except Exception: + import traceback + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/onedrive_planner_http.py b/onedrive_planner_http.py new file mode 100644 index 0000000..9f293bf --- /dev/null +++ b/onedrive_planner_http.py @@ -0,0 +1,1088 @@ +""" +MISF VOB — OneDrive / Planner HTTP download (API-first, no Selenium). + +Architecture: + 1. Playwright opens Microsoft login, captures the Bearer token from the first + authenticated request to graph.microsoft.com, then closes immediately. + 2. All subsequent work uses httpx against the Microsoft Graph API: + - Discover plans via user's group memberships + - List Not-Started tasks per facility plan + - Resolve SharePoint file references via Graph shares endpoint + - Stream-download each PDF + 3. Identical output structure to the Selenium bot: + Output//Original// + Output//Workable/ (DD-MM-YYYY_FacilityName[_N].pdf) + Output//Workable_Medicare/ + Output//Summary/Download_Summary_HTTP_.xlsx + Output//checkpoint_http.json + +Usage: + python onedrive_planner_http.py + python onedrive_planner_http.py --limit 3 + python onedrive_planner_http.py --facility "Arbor Hills" + python onedrive_planner_http.py --resume +""" +from __future__ import annotations + +import argparse +import base64 +import logging +import shutil +import sys +import time +import traceback +from datetime import datetime +from pathlib import Path +from urllib.parse import unquote, urlparse + +import httpx +import pandas as pd +from playwright.sync_api import sync_playwright + +ROOT = Path(__file__).resolve().parent +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from checkpoint import clear_checkpoint, load_checkpoint, save_checkpoint # noqa: E402 +from config import ( # noqa: E402 + PROJECT_ROOT, + TODAY_ISO_DATE, + AppConfig, + credentials_setup_message, + missing_onedrive_credentials, + read_app_config, + read_facility_list, +) +from onedrive_planner import _sanitize_folder_name # noqa: E402 +from totp_utils import validate_totp_secret # noqa: E402 + +_log = logging.getLogger(__name__) + +GRAPH_BASE = "https://graph.microsoft.com/v1.0" + + +def _find_chromium_exe() -> str | None: + """ + Locate the Playwright-downloaded Chromium executable without relying on + Playwright's internal path detection, which fails on some Windows setups. + Searches ms-playwright in LOCALAPPDATA and PLAYWRIGHT_BROWSERS_PATH. + """ + import os + search_roots = [] + if os.environ.get("PLAYWRIGHT_BROWSERS_PATH"): + search_roots.append(Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"])) + local_app = os.environ.get("LOCALAPPDATA", "") + if local_app: + search_roots.append(Path(local_app) / "ms-playwright") + search_roots.append(Path.home() / "AppData" / "Local" / "ms-playwright") + + for base in search_roots: + if not base.is_dir(): + continue + # Prefer highest version number + for d in sorted(base.glob("chromium-*"), reverse=True): + for rel in ("chrome-win64/chrome.exe", "chrome-win/chrome.exe"): + exe = d / rel + if exe.is_file(): + _log.debug("Found Chromium at: %s", exe) + return str(exe) + return None + +# --------------------------------------------------------------------------- +# First-run browser install +# --------------------------------------------------------------------------- + +def _ensure_playwright_browsers() -> None: + """Auto-install Playwright Chromium browser on first run on a new machine. + + PyInstaller bundles node.exe + package/cli.js inside _internal/playwright/driver/ + but does NOT include playwright.cmd (a batch file). So we invoke node.exe directly. + """ + if _find_chromium_exe(): + return # Already installed + + import subprocess + + print( + "\n[First-time setup] Playwright Chromium browser not found.\n" + "Downloading now (~150 MB). This only happens once on this machine.\n" + ) + + # Locate the driver directory. + # Frozen (PyInstaller onedir): _MEIPASS/playwright/driver/ + # Non-frozen (dev): next to the playwright package itself. + driver_dir: Path | None = None + if getattr(sys, "frozen", False): + driver_dir = Path(sys._MEIPASS) / "playwright" / "driver" + else: + try: + import playwright as _pw + driver_dir = Path(_pw.__file__).parent / "driver" + except Exception: + pass + + cmd: list[str] | None = None + if driver_dir and driver_dir.is_dir(): + node_exe = driver_dir / "node.exe" + cli_js = driver_dir / "package" / "cli.js" + if node_exe.is_file() and cli_js.is_file(): + # PyInstaller bundle: run node.exe directly (no playwright.cmd present) + cmd = [str(node_exe), str(cli_js), "install", "chromium"] + else: + # Dev / non-frozen fallback: try playwright.cmd / playwright shell script + for name in ("playwright.cmd", "playwright"): + p = driver_dir / name + if p.is_file(): + cmd = [str(p), "install", "chromium"] + break + + if cmd is None: + raise RuntimeError( + "Could not locate the Playwright driver inside the bundle.\n" + "Expected: _internal/playwright/driver/node.exe\n" + "Please reinstall the application." + ) + + try: + result = subprocess.run(cmd, timeout=600) + if result.returncode != 0: + raise RuntimeError(f"playwright install chromium exited {result.returncode}") + print("[Setup complete] Browser ready. Continuing...\n") + except Exception as exc: + raise RuntimeError( + f"Could not auto-install Playwright browser: {exc}\n" + "Run manually: playwright install chromium" + ) from exc + + +# --------------------------------------------------------------------------- +# Logging setup +# --------------------------------------------------------------------------- + +def _setup_logging() -> Path: + log_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / "planner_http.log" + + root = logging.getLogger() + root.setLevel(logging.DEBUG) + + # Guard: don't add duplicate handlers if called more than once in the same process + if root.handlers: + return log_path + + fmt = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s — %(message)s") + + fh = logging.FileHandler(log_path, encoding="utf-8") + fh.setLevel(logging.DEBUG) + fh.setFormatter(fmt) + root.addHandler(fh) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + ch.setFormatter(fmt) + root.addHandler(ch) + + return log_path + + +# --------------------------------------------------------------------------- +# Auth — Playwright login, Bearer token capture +# --------------------------------------------------------------------------- + +class _TokenStore: + """Collects the latest Bearer token from intercepted Playwright requests. + + Only accepts tokens from Planner/group-scoped Graph endpoints to avoid + capturing narrow User.Read tokens that fire during the auth flow (e.g. + /me, /me/photo) before the Planner-scoped tokens arrive. + """ + + # Endpoints whose tokens are guaranteed to carry Planner/Group read scope + _PLANNER_PATHS = ( + "/planner/", + "/groups/", + "/me/memberOf", + "/me/transitiveMemberOf", + ) + + def __init__(self) -> None: + self.token: str | None = None + self._at: float = 0.0 + + def on_request(self, request) -> None: + url = request.url + if "graph.microsoft.com" not in url: + return + if not any(p in url for p in self._PLANNER_PATHS): + return # ignore narrow auth-flow tokens (e.g. /me profile) + auth = request.headers.get("authorization", "") + if auth.startswith("Bearer "): + candidate = auth[7:] + if candidate != self.token: + self.token = candidate + self._at = time.monotonic() + _log.debug("Planner-scoped token captured from: %.80s", url) + + @property + def age_s(self) -> float: + return time.monotonic() - self._at if self.token else float("inf") + + +def _playwright_login_and_capture_token( + cfg: AppConfig, + *, + headless: bool = True, + timeout_s: float = 120, +) -> tuple[str, dict[str, str]]: + """ + Open Playwright, complete Microsoft login, wait until the Planner app + fires its first Graph API call, capture the Bearer token, then close. + Returns (bearer_token, cookies_dict). + """ + import pyotp # available via existing totp_utils dependency + + store = _TokenStore() + + proxy_cfg: dict | None = None + if cfg.proxy_ip and cfg.proxy_port: + proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}" + proxy_cfg = {"server": proxy_url} + if cfg.proxy_username and cfg.proxy_password: + proxy_cfg["username"] = cfg.proxy_username + proxy_cfg["password"] = str(cfg.proxy_password) + + chromium_exe = _find_chromium_exe() + if chromium_exe: + _log.info("Using Chromium at: %s", chromium_exe) + else: + _log.warning("Chromium not found via dynamic search — letting Playwright auto-detect") + + with sync_playwright() as pw: + launch_kwargs: dict = { + "headless": headless, + "proxy": proxy_cfg, + } + if chromium_exe: + # Bypass Playwright's broken path detection on this machine + launch_kwargs["executable_path"] = chromium_exe + browser = pw.chromium.launch(**launch_kwargs) + ctx = browser.new_context( + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ) + ) + page = ctx.new_page() + page.on("request", store.on_request) + + planner_url = cfg.planner_url or "https://planner.cloud.microsoft/" + _log.info("Navigating to Planner for login (%s)", planner_url[:80]) + page.goto(planner_url, timeout=60_000) + + # Username + page.wait_for_selector("input[type='email'], input[name='loginfmt']", timeout=30_000) + page.fill("input[type='email'], input[name='loginfmt']", cfg.onedrive_username) + page.click("input[type='submit'], button[type='submit']") + + # Password + page.wait_for_selector("input[type='password'], input[name='passwd']", timeout=20_000) + page.fill("input[type='password'], input[name='passwd']", cfg.onedrive_password) + page.click("input[type='submit'], button[type='submit']") + + # TOTP (optional) + try: + page.wait_for_selector( + "input[name='otc'], input[autocomplete='one-time-code']", + timeout=10_000, + ) + otp = pyotp.TOTP(cfg.microsoft_totp_secret).now() + page.fill("input[name='otc'], input[autocomplete='one-time-code']", otp) + page.click("input[type='submit'], button[type='submit']") + _log.info("TOTP submitted") + except Exception: + _log.debug("No TOTP prompt detected") + + # "Stay signed in?" — click No + try: + page.wait_for_selector("input[id='idBtn_Back']", timeout=8_000) + page.click("input[id='idBtn_Back']") + except Exception: + pass + + # Wait until Planner fires a Graph API call so we capture the token + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline and not store.token: + try: + page.wait_for_selector( + "[data-testid*='planner'], [class*='planner'], [role='main']", + timeout=4_000, + ) + except Exception: + pass + if not store.token: + time.sleep(1) + + if not store.token: + raise RuntimeError( + "Could not capture Bearer token — Microsoft login may have failed " + "or Planner did not fire any Graph API requests within " + f"{timeout_s}s." + ) + + cookies = {c["name"]: c["value"] for c in ctx.cookies()} + browser.close() + + _log.info( + "Login complete — token captured (age %.1fs), %d cookies harvested", + store.age_s, len(cookies), + ) + return store.token, cookies + + +# --------------------------------------------------------------------------- +# httpx client +# --------------------------------------------------------------------------- + +def _build_client(cfg: AppConfig, token: str) -> httpx.Client: + proxy_url: str | None = None + if cfg.proxy_ip and cfg.proxy_port: + proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}" + if cfg.proxy_username and cfg.proxy_password: + proxy_url = ( + f"http://{cfg.proxy_username}:{cfg.proxy_password}" + f"@{cfg.proxy_ip}:{cfg.proxy_port}" + ) + + return httpx.Client( + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + }, + timeout=60.0, + follow_redirects=True, + proxy=proxy_url, # httpx >= 0.28: singular `proxy=`, not `proxies=` + ) + + +def _refresh_token(client: httpx.Client, cfg: AppConfig) -> None: + """Re-login via Playwright and update the client's Authorization header in place.""" + _log.warning("Token expired (401) — re-harvesting via Playwright...") + new_token, _ = _playwright_login_and_capture_token(cfg, headless=True) + client.headers["Authorization"] = f"Bearer {new_token}" + _log.info("Token refreshed.") + + +# --------------------------------------------------------------------------- +# Graph API helpers +# --------------------------------------------------------------------------- + +def _get(client: httpx.Client, path: str, cfg: AppConfig, **params) -> dict: + """GET a Graph API path, refreshing token once on 401.""" + url = f"{GRAPH_BASE}/{path.lstrip('/')}" + for attempt in range(2): + r = client.get(url, params=params or None) + if r.status_code == 401 and attempt == 0: + _refresh_token(client, cfg) + continue + r.raise_for_status() + return r.json() + raise RuntimeError(f"Unreachable: {url}") # pragma: no cover + + +def _get_paged(client: httpx.Client, path: str, cfg: AppConfig, **params) -> list[dict]: + """Follow @odata.nextLink pagination until exhausted.""" + results: list[dict] = [] + url = f"{GRAPH_BASE}/{path.lstrip('/')}" + first = True + while url: + for attempt in range(2): + if first: + r = client.get(url, params=params or None) + else: + r = client.get(url) + if r.status_code == 401 and attempt == 0: + _refresh_token(client, cfg) + continue + r.raise_for_status() + break + data = r.json() + results.extend(data.get("value", [])) + url = data.get("@odata.nextLink") + first = False + return results + + +def _update_task_status( + client: httpx.Client, + task_id: str, + etag: str, + percent_complete: int, + cfg: AppConfig, +) -> bool: + """PATCH a Planner task's percentComplete (0=Not Started, 50=In Progress, 100=Complete). + + Planner requires the If-Match header with the task's current ETag for + optimistic concurrency — without it the API returns 428 Precondition Required. + Returns True on success, False on any error (warning is logged but run continues). + """ + url = f"{GRAPH_BASE}/planner/tasks/{task_id}" + req_headers = { + "If-Match": etag, + "Content-Type": "application/json", + } + body = {"percentComplete": percent_complete} + try: + for attempt in range(2): + r = client.patch(url, json=body, headers=req_headers) + if r.status_code == 401 and attempt == 0: + _refresh_token(client, cfg) + continue + r.raise_for_status() + return True + except httpx.HTTPStatusError as exc: + _log.warning( + " Status update failed for task %s: HTTP %s — %s", + task_id, exc.response.status_code, exc.response.text[:200], + ) + except Exception as exc: + _log.warning(" Status update failed for task %s: %s", task_id, exc) + return False + + +# --------------------------------------------------------------------------- +# Plan discovery via group memberships +# --------------------------------------------------------------------------- + +def _discover_all_plans(client: httpx.Client, cfg: AppConfig) -> list[dict]: + """ + Return every Planner plan visible to the user through their Microsoft 365 group + memberships. Each dict has at minimum: id, title, _group_name. + """ + _log.info("Fetching user's group memberships...") + groups = _get_paged( + client, + "me/memberOf/microsoft.graph.group", + cfg, + **{"$select": "id,displayName", "$top": "999"}, + ) + _log.info("%d groups found", len(groups)) + + all_plans: list[dict] = [] + for group in groups: + gid = group.get("id", "") + if not gid: + continue + try: + plans = _get_paged( + client, + f"groups/{gid}/planner/plans", + cfg, + **{"$select": "id,title"}, + ) + for p in plans: + p["_group_id"] = gid + p["_group_name"] = group.get("displayName", "") + all_plans.extend(plans) + except httpx.HTTPStatusError as exc: + if exc.response.status_code in (403, 404): + pass # group has no Planner or access denied + else: + _log.warning("plans for group %s: HTTP %s", gid, exc.response.status_code) + time.sleep(0.1) # gentle rate-limit courtesy + + _log.info("Total plans discovered: %d", len(all_plans)) + return all_plans + + +def _match_plan(plans: list[dict], facility_name: str) -> dict | None: + """Match a facility name to a Planner plan title (exact → prefix → substring).""" + fac = facility_name.strip().lower() + + # Pass 1: exact match + for p in plans: + if (p.get("title") or "").strip().lower() == fac: + return p + + # Pass 2: one string is a prefix of the other (handles trailing words) + for p in plans: + title = (p.get("title") or "").strip().lower() + if title.startswith(fac) or fac.startswith(title): + _log.warning("Prefix plan match: '%s' -> '%s'", facility_name, p.get("title")) + return p + + # Pass 3: substring match — only for names long enough to be unambiguous + # (short names like "Hills" would falsely match "Arbor Hills") + if len(fac) >= 8: + for p in plans: + title = (p.get("title") or "").strip().lower() + if fac in title: + _log.warning("Substring plan match: '%s' -> '%s'", facility_name, p.get("title")) + return p + + return None + + +# --------------------------------------------------------------------------- +# Tasks + file references +# --------------------------------------------------------------------------- + +def _not_started_tasks(client: httpx.Client, plan_id: str, cfg: AppConfig) -> list[dict]: + tasks = _get_paged( + client, + f"planner/plans/{plan_id}/tasks", + cfg, + **{"$select": "id,title,percentComplete"}, + ) + ns = [t for t in tasks if (t.get("percentComplete") or 0) == 0] + _log.info("Plan %s: %d tasks total, %d not started", plan_id, len(tasks), len(ns)) + return ns + + +def _task_file_urls(client: httpx.Client, task_id: str, cfg: AppConfig) -> list[str]: + """Return SharePoint file URLs from a task's references (details endpoint).""" + try: + details = _get(client, f"planner/tasks/{task_id}/details", cfg) + except httpx.HTTPStatusError as exc: + _log.warning("task details %s: HTTP %s", task_id, exc.response.status_code) + return [] + + refs: dict = details.get("references", {}) or {} + urls: list[str] = [] + for encoded_key in refs: + try: + url = unquote(encoded_key) + if url.startswith("http"): + urls.append(url) + except Exception: + pass + return urls + + +# --------------------------------------------------------------------------- +# SharePoint URL resolution + download +# --------------------------------------------------------------------------- + +def _encode_share_url(url: str) -> str: + """Convert a SharePoint URL to Graph's u!{base64url} share-id format.""" + b64 = base64.urlsafe_b64encode(url.encode("utf-8")).rstrip(b"=").decode() + return f"u!{b64}" + + +def _resolve_sp_url( + client: httpx.Client, sp_url: str, cfg: AppConfig +) -> tuple[str | None, str | None]: + """ + Resolve a SharePoint URL to (pre-auth download URL, filename). + Uses the Graph shares endpoint — same call visible in DevTools. + """ + share_id = _encode_share_url(sp_url) + try: + # Do NOT use $select here: @microsoft.graph.downloadUrl is an OData + # annotation and is only present in the full (unfiltered) response. + # Requesting it via $select causes Graph to return 200 with no annotation. + item = _get(client, f"shares/{share_id}/driveItem", cfg) + dl_url = item.get("@microsoft.graph.downloadUrl") + name = item.get("name") or Path(urlparse(sp_url).path).name or "download.pdf" + return dl_url, name + except httpx.HTTPStatusError as exc: + _log.warning("shares resolve %s…: HTTP %s", sp_url[:60], exc.response.status_code) + return None, None + + +def _download_file( + download_url: str, + dest_folder: Path, + filename: str, +) -> Path | None: + """Stream-download a pre-authenticated Azure Blob SAS URL to dest_folder/filename. + + If the file already exists (from a previous interrupted run) it is kept as-is + and its path is returned — no re-download, no duplicate _2/_3 copies. + + Uses a fresh proxy-free client: the SAS URL authenticates via query-string + and must not be routed through the corporate proxy. + """ + dest_folder.mkdir(parents=True, exist_ok=True) + dest = dest_folder / filename + + # Skip re-download if file already exists and is non-empty + if dest.exists() and dest.stat().st_size > 0: + _log.info(" Already exists, skipping download: %s (%d bytes)", + dest.name, dest.stat().st_size) + return dest + + try: + # Proxy-free: Azure Blob SAS URLs authenticate via query-string, + # routing through the corporate proxy wastes bandwidth and can timeout. + with httpx.Client(timeout=120.0, follow_redirects=True) as blob_client: + with blob_client.stream("GET", download_url) as r: + r.raise_for_status() + with open(dest, "wb") as f: + for chunk in r.iter_bytes(65536): + f.write(chunk) + _log.info(" Downloaded: %s (%d bytes)", dest.name, dest.stat().st_size) + return dest + except Exception as exc: + _log.error(" Download failed for %s: %s", filename, exc) + dest.unlink(missing_ok=True) + return None + + +# --------------------------------------------------------------------------- +# Workable file copy (same naming as Selenium bot) +# --------------------------------------------------------------------------- + +def _unique_workable_path(root: Path, base: str, ext: str) -> Path: + ext = ext if ext.startswith(".") else f".{ext}" + first = root / f"{base}{ext}" + if not first.exists(): + return first + for n in range(2, 1000): + candidate = root / f"{base}_{n}{ext}" + if not candidate.exists(): + return candidate + raise RuntimeError(f"Cannot allocate workable filename for {base}{ext}") + + +def _build_workable_files( + *, + facility_name: str, + downloaded: list[Path], + output_root: Path, + today_iso: str, +) -> tuple[list[Path], Path]: + ddmmyyyy = datetime.strptime(today_iso, "%Y-%m-%d").strftime("%d-%m-%Y") + fac_safe = _sanitize_folder_name(facility_name) + workable_root = output_root / today_iso / "Workable" + medicare_root = output_root / today_iso / "Workable_Medicare" + + if not downloaded: + return [], workable_root + + workable_root.mkdir(parents=True, exist_ok=True) + medicare_root.mkdir(parents=True, exist_ok=True) + + workable_paths: list[Path] = [] + for i, src in enumerate(downloaded, start=1): + ext = src.suffix or ".pdf" + base = ( + f"{ddmmyyyy}_{fac_safe}" + if len(downloaded) == 1 + else f"{ddmmyyyy}_{fac_safe}_{i}" + ) + # Check if this workable file already exists (previous interrupted run) + dst_first = workable_root / f"{base}{ext}" + if dst_first.exists() and dst_first.stat().st_size > 0: + _log.info(" Workable already exists, keeping: %s", dst_first.name) + dst = dst_first + else: + dst = _unique_workable_path(workable_root, base, ext) + shutil.copy2(src, dst) + workable_paths.append(dst) + med_dst_first = medicare_root / f"{dst.stem}{ext}" + if med_dst_first.exists() and med_dst_first.stat().st_size > 0: + _log.info(" Medicare already exists, keeping: %s", med_dst_first.name) + else: + med_dst = _unique_workable_path(medicare_root, dst.stem, ext) + shutil.copy2(dst, med_dst) + + _log.info( + "Workable: %d file(s) -> %s | Medicare -> %s", + len(workable_paths), workable_root, medicare_root, + ) + return workable_paths, workable_root + + +# --------------------------------------------------------------------------- +# Summary Excel (same schema as Selenium bot) +# --------------------------------------------------------------------------- + +def _write_summary( + results: dict[str, dict], + failures: list[tuple[str, str, str]], + summary_path: Path, + run_timestamp: str, +) -> None: + failure_names = {f for f, _, _ in failures} + rows: list[dict] = [] + + for facility, payload in results.items(): + if facility in failure_names: + continue + file_count = len(payload.get("original") or []) + tasks_total = int(payload.get("tasks_total") or 0) + tasks_att = int(payload.get("tasks_with_attachments") or 0) + tasks_dl = int(payload.get("tasks_with_downloads") or 0) + tasks_skip = int(payload.get("tasks_skipped_no_attachment") or 0) + status_changed = int(payload.get("status_changed") or 0) + + if tasks_total <= 0: + status = "No tasks" + elif tasks_att <= 0: + status = "No attachments" + elif tasks_dl <= 0 or file_count <= 0: + status = "Download failed" + elif tasks_dl >= tasks_att: + status = "Success" + else: + status = "Partial" + + if file_count <= 0: + status_changed_label = "N/A" + elif status_changed >= tasks_dl: + status_changed_label = f"Yes — {status_changed} task(s) → In Progress" + elif status_changed > 0: + status_changed_label = f"Partial — {status_changed}/{tasks_dl} task(s) → In Progress" + else: + status_changed_label = "No — Planner PATCH failed" + + rows.append({ + "Run Timestamp": run_timestamp, + "Facility Name": facility, + "Files Downloaded": file_count, + "Tasks Not Started": tasks_total, + "Tasks With Attachments": tasks_att, + "Tasks Skipped (No File)": tasks_skip, + "Status": status, + "Status Changed": status_changed_label, + }) + + for facility, err, status in failures: + rows.append({ + "Run Timestamp": run_timestamp, + "Facility Name": facility, + "Files Downloaded": 0, + "Tasks Not Started": 0, + "Tasks With Attachments": 0, + "Tasks Skipped (No File)": 0, + "Status": status, + "Status Changed": "N/A", + }) + + summary_path.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame(rows).to_excel(summary_path, index=False, sheet_name="Download") + _log.info("Summary -> %s", summary_path) + + +# --------------------------------------------------------------------------- +# Per-facility processing +# --------------------------------------------------------------------------- + +def _process_facility( + client: httpx.Client, + cfg: AppConfig, + plans: list[dict], + facility_name: str, + dest_folder: Path, +) -> dict: + plan = _match_plan(plans, facility_name) + if plan is None: + raise ValueError(f"No Planner plan found matching '{facility_name}'") + + plan_id = plan["id"] + _log.info( + "%s -> plan '%s' (group: %s)", + facility_name, plan.get("title"), plan.get("_group_name", ""), + ) + + # ── Only Not Started tasks (percentComplete == 0) are ever processed. + # ── Within those, only tasks that have at least one file attachment are + # downloaded. Tasks that are In Progress, Completed, or have no + # attachments are silently skipped. + tasks = _not_started_tasks(client, plan_id, cfg) + tasks_total = len(tasks) + tasks_with_att = 0 + tasks_with_dl = 0 + tasks_no_att = 0 # Not Started but no file attachment → skip + status_changed = 0 + downloaded: list[Path] = [] + + _log.info( + " Processing ONLY Not Started tasks with attachments (%d Not Started found)", + tasks_total, + ) + + for task in tasks: + tid = task["id"] + title = task.get("title", "") + file_urls = _task_file_urls(client, tid, cfg) + + if not file_urls: + # Not Started but no attachment — skip entirely + _log.info(" SKIP '%s': Not Started but no attachment", title[:60]) + tasks_no_att += 1 + continue + + # Not Started WITH attachment(s) — download ALL of them + tasks_with_att += 1 + total_urls = len(file_urls) + task_dl = 0 + _log.info(" PROCESS '%s': %d attachment(s)", title[:60], total_urls) + + for att_idx, sp_url in enumerate(file_urls, start=1): + dl_url, filename = _resolve_sp_url(client, sp_url, cfg) + if not dl_url or not filename: + _log.warning( + " [%d/%d] Could not resolve attachment URL: %.80s", + att_idx, total_urls, sp_url, + ) + continue + _log.info(" [%d/%d] Downloading: %s", att_idx, total_urls, filename) + path = _download_file(dl_url, dest_folder, filename) + if path: + downloaded.append(path) + task_dl += 1 + + if task_dl == total_urls: + # Every attachment downloaded — mark In Progress + tasks_with_dl += 1 + etag = task.get("@odata.etag", "") + if etag: + ok = _update_task_status(client, tid, etag, 50, cfg) + if ok: + _log.info( + " Status -> In Progress: '%s' (%d/%d files downloaded)", + title[:60], task_dl, total_urls, + ) + status_changed += 1 + else: + _log.warning(" No ETag for task '%s' — status not updated", title[:60]) + elif task_dl > 0: + # Partial — some failed; don't mark In Progress, leave for retry + tasks_with_dl += 1 + _log.warning( + " '%s': partial download %d/%d — status NOT changed (will retry next run)", + title[:60], task_dl, total_urls, + ) + else: + _log.warning( + " '%s': 0/%d attachments downloaded — status NOT changed", + title[:60], total_urls, + ) + + _log.info( + " Summary: %d Not Started | %d with attachments (processed) | " + "%d no attachment (skipped) | %d file(s) downloaded", + tasks_total, tasks_with_att, tasks_no_att, len(downloaded), + ) + + return { + "original": downloaded, + "tasks_total": tasks_total, + "tasks_with_attachments": tasks_with_att, + "tasks_with_downloads": tasks_with_dl, + "tasks_skipped_no_attachment": tasks_no_att, + "status_changed": status_changed, + "status_via_task_details": 0, + } + + +# --------------------------------------------------------------------------- +# Main workflow +# --------------------------------------------------------------------------- + +def _filter_facilities( + facilities: list[str], + *, + facility_limit: int, + facility_name: str | None, +) -> list[str]: + if facility_name: + name = facility_name.strip() + matches = [f for f in facilities if f.lower() == name.lower()] + if not matches: + raise ValueError(f"Facility '{name}' not found in Facilities sheet.") + return matches + if facility_limit > 0: + return facilities[:facility_limit] + return facilities + + +def run_planner_http_workflow( + *, + facility_limit: int = 0, + facility_name: str | None = None, + resume: bool = False, +) -> None: + run_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log_path = _setup_logging() + _log.info("=" * 66) + _log.info(" MISF Planner HTTP Download — Start [%s]", run_timestamp) + _log.info(" Run date : %s", TODAY_ISO_DATE) + _log.info(" Log file : %s", log_path) + _log.info("=" * 66) + + cfg = read_app_config() + output_root = PROJECT_ROOT / "Output" + original_root = output_root / TODAY_ISO_DATE / "Original" + checkpoint_path = output_root / TODAY_ISO_DATE / "checkpoint_http.json" + + facilities = read_facility_list() + facilities = _filter_facilities( + facilities, + facility_limit=facility_limit, + facility_name=facility_name, + ) + + # Resume state + start_index = 0 + completed: set[str] = set() + if resume: + ckpt = load_checkpoint(checkpoint_path) + start_index = int(ckpt.get("last_completed_index", 0) or 0) + completed = {str(x) for x in (ckpt.get("completed_facilities", []) or [])} + _log.info("Resuming from index %d, %d already done", start_index, len(completed)) + else: + clear_checkpoint(checkpoint_path) + + # Validate credentials before launching the browser — fail fast with a clear message + missing = missing_onedrive_credentials( + cfg.onedrive_username, + cfg.onedrive_password, + cfg.microsoft_totp_secret, + ) + if missing: + raise ValueError(credentials_setup_message(missing)) + validate_totp_secret(cfg.microsoft_totp_secret) + + # --- Ensure browser is installed (auto-downloads on first run) --- + _ensure_playwright_browsers() + + # --- Login (Playwright, headless) --- + _log.info("Launching Playwright for Microsoft login...") + token, _cookies = _playwright_login_and_capture_token(cfg, headless=True) + client = _build_client(cfg, token) + + # --- Discover all plans once --- + _log.info("Discovering Planner plans via Graph API...") + plans = _discover_all_plans(client, cfg) + _log.info("Plan discovery done: %d plans available", len(plans)) + + results: dict[str, dict] = {} + failures: list[tuple[str, str, str]] = [] + total_files = 0 + + for idx, fac in enumerate(facilities, start=1): + if idx < start_index or fac in completed: + _log.info("Skipping %s (already processed)", fac) + continue + + _log.info("-" * 50) + _log.info("Facility %d / %d: %s", idx, len(facilities), fac) + dest = original_root / _sanitize_folder_name(fac) + + try: + result = _process_facility(client, cfg, plans, fac, dest) + downloaded = result["original"] + file_count = len(downloaded) + + if downloaded: + workable_paths, workable_root = _build_workable_files( + facility_name=fac, + downloaded=downloaded, + output_root=output_root, + today_iso=TODAY_ISO_DATE, + ) + else: + workable_paths = [] + workable_root = output_root / TODAY_ISO_DATE / "Workable" + if dest.exists() and not any(dest.iterdir()): + dest.rmdir() + + results[fac] = {**result, "workable": workable_paths} + total_files += file_count + completed.add(fac) + save_checkpoint( + checkpoint_path, + last_completed_index=idx, + facility_name=fac, + completed_facilities=sorted(completed), + ) + _log.info("%s: %d file(s) -> %s", fac, file_count, workable_root) + + except Exception as exc: + err_msg = str(exc) + _log.error("%s FAILED: %s", fac, err_msg) + status = ( + "Not appear" + if "no planner plan found" in err_msg.lower() + else "FAILED" + ) + failures.append((fac, err_msg, status)) + results[fac] = { + "original": [], "workable": [], "comment": err_msg[:500], + "tasks_total": 0, "tasks_with_attachments": 0, + "tasks_with_downloads": 0, "tasks_skipped_no_attachment": 0, + "status_changed": 0, "status_via_task_details": 0, + } + save_checkpoint( + checkpoint_path, + last_completed_index=idx, + facility_name=fac, + completed_facilities=sorted(completed), + ) + + summary_path = ( + output_root / TODAY_ISO_DATE / "Summary" + / f"Download_Summary_HTTP_{TODAY_ISO_DATE}.xlsx" + ) + _write_summary(results, failures, summary_path, run_timestamp) + clear_checkpoint(checkpoint_path) + + _log.info("=" * 66) + _log.info( + " COMPLETE — %d file(s) downloaded, %d failure(s)", + total_files, len(failures), + ) + _log.info("=" * 66) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + description="Download face-sheet PDFs from Planner via Microsoft Graph API (no Selenium)." + ) + p.add_argument( + "--limit", type=int, default=0, + help="Process only the first N facilities (0 = all).", + ) + p.add_argument( + "--facility", type=str, default="", + help="Process a single facility by exact name.", + ) + p.add_argument( + "--resume", action="store_true", + help="Resume from checkpoint.", + ) + return p + + +if __name__ == "__main__": + _frozen = getattr(sys, "frozen", False) # True when running as .exe + + args = _build_arg_parser().parse_args() + exit_code = 0 + try: + run_planner_http_workflow( + facility_limit=max(0, args.limit), + facility_name=args.facility.strip() or None, + resume=args.resume, + ) + except KeyboardInterrupt: + print("\nCancelled.") + exit_code = 130 + except Exception: + traceback.print_exc() + exit_code = 1 + + if _frozen and exit_code != 0: + # On error only: keep the window open so the user can read what went wrong. + input("\n✗ An error occurred (see above). Press Enter to close...") + + sys.exit(exit_code) diff --git a/src/chrome.py b/src/chrome.py index 7ef52d4..9403e32 100644 --- a/src/chrome.py +++ b/src/chrome.py @@ -397,10 +397,33 @@ def _is_image_mostly_black(path: Path, *, threshold: int = 18) -> bool: return False +def _load_timestamp_font(size: int = 22): + """Return a PIL ImageFont at the requested size, falling back to the built-in bitmap font.""" + try: + from PIL import ImageFont + + candidates = [ + "C:/Windows/Fonts/segoeui.ttf", + "C:/Windows/Fonts/arial.ttf", + "C:/Windows/Fonts/calibri.ttf", + "C:/Windows/Fonts/verdana.ttf", + ] + for path in candidates: + try: + return ImageFont.truetype(path, size) + except Exception: + continue + except Exception: + pass + try: + from PIL import ImageFont + return ImageFont.load_default() + except Exception: + return None + + def _capture_browser_with_timestamp_bar(driver: webdriver.Chrome, path: Path) -> bool: - """ - Fallback when desktop capture is black: maximized browser PNG plus a taskbar-style clock strip. - """ + """Browser viewport PNG with a timestamp bar showing machine time.""" try: import io @@ -409,18 +432,36 @@ def _capture_browser_with_timestamp_bar(driver: webdriver.Chrome, path: Path) -> prepare_browser_for_screenshot(driver) png = driver.get_screenshot_as_png() browser = Image.open(io.BytesIO(png)).convert("RGB") - bar_h = 48 - canvas = Image.new("RGB", (browser.width, browser.height + bar_h), (30, 30, 30)) + bar_h = 36 + canvas = Image.new("RGB", (browser.width, browser.height + bar_h), (24, 24, 24)) canvas.paste(browser, (0, 0)) draw = ImageDraw.Draw(canvas) - stamp = datetime.now().strftime("%I:%M %p %m/%d/%Y").lstrip("0") - draw.text((max(browser.width - 260, 8), browser.height + 14), stamp, fill=(255, 255, 255)) + stamp = datetime.now().strftime("%I:%M:%S %p %m/%d/%Y").lstrip("0") + font = _load_timestamp_font(22) + y = browser.height + (bar_h - 22) // 2 + if font: + draw.text((12, y), stamp, fill=(220, 220, 220), font=font) + else: + draw.text((12, y), stamp, fill=(220, 220, 220)) canvas.save(str(path), "PNG") return path.is_file() and path.stat().st_size > 0 except Exception: return False +def capture_browser_screenshot(driver: webdriver.Chrome, path: Path) -> bool: + """Capture only the Chromium browser viewport — no dependency on the host desktop.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + log_step("Browser screenshot", path=str(path)) + if _capture_browser_with_timestamp_bar(driver, path): + log_step("Browser screenshot saved", path=str(path)) + return True + driver.save_screenshot(str(path)) + log_step("Viewport screenshot saved", path=str(path)) + return path.is_file() + + def capture_fullscreen_screenshot(driver: webdriver.Chrome, path: Path) -> bool: """ Full-screen capture with system taskbar visible when an interactive desktop exists. diff --git a/src/onedrive_planner.py b/src/onedrive_planner.py index 5b5ffd0..e53a528 100644 --- a/src/onedrive_planner.py +++ b/src/onedrive_planner.py @@ -19,6 +19,7 @@ from pathlib import Path import pyotp from chrome import DOWNLOADS_DIR as _CHROME_DOWNLOADS_DIR +from chrome import capture_browser_screenshot from chrome import capture_fullscreen_screenshot from config import DEFAULT_ONEDRIVE_URL, DEFAULT_PLANNER_URL, PROJECT_ROOT, TODAY_ISO_DATE from debug_log import write_agent_session_debug as _agent_session_log @@ -367,11 +368,15 @@ def _on_planner_plan_board(driver: WebDriver) -> bool: def _plan_load_error_present(driver: WebDriver) -> bool: - """True when Planner shows 'Couldn't open your plan' (URL may still be /plan/).""" + """True when Planner shows any error overlay that blocks normal operation.""" phrases = ( "couldn't open your plan", "could not open your plan", "try again in a few minutes", + "something went wrong", + "something's not right", + "we couldn't load", + "please try again", ) try: blob = (driver.find_element(By.TAG_NAME, "body").text or "").lower() @@ -380,6 +385,33 @@ def _plan_load_error_present(driver: WebDriver) -> bool: return False +def _recover_planner_error(driver: WebDriver) -> bool: + """Detect and dismiss a Planner error overlay ('Something went wrong', etc). + + Tries clicking the 'Try again' button first; falls back to a full page refresh. + Returns True if an error was detected (regardless of whether recovery succeeded). + """ + if not _plan_load_error_present(driver): + return False + log_step("Planner error overlay detected — attempting recovery") + if _click_plan_try_again(driver): + time.sleep(4) + if not _plan_load_error_present(driver): + log_step("Planner error overlay cleared via Try again") + return True + # Fallback: full page refresh + try: + log_step("Refreshing page to clear Planner error overlay") + driver.refresh() + time.sleep(6) + driver.switch_to.default_content() + _switch_to_planner_context(driver) + log_step("Page refreshed — error overlay cleared") + except Exception as exc: + log_step("Refresh failed during error recovery", error=str(exc)[:120]) + return True + + def _plan_page_ready(driver: WebDriver) -> bool: """Plan UI is usable — not the error page.""" if _plan_load_error_present(driver): @@ -1157,6 +1189,7 @@ def open_my_plans_shared(driver: WebDriver) -> None: This matches your current UI flow and avoids search-based plan opening. """ + _recover_planner_error(driver) log_step("Navigate: My plans -> Shared") log_url(driver, "before") driver.switch_to.default_content() @@ -1374,7 +1407,26 @@ def _open_facility_from_shared_list(driver: WebDriver, facility_name: str) -> No _type_facility_filter(driver, term) if _filter_shows_no_results(driver): log_step("Filter returned no results", query=term[:60], facility=facility_name) - continue + # A Planner error overlay may be blocking the search UI — try to recover + if _recover_planner_error(driver): + log_step("Recovered from error overlay during filter search", facility=facility_name) + # Re-open My plans -> Shared after the page refreshed + _click_any(driver, _MY_PLANS_NAV_LOCATORS, timeout=WAIT_LONG) + time.sleep(2) + shared_locators = [ + (By.XPATH, "//*[self::button or self::div or self::span][normalize-space()='Shared']"), + (By.XPATH, "//*[@role='tab' and normalize-space()='Shared']"), + ] + try: + _click_any(driver, shared_locators, timeout=WAIT_LONG) + time.sleep(2) + except TimeoutException: + pass + _type_facility_filter(driver, term) + if _filter_shows_no_results(driver): + continue + else: + continue rows = _shared_list_data_rows(driver) log_step( "Filter results", @@ -1611,6 +1663,53 @@ def _ensure_board_view(driver: WebDriver) -> None: log_step("Board tab not found (continuing on current view)") +def _wait_for_board_loaded(driver: WebDriver, timeout: int = 20) -> None: + """Switch to Board tab and wait until board content is visible before screenshotting.""" + board_locators = [ + (By.XPATH, "//*[@role='tab' and normalize-space()='Board']"), + (By.XPATH, "//*[self::button or self::div][normalize-space()='Board']"), + ] + + # Click the Board tab (ignore if already on board or tab not found) + try: + tab = _first_clickable(driver, board_locators, timeout=8) + # Only click if the tab isn't already selected + selected = (tab.get_attribute("aria-selected") or "").lower() + if selected != "true": + log_step("Switch to Board view") + _js_click(driver, tab) + else: + log_step("Board tab already active") + except TimeoutException: + log_step("Board tab not found (continuing on current view)") + + # Poll until board content is visible (task cards, bucket columns, or Add task) + content_xpaths = [ + "//button[@data-testid='attachment-indicator']", + "//*[@role='checkbox']", + "//label[@role='checkbox']", + "//*[contains(@data-testid,'TaskCard') or contains(@class,'TaskCard')]", + "//*[@role='list' and ancestor::*[@role='group']]", + "//*[normalize-space()='Add task' and ancestor::*[@role='group']]", + "//*[normalize-space()='Add task']", + ] + + log_step("Waiting for board content to load") + deadline = time.time() + timeout + while time.time() < deadline: + for xpath in content_xpaths: + try: + for el in driver.find_elements(By.XPATH, xpath)[:6]: + if el.is_displayed(): + log_step("Board content visible — ready for screenshot") + return + except Exception: + continue + time.sleep(0.5) + + log_step("Board content wait timed out — proceeding with screenshot anyway") + + def _ensure_grid_view(driver: WebDriver) -> None: grid_locators = [ (By.CSS_SELECTOR, "button#gridViewPivot"), @@ -1642,13 +1741,13 @@ def _is_status_sorted_descending(header) -> bool: def _sort_descending_menu_selected(driver: WebDriver) -> bool: - """True when the column header menu shows Sort Descending as active.""" + """True when the column header menu shows Sort descending as active.""" probes = [ - "//button[@name='Sort Descending' and @aria-checked='true']", - "//button[@name='Sort Descending' and contains(@class,'is-checked')]", - "//button[@name='Sort Descending']//*[@data-icon-name='CheckMark']", + "//button[@name='Sort descending' and @aria-checked='true']", + "//button[@name='Sort descending' and contains(@class,'is-checked')]", + "//button[@name='Sort descending']//*[@data-icon-name='CheckMark']", ( - "//button[@name='Sort Descending']" + "//button[@name='Sort descending']" "/ancestor::li[contains(@class,'is-checked') or contains(@class,'is-selected')]" ), ] @@ -1692,13 +1791,18 @@ def _ensure_status_sort_descending(driver: WebDriver) -> None: return sort_locators = [ + # Exact match — actual Planner DOM uses lowercase 'd' + (By.XPATH, "//button[@name='Sort descending' and @role='menuitem']"), + (By.XPATH, "//ul[contains(@class,'ContextualMenu')]//button[normalize-space()='Sort descending']"), + (By.XPATH, "//button[@role='menuitem' and normalize-space()='Sort descending']"), + (By.XPATH, "//span[normalize-space()='Sort descending']/ancestor::button[@role='menuitem']"), + # Uppercase fallback (in case Planner localises differently) (By.XPATH, "//button[@name='Sort Descending' and @role='menuitem']"), - ( - By.XPATH, - "//ul[contains(@class,'ContextualMenu')]" - "//button[normalize-space()='Sort Descending']", - ), (By.XPATH, "//button[@role='menuitem' and normalize-space()='Sort Descending']"), + # Broad contains — catches any capitalisation + (By.XPATH, "//*[@role='menuitem' and contains(normalize-space(),'Sort descending')]"), + (By.XPATH, "//*[@role='menuitem' and contains(normalize-space(),'Sort Descending')]"), + (By.XPATH, "//li[contains(@class,'ms-ContextualMenu-item')]//button[contains(normalize-space(),'escending')]"), ] try: _click_any(driver, sort_locators, timeout=5) @@ -1712,8 +1816,30 @@ def _ensure_status_sort_descending(driver: WebDriver) -> None: except TimeoutException: log_step("Status column sort descending clicked") except TimeoutException: - log_step("Sort Descending menu item not found (continuing on current sort)") - _dismiss_open_menu(driver) + log_step("Sort descending menu item not found via XPath — trying JS click") + clicked = False + try: + clicked = driver.execute_script( + """ + var items = Array.from(document.querySelectorAll( + '[role="menuitem"], [role="menuitem"] button, button[role="menuitem"]' + )); + var target = items.find(function(el) { + var t = (el.innerText || el.textContent || '').trim().toLowerCase(); + return t === 'sort descending'; + }); + if (target) { target.click(); return true; } + return false; + """ + ) + except Exception: + pass + if clicked: + time.sleep(1) + log_step("Sort descending clicked via JS") + else: + log_step("Sort descending not found (continuing on current sort)") + _dismiss_open_menu(driver) def _grid_probes_present(driver: WebDriver) -> bool: @@ -1902,11 +2028,14 @@ def _scroll_grid_down(driver: WebDriver, pixels: int = 450) -> None: time.sleep(0.4) -def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100) -> list[str]: - """Scroll the virtualized grid and collect Not started task titles.""" +def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100) -> list[tuple[str, int]]: + """Scroll the virtualized grid and collect (title, aria-rowindex) for Not started tasks. + + Uses aria-rowindex as the unique key so duplicate task names are all captured. + """ _scroll_grid_to_top(driver) - seen: set[str] = set() - titles: list[str] = [] + seen_indices: set[int] = set() + tasks: list[tuple[str, int]] = [] stagnant = 0 for _ in range(max_scrolls): new_this_pass = 0 @@ -1914,14 +2043,15 @@ def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100) status = _grid_row_status(row) if status != "not started": continue + idx_raw = row.get_attribute("aria-rowindex") or "" + row_idx = int(idx_raw) if idx_raw.isdigit() else -1 + if row_idx < 1 or row_idx in seen_indices: + continue title = _grid_row_task_name(row).strip() if not title or _is_weak_task_title_key(title.lower()): continue - key = title.lower() - if key in seen: - continue - seen.add(key) - titles.append(title) + seen_indices.add(row_idx) + tasks.append((title, row_idx)) new_this_pass += 1 if new_this_pass == 0: stagnant += 1 @@ -1931,8 +2061,22 @@ def _scan_grid_not_started_titles(driver: WebDriver, *, max_scrolls: int = 100) stagnant = 0 _scroll_grid_down(driver) _scroll_grid_to_top(driver) - log_step("Grid not-started tasks scanned", count=len(titles)) - return titles + log_step("Grid not-started tasks scanned", count=len(tasks)) + return tasks + + +def _find_grid_row_by_index(driver: WebDriver, row_index: int, *, max_scrolls: int = 100): + """Find a grid row by its aria-rowindex — stable even when multiple rows share the same title.""" + if row_index < 1: + return None + _scroll_grid_to_top(driver) + for _ in range(max_scrolls): + for row in _grid_visible_data_rows(driver): + idx_raw = row.get_attribute("aria-rowindex") or "" + if idx_raw.isdigit() and int(idx_raw) == row_index: + return row + _scroll_grid_down(driver) + return None def _find_grid_row_by_title(driver: WebDriver, title: str, *, max_scrolls: int = 100): @@ -6047,13 +6191,13 @@ def save_board_screenshot( safe = _sanitize_folder_name(facility_name) taken_at = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") path = shots_dir / f"{safe}_{taken_at}.png" - capture_fullscreen_screenshot(driver, path) + capture_browser_screenshot(driver, path) log_step( "Board screenshot saved", facility=facility_name, tasks=task_count, path=str(path), - fullscreen="true", + fullscreen="false", view="board", ) if task_count: @@ -6114,6 +6258,7 @@ def process_facility( try: open_my_plans_shared(driver) _open_facility_from_shared_list(driver, facility_name) + _wait_for_board_loaded(driver) save_board_screenshot(driver, facility_name) _ensure_grid_view(driver) _ensure_status_sort_descending(driver) @@ -6135,8 +6280,8 @@ def process_facility( log_step("Grid not visible — waiting for grid to settle", facility=facility_name) time.sleep(6) - task_titles = _scan_grid_not_started_titles(driver) - task_count = len(task_titles) + task_entries = _scan_grid_not_started_titles(driver) + task_count = len(task_entries) log_step( "Scan tasks on grid", task_count=task_count, @@ -6147,19 +6292,21 @@ def process_facility( return FacilityRunResult() download_dir_ready = False - processed_titles: set[str] = set() + processed_indices: set[int] = set() - for task_num, grid_title in enumerate(task_titles, start=1): + for task_num, (grid_title, row_index) in enumerate(task_entries, start=1): title_key = grid_title.strip().lower() - if title_key in processed_titles: + if row_index in processed_indices: continue _close_task_panel(driver) task_title = "" try: - row = _find_grid_row_by_title(driver, grid_title) + # Locate by row index first (handles duplicate task names correctly), + # fall back to title search if the row has no aria-rowindex. + row = _find_grid_row_by_index(driver, row_index) or _find_grid_row_by_title(driver, grid_title) if row is None: log_step("Grid row not found", title=grid_title[:60], facility=facility_name) - processed_titles.add(title_key) + processed_indices.add(row_index) continue row_status = _grid_row_status(row) @@ -6170,7 +6317,7 @@ def process_facility( status=row_status or "", facility=facility_name, ) - processed_titles.add(title_key) + processed_indices.add(row_index) continue log_step( @@ -6180,7 +6327,7 @@ def process_facility( facility=facility_name, ) # Re-find row right before click — virtualized grid can shift rows. - row = _find_grid_row_by_title(driver, grid_title) + row = _find_grid_row_by_index(driver, row_index) or _find_grid_row_by_title(driver, grid_title) if row is None or not _open_grid_row_details(driver, row): log_step("Grid task open failed", title=grid_title[:60], facility=facility_name) continue @@ -6188,7 +6335,7 @@ def process_facility( # Grid Status column already filtered to Not started — go straight # to Attachments tab (no panel status gate). task_title = _clean_grid_task_title(grid_title) - processed_titles.add(task_title.strip().lower() or title_key) + processed_indices.add(row_index) log_step("Task opened from grid", title=task_title[:60]) time.sleep(PANEL_TAB_LOAD_DELAY) @@ -6280,10 +6427,7 @@ def process_facility( ) _log.warning("Task %d failed: %s", task_num, exc) _close_task_panel(driver) - if task_title: - processed_titles.add(task_title.strip().lower()) - elif title_key: - processed_titles.add(title_key) + processed_indices.add(row_index) _remove_empty_directory(dest_folder) log_step( diff --git a/src/planner_download.py b/src/planner_download.py index 946236e..c193a7b 100644 --- a/src/planner_download.py +++ b/src/planner_download.py @@ -58,6 +58,64 @@ from debug_log import write_debug as _agent_log _log = logging.getLogger(__name__) +def _is_browser_dead(exc: Exception) -> bool: + """True when Chrome itself has crashed and the WebDriver session is gone.""" + err = str(exc).lower() + return any(phrase in err for phrase in ( + "invalid session id", + "session deleted", + "browser has closed the connection", + "chrome not reachable", + "cannot connect to chrome", + "no such window", + )) + + +def _restart_browser(cfg: "AppConfig") -> "webdriver.Chrome | None": + """Kill Chrome, relaunch it, and re-login to Planner. Returns new driver on success.""" + log_step("Browser crash detected — restarting Chrome") + _log.warning("Restarting Chrome after crash...") + try: + stop_chrome_browser() + except Exception: + pass + time.sleep(4) + try: + if not start_chrome_debug( + cfg.proxy_ip, cfg.proxy_port, cfg.proxy_username, cfg.proxy_password + ): + log_step("Chrome restart failed — start_chrome_debug returned False") + return None + new_driver = attach_browser(create_download_dir=False) + if new_driver is None: + log_step("Chrome restart failed — attach_browser returned None") + return None + consolidate_browser_to_single_tab(new_driver) + if cfg.proxy_username and cfg.proxy_password: + dismiss_proxy_auth_popup( + new_driver, + cfg.proxy_username.strip(), + str(cfg.proxy_password), + wait_secs=8.0, + max_attempts=3, + ) + time.sleep(2) + login_and_open_planner( + new_driver, + cfg.onedrive_username, + cfg.onedrive_password, + cfg.microsoft_totp_secret, + planner_url=cfg.planner_url, + ) + log_step("Chrome restarted and Planner re-opened after crash") + _log.info("Chrome restarted successfully after crash.") + return new_driver + except Exception as exc: + log_step("Browser restart failed", error=str(exc)[:200]) + _log.error("Browser restart failed: %s", exc) + return None + + def _show_error_dialog(title: str, message: str) -> None: try: root = tk.Tk() @@ -624,7 +682,10 @@ def run_download_workflow( log_step("Facility failed", facility=fac, error=err_msg[:200], status=status) _log.error("%s: %s", fac, err_msg) if screenshot_on_error and driver is not None: - save_failure_screenshot(driver, fac, failure_log_dir=failure_log_dir) + try: + save_failure_screenshot(driver, fac, failure_log_dir=failure_log_dir) + except Exception: + pass failures.append((fac, err_msg, status)) results[fac] = { "original": [], @@ -644,6 +705,21 @@ def run_download_workflow( completed_facilities=sorted(completed), ) + # Chrome crashed — restart it so remaining facilities can still run + if _is_browser_dead(exc): + _log.warning( + "Browser crash detected after '%s' — restarting Chrome to continue run", + fac, + ) + new_driver = _restart_browser(cfg) + if new_driver is not None: + driver = new_driver + else: + _log.error( + "Could not restart browser after crash — aborting remaining facilities" + ) + break + summary_path = ( PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "Summary" / f"Download_Summary_{TODAY_ISO_DATE}.xlsx"