""" MISF VOB — OneDrive / Planner HTTP download (API-first, no Selenium). Architecture: 1. Playwright opens Microsoft login, captures the Bearer token from the first authenticated request to graph.microsoft.com, then closes immediately. 2. All subsequent work uses httpx against the Microsoft Graph API: - Discover plans via user's group memberships - List Not-Started tasks per facility plan, skipping only "Facesheets Needed (GG)" tasks (assigned or not, all others download) - Resolve SharePoint file references via Graph shares endpoint - Stream-download each PDF 3. Identical output structure to the Selenium bot: Output//Original// Output//Workable/ (DD-MM-YYYY_FacilityName[_N].pdf) Output//Workable_Medicare/ Output//Summary/Download_Summary_HTTP_.xlsx Output//checkpoint_http.json Usage: python onedrive_planner_http.py python onedrive_planner_http.py --limit 3 python onedrive_planner_http.py --facility "Arbor Hills" python onedrive_planner_http.py --resume """ from __future__ import annotations import argparse import base64 import logging import shutil import sys import time import traceback from datetime import datetime from pathlib import Path from urllib.parse import unquote, urlparse import httpx import pandas as pd from playwright.sync_api import sync_playwright ROOT = Path(__file__).resolve().parent SRC = ROOT / "src" if str(SRC) not in sys.path: sys.path.insert(0, str(SRC)) from checkpoint import clear_checkpoint, load_checkpoint, save_checkpoint # noqa: E402 from config import ( # noqa: E402 PROJECT_ROOT, TODAY_ISO_DATE, AppConfig, credentials_setup_message, missing_onedrive_credentials, read_app_config, read_facility_list, ) from onedrive_planner import _sanitize_folder_name # noqa: E402 from totp_utils import validate_totp_secret # noqa: E402 _log = logging.getLogger(__name__) GRAPH_BASE = "https://graph.microsoft.com/v1.0" def _find_chromium_exe() -> str | None: """ Locate the Playwright-downloaded Chromium executable without relying on Playwright's internal path detection, which fails on some Windows setups. Searches ms-playwright in LOCALAPPDATA and PLAYWRIGHT_BROWSERS_PATH. """ import os search_roots = [] if os.environ.get("PLAYWRIGHT_BROWSERS_PATH"): search_roots.append(Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"])) local_app = os.environ.get("LOCALAPPDATA", "") if local_app: search_roots.append(Path(local_app) / "ms-playwright") search_roots.append(Path.home() / "AppData" / "Local" / "ms-playwright") for base in search_roots: if not base.is_dir(): continue # Prefer highest version number for d in sorted(base.glob("chromium-*"), reverse=True): for rel in ("chrome-win64/chrome.exe", "chrome-win/chrome.exe"): exe = d / rel if exe.is_file(): _log.debug("Found Chromium at: %s", exe) return str(exe) return None # --------------------------------------------------------------------------- # First-run browser install # --------------------------------------------------------------------------- def _ensure_playwright_browsers() -> None: """Auto-install Playwright Chromium browser on first run on a new machine. PyInstaller bundles node.exe + package/cli.js inside _internal/playwright/driver/ but does NOT include playwright.cmd (a batch file). So we invoke node.exe directly. """ if _find_chromium_exe(): return # Already installed import subprocess print( "\n[First-time setup] Playwright Chromium browser not found.\n" "Downloading now (~150 MB). This only happens once on this machine.\n" ) # Locate the driver directory. # Frozen (PyInstaller onedir): _MEIPASS/playwright/driver/ # Non-frozen (dev): next to the playwright package itself. driver_dir: Path | None = None if getattr(sys, "frozen", False): driver_dir = Path(sys._MEIPASS) / "playwright" / "driver" else: try: import playwright as _pw driver_dir = Path(_pw.__file__).parent / "driver" except Exception: pass cmd: list[str] | None = None if driver_dir and driver_dir.is_dir(): node_exe = driver_dir / "node.exe" cli_js = driver_dir / "package" / "cli.js" if node_exe.is_file() and cli_js.is_file(): # PyInstaller bundle: run node.exe directly (no playwright.cmd present) cmd = [str(node_exe), str(cli_js), "install", "chromium"] else: # Dev / non-frozen fallback: try playwright.cmd / playwright shell script for name in ("playwright.cmd", "playwright"): p = driver_dir / name if p.is_file(): cmd = [str(p), "install", "chromium"] break if cmd is None: raise RuntimeError( "Could not locate the Playwright driver inside the bundle.\n" "Expected: _internal/playwright/driver/node.exe\n" "Please reinstall the application." ) try: result = subprocess.run(cmd, timeout=600) if result.returncode != 0: raise RuntimeError(f"playwright install chromium exited {result.returncode}") print("[Setup complete] Browser ready. Continuing...\n") except Exception as exc: raise RuntimeError( f"Could not auto-install Playwright browser: {exc}\n" "Run manually: playwright install chromium" ) from exc # --------------------------------------------------------------------------- # Logging setup # --------------------------------------------------------------------------- def _force_utf8_console() -> None: """Make stdout/stderr tolerate non-ASCII. Windows consoles default to cp1252, and both the log format and the error banner contain characters outside it — printing them raises UnicodeEncodeError and hides the real error from the operator. """ for stream in (sys.stdout, sys.stderr): try: stream.reconfigure(encoding="utf-8", errors="replace") except Exception: pass def _setup_logging() -> Path: _force_utf8_console() log_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs" log_dir.mkdir(parents=True, exist_ok=True) log_path = log_dir / "planner_http.log" root = logging.getLogger() root.setLevel(logging.DEBUG) # Guard: don't add duplicate handlers if called more than once in the same process if root.handlers: return log_path fmt = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s — %(message)s") fh = logging.FileHandler(log_path, encoding="utf-8") fh.setLevel(logging.DEBUG) fh.setFormatter(fmt) root.addHandler(fh) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) ch.setFormatter(fmt) root.addHandler(ch) return log_path # --------------------------------------------------------------------------- # Auth — Playwright login, Bearer token capture # --------------------------------------------------------------------------- # Microsoft's converged sign-in page keeps the email, password and OTC inputs # in the same document, so a bare wait_for_selector() for a later step returns # immediately while an earlier view is still on screen. Every fill therefore has # to be verified before it is submitted — see _fill_verified(). EMAIL_SELECTOR = "input[name='loginfmt'], #i0116, input[type='email']" PASSWORD_SELECTOR = "input[name='passwd'], #i0118, input[type='password']" OTP_SELECTOR = ( "input[name='otc'], #idTxtBx_SAOTCC_OTC, input[autocomplete='one-time-code']" ) # Text Microsoft shows when it rejects a credential outright (as opposed to the # "you left the box empty" validation message, which we retry). _FATAL_LOGIN_MESSAGES = ( "your account or password is incorrect", "we couldn't find an account", "that microsoft account doesn't exist", "your account has been locked", "sign-in was blocked", "you can't get there from here", "code you entered isn't working", "verification code is incorrect", ) class _TokenStore: """Collects Bearer tokens from intercepted Playwright requests. Two tiers. Planner/group-scoped Graph endpoints are preferred, because the narrow tokens that fire early in the auth flow (``/me``, ``/me/photo``) lack the Planner scopes we need. If no preferred token has shown up after ``fallback_after_s``, any graph.microsoft.com Bearer is accepted rather than failing the whole run. """ # Endpoints whose tokens are guaranteed to carry Planner/Group read scope _PLANNER_PATHS = ( "/planner/", "/groups/", "/me/memberOf", "/me/transitiveMemberOf", ) def __init__(self, *, fallback_after_s: float = 45.0) -> None: self.token: str | None = None self.preferred: bool = False self.graph_urls: set[str] = set() self.bearer_hosts: set[str] = set() self._at: float = 0.0 self._start = time.monotonic() self._fallback_after_s = fallback_after_s def on_request(self, request) -> None: url = request.url auth = request.headers.get("authorization", "") if not auth.startswith("Bearer "): return try: self.bearer_hosts.add(url.split("/")[2]) except IndexError: pass if "graph.microsoft.com" not in url: return self.graph_urls.add(url.split("?")[0]) candidate = auth[7:] is_preferred = any(p in url for p in self._PLANNER_PATHS) if is_preferred: if candidate != self.token or not self.preferred: self.token = candidate self.preferred = True self._at = time.monotonic() _log.debug("Planner-scoped token captured from: %.100s", url) return # Non-preferred Graph call. Only fall back to it once we have waited # long enough that a Planner-scoped call clearly isn't coming. if self.preferred: return if time.monotonic() - self._start < self._fallback_after_s: _log.debug("Ignoring non-Planner Graph token (still waiting): %.100s", url) return if candidate != self.token: self.token = candidate self._at = time.monotonic() _log.warning( "No Planner-scoped Graph call seen in %.0fs — falling back to the " "token from %.100s", self._fallback_after_s, url, ) @property def age_s(self) -> float: return time.monotonic() - self._at if self.token else float("inf") def _is_visible(page, selector: str) -> bool: try: return page.locator(selector).first.is_visible(timeout=1_000) except Exception: return False def _page_text(page, limit: int = 3000) -> str: try: return page.inner_text("body")[:limit] except Exception: return "" def _fatal_login_message(page) -> str | None: """Return Microsoft's error text when it has rejected a credential.""" text = _page_text(page).lower() for phrase in _FATAL_LOGIN_MESSAGES: if phrase in text: return phrase return None def _fill_verified(page, selector: str, value: str, *, label: str, attempts: int = 3) -> None: """Fill a login field and confirm the value actually stuck. The sign-in page pre-renders later steps, so a fill issued while the view is still switching gets wiped and the form submits empty ("Please enter your password."). Reading the value back is the only reliable guard. """ for attempt in range(1, attempts + 1): try: box = page.locator(selector).first # The field can be in the DOM but not yet interactive while the # sign-in page switches views, so wait for it to settle first. box.wait_for(state="visible", timeout=20_000) box.click(timeout=10_000) box.fill("", timeout=10_000) box.fill(value, timeout=10_000) if box.input_value(timeout=5_000) == value: return except Exception as exc: _log.debug("%s fill attempt %d raised: %s", label, attempt, exc) _log.warning( "%s field did not retain its value (attempt %d/%d) — retrying", label, attempt, attempts, ) time.sleep(1.5) raise RuntimeError( f"Could not enter the {label} on the Microsoft sign-in page — the field " "kept clearing itself. The sign-in page layout may have changed again." ) def _submit_password(page, password: str, *, attempts: int = 3) -> None: """Enter the password and confirm the page moved on to the next step.""" page.wait_for_selector(PASSWORD_SELECTOR, timeout=30_000) for attempt in range(1, attempts + 1): # Give the view switch time to finish; the box exists in the DOM while # the email view is still showing, so wait_for_selector returns at once. page.wait_for_timeout(1_500) _fill_verified(page, PASSWORD_SELECTOR, password, label="password") page.locator(PASSWORD_SELECTOR).first.press("Enter") deadline = time.monotonic() + 25 while time.monotonic() < deadline: if _is_visible(page, OTP_SELECTOR) or _login_interrupt_visible(page): _log.info("Password accepted") return if "login.microsoftonline.com" not in page.url and "login.live.com" not in page.url: _log.info("Password accepted (left the sign-in host)") return fatal = _fatal_login_message(page) if fatal: raise RuntimeError(f"Microsoft rejected the sign-in: {fatal!r}") time.sleep(1) _log.warning( "Still on the password page after attempt %d/%d — retrying", attempt, attempts ) raise RuntimeError( "Could not get past the Microsoft password page. Page said: " + " | ".join(ln.strip() for ln in _page_text(page, 600).splitlines() if ln.strip()) ) def _submit_totp(page, totp_secret: str, *, attempts: int = 2) -> bool: """Enter the authenticator code if prompted. Returns True when submitted.""" import pyotp if not _is_visible(page, OTP_SELECTOR): _log.debug("No TOTP prompt detected") return False for attempt in range(1, attempts + 1): code = pyotp.TOTP(totp_secret).now() _fill_verified(page, OTP_SELECTOR, code, label="authenticator code") page.locator(OTP_SELECTOR).first.press("Enter") _log.info("TOTP submitted (attempt %d)", attempt) deadline = time.monotonic() + 25 while time.monotonic() < deadline: if _login_interrupt_visible(page): return True if not _is_visible(page, OTP_SELECTOR): return True fatal = _fatal_login_message(page) if fatal: raise RuntimeError(f"Microsoft rejected the authenticator code: {fatal!r}") time.sleep(1) _log.warning("Code page still showing — retrying with a fresh code") time.sleep(5) raise RuntimeError("Microsoft kept asking for the authenticator code.") # Pages Microsoft inserts between a successful sign-in and the app. Each entry # is (text identifying the page, selectors that skip it, label for the log). # Order matters only in that the first matching entry wins. # # The selectors are deliberately narrow: #idBtn_Back is also the back arrow on # the password view, so these are only ever clicked when the page text confirms # which interrupt we are looking at. _LOGIN_INTERRUPTS: tuple[tuple[str, tuple[str, ...], str], ...] = ( ( "keep your account secure", ( "#iShowSkip", "a:has-text('Not now')", "button:has-text('Not now')", "a:has-text('Skip setup')", "#iCancel", ), "MFA setup prompt", ), ( "stay signed in", ( "#idBtn_Back", "input[type='submit'][value='No']", "button:has-text('No')", ), "Stay signed in?", ), ( "help us protect your account", ("#iShowSkip", "a:has-text('Skip for now')", "a:has-text('Not now')"), "account protection prompt", ), ) def _login_interrupt_visible(page) -> bool: text = _page_text(page, 1500).lower() return any(needle in text for needle, _, _ in _LOGIN_INTERRUPTS) def _skip_login_interrupt(page) -> bool: """Dismiss one post-sign-in interstitial. Returns True when one was clicked.""" text = _page_text(page, 1500).lower() for needle, selectors, label in _LOGIN_INTERRUPTS: if needle not in text: continue for selector in selectors: try: page.click(selector, timeout=2_000) _log.info("Dismissed Microsoft %s via %s", label, selector) page.wait_for_timeout(2_500) return True except Exception: continue _log.warning( "Microsoft %s is on screen but no skip control matched — page says: %s", label, " | ".join(ln.strip() for ln in _page_text(page, 400).splitlines() if ln.strip()), ) return False return False def _token_from_msal_storage(page) -> str | None: """Fallback: read a Graph access token out of the SPA's MSAL localStorage. Used when the Planner app completes login but never fires a Graph request we can intercept, so a future change to the SPA's network calls does not break the whole download again. """ try: entries = page.evaluate( """() => { const out = []; for (let i = 0; i < localStorage.length; i++) { const k = localStorage.key(i); if (!k || !k.toLowerCase().includes('accesstoken')) continue; try { out.push(JSON.parse(localStorage.getItem(k))); } catch (e) {} } return out; }""" ) except Exception as exc: _log.debug("Could not read localStorage: %s", exc) return None return _pick_graph_token(entries or []) def _pick_graph_token(entries: list[dict]) -> str | None: """Choose the MSAL cache entry that carries Graph Planner/Group scopes.""" wanted = ("tasks.read", "tasks.readwrite", "group.read.all", "groupmember.read.all") best: tuple[int, str] | None = None for entry in entries: if not isinstance(entry, dict): continue secret = entry.get("secret") or "" if not secret: continue target = str(entry.get("target") or "").lower() if "graph.microsoft.com" not in target and not any(w in target for w in wanted): continue score = sum(1 for w in wanted if w in target) if best is None or score > best[0]: best = (score, secret) if best: _log.warning("Using Graph token from the MSAL localStorage cache (no Graph request seen)") return best[1] return None def _dump_login_failure(page, store: "_TokenStore") -> Path | None: """Save a screenshot, the page HTML and what we saw, next to the run log.""" try: out_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs" out_dir.mkdir(parents=True, exist_ok=True) stamp = datetime.now().strftime("%H%M%S") stem = out_dir / f"login_failure_{stamp}" try: page.screenshot(path=str(stem.with_suffix(".png")), full_page=True) except Exception: pass try: stem.with_suffix(".html").write_text(page.content(), encoding="utf-8", errors="replace") except Exception: pass graph_lines = [f" {u}" for u in sorted(store.graph_urls)] or [" none"] summary = [ f"url : {page.url}", f"text : {_page_text(page, 2000)}", "", f"hosts that received a Bearer: {sorted(store.bearer_hosts) or 'none'}", "graph.microsoft.com URLs seen:", *graph_lines, ] stem.with_suffix(".txt").write_text("\n".join(summary), encoding="utf-8", errors="replace") _log.error("Login failure diagnostics written to %s.*", stem) return stem except Exception as exc: _log.debug("Could not write login diagnostics: %s", exc) return None def _playwright_login_and_capture_token( cfg: AppConfig, *, headless: bool = True, timeout_s: float = 180, ) -> tuple[str, dict[str, str]]: """ Open Playwright, complete Microsoft login, wait until the Planner app fires its first Graph API call, capture the Bearer token, then close. Returns (bearer_token, cookies_dict). """ store = _TokenStore() proxy_cfg: dict | None = None if cfg.proxy_ip and cfg.proxy_port: proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}" proxy_cfg = {"server": proxy_url} if cfg.proxy_username and cfg.proxy_password: proxy_cfg["username"] = cfg.proxy_username proxy_cfg["password"] = str(cfg.proxy_password) chromium_exe = _find_chromium_exe() if chromium_exe: _log.info("Using Chromium at: %s", chromium_exe) else: _log.warning("Chromium not found via dynamic search — letting Playwright auto-detect") with sync_playwright() as pw: launch_kwargs: dict = { "headless": headless, "proxy": proxy_cfg, } if chromium_exe: # Bypass Playwright's broken path detection on this machine launch_kwargs["executable_path"] = chromium_exe browser = pw.chromium.launch(**launch_kwargs) ctx = browser.new_context( user_agent=( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/124.0.0.0 Safari/537.36" ) ) # Context-level so popups and secondary pages are observed too. ctx.on("request", store.on_request) page = ctx.new_page() try: planner_url = cfg.planner_url or "https://planner.cloud.microsoft/" _log.info("Navigating to Planner for login (%s)", planner_url[:80]) page.goto(planner_url, timeout=60_000) # Username page.wait_for_selector(EMAIL_SELECTOR, timeout=30_000) _fill_verified(page, EMAIL_SELECTOR, cfg.onedrive_username, label="username") page.locator(EMAIL_SELECTOR).first.press("Enter") # Password _submit_password(page, cfg.onedrive_password) # TOTP _submit_totp(page, cfg.microsoft_totp_secret) # "Let's keep your account secure", "Stay signed in?", etc. for _ in range(4): if not _skip_login_interrupt(page): break # Wait until Planner fires a Graph API call so we capture the token deadline = time.monotonic() + timeout_s while time.monotonic() < deadline and not store.token: try: page.wait_for_selector( "[data-testid*='planner'], [class*='planner'], [role='main']", timeout=4_000, ) except Exception: pass if not store.token: _skip_login_interrupt(page) time.sleep(1) if not store.token: store.token = _token_from_msal_storage(page) if not store.token: _dump_login_failure(page, store) raise RuntimeError( "Could not capture Bearer token — Microsoft login may have failed " f"or Planner did not fire any Graph API requests within {timeout_s}s. " f"Last URL: {page.url[:120]}. " f"Graph URLs seen: {sorted(store.graph_urls) or 'none'}. " "See the login_failure_*.png/.html/.txt files next to this log." ) cookies = {c["name"]: c["value"] for c in ctx.cookies()} finally: browser.close() _log.info( "Login complete — token captured (%s, age %.1fs), %d cookies harvested", "Planner-scoped" if store.preferred else "fallback scope", store.age_s, len(cookies), ) return store.token, cookies # --------------------------------------------------------------------------- # httpx client # --------------------------------------------------------------------------- def _build_client(cfg: AppConfig, token: str) -> httpx.Client: proxy_url: str | None = None if cfg.proxy_ip and cfg.proxy_port: proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}" if cfg.proxy_username and cfg.proxy_password: proxy_url = ( f"http://{cfg.proxy_username}:{cfg.proxy_password}" f"@{cfg.proxy_ip}:{cfg.proxy_port}" ) return httpx.Client( headers={ "Authorization": f"Bearer {token}", "Accept": "application/json", }, timeout=60.0, follow_redirects=True, proxy=proxy_url, # httpx >= 0.28: singular `proxy=`, not `proxies=` ) def _login_with_retry( cfg: AppConfig, *, headless: bool = True, attempts: int = 2 ) -> tuple[str, dict[str, str]]: """Run the Playwright login, retrying once on a transient sign-in hiccup.""" last: Exception | None = None for attempt in range(1, attempts + 1): try: return _playwright_login_and_capture_token(cfg, headless=headless) except Exception as exc: last = exc _log.error("Login attempt %d/%d failed: %s", attempt, attempts, exc) if attempt < attempts: time.sleep(10) assert last is not None raise last def _refresh_token(client: httpx.Client, cfg: AppConfig) -> None: """Re-login via Playwright and update the client's Authorization header in place.""" _log.warning("Token expired (401) — re-harvesting via Playwright...") new_token, _ = _login_with_retry(cfg, headless=True) client.headers["Authorization"] = f"Bearer {new_token}" _log.info("Token refreshed.") # --------------------------------------------------------------------------- # Graph API helpers # --------------------------------------------------------------------------- def _get(client: httpx.Client, path: str, cfg: AppConfig, **params) -> dict: """GET a Graph API path, refreshing token once on 401.""" url = f"{GRAPH_BASE}/{path.lstrip('/')}" for attempt in range(2): r = client.get(url, params=params or None) if r.status_code == 401 and attempt == 0: _refresh_token(client, cfg) continue r.raise_for_status() return r.json() raise RuntimeError(f"Unreachable: {url}") # pragma: no cover def _get_paged(client: httpx.Client, path: str, cfg: AppConfig, **params) -> list[dict]: """Follow @odata.nextLink pagination until exhausted.""" results: list[dict] = [] url = f"{GRAPH_BASE}/{path.lstrip('/')}" first = True while url: for attempt in range(2): if first: r = client.get(url, params=params or None) else: r = client.get(url) if r.status_code == 401 and attempt == 0: _refresh_token(client, cfg) continue r.raise_for_status() break data = r.json() results.extend(data.get("value", [])) url = data.get("@odata.nextLink") first = False return results def _update_task_status( client: httpx.Client, task_id: str, etag: str, percent_complete: int, cfg: AppConfig, ) -> bool: """PATCH a Planner task's percentComplete (0=Not Started, 50=In Progress, 100=Complete). Planner requires the If-Match header with the task's current ETag for optimistic concurrency — without it the API returns 428 Precondition Required. Returns True on success, False on any error (warning is logged but run continues). """ url = f"{GRAPH_BASE}/planner/tasks/{task_id}" req_headers = { "If-Match": etag, "Content-Type": "application/json", } body = {"percentComplete": percent_complete} try: for attempt in range(2): r = client.patch(url, json=body, headers=req_headers) if r.status_code == 401 and attempt == 0: _refresh_token(client, cfg) continue r.raise_for_status() return True except httpx.HTTPStatusError as exc: _log.warning( " Status update failed for task %s: HTTP %s — %s", task_id, exc.response.status_code, exc.response.text[:200], ) except Exception as exc: _log.warning(" Status update failed for task %s: %s", task_id, exc) return False # --------------------------------------------------------------------------- # Plan discovery via group memberships # --------------------------------------------------------------------------- def _discover_all_plans(client: httpx.Client, cfg: AppConfig) -> list[dict]: """ Return every Planner plan visible to the user through their Microsoft 365 group memberships. Each dict has at minimum: id, title, _group_name. """ _log.info("Fetching user's group memberships...") groups = _get_paged( client, "me/memberOf/microsoft.graph.group", cfg, **{"$select": "id,displayName", "$top": "999"}, ) _log.info("%d groups found", len(groups)) all_plans: list[dict] = [] for group in groups: gid = group.get("id", "") if not gid: continue try: plans = _get_paged( client, f"groups/{gid}/planner/plans", cfg, **{"$select": "id,title"}, ) for p in plans: p["_group_id"] = gid p["_group_name"] = group.get("displayName", "") all_plans.extend(plans) except httpx.HTTPStatusError as exc: if exc.response.status_code in (403, 404): pass # group has no Planner or access denied else: _log.warning("plans for group %s: HTTP %s", gid, exc.response.status_code) time.sleep(0.1) # gentle rate-limit courtesy _log.info("Total plans discovered: %d", len(all_plans)) return all_plans def _match_plan(plans: list[dict], facility_name: str) -> dict | None: """Match a facility name to a Planner plan title (exact → prefix → substring).""" fac = facility_name.strip().lower() # Pass 1: exact match for p in plans: if (p.get("title") or "").strip().lower() == fac: return p # Pass 2: one string is a prefix of the other (handles trailing words) for p in plans: title = (p.get("title") or "").strip().lower() if title.startswith(fac) or fac.startswith(title): _log.warning("Prefix plan match: '%s' -> '%s'", facility_name, p.get("title")) return p # Pass 3: substring match — only for names long enough to be unambiguous # (short names like "Hills" would falsely match "Arbor Hills") if len(fac) >= 8: for p in plans: title = (p.get("title") or "").strip().lower() if fac in title: _log.warning("Substring plan match: '%s' -> '%s'", facility_name, p.get("title")) return p return None # --------------------------------------------------------------------------- # Tasks + file references # --------------------------------------------------------------------------- def _not_started_tasks(client: httpx.Client, plan_id: str, cfg: AppConfig) -> list[dict]: tasks = _get_paged( client, f"planner/plans/{plan_id}/tasks", cfg, **{"$select": "id,title,percentComplete,assignments"}, ) ns = [t for t in tasks if (t.get("percentComplete") or 0) == 0] _log.info("Plan %s: %d tasks total, %d not started", plan_id, len(tasks), len(ns)) return ns def _task_assignee_ids(task: dict) -> list[str]: """User IDs a task is assigned to (the Planner grid's 'Assigned to' column).""" return [uid for uid in (task.get("assignments") or {}) if uid] def _is_gg_task(task: dict) -> bool: """'Facesheets Needed (GG)' tasks are the only ones the bot must not download.""" # ponytail: title match only; also check task details notes if GG ever appears there return "facesheets needed (gg)" in " ".join((task.get("title") or "").lower().split()) def _resolve_user_names( client: httpx.Client, user_ids: list[str], cfg: AppConfig, cache: dict[str, str], ) -> list[str]: """Display names for assignee IDs, cached across the run. A lookup failure is never fatal — the ID is shown instead so the summary still records who the task was assigned to. """ names: list[str] = [] for uid in user_ids: if uid not in cache: try: user = _get(client, f"users/{uid}", cfg, **{"$select": "displayName"}) cache[uid] = user.get("displayName") or uid except Exception as exc: _log.debug("user lookup %s failed: %s", uid, exc) cache[uid] = uid names.append(cache[uid]) return names def _task_file_urls(client: httpx.Client, task_id: str, cfg: AppConfig) -> list[str]: """Return SharePoint file URLs from a task's references (details endpoint).""" try: details = _get(client, f"planner/tasks/{task_id}/details", cfg) except httpx.HTTPStatusError as exc: _log.warning("task details %s: HTTP %s", task_id, exc.response.status_code) return [] refs: dict = details.get("references", {}) or {} urls: list[str] = [] for encoded_key in refs: try: url = unquote(encoded_key) if url.startswith("http"): urls.append(url) except Exception: pass return urls # --------------------------------------------------------------------------- # SharePoint URL resolution + download # --------------------------------------------------------------------------- def _encode_share_url(url: str) -> str: """Convert a SharePoint URL to Graph's u!{base64url} share-id format.""" b64 = base64.urlsafe_b64encode(url.encode("utf-8")).rstrip(b"=").decode() return f"u!{b64}" def _resolve_sp_url( client: httpx.Client, sp_url: str, cfg: AppConfig ) -> tuple[str | None, str | None]: """ Resolve a SharePoint URL to (pre-auth download URL, filename). Uses the Graph shares endpoint — same call visible in DevTools. """ share_id = _encode_share_url(sp_url) try: # Do NOT use $select here: @microsoft.graph.downloadUrl is an OData # annotation and is only present in the full (unfiltered) response. # Requesting it via $select causes Graph to return 200 with no annotation. item = _get(client, f"shares/{share_id}/driveItem", cfg) dl_url = item.get("@microsoft.graph.downloadUrl") name = item.get("name") or Path(urlparse(sp_url).path).name or "download.pdf" return dl_url, name except httpx.HTTPStatusError as exc: _log.warning("shares resolve %s…: HTTP %s", sp_url[:60], exc.response.status_code) return None, None def _download_file( download_url: str, dest_folder: Path, filename: str, ) -> Path | None: """Stream-download a pre-authenticated Azure Blob SAS URL to dest_folder/filename. If the file already exists (from a previous interrupted run) it is kept as-is and its path is returned — no re-download, no duplicate _2/_3 copies. Uses a fresh proxy-free client: the SAS URL authenticates via query-string and must not be routed through the corporate proxy. """ dest_folder.mkdir(parents=True, exist_ok=True) dest = dest_folder / filename # Skip re-download if file already exists and is non-empty if dest.exists() and dest.stat().st_size > 0: _log.info(" Already exists, skipping download: %s (%d bytes)", dest.name, dest.stat().st_size) return dest try: # Proxy-free: Azure Blob SAS URLs authenticate via query-string, # routing through the corporate proxy wastes bandwidth and can timeout. with httpx.Client(timeout=120.0, follow_redirects=True) as blob_client: with blob_client.stream("GET", download_url) as r: r.raise_for_status() with open(dest, "wb") as f: for chunk in r.iter_bytes(65536): f.write(chunk) _log.info(" Downloaded: %s (%d bytes)", dest.name, dest.stat().st_size) return dest except Exception as exc: _log.error(" Download failed for %s: %s", filename, exc) dest.unlink(missing_ok=True) return None # --------------------------------------------------------------------------- # Workable file copy (same naming as Selenium bot) # --------------------------------------------------------------------------- def _unique_workable_path(root: Path, base: str, ext: str) -> Path: ext = ext if ext.startswith(".") else f".{ext}" first = root / f"{base}{ext}" if not first.exists(): return first for n in range(2, 1000): candidate = root / f"{base}_{n}{ext}" if not candidate.exists(): return candidate raise RuntimeError(f"Cannot allocate workable filename for {base}{ext}") def _build_workable_files( *, facility_name: str, downloaded: list[Path], output_root: Path, today_iso: str, ) -> tuple[list[Path], Path]: mmddyyyy = datetime.strptime(today_iso, "%Y-%m-%d").strftime("%m-%d-%Y") fac_safe = _sanitize_folder_name(facility_name) workable_root = output_root / today_iso / "Workable" medicare_root = output_root / today_iso / "Workable_Medicare" if not downloaded: return [], workable_root workable_root.mkdir(parents=True, exist_ok=True) medicare_root.mkdir(parents=True, exist_ok=True) workable_paths: list[Path] = [] for i, src in enumerate(downloaded, start=1): ext = src.suffix or ".pdf" base = ( f"{mmddyyyy}_{fac_safe}" if len(downloaded) == 1 else f"{mmddyyyy}_{fac_safe}_{i}" ) # Check if this workable file already exists (previous interrupted run) dst_first = workable_root / f"{base}{ext}" if dst_first.exists() and dst_first.stat().st_size > 0: _log.info(" Workable already exists, keeping: %s", dst_first.name) dst = dst_first else: dst = _unique_workable_path(workable_root, base, ext) shutil.copy2(src, dst) workable_paths.append(dst) med_dst_first = medicare_root / f"{dst.stem}{ext}" if med_dst_first.exists() and med_dst_first.stat().st_size > 0: _log.info(" Medicare already exists, keeping: %s", med_dst_first.name) else: med_dst = _unique_workable_path(medicare_root, dst.stem, ext) shutil.copy2(dst, med_dst) _log.info( "Workable: %d file(s) -> %s | Medicare -> %s", len(workable_paths), workable_root, medicare_root, ) return workable_paths, workable_root # --------------------------------------------------------------------------- # Summary Excel (same schema as Selenium bot) # --------------------------------------------------------------------------- def _write_summary( results: dict[str, dict], failures: list[tuple[str, str, str]], summary_path: Path, run_timestamp: str, ) -> None: failure_names = {f for f, _, _ in failures} rows: list[dict] = [] task_rows: list[dict] = [] for facility, payload in results.items(): if facility in failure_names: continue file_count = len(payload.get("original") or []) tasks_total = int(payload.get("tasks_total") or 0) tasks_att = int(payload.get("tasks_with_attachments") or 0) tasks_dl = int(payload.get("tasks_with_downloads") or 0) tasks_skip = int(payload.get("tasks_skipped_no_attachment") or 0) tasks_gg = int(payload.get("tasks_skipped_gg") or 0) status_changed = int(payload.get("status_changed") or 0) task_rows.extend(payload.get("task_rows") or []) if tasks_total <= 0: status = "No tasks" elif tasks_att <= 0 and tasks_gg > 0: status = "All GG — skipped" elif tasks_att <= 0: status = "No attachments" elif tasks_dl <= 0 or file_count <= 0: status = "Download failed" elif tasks_dl >= tasks_att: status = "Success" else: status = "Partial" if file_count <= 0: status_changed_label = "N/A" elif status_changed >= tasks_dl: status_changed_label = f"Yes — {status_changed} task(s) → In Progress" elif status_changed > 0: status_changed_label = f"Partial — {status_changed}/{tasks_dl} task(s) → In Progress" else: status_changed_label = "No — Planner PATCH failed" rows.append({ "Run Timestamp": run_timestamp, "Facility Name": facility, "Files Downloaded": file_count, "Tasks Not Started": tasks_total, "Tasks With Attachments": tasks_att, "Tasks Skipped (GG)": tasks_gg, "Tasks Skipped (No File)": tasks_skip, "Status": status, "Status Changed": status_changed_label, }) for facility, err, status in failures: rows.append({ "Run Timestamp": run_timestamp, "Facility Name": facility, "Files Downloaded": 0, "Tasks Not Started": 0, "Tasks With Attachments": 0, "Tasks Skipped (GG)": 0, "Tasks Skipped (No File)": 0, "Status": status, "Status Changed": "N/A", }) task_columns = [ "Facility Name", "Task Name", "Assigned To", "Attachments", "Downloaded", "Reason", ] tasks_df = pd.DataFrame(task_rows, columns=task_columns) summary_path.parent.mkdir(parents=True, exist_ok=True) with pd.ExcelWriter(summary_path, engine="openpyxl") as writer: pd.DataFrame(rows).to_excel(writer, index=False, sheet_name="Download") tasks_df.to_excel(writer, index=False, sheet_name="Tasks") _log.info( "Summary -> %s (%d facility row(s), %d task row(s))", summary_path, len(rows), len(tasks_df), ) # --------------------------------------------------------------------------- # Per-facility processing # --------------------------------------------------------------------------- def _process_facility( client: httpx.Client, cfg: AppConfig, plans: list[dict], facility_name: str, dest_folder: Path, ) -> dict: plan = _match_plan(plans, facility_name) if plan is None: raise ValueError(f"No Planner plan found matching '{facility_name}'") plan_id = plan["id"] _log.info( "%s -> plan '%s' (group: %s)", facility_name, plan.get("title"), plan.get("_group_name", ""), ) # ── Only Not Started tasks (percentComplete == 0) are ever processed. # ── Within those, only tasks that have at least one file attachment are # downloaded. Tasks that are In Progress, Completed, or have no # attachments are silently skipped. tasks = _not_started_tasks(client, plan_id, cfg) tasks_total = len(tasks) tasks_with_att = 0 tasks_with_dl = 0 tasks_no_att = 0 # Not Started but no file attachment → skip tasks_gg = 0 # "Facesheets Needed (GG)" → skip status_changed = 0 downloaded: list[Path] = [] task_rows: list[dict] = [] name_cache: dict[str, str] = {} _log.info( " Processing Not Started tasks with attachments, assigned or not " "(%d Not Started found)", tasks_total, ) for task in tasks: tid = task["id"] title = task.get("title", "") assignee_ids = _task_assignee_ids(task) assigned_to = ( ", ".join(_resolve_user_names(client, assignee_ids, cfg, name_cache)) if assignee_ids else "" ) # GG tasks are never downloaded and their status is never touched. # Checked before the attachment lookup so they cost no extra Graph call. if _is_gg_task(task): _log.info(" SKIP '%s': Facesheets Needed (GG)", title[:60]) tasks_gg += 1 task_rows.append({ "Facility Name": facility_name, "Task Name": title, "Assigned To": assigned_to, "Attachments": "", "Downloaded": "No", "Reason": "Skipped — Facesheets Needed (GG)", }) continue file_urls = _task_file_urls(client, tid, cfg) if not file_urls: # Not Started but no attachment — skip entirely _log.info(" SKIP '%s': Not Started but no attachment", title[:60]) tasks_no_att += 1 task_rows.append({ "Facility Name": facility_name, "Task Name": title, "Assigned To": assigned_to, "Attachments": 0, "Downloaded": "No", "Reason": "Skipped — no attachment", }) continue # Not Started, WITH attachment(s) — download ALL of them tasks_with_att += 1 total_urls = len(file_urls) task_dl = 0 _log.info(" PROCESS '%s': %d attachment(s)", title[:60], total_urls) for att_idx, sp_url in enumerate(file_urls, start=1): dl_url, filename = _resolve_sp_url(client, sp_url, cfg) if not dl_url or not filename: _log.warning( " [%d/%d] Could not resolve attachment URL: %.80s", att_idx, total_urls, sp_url, ) continue _log.info(" [%d/%d] Downloading: %s", att_idx, total_urls, filename) path = _download_file(dl_url, dest_folder, filename) if path: downloaded.append(path) task_dl += 1 if task_dl == total_urls: reason = f"Downloaded {task_dl}/{total_urls} attachment(s)" elif task_dl > 0: reason = f"Partial — {task_dl}/{total_urls} attachment(s) downloaded" else: reason = f"Download failed — 0/{total_urls} attachment(s)" task_rows.append({ "Facility Name": facility_name, "Task Name": title, "Assigned To": assigned_to, "Attachments": total_urls, "Downloaded": "Yes" if task_dl else "No", "Reason": reason, }) if task_dl == total_urls: # Every attachment downloaded — mark In Progress tasks_with_dl += 1 etag = task.get("@odata.etag", "") if etag: ok = _update_task_status(client, tid, etag, 50, cfg) if ok: _log.info( " Status -> In Progress: '%s' (%d/%d files downloaded)", title[:60], task_dl, total_urls, ) status_changed += 1 else: _log.warning(" No ETag for task '%s' — status not updated", title[:60]) elif task_dl > 0: # Partial — some failed; don't mark In Progress, leave for retry tasks_with_dl += 1 _log.warning( " '%s': partial download %d/%d — status NOT changed (will retry next run)", title[:60], task_dl, total_urls, ) else: _log.warning( " '%s': 0/%d attachments downloaded — status NOT changed", title[:60], total_urls, ) _log.info( " Summary: %d Not Started | %d GG (skipped) | " "%d with attachments (processed) | %d no attachment (skipped) | " "%d file(s) downloaded", tasks_total, tasks_gg, tasks_with_att, tasks_no_att, len(downloaded), ) return { "original": downloaded, "tasks_total": tasks_total, "tasks_with_attachments": tasks_with_att, "tasks_with_downloads": tasks_with_dl, "tasks_skipped_no_attachment": tasks_no_att, "tasks_skipped_gg": tasks_gg, "task_rows": task_rows, "status_changed": status_changed, "status_via_task_details": 0, } # --------------------------------------------------------------------------- # Main workflow # --------------------------------------------------------------------------- def _filter_facilities( facilities: list[str], *, facility_limit: int, facility_name: str | None, ) -> list[str]: if facility_name: name = facility_name.strip() matches = [f for f in facilities if f.lower() == name.lower()] if not matches: raise ValueError(f"Facility '{name}' not found in Facilities sheet.") return matches if facility_limit > 0: return facilities[:facility_limit] return facilities def run_planner_http_workflow( *, facility_limit: int = 0, facility_name: str | None = None, resume: bool = False, ) -> None: run_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") log_path = _setup_logging() _log.info("=" * 66) _log.info(" MISF Planner HTTP Download — Start [%s]", run_timestamp) _log.info(" Run date : %s", TODAY_ISO_DATE) _log.info(" Log file : %s", log_path) _log.info("=" * 66) cfg = read_app_config() output_root = PROJECT_ROOT / "Output" original_root = output_root / TODAY_ISO_DATE / "Original" checkpoint_path = output_root / TODAY_ISO_DATE / "checkpoint_http.json" facilities = read_facility_list() facilities = _filter_facilities( facilities, facility_limit=facility_limit, facility_name=facility_name, ) # Resume state start_index = 0 completed: set[str] = set() if resume: ckpt = load_checkpoint(checkpoint_path) start_index = int(ckpt.get("last_completed_index", 0) or 0) completed = {str(x) for x in (ckpt.get("completed_facilities", []) or [])} _log.info("Resuming from index %d, %d already done", start_index, len(completed)) else: clear_checkpoint(checkpoint_path) # Validate credentials before launching the browser — fail fast with a clear message missing = missing_onedrive_credentials( cfg.onedrive_username, cfg.onedrive_password, cfg.microsoft_totp_secret, ) if missing: raise ValueError(credentials_setup_message(missing)) validate_totp_secret(cfg.microsoft_totp_secret) # --- Ensure browser is installed (auto-downloads on first run) --- _ensure_playwright_browsers() # --- Login (Playwright, headless) --- _log.info("Launching Playwright for Microsoft login...") token, _cookies = _login_with_retry(cfg, headless=True) client = _build_client(cfg, token) # --- Discover all plans once --- _log.info("Discovering Planner plans via Graph API...") plans = _discover_all_plans(client, cfg) _log.info("Plan discovery done: %d plans available", len(plans)) results: dict[str, dict] = {} failures: list[tuple[str, str, str]] = [] total_files = 0 for idx, fac in enumerate(facilities, start=1): if idx < start_index or fac in completed: _log.info("Skipping %s (already processed)", fac) continue _log.info("-" * 50) _log.info("Facility %d / %d: %s", idx, len(facilities), fac) dest = original_root / _sanitize_folder_name(fac) try: result = _process_facility(client, cfg, plans, fac, dest) downloaded = result["original"] file_count = len(downloaded) if downloaded: workable_paths, workable_root = _build_workable_files( facility_name=fac, downloaded=downloaded, output_root=output_root, today_iso=TODAY_ISO_DATE, ) else: workable_paths = [] workable_root = output_root / TODAY_ISO_DATE / "Workable" if dest.exists() and not any(dest.iterdir()): dest.rmdir() results[fac] = {**result, "workable": workable_paths} total_files += file_count completed.add(fac) save_checkpoint( checkpoint_path, last_completed_index=idx, facility_name=fac, completed_facilities=sorted(completed), ) _log.info("%s: %d file(s) -> %s", fac, file_count, workable_root) except Exception as exc: err_msg = str(exc) _log.error("%s FAILED: %s", fac, err_msg) status = ( "Not appear" if "no planner plan found" in err_msg.lower() else "FAILED" ) failures.append((fac, err_msg, status)) results[fac] = { "original": [], "workable": [], "comment": err_msg[:500], "tasks_total": 0, "tasks_with_attachments": 0, "tasks_with_downloads": 0, "tasks_skipped_no_attachment": 0, "tasks_skipped_gg": 0, "task_rows": [], "status_changed": 0, "status_via_task_details": 0, } save_checkpoint( checkpoint_path, last_completed_index=idx, facility_name=fac, completed_facilities=sorted(completed), ) summary_path = ( output_root / TODAY_ISO_DATE / "Summary" / f"Download_Summary_HTTP_{TODAY_ISO_DATE}.xlsx" ) _write_summary(results, failures, summary_path, run_timestamp) clear_checkpoint(checkpoint_path) _log.info("=" * 66) _log.info( " COMPLETE — %d file(s) downloaded, %d failure(s)", total_files, len(failures), ) _log.info("=" * 66) # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- def _build_arg_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( description="Download face-sheet PDFs from Planner via Microsoft Graph API (no Selenium)." ) p.add_argument( "--limit", type=int, default=0, help="Process only the first N facilities (0 = all).", ) p.add_argument( "--facility", type=str, default="", help="Process a single facility by exact name.", ) p.add_argument( "--resume", action="store_true", help="Resume from checkpoint.", ) return p if __name__ == "__main__": _frozen = getattr(sys, "frozen", False) # True when running as .exe _force_utf8_console() args = _build_arg_parser().parse_args() exit_code = 0 try: run_planner_http_workflow( facility_limit=max(0, args.limit), facility_name=args.facility.strip() or None, resume=args.resume, ) except KeyboardInterrupt: print("\nCancelled.") exit_code = 130 except Exception: traceback.print_exc() exit_code = 1 # On error only, and only for a real console: keep the window open so the # user can read what went wrong. Scheduled runs have no stdin and would # otherwise die with an EOFError that buries the actual failure. if _frozen and exit_code != 0 and sys.stdin is not None and sys.stdin.isatty(): input("\n[X] An error occurred (see above). Press Enter to close...") sys.exit(exit_code)