Lubna Fathima
291507c2f7
- Add ecw_download.py and onedrive_planner_http.py for ECW and HTTP-based Planner access - Add build_exe.bat for packaging - Restart Chrome automatically on crash mid-run in planner_download.py - Detect and recover from Planner "something went wrong" error overlays - Improve board-load waiting, sort-descending menu matching, and screenshot timestamp rendering in chrome.py/onedrive_planner.py
1089 lines
39 KiB
Python
1089 lines
39 KiB
Python
"""
|
|
MISF VOB — OneDrive / Planner HTTP download (API-first, no Selenium).
|
|
|
|
Architecture:
|
|
1. Playwright opens Microsoft login, captures the Bearer token from the first
|
|
authenticated request to graph.microsoft.com, then closes immediately.
|
|
2. All subsequent work uses httpx against the Microsoft Graph API:
|
|
- Discover plans via user's group memberships
|
|
- List Not-Started tasks per facility plan
|
|
- Resolve SharePoint file references via Graph shares endpoint
|
|
- Stream-download each PDF
|
|
3. Identical output structure to the Selenium bot:
|
|
Output/<date>/Original/<Facility>/
|
|
Output/<date>/Workable/ (DD-MM-YYYY_FacilityName[_N].pdf)
|
|
Output/<date>/Workable_Medicare/
|
|
Output/<date>/Summary/Download_Summary_HTTP_<date>.xlsx
|
|
Output/<date>/checkpoint_http.json
|
|
|
|
Usage:
|
|
python onedrive_planner_http.py
|
|
python onedrive_planner_http.py --limit 3
|
|
python onedrive_planner_http.py --facility "Arbor Hills"
|
|
python onedrive_planner_http.py --resume
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import logging
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from urllib.parse import unquote, urlparse
|
|
|
|
import httpx
|
|
import pandas as pd
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
SRC = ROOT / "src"
|
|
if str(SRC) not in sys.path:
|
|
sys.path.insert(0, str(SRC))
|
|
|
|
from checkpoint import clear_checkpoint, load_checkpoint, save_checkpoint # noqa: E402
|
|
from config import ( # noqa: E402
|
|
PROJECT_ROOT,
|
|
TODAY_ISO_DATE,
|
|
AppConfig,
|
|
credentials_setup_message,
|
|
missing_onedrive_credentials,
|
|
read_app_config,
|
|
read_facility_list,
|
|
)
|
|
from onedrive_planner import _sanitize_folder_name # noqa: E402
|
|
from totp_utils import validate_totp_secret # noqa: E402
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
GRAPH_BASE = "https://graph.microsoft.com/v1.0"
|
|
|
|
|
|
def _find_chromium_exe() -> str | None:
|
|
"""
|
|
Locate the Playwright-downloaded Chromium executable without relying on
|
|
Playwright's internal path detection, which fails on some Windows setups.
|
|
Searches ms-playwright in LOCALAPPDATA and PLAYWRIGHT_BROWSERS_PATH.
|
|
"""
|
|
import os
|
|
search_roots = []
|
|
if os.environ.get("PLAYWRIGHT_BROWSERS_PATH"):
|
|
search_roots.append(Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"]))
|
|
local_app = os.environ.get("LOCALAPPDATA", "")
|
|
if local_app:
|
|
search_roots.append(Path(local_app) / "ms-playwright")
|
|
search_roots.append(Path.home() / "AppData" / "Local" / "ms-playwright")
|
|
|
|
for base in search_roots:
|
|
if not base.is_dir():
|
|
continue
|
|
# Prefer highest version number
|
|
for d in sorted(base.glob("chromium-*"), reverse=True):
|
|
for rel in ("chrome-win64/chrome.exe", "chrome-win/chrome.exe"):
|
|
exe = d / rel
|
|
if exe.is_file():
|
|
_log.debug("Found Chromium at: %s", exe)
|
|
return str(exe)
|
|
return None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# First-run browser install
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _ensure_playwright_browsers() -> None:
|
|
"""Auto-install Playwright Chromium browser on first run on a new machine.
|
|
|
|
PyInstaller bundles node.exe + package/cli.js inside _internal/playwright/driver/
|
|
but does NOT include playwright.cmd (a batch file). So we invoke node.exe directly.
|
|
"""
|
|
if _find_chromium_exe():
|
|
return # Already installed
|
|
|
|
import subprocess
|
|
|
|
print(
|
|
"\n[First-time setup] Playwright Chromium browser not found.\n"
|
|
"Downloading now (~150 MB). This only happens once on this machine.\n"
|
|
)
|
|
|
|
# Locate the driver directory.
|
|
# Frozen (PyInstaller onedir): _MEIPASS/playwright/driver/
|
|
# Non-frozen (dev): next to the playwright package itself.
|
|
driver_dir: Path | None = None
|
|
if getattr(sys, "frozen", False):
|
|
driver_dir = Path(sys._MEIPASS) / "playwright" / "driver"
|
|
else:
|
|
try:
|
|
import playwright as _pw
|
|
driver_dir = Path(_pw.__file__).parent / "driver"
|
|
except Exception:
|
|
pass
|
|
|
|
cmd: list[str] | None = None
|
|
if driver_dir and driver_dir.is_dir():
|
|
node_exe = driver_dir / "node.exe"
|
|
cli_js = driver_dir / "package" / "cli.js"
|
|
if node_exe.is_file() and cli_js.is_file():
|
|
# PyInstaller bundle: run node.exe directly (no playwright.cmd present)
|
|
cmd = [str(node_exe), str(cli_js), "install", "chromium"]
|
|
else:
|
|
# Dev / non-frozen fallback: try playwright.cmd / playwright shell script
|
|
for name in ("playwright.cmd", "playwright"):
|
|
p = driver_dir / name
|
|
if p.is_file():
|
|
cmd = [str(p), "install", "chromium"]
|
|
break
|
|
|
|
if cmd is None:
|
|
raise RuntimeError(
|
|
"Could not locate the Playwright driver inside the bundle.\n"
|
|
"Expected: _internal/playwright/driver/node.exe\n"
|
|
"Please reinstall the application."
|
|
)
|
|
|
|
try:
|
|
result = subprocess.run(cmd, timeout=600)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"playwright install chromium exited {result.returncode}")
|
|
print("[Setup complete] Browser ready. Continuing...\n")
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"Could not auto-install Playwright browser: {exc}\n"
|
|
"Run manually: playwright install chromium"
|
|
) from exc
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Logging setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _setup_logging() -> Path:
|
|
log_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs"
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
log_path = log_dir / "planner_http.log"
|
|
|
|
root = logging.getLogger()
|
|
root.setLevel(logging.DEBUG)
|
|
|
|
# Guard: don't add duplicate handlers if called more than once in the same process
|
|
if root.handlers:
|
|
return log_path
|
|
|
|
fmt = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s — %(message)s")
|
|
|
|
fh = logging.FileHandler(log_path, encoding="utf-8")
|
|
fh.setLevel(logging.DEBUG)
|
|
fh.setFormatter(fmt)
|
|
root.addHandler(fh)
|
|
|
|
ch = logging.StreamHandler(sys.stdout)
|
|
ch.setLevel(logging.INFO)
|
|
ch.setFormatter(fmt)
|
|
root.addHandler(ch)
|
|
|
|
return log_path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Auth — Playwright login, Bearer token capture
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _TokenStore:
|
|
"""Collects the latest Bearer token from intercepted Playwright requests.
|
|
|
|
Only accepts tokens from Planner/group-scoped Graph endpoints to avoid
|
|
capturing narrow User.Read tokens that fire during the auth flow (e.g.
|
|
/me, /me/photo) before the Planner-scoped tokens arrive.
|
|
"""
|
|
|
|
# Endpoints whose tokens are guaranteed to carry Planner/Group read scope
|
|
_PLANNER_PATHS = (
|
|
"/planner/",
|
|
"/groups/",
|
|
"/me/memberOf",
|
|
"/me/transitiveMemberOf",
|
|
)
|
|
|
|
def __init__(self) -> None:
|
|
self.token: str | None = None
|
|
self._at: float = 0.0
|
|
|
|
def on_request(self, request) -> None:
|
|
url = request.url
|
|
if "graph.microsoft.com" not in url:
|
|
return
|
|
if not any(p in url for p in self._PLANNER_PATHS):
|
|
return # ignore narrow auth-flow tokens (e.g. /me profile)
|
|
auth = request.headers.get("authorization", "")
|
|
if auth.startswith("Bearer "):
|
|
candidate = auth[7:]
|
|
if candidate != self.token:
|
|
self.token = candidate
|
|
self._at = time.monotonic()
|
|
_log.debug("Planner-scoped token captured from: %.80s", url)
|
|
|
|
@property
|
|
def age_s(self) -> float:
|
|
return time.monotonic() - self._at if self.token else float("inf")
|
|
|
|
|
|
def _playwright_login_and_capture_token(
|
|
cfg: AppConfig,
|
|
*,
|
|
headless: bool = True,
|
|
timeout_s: float = 120,
|
|
) -> tuple[str, dict[str, str]]:
|
|
"""
|
|
Open Playwright, complete Microsoft login, wait until the Planner app
|
|
fires its first Graph API call, capture the Bearer token, then close.
|
|
Returns (bearer_token, cookies_dict).
|
|
"""
|
|
import pyotp # available via existing totp_utils dependency
|
|
|
|
store = _TokenStore()
|
|
|
|
proxy_cfg: dict | None = None
|
|
if cfg.proxy_ip and cfg.proxy_port:
|
|
proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}"
|
|
proxy_cfg = {"server": proxy_url}
|
|
if cfg.proxy_username and cfg.proxy_password:
|
|
proxy_cfg["username"] = cfg.proxy_username
|
|
proxy_cfg["password"] = str(cfg.proxy_password)
|
|
|
|
chromium_exe = _find_chromium_exe()
|
|
if chromium_exe:
|
|
_log.info("Using Chromium at: %s", chromium_exe)
|
|
else:
|
|
_log.warning("Chromium not found via dynamic search — letting Playwright auto-detect")
|
|
|
|
with sync_playwright() as pw:
|
|
launch_kwargs: dict = {
|
|
"headless": headless,
|
|
"proxy": proxy_cfg,
|
|
}
|
|
if chromium_exe:
|
|
# Bypass Playwright's broken path detection on this machine
|
|
launch_kwargs["executable_path"] = chromium_exe
|
|
browser = pw.chromium.launch(**launch_kwargs)
|
|
ctx = browser.new_context(
|
|
user_agent=(
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
|
)
|
|
)
|
|
page = ctx.new_page()
|
|
page.on("request", store.on_request)
|
|
|
|
planner_url = cfg.planner_url or "https://planner.cloud.microsoft/"
|
|
_log.info("Navigating to Planner for login (%s)", planner_url[:80])
|
|
page.goto(planner_url, timeout=60_000)
|
|
|
|
# Username
|
|
page.wait_for_selector("input[type='email'], input[name='loginfmt']", timeout=30_000)
|
|
page.fill("input[type='email'], input[name='loginfmt']", cfg.onedrive_username)
|
|
page.click("input[type='submit'], button[type='submit']")
|
|
|
|
# Password
|
|
page.wait_for_selector("input[type='password'], input[name='passwd']", timeout=20_000)
|
|
page.fill("input[type='password'], input[name='passwd']", cfg.onedrive_password)
|
|
page.click("input[type='submit'], button[type='submit']")
|
|
|
|
# TOTP (optional)
|
|
try:
|
|
page.wait_for_selector(
|
|
"input[name='otc'], input[autocomplete='one-time-code']",
|
|
timeout=10_000,
|
|
)
|
|
otp = pyotp.TOTP(cfg.microsoft_totp_secret).now()
|
|
page.fill("input[name='otc'], input[autocomplete='one-time-code']", otp)
|
|
page.click("input[type='submit'], button[type='submit']")
|
|
_log.info("TOTP submitted")
|
|
except Exception:
|
|
_log.debug("No TOTP prompt detected")
|
|
|
|
# "Stay signed in?" — click No
|
|
try:
|
|
page.wait_for_selector("input[id='idBtn_Back']", timeout=8_000)
|
|
page.click("input[id='idBtn_Back']")
|
|
except Exception:
|
|
pass
|
|
|
|
# Wait until Planner fires a Graph API call so we capture the token
|
|
deadline = time.monotonic() + timeout_s
|
|
while time.monotonic() < deadline and not store.token:
|
|
try:
|
|
page.wait_for_selector(
|
|
"[data-testid*='planner'], [class*='planner'], [role='main']",
|
|
timeout=4_000,
|
|
)
|
|
except Exception:
|
|
pass
|
|
if not store.token:
|
|
time.sleep(1)
|
|
|
|
if not store.token:
|
|
raise RuntimeError(
|
|
"Could not capture Bearer token — Microsoft login may have failed "
|
|
"or Planner did not fire any Graph API requests within "
|
|
f"{timeout_s}s."
|
|
)
|
|
|
|
cookies = {c["name"]: c["value"] for c in ctx.cookies()}
|
|
browser.close()
|
|
|
|
_log.info(
|
|
"Login complete — token captured (age %.1fs), %d cookies harvested",
|
|
store.age_s, len(cookies),
|
|
)
|
|
return store.token, cookies
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# httpx client
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_client(cfg: AppConfig, token: str) -> httpx.Client:
|
|
proxy_url: str | None = None
|
|
if cfg.proxy_ip and cfg.proxy_port:
|
|
proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}"
|
|
if cfg.proxy_username and cfg.proxy_password:
|
|
proxy_url = (
|
|
f"http://{cfg.proxy_username}:{cfg.proxy_password}"
|
|
f"@{cfg.proxy_ip}:{cfg.proxy_port}"
|
|
)
|
|
|
|
return httpx.Client(
|
|
headers={
|
|
"Authorization": f"Bearer {token}",
|
|
"Accept": "application/json",
|
|
},
|
|
timeout=60.0,
|
|
follow_redirects=True,
|
|
proxy=proxy_url, # httpx >= 0.28: singular `proxy=`, not `proxies=`
|
|
)
|
|
|
|
|
|
def _refresh_token(client: httpx.Client, cfg: AppConfig) -> None:
|
|
"""Re-login via Playwright and update the client's Authorization header in place."""
|
|
_log.warning("Token expired (401) — re-harvesting via Playwright...")
|
|
new_token, _ = _playwright_login_and_capture_token(cfg, headless=True)
|
|
client.headers["Authorization"] = f"Bearer {new_token}"
|
|
_log.info("Token refreshed.")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Graph API helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _get(client: httpx.Client, path: str, cfg: AppConfig, **params) -> dict:
|
|
"""GET a Graph API path, refreshing token once on 401."""
|
|
url = f"{GRAPH_BASE}/{path.lstrip('/')}"
|
|
for attempt in range(2):
|
|
r = client.get(url, params=params or None)
|
|
if r.status_code == 401 and attempt == 0:
|
|
_refresh_token(client, cfg)
|
|
continue
|
|
r.raise_for_status()
|
|
return r.json()
|
|
raise RuntimeError(f"Unreachable: {url}") # pragma: no cover
|
|
|
|
|
|
def _get_paged(client: httpx.Client, path: str, cfg: AppConfig, **params) -> list[dict]:
|
|
"""Follow @odata.nextLink pagination until exhausted."""
|
|
results: list[dict] = []
|
|
url = f"{GRAPH_BASE}/{path.lstrip('/')}"
|
|
first = True
|
|
while url:
|
|
for attempt in range(2):
|
|
if first:
|
|
r = client.get(url, params=params or None)
|
|
else:
|
|
r = client.get(url)
|
|
if r.status_code == 401 and attempt == 0:
|
|
_refresh_token(client, cfg)
|
|
continue
|
|
r.raise_for_status()
|
|
break
|
|
data = r.json()
|
|
results.extend(data.get("value", []))
|
|
url = data.get("@odata.nextLink")
|
|
first = False
|
|
return results
|
|
|
|
|
|
def _update_task_status(
|
|
client: httpx.Client,
|
|
task_id: str,
|
|
etag: str,
|
|
percent_complete: int,
|
|
cfg: AppConfig,
|
|
) -> bool:
|
|
"""PATCH a Planner task's percentComplete (0=Not Started, 50=In Progress, 100=Complete).
|
|
|
|
Planner requires the If-Match header with the task's current ETag for
|
|
optimistic concurrency — without it the API returns 428 Precondition Required.
|
|
Returns True on success, False on any error (warning is logged but run continues).
|
|
"""
|
|
url = f"{GRAPH_BASE}/planner/tasks/{task_id}"
|
|
req_headers = {
|
|
"If-Match": etag,
|
|
"Content-Type": "application/json",
|
|
}
|
|
body = {"percentComplete": percent_complete}
|
|
try:
|
|
for attempt in range(2):
|
|
r = client.patch(url, json=body, headers=req_headers)
|
|
if r.status_code == 401 and attempt == 0:
|
|
_refresh_token(client, cfg)
|
|
continue
|
|
r.raise_for_status()
|
|
return True
|
|
except httpx.HTTPStatusError as exc:
|
|
_log.warning(
|
|
" Status update failed for task %s: HTTP %s — %s",
|
|
task_id, exc.response.status_code, exc.response.text[:200],
|
|
)
|
|
except Exception as exc:
|
|
_log.warning(" Status update failed for task %s: %s", task_id, exc)
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plan discovery via group memberships
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _discover_all_plans(client: httpx.Client, cfg: AppConfig) -> list[dict]:
|
|
"""
|
|
Return every Planner plan visible to the user through their Microsoft 365 group
|
|
memberships. Each dict has at minimum: id, title, _group_name.
|
|
"""
|
|
_log.info("Fetching user's group memberships...")
|
|
groups = _get_paged(
|
|
client,
|
|
"me/memberOf/microsoft.graph.group",
|
|
cfg,
|
|
**{"$select": "id,displayName", "$top": "999"},
|
|
)
|
|
_log.info("%d groups found", len(groups))
|
|
|
|
all_plans: list[dict] = []
|
|
for group in groups:
|
|
gid = group.get("id", "")
|
|
if not gid:
|
|
continue
|
|
try:
|
|
plans = _get_paged(
|
|
client,
|
|
f"groups/{gid}/planner/plans",
|
|
cfg,
|
|
**{"$select": "id,title"},
|
|
)
|
|
for p in plans:
|
|
p["_group_id"] = gid
|
|
p["_group_name"] = group.get("displayName", "")
|
|
all_plans.extend(plans)
|
|
except httpx.HTTPStatusError as exc:
|
|
if exc.response.status_code in (403, 404):
|
|
pass # group has no Planner or access denied
|
|
else:
|
|
_log.warning("plans for group %s: HTTP %s", gid, exc.response.status_code)
|
|
time.sleep(0.1) # gentle rate-limit courtesy
|
|
|
|
_log.info("Total plans discovered: %d", len(all_plans))
|
|
return all_plans
|
|
|
|
|
|
def _match_plan(plans: list[dict], facility_name: str) -> dict | None:
|
|
"""Match a facility name to a Planner plan title (exact → prefix → substring)."""
|
|
fac = facility_name.strip().lower()
|
|
|
|
# Pass 1: exact match
|
|
for p in plans:
|
|
if (p.get("title") or "").strip().lower() == fac:
|
|
return p
|
|
|
|
# Pass 2: one string is a prefix of the other (handles trailing words)
|
|
for p in plans:
|
|
title = (p.get("title") or "").strip().lower()
|
|
if title.startswith(fac) or fac.startswith(title):
|
|
_log.warning("Prefix plan match: '%s' -> '%s'", facility_name, p.get("title"))
|
|
return p
|
|
|
|
# Pass 3: substring match — only for names long enough to be unambiguous
|
|
# (short names like "Hills" would falsely match "Arbor Hills")
|
|
if len(fac) >= 8:
|
|
for p in plans:
|
|
title = (p.get("title") or "").strip().lower()
|
|
if fac in title:
|
|
_log.warning("Substring plan match: '%s' -> '%s'", facility_name, p.get("title"))
|
|
return p
|
|
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tasks + file references
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _not_started_tasks(client: httpx.Client, plan_id: str, cfg: AppConfig) -> list[dict]:
|
|
tasks = _get_paged(
|
|
client,
|
|
f"planner/plans/{plan_id}/tasks",
|
|
cfg,
|
|
**{"$select": "id,title,percentComplete"},
|
|
)
|
|
ns = [t for t in tasks if (t.get("percentComplete") or 0) == 0]
|
|
_log.info("Plan %s: %d tasks total, %d not started", plan_id, len(tasks), len(ns))
|
|
return ns
|
|
|
|
|
|
def _task_file_urls(client: httpx.Client, task_id: str, cfg: AppConfig) -> list[str]:
|
|
"""Return SharePoint file URLs from a task's references (details endpoint)."""
|
|
try:
|
|
details = _get(client, f"planner/tasks/{task_id}/details", cfg)
|
|
except httpx.HTTPStatusError as exc:
|
|
_log.warning("task details %s: HTTP %s", task_id, exc.response.status_code)
|
|
return []
|
|
|
|
refs: dict = details.get("references", {}) or {}
|
|
urls: list[str] = []
|
|
for encoded_key in refs:
|
|
try:
|
|
url = unquote(encoded_key)
|
|
if url.startswith("http"):
|
|
urls.append(url)
|
|
except Exception:
|
|
pass
|
|
return urls
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SharePoint URL resolution + download
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _encode_share_url(url: str) -> str:
|
|
"""Convert a SharePoint URL to Graph's u!{base64url} share-id format."""
|
|
b64 = base64.urlsafe_b64encode(url.encode("utf-8")).rstrip(b"=").decode()
|
|
return f"u!{b64}"
|
|
|
|
|
|
def _resolve_sp_url(
|
|
client: httpx.Client, sp_url: str, cfg: AppConfig
|
|
) -> tuple[str | None, str | None]:
|
|
"""
|
|
Resolve a SharePoint URL to (pre-auth download URL, filename).
|
|
Uses the Graph shares endpoint — same call visible in DevTools.
|
|
"""
|
|
share_id = _encode_share_url(sp_url)
|
|
try:
|
|
# Do NOT use $select here: @microsoft.graph.downloadUrl is an OData
|
|
# annotation and is only present in the full (unfiltered) response.
|
|
# Requesting it via $select causes Graph to return 200 with no annotation.
|
|
item = _get(client, f"shares/{share_id}/driveItem", cfg)
|
|
dl_url = item.get("@microsoft.graph.downloadUrl")
|
|
name = item.get("name") or Path(urlparse(sp_url).path).name or "download.pdf"
|
|
return dl_url, name
|
|
except httpx.HTTPStatusError as exc:
|
|
_log.warning("shares resolve %s…: HTTP %s", sp_url[:60], exc.response.status_code)
|
|
return None, None
|
|
|
|
|
|
def _download_file(
|
|
download_url: str,
|
|
dest_folder: Path,
|
|
filename: str,
|
|
) -> Path | None:
|
|
"""Stream-download a pre-authenticated Azure Blob SAS URL to dest_folder/filename.
|
|
|
|
If the file already exists (from a previous interrupted run) it is kept as-is
|
|
and its path is returned — no re-download, no duplicate _2/_3 copies.
|
|
|
|
Uses a fresh proxy-free client: the SAS URL authenticates via query-string
|
|
and must not be routed through the corporate proxy.
|
|
"""
|
|
dest_folder.mkdir(parents=True, exist_ok=True)
|
|
dest = dest_folder / filename
|
|
|
|
# Skip re-download if file already exists and is non-empty
|
|
if dest.exists() and dest.stat().st_size > 0:
|
|
_log.info(" Already exists, skipping download: %s (%d bytes)",
|
|
dest.name, dest.stat().st_size)
|
|
return dest
|
|
|
|
try:
|
|
# Proxy-free: Azure Blob SAS URLs authenticate via query-string,
|
|
# routing through the corporate proxy wastes bandwidth and can timeout.
|
|
with httpx.Client(timeout=120.0, follow_redirects=True) as blob_client:
|
|
with blob_client.stream("GET", download_url) as r:
|
|
r.raise_for_status()
|
|
with open(dest, "wb") as f:
|
|
for chunk in r.iter_bytes(65536):
|
|
f.write(chunk)
|
|
_log.info(" Downloaded: %s (%d bytes)", dest.name, dest.stat().st_size)
|
|
return dest
|
|
except Exception as exc:
|
|
_log.error(" Download failed for %s: %s", filename, exc)
|
|
dest.unlink(missing_ok=True)
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Workable file copy (same naming as Selenium bot)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _unique_workable_path(root: Path, base: str, ext: str) -> Path:
|
|
ext = ext if ext.startswith(".") else f".{ext}"
|
|
first = root / f"{base}{ext}"
|
|
if not first.exists():
|
|
return first
|
|
for n in range(2, 1000):
|
|
candidate = root / f"{base}_{n}{ext}"
|
|
if not candidate.exists():
|
|
return candidate
|
|
raise RuntimeError(f"Cannot allocate workable filename for {base}{ext}")
|
|
|
|
|
|
def _build_workable_files(
|
|
*,
|
|
facility_name: str,
|
|
downloaded: list[Path],
|
|
output_root: Path,
|
|
today_iso: str,
|
|
) -> tuple[list[Path], Path]:
|
|
ddmmyyyy = datetime.strptime(today_iso, "%Y-%m-%d").strftime("%d-%m-%Y")
|
|
fac_safe = _sanitize_folder_name(facility_name)
|
|
workable_root = output_root / today_iso / "Workable"
|
|
medicare_root = output_root / today_iso / "Workable_Medicare"
|
|
|
|
if not downloaded:
|
|
return [], workable_root
|
|
|
|
workable_root.mkdir(parents=True, exist_ok=True)
|
|
medicare_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
workable_paths: list[Path] = []
|
|
for i, src in enumerate(downloaded, start=1):
|
|
ext = src.suffix or ".pdf"
|
|
base = (
|
|
f"{ddmmyyyy}_{fac_safe}"
|
|
if len(downloaded) == 1
|
|
else f"{ddmmyyyy}_{fac_safe}_{i}"
|
|
)
|
|
# Check if this workable file already exists (previous interrupted run)
|
|
dst_first = workable_root / f"{base}{ext}"
|
|
if dst_first.exists() and dst_first.stat().st_size > 0:
|
|
_log.info(" Workable already exists, keeping: %s", dst_first.name)
|
|
dst = dst_first
|
|
else:
|
|
dst = _unique_workable_path(workable_root, base, ext)
|
|
shutil.copy2(src, dst)
|
|
workable_paths.append(dst)
|
|
med_dst_first = medicare_root / f"{dst.stem}{ext}"
|
|
if med_dst_first.exists() and med_dst_first.stat().st_size > 0:
|
|
_log.info(" Medicare already exists, keeping: %s", med_dst_first.name)
|
|
else:
|
|
med_dst = _unique_workable_path(medicare_root, dst.stem, ext)
|
|
shutil.copy2(dst, med_dst)
|
|
|
|
_log.info(
|
|
"Workable: %d file(s) -> %s | Medicare -> %s",
|
|
len(workable_paths), workable_root, medicare_root,
|
|
)
|
|
return workable_paths, workable_root
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Summary Excel (same schema as Selenium bot)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _write_summary(
|
|
results: dict[str, dict],
|
|
failures: list[tuple[str, str, str]],
|
|
summary_path: Path,
|
|
run_timestamp: str,
|
|
) -> None:
|
|
failure_names = {f for f, _, _ in failures}
|
|
rows: list[dict] = []
|
|
|
|
for facility, payload in results.items():
|
|
if facility in failure_names:
|
|
continue
|
|
file_count = len(payload.get("original") or [])
|
|
tasks_total = int(payload.get("tasks_total") or 0)
|
|
tasks_att = int(payload.get("tasks_with_attachments") or 0)
|
|
tasks_dl = int(payload.get("tasks_with_downloads") or 0)
|
|
tasks_skip = int(payload.get("tasks_skipped_no_attachment") or 0)
|
|
status_changed = int(payload.get("status_changed") or 0)
|
|
|
|
if tasks_total <= 0:
|
|
status = "No tasks"
|
|
elif tasks_att <= 0:
|
|
status = "No attachments"
|
|
elif tasks_dl <= 0 or file_count <= 0:
|
|
status = "Download failed"
|
|
elif tasks_dl >= tasks_att:
|
|
status = "Success"
|
|
else:
|
|
status = "Partial"
|
|
|
|
if file_count <= 0:
|
|
status_changed_label = "N/A"
|
|
elif status_changed >= tasks_dl:
|
|
status_changed_label = f"Yes — {status_changed} task(s) → In Progress"
|
|
elif status_changed > 0:
|
|
status_changed_label = f"Partial — {status_changed}/{tasks_dl} task(s) → In Progress"
|
|
else:
|
|
status_changed_label = "No — Planner PATCH failed"
|
|
|
|
rows.append({
|
|
"Run Timestamp": run_timestamp,
|
|
"Facility Name": facility,
|
|
"Files Downloaded": file_count,
|
|
"Tasks Not Started": tasks_total,
|
|
"Tasks With Attachments": tasks_att,
|
|
"Tasks Skipped (No File)": tasks_skip,
|
|
"Status": status,
|
|
"Status Changed": status_changed_label,
|
|
})
|
|
|
|
for facility, err, status in failures:
|
|
rows.append({
|
|
"Run Timestamp": run_timestamp,
|
|
"Facility Name": facility,
|
|
"Files Downloaded": 0,
|
|
"Tasks Not Started": 0,
|
|
"Tasks With Attachments": 0,
|
|
"Tasks Skipped (No File)": 0,
|
|
"Status": status,
|
|
"Status Changed": "N/A",
|
|
})
|
|
|
|
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
|
pd.DataFrame(rows).to_excel(summary_path, index=False, sheet_name="Download")
|
|
_log.info("Summary -> %s", summary_path)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-facility processing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _process_facility(
|
|
client: httpx.Client,
|
|
cfg: AppConfig,
|
|
plans: list[dict],
|
|
facility_name: str,
|
|
dest_folder: Path,
|
|
) -> dict:
|
|
plan = _match_plan(plans, facility_name)
|
|
if plan is None:
|
|
raise ValueError(f"No Planner plan found matching '{facility_name}'")
|
|
|
|
plan_id = plan["id"]
|
|
_log.info(
|
|
"%s -> plan '%s' (group: %s)",
|
|
facility_name, plan.get("title"), plan.get("_group_name", ""),
|
|
)
|
|
|
|
# ── Only Not Started tasks (percentComplete == 0) are ever processed.
|
|
# ── Within those, only tasks that have at least one file attachment are
|
|
# downloaded. Tasks that are In Progress, Completed, or have no
|
|
# attachments are silently skipped.
|
|
tasks = _not_started_tasks(client, plan_id, cfg)
|
|
tasks_total = len(tasks)
|
|
tasks_with_att = 0
|
|
tasks_with_dl = 0
|
|
tasks_no_att = 0 # Not Started but no file attachment → skip
|
|
status_changed = 0
|
|
downloaded: list[Path] = []
|
|
|
|
_log.info(
|
|
" Processing ONLY Not Started tasks with attachments (%d Not Started found)",
|
|
tasks_total,
|
|
)
|
|
|
|
for task in tasks:
|
|
tid = task["id"]
|
|
title = task.get("title", "")
|
|
file_urls = _task_file_urls(client, tid, cfg)
|
|
|
|
if not file_urls:
|
|
# Not Started but no attachment — skip entirely
|
|
_log.info(" SKIP '%s': Not Started but no attachment", title[:60])
|
|
tasks_no_att += 1
|
|
continue
|
|
|
|
# Not Started WITH attachment(s) — download ALL of them
|
|
tasks_with_att += 1
|
|
total_urls = len(file_urls)
|
|
task_dl = 0
|
|
_log.info(" PROCESS '%s': %d attachment(s)", title[:60], total_urls)
|
|
|
|
for att_idx, sp_url in enumerate(file_urls, start=1):
|
|
dl_url, filename = _resolve_sp_url(client, sp_url, cfg)
|
|
if not dl_url or not filename:
|
|
_log.warning(
|
|
" [%d/%d] Could not resolve attachment URL: %.80s",
|
|
att_idx, total_urls, sp_url,
|
|
)
|
|
continue
|
|
_log.info(" [%d/%d] Downloading: %s", att_idx, total_urls, filename)
|
|
path = _download_file(dl_url, dest_folder, filename)
|
|
if path:
|
|
downloaded.append(path)
|
|
task_dl += 1
|
|
|
|
if task_dl == total_urls:
|
|
# Every attachment downloaded — mark In Progress
|
|
tasks_with_dl += 1
|
|
etag = task.get("@odata.etag", "")
|
|
if etag:
|
|
ok = _update_task_status(client, tid, etag, 50, cfg)
|
|
if ok:
|
|
_log.info(
|
|
" Status -> In Progress: '%s' (%d/%d files downloaded)",
|
|
title[:60], task_dl, total_urls,
|
|
)
|
|
status_changed += 1
|
|
else:
|
|
_log.warning(" No ETag for task '%s' — status not updated", title[:60])
|
|
elif task_dl > 0:
|
|
# Partial — some failed; don't mark In Progress, leave for retry
|
|
tasks_with_dl += 1
|
|
_log.warning(
|
|
" '%s': partial download %d/%d — status NOT changed (will retry next run)",
|
|
title[:60], task_dl, total_urls,
|
|
)
|
|
else:
|
|
_log.warning(
|
|
" '%s': 0/%d attachments downloaded — status NOT changed",
|
|
title[:60], total_urls,
|
|
)
|
|
|
|
_log.info(
|
|
" Summary: %d Not Started | %d with attachments (processed) | "
|
|
"%d no attachment (skipped) | %d file(s) downloaded",
|
|
tasks_total, tasks_with_att, tasks_no_att, len(downloaded),
|
|
)
|
|
|
|
return {
|
|
"original": downloaded,
|
|
"tasks_total": tasks_total,
|
|
"tasks_with_attachments": tasks_with_att,
|
|
"tasks_with_downloads": tasks_with_dl,
|
|
"tasks_skipped_no_attachment": tasks_no_att,
|
|
"status_changed": status_changed,
|
|
"status_via_task_details": 0,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main workflow
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _filter_facilities(
|
|
facilities: list[str],
|
|
*,
|
|
facility_limit: int,
|
|
facility_name: str | None,
|
|
) -> list[str]:
|
|
if facility_name:
|
|
name = facility_name.strip()
|
|
matches = [f for f in facilities if f.lower() == name.lower()]
|
|
if not matches:
|
|
raise ValueError(f"Facility '{name}' not found in Facilities sheet.")
|
|
return matches
|
|
if facility_limit > 0:
|
|
return facilities[:facility_limit]
|
|
return facilities
|
|
|
|
|
|
def run_planner_http_workflow(
|
|
*,
|
|
facility_limit: int = 0,
|
|
facility_name: str | None = None,
|
|
resume: bool = False,
|
|
) -> None:
|
|
run_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
log_path = _setup_logging()
|
|
_log.info("=" * 66)
|
|
_log.info(" MISF Planner HTTP Download — Start [%s]", run_timestamp)
|
|
_log.info(" Run date : %s", TODAY_ISO_DATE)
|
|
_log.info(" Log file : %s", log_path)
|
|
_log.info("=" * 66)
|
|
|
|
cfg = read_app_config()
|
|
output_root = PROJECT_ROOT / "Output"
|
|
original_root = output_root / TODAY_ISO_DATE / "Original"
|
|
checkpoint_path = output_root / TODAY_ISO_DATE / "checkpoint_http.json"
|
|
|
|
facilities = read_facility_list()
|
|
facilities = _filter_facilities(
|
|
facilities,
|
|
facility_limit=facility_limit,
|
|
facility_name=facility_name,
|
|
)
|
|
|
|
# Resume state
|
|
start_index = 0
|
|
completed: set[str] = set()
|
|
if resume:
|
|
ckpt = load_checkpoint(checkpoint_path)
|
|
start_index = int(ckpt.get("last_completed_index", 0) or 0)
|
|
completed = {str(x) for x in (ckpt.get("completed_facilities", []) or [])}
|
|
_log.info("Resuming from index %d, %d already done", start_index, len(completed))
|
|
else:
|
|
clear_checkpoint(checkpoint_path)
|
|
|
|
# Validate credentials before launching the browser — fail fast with a clear message
|
|
missing = missing_onedrive_credentials(
|
|
cfg.onedrive_username,
|
|
cfg.onedrive_password,
|
|
cfg.microsoft_totp_secret,
|
|
)
|
|
if missing:
|
|
raise ValueError(credentials_setup_message(missing))
|
|
validate_totp_secret(cfg.microsoft_totp_secret)
|
|
|
|
# --- Ensure browser is installed (auto-downloads on first run) ---
|
|
_ensure_playwright_browsers()
|
|
|
|
# --- Login (Playwright, headless) ---
|
|
_log.info("Launching Playwright for Microsoft login...")
|
|
token, _cookies = _playwright_login_and_capture_token(cfg, headless=True)
|
|
client = _build_client(cfg, token)
|
|
|
|
# --- Discover all plans once ---
|
|
_log.info("Discovering Planner plans via Graph API...")
|
|
plans = _discover_all_plans(client, cfg)
|
|
_log.info("Plan discovery done: %d plans available", len(plans))
|
|
|
|
results: dict[str, dict] = {}
|
|
failures: list[tuple[str, str, str]] = []
|
|
total_files = 0
|
|
|
|
for idx, fac in enumerate(facilities, start=1):
|
|
if idx < start_index or fac in completed:
|
|
_log.info("Skipping %s (already processed)", fac)
|
|
continue
|
|
|
|
_log.info("-" * 50)
|
|
_log.info("Facility %d / %d: %s", idx, len(facilities), fac)
|
|
dest = original_root / _sanitize_folder_name(fac)
|
|
|
|
try:
|
|
result = _process_facility(client, cfg, plans, fac, dest)
|
|
downloaded = result["original"]
|
|
file_count = len(downloaded)
|
|
|
|
if downloaded:
|
|
workable_paths, workable_root = _build_workable_files(
|
|
facility_name=fac,
|
|
downloaded=downloaded,
|
|
output_root=output_root,
|
|
today_iso=TODAY_ISO_DATE,
|
|
)
|
|
else:
|
|
workable_paths = []
|
|
workable_root = output_root / TODAY_ISO_DATE / "Workable"
|
|
if dest.exists() and not any(dest.iterdir()):
|
|
dest.rmdir()
|
|
|
|
results[fac] = {**result, "workable": workable_paths}
|
|
total_files += file_count
|
|
completed.add(fac)
|
|
save_checkpoint(
|
|
checkpoint_path,
|
|
last_completed_index=idx,
|
|
facility_name=fac,
|
|
completed_facilities=sorted(completed),
|
|
)
|
|
_log.info("%s: %d file(s) -> %s", fac, file_count, workable_root)
|
|
|
|
except Exception as exc:
|
|
err_msg = str(exc)
|
|
_log.error("%s FAILED: %s", fac, err_msg)
|
|
status = (
|
|
"Not appear"
|
|
if "no planner plan found" in err_msg.lower()
|
|
else "FAILED"
|
|
)
|
|
failures.append((fac, err_msg, status))
|
|
results[fac] = {
|
|
"original": [], "workable": [], "comment": err_msg[:500],
|
|
"tasks_total": 0, "tasks_with_attachments": 0,
|
|
"tasks_with_downloads": 0, "tasks_skipped_no_attachment": 0,
|
|
"status_changed": 0, "status_via_task_details": 0,
|
|
}
|
|
save_checkpoint(
|
|
checkpoint_path,
|
|
last_completed_index=idx,
|
|
facility_name=fac,
|
|
completed_facilities=sorted(completed),
|
|
)
|
|
|
|
summary_path = (
|
|
output_root / TODAY_ISO_DATE / "Summary"
|
|
/ f"Download_Summary_HTTP_{TODAY_ISO_DATE}.xlsx"
|
|
)
|
|
_write_summary(results, failures, summary_path, run_timestamp)
|
|
clear_checkpoint(checkpoint_path)
|
|
|
|
_log.info("=" * 66)
|
|
_log.info(
|
|
" COMPLETE — %d file(s) downloaded, %d failure(s)",
|
|
total_files, len(failures),
|
|
)
|
|
_log.info("=" * 66)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_arg_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(
|
|
description="Download face-sheet PDFs from Planner via Microsoft Graph API (no Selenium)."
|
|
)
|
|
p.add_argument(
|
|
"--limit", type=int, default=0,
|
|
help="Process only the first N facilities (0 = all).",
|
|
)
|
|
p.add_argument(
|
|
"--facility", type=str, default="",
|
|
help="Process a single facility by exact name.",
|
|
)
|
|
p.add_argument(
|
|
"--resume", action="store_true",
|
|
help="Resume from checkpoint.",
|
|
)
|
|
return p
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_frozen = getattr(sys, "frozen", False) # True when running as .exe
|
|
|
|
args = _build_arg_parser().parse_args()
|
|
exit_code = 0
|
|
try:
|
|
run_planner_http_workflow(
|
|
facility_limit=max(0, args.limit),
|
|
facility_name=args.facility.strip() or None,
|
|
resume=args.resume,
|
|
)
|
|
except KeyboardInterrupt:
|
|
print("\nCancelled.")
|
|
exit_code = 130
|
|
except Exception:
|
|
traceback.print_exc()
|
|
exit_code = 1
|
|
|
|
if _frozen and exit_code != 0:
|
|
# On error only: keep the window open so the user can read what went wrong.
|
|
input("\n✗ An error occurred (see above). Press Enter to close...")
|
|
|
|
sys.exit(exit_code)
|