VasanthandClaude Opus 5
bfd72825b4
A task carrying an "Assigned to" avatar in the Planner grid has been picked up by a person, so its attachments must not be downloaded and its status must not be moved to In Progress. Until now the only filter was percentComplete == 0, so an assigned task with an attachment was downloaded like any other — on the Balboa Nursing and Rehab plan that was one of the two "Facesheets Needed (GG)" tasks assigned to Dr. Mark Lukasik and Adina Smets. - _not_started_tasks() now selects `assignments` as well. - _task_assignee_ids()/_resolve_user_names(): the assignee check runs before the task-details call, so an assigned task costs no extra Graph request; display names are resolved once per user and cached for the run, falling back to the raw ID when the lookup fails. - Summary workbook gains a "Tasks Skipped (Assigned)" column and a second "Tasks" sheet with one row per task — facility, task name, assigned to, attachment count, Downloaded Yes/No and the reason — so a skip can be explained without reading the log. - New "All assigned — skipped" facility status, distinct from "No attachments". scripts/probe_assignments.py is the read-only probe used to confirm the rule against the live plan before changing behaviour. Verified on Balboa Nursing and Rehab: 4 Not Started -> 8.24.26 and 8.21.26 downloaded (unassigned), both Facesheets Needed (GG) skipped. 110 tests pass (tests/test_uipath_pilot.py and tests/test_config.py fail on this checkout for pre-existing, unrelated reasons). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1537 lines
56 KiB
Python
1537 lines
56 KiB
Python
"""
|
|
MISF VOB — OneDrive / Planner HTTP download (API-first, no Selenium).
|
|
|
|
Architecture:
|
|
1. Playwright opens Microsoft login, captures the Bearer token from the first
|
|
authenticated request to graph.microsoft.com, then closes immediately.
|
|
2. All subsequent work uses httpx against the Microsoft Graph API:
|
|
- Discover plans via user's group memberships
|
|
- List Not-Started tasks per facility plan, skipping any task that is
|
|
already assigned to someone (their work — not ours to download)
|
|
- Resolve SharePoint file references via Graph shares endpoint
|
|
- Stream-download each PDF
|
|
3. Identical output structure to the Selenium bot:
|
|
Output/<date>/Original/<Facility>/
|
|
Output/<date>/Workable/ (DD-MM-YYYY_FacilityName[_N].pdf)
|
|
Output/<date>/Workable_Medicare/
|
|
Output/<date>/Summary/Download_Summary_HTTP_<date>.xlsx
|
|
Output/<date>/checkpoint_http.json
|
|
|
|
Usage:
|
|
python onedrive_planner_http.py
|
|
python onedrive_planner_http.py --limit 3
|
|
python onedrive_planner_http.py --facility "Arbor Hills"
|
|
python onedrive_planner_http.py --resume
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import logging
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from urllib.parse import unquote, urlparse
|
|
|
|
import httpx
|
|
import pandas as pd
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
SRC = ROOT / "src"
|
|
if str(SRC) not in sys.path:
|
|
sys.path.insert(0, str(SRC))
|
|
|
|
from checkpoint import clear_checkpoint, load_checkpoint, save_checkpoint # noqa: E402
|
|
from config import ( # noqa: E402
|
|
PROJECT_ROOT,
|
|
TODAY_ISO_DATE,
|
|
AppConfig,
|
|
credentials_setup_message,
|
|
missing_onedrive_credentials,
|
|
read_app_config,
|
|
read_facility_list,
|
|
)
|
|
from onedrive_planner import _sanitize_folder_name # noqa: E402
|
|
from totp_utils import validate_totp_secret # noqa: E402
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
GRAPH_BASE = "https://graph.microsoft.com/v1.0"
|
|
|
|
|
|
def _find_chromium_exe() -> str | None:
|
|
"""
|
|
Locate the Playwright-downloaded Chromium executable without relying on
|
|
Playwright's internal path detection, which fails on some Windows setups.
|
|
Searches ms-playwright in LOCALAPPDATA and PLAYWRIGHT_BROWSERS_PATH.
|
|
"""
|
|
import os
|
|
search_roots = []
|
|
if os.environ.get("PLAYWRIGHT_BROWSERS_PATH"):
|
|
search_roots.append(Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"]))
|
|
local_app = os.environ.get("LOCALAPPDATA", "")
|
|
if local_app:
|
|
search_roots.append(Path(local_app) / "ms-playwright")
|
|
search_roots.append(Path.home() / "AppData" / "Local" / "ms-playwright")
|
|
|
|
for base in search_roots:
|
|
if not base.is_dir():
|
|
continue
|
|
# Prefer highest version number
|
|
for d in sorted(base.glob("chromium-*"), reverse=True):
|
|
for rel in ("chrome-win64/chrome.exe", "chrome-win/chrome.exe"):
|
|
exe = d / rel
|
|
if exe.is_file():
|
|
_log.debug("Found Chromium at: %s", exe)
|
|
return str(exe)
|
|
return None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# First-run browser install
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _ensure_playwright_browsers() -> None:
|
|
"""Auto-install Playwright Chromium browser on first run on a new machine.
|
|
|
|
PyInstaller bundles node.exe + package/cli.js inside _internal/playwright/driver/
|
|
but does NOT include playwright.cmd (a batch file). So we invoke node.exe directly.
|
|
"""
|
|
if _find_chromium_exe():
|
|
return # Already installed
|
|
|
|
import subprocess
|
|
|
|
print(
|
|
"\n[First-time setup] Playwright Chromium browser not found.\n"
|
|
"Downloading now (~150 MB). This only happens once on this machine.\n"
|
|
)
|
|
|
|
# Locate the driver directory.
|
|
# Frozen (PyInstaller onedir): _MEIPASS/playwright/driver/
|
|
# Non-frozen (dev): next to the playwright package itself.
|
|
driver_dir: Path | None = None
|
|
if getattr(sys, "frozen", False):
|
|
driver_dir = Path(sys._MEIPASS) / "playwright" / "driver"
|
|
else:
|
|
try:
|
|
import playwright as _pw
|
|
driver_dir = Path(_pw.__file__).parent / "driver"
|
|
except Exception:
|
|
pass
|
|
|
|
cmd: list[str] | None = None
|
|
if driver_dir and driver_dir.is_dir():
|
|
node_exe = driver_dir / "node.exe"
|
|
cli_js = driver_dir / "package" / "cli.js"
|
|
if node_exe.is_file() and cli_js.is_file():
|
|
# PyInstaller bundle: run node.exe directly (no playwright.cmd present)
|
|
cmd = [str(node_exe), str(cli_js), "install", "chromium"]
|
|
else:
|
|
# Dev / non-frozen fallback: try playwright.cmd / playwright shell script
|
|
for name in ("playwright.cmd", "playwright"):
|
|
p = driver_dir / name
|
|
if p.is_file():
|
|
cmd = [str(p), "install", "chromium"]
|
|
break
|
|
|
|
if cmd is None:
|
|
raise RuntimeError(
|
|
"Could not locate the Playwright driver inside the bundle.\n"
|
|
"Expected: _internal/playwright/driver/node.exe\n"
|
|
"Please reinstall the application."
|
|
)
|
|
|
|
try:
|
|
result = subprocess.run(cmd, timeout=600)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"playwright install chromium exited {result.returncode}")
|
|
print("[Setup complete] Browser ready. Continuing...\n")
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"Could not auto-install Playwright browser: {exc}\n"
|
|
"Run manually: playwright install chromium"
|
|
) from exc
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Logging setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _force_utf8_console() -> None:
|
|
"""Make stdout/stderr tolerate non-ASCII.
|
|
|
|
Windows consoles default to cp1252, and both the log format and the error
|
|
banner contain characters outside it — printing them raises
|
|
UnicodeEncodeError and hides the real error from the operator.
|
|
"""
|
|
for stream in (sys.stdout, sys.stderr):
|
|
try:
|
|
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _setup_logging() -> Path:
|
|
_force_utf8_console()
|
|
log_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs"
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
log_path = log_dir / "planner_http.log"
|
|
|
|
root = logging.getLogger()
|
|
root.setLevel(logging.DEBUG)
|
|
|
|
# Guard: don't add duplicate handlers if called more than once in the same process
|
|
if root.handlers:
|
|
return log_path
|
|
|
|
fmt = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s — %(message)s")
|
|
|
|
fh = logging.FileHandler(log_path, encoding="utf-8")
|
|
fh.setLevel(logging.DEBUG)
|
|
fh.setFormatter(fmt)
|
|
root.addHandler(fh)
|
|
|
|
ch = logging.StreamHandler(sys.stdout)
|
|
ch.setLevel(logging.INFO)
|
|
ch.setFormatter(fmt)
|
|
root.addHandler(ch)
|
|
|
|
return log_path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Auth — Playwright login, Bearer token capture
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Microsoft's converged sign-in page keeps the email, password and OTC inputs
|
|
# in the same document, so a bare wait_for_selector() for a later step returns
|
|
# immediately while an earlier view is still on screen. Every fill therefore has
|
|
# to be verified before it is submitted — see _fill_verified().
|
|
EMAIL_SELECTOR = "input[name='loginfmt'], #i0116, input[type='email']"
|
|
PASSWORD_SELECTOR = "input[name='passwd'], #i0118, input[type='password']"
|
|
OTP_SELECTOR = (
|
|
"input[name='otc'], #idTxtBx_SAOTCC_OTC, input[autocomplete='one-time-code']"
|
|
)
|
|
|
|
# Text Microsoft shows when it rejects a credential outright (as opposed to the
|
|
# "you left the box empty" validation message, which we retry).
|
|
_FATAL_LOGIN_MESSAGES = (
|
|
"your account or password is incorrect",
|
|
"we couldn't find an account",
|
|
"that microsoft account doesn't exist",
|
|
"your account has been locked",
|
|
"sign-in was blocked",
|
|
"you can't get there from here",
|
|
"code you entered isn't working",
|
|
"verification code is incorrect",
|
|
)
|
|
|
|
|
|
class _TokenStore:
|
|
"""Collects Bearer tokens from intercepted Playwright requests.
|
|
|
|
Two tiers. Planner/group-scoped Graph endpoints are preferred, because the
|
|
narrow tokens that fire early in the auth flow (``/me``, ``/me/photo``)
|
|
lack the Planner scopes we need. If no preferred token has shown up after
|
|
``fallback_after_s``, any graph.microsoft.com Bearer is accepted rather
|
|
than failing the whole run.
|
|
"""
|
|
|
|
# Endpoints whose tokens are guaranteed to carry Planner/Group read scope
|
|
_PLANNER_PATHS = (
|
|
"/planner/",
|
|
"/groups/",
|
|
"/me/memberOf",
|
|
"/me/transitiveMemberOf",
|
|
)
|
|
|
|
def __init__(self, *, fallback_after_s: float = 45.0) -> None:
|
|
self.token: str | None = None
|
|
self.preferred: bool = False
|
|
self.graph_urls: set[str] = set()
|
|
self.bearer_hosts: set[str] = set()
|
|
self._at: float = 0.0
|
|
self._start = time.monotonic()
|
|
self._fallback_after_s = fallback_after_s
|
|
|
|
def on_request(self, request) -> None:
|
|
url = request.url
|
|
auth = request.headers.get("authorization", "")
|
|
if not auth.startswith("Bearer "):
|
|
return
|
|
try:
|
|
self.bearer_hosts.add(url.split("/")[2])
|
|
except IndexError:
|
|
pass
|
|
if "graph.microsoft.com" not in url:
|
|
return
|
|
|
|
self.graph_urls.add(url.split("?")[0])
|
|
candidate = auth[7:]
|
|
is_preferred = any(p in url for p in self._PLANNER_PATHS)
|
|
|
|
if is_preferred:
|
|
if candidate != self.token or not self.preferred:
|
|
self.token = candidate
|
|
self.preferred = True
|
|
self._at = time.monotonic()
|
|
_log.debug("Planner-scoped token captured from: %.100s", url)
|
|
return
|
|
|
|
# Non-preferred Graph call. Only fall back to it once we have waited
|
|
# long enough that a Planner-scoped call clearly isn't coming.
|
|
if self.preferred:
|
|
return
|
|
if time.monotonic() - self._start < self._fallback_after_s:
|
|
_log.debug("Ignoring non-Planner Graph token (still waiting): %.100s", url)
|
|
return
|
|
if candidate != self.token:
|
|
self.token = candidate
|
|
self._at = time.monotonic()
|
|
_log.warning(
|
|
"No Planner-scoped Graph call seen in %.0fs — falling back to the "
|
|
"token from %.100s",
|
|
self._fallback_after_s, url,
|
|
)
|
|
|
|
@property
|
|
def age_s(self) -> float:
|
|
return time.monotonic() - self._at if self.token else float("inf")
|
|
|
|
|
|
def _is_visible(page, selector: str) -> bool:
|
|
try:
|
|
return page.locator(selector).first.is_visible(timeout=1_000)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _page_text(page, limit: int = 3000) -> str:
|
|
try:
|
|
return page.inner_text("body")[:limit]
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def _fatal_login_message(page) -> str | None:
|
|
"""Return Microsoft's error text when it has rejected a credential."""
|
|
text = _page_text(page).lower()
|
|
for phrase in _FATAL_LOGIN_MESSAGES:
|
|
if phrase in text:
|
|
return phrase
|
|
return None
|
|
|
|
|
|
def _fill_verified(page, selector: str, value: str, *, label: str, attempts: int = 3) -> None:
|
|
"""Fill a login field and confirm the value actually stuck.
|
|
|
|
The sign-in page pre-renders later steps, so a fill issued while the view is
|
|
still switching gets wiped and the form submits empty ("Please enter your
|
|
password."). Reading the value back is the only reliable guard.
|
|
"""
|
|
for attempt in range(1, attempts + 1):
|
|
try:
|
|
box = page.locator(selector).first
|
|
# The field can be in the DOM but not yet interactive while the
|
|
# sign-in page switches views, so wait for it to settle first.
|
|
box.wait_for(state="visible", timeout=20_000)
|
|
box.click(timeout=10_000)
|
|
box.fill("", timeout=10_000)
|
|
box.fill(value, timeout=10_000)
|
|
if box.input_value(timeout=5_000) == value:
|
|
return
|
|
except Exception as exc:
|
|
_log.debug("%s fill attempt %d raised: %s", label, attempt, exc)
|
|
_log.warning(
|
|
"%s field did not retain its value (attempt %d/%d) — retrying",
|
|
label, attempt, attempts,
|
|
)
|
|
time.sleep(1.5)
|
|
raise RuntimeError(
|
|
f"Could not enter the {label} on the Microsoft sign-in page — the field "
|
|
"kept clearing itself. The sign-in page layout may have changed again."
|
|
)
|
|
|
|
|
|
def _submit_password(page, password: str, *, attempts: int = 3) -> None:
|
|
"""Enter the password and confirm the page moved on to the next step."""
|
|
page.wait_for_selector(PASSWORD_SELECTOR, timeout=30_000)
|
|
|
|
for attempt in range(1, attempts + 1):
|
|
# Give the view switch time to finish; the box exists in the DOM while
|
|
# the email view is still showing, so wait_for_selector returns at once.
|
|
page.wait_for_timeout(1_500)
|
|
_fill_verified(page, PASSWORD_SELECTOR, password, label="password")
|
|
page.locator(PASSWORD_SELECTOR).first.press("Enter")
|
|
|
|
deadline = time.monotonic() + 25
|
|
while time.monotonic() < deadline:
|
|
if _is_visible(page, OTP_SELECTOR) or _login_interrupt_visible(page):
|
|
_log.info("Password accepted")
|
|
return
|
|
if "login.microsoftonline.com" not in page.url and "login.live.com" not in page.url:
|
|
_log.info("Password accepted (left the sign-in host)")
|
|
return
|
|
fatal = _fatal_login_message(page)
|
|
if fatal:
|
|
raise RuntimeError(f"Microsoft rejected the sign-in: {fatal!r}")
|
|
time.sleep(1)
|
|
|
|
_log.warning(
|
|
"Still on the password page after attempt %d/%d — retrying", attempt, attempts
|
|
)
|
|
|
|
raise RuntimeError(
|
|
"Could not get past the Microsoft password page. Page said: "
|
|
+ " | ".join(ln.strip() for ln in _page_text(page, 600).splitlines() if ln.strip())
|
|
)
|
|
|
|
|
|
def _submit_totp(page, totp_secret: str, *, attempts: int = 2) -> bool:
|
|
"""Enter the authenticator code if prompted. Returns True when submitted."""
|
|
import pyotp
|
|
|
|
if not _is_visible(page, OTP_SELECTOR):
|
|
_log.debug("No TOTP prompt detected")
|
|
return False
|
|
|
|
for attempt in range(1, attempts + 1):
|
|
code = pyotp.TOTP(totp_secret).now()
|
|
_fill_verified(page, OTP_SELECTOR, code, label="authenticator code")
|
|
page.locator(OTP_SELECTOR).first.press("Enter")
|
|
_log.info("TOTP submitted (attempt %d)", attempt)
|
|
|
|
deadline = time.monotonic() + 25
|
|
while time.monotonic() < deadline:
|
|
if _login_interrupt_visible(page):
|
|
return True
|
|
if not _is_visible(page, OTP_SELECTOR):
|
|
return True
|
|
fatal = _fatal_login_message(page)
|
|
if fatal:
|
|
raise RuntimeError(f"Microsoft rejected the authenticator code: {fatal!r}")
|
|
time.sleep(1)
|
|
|
|
_log.warning("Code page still showing — retrying with a fresh code")
|
|
time.sleep(5)
|
|
|
|
raise RuntimeError("Microsoft kept asking for the authenticator code.")
|
|
|
|
|
|
# Pages Microsoft inserts between a successful sign-in and the app. Each entry
|
|
# is (text identifying the page, selectors that skip it, label for the log).
|
|
# Order matters only in that the first matching entry wins.
|
|
#
|
|
# The selectors are deliberately narrow: #idBtn_Back is also the back arrow on
|
|
# the password view, so these are only ever clicked when the page text confirms
|
|
# which interrupt we are looking at.
|
|
_LOGIN_INTERRUPTS: tuple[tuple[str, tuple[str, ...], str], ...] = (
|
|
(
|
|
"keep your account secure",
|
|
(
|
|
"#iShowSkip",
|
|
"a:has-text('Not now')",
|
|
"button:has-text('Not now')",
|
|
"a:has-text('Skip setup')",
|
|
"#iCancel",
|
|
),
|
|
"MFA setup prompt",
|
|
),
|
|
(
|
|
"stay signed in",
|
|
(
|
|
"#idBtn_Back",
|
|
"input[type='submit'][value='No']",
|
|
"button:has-text('No')",
|
|
),
|
|
"Stay signed in?",
|
|
),
|
|
(
|
|
"help us protect your account",
|
|
("#iShowSkip", "a:has-text('Skip for now')", "a:has-text('Not now')"),
|
|
"account protection prompt",
|
|
),
|
|
)
|
|
|
|
|
|
def _login_interrupt_visible(page) -> bool:
|
|
text = _page_text(page, 1500).lower()
|
|
return any(needle in text for needle, _, _ in _LOGIN_INTERRUPTS)
|
|
|
|
|
|
def _skip_login_interrupt(page) -> bool:
|
|
"""Dismiss one post-sign-in interstitial. Returns True when one was clicked."""
|
|
text = _page_text(page, 1500).lower()
|
|
for needle, selectors, label in _LOGIN_INTERRUPTS:
|
|
if needle not in text:
|
|
continue
|
|
for selector in selectors:
|
|
try:
|
|
page.click(selector, timeout=2_000)
|
|
_log.info("Dismissed Microsoft %s via %s", label, selector)
|
|
page.wait_for_timeout(2_500)
|
|
return True
|
|
except Exception:
|
|
continue
|
|
_log.warning(
|
|
"Microsoft %s is on screen but no skip control matched — page says: %s",
|
|
label,
|
|
" | ".join(ln.strip() for ln in _page_text(page, 400).splitlines() if ln.strip()),
|
|
)
|
|
return False
|
|
return False
|
|
|
|
|
|
def _token_from_msal_storage(page) -> str | None:
|
|
"""Fallback: read a Graph access token out of the SPA's MSAL localStorage.
|
|
|
|
Used when the Planner app completes login but never fires a Graph request we
|
|
can intercept, so a future change to the SPA's network calls does not break
|
|
the whole download again.
|
|
"""
|
|
try:
|
|
entries = page.evaluate(
|
|
"""() => {
|
|
const out = [];
|
|
for (let i = 0; i < localStorage.length; i++) {
|
|
const k = localStorage.key(i);
|
|
if (!k || !k.toLowerCase().includes('accesstoken')) continue;
|
|
try { out.push(JSON.parse(localStorage.getItem(k))); } catch (e) {}
|
|
}
|
|
return out;
|
|
}"""
|
|
)
|
|
except Exception as exc:
|
|
_log.debug("Could not read localStorage: %s", exc)
|
|
return None
|
|
return _pick_graph_token(entries or [])
|
|
|
|
|
|
def _pick_graph_token(entries: list[dict]) -> str | None:
|
|
"""Choose the MSAL cache entry that carries Graph Planner/Group scopes."""
|
|
wanted = ("tasks.read", "tasks.readwrite", "group.read.all", "groupmember.read.all")
|
|
best: tuple[int, str] | None = None
|
|
for entry in entries:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
secret = entry.get("secret") or ""
|
|
if not secret:
|
|
continue
|
|
target = str(entry.get("target") or "").lower()
|
|
if "graph.microsoft.com" not in target and not any(w in target for w in wanted):
|
|
continue
|
|
score = sum(1 for w in wanted if w in target)
|
|
if best is None or score > best[0]:
|
|
best = (score, secret)
|
|
if best:
|
|
_log.warning("Using Graph token from the MSAL localStorage cache (no Graph request seen)")
|
|
return best[1]
|
|
return None
|
|
|
|
|
|
def _dump_login_failure(page, store: "_TokenStore") -> Path | None:
|
|
"""Save a screenshot, the page HTML and what we saw, next to the run log."""
|
|
try:
|
|
out_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs"
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
stamp = datetime.now().strftime("%H%M%S")
|
|
stem = out_dir / f"login_failure_{stamp}"
|
|
try:
|
|
page.screenshot(path=str(stem.with_suffix(".png")), full_page=True)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
stem.with_suffix(".html").write_text(page.content(), encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
pass
|
|
graph_lines = [f" {u}" for u in sorted(store.graph_urls)] or [" none"]
|
|
summary = [
|
|
f"url : {page.url}",
|
|
f"text : {_page_text(page, 2000)}",
|
|
"",
|
|
f"hosts that received a Bearer: {sorted(store.bearer_hosts) or 'none'}",
|
|
"graph.microsoft.com URLs seen:",
|
|
*graph_lines,
|
|
]
|
|
stem.with_suffix(".txt").write_text("\n".join(summary), encoding="utf-8", errors="replace")
|
|
_log.error("Login failure diagnostics written to %s.*", stem)
|
|
return stem
|
|
except Exception as exc:
|
|
_log.debug("Could not write login diagnostics: %s", exc)
|
|
return None
|
|
|
|
|
|
def _playwright_login_and_capture_token(
|
|
cfg: AppConfig,
|
|
*,
|
|
headless: bool = True,
|
|
timeout_s: float = 180,
|
|
) -> tuple[str, dict[str, str]]:
|
|
"""
|
|
Open Playwright, complete Microsoft login, wait until the Planner app
|
|
fires its first Graph API call, capture the Bearer token, then close.
|
|
Returns (bearer_token, cookies_dict).
|
|
"""
|
|
store = _TokenStore()
|
|
|
|
proxy_cfg: dict | None = None
|
|
if cfg.proxy_ip and cfg.proxy_port:
|
|
proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}"
|
|
proxy_cfg = {"server": proxy_url}
|
|
if cfg.proxy_username and cfg.proxy_password:
|
|
proxy_cfg["username"] = cfg.proxy_username
|
|
proxy_cfg["password"] = str(cfg.proxy_password)
|
|
|
|
chromium_exe = _find_chromium_exe()
|
|
if chromium_exe:
|
|
_log.info("Using Chromium at: %s", chromium_exe)
|
|
else:
|
|
_log.warning("Chromium not found via dynamic search — letting Playwright auto-detect")
|
|
|
|
with sync_playwright() as pw:
|
|
launch_kwargs: dict = {
|
|
"headless": headless,
|
|
"proxy": proxy_cfg,
|
|
}
|
|
if chromium_exe:
|
|
# Bypass Playwright's broken path detection on this machine
|
|
launch_kwargs["executable_path"] = chromium_exe
|
|
browser = pw.chromium.launch(**launch_kwargs)
|
|
ctx = browser.new_context(
|
|
user_agent=(
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
|
)
|
|
)
|
|
# Context-level so popups and secondary pages are observed too.
|
|
ctx.on("request", store.on_request)
|
|
page = ctx.new_page()
|
|
|
|
try:
|
|
planner_url = cfg.planner_url or "https://planner.cloud.microsoft/"
|
|
_log.info("Navigating to Planner for login (%s)", planner_url[:80])
|
|
page.goto(planner_url, timeout=60_000)
|
|
|
|
# Username
|
|
page.wait_for_selector(EMAIL_SELECTOR, timeout=30_000)
|
|
_fill_verified(page, EMAIL_SELECTOR, cfg.onedrive_username, label="username")
|
|
page.locator(EMAIL_SELECTOR).first.press("Enter")
|
|
|
|
# Password
|
|
_submit_password(page, cfg.onedrive_password)
|
|
|
|
# TOTP
|
|
_submit_totp(page, cfg.microsoft_totp_secret)
|
|
|
|
# "Let's keep your account secure", "Stay signed in?", etc.
|
|
for _ in range(4):
|
|
if not _skip_login_interrupt(page):
|
|
break
|
|
|
|
# Wait until Planner fires a Graph API call so we capture the token
|
|
deadline = time.monotonic() + timeout_s
|
|
while time.monotonic() < deadline and not store.token:
|
|
try:
|
|
page.wait_for_selector(
|
|
"[data-testid*='planner'], [class*='planner'], [role='main']",
|
|
timeout=4_000,
|
|
)
|
|
except Exception:
|
|
pass
|
|
if not store.token:
|
|
_skip_login_interrupt(page)
|
|
time.sleep(1)
|
|
|
|
if not store.token:
|
|
store.token = _token_from_msal_storage(page)
|
|
|
|
if not store.token:
|
|
_dump_login_failure(page, store)
|
|
raise RuntimeError(
|
|
"Could not capture Bearer token — Microsoft login may have failed "
|
|
f"or Planner did not fire any Graph API requests within {timeout_s}s. "
|
|
f"Last URL: {page.url[:120]}. "
|
|
f"Graph URLs seen: {sorted(store.graph_urls) or 'none'}. "
|
|
"See the login_failure_*.png/.html/.txt files next to this log."
|
|
)
|
|
|
|
cookies = {c["name"]: c["value"] for c in ctx.cookies()}
|
|
finally:
|
|
browser.close()
|
|
|
|
_log.info(
|
|
"Login complete — token captured (%s, age %.1fs), %d cookies harvested",
|
|
"Planner-scoped" if store.preferred else "fallback scope",
|
|
store.age_s, len(cookies),
|
|
)
|
|
return store.token, cookies
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# httpx client
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_client(cfg: AppConfig, token: str) -> httpx.Client:
|
|
proxy_url: str | None = None
|
|
if cfg.proxy_ip and cfg.proxy_port:
|
|
proxy_url = f"http://{cfg.proxy_ip}:{cfg.proxy_port}"
|
|
if cfg.proxy_username and cfg.proxy_password:
|
|
proxy_url = (
|
|
f"http://{cfg.proxy_username}:{cfg.proxy_password}"
|
|
f"@{cfg.proxy_ip}:{cfg.proxy_port}"
|
|
)
|
|
|
|
return httpx.Client(
|
|
headers={
|
|
"Authorization": f"Bearer {token}",
|
|
"Accept": "application/json",
|
|
},
|
|
timeout=60.0,
|
|
follow_redirects=True,
|
|
proxy=proxy_url, # httpx >= 0.28: singular `proxy=`, not `proxies=`
|
|
)
|
|
|
|
|
|
def _login_with_retry(
|
|
cfg: AppConfig, *, headless: bool = True, attempts: int = 2
|
|
) -> tuple[str, dict[str, str]]:
|
|
"""Run the Playwright login, retrying once on a transient sign-in hiccup."""
|
|
last: Exception | None = None
|
|
for attempt in range(1, attempts + 1):
|
|
try:
|
|
return _playwright_login_and_capture_token(cfg, headless=headless)
|
|
except Exception as exc:
|
|
last = exc
|
|
_log.error("Login attempt %d/%d failed: %s", attempt, attempts, exc)
|
|
if attempt < attempts:
|
|
time.sleep(10)
|
|
assert last is not None
|
|
raise last
|
|
|
|
|
|
def _refresh_token(client: httpx.Client, cfg: AppConfig) -> None:
|
|
"""Re-login via Playwright and update the client's Authorization header in place."""
|
|
_log.warning("Token expired (401) — re-harvesting via Playwright...")
|
|
new_token, _ = _login_with_retry(cfg, headless=True)
|
|
client.headers["Authorization"] = f"Bearer {new_token}"
|
|
_log.info("Token refreshed.")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Graph API helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _get(client: httpx.Client, path: str, cfg: AppConfig, **params) -> dict:
|
|
"""GET a Graph API path, refreshing token once on 401."""
|
|
url = f"{GRAPH_BASE}/{path.lstrip('/')}"
|
|
for attempt in range(2):
|
|
r = client.get(url, params=params or None)
|
|
if r.status_code == 401 and attempt == 0:
|
|
_refresh_token(client, cfg)
|
|
continue
|
|
r.raise_for_status()
|
|
return r.json()
|
|
raise RuntimeError(f"Unreachable: {url}") # pragma: no cover
|
|
|
|
|
|
def _get_paged(client: httpx.Client, path: str, cfg: AppConfig, **params) -> list[dict]:
|
|
"""Follow @odata.nextLink pagination until exhausted."""
|
|
results: list[dict] = []
|
|
url = f"{GRAPH_BASE}/{path.lstrip('/')}"
|
|
first = True
|
|
while url:
|
|
for attempt in range(2):
|
|
if first:
|
|
r = client.get(url, params=params or None)
|
|
else:
|
|
r = client.get(url)
|
|
if r.status_code == 401 and attempt == 0:
|
|
_refresh_token(client, cfg)
|
|
continue
|
|
r.raise_for_status()
|
|
break
|
|
data = r.json()
|
|
results.extend(data.get("value", []))
|
|
url = data.get("@odata.nextLink")
|
|
first = False
|
|
return results
|
|
|
|
|
|
def _update_task_status(
|
|
client: httpx.Client,
|
|
task_id: str,
|
|
etag: str,
|
|
percent_complete: int,
|
|
cfg: AppConfig,
|
|
) -> bool:
|
|
"""PATCH a Planner task's percentComplete (0=Not Started, 50=In Progress, 100=Complete).
|
|
|
|
Planner requires the If-Match header with the task's current ETag for
|
|
optimistic concurrency — without it the API returns 428 Precondition Required.
|
|
Returns True on success, False on any error (warning is logged but run continues).
|
|
"""
|
|
url = f"{GRAPH_BASE}/planner/tasks/{task_id}"
|
|
req_headers = {
|
|
"If-Match": etag,
|
|
"Content-Type": "application/json",
|
|
}
|
|
body = {"percentComplete": percent_complete}
|
|
try:
|
|
for attempt in range(2):
|
|
r = client.patch(url, json=body, headers=req_headers)
|
|
if r.status_code == 401 and attempt == 0:
|
|
_refresh_token(client, cfg)
|
|
continue
|
|
r.raise_for_status()
|
|
return True
|
|
except httpx.HTTPStatusError as exc:
|
|
_log.warning(
|
|
" Status update failed for task %s: HTTP %s — %s",
|
|
task_id, exc.response.status_code, exc.response.text[:200],
|
|
)
|
|
except Exception as exc:
|
|
_log.warning(" Status update failed for task %s: %s", task_id, exc)
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plan discovery via group memberships
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _discover_all_plans(client: httpx.Client, cfg: AppConfig) -> list[dict]:
|
|
"""
|
|
Return every Planner plan visible to the user through their Microsoft 365 group
|
|
memberships. Each dict has at minimum: id, title, _group_name.
|
|
"""
|
|
_log.info("Fetching user's group memberships...")
|
|
groups = _get_paged(
|
|
client,
|
|
"me/memberOf/microsoft.graph.group",
|
|
cfg,
|
|
**{"$select": "id,displayName", "$top": "999"},
|
|
)
|
|
_log.info("%d groups found", len(groups))
|
|
|
|
all_plans: list[dict] = []
|
|
for group in groups:
|
|
gid = group.get("id", "")
|
|
if not gid:
|
|
continue
|
|
try:
|
|
plans = _get_paged(
|
|
client,
|
|
f"groups/{gid}/planner/plans",
|
|
cfg,
|
|
**{"$select": "id,title"},
|
|
)
|
|
for p in plans:
|
|
p["_group_id"] = gid
|
|
p["_group_name"] = group.get("displayName", "")
|
|
all_plans.extend(plans)
|
|
except httpx.HTTPStatusError as exc:
|
|
if exc.response.status_code in (403, 404):
|
|
pass # group has no Planner or access denied
|
|
else:
|
|
_log.warning("plans for group %s: HTTP %s", gid, exc.response.status_code)
|
|
time.sleep(0.1) # gentle rate-limit courtesy
|
|
|
|
_log.info("Total plans discovered: %d", len(all_plans))
|
|
return all_plans
|
|
|
|
|
|
def _match_plan(plans: list[dict], facility_name: str) -> dict | None:
|
|
"""Match a facility name to a Planner plan title (exact → prefix → substring)."""
|
|
fac = facility_name.strip().lower()
|
|
|
|
# Pass 1: exact match
|
|
for p in plans:
|
|
if (p.get("title") or "").strip().lower() == fac:
|
|
return p
|
|
|
|
# Pass 2: one string is a prefix of the other (handles trailing words)
|
|
for p in plans:
|
|
title = (p.get("title") or "").strip().lower()
|
|
if title.startswith(fac) or fac.startswith(title):
|
|
_log.warning("Prefix plan match: '%s' -> '%s'", facility_name, p.get("title"))
|
|
return p
|
|
|
|
# Pass 3: substring match — only for names long enough to be unambiguous
|
|
# (short names like "Hills" would falsely match "Arbor Hills")
|
|
if len(fac) >= 8:
|
|
for p in plans:
|
|
title = (p.get("title") or "").strip().lower()
|
|
if fac in title:
|
|
_log.warning("Substring plan match: '%s' -> '%s'", facility_name, p.get("title"))
|
|
return p
|
|
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tasks + file references
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _not_started_tasks(client: httpx.Client, plan_id: str, cfg: AppConfig) -> list[dict]:
|
|
tasks = _get_paged(
|
|
client,
|
|
f"planner/plans/{plan_id}/tasks",
|
|
cfg,
|
|
**{"$select": "id,title,percentComplete,assignments"},
|
|
)
|
|
ns = [t for t in tasks if (t.get("percentComplete") or 0) == 0]
|
|
_log.info("Plan %s: %d tasks total, %d not started", plan_id, len(tasks), len(ns))
|
|
return ns
|
|
|
|
|
|
def _task_assignee_ids(task: dict) -> list[str]:
|
|
"""User IDs a task is assigned to (the Planner grid's 'Assigned to' column)."""
|
|
return [uid for uid in (task.get("assignments") or {}) if uid]
|
|
|
|
|
|
def _resolve_user_names(
|
|
client: httpx.Client,
|
|
user_ids: list[str],
|
|
cfg: AppConfig,
|
|
cache: dict[str, str],
|
|
) -> list[str]:
|
|
"""Display names for assignee IDs, cached across the run.
|
|
|
|
A lookup failure is never fatal — the ID is shown instead so the summary
|
|
still records who the task was assigned to.
|
|
"""
|
|
names: list[str] = []
|
|
for uid in user_ids:
|
|
if uid not in cache:
|
|
try:
|
|
user = _get(client, f"users/{uid}", cfg, **{"$select": "displayName"})
|
|
cache[uid] = user.get("displayName") or uid
|
|
except Exception as exc:
|
|
_log.debug("user lookup %s failed: %s", uid, exc)
|
|
cache[uid] = uid
|
|
names.append(cache[uid])
|
|
return names
|
|
|
|
|
|
def _task_file_urls(client: httpx.Client, task_id: str, cfg: AppConfig) -> list[str]:
|
|
"""Return SharePoint file URLs from a task's references (details endpoint)."""
|
|
try:
|
|
details = _get(client, f"planner/tasks/{task_id}/details", cfg)
|
|
except httpx.HTTPStatusError as exc:
|
|
_log.warning("task details %s: HTTP %s", task_id, exc.response.status_code)
|
|
return []
|
|
|
|
refs: dict = details.get("references", {}) or {}
|
|
urls: list[str] = []
|
|
for encoded_key in refs:
|
|
try:
|
|
url = unquote(encoded_key)
|
|
if url.startswith("http"):
|
|
urls.append(url)
|
|
except Exception:
|
|
pass
|
|
return urls
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SharePoint URL resolution + download
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _encode_share_url(url: str) -> str:
|
|
"""Convert a SharePoint URL to Graph's u!{base64url} share-id format."""
|
|
b64 = base64.urlsafe_b64encode(url.encode("utf-8")).rstrip(b"=").decode()
|
|
return f"u!{b64}"
|
|
|
|
|
|
def _resolve_sp_url(
|
|
client: httpx.Client, sp_url: str, cfg: AppConfig
|
|
) -> tuple[str | None, str | None]:
|
|
"""
|
|
Resolve a SharePoint URL to (pre-auth download URL, filename).
|
|
Uses the Graph shares endpoint — same call visible in DevTools.
|
|
"""
|
|
share_id = _encode_share_url(sp_url)
|
|
try:
|
|
# Do NOT use $select here: @microsoft.graph.downloadUrl is an OData
|
|
# annotation and is only present in the full (unfiltered) response.
|
|
# Requesting it via $select causes Graph to return 200 with no annotation.
|
|
item = _get(client, f"shares/{share_id}/driveItem", cfg)
|
|
dl_url = item.get("@microsoft.graph.downloadUrl")
|
|
name = item.get("name") or Path(urlparse(sp_url).path).name or "download.pdf"
|
|
return dl_url, name
|
|
except httpx.HTTPStatusError as exc:
|
|
_log.warning("shares resolve %s…: HTTP %s", sp_url[:60], exc.response.status_code)
|
|
return None, None
|
|
|
|
|
|
def _download_file(
|
|
download_url: str,
|
|
dest_folder: Path,
|
|
filename: str,
|
|
) -> Path | None:
|
|
"""Stream-download a pre-authenticated Azure Blob SAS URL to dest_folder/filename.
|
|
|
|
If the file already exists (from a previous interrupted run) it is kept as-is
|
|
and its path is returned — no re-download, no duplicate _2/_3 copies.
|
|
|
|
Uses a fresh proxy-free client: the SAS URL authenticates via query-string
|
|
and must not be routed through the corporate proxy.
|
|
"""
|
|
dest_folder.mkdir(parents=True, exist_ok=True)
|
|
dest = dest_folder / filename
|
|
|
|
# Skip re-download if file already exists and is non-empty
|
|
if dest.exists() and dest.stat().st_size > 0:
|
|
_log.info(" Already exists, skipping download: %s (%d bytes)",
|
|
dest.name, dest.stat().st_size)
|
|
return dest
|
|
|
|
try:
|
|
# Proxy-free: Azure Blob SAS URLs authenticate via query-string,
|
|
# routing through the corporate proxy wastes bandwidth and can timeout.
|
|
with httpx.Client(timeout=120.0, follow_redirects=True) as blob_client:
|
|
with blob_client.stream("GET", download_url) as r:
|
|
r.raise_for_status()
|
|
with open(dest, "wb") as f:
|
|
for chunk in r.iter_bytes(65536):
|
|
f.write(chunk)
|
|
_log.info(" Downloaded: %s (%d bytes)", dest.name, dest.stat().st_size)
|
|
return dest
|
|
except Exception as exc:
|
|
_log.error(" Download failed for %s: %s", filename, exc)
|
|
dest.unlink(missing_ok=True)
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Workable file copy (same naming as Selenium bot)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _unique_workable_path(root: Path, base: str, ext: str) -> Path:
|
|
ext = ext if ext.startswith(".") else f".{ext}"
|
|
first = root / f"{base}{ext}"
|
|
if not first.exists():
|
|
return first
|
|
for n in range(2, 1000):
|
|
candidate = root / f"{base}_{n}{ext}"
|
|
if not candidate.exists():
|
|
return candidate
|
|
raise RuntimeError(f"Cannot allocate workable filename for {base}{ext}")
|
|
|
|
|
|
def _build_workable_files(
|
|
*,
|
|
facility_name: str,
|
|
downloaded: list[Path],
|
|
output_root: Path,
|
|
today_iso: str,
|
|
) -> tuple[list[Path], Path]:
|
|
ddmmyyyy = datetime.strptime(today_iso, "%Y-%m-%d").strftime("%d-%m-%Y")
|
|
fac_safe = _sanitize_folder_name(facility_name)
|
|
workable_root = output_root / today_iso / "Workable"
|
|
medicare_root = output_root / today_iso / "Workable_Medicare"
|
|
|
|
if not downloaded:
|
|
return [], workable_root
|
|
|
|
workable_root.mkdir(parents=True, exist_ok=True)
|
|
medicare_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
workable_paths: list[Path] = []
|
|
for i, src in enumerate(downloaded, start=1):
|
|
ext = src.suffix or ".pdf"
|
|
base = (
|
|
f"{ddmmyyyy}_{fac_safe}"
|
|
if len(downloaded) == 1
|
|
else f"{ddmmyyyy}_{fac_safe}_{i}"
|
|
)
|
|
# Check if this workable file already exists (previous interrupted run)
|
|
dst_first = workable_root / f"{base}{ext}"
|
|
if dst_first.exists() and dst_first.stat().st_size > 0:
|
|
_log.info(" Workable already exists, keeping: %s", dst_first.name)
|
|
dst = dst_first
|
|
else:
|
|
dst = _unique_workable_path(workable_root, base, ext)
|
|
shutil.copy2(src, dst)
|
|
workable_paths.append(dst)
|
|
med_dst_first = medicare_root / f"{dst.stem}{ext}"
|
|
if med_dst_first.exists() and med_dst_first.stat().st_size > 0:
|
|
_log.info(" Medicare already exists, keeping: %s", med_dst_first.name)
|
|
else:
|
|
med_dst = _unique_workable_path(medicare_root, dst.stem, ext)
|
|
shutil.copy2(dst, med_dst)
|
|
|
|
_log.info(
|
|
"Workable: %d file(s) -> %s | Medicare -> %s",
|
|
len(workable_paths), workable_root, medicare_root,
|
|
)
|
|
return workable_paths, workable_root
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Summary Excel (same schema as Selenium bot)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _write_summary(
|
|
results: dict[str, dict],
|
|
failures: list[tuple[str, str, str]],
|
|
summary_path: Path,
|
|
run_timestamp: str,
|
|
) -> None:
|
|
failure_names = {f for f, _, _ in failures}
|
|
rows: list[dict] = []
|
|
task_rows: list[dict] = []
|
|
|
|
for facility, payload in results.items():
|
|
if facility in failure_names:
|
|
continue
|
|
file_count = len(payload.get("original") or [])
|
|
tasks_total = int(payload.get("tasks_total") or 0)
|
|
tasks_att = int(payload.get("tasks_with_attachments") or 0)
|
|
tasks_dl = int(payload.get("tasks_with_downloads") or 0)
|
|
tasks_skip = int(payload.get("tasks_skipped_no_attachment") or 0)
|
|
tasks_assigned = int(payload.get("tasks_skipped_assigned") or 0)
|
|
status_changed = int(payload.get("status_changed") or 0)
|
|
task_rows.extend(payload.get("task_rows") or [])
|
|
|
|
if tasks_total <= 0:
|
|
status = "No tasks"
|
|
elif tasks_att <= 0 and tasks_assigned > 0:
|
|
status = "All assigned — skipped"
|
|
elif tasks_att <= 0:
|
|
status = "No attachments"
|
|
elif tasks_dl <= 0 or file_count <= 0:
|
|
status = "Download failed"
|
|
elif tasks_dl >= tasks_att:
|
|
status = "Success"
|
|
else:
|
|
status = "Partial"
|
|
|
|
if file_count <= 0:
|
|
status_changed_label = "N/A"
|
|
elif status_changed >= tasks_dl:
|
|
status_changed_label = f"Yes — {status_changed} task(s) → In Progress"
|
|
elif status_changed > 0:
|
|
status_changed_label = f"Partial — {status_changed}/{tasks_dl} task(s) → In Progress"
|
|
else:
|
|
status_changed_label = "No — Planner PATCH failed"
|
|
|
|
rows.append({
|
|
"Run Timestamp": run_timestamp,
|
|
"Facility Name": facility,
|
|
"Files Downloaded": file_count,
|
|
"Tasks Not Started": tasks_total,
|
|
"Tasks With Attachments": tasks_att,
|
|
"Tasks Skipped (Assigned)": tasks_assigned,
|
|
"Tasks Skipped (No File)": tasks_skip,
|
|
"Status": status,
|
|
"Status Changed": status_changed_label,
|
|
})
|
|
|
|
for facility, err, status in failures:
|
|
rows.append({
|
|
"Run Timestamp": run_timestamp,
|
|
"Facility Name": facility,
|
|
"Files Downloaded": 0,
|
|
"Tasks Not Started": 0,
|
|
"Tasks With Attachments": 0,
|
|
"Tasks Skipped (Assigned)": 0,
|
|
"Tasks Skipped (No File)": 0,
|
|
"Status": status,
|
|
"Status Changed": "N/A",
|
|
})
|
|
|
|
task_columns = [
|
|
"Facility Name", "Task Name", "Assigned To",
|
|
"Attachments", "Downloaded", "Reason",
|
|
]
|
|
tasks_df = pd.DataFrame(task_rows, columns=task_columns)
|
|
|
|
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with pd.ExcelWriter(summary_path, engine="openpyxl") as writer:
|
|
pd.DataFrame(rows).to_excel(writer, index=False, sheet_name="Download")
|
|
tasks_df.to_excel(writer, index=False, sheet_name="Tasks")
|
|
_log.info(
|
|
"Summary -> %s (%d facility row(s), %d task row(s))",
|
|
summary_path, len(rows), len(tasks_df),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-facility processing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _process_facility(
|
|
client: httpx.Client,
|
|
cfg: AppConfig,
|
|
plans: list[dict],
|
|
facility_name: str,
|
|
dest_folder: Path,
|
|
) -> dict:
|
|
plan = _match_plan(plans, facility_name)
|
|
if plan is None:
|
|
raise ValueError(f"No Planner plan found matching '{facility_name}'")
|
|
|
|
plan_id = plan["id"]
|
|
_log.info(
|
|
"%s -> plan '%s' (group: %s)",
|
|
facility_name, plan.get("title"), plan.get("_group_name", ""),
|
|
)
|
|
|
|
# ── Only Not Started tasks (percentComplete == 0) are ever processed.
|
|
# ── Within those, only tasks that have at least one file attachment are
|
|
# downloaded. Tasks that are In Progress, Completed, or have no
|
|
# attachments are silently skipped.
|
|
tasks = _not_started_tasks(client, plan_id, cfg)
|
|
tasks_total = len(tasks)
|
|
tasks_with_att = 0
|
|
tasks_with_dl = 0
|
|
tasks_no_att = 0 # Not Started but no file attachment → skip
|
|
tasks_assigned = 0 # Not Started but already assigned to someone → skip
|
|
status_changed = 0
|
|
downloaded: list[Path] = []
|
|
task_rows: list[dict] = []
|
|
name_cache: dict[str, str] = {}
|
|
|
|
_log.info(
|
|
" Processing ONLY unassigned Not Started tasks with attachments "
|
|
"(%d Not Started found)",
|
|
tasks_total,
|
|
)
|
|
|
|
for task in tasks:
|
|
tid = task["id"]
|
|
title = task.get("title", "")
|
|
assignee_ids = _task_assignee_ids(task)
|
|
|
|
# A task someone has already picked up is theirs to work — never
|
|
# download it and never touch its status. Checked before the
|
|
# attachment lookup so an assigned task costs no extra Graph call.
|
|
if assignee_ids:
|
|
names = _resolve_user_names(client, assignee_ids, cfg, name_cache)
|
|
assigned_to = ", ".join(names)
|
|
_log.info(" SKIP '%s': assigned to %s", title[:60], assigned_to)
|
|
tasks_assigned += 1
|
|
task_rows.append({
|
|
"Facility Name": facility_name,
|
|
"Task Name": title,
|
|
"Assigned To": assigned_to,
|
|
"Attachments": "",
|
|
"Downloaded": "No",
|
|
"Reason": f"Skipped — assigned to {assigned_to}",
|
|
})
|
|
continue
|
|
|
|
file_urls = _task_file_urls(client, tid, cfg)
|
|
|
|
if not file_urls:
|
|
# Not Started but no attachment — skip entirely
|
|
_log.info(" SKIP '%s': Not Started but no attachment", title[:60])
|
|
tasks_no_att += 1
|
|
task_rows.append({
|
|
"Facility Name": facility_name,
|
|
"Task Name": title,
|
|
"Assigned To": "",
|
|
"Attachments": 0,
|
|
"Downloaded": "No",
|
|
"Reason": "Skipped — no attachment",
|
|
})
|
|
continue
|
|
|
|
# Unassigned, Not Started, WITH attachment(s) — download ALL of them
|
|
tasks_with_att += 1
|
|
total_urls = len(file_urls)
|
|
task_dl = 0
|
|
_log.info(" PROCESS '%s': %d attachment(s)", title[:60], total_urls)
|
|
|
|
for att_idx, sp_url in enumerate(file_urls, start=1):
|
|
dl_url, filename = _resolve_sp_url(client, sp_url, cfg)
|
|
if not dl_url or not filename:
|
|
_log.warning(
|
|
" [%d/%d] Could not resolve attachment URL: %.80s",
|
|
att_idx, total_urls, sp_url,
|
|
)
|
|
continue
|
|
_log.info(" [%d/%d] Downloading: %s", att_idx, total_urls, filename)
|
|
path = _download_file(dl_url, dest_folder, filename)
|
|
if path:
|
|
downloaded.append(path)
|
|
task_dl += 1
|
|
|
|
if task_dl == total_urls:
|
|
reason = f"Downloaded {task_dl}/{total_urls} attachment(s)"
|
|
elif task_dl > 0:
|
|
reason = f"Partial — {task_dl}/{total_urls} attachment(s) downloaded"
|
|
else:
|
|
reason = f"Download failed — 0/{total_urls} attachment(s)"
|
|
task_rows.append({
|
|
"Facility Name": facility_name,
|
|
"Task Name": title,
|
|
"Assigned To": "",
|
|
"Attachments": total_urls,
|
|
"Downloaded": "Yes" if task_dl else "No",
|
|
"Reason": reason,
|
|
})
|
|
|
|
if task_dl == total_urls:
|
|
# Every attachment downloaded — mark In Progress
|
|
tasks_with_dl += 1
|
|
etag = task.get("@odata.etag", "")
|
|
if etag:
|
|
ok = _update_task_status(client, tid, etag, 50, cfg)
|
|
if ok:
|
|
_log.info(
|
|
" Status -> In Progress: '%s' (%d/%d files downloaded)",
|
|
title[:60], task_dl, total_urls,
|
|
)
|
|
status_changed += 1
|
|
else:
|
|
_log.warning(" No ETag for task '%s' — status not updated", title[:60])
|
|
elif task_dl > 0:
|
|
# Partial — some failed; don't mark In Progress, leave for retry
|
|
tasks_with_dl += 1
|
|
_log.warning(
|
|
" '%s': partial download %d/%d — status NOT changed (will retry next run)",
|
|
title[:60], task_dl, total_urls,
|
|
)
|
|
else:
|
|
_log.warning(
|
|
" '%s': 0/%d attachments downloaded — status NOT changed",
|
|
title[:60], total_urls,
|
|
)
|
|
|
|
_log.info(
|
|
" Summary: %d Not Started | %d assigned (skipped) | "
|
|
"%d with attachments (processed) | %d no attachment (skipped) | "
|
|
"%d file(s) downloaded",
|
|
tasks_total, tasks_assigned, tasks_with_att, tasks_no_att, len(downloaded),
|
|
)
|
|
|
|
return {
|
|
"original": downloaded,
|
|
"tasks_total": tasks_total,
|
|
"tasks_with_attachments": tasks_with_att,
|
|
"tasks_with_downloads": tasks_with_dl,
|
|
"tasks_skipped_no_attachment": tasks_no_att,
|
|
"tasks_skipped_assigned": tasks_assigned,
|
|
"task_rows": task_rows,
|
|
"status_changed": status_changed,
|
|
"status_via_task_details": 0,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main workflow
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _filter_facilities(
|
|
facilities: list[str],
|
|
*,
|
|
facility_limit: int,
|
|
facility_name: str | None,
|
|
) -> list[str]:
|
|
if facility_name:
|
|
name = facility_name.strip()
|
|
matches = [f for f in facilities if f.lower() == name.lower()]
|
|
if not matches:
|
|
raise ValueError(f"Facility '{name}' not found in Facilities sheet.")
|
|
return matches
|
|
if facility_limit > 0:
|
|
return facilities[:facility_limit]
|
|
return facilities
|
|
|
|
|
|
def run_planner_http_workflow(
|
|
*,
|
|
facility_limit: int = 0,
|
|
facility_name: str | None = None,
|
|
resume: bool = False,
|
|
) -> None:
|
|
run_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
log_path = _setup_logging()
|
|
_log.info("=" * 66)
|
|
_log.info(" MISF Planner HTTP Download — Start [%s]", run_timestamp)
|
|
_log.info(" Run date : %s", TODAY_ISO_DATE)
|
|
_log.info(" Log file : %s", log_path)
|
|
_log.info("=" * 66)
|
|
|
|
cfg = read_app_config()
|
|
output_root = PROJECT_ROOT / "Output"
|
|
original_root = output_root / TODAY_ISO_DATE / "Original"
|
|
checkpoint_path = output_root / TODAY_ISO_DATE / "checkpoint_http.json"
|
|
|
|
facilities = read_facility_list()
|
|
facilities = _filter_facilities(
|
|
facilities,
|
|
facility_limit=facility_limit,
|
|
facility_name=facility_name,
|
|
)
|
|
|
|
# Resume state
|
|
start_index = 0
|
|
completed: set[str] = set()
|
|
if resume:
|
|
ckpt = load_checkpoint(checkpoint_path)
|
|
start_index = int(ckpt.get("last_completed_index", 0) or 0)
|
|
completed = {str(x) for x in (ckpt.get("completed_facilities", []) or [])}
|
|
_log.info("Resuming from index %d, %d already done", start_index, len(completed))
|
|
else:
|
|
clear_checkpoint(checkpoint_path)
|
|
|
|
# Validate credentials before launching the browser — fail fast with a clear message
|
|
missing = missing_onedrive_credentials(
|
|
cfg.onedrive_username,
|
|
cfg.onedrive_password,
|
|
cfg.microsoft_totp_secret,
|
|
)
|
|
if missing:
|
|
raise ValueError(credentials_setup_message(missing))
|
|
validate_totp_secret(cfg.microsoft_totp_secret)
|
|
|
|
# --- Ensure browser is installed (auto-downloads on first run) ---
|
|
_ensure_playwright_browsers()
|
|
|
|
# --- Login (Playwright, headless) ---
|
|
_log.info("Launching Playwright for Microsoft login...")
|
|
token, _cookies = _login_with_retry(cfg, headless=True)
|
|
client = _build_client(cfg, token)
|
|
|
|
# --- Discover all plans once ---
|
|
_log.info("Discovering Planner plans via Graph API...")
|
|
plans = _discover_all_plans(client, cfg)
|
|
_log.info("Plan discovery done: %d plans available", len(plans))
|
|
|
|
results: dict[str, dict] = {}
|
|
failures: list[tuple[str, str, str]] = []
|
|
total_files = 0
|
|
|
|
for idx, fac in enumerate(facilities, start=1):
|
|
if idx < start_index or fac in completed:
|
|
_log.info("Skipping %s (already processed)", fac)
|
|
continue
|
|
|
|
_log.info("-" * 50)
|
|
_log.info("Facility %d / %d: %s", idx, len(facilities), fac)
|
|
dest = original_root / _sanitize_folder_name(fac)
|
|
|
|
try:
|
|
result = _process_facility(client, cfg, plans, fac, dest)
|
|
downloaded = result["original"]
|
|
file_count = len(downloaded)
|
|
|
|
if downloaded:
|
|
workable_paths, workable_root = _build_workable_files(
|
|
facility_name=fac,
|
|
downloaded=downloaded,
|
|
output_root=output_root,
|
|
today_iso=TODAY_ISO_DATE,
|
|
)
|
|
else:
|
|
workable_paths = []
|
|
workable_root = output_root / TODAY_ISO_DATE / "Workable"
|
|
if dest.exists() and not any(dest.iterdir()):
|
|
dest.rmdir()
|
|
|
|
results[fac] = {**result, "workable": workable_paths}
|
|
total_files += file_count
|
|
completed.add(fac)
|
|
save_checkpoint(
|
|
checkpoint_path,
|
|
last_completed_index=idx,
|
|
facility_name=fac,
|
|
completed_facilities=sorted(completed),
|
|
)
|
|
_log.info("%s: %d file(s) -> %s", fac, file_count, workable_root)
|
|
|
|
except Exception as exc:
|
|
err_msg = str(exc)
|
|
_log.error("%s FAILED: %s", fac, err_msg)
|
|
status = (
|
|
"Not appear"
|
|
if "no planner plan found" in err_msg.lower()
|
|
else "FAILED"
|
|
)
|
|
failures.append((fac, err_msg, status))
|
|
results[fac] = {
|
|
"original": [], "workable": [], "comment": err_msg[:500],
|
|
"tasks_total": 0, "tasks_with_attachments": 0,
|
|
"tasks_with_downloads": 0, "tasks_skipped_no_attachment": 0,
|
|
"tasks_skipped_assigned": 0, "task_rows": [],
|
|
"status_changed": 0, "status_via_task_details": 0,
|
|
}
|
|
save_checkpoint(
|
|
checkpoint_path,
|
|
last_completed_index=idx,
|
|
facility_name=fac,
|
|
completed_facilities=sorted(completed),
|
|
)
|
|
|
|
summary_path = (
|
|
output_root / TODAY_ISO_DATE / "Summary"
|
|
/ f"Download_Summary_HTTP_{TODAY_ISO_DATE}.xlsx"
|
|
)
|
|
_write_summary(results, failures, summary_path, run_timestamp)
|
|
clear_checkpoint(checkpoint_path)
|
|
|
|
_log.info("=" * 66)
|
|
_log.info(
|
|
" COMPLETE — %d file(s) downloaded, %d failure(s)",
|
|
total_files, len(failures),
|
|
)
|
|
_log.info("=" * 66)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_arg_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(
|
|
description="Download face-sheet PDFs from Planner via Microsoft Graph API (no Selenium)."
|
|
)
|
|
p.add_argument(
|
|
"--limit", type=int, default=0,
|
|
help="Process only the first N facilities (0 = all).",
|
|
)
|
|
p.add_argument(
|
|
"--facility", type=str, default="",
|
|
help="Process a single facility by exact name.",
|
|
)
|
|
p.add_argument(
|
|
"--resume", action="store_true",
|
|
help="Resume from checkpoint.",
|
|
)
|
|
return p
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_frozen = getattr(sys, "frozen", False) # True when running as .exe
|
|
_force_utf8_console()
|
|
|
|
args = _build_arg_parser().parse_args()
|
|
exit_code = 0
|
|
try:
|
|
run_planner_http_workflow(
|
|
facility_limit=max(0, args.limit),
|
|
facility_name=args.facility.strip() or None,
|
|
resume=args.resume,
|
|
)
|
|
except KeyboardInterrupt:
|
|
print("\nCancelled.")
|
|
exit_code = 130
|
|
except Exception:
|
|
traceback.print_exc()
|
|
exit_code = 1
|
|
|
|
# On error only, and only for a real console: keep the window open so the
|
|
# user can read what went wrong. Scheduled runs have no stdin and would
|
|
# otherwise die with an EOFError that buries the actual failure.
|
|
if _frozen and exit_code != 0 and sys.stdin is not None and sys.stdin.isatty():
|
|
input("\n[X] An error occurred (see above). Press Enter to close...")
|
|
|
|
sys.exit(exit_code)
|