Fix Planner login: password field submitted empty since 2026-07-31

Microsoft changed the converged sign-in page to pre-render the password
view, so input#i0118 is present and visible while the email view is still
on screen. wait_for_selector() for the password field therefore returned
immediately, the fill landed mid view-switch and was wiped, and the Sign in
click submitted an empty box ("Please enter your password."). Planner never
loaded, no Graph request fired, and the run died with "Could not capture
Bearer token" on every machine from 2026-07-31 onwards.

Login flow:
- _fill_verified(): wait for the field to be interactive, fill, and read the
  value back before submitting; retry when it does not stick.
- _submit_password()/_submit_totp(): confirm the page actually advanced, and
  raise a specific error naming Microsoft's message when a credential is
  rejected instead of timing out silently.
- _skip_login_interrupt(): dismiss the post-sign-in interstitials, including
  "Let's keep your account secure", which was also never handled. The old
  "Stay signed in?" handler only matched input#idBtn_Back and Microsoft now
  renders that control as a <button>.

Token capture:
- Listen at context level so popups are observed.
- _TokenStore falls back to any graph.microsoft.com Bearer if no Planner-scoped
  call appears, and records every Graph URL seen for diagnosis.
- MSAL localStorage fallback when no Graph request is intercepted at all.
- On failure, dump a screenshot, the page HTML and the request summary next to
  the run log so the next regression does not need a code change to diagnose.
- Login timeout 120s -> 180s, with one retry of the whole login.

Also: force UTF-8 on stdout/stderr (the error banner crashed with
UnicodeEncodeError on a cp1252 console, hiding the real error), and only
prompt "Press Enter to close" when stdin is a TTY so scheduled runs do not
fail with EOFError.

Build: onedrive_planner_http.spec was gitignored and missing from the repo, so
build_exe.bat could not run; it is now tracked. requirements.txt was missing
httpx, playwright and python-dotenv.

Verified: scripts/verify_login.py completes login, token capture and read-only
Graph access (63 plans discovered). 103 tests pass.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.gitignore

@@ -4,6 +4,11 @@ dist/
 _internal/
 *.exe
 *.spec
+# ...except the build recipe itself — build_exe.bat cannot build without it.
+!onedrive_planner_http.spec
+
+# Diagnostics written by scripts/diag_login.py
+diag/
 
 # Chrome binary
 chrome-win/

onedrive_planner_http.py

@@ -160,7 +160,22 @@ def _ensure_playwright_browsers() -> None:
 # Logging setup
 # ---------------------------------------------------------------------------
 
+def _force_utf8_console() -> None:
+    """Make stdout/stderr tolerate non-ASCII.
+
+    Windows consoles default to cp1252, and both the log format and the error
+    banner contain characters outside it — printing them raises
+    UnicodeEncodeError and hides the real error from the operator.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+
+
 def _setup_logging() -> Path:
+    _force_utf8_console()
     log_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs"
     log_dir.mkdir(parents=True, exist_ok=True)
     log_path = log_dir / "planner_http.log"
@@ -191,12 +206,38 @@ def _setup_logging() -> Path:
 # Auth — Playwright login, Bearer token capture
 # ---------------------------------------------------------------------------
 
-class _TokenStore:
-    """Collects the latest Bearer token from intercepted Playwright requests.
+# Microsoft's converged sign-in page keeps the email, password and OTC inputs
+# in the same document, so a bare wait_for_selector() for a later step returns
+# immediately while an earlier view is still on screen. Every fill therefore has
+# to be verified before it is submitted — see _fill_verified().
+EMAIL_SELECTOR = "input[name='loginfmt'], #i0116, input[type='email']"
+PASSWORD_SELECTOR = "input[name='passwd'], #i0118, input[type='password']"
+OTP_SELECTOR = (
+    "input[name='otc'], #idTxtBx_SAOTCC_OTC, input[autocomplete='one-time-code']"
+)
 
-    Only accepts tokens from Planner/group-scoped Graph endpoints to avoid
-    capturing narrow User.Read tokens that fire during the auth flow (e.g.
-    /me, /me/photo) before the Planner-scoped tokens arrive.
+# Text Microsoft shows when it rejects a credential outright (as opposed to the
+# "you left the box empty" validation message, which we retry).
+_FATAL_LOGIN_MESSAGES = (
+    "your account or password is incorrect",
+    "we couldn't find an account",
+    "that microsoft account doesn't exist",
+    "your account has been locked",
+    "sign-in was blocked",
+    "you can't get there from here",
+    "code you entered isn't working",
+    "verification code is incorrect",
+)
+
+
+class _TokenStore:
+    """Collects Bearer tokens from intercepted Playwright requests.
+
+    Two tiers. Planner/group-scoped Graph endpoints are preferred, because the
+    narrow tokens that fire early in the auth flow (``/me``, ``/me/photo``)
+    lack the Planner scopes we need. If no preferred token has shown up after
+    ``fallback_after_s``, any graph.microsoft.com Bearer is accepted rather
+    than failing the whole run.
     """
 
     # Endpoints whose tokens are guaranteed to carry Planner/Group read scope
@@ -207,42 +248,333 @@ class _TokenStore:
         "/me/transitiveMemberOf",
     )
 
-    def __init__(self) -> None:
+    def __init__(self, *, fallback_after_s: float = 45.0) -> None:
         self.token: str | None = None
+        self.preferred: bool = False
+        self.graph_urls: set[str] = set()
+        self.bearer_hosts: set[str] = set()
         self._at: float = 0.0
+        self._start = time.monotonic()
+        self._fallback_after_s = fallback_after_s
 
     def on_request(self, request) -> None:
         url = request.url
+        auth = request.headers.get("authorization", "")
+        if not auth.startswith("Bearer "):
+            return
+        try:
+            self.bearer_hosts.add(url.split("/")[2])
+        except IndexError:
+            pass
         if "graph.microsoft.com" not in url:
             return
-        if not any(p in url for p in self._PLANNER_PATHS):
-            return  # ignore narrow auth-flow tokens (e.g. /me profile)
-        auth = request.headers.get("authorization", "")
-        if auth.startswith("Bearer "):
-            candidate = auth[7:]
-            if candidate != self.token:
+
+        self.graph_urls.add(url.split("?")[0])
+        candidate = auth[7:]
+        is_preferred = any(p in url for p in self._PLANNER_PATHS)
+
+        if is_preferred:
+            if candidate != self.token or not self.preferred:
                 self.token = candidate
+                self.preferred = True
                 self._at = time.monotonic()
-                _log.debug("Planner-scoped token captured from: %.80s", url)
+                _log.debug("Planner-scoped token captured from: %.100s", url)
+            return
+
+        # Non-preferred Graph call. Only fall back to it once we have waited
+        # long enough that a Planner-scoped call clearly isn't coming.
+        if self.preferred:
+            return
+        if time.monotonic() - self._start < self._fallback_after_s:
+            _log.debug("Ignoring non-Planner Graph token (still waiting): %.100s", url)
+            return
+        if candidate != self.token:
+            self.token = candidate
+            self._at = time.monotonic()
+            _log.warning(
+                "No Planner-scoped Graph call seen in %.0fs — falling back to the "
+                "token from %.100s",
+                self._fallback_after_s, url,
+            )
 
     @property
     def age_s(self) -> float:
         return time.monotonic() - self._at if self.token else float("inf")
 
 
+def _is_visible(page, selector: str) -> bool:
+    try:
+        return page.locator(selector).first.is_visible(timeout=1_000)
+    except Exception:
+        return False
+
+
+def _page_text(page, limit: int = 3000) -> str:
+    try:
+        return page.inner_text("body")[:limit]
+    except Exception:
+        return ""
+
+
+def _fatal_login_message(page) -> str | None:
+    """Return Microsoft's error text when it has rejected a credential."""
+    text = _page_text(page).lower()
+    for phrase in _FATAL_LOGIN_MESSAGES:
+        if phrase in text:
+            return phrase
+    return None
+
+
+def _fill_verified(page, selector: str, value: str, *, label: str, attempts: int = 3) -> None:
+    """Fill a login field and confirm the value actually stuck.
+
+    The sign-in page pre-renders later steps, so a fill issued while the view is
+    still switching gets wiped and the form submits empty ("Please enter your
+    password."). Reading the value back is the only reliable guard.
+    """
+    for attempt in range(1, attempts + 1):
+        try:
+            box = page.locator(selector).first
+            # The field can be in the DOM but not yet interactive while the
+            # sign-in page switches views, so wait for it to settle first.
+            box.wait_for(state="visible", timeout=20_000)
+            box.click(timeout=10_000)
+            box.fill("", timeout=10_000)
+            box.fill(value, timeout=10_000)
+            if box.input_value(timeout=5_000) == value:
+                return
+        except Exception as exc:
+            _log.debug("%s fill attempt %d raised: %s", label, attempt, exc)
+        _log.warning(
+            "%s field did not retain its value (attempt %d/%d) — retrying",
+            label, attempt, attempts,
+        )
+        time.sleep(1.5)
+    raise RuntimeError(
+        f"Could not enter the {label} on the Microsoft sign-in page — the field "
+        "kept clearing itself. The sign-in page layout may have changed again."
+    )
+
+
+def _submit_password(page, password: str, *, attempts: int = 3) -> None:
+    """Enter the password and confirm the page moved on to the next step."""
+    page.wait_for_selector(PASSWORD_SELECTOR, timeout=30_000)
+
+    for attempt in range(1, attempts + 1):
+        # Give the view switch time to finish; the box exists in the DOM while
+        # the email view is still showing, so wait_for_selector returns at once.
+        page.wait_for_timeout(1_500)
+        _fill_verified(page, PASSWORD_SELECTOR, password, label="password")
+        page.locator(PASSWORD_SELECTOR).first.press("Enter")
+
+        deadline = time.monotonic() + 25
+        while time.monotonic() < deadline:
+            if _is_visible(page, OTP_SELECTOR) or _login_interrupt_visible(page):
+                _log.info("Password accepted")
+                return
+            if "login.microsoftonline.com" not in page.url and "login.live.com" not in page.url:
+                _log.info("Password accepted (left the sign-in host)")
+                return
+            fatal = _fatal_login_message(page)
+            if fatal:
+                raise RuntimeError(f"Microsoft rejected the sign-in: {fatal!r}")
+            time.sleep(1)
+
+        _log.warning(
+            "Still on the password page after attempt %d/%d — retrying", attempt, attempts
+        )
+
+    raise RuntimeError(
+        "Could not get past the Microsoft password page. Page said: "
+        + " | ".join(ln.strip() for ln in _page_text(page, 600).splitlines() if ln.strip())
+    )
+
+
+def _submit_totp(page, totp_secret: str, *, attempts: int = 2) -> bool:
+    """Enter the authenticator code if prompted. Returns True when submitted."""
+    import pyotp
+
+    if not _is_visible(page, OTP_SELECTOR):
+        _log.debug("No TOTP prompt detected")
+        return False
+
+    for attempt in range(1, attempts + 1):
+        code = pyotp.TOTP(totp_secret).now()
+        _fill_verified(page, OTP_SELECTOR, code, label="authenticator code")
+        page.locator(OTP_SELECTOR).first.press("Enter")
+        _log.info("TOTP submitted (attempt %d)", attempt)
+
+        deadline = time.monotonic() + 25
+        while time.monotonic() < deadline:
+            if _login_interrupt_visible(page):
+                return True
+            if not _is_visible(page, OTP_SELECTOR):
+                return True
+            fatal = _fatal_login_message(page)
+            if fatal:
+                raise RuntimeError(f"Microsoft rejected the authenticator code: {fatal!r}")
+            time.sleep(1)
+
+        _log.warning("Code page still showing — retrying with a fresh code")
+        time.sleep(5)
+
+    raise RuntimeError("Microsoft kept asking for the authenticator code.")
+
+
+# Pages Microsoft inserts between a successful sign-in and the app. Each entry
+# is (text identifying the page, selectors that skip it, label for the log).
+# Order matters only in that the first matching entry wins.
+#
+# The selectors are deliberately narrow: #idBtn_Back is also the back arrow on
+# the password view, so these are only ever clicked when the page text confirms
+# which interrupt we are looking at.
+_LOGIN_INTERRUPTS: tuple[tuple[str, tuple[str, ...], str], ...] = (
+    (
+        "keep your account secure",
+        (
+            "#iShowSkip",
+            "a:has-text('Not now')",
+            "button:has-text('Not now')",
+            "a:has-text('Skip setup')",
+            "#iCancel",
+        ),
+        "MFA setup prompt",
+    ),
+    (
+        "stay signed in",
+        (
+            "#idBtn_Back",
+            "input[type='submit'][value='No']",
+            "button:has-text('No')",
+        ),
+        "Stay signed in?",
+    ),
+    (
+        "help us protect your account",
+        ("#iShowSkip", "a:has-text('Skip for now')", "a:has-text('Not now')"),
+        "account protection prompt",
+    ),
+)
+
+
+def _login_interrupt_visible(page) -> bool:
+    text = _page_text(page, 1500).lower()
+    return any(needle in text for needle, _, _ in _LOGIN_INTERRUPTS)
+
+
+def _skip_login_interrupt(page) -> bool:
+    """Dismiss one post-sign-in interstitial. Returns True when one was clicked."""
+    text = _page_text(page, 1500).lower()
+    for needle, selectors, label in _LOGIN_INTERRUPTS:
+        if needle not in text:
+            continue
+        for selector in selectors:
+            try:
+                page.click(selector, timeout=2_000)
+                _log.info("Dismissed Microsoft %s via %s", label, selector)
+                page.wait_for_timeout(2_500)
+                return True
+            except Exception:
+                continue
+        _log.warning(
+            "Microsoft %s is on screen but no skip control matched — page says: %s",
+            label,
+            " | ".join(ln.strip() for ln in _page_text(page, 400).splitlines() if ln.strip()),
+        )
+        return False
+    return False
+
+
+def _token_from_msal_storage(page) -> str | None:
+    """Fallback: read a Graph access token out of the SPA's MSAL localStorage.
+
+    Used when the Planner app completes login but never fires a Graph request we
+    can intercept, so a future change to the SPA's network calls does not break
+    the whole download again.
+    """
+    try:
+        entries = page.evaluate(
+            """() => {
+                const out = [];
+                for (let i = 0; i < localStorage.length; i++) {
+                    const k = localStorage.key(i);
+                    if (!k || !k.toLowerCase().includes('accesstoken')) continue;
+                    try { out.push(JSON.parse(localStorage.getItem(k))); } catch (e) {}
+                }
+                return out;
+            }"""
+        )
+    except Exception as exc:
+        _log.debug("Could not read localStorage: %s", exc)
+        return None
+    return _pick_graph_token(entries or [])
+
+
+def _pick_graph_token(entries: list[dict]) -> str | None:
+    """Choose the MSAL cache entry that carries Graph Planner/Group scopes."""
+    wanted = ("tasks.read", "tasks.readwrite", "group.read.all", "groupmember.read.all")
+    best: tuple[int, str] | None = None
+    for entry in entries:
+        if not isinstance(entry, dict):
+            continue
+        secret = entry.get("secret") or ""
+        if not secret:
+            continue
+        target = str(entry.get("target") or "").lower()
+        if "graph.microsoft.com" not in target and not any(w in target for w in wanted):
+            continue
+        score = sum(1 for w in wanted if w in target)
+        if best is None or score > best[0]:
+            best = (score, secret)
+    if best:
+        _log.warning("Using Graph token from the MSAL localStorage cache (no Graph request seen)")
+        return best[1]
+    return None
+
+
+def _dump_login_failure(page, store: "_TokenStore") -> Path | None:
+    """Save a screenshot, the page HTML and what we saw, next to the run log."""
+    try:
+        out_dir = PROJECT_ROOT / "Output" / TODAY_ISO_DATE / "logs"
+        out_dir.mkdir(parents=True, exist_ok=True)
+        stamp = datetime.now().strftime("%H%M%S")
+        stem = out_dir / f"login_failure_{stamp}"
+        try:
+            page.screenshot(path=str(stem.with_suffix(".png")), full_page=True)
+        except Exception:
+            pass
+        try:
+            stem.with_suffix(".html").write_text(page.content(), encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+        graph_lines = [f"  {u}" for u in sorted(store.graph_urls)] or ["  none"]
+        summary = [
+            f"url   : {page.url}",
+            f"text  : {_page_text(page, 2000)}",
+            "",
+            f"hosts that received a Bearer: {sorted(store.bearer_hosts) or 'none'}",
+            "graph.microsoft.com URLs seen:",
+            *graph_lines,
+        ]
+        stem.with_suffix(".txt").write_text("\n".join(summary), encoding="utf-8", errors="replace")
+        _log.error("Login failure diagnostics written to %s.*", stem)
+        return stem
+    except Exception as exc:
+        _log.debug("Could not write login diagnostics: %s", exc)
+        return None
+
+
 def _playwright_login_and_capture_token(
     cfg: AppConfig,
     *,
     headless: bool = True,
-    timeout_s: float = 120,
+    timeout_s: float = 180,
 ) -> tuple[str, dict[str, str]]:
     """
     Open Playwright, complete Microsoft login, wait until the Planner app
     fires its first Graph API call, capture the Bearer token, then close.
     Returns (bearer_token, cookies_dict).
     """
-    import pyotp  # available via existing totp_utils dependency
-
     store = _TokenStore()
 
     proxy_cfg: dict | None = None
@@ -275,68 +607,65 @@ def _playwright_login_and_capture_token(
                 "Chrome/124.0.0.0 Safari/537.36"
             )
         )
+        # Context-level so popups and secondary pages are observed too.
+        ctx.on("request", store.on_request)
         page = ctx.new_page()
-        page.on("request", store.on_request)
 
-        planner_url = cfg.planner_url or "https://planner.cloud.microsoft/"
-        _log.info("Navigating to Planner for login (%s)", planner_url[:80])
-        page.goto(planner_url, timeout=60_000)
-
-        # Username
-        page.wait_for_selector("input[type='email'], input[name='loginfmt']", timeout=30_000)
-        page.fill("input[type='email'], input[name='loginfmt']", cfg.onedrive_username)
-        page.click("input[type='submit'], button[type='submit']")
-
-        # Password
-        page.wait_for_selector("input[type='password'], input[name='passwd']", timeout=20_000)
-        page.fill("input[type='password'], input[name='passwd']", cfg.onedrive_password)
-        page.click("input[type='submit'], button[type='submit']")
-
-        # TOTP (optional)
         try:
-            page.wait_for_selector(
-                "input[name='otc'], input[autocomplete='one-time-code']",
-                timeout=10_000,
-            )
-            otp = pyotp.TOTP(cfg.microsoft_totp_secret).now()
-            page.fill("input[name='otc'], input[autocomplete='one-time-code']", otp)
-            page.click("input[type='submit'], button[type='submit']")
-            _log.info("TOTP submitted")
-        except Exception:
-            _log.debug("No TOTP prompt detected")
+            planner_url = cfg.planner_url or "https://planner.cloud.microsoft/"
+            _log.info("Navigating to Planner for login (%s)", planner_url[:80])
+            page.goto(planner_url, timeout=60_000)
 
-        # "Stay signed in?" — click No
-        try:
-            page.wait_for_selector("input[id='idBtn_Back']", timeout=8_000)
-            page.click("input[id='idBtn_Back']")
-        except Exception:
-            pass
+            # Username
+            page.wait_for_selector(EMAIL_SELECTOR, timeout=30_000)
+            _fill_verified(page, EMAIL_SELECTOR, cfg.onedrive_username, label="username")
+            page.locator(EMAIL_SELECTOR).first.press("Enter")
+
+            # Password
+            _submit_password(page, cfg.onedrive_password)
+
+            # TOTP
+            _submit_totp(page, cfg.microsoft_totp_secret)
+
+            # "Let's keep your account secure", "Stay signed in?", etc.
+            for _ in range(4):
+                if not _skip_login_interrupt(page):
+                    break
+
+            # Wait until Planner fires a Graph API call so we capture the token
+            deadline = time.monotonic() + timeout_s
+            while time.monotonic() < deadline and not store.token:
+                try:
+                    page.wait_for_selector(
+                        "[data-testid*='planner'], [class*='planner'], [role='main']",
+                        timeout=4_000,
+                    )
+                except Exception:
+                    pass
+                if not store.token:
+                    _skip_login_interrupt(page)
+                    time.sleep(1)
 
-        # Wait until Planner fires a Graph API call so we capture the token
-        deadline = time.monotonic() + timeout_s
-        while time.monotonic() < deadline and not store.token:
-            try:
-                page.wait_for_selector(
-                    "[data-testid*='planner'], [class*='planner'], [role='main']",
-                    timeout=4_000,
-                )
-            except Exception:
-                pass
             if not store.token:
-                time.sleep(1)
+                store.token = _token_from_msal_storage(page)
 
-        if not store.token:
-            raise RuntimeError(
-                "Could not capture Bearer token — Microsoft login may have failed "
-                "or Planner did not fire any Graph API requests within "
-                f"{timeout_s}s."
-            )
+            if not store.token:
+                _dump_login_failure(page, store)
+                raise RuntimeError(
+                    "Could not capture Bearer token — Microsoft login may have failed "
+                    f"or Planner did not fire any Graph API requests within {timeout_s}s. "
+                    f"Last URL: {page.url[:120]}. "
+                    f"Graph URLs seen: {sorted(store.graph_urls) or 'none'}. "
+                    "See the login_failure_*.png/.html/.txt files next to this log."
+                )
 
-        cookies = {c["name"]: c["value"] for c in ctx.cookies()}
-        browser.close()
+            cookies = {c["name"]: c["value"] for c in ctx.cookies()}
+        finally:
+            browser.close()
 
     _log.info(
-        "Login complete — token captured (age %.1fs), %d cookies harvested",
+        "Login complete — token captured (%s, age %.1fs), %d cookies harvested",
+        "Planner-scoped" if store.preferred else "fallback scope",
         store.age_s, len(cookies),
     )
     return store.token, cookies
@@ -367,10 +696,27 @@ def _build_client(cfg: AppConfig, token: str) -> httpx.Client:
     )
 
 
+def _login_with_retry(
+    cfg: AppConfig, *, headless: bool = True, attempts: int = 2
+) -> tuple[str, dict[str, str]]:
+    """Run the Playwright login, retrying once on a transient sign-in hiccup."""
+    last: Exception | None = None
+    for attempt in range(1, attempts + 1):
+        try:
+            return _playwright_login_and_capture_token(cfg, headless=headless)
+        except Exception as exc:
+            last = exc
+            _log.error("Login attempt %d/%d failed: %s", attempt, attempts, exc)
+            if attempt < attempts:
+                time.sleep(10)
+    assert last is not None
+    raise last
+
+
 def _refresh_token(client: httpx.Client, cfg: AppConfig) -> None:
     """Re-login via Playwright and update the client's Authorization header in place."""
     _log.warning("Token expired (401) — re-harvesting via Playwright...")
-    new_token, _ = _playwright_login_and_capture_token(cfg, headless=True)
+    new_token, _ = _login_with_retry(cfg, headless=True)
     client.headers["Authorization"] = f"Bearer {new_token}"
     _log.info("Token refreshed.")
 
@@ -953,7 +1299,7 @@ def run_planner_http_workflow(
 
     # --- Login (Playwright, headless) ---
     _log.info("Launching Playwright for Microsoft login...")
-    token, _cookies = _playwright_login_and_capture_token(cfg, headless=True)
+    token, _cookies = _login_with_retry(cfg, headless=True)
     client = _build_client(cfg, token)
 
     # --- Discover all plans once ---
@@ -1065,6 +1411,7 @@ def _build_arg_parser() -> argparse.ArgumentParser:
 
 if __name__ == "__main__":
     _frozen = getattr(sys, "frozen", False)  # True when running as .exe
+    _force_utf8_console()
 
     args = _build_arg_parser().parse_args()
     exit_code = 0
@@ -1081,8 +1428,10 @@ if __name__ == "__main__":
         traceback.print_exc()
         exit_code = 1
 
-    if _frozen and exit_code != 0:
-        # On error only: keep the window open so the user can read what went wrong.
-        input("\n✗ An error occurred (see above). Press Enter to close...")
+    # On error only, and only for a real console: keep the window open so the
+    # user can read what went wrong. Scheduled runs have no stdin and would
+    # otherwise die with an EOFError that buries the actual failure.
+    if _frozen and exit_code != 0 and sys.stdin is not None and sys.stdin.isatty():
+        input("\n[X] An error occurred (see above). Press Enter to close...")
 
     sys.exit(exit_code)

onedrive_planner_http.spec

@@ -0,0 +1,83 @@
+# -*- mode: python ; coding: utf-8 -*-
+"""PyInstaller spec for MISF_Planner_Download.exe (onedir build).
+
+Layout produced (matches what is deployed to D:\\MISF_Planner_Download):
+
+    MISF_Planner_Download.exe
+    _internal/
+        src/*.py                 <- imported via sys.path at runtime
+        playwright/driver/...    <- node.exe + cli.js, used to auto-install Chromium
+        <bundled dependencies>
+"""
+from PyInstaller.utils.hooks import collect_data_files, collect_submodules
+
+block_cipher = None
+
+# src/ is shipped as data, not as a package: onedrive_planner_http.py puts it on
+# sys.path at import time (ROOT / "src"), and PROJECT_ROOT resolves next to the
+# .exe so Input/ and Output/ stay outside the bundle.
+datas = [("src", "src")]
+
+# node.exe + package/cli.js — _ensure_playwright_browsers() shells out to these
+# on a machine that has no Chromium yet.
+datas += collect_data_files("playwright")
+
+hiddenimports = [
+    "checkpoint",
+    "config",
+    "logging_setup",
+    "onedrive_planner",
+    "run_log",
+    "totp_utils",
+    "pyotp",
+    "openpyxl",
+    "openpyxl.cell._writer",
+]
+hiddenimports += collect_submodules("playwright")
+
+a = Analysis(
+    ["onedrive_planner_http.py"],
+    pathex=["src"],
+    binaries=[],
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=["tkinter", "matplotlib", "PyQt5", "PySide2", "IPython"],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name="MISF_Planner_Download",
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=False,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=False,
+    upx_exclude=[],
+    name="MISF_Planner_Download",
+)

requirements.txt

@@ -4,3 +4,8 @@ selenium==4.27.1
 webdriver-manager==4.0.2
 pyotp==2.9.0
 pytest==8.3.4
+
+# Planner HTTP download (onedrive_planner_http.py)
+httpx>=0.28
+playwright>=1.49
+python-dotenv>=1.0

scripts/diag_login.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""
+Diagnostic: instrumented Microsoft login for the Planner HTTP download.
+
+Mirrors ``_playwright_login_and_capture_token`` in onedrive_planner_http.py but
+records everything that function is blind to:
+
+  * every request the browser context makes (host, path, whether it carries a
+    Bearer token)                                   -> diag/requests.txt
+  * page URL / title / visible text after each step -> diag/NN_<step>.txt
+  * screenshot + full HTML after each step          -> diag/NN_<step>.png/.html
+  * the SPA's localStorage keys and any MSAL access
+    tokens sitting in it                            -> diag/msal.json
+
+Run:  .venv\\Scripts\\python scripts\\diag_login.py [--headless] [--wait 180]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+SRC = ROOT / "src"
+for p in (str(ROOT), str(SRC)):
+    if p not in sys.path:
+        sys.path.insert(0, p)
+
+import pyotp  # noqa: E402
+from playwright.sync_api import sync_playwright  # noqa: E402
+
+from config import read_app_config  # noqa: E402
+
+DIAG = ROOT / "diag" / datetime.now().strftime("%Y%m%d_%H%M%S")
+
+
+class _Recorder:
+    """Records every request; flags the ones carrying an Authorization header."""
+
+    def __init__(self) -> None:
+        self.rows: list[tuple[float, str, bool, str]] = []
+        self.t0 = time.monotonic()
+        self.bearer_hosts: dict[str, int] = {}
+
+    def on_request(self, request) -> None:
+        auth = request.headers.get("authorization", "")
+        has_bearer = auth.startswith("Bearer ")
+        url = request.url
+        self.rows.append((time.monotonic() - self.t0, url, has_bearer, request.method))
+        if has_bearer:
+            host = url.split("/")[2] if "//" in url else url
+            self.bearer_hosts[host] = self.bearer_hosts.get(host, 0) + 1
+
+    def dump(self, path: Path) -> None:
+        lines = [
+            f"{t:7.1f}s  {'BEARER' if b else '      '}  {m:6s}  {u}"
+            for t, u, b, m in self.rows
+        ]
+        path.write_text("\n".join(lines), encoding="utf-8", errors="replace")
+
+
+_step_no = 0
+
+
+def snapshot(page, label: str) -> None:
+    """Dump URL, title, visible text, HTML and a screenshot for one step."""
+    global _step_no
+    _step_no += 1
+    stem = DIAG / f"{_step_no:02d}_{label}"
+    try:
+        url, title = page.url, page.title()
+    except Exception as exc:  # page closed / navigating
+        url, title = f"<unavailable: {exc}>", ""
+    text = ""
+    try:
+        text = page.inner_text("body")[:4000]
+    except Exception as exc:
+        text = f"<no body text: {exc}>"
+    stem.with_suffix(".txt").write_text(
+        f"URL   : {url}\nTITLE : {title}\n\n--- visible text ---\n{text}\n",
+        encoding="utf-8",
+        errors="replace",
+    )
+    try:
+        stem.with_suffix(".html").write_text(page.content(), encoding="utf-8", errors="replace")
+    except Exception:
+        pass
+    try:
+        page.screenshot(path=str(stem.with_suffix(".png")), full_page=True)
+    except Exception:
+        pass
+    print(f"\n[{_step_no:02d}] {label}")
+    print(f"     url   : {url}")
+    print(f"     title : {title}")
+    first = " | ".join(ln.strip() for ln in text.splitlines() if ln.strip())[:300]
+    print(f"     text  : {first}")
+
+
+def dump_storage(page) -> None:
+    """Dump localStorage keys and any MSAL access-token entries."""
+    try:
+        data = page.evaluate(
+            """() => {
+                const out = {};
+                for (let i = 0; i < localStorage.length; i++) {
+                    const k = localStorage.key(i);
+                    out[k] = localStorage.getItem(k);
+                }
+                return out;
+            }"""
+        )
+    except Exception as exc:
+        print(f"     localStorage unreadable: {exc}")
+        return
+
+    summary = {}
+    for key, raw in data.items():
+        entry = {"len": len(raw or "")}
+        if "accesstoken" in key.lower():
+            try:
+                parsed = json.loads(raw)
+                entry["target"] = parsed.get("target")
+                entry["realm"] = parsed.get("realm")
+                entry["clientId"] = parsed.get("clientId")
+                entry["expiresOn"] = parsed.get("expiresOn")
+                entry["secret_len"] = len(parsed.get("secret", "") or "")
+            except Exception:
+                entry["parse"] = "failed"
+        summary[key] = entry
+
+    (DIAG / "msal.json").write_text(
+        json.dumps(summary, indent=2), encoding="utf-8", errors="replace"
+    )
+    tokens = [k for k in summary if "accesstoken" in k.lower()]
+    print(f"     localStorage keys: {len(summary)}, MSAL access-token entries: {len(tokens)}")
+    for k in tokens:
+        print(f"       - target={summary[k].get('target')!r} secret_len={summary[k].get('secret_len')}")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--headless", action="store_true", help="run headless (default: visible)")
+    ap.add_argument("--wait", type=float, default=180, help="seconds to observe after login")
+    args = ap.parse_args()
+
+    DIAG.mkdir(parents=True, exist_ok=True)
+    print(f"Diagnostics -> {DIAG}")
+
+    cfg = read_app_config()
+
+    proxy_cfg = None
+    if cfg.proxy_ip and cfg.proxy_port:
+        proxy_cfg = {"server": f"http://{cfg.proxy_ip}:{cfg.proxy_port}"}
+        if cfg.proxy_username and cfg.proxy_password:
+            proxy_cfg["username"] = cfg.proxy_username
+            proxy_cfg["password"] = str(cfg.proxy_password)
+    print(f"Proxy: {proxy_cfg}")
+
+    rec = _Recorder()
+
+    with sync_playwright() as pw:
+        browser = pw.chromium.launch(headless=args.headless, proxy=proxy_cfg)
+        ctx = browser.new_context(
+            user_agent=(
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/124.0.0.0 Safari/537.36"
+            )
+        )
+        ctx.on("request", rec.on_request)
+        page = ctx.new_page()
+
+        planner_url = cfg.planner_url or "https://planner.cloud.microsoft/"
+        print(f"Navigating: {planner_url}")
+        page.goto(planner_url, timeout=60_000)
+        snapshot(page, "after_goto")
+
+        # --- username ---
+        try:
+            page.wait_for_selector("input[type='email'], input[name='loginfmt']", timeout=30_000)
+            page.fill("input[type='email'], input[name='loginfmt']", cfg.onedrive_username)
+            page.click("input[type='submit'], button[type='submit']")
+            snapshot(page, "after_username")
+        except Exception as exc:
+            print(f"     username step: {exc}")
+            snapshot(page, "username_failed")
+
+        # --- password ---
+        try:
+            page.wait_for_selector("input[type='password'], input[name='passwd']", timeout=20_000)
+            page.fill("input[type='password'], input[name='passwd']", cfg.onedrive_password)
+            page.click("input[type='submit'], button[type='submit']")
+            time.sleep(3)
+            snapshot(page, "after_password")
+        except Exception as exc:
+            print(f"     password step: {exc}")
+            snapshot(page, "password_failed")
+
+        # --- TOTP ---
+        try:
+            page.wait_for_selector(
+                "input[name='otc'], input[autocomplete='one-time-code']", timeout=15_000
+            )
+            snapshot(page, "otp_prompt")
+            otp = pyotp.TOTP(cfg.microsoft_totp_secret).now()
+            page.fill("input[name='otc'], input[autocomplete='one-time-code']", otp)
+            page.click("input[type='submit'], button[type='submit']")
+            print(f"     TOTP submitted ({otp})")
+            time.sleep(5)
+            snapshot(page, "after_totp")
+        except Exception as exc:
+            print(f"     no TOTP prompt: {exc}")
+            snapshot(page, "no_otp_prompt")
+
+        # --- whatever comes next: DO NOT click anything, just observe ---
+        for wait_s in (5, 10, 20):
+            time.sleep(wait_s)
+            snapshot(page, f"observe_t{wait_s}")
+
+        # Report what the current page offers as clickable controls, so we can
+        # see exactly how the "Stay signed in?" prompt is rendered now.
+        try:
+            controls = page.evaluate(
+                """() => Array.from(
+                    document.querySelectorAll('button, input[type=submit], input[type=button], a[role=button]')
+                ).slice(0, 40).map(e => ({
+                    tag: e.tagName, id: e.id, type: e.type || '',
+                    value: e.value || '', text: (e.innerText || '').trim().slice(0, 60),
+                    visible: !!(e.offsetWidth || e.offsetHeight)
+                }))"""
+            )
+            (DIAG / "controls.json").write_text(
+                json.dumps(controls, indent=2), encoding="utf-8", errors="replace"
+            )
+            print("\n     clickable controls on current page:")
+            for c in controls:
+                if c["visible"]:
+                    print(f"       {c['tag']}#{c['id'] or '-'} type={c['type']!r} "
+                          f"value={c['value']!r} text={c['text']!r}")
+        except Exception as exc:
+            print(f"     control scan failed: {exc}")
+
+        dump_storage(page)
+
+        # --- observe for the remaining budget, watching for a Graph bearer ---
+        deadline = time.monotonic() + args.wait
+        seen_graph = False
+        while time.monotonic() < deadline:
+            graph = [r for r in rec.rows if "graph.microsoft.com" in r[1] and r[2]]
+            if graph and not seen_graph:
+                seen_graph = True
+                print(f"\n     *** first Graph bearer request at {graph[0][0]:.1f}s: {graph[0][1][:140]}")
+            time.sleep(2)
+
+        snapshot(page, "final")
+        dump_storage(page)
+
+        rec.dump(DIAG / "requests.txt")
+
+        print("\n" + "=" * 70)
+        print(f"Total requests recorded : {len(rec.rows)}")
+        print("Hosts that received a Bearer token:")
+        for host, n in sorted(rec.bearer_hosts.items(), key=lambda kv: -kv[1]):
+            print(f"   {n:5d}  {host}")
+        graph_urls = sorted({r[1].split("?")[0] for r in rec.rows if "graph.microsoft.com" in r[1]})
+        print(f"\nDistinct graph.microsoft.com paths seen: {len(graph_urls)}")
+        for u in graph_urls[:40]:
+            print(f"   {u}")
+        print("=" * 70)
+        print(f"Artifacts: {DIAG}")
+
+        browser.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/verify_login.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""
+Read-only end-to-end check of the login fix.
+
+Runs the real Playwright login, then exercises the Graph API with the captured
+token: plan discovery plus a Not-Started task listing for the first facility.
+Downloads nothing and never PATCHes a task, so it is safe to run against
+production Planner.
+
+Run:  .venv\\Scripts\\python scripts\\verify_login.py [--headed]
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+SRC = ROOT / "src"
+for p in (str(ROOT), str(SRC)):
+    if p not in sys.path:
+        sys.path.insert(0, p)
+
+from config import read_app_config, read_facility_list  # noqa: E402
+from onedrive_planner_http import (  # noqa: E402
+    _build_client,
+    _discover_all_plans,
+    _login_with_retry,
+    _match_plan,
+    _not_started_tasks,
+)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--headed", action="store_true", help="show the browser window")
+    args = ap.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG,
+        format="%(asctime)s %(levelname)-8s %(name)s — %(message)s",
+        stream=sys.stdout,
+    )
+
+    cfg = read_app_config()
+    facilities = read_facility_list()
+
+    print("\n--- login ---")
+    token, cookies = _login_with_retry(cfg, headless=not args.headed)
+    print(f"token length : {len(token)}")
+    print(f"cookies      : {len(cookies)}")
+
+    client = _build_client(cfg, token)
+
+    print("\n--- plan discovery ---")
+    plans = _discover_all_plans(client, cfg)
+    print(f"plans visible: {len(plans)}")
+    for p in plans[:5]:
+        print(f"   {p.get('title')!r}  (group {p.get('_group_name')!r})")
+
+    facility = facilities[0]
+    print(f"\n--- read-only task listing for {facility!r} ---")
+    plan = _match_plan(plans, facility)
+    if not plan:
+        print("   no matching plan found for this facility")
+        return 1
+    tasks = _not_started_tasks(client, plan["id"], cfg)
+    print(f"   plan {plan.get('title')!r}: {len(tasks)} Not-Started task(s)")
+    for t in tasks[:5]:
+        print(f"     - {t.get('title')!r}")
+
+    print("\nOK — login, token capture and Graph API access all work.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_http_login.py

@@ -0,0 +1,96 @@
+"""Unit tests for the Planner HTTP login helpers (token capture + MSAL cache)."""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+SRC = ROOT / "src"
+for _p in (str(ROOT), str(SRC)):
+    if _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from onedrive_planner_http import _pick_graph_token, _TokenStore  # noqa: E402
+
+GRAPH_PLANNER = "https://graph.microsoft.com/beta/planner/plans/TestPlanId?$expand=details"
+GRAPH_ME = "https://graph.microsoft.com/v1.0/me/photo/$value"
+NON_GRAPH = "https://planner.cloud.microsoft/webui/api/something"
+
+
+class _Request:
+    def __init__(self, url: str, token: str | None = None) -> None:
+        self.url = url
+        self.headers = {"authorization": f"Bearer {token}"} if token else {}
+
+
+class TestTokenStore:
+    def test_captures_planner_scoped_token(self):
+        store = _TokenStore()
+        store.on_request(_Request(GRAPH_PLANNER, "planner-tok"))
+        assert store.token == "planner-tok"
+        assert store.preferred is True
+
+    def test_ignores_narrow_graph_token_inside_wait_window(self):
+        store = _TokenStore(fallback_after_s=45)
+        store.on_request(_Request(GRAPH_ME, "narrow-tok"))
+        assert store.token is None
+        # ...and records the URL so a failure is diagnosable from the log
+        assert GRAPH_ME.split("?")[0] in store.graph_urls
+
+    def test_falls_back_to_any_graph_token_after_window(self):
+        store = _TokenStore(fallback_after_s=0)
+        store.on_request(_Request(GRAPH_ME, "narrow-tok"))
+        assert store.token == "narrow-tok"
+        assert store.preferred is False
+
+    def test_planner_token_overrides_earlier_fallback(self):
+        store = _TokenStore(fallback_after_s=0)
+        store.on_request(_Request(GRAPH_ME, "narrow-tok"))
+        store.on_request(_Request(GRAPH_PLANNER, "planner-tok"))
+        assert store.token == "planner-tok"
+        assert store.preferred is True
+
+    def test_fallback_does_not_override_planner_token(self):
+        store = _TokenStore(fallback_after_s=0)
+        store.on_request(_Request(GRAPH_PLANNER, "planner-tok"))
+        store.on_request(_Request(GRAPH_ME, "narrow-tok"))
+        assert store.token == "planner-tok"
+
+    def test_non_graph_bearer_is_recorded_but_not_used(self):
+        store = _TokenStore(fallback_after_s=0)
+        store.on_request(_Request(NON_GRAPH, "spa-tok"))
+        assert store.token is None
+        assert "planner.cloud.microsoft" in store.bearer_hosts
+
+    def test_requests_without_authorization_are_ignored(self):
+        store = _TokenStore(fallback_after_s=0)
+        store.on_request(_Request(GRAPH_PLANNER))
+        assert store.token is None
+        assert store.graph_urls == set()
+
+
+class TestPickGraphToken:
+    def test_picks_entry_with_graph_scopes(self):
+        entries = [
+            {"target": "openid profile", "secret": "id-tok"},
+            {"target": "Tasks.Read Group.Read.All", "secret": "graph-tok"},
+        ]
+        assert _pick_graph_token(entries) == "graph-tok"
+
+    def test_prefers_the_entry_with_more_wanted_scopes(self):
+        entries = [
+            {"target": "https://graph.microsoft.com/Tasks.Read", "secret": "one"},
+            {"target": "Tasks.Read Group.Read.All GroupMember.Read.All", "secret": "many"},
+        ]
+        assert _pick_graph_token(entries) == "many"
+
+    def test_returns_none_without_a_graph_entry(self):
+        entries = [{"target": "openid profile offline_access", "secret": "id-tok"}]
+        assert _pick_graph_token(entries) is None
+
+    def test_skips_entries_with_no_secret(self):
+        entries = [{"target": "Tasks.Read", "secret": ""}]
+        assert _pick_graph_token(entries) is None
+
+    def test_tolerates_junk_entries(self):
+        assert _pick_graph_token([None, "nonsense", {}, 42]) is None