Initial commit: MISF VOB download and upload automation
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
c2dec96b1f
43 files changed
+10944
No files matched your search
+891
@@ -0,0 +1,891 @@
|
||||
"""Chromium (Chrome-compatible) remote-debug startup and Selenium attach."""
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.error import URLError
|
||||
from urllib.request import urlopen
|
||||
|
||||
from selenium import webdriver
|
||||
from selenium.common.exceptions import WebDriverException
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
|
||||
from config import PROJECT_ROOT
|
||||
from debug_log import write_debug as _agent_log
|
||||
from run_log import log_step
|
||||
|
||||
_DEBUG_PORT = 9222
|
||||
DEBUG_PORT = _DEBUG_PORT
|
||||
_PROXY_CREDENTIALS: dict[str, str] = {}
|
||||
_LAST_BROWSER_PATH: str | None = None
|
||||
_BROWSER_PROCESS: subprocess.Popen | None = None
|
||||
|
||||
USER_DATA_DIR = os.path.join(
|
||||
os.environ.get("USERPROFILE", str(Path.home())),
|
||||
"Automation",
|
||||
"ChromiumProfile_MISF_Download",
|
||||
)
|
||||
|
||||
_DRIVERS_DIR = PROJECT_ROOT / ".drivers"
|
||||
_CFT_DOWNLOAD_BASE = "https://storage.googleapis.com/chrome-for-testing-public"
|
||||
DOWNLOADS_DIR = PROJECT_ROOT / "Input" / "PDFs"
|
||||
_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+\.\d+)")
|
||||
|
||||
# Microsoft 365 hosts bypass HTTP proxy when split-tunnel is configured.
|
||||
CHROME_PROXY_BYPASS_OFFICE = (
|
||||
"*.office.com;*.office365.com;*.microsoft.com;*.live.com;*.outlook.com;"
|
||||
"outlook.office.com;outlook.office365.com;login.microsoftonline.com;login.live.com;"
|
||||
"aadcdn.msauth.net;aadcdn.msftauth.net"
|
||||
)
|
||||
|
||||
|
||||
|
||||
def set_default_download_directory(path: Path | str, *, create: bool = False) -> None:
|
||||
"""Set browser download prefs root (used at attach time). Per-facility CDP still applies."""
|
||||
global DOWNLOADS_DIR
|
||||
DOWNLOADS_DIR = Path(path).resolve()
|
||||
if create:
|
||||
DOWNLOADS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _allow_google_chrome_fallback() -> bool:
|
||||
return os.environ.get("MISF_ALLOW_CHROME", "").strip().lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def _portable_chrome_win_roots() -> list[Path]:
|
||||
return [
|
||||
PROJECT_ROOT / "chrome-win" / "chrome-win",
|
||||
PROJECT_ROOT / "chrome-win",
|
||||
]
|
||||
|
||||
|
||||
def _portable_chromium_executable() -> str | None:
|
||||
"""Bundled Chromium under project ``chrome-win/`` (preferred for production)."""
|
||||
for root in _portable_chrome_win_roots():
|
||||
for name in ("chrome.exe", "chromium.exe"):
|
||||
path = root / name
|
||||
if path.is_file():
|
||||
return str(path)
|
||||
return None
|
||||
|
||||
|
||||
def _bundled_chromedriver_for(browser_path: str) -> str | None:
|
||||
"""``chromedriver.exe`` shipped next to portable ``chrome.exe``."""
|
||||
if not browser_path:
|
||||
return None
|
||||
driver = Path(browser_path).parent / "chromedriver.exe"
|
||||
if driver.is_file():
|
||||
return str(driver)
|
||||
return None
|
||||
|
||||
|
||||
def _installed_chromium_paths() -> list[str]:
|
||||
"""System-installed Chromium (not Google Chrome)."""
|
||||
local = os.environ.get("LOCALAPPDATA", "")
|
||||
profile = os.environ.get("USERPROFILE", str(Path.home()))
|
||||
return [
|
||||
r"C:\Program Files\Chromium\Application\chrome.exe",
|
||||
r"C:\Program Files\Chromium\Application\chromium.exe",
|
||||
r"C:\Program Files (x86)\Chromium\Application\chrome.exe",
|
||||
os.path.join(local, r"Chromium\Application\chrome.exe"),
|
||||
os.path.join(profile, r"scoop\apps\chromium\current\chrome.exe"),
|
||||
]
|
||||
|
||||
|
||||
def _prefer_installed_chromium() -> bool:
|
||||
"""Skip ``chrome-win/`` and use a system Chromium install instead."""
|
||||
return os.environ.get("MISF_USE_INSTALLED_CHROME", "").strip().lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
|
||||
def _google_chrome_fallback_paths() -> list[str]:
|
||||
local = os.environ.get("LOCALAPPDATA", "")
|
||||
return [
|
||||
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
|
||||
r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
|
||||
os.path.join(local, r"Google\Chrome\Application\chrome.exe"),
|
||||
]
|
||||
|
||||
|
||||
def find_chromium_executable() -> str | None:
|
||||
"""
|
||||
Resolve Chromium only (Google Chrome is opt-in via ``MISF_ALLOW_CHROME=1``).
|
||||
|
||||
Default order: ``chrome-win/`` portable bundle, then system Chromium.
|
||||
Override: ``CHROMIUM_PATH`` / ``BROWSER_PATH``, or ``MISF_USE_INSTALLED_CHROME=1``.
|
||||
"""
|
||||
for env_key in ("CHROMIUM_PATH", "BROWSER_PATH"):
|
||||
raw = os.environ.get(env_key, "").strip().strip('"')
|
||||
if raw and os.path.isfile(raw):
|
||||
return raw
|
||||
|
||||
portable = _portable_chromium_executable()
|
||||
installed = [p for p in _installed_chromium_paths() if os.path.isfile(p)]
|
||||
|
||||
if _prefer_installed_chromium():
|
||||
candidates = installed + ([portable] if portable else [])
|
||||
else:
|
||||
candidates = ([portable] if portable else []) + installed
|
||||
|
||||
if _allow_google_chrome_fallback():
|
||||
candidates = candidates + [
|
||||
p for p in _google_chrome_fallback_paths() if os.path.isfile(p)
|
||||
]
|
||||
|
||||
for path in candidates:
|
||||
if path:
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:find_chromium_executable",
|
||||
"Selected browser executable",
|
||||
{"basename": os.path.basename(path), "from_portable": "chrome-win" in path.lower()},
|
||||
)
|
||||
return path
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:find_chromium_executable",
|
||||
"No Chromium executable found",
|
||||
{"chrome_win_checked": [str(r) for r in _portable_chrome_win_roots()]},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _browser_runs_in_background() -> bool:
|
||||
"""Default: foreground (visible). Set MISF_BROWSER_BACKGROUND=1 to start minimized."""
|
||||
return os.environ.get("MISF_BROWSER_BACKGROUND", "").strip().lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
|
||||
def _launch_browser_process(cmd: list[str]) -> subprocess.Popen:
|
||||
"""Start Chromium detached; minimized on Windows when running in background."""
|
||||
global _BROWSER_PROCESS
|
||||
kwargs: dict = {
|
||||
"stdout": subprocess.DEVNULL,
|
||||
"stderr": subprocess.DEVNULL,
|
||||
}
|
||||
if sys.platform == "win32" and _browser_runs_in_background():
|
||||
startupinfo = subprocess.STARTUPINFO()
|
||||
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||
startupinfo.wShowWindow = 7 # SW_SHOWMINNOACTIVE — minimized, no focus steal
|
||||
kwargs["startupinfo"] = startupinfo
|
||||
_BROWSER_PROCESS = subprocess.Popen(cmd, **kwargs)
|
||||
return _BROWSER_PROCESS
|
||||
|
||||
|
||||
def stop_chrome_browser(driver: webdriver.Chrome | None = None) -> None:
|
||||
"""Disconnect Selenium and terminate the Chromium process we launched."""
|
||||
if driver is not None:
|
||||
try:
|
||||
driver.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
global _BROWSER_PROCESS
|
||||
proc = _BROWSER_PROCESS
|
||||
_BROWSER_PROCESS = None
|
||||
if proc is not None and proc.poll() is None:
|
||||
try:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=8)
|
||||
except Exception:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if wait_for_debug_port(DEBUG_PORT, timeout=1):
|
||||
_kill_processes_listening_on_port(DEBUG_PORT)
|
||||
|
||||
|
||||
def _kill_processes_listening_on_port(port: int) -> None:
|
||||
"""Best-effort cleanup when the stored Popen handle is unavailable."""
|
||||
if sys.platform != "win32":
|
||||
return
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
["netstat", "-ano", "-p", "tcp"],
|
||||
text=True,
|
||||
errors="replace",
|
||||
**_hidden_subprocess_kwargs(),
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
target_pids: set[int] = set()
|
||||
needle = f":{port}"
|
||||
for line in output.splitlines():
|
||||
if needle not in line or "LISTENING" not in line.upper():
|
||||
continue
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
try:
|
||||
target_pids.add(int(parts[-1]))
|
||||
except ValueError:
|
||||
continue
|
||||
for pid in target_pids:
|
||||
if pid <= 0:
|
||||
continue
|
||||
try:
|
||||
subprocess.run(
|
||||
["taskkill", "/PID", str(pid), "/T", "/F"],
|
||||
check=False,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
**_hidden_subprocess_kwargs(),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _apply_background_window(driver: webdriver.Chrome) -> None:
|
||||
try:
|
||||
driver.maximize_window()
|
||||
except WebDriverException:
|
||||
pass
|
||||
|
||||
|
||||
def _bring_browser_to_front(driver: webdriver.Chrome) -> None:
|
||||
"""Best-effort focus for scheduled/server runs so captures are not blank."""
|
||||
if sys.platform != "win32":
|
||||
return
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
user32 = ctypes.windll.user32
|
||||
hwnd = user32.GetForegroundWindow()
|
||||
try:
|
||||
rect = driver.get_window_rect()
|
||||
target_x = int(rect.get("x", 0) + max(rect.get("width", 800) // 2, 1))
|
||||
target_y = int(rect.get("y", 0) + max(rect.get("height", 600) // 2, 1))
|
||||
user32.SetCursorPos(target_x, target_y)
|
||||
except Exception:
|
||||
pass
|
||||
if hwnd:
|
||||
user32.SetForegroundWindow(hwnd)
|
||||
driver.switch_to.window(driver.current_window_handle)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def prepare_browser_for_screenshot(driver: webdriver.Chrome) -> None:
|
||||
"""Maximize and focus the automation browser before board screenshots."""
|
||||
try:
|
||||
driver.switch_to.default_content()
|
||||
except Exception:
|
||||
pass
|
||||
_apply_background_window(driver)
|
||||
_bring_browser_to_front(driver)
|
||||
time.sleep(0.6)
|
||||
|
||||
|
||||
def _hidden_subprocess_kwargs() -> dict:
|
||||
"""Run helper processes without flashing a console window on Windows."""
|
||||
if sys.platform != "win32":
|
||||
return {}
|
||||
kwargs: dict = {"creationflags": subprocess.CREATE_NO_WINDOW}
|
||||
startupinfo = subprocess.STARTUPINFO()
|
||||
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||
startupinfo.wShowWindow = 0
|
||||
kwargs["startupinfo"] = startupinfo
|
||||
return kwargs
|
||||
|
||||
|
||||
def _capture_desktop_powershell(path: Path) -> bool:
|
||||
"""Primary monitor capture on Windows (includes taskbar date/time)."""
|
||||
if sys.platform != "win32":
|
||||
return False
|
||||
out = str(path.resolve())
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms,System.Drawing
|
||||
$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
|
||||
$bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bmp)
|
||||
$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
|
||||
$bmp.Save('{out.replace("'", "''")}', [System.Drawing.Imaging.ImageFormat]::Png)
|
||||
$graphics.Dispose()
|
||||
$bmp.Dispose()
|
||||
"""
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"powershell",
|
||||
"-NoProfile",
|
||||
"-WindowStyle",
|
||||
"Hidden",
|
||||
"-Command",
|
||||
ps_script,
|
||||
],
|
||||
check=True,
|
||||
timeout=20,
|
||||
capture_output=True,
|
||||
**_hidden_subprocess_kwargs(),
|
||||
)
|
||||
return path.is_file() and path.stat().st_size > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def capture_desktop_screenshot(path: Path) -> bool:
|
||||
"""Save a full-desktop PNG (taskbar clock visible). Falls back to False if unavailable."""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
from PIL import ImageGrab
|
||||
|
||||
ImageGrab.grab(all_screens=False).save(str(path), "PNG")
|
||||
if path.is_file():
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import mss
|
||||
import mss.tools
|
||||
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[1]
|
||||
shot = sct.grab(monitor)
|
||||
mss.tools.to_png(shot.rgb, shot.size, output=str(path))
|
||||
return path.is_file()
|
||||
except Exception:
|
||||
pass
|
||||
if _capture_desktop_powershell(path):
|
||||
return True
|
||||
try:
|
||||
import pyautogui
|
||||
|
||||
pyautogui.screenshot(str(path))
|
||||
return path.is_file()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _is_image_mostly_black(path: Path, *, threshold: int = 18) -> bool:
|
||||
"""True when a PNG is nearly all black (common on headless/disconnected server sessions)."""
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(path) as img:
|
||||
rgb = img.convert("RGB")
|
||||
sample = rgb.resize((64, 64))
|
||||
pixels = list(sample.getdata())
|
||||
if not pixels:
|
||||
return True
|
||||
avg = sum(sum(p) for p in pixels) / (len(pixels) * 3)
|
||||
return avg < threshold
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _capture_browser_with_timestamp_bar(driver: webdriver.Chrome, path: Path) -> bool:
|
||||
"""
|
||||
Fallback when desktop capture is black: maximized browser PNG plus a taskbar-style clock strip.
|
||||
"""
|
||||
try:
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
prepare_browser_for_screenshot(driver)
|
||||
png = driver.get_screenshot_as_png()
|
||||
browser = Image.open(io.BytesIO(png)).convert("RGB")
|
||||
bar_h = 48
|
||||
canvas = Image.new("RGB", (browser.width, browser.height + bar_h), (30, 30, 30))
|
||||
canvas.paste(browser, (0, 0))
|
||||
draw = ImageDraw.Draw(canvas)
|
||||
stamp = datetime.now().strftime("%I:%M %p %m/%d/%Y").lstrip("0")
|
||||
draw.text((max(browser.width - 260, 8), browser.height + 14), stamp, fill=(255, 255, 255))
|
||||
canvas.save(str(path), "PNG")
|
||||
return path.is_file() and path.stat().st_size > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def capture_fullscreen_screenshot(driver: webdriver.Chrome, path: Path) -> bool:
|
||||
"""
|
||||
Full-screen capture with system taskbar visible when an interactive desktop exists.
|
||||
|
||||
On scheduled/server sessions without a visible desktop, falls back to a maximized
|
||||
browser screenshot plus a timestamp bar (instead of an all-black PNG).
|
||||
"""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_step("Fullscreen screenshot", path=str(path))
|
||||
prepare_browser_for_screenshot(driver)
|
||||
if capture_desktop_screenshot(path) and not _is_image_mostly_black(path):
|
||||
log_step("Fullscreen screenshot saved", path=str(path))
|
||||
return True
|
||||
if _is_image_mostly_black(path):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
if _capture_browser_with_timestamp_bar(driver, path):
|
||||
log_step("Browser screenshot with timestamp bar saved", path=str(path), mode="server_fallback")
|
||||
return True
|
||||
driver.save_screenshot(str(path))
|
||||
log_step("Viewport screenshot saved (desktop capture unavailable)", path=str(path))
|
||||
return path.is_file()
|
||||
|
||||
|
||||
def wait_for_debug_port(port: int = DEBUG_PORT, timeout: int = 40) -> bool:
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
if s.connect_ex(("127.0.0.1", port)) == 0:
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def is_debug_port_open(port: int = DEBUG_PORT) -> bool:
|
||||
return wait_for_debug_port(port, timeout=1)
|
||||
|
||||
|
||||
def _version_from_portable_dir(browser_path: str) -> str:
|
||||
"""Read e.g. 150.0.7852.0 from 150.0.7852.0.manifest next to chrome.exe."""
|
||||
folder = Path(browser_path).parent
|
||||
for manifest in folder.glob("*.manifest"):
|
||||
match = _VERSION_RE.search(manifest.name)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_browser_version(browser_path: str) -> str:
|
||||
"""Best-effort version for the browser we launched (no hanging --version)."""
|
||||
version = _get_browser_version_from_debug_port(timeout_secs=5.0)
|
||||
if version:
|
||||
return version
|
||||
if browser_path:
|
||||
version = _version_from_portable_dir(browser_path)
|
||||
if version:
|
||||
return version
|
||||
return ""
|
||||
|
||||
|
||||
def _ensure_chromedriver_cft(browser_version: str) -> str:
|
||||
"""
|
||||
Download chromedriver matching ``browser_version`` from Chrome for Testing.
|
||||
Cached under ``.drivers/<version>/chromedriver.exe``.
|
||||
"""
|
||||
dest = _DRIVERS_DIR / browser_version / "chromedriver.exe"
|
||||
if dest.is_file():
|
||||
return str(dest)
|
||||
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
zip_url = f"{_CFT_DOWNLOAD_BASE}/{browser_version}/win64/chromedriver-win64.zip"
|
||||
zip_path = dest.parent / "chromedriver-win64.zip"
|
||||
print(f"[INFO] Downloading ChromeDriver {browser_version} (Chrome for Testing)...")
|
||||
|
||||
_download_timeout = int(
|
||||
os.environ.get("MISF_CHROMEDRIVER_DOWNLOAD_TIMEOUT", "60")
|
||||
)
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(1, 4):
|
||||
try:
|
||||
req = urlopen(zip_url, timeout=_download_timeout)
|
||||
with open(zip_path, "wb") as fh:
|
||||
shutil.copyfileobj(req, fh)
|
||||
break
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
print(f"[WARN] ChromeDriver download attempt {attempt}/3 failed: {exc}")
|
||||
if zip_path.exists():
|
||||
zip_path.unlink(missing_ok=True)
|
||||
time.sleep(2 * attempt)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"ChromeDriver {browser_version} download failed after 3 attempts: {last_exc}"
|
||||
)
|
||||
|
||||
if not zipfile.is_zipfile(zip_path):
|
||||
zip_path.unlink(missing_ok=True)
|
||||
raise RuntimeError(
|
||||
f"Downloaded ChromeDriver archive is not a valid ZIP: {zip_url}"
|
||||
)
|
||||
|
||||
with zipfile.ZipFile(zip_path) as archive:
|
||||
member = next(
|
||||
(name for name in archive.namelist() if name.endswith("chromedriver.exe")),
|
||||
None,
|
||||
)
|
||||
if not member:
|
||||
raise RuntimeError(f"chromedriver.exe not found in {zip_url}")
|
||||
archive.extract(member, dest.parent)
|
||||
extracted = dest.parent / member
|
||||
extracted.rename(dest)
|
||||
|
||||
try:
|
||||
zip_path.unlink(missing_ok=True)
|
||||
nested = dest.parent / "chromedriver-win64"
|
||||
if nested.is_dir():
|
||||
shutil.rmtree(nested, ignore_errors=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
print(f"[INFO] ChromeDriver installed -> {dest}")
|
||||
return str(dest)
|
||||
|
||||
|
||||
def _get_browser_version_from_debug_port(timeout_secs: float = 2.0) -> str:
|
||||
"""
|
||||
Read Chromium version from the running remote-debug instance.
|
||||
|
||||
Example payload: {"Browser":"Chrome/150.0.7852.0", ...}
|
||||
"""
|
||||
try:
|
||||
with urlopen(f"http://127.0.0.1:{DEBUG_PORT}/json/version", timeout=timeout_secs) as resp:
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
data = json.loads(raw)
|
||||
browser = str(data.get("Browser", "")).strip()
|
||||
# "Chrome/150.0.7852.0"
|
||||
if "/" in browser:
|
||||
return browser.split("/", 1)[1].strip()
|
||||
return browser
|
||||
except (URLError, TimeoutError, ValueError, OSError):
|
||||
return ""
|
||||
|
||||
|
||||
def create_proxy_auth_extension(
|
||||
proxy_ip: str, proxy_port: str, proxy_username: str, proxy_password: str
|
||||
) -> str:
|
||||
manifest_json = """{
|
||||
"version": "1.0.0",
|
||||
"manifest_version": 2,
|
||||
"name": "Chrome Proxy Auth Extension",
|
||||
"permissions": [
|
||||
"proxy", "tabs", "unlimitedStorage", "storage",
|
||||
"<all_urls>", "webRequest", "webRequestBlocking"
|
||||
],
|
||||
"background": { "scripts": ["background.js"] },
|
||||
"minimum_chrome_version": "22.0.0"
|
||||
}"""
|
||||
_u = json.dumps(proxy_username)
|
||||
_p = json.dumps(proxy_password)
|
||||
background_js = f"""function callbackFn(details) {{
|
||||
return {{ authCredentials: {{ username: {_u}, password: {_p} }} }};
|
||||
}}
|
||||
chrome.webRequest.onAuthRequired.addListener(
|
||||
callbackFn, {{urls: ["<all_urls>"]}}, ["blocking"]
|
||||
);"""
|
||||
ext_dir = Path(tempfile.mkdtemp(prefix="proxy_auth_ext_"))
|
||||
atexit.register(shutil.rmtree, str(ext_dir), True)
|
||||
(ext_dir / "manifest.json").write_text(manifest_json, encoding="utf-8")
|
||||
(ext_dir / "background.js").write_text(background_js, encoding="utf-8")
|
||||
print(f"[INFO] Proxy auth extension folder -> {ext_dir}")
|
||||
return str(ext_dir)
|
||||
|
||||
|
||||
def start_chrome_debug(
|
||||
proxy_ip=None,
|
||||
proxy_port=None,
|
||||
proxy_username=None,
|
||||
proxy_password=None,
|
||||
) -> bool:
|
||||
global _PROXY_CREDENTIALS
|
||||
|
||||
browser_path = find_chromium_executable()
|
||||
if not browser_path:
|
||||
raise RuntimeError(
|
||||
"Chromium not found. Place the portable bundle in chrome-win\\chrome-win\\ "
|
||||
"(chrome.exe + chromedriver.exe), install Chromium, or set CHROMIUM_PATH. "
|
||||
"(Google Chrome: MISF_ALLOW_CHROME=1.)"
|
||||
)
|
||||
|
||||
log_step(
|
||||
"Launch Chromium",
|
||||
executable=browser_path,
|
||||
background=str(_browser_runs_in_background()).lower(),
|
||||
)
|
||||
global _LAST_BROWSER_PATH
|
||||
_LAST_BROWSER_PATH = browser_path
|
||||
os.makedirs(USER_DATA_DIR, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
browser_path,
|
||||
f"--remote-debugging-port={DEBUG_PORT}",
|
||||
"--remote-allow-origins=*",
|
||||
f"--user-data-dir={USER_DATA_DIR}",
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-popup-blocking",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--password-store=basic",
|
||||
"--disable-restore-session-state",
|
||||
"--disable-session-crashed-bubble",
|
||||
"--hide-crash-restore-bubble",
|
||||
"about:blank",
|
||||
]
|
||||
if _browser_runs_in_background():
|
||||
cmd.append("--start-minimized")
|
||||
|
||||
if proxy_ip and proxy_port:
|
||||
port_str = str(proxy_port).strip()
|
||||
if not port_str.isdigit():
|
||||
print(f"[WARN] Invalid proxy port '{port_str}' — skipping proxy")
|
||||
_PROXY_CREDENTIALS = {}
|
||||
else:
|
||||
cmd.append(f"--proxy-server=http://{proxy_ip.strip()}:{port_str}")
|
||||
cmd.append(
|
||||
"--proxy-bypass-list=127.0.0.1;localhost;<-loopback>;"
|
||||
+ CHROME_PROXY_BYPASS_OFFICE
|
||||
)
|
||||
log_step("Proxy configured", host=proxy_ip.strip(), port=port_str)
|
||||
if proxy_username and proxy_password:
|
||||
ext_folder = create_proxy_auth_extension(
|
||||
proxy_ip.strip(),
|
||||
port_str,
|
||||
proxy_username.strip(),
|
||||
proxy_password,
|
||||
)
|
||||
cmd.append(f"--load-extension={ext_folder}")
|
||||
_PROXY_CREDENTIALS = {
|
||||
"username": proxy_username.strip(),
|
||||
"password": str(proxy_password),
|
||||
}
|
||||
else:
|
||||
_PROXY_CREDENTIALS = {}
|
||||
else:
|
||||
print("[INFO] No proxy configured — direct connection")
|
||||
_PROXY_CREDENTIALS = {}
|
||||
|
||||
if is_debug_port_open(DEBUG_PORT):
|
||||
log_step("Debug port already in use — closing previous browser session")
|
||||
stop_chrome_browser()
|
||||
time.sleep(2)
|
||||
|
||||
_launch_browser_process(cmd)
|
||||
|
||||
if not wait_for_debug_port(DEBUG_PORT, timeout=40):
|
||||
print("[ERROR] Browser debug port did not open in time")
|
||||
return False
|
||||
|
||||
print("[INFO] Browser debug port ready")
|
||||
return True
|
||||
|
||||
|
||||
def _find_chromedriver_path() -> str | None:
|
||||
"""
|
||||
Return a chromedriver path matching the browser we started.
|
||||
|
||||
Prefers ``chromedriver.exe`` bundled in ``chrome-win/``. Otherwise downloads
|
||||
a matching Chrome-for-Testing driver, then Selenium Manager.
|
||||
"""
|
||||
for env_key in ("CHROMEDRIVER_PATH", "WEBDRIVER_CHROME_DRIVER"):
|
||||
raw = os.environ.get(env_key, "").strip().strip('"')
|
||||
if raw and os.path.isfile(raw):
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:_find_chromedriver_path",
|
||||
"Using explicit chromedriver path",
|
||||
{"basename": os.path.basename(raw)},
|
||||
)
|
||||
return raw
|
||||
|
||||
browser_path = _LAST_BROWSER_PATH or ""
|
||||
bundled = _bundled_chromedriver_for(browser_path)
|
||||
if bundled:
|
||||
log_step("Using bundled chromedriver", path=bundled)
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:_find_chromedriver_path",
|
||||
"Using chrome-win chromedriver",
|
||||
{"basename": os.path.basename(bundled)},
|
||||
)
|
||||
return bundled
|
||||
|
||||
browser_version = _resolve_browser_version(browser_path)
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:_find_chromedriver_path",
|
||||
"Detected browser version",
|
||||
{
|
||||
"browser_version": browser_version,
|
||||
"from_portable": "chrome-win" in (browser_path or "").lower(),
|
||||
},
|
||||
)
|
||||
|
||||
if browser_version:
|
||||
try:
|
||||
driver_path = _ensure_chromedriver_cft(browser_version)
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:_find_chromedriver_path",
|
||||
"Chrome for Testing chromedriver ready",
|
||||
{"browser_version": browser_version},
|
||||
)
|
||||
return driver_path
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"[WARN] Could not download ChromeDriver {browser_version}: {exc}"
|
||||
)
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:_find_chromedriver_path",
|
||||
"CfT chromedriver download failed",
|
||||
{"error": str(exc)[:200]},
|
||||
)
|
||||
|
||||
_agent_log(
|
||||
"A",
|
||||
"src/chrome.py:_find_chromedriver_path",
|
||||
"Using Selenium Manager for chromedriver",
|
||||
{},
|
||||
)
|
||||
print(
|
||||
"[WARN] Browser version unknown — Selenium Manager may pick the wrong driver. "
|
||||
"Set CHROMEDRIVER_PATH or use installed Chrome (default)."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _fill_proxy_auth_js_alert_if_present(
|
||||
driver, username: str, password: str, wait_secs: float
|
||||
) -> bool:
|
||||
deadline = time.monotonic() + wait_secs
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
alert = driver.switch_to.alert
|
||||
alert.send_keys(Keys.CONTROL + "a")
|
||||
alert.send_keys(username)
|
||||
alert.send_keys(Keys.TAB)
|
||||
alert.send_keys(Keys.CONTROL + "a")
|
||||
alert.send_keys(password)
|
||||
alert.accept()
|
||||
return True
|
||||
except Exception:
|
||||
time.sleep(0.25)
|
||||
return False
|
||||
|
||||
|
||||
def _try_fill_proxy_auth_native_dialog(username: str, password: str) -> bool:
|
||||
if sys.platform != "win32":
|
||||
return False
|
||||
try:
|
||||
import pyautogui
|
||||
except ImportError:
|
||||
print("[INFO] pyautogui not installed — skip native proxy dialog fill")
|
||||
return False
|
||||
try:
|
||||
time.sleep(0.6)
|
||||
pyautogui.FAILSAFE = True
|
||||
pyautogui.hotkey("ctrl", "a")
|
||||
pyautogui.press("backspace")
|
||||
pyautogui.write(username, interval=0.02)
|
||||
pyautogui.press("tab")
|
||||
pyautogui.hotkey("ctrl", "a")
|
||||
pyautogui.press("backspace")
|
||||
pyautogui.write(password, interval=0.02)
|
||||
pyautogui.press("enter")
|
||||
return True
|
||||
except Exception as exc:
|
||||
print(f"[WARN] Native proxy keyboard fill failed: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def _microsoft_login_visible(driver) -> bool:
|
||||
try:
|
||||
url = (driver.current_url or "").lower()
|
||||
if "login.microsoftonline.com" in url or "login.live.com" in url:
|
||||
return True
|
||||
return bool(
|
||||
driver.find_elements(By.CSS_SELECTOR, "input[type='email']")
|
||||
or driver.find_elements(By.NAME, "loginfmt")
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def dismiss_proxy_auth_popup(
|
||||
driver,
|
||||
username: str,
|
||||
password: str,
|
||||
wait_secs: float = 5.0,
|
||||
max_attempts: int = 3,
|
||||
) -> bool:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
if _fill_proxy_auth_js_alert_if_present(driver, username, password, wait_secs):
|
||||
print(f"[INFO] Proxy auth handled (JS dialog) attempt {attempt}/{max_attempts}")
|
||||
return True
|
||||
if attempt < max_attempts:
|
||||
time.sleep(2)
|
||||
|
||||
if _try_fill_proxy_auth_native_dialog(username, password):
|
||||
time.sleep(2)
|
||||
if _microsoft_login_visible(driver):
|
||||
print("[INFO] Microsoft login visible after proxy auth — proceeding")
|
||||
return True
|
||||
|
||||
if _microsoft_login_visible(driver):
|
||||
print("[INFO] Microsoft login page present — proxy session usable")
|
||||
return True
|
||||
|
||||
print("[WARN] Proxy auth popup could not be dismissed")
|
||||
return False
|
||||
|
||||
|
||||
def attach_browser(
|
||||
retries: int = 5,
|
||||
download_dir: Path | str | None = None,
|
||||
*,
|
||||
create_download_dir: bool = False,
|
||||
) -> webdriver.Chrome | None:
|
||||
driver_path = _find_chromedriver_path()
|
||||
service = Service(driver_path) if driver_path else Service()
|
||||
dl_dir = Path(download_dir) if download_dir else DOWNLOADS_DIR
|
||||
if create_download_dir:
|
||||
dl_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
log_step("Attach browser", attempt=f"{attempt}/{retries}", download_dir=str(dl_dir))
|
||||
chrome_options = Options()
|
||||
chrome_options.add_experimental_option(
|
||||
"debuggerAddress", f"127.0.0.1:{DEBUG_PORT}"
|
||||
)
|
||||
chrome_options.add_experimental_option(
|
||||
"prefs",
|
||||
{
|
||||
"download.default_directory": str(dl_dir),
|
||||
"download.prompt_for_download": False,
|
||||
"plugins.always_open_pdf_externally": True,
|
||||
"download.directory_upgrade": True,
|
||||
"credentials_enable_service": False,
|
||||
"profile.password_manager_enabled": False,
|
||||
},
|
||||
)
|
||||
driver = webdriver.Chrome(service=service, options=chrome_options)
|
||||
_ = driver.current_url
|
||||
_apply_background_window(driver)
|
||||
driver.set_page_load_timeout(90)
|
||||
driver.set_script_timeout(30)
|
||||
log_step("Browser attached successfully")
|
||||
return driver
|
||||
except WebDriverException as exc:
|
||||
log_step("Attach failed", attempt=f"{attempt}/{retries}", error=str(exc)[:200])
|
||||
if attempt < retries:
|
||||
time.sleep(3)
|
||||
|
||||
print("[ERROR] Failed to attach browser after all retries")
|
||||
return None
|
||||
Reference in new issue
Block a user