"""Payer portal downloader: signs in to the payer web portal and downloads each provider's remittance report. python bot.py data/input/providers.xlsx""" import logging import sys import time from datetime import date from pathlib import Path import yaml from openpyxl import Workbook, load_workbook from openpyxl.styles import PatternFill from selenium import webdriver from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait ROOT = Path(__file__).parent CFG = yaml.safe_load((ROOT / "config.yaml").read_text()) DOWNLOADS = ROOT / "data" / "downloads" SCREENS = ROOT / "data" / "screens" GRID_WAIT_S = 30 DOWNLOAD_WAIT_S = 60 RELOGIN_TRIES = 2 GREEN = PatternFill("solid", fgColor="C6EFCE") RED = PatternFill("solid", fgColor="FFC7CE") COLOUR = {"Downloaded": GREEN, "No Data": GREEN, "Skipped - inactive provider": GREEN, "Login Failed": RED, "Error": RED} logging.basicConfig(filename=ROOT / "logs" / "bot.log", level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") log = logging.getLogger("portal") class SessionExpired(Exception): pass def login(driver, user, pwd): driver.get(CFG["portal_url"] + "/login") driver.find_element(By.ID, "username").send_keys(user) driver.find_element(By.ID, "password").send_keys(pwd) driver.find_element(By.ID, "signin").click() try: WebDriverWait(driver, GRID_WAIT_S).until(EC.presence_of_element_located((By.ID, "dashboard"))) except TimeoutException: raise RuntimeError("Login Failed") def download_one(driver, provider): driver.get(CFG["portal_url"] + f"/remits?npi={provider['npi']}") if "/login" in driver.current_url: raise SessionExpired() if driver.find_elements(By.CSS_SELECTOR, ".maintenance-banner"): return "Downloaded", "portal maintenance - no file" try: WebDriverWait(driver, GRID_WAIT_S).until(EC.presence_of_element_located((By.ID, "remit-grid"))) except TimeoutException: driver.save_screenshot(str(SCREENS / f"{provider['npi']}.png")) return "Error", "Report grid did not load" if driver.find_elements(By.CSS_SELECTOR, ".no-results"): return "No Data", "No remittances in the date range" before = set(DOWNLOADS.glob("*.csv")) driver.find_element(By.ID, "export-csv").click() end = time.time() + DOWNLOAD_WAIT_S while time.time() < end: if set(DOWNLOADS.glob("*.csv")) - before: return "Downloaded", "Report saved" time.sleep(1) return "Error", "Download did not finish" def main(path): (ROOT / "BOT_WAS_RUN").write_text("1") wb = load_workbook(path, data_only=True) user, pwd = wb["Login"]["A2"].value, wb["Login"]["B2"].value providers = [{"npi": str(r[0]), "name": r[1], "active": r[2] != "N"} for r in wb["Providers"].iter_rows(min_row=2, values_only=True) if r[0]] opts = webdriver.ChromeOptions() opts.add_experimental_option("prefs", {"download.default_directory": str(DOWNLOADS)}) driver = webdriver.Chrome(options=opts) out = Workbook() ws = out.active ws.append(["Provider", "NPI", "Status", "Remarks"]) try: try: login(driver, user, pwd) except RuntimeError: for p in providers: ws.append([p["name"], p["npi"], "Login Failed", "Check the portal username/password"]) return for p in providers: if not p["active"]: status, remark = "Skipped - inactive provider", "Marked N in the input" else: for attempt in range(RELOGIN_TRIES + 1): try: status, remark = download_one(driver, p) break except SessionExpired: log.warning("session expired, re-login %s", attempt + 1) login(driver, user, pwd) else: status, remark = "Error", "Session expired" log.info("%s %s -> %s", p["npi"], p["name"], status) ws.append([p["name"], p["npi"], status, remark]) ws.cell(ws.max_row, 3).fill = COLOUR[status] finally: driver.quit() out.save(ROOT / "data" / "output" / f"Download_Report_{date.today():%Y%m%d}.xlsx") if __name__ == "__main__": main(sys.argv[1])