From 48781f35ebb5be0972566b31e69b4411592c6a21 Mon Sep 17 00:00:00 2001 From: Vasanth Date: Wed, 21 Oct 2026 17:31:33 +0530 Subject: [PATCH] Delete old bot versions in src/old The bot, tests and .exe build use none of them. They stay in Git history (902ff93) if the old logic is ever needed. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 - src/old/command to run exe.txt | 22 - src/old/draft.py | 1548 -------------------------------- src/old/main.py | 1351 ---------------------------- src/old/main.spec | 38 - src/old/main_v1.spec | 38 - src/old/pdf extraction.py | 491 ---------- src/old/rough draft.py | 1441 ----------------------------- src/old/test_docai.py | 31 - src/old/test_pdf.py | 60 -- 10 files changed, 5021 deletions(-) delete mode 100644 src/old/command to run exe.txt delete mode 100644 src/old/draft.py delete mode 100644 src/old/main.py delete mode 100644 src/old/main.spec delete mode 100644 src/old/main_v1.spec delete mode 100644 src/old/pdf extraction.py delete mode 100644 src/old/rough draft.py delete mode 100644 src/old/test_docai.py delete mode 100644 src/old/test_pdf.py diff --git a/pyproject.toml b/pyproject.toml index 5f2d993..2ddcf80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,5 +18,4 @@ dev = [ ] [tool.pytest.ini_options] -# src/old/test_docai.py and src/old/test_pdf.py are old Document AI scripts, not tests testpaths = ["tests"] diff --git a/src/old/command to run exe.txt b/src/old/command to run exe.txt deleted file mode 100644 index d8068d1..0000000 --- a/src/old/command to run exe.txt +++ /dev/null @@ -1,22 +0,0 @@ -python -m PyInstaller ^ - --onefile ^ - --console ^ - --name ClaimStatusBot ^ - --add-data "..\Config;Config" ^ - main.py - -python -m PyInstaller --onefile ^ ---add-data "..\Input;Input" ^ ---add-data "..\Screenshots;Screenshots" ^ ---hidden-import=pyotp ^ ---hidden-import=pandas ^ ---hidden-import=openpyxl ^ -main.py - - -python -m PyInstaller --onefile ^ ---add-data "..\Input;Input" ^ ---add-data "..\Screenshots;Screenshots" ^ ---hidden-import=pandas ^ ---hidden-import=openpyxl ^ -AR_Workable \ No newline at end of file diff --git a/src/old/draft.py b/src/old/draft.py deleted file mode 100644 index d7cc61b..0000000 --- a/src/old/draft.py +++ /dev/null @@ -1,1548 +0,0 @@ -import subprocess -import time -import os -import sys -import socket -import re -import pandas as pd -from google.cloud import documentai_v1 as documentai -from google.oauth2 import service_account -import requests -from pathlib import Path -from datetime import datetime -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.common.action_chains import ActionChains -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.chrome.service import Service -from selenium.common.exceptions import WebDriverException -from selenium.common.exceptions import NoSuchElementException -from selenium.common.exceptions import StaleElementReferenceException -from selenium.webdriver.support.ui import Select -from selenium.webdriver.common.keys import Keys - -try: - from pywinauto import Application -except Exception: - Application = None - -# ================= DOCUMENT AI CONFIG ================= -DOCAI_CREDENTIALS_PATH = r"C:\Potential Project G2\.env\pdf-extraction-492807-41531f078f92.json" -DOCAI_PROJECT_ID = "1065395041789" -DOCAI_LOCATION = "us" -DOCAI_PROCESSOR_ID = "2d4c7121ce241b13" - -# ================= PROJECT ROOT ================= -if getattr(sys, "frozen", False): - PROJECT_ROOT = Path(sys.executable).parent -else: - PROJECT_ROOT = Path(__file__).resolve().parent.parent - -# ================= GLOBAL CONFIG ================= -DEBUG_PORT = 9222 -USER_DATA_DIR = os.path.join( - os.environ["USERPROFILE"], "Automation", "ChromeProfile_CF" -) -today_date = datetime.now().strftime("%m-%d-%Y") -DOWNLOADS_DIR = Path.home() / "Downloads" - - -# ================= CHROME PATH ================= -def find_chrome_path(): - possible_paths = [ - r"C:\Program Files\Google\Chrome\Application\chrome.exe", - r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", - os.path.join( - os.environ.get("LOCALAPPDATA", ""), - r"Google\Chrome\Application\chrome.exe" - ), - ] - for path in possible_paths: - if os.path.exists(path): - return path - return None - - -# ================= PROXY CONFIG ================= -def read_input_config(): - config_path = PROJECT_ROOT / "Input" / "input.xlsx" - if not config_path.exists(): - print("[ERROR] input.xlsx not found.") - sys.exit(1) - df = pd.read_excel(config_path, sheet_name="Sheet1", header=None) - config = {} - for _, row in df.iterrows(): - key = str(row[0]).strip() - value = str(row[1]).strip() - config[key] = value - proxy_ip = config.get("Proxy IP", "") - proxy_port = config.get("Proxy Port", "") - waystar_username = config.get("Waystar Username", "") - waystar_password = config.get("Waystar Password", "") - portal_url = config.get("Waystar URL", "") - first_car_answer = config.get("Waystar SQ First Car", "") - first_job_answer = config.get("Waystar SQ First Job", "") - print(f"[INFO] Proxy → {proxy_ip}:{proxy_port}") - print(f"[INFO] Username → {waystar_username}") - print(f"[INFO] Portal URL → {portal_url}") - return ( - proxy_ip, proxy_port, waystar_username, waystar_password, - portal_url, first_car_answer, first_job_answer, - ) - - -# ================= DEBUG PORT CHECK ================= -def wait_for_debug_port(port=9222, timeout=40): - start = time.time() - while time.time() - start < timeout: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - if s.connect_ex(("127.0.0.1", port)) == 0: - return True - time.sleep(1) - return False - - -# ================= KILL CHROME ================= -def kill_all_chrome(): - try: - os.system("taskkill /F /IM chrome.exe /T >nul 2>&1") - time.sleep(3) - print("[INFO] All Chrome instances closed") - except Exception as e: - print(f"[WARN] Unable to kill Chrome: {e}") - - -# ================= START CHROME DEBUG ================= -def start_chrome_debug(proxy_ip=None, proxy_port=None): - chrome_path = find_chrome_path() - if not chrome_path: - raise Exception("Google Chrome not found.") - print(f"[INFO] Using Chrome path: {chrome_path}") - os.makedirs(USER_DATA_DIR, exist_ok=True) - cmd = [ - chrome_path, - f"--remote-debugging-port={DEBUG_PORT}", - f"--user-data-dir={USER_DATA_DIR}", - "--disable-blink-features=AutomationControlled", - "--ignore-certificate-errors", - "about:blank", - ] - if proxy_ip and proxy_port: - proxy = f"{proxy_ip}:{proxy_port}" - cmd.append(f"--proxy-server=http://{proxy}") - print(f"[INFO] Chrome started with proxy → {proxy}") - else: - print("[INFO] Chrome started WITHOUT proxy") - print("[INFO] Launching Chrome...") - subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - if not wait_for_debug_port(DEBUG_PORT, timeout=40): - print("[ERROR] Chrome debug port did not open") - return False - print("[INFO] Chrome debug port is ready") - return True - - -# ================= SCREENSHOT ================= -def capture_screenshot(driver, client_name, step_name): - if not driver: - return None - safe_client = "".join(c for c in client_name if c.isalnum() or c in ("_", "-")) - safe_step = "".join(c for c in step_name if c.isalnum() or c in ("_", "-")) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - screenshots_dir = PROJECT_ROOT / "Screenshots" / today_date / safe_client - screenshots_dir.mkdir(parents=True, exist_ok=True) - screenshot_path = screenshots_dir / f"{safe_step}_{timestamp}.png" - try: - driver.save_screenshot(str(screenshot_path)) - print(f"[INFO] Screenshot saved: {screenshot_path}") - return screenshot_path - except Exception as e: - print(f"[ERROR] Screenshot failed: {e}") - return None - - -# ================= ATTACH BROWSER ================= -def attach_browser(download_path, retries=5): - for attempt in range(1, retries + 1): - try: - print(f"[INFO] Attaching browser (attempt {attempt})") - chrome_options = Options() - chrome_options.add_experimental_option( - "debuggerAddress", f"127.0.0.1:{DEBUG_PORT}" - ) - chrome_options.set_capability("webSocketUrl", False) - prefs = { - "download.default_directory": str(download_path), - "download.prompt_for_download": False, - "download.directory_upgrade": True, - "plugins.always_open_pdf_externally": True, - "safebrowsing.enabled": True, - } - chrome_options.add_experimental_option("prefs", prefs) - service = Service() - driver = webdriver.Chrome(service=service, options=chrome_options) - _ = driver.current_url - driver.maximize_window() - print("[INFO] Browser attached successfully") - try: - driver.execute_cdp_cmd( - "Page.setDownloadBehavior", - {"behavior": "allow", "downloadPath": str(download_path)}, - ) - except Exception as e: - print(f"[WARN] Unable to set download behavior: {e}") - return driver - except WebDriverException as e: - print(f"[WARN] Attach attempt {attempt} failed: {e}") - time.sleep(3) - print("[ERROR] Failed to attach Chrome after retries") - return None - - -# ================= LOGIN ================= -def login_to_portal(driver, waystar_username, waystar_password, - first_car_answer, first_job_answer): - wait = WebDriverWait(driver, 30) - try: - time.sleep(5) - print("[INFO] Starting login process") - username_field = wait.until(EC.presence_of_element_located((By.ID, "loginName"))) - username_field.clear() - username_field.send_keys(waystar_username) - time.sleep(2) - password_field = wait.until(EC.presence_of_element_located((By.ID, "password"))) - password_field.clear() - password_field.send_keys(waystar_password) - time.sleep(2) - driver.find_element(By.ID, "loginButton").click() - time.sleep(5) - print("[INFO] Login clicked") - try: - alert_page = driver.find_elements( - By.XPATH, "//span[contains(text(),'Proactive Alert')]" - ) - if alert_page: - wait.until( - EC.element_to_be_clickable((By.XPATH, "//button[@id='continueButton']")) - ).click() - time.sleep(3) - except Exception: - pass - try: - question_exists = driver.find_elements( - By.XPATH, "//*[@id='authDiv']//strong" - ) - if question_exists: - question = wait.until( - EC.presence_of_element_located((By.XPATH, "//*[@id='authDiv']//strong")) - ).text.lower() - print(f"[INFO] Security Question → {question}") - if "first car" in question: - answer = first_car_answer - elif "first job" in question: - answer = first_job_answer - else: - raise Exception(f"Unknown security question: {question}") - answer_box = wait.until( - EC.presence_of_element_located((By.ID, "verifyAnswer")) - ) - answer_box.clear() - answer_box.send_keys(answer) - driver.find_element(By.ID, "VerifyButton").click() - print("[INFO] Security answer submitted") - time.sleep(3) - except Exception: - pass - try: - security_qpage = driver.find_elements( - By.XPATH, "//h2[normalize-space()='Security Information']" - ) - if security_qpage: - print("[INFO] Security page appeared, skipping") - wait.until(EC.element_to_be_clickable((By.ID, "NextButton"))).click() - time.sleep(2) - wait.until( - EC.element_to_be_clickable( - (By.XPATH, "//a[normalize-space()='Skip This Step']") - ) - ).click() - time.sleep(5) - except Exception: - pass - print("[INFO] Login completed successfully") - except Exception as e: - print(f"[ERROR] Login failed: {e}") - raise - - -# ================= CALCULATE DATE RANGE ================= -def get_date_range(): - now = datetime.now() - first_of_current = datetime(now.year, now.month, 1) - last_of_prev = first_of_current - pd.Timedelta(days=1) - first_of_prev = datetime(last_of_prev.year, last_of_prev.month, 1) - return ( - first_of_prev.strftime("%m/%d/%Y"), - last_of_prev.strftime("%m/%d/%Y"), - ) - - -# ================= APPLY FILTERS ================= -def apply_filters(driver): - wait = WebDriverWait(driver, 40) - print("[INFO] Applying filters") - wait.until(EC.presence_of_element_located((By.ID, "ClaimListingSearchButtonBottom"))) - wait.until(EC.presence_of_element_located((By.ID, "SearchCriteria_Status"))) - status_dropdown = Select( - wait.until(EC.element_to_be_clickable((By.ID, "SearchCriteria_Status"))) - ) - status_dropdown.select_by_visible_text("All") - date_dropdown = Select( - wait.until(EC.element_to_be_clickable((By.ID, "SearchCriteria_TransDate"))) - ) - date_dropdown.select_by_visible_text("Custom") - time.sleep(2) - start_date, end_date = get_date_range() - print(f"[INFO] Setting date range: {start_date} to {end_date}") - try: - start_date_field = wait.until( - EC.presence_of_element_located((By.ID, "transactionDateFromDateBox")) - ) - start_date_field.clear() - start_date_field.send_keys(start_date) - time.sleep(1) - except Exception as e: - print(f"[WARN] Could not set start date: {e}") - try: - end_date_field = wait.until( - EC.presence_of_element_located((By.ID, "transactionDateToDateBox")) - ) - end_date_field.clear() - end_date_field.send_keys(end_date) - time.sleep(1) - except Exception as e: - print(f"[WARN] Could not set end date: {e}") - search_btn = wait.until( - EC.element_to_be_clickable((By.ID, "ClaimListingSearchButtonBottom")) - ) - driver.execute_script("arguments[0].click();", search_btn) - wait.until( - EC.presence_of_element_located( - (By.XPATH, "//table[@id='claimsGrid']//tbody//tr") - ) - ) - time.sleep(3) - print("[INFO] Filters applied successfully") - - -# ================= CLICK NEXT PAGE ================= -def click_next_page(driver): - try: - next_btn = driver.find_element(By.XPATH, "//button[@id='innerPagerRight']") - if "disabled" in next_btn.get_attribute("class"): - return False - driver.execute_script("arguments[0].click();", next_btn) - WebDriverWait(driver, 10).until(EC.staleness_of(next_btn)) - time.sleep(2) - return True - except Exception: - return False - - -# ================= DIRECT DOWNLOAD ================= -def download_pdf_direct(driver, pdf_url, save_path): - try: - cookies = {c["name"]: c["value"] for c in driver.get_cookies()} - headers = { - "User-Agent": driver.execute_script("return navigator.userAgent;"), - "Referer": driver.current_url, - "Accept": "application/pdf,*/*", - } - response = requests.get( - pdf_url, headers=headers, cookies=cookies, stream=True, timeout=60 - ) - if response.status_code != 200: - print(f"[ERROR] HTTP {response.status_code} → {pdf_url}") - return False - content_type = response.headers.get("Content-Type", "") - if "pdf" not in content_type.lower() and "octet-stream" not in content_type.lower(): - print(f"[WARN] Unexpected content type → {content_type}") - save_path = Path(save_path) - save_path.parent.mkdir(parents=True, exist_ok=True) - with open(save_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - print(f"[SUCCESS] Downloaded → {save_path.name}") - return True - except Exception as e: - print(f"[ERROR] Direct download failed → {e}") - return False - - -# ============================================================ -# CMS-1500 DOCUMENT AI EXTRACTOR — v7.0 (PRODUCTION) -# -# Key guarantees: -# • DOS : Serial-number gutter (x < 0.045) excluded — row -# numbers like "01","02" never picked up as MM. -# • DX : Single, authoritative _is_icd10() covering all -# ICD-10-CM forms up to 7 chars (S81812A, G20C, -# R2681, Z7409, dotted A01.0 etc.). -# • CPT : Accepts standard 5-digit AND HCPCS alpha codes -# (Q4250, G0283, L8699). -# • Units: Primary lookup confined to tight x-range; fallback -# also constrained to same x-range — charges column -# never bleeds in. Diagnosis-pointer letters (A-L) -# are explicitly rejected. -# ============================================================ - -# ── Diagnosis pointer letters (column E of section 24) ────── -_DIAG_POINTER_LETTERS = set("ABCDEFGHIJKL") - -# ── Normalized field regions ───────────────────────────────── -FIELD_1A = { - "x_min": 0.55, "x_max": 0.95, - "y_min": 0.095, "y_max": 0.185, -} - -FIELD_21 = { - "x_min": 0.02, "x_max": 0.68, - "y_min": 0.510, "y_max": 0.590, -} - -SECTION_24_Y_MIN = 0.615 -SECTION_24_Y_MAX = 0.880 - -# ── Column x-ranges (normalized 0–1) ───────────────────────── -# DOS_FROM_MM x_min = 0.045 ← excludes left-gutter row serial -# numbers printed at x ≈ 0.01–0.035. -COL = { - "DOS_FROM_MM": (0.045, 0.085), - "DOS_FROM_DD": (0.078, 0.120), - "DOS_FROM_YY": (0.112, 0.165), - "POS": (0.215, 0.272), - "CPT": (0.285, 0.430), - "MODIFIER": (0.410, 0.540), - "UNITS": (0.648, 0.695), - "RENDERING": (0.810, 0.998), -} - -_VALID_MODIFIERS = { - "GP","GN","GO","GQ","GX","GY","GZ", - "LT","RT", - "E1","E2","E3","E4", - "FA","F1","F2","F3","F4","F5","F6","F7","F8","F9", - "TA","T1","T2","T3","T4","T5","T6","T7","T8","T9", - "TC","26","59","25","51","76","77","78","79","91", - "24","57","58","80","81","82", - "AA","AD","AE","AF","AG","AH","AI","AJ","AK","AM","AS","AT", - "CQ","CR","CS","GA","GC","GJ","GK","GL","GT", - "JA","JB","JW","JZ","KX","KU","KV","NU","RR","UE", - "Q0","Q1","Q2","Q3","Q4","Q5","Q6", - "SA","SB","SC","SD","SF","SG","XE","XP","XS","XU", - "22","23","32","33","47","50","52","53", - "54","55","56","62","63","66", - "73","74","90","92","93","95","96","97","99", - "AB", -} - -_NOT_MODIFIERS = { - "MM","DD","YY","TO","OF","OR","IN","IS","NO","BY", - "AT","BE","DO","HE","HI","IF","ME","MY","OK","ON", - "SO","UP","US","WE","NPI","ICD","CPT","POS","DOS", - "ID","AN","AC","EMG","FROM","THE","AND", -} - -_VALID_POS = { - "11","12","13","14","15","16","17","18","19","20", - "21","22","23","24","25","31","32","33","34", - "41","42","49","50","51","52","53","54","55","56","57", - "60","61","62","65","71","72","81","99", -} - -_SKIP_ID_LABELS = { - "MEDICARE","MEDICAID","CHAMPVA","TRICARE","CHAMPUS", - "NUMBER","INSURED","MEMBER","HEALTH","PLAN","PICA", - "PROGRAM","ITEM","FOR","OTHER","GROUP","FECA","CARRIER", - "1A","FOR","IN","ITEM","NPI","SSN","EIN","QUAL", - "ID","INSURED'S","I.D.","OTHER","(FOR","(",")", - "1)","IN","ITEM","1", -} - -# ── DX slot labels printed on the form (A. B. … L.) ───────── -_DX_SLOT_LABEL_PAT = re.compile(r"^[A-L]\.?$", re.IGNORECASE) - -# ── Form section / header tokens to skip in field 21 ──────── -_DX_SKIP_TOKENS = { - "21", "22", "23", "24", "0", "O", - "ICD", "IND", "ICO", "ICD-10-CM", - "DIAGNOSIS", "NATURE", "ILLNESS", "INJURY", -} - - -# ============================================================ -# DOCUMENT AI CLIENT -# ============================================================ - -def _get_docai_client(): - credentials = service_account.Credentials.from_service_account_file( - DOCAI_CREDENTIALS_PATH, - scopes=["https://www.googleapis.com/auth/cloud-platform"], - ) - return documentai.DocumentProcessorServiceClient( - credentials=credentials, - client_options={"api_endpoint": "us-documentai.googleapis.com"}, - ) - - -def _process_pdf_with_docai(pdf_path: str): - client = _get_docai_client() - with open(pdf_path, "rb") as f: - pdf_bytes = f.read() - processor_name = ( - f"projects/{DOCAI_PROJECT_ID}/locations/{DOCAI_LOCATION}" - f"/processors/{DOCAI_PROCESSOR_ID}" - ) - result = client.process_document( - request=documentai.ProcessRequest( - name=processor_name, - raw_document=documentai.RawDocument( - content=pdf_bytes, mime_type="application/pdf" - ), - ) - ) - return result.document - - -# ============================================================ -# TOKEN / WORD HELPERS -# ============================================================ - -def _token_text(document, token) -> str: - seg = token.layout.text_anchor.text_segments - if not seg: - return "" - start = int(seg[0].start_index) if seg[0].start_index else 0 - end = int(seg[0].end_index) - return document.text[start:end].strip() - - -def _token_norm_bounds(token): - verts = token.layout.bounding_poly.normalized_vertices - if len(verts) < 4: - return None - xs = [v.x for v in verts] - ys = [v.y for v in verts] - return min(xs), min(ys), max(xs), max(ys) - - -def _build_words(document, page) -> list: - words = [] - for token in page.tokens: - text = _token_text(document, token).strip() - if not text: - continue - bounds = _token_norm_bounds(token) - if bounds is None: - continue - x0, y_top, x1, y_bot = bounds - words.append({ - "text": text, - "x0": x0, - "x1": x1, - "y_top": y_top, - "y_bot": y_bot, - "x_mid": (x0 + x1) / 2, - "y_mid": (y_top + y_bot) / 2, - }) - return words - - -def _words_in_region(words: list, region: dict) -> list: - return [ - w for w in words - if region["x_min"] <= w["x_mid"] <= region["x_max"] - and region["y_min"] <= w["y_mid"] <= region["y_max"] - ] - - -def _words_in_col(words: list, row_y_mid: float, - col_name: str, y_tol: float = 0.013) -> list: - x_min, x_max = COL[col_name] - return [ - w for w in words - if x_min <= w["x_mid"] <= x_max - and abs(w["y_mid"] - row_y_mid) <= y_tol - ] - - -# ============================================================ -# VALIDATORS -# ============================================================ - -def _is_icd10(t: str) -> bool: - """ - Validates ICD-10-CM codes — single authoritative definition. - - Accepts ALL of: - • Dotted : A01.0 M54.51 S72.001A (letter + 2-3 digits + dot + 1-4 chars + optional suffix) - • Undotted: F329 Z7409 R2681 G20C S81812A M80011A - (letter + digit + 1-5 alphanumeric, total 3-7 chars) - - Rule: first char = letter, second char = digit (ICD-10-CM spec). - Maximum valid length = 7 characters. - """ - t = t.upper().strip() - if len(t) < 3 or len(t) > 7: - return False - if not t[0].isalpha() or not t[1].isdigit(): - return False - # Dotted form: e.g. A01.0, M54.51, S72.001A - if re.fullmatch(r"[A-Z]\d{2,3}\.\d{1,4}[A-Z]?", t): - return True - # Undotted form: letter + digit + 1-5 alphanumeric chars - if re.fullmatch(r"[A-Z]\d[0-9A-Z]{1,5}", t): - return True - return False - - -def _is_cpt(t: str) -> bool: - """ - Validates CPT / HCPCS codes. - - Accepts: - • Standard CPT (5 digits) : 97110, 99348, 15271 - • HCPCS Level II (letter + 4 digits) : Q4250, G0283, L8699, A6530 - • Category III CPT (4 digits + T/F) : 0596T, 0163F - """ - t = t.strip().upper() - # Standard 5-digit numeric CPT - if re.fullmatch(r"\d{5}", t): - return 10000 <= int(t) <= 99999 - # HCPCS Level II: A–V letter + 4 digits - if re.fullmatch(r"[A-V]\d{4}", t): - return True - # Category III CPT: 4 digits + T or F suffix - if re.fullmatch(r"\d{4}[TF]", t): - return True - return False - - -def _is_valid_insured_id(t: str) -> bool: - t_clean = re.sub(r"[-_.\s()]", "", t.upper()) - if not re.match(r"^[A-Z0-9]+$", t_clean): - return False - length = len(t_clean) - if length < 5 or length > 20: - return False - base = t.upper().rstrip(".,)(") - if base in _SKIP_ID_LABELS: - return False - if any(label in t.upper() - for label in { - "INSURED", "NUMBER", "PROGRAM", "MEDICARE", "MEDICAID", - "PICA", "CARRIER", "HEALTH", "PLAN", "CHAMPVA", "TRICARE", - }): - return False - if re.fullmatch(r"\d{10}", t_clean): # NPI - return False - if _is_cpt(t_clean): # CPT / HCPCS - return False - if re.fullmatch(r"[A-Z]{1,5}", t_clean): # abbreviations - return False - if re.fullmatch(r"\d+", t_clean): - return 7 <= length <= 12 - if re.search(r"[A-Z]", t_clean) and re.search(r"\d", t_clean): - return True - return False - - -def _is_valid_dos(mm: str, dd: str, yy: str) -> bool: - try: - m, d, y = int(mm), int(dd), int(yy) - return (1 <= m <= 12) and (1 <= d <= 31) and (2000 <= y <= 2099) - except Exception: - return False - - -# ============================================================ -# FIELD EXTRACTORS -# ============================================================ - -def _extract_dob(text: str) -> str: - m = re.search(r"\b(\d{2})\s+(\d{2})\s+(\d{4})\b", text) - if m: - return f"{m.group(1)}/{m.group(2)}/{m.group(3)}" - m = re.search(r"\b(\d{2})[\/\-](\d{2})[\/\-](\d{4})\b", text) - if m: - return f"{m.group(1)}/{m.group(2)}/{m.group(3)}" - return "" - - -def _extract_insured_id(words: list, text: str) -> str: - wide_region = {"x_min": 0.50, "x_max": 0.95, - "y_min": 0.08, "y_max": 0.20} - wide_tokens = _words_in_region(words, wide_region) - print(f"[DEBUG] Field 1a wide region tokens: " - f"{[(w['text'], round(w['y_mid'], 3)) for w in wide_tokens]}") - - # Primary: region scan - candidates = _words_in_region(words, FIELD_1A) - for w in sorted(candidates, key=lambda w: (w["y_top"], w["x0"])): - t = w["text"].strip() - if _is_valid_insured_id(t): - print(f"[DEBUG] Insured ID via region: {t}") - return t.upper() - - # Fallback 1: look for label "1a" then scan right - for i, w in enumerate(words): - if re.fullmatch(r"1[aA]\.?", w["text"].strip()): - for nxt in words[i + 1: i + 15]: - if nxt["x_mid"] > 0.50 and nxt["y_mid"] < 0.22: - if _is_valid_insured_id(nxt["text"].strip()): - print(f"[DEBUG] Insured ID via 1a label: {nxt['text']}") - return nxt["text"].strip().upper() - - # Fallback 2: top-right area scan - top_right = [w for w in words if w["x_mid"] > 0.52 and w["y_mid"] < 0.22] - for w in sorted(top_right, key=lambda w: (w["y_top"], w["x0"])): - t = w["text"].strip() - if _is_valid_insured_id(t): - print(f"[DEBUG] Insured ID via top-right scan: {t}") - return t.upper() - - print("[DEBUG] Insured ID: not found") - return "" - - -def _extract_diagnosis_codes(words: list, text: str) -> list: - """ - Extract ICD-10-CM diagnosis codes from Field 21 of the CMS-1500. - Captures everything: dotted, undotted, suffix letters, 3-7 chars. - """ - region_words = _words_in_region(words, FIELD_21) - print(f"[DEBUG] Field 21 region token count: {len(region_words)}") - print(f"[DEBUG] Field 21 tokens: {[w['text'] for w in region_words]}") - - codes = [] - seen = set() - - for w in sorted(region_words, key=lambda w: (w["y_top"], w["x0"])): - t = w["text"].strip().upper() - - # Skip slot label letters printed on the form (A. B. … L.) - if _DX_SLOT_LABEL_PAT.match(t): - continue - - # Skip form header / section labels - if t in _DX_SKIP_TOKENS: - continue - - # Skip pure digits — no ICD-10 code is all-numeric - if t.isdigit(): - continue - - # Accept valid ICD-10-CM codes - if _is_icd10(t) and t not in seen: - codes.append(t) - seen.add(t) - print(f"[DEBUG] DX captured: {t}") - - # ── Text-regex fallback when region yields nothing ──────── - if not codes: - print("[DEBUG] Region empty — trying text regex fallback") - seg = re.search( - r"(?:DIAGNOSIS|21\.?)([\s\S]{0,600})" - r"(?:24\.|DATE.S.\s*OF\s*SERVICE)", - text, re.IGNORECASE, - ) - if seg: - # Broadened fallback: letter + digit + 1-5 alphanum, - # with optional dot + 1-4 chars - for c in re.findall( - r"\b([A-Z]\d[0-9A-Z]{1,5}(?:\.[0-9A-Z]{1,4})?)\b", - seg.group(1), - ): - cu = c.upper() - if cu not in seen and _is_icd10(cu): - seen.add(cu) - codes.append(cu) - print(f"[DEBUG] DX fallback: {cu}") - - print(f"[DEBUG] Diagnosis codes final ({len(codes)}): {codes}") - return codes - - -# ============================================================ -# SERVICE LINE HELPERS -# ============================================================ - -def _is_real_cpt_location(w: dict, words: list) -> bool: - """ - Confirm a CPT token truly belongs to section 24 by requiring - a DOS digit OR a 10-digit NPI on the same row. - """ - row_y = w["y_mid"] - y_tol = 0.018 - - # DOS columns — use updated x_min values (gutter excluded) - for col in ("DOS_FROM_MM", "DOS_FROM_DD", "DOS_FROM_YY"): - x_min, x_max = COL[col] - dos_hit = [ - d for d in words - if x_min <= d["x_mid"] <= x_max - and abs(d["y_mid"] - row_y) <= y_tol - and re.fullmatch(r"\d{1,4}", d["text"].strip()) - ] - if dos_hit: - return True - - # Rendering NPI column - x_min_npi, x_max_npi = COL["RENDERING"] - npi_hit = [ - d for d in words - if x_min_npi <= d["x_mid"] <= x_max_npi - and abs(d["y_mid"] - row_y) <= y_tol - and re.fullmatch(r"\d{10}", d["text"].strip()) - ] - return bool(npi_hit) - - -def _extract_dos_for_row(words: list, row_y: float) -> str: - """ - Extract Date of Service for a service line row. - Gutter guard: skip tokens at x_mid < 0.025 only - (row serial numbers are printed at x < 0.02). - """ - def _first_digit(col_name): - ws = _words_in_col(words, row_y, col_name, y_tol=0.018) - for w in sorted(ws, key=lambda w: w["x0"]): - # Only skip extreme left gutter (row numbers like "1","2","3") - if w["x_mid"] < 0.025: - continue - if re.fullmatch(r"\d{1,4}", w["text"].strip()): - return w["text"].strip() - return "" - - mm = _first_digit("DOS_FROM_MM").zfill(2) - dd = _first_digit("DOS_FROM_DD").zfill(2) - yy = _first_digit("DOS_FROM_YY") - - if re.fullmatch(r"\d{2}", yy): - yy = ("20" + yy) if int(yy) < 50 else ("19" + yy) - - if _is_valid_dos(mm, dd, yy): - return f"{mm}/{dd}/{yy}" - - # Fallback: collect all digit tokens across DOS columns sorted left→right - all_w = [] - for col in ("DOS_FROM_MM", "DOS_FROM_DD", "DOS_FROM_YY"): - all_w += [ - w for w in _words_in_col(words, row_y, col, y_tol=0.018) - if w["x_mid"] >= 0.025 # only skip extreme gutter - ] - - parts = [ - w["text"].strip() - for w in sorted(all_w, key=lambda w: w["x0"]) - if re.fullmatch(r"\d+", w["text"].strip()) - ] - - print(f"[DEBUG DOS] row_y={row_y:.4f} | mm={mm} dd={dd} yy={yy} | " - f"fallback parts={parts}") - - if len(parts) >= 3: - mm2 = parts[0].zfill(2) - dd2 = parts[1].zfill(2) - yy2 = parts[2] - if re.fullmatch(r"\d{2}", yy2): - yy2 = ("20" + yy2) if int(yy2) < 50 else ("19" + yy2) - if _is_valid_dos(mm2, dd2, yy2): - return f"{mm2}/{dd2}/{yy2}" - - return "" - -def _extract_pos_for_row(words: list, row_y: float) -> str: - for w in _words_in_col(words, row_y, "POS"): - if w["text"].strip() in _VALID_POS: - return w["text"].strip() - return "" - - -def _extract_modifier_for_row(words: list, row_y: float) -> str: - mod_words = _words_in_col(words, row_y, "MODIFIER", y_tol=0.016) - found = [] - for w in sorted(mod_words, key=lambda w: w["x0"]): - t = w["text"].strip().upper() - if not t: - continue - # Skip diagnosis pointer letters - if re.fullmatch(r"[A-L]", t): - continue - if t in _NOT_MODIFIERS: - continue - if t in _VALID_MODIFIERS and t not in found: - found.append(t) - if len(found) == 4: - break - return " ".join(found) - - -def _extract_units_for_row(words: list, row_y: float, - cpt_val: str = "", mod_val: str = "", - pos_val: str = "", npi_val: str = "") -> str: - """ - Extract units (G column) for a service line row. - - Strategy: elimination-based. - On the CMS-1500, section 24 columns left→right are: - DOS | POS | EMG | CPT | MODIFIER | DIAG-POINTER | CHARGES | UNITS | RENDERING - Units is a small integer (1-99) that sits between the - diagnosis-pointer column and the rendering NPI column. - We collect ALL tokens on the row, ban every token already - claimed by a named column, then pick the leftmost remaining - 1-2 digit integer that falls in the right half of the form - (x_mid > 0.54) but left of the rendering column (x_mid < 0.81). - """ - y_tol = 0.022 - - # ── Debug: print all tokens on this row so we can calibrate ── - all_row = [ - w for w in words - if abs(w["y_mid"] - row_y) <= y_tol - and SECTION_24_Y_MIN <= w["y_mid"] <= SECTION_24_Y_MAX - ] - print(f"[DEBUG UNITS] All tokens on row y≈{row_y:.4f}: " - f"{[(w['text'], round(w['x_mid'], 4)) for w in sorted(all_row, key=lambda w: w['x_mid'])]}") - - # ── Build ban-set from all other named columns ──────────── - ban = set() - # Values already extracted for this row - for v in (cpt_val, pos_val, npi_val): - if v: - ban.add(v.upper()) - if mod_val: - for m in mod_val.split(): - ban.add(m.upper()) - - # Tokens from every other named column - for col in ("DOS_FROM_MM", "DOS_FROM_DD", "DOS_FROM_YY", - "POS", "CPT", "MODIFIER", "RENDERING"): - for w in _words_in_col(words, row_y, col, y_tol=y_tol + 0.005): - ban.add(w["text"].strip().upper()) - - print(f"[DEBUG UNITS] Ban set: {ban}") - - # ── Find the actual NPI left-edge to use as hard right wall ─ - # This adapts to whatever x the NPI lands on for this form. - npi_x_boundary = 0.998 # fallback if no NPI found on this row - for w in words: - if (abs(w["y_mid"] - row_y) <= y_tol - and re.fullmatch(r"\d{10}", w["text"].strip())): - npi_x_boundary = min(npi_x_boundary, w["x0"]) - - # ── Candidate zone: between modifier area and NPI left edge ─ - UNIT_X_MIN = 0.540 # right of modifier / diag-pointer - UNIT_X_MAX = npi_x_boundary # left wall of NPI token - - print(f"[DEBUG UNITS] Search zone x={UNIT_X_MIN:.3f}→{UNIT_X_MAX:.3f}") - - candidates = [ - w for w in words - if UNIT_X_MIN <= w["x_mid"] <= UNIT_X_MAX - and abs(w["y_mid"] - row_y) <= y_tol - and SECTION_24_Y_MIN <= w["y_mid"] <= SECTION_24_Y_MAX - and w["text"].strip().upper() not in ban - and w["text"].strip().upper() not in _DIAG_POINTER_LETTERS - and re.fullmatch(r"[1-9]\d?", w["text"].strip()) # 1-99, never 4-digit charges - ] - - print(f"[DEBUG UNITS] Candidates after ban: " - f"{[(w['text'], round(w['x_mid'], 4)) for w in sorted(candidates, key=lambda w: w['x_mid'])]}") - - if candidates: - # Rightmost qualifying token = closest to NPI = units column - best = sorted(candidates, key=lambda w: w["x_mid"], reverse=True)[0] - print(f"[DEBUG UNITS] → Selected: {best['text']} at x={best['x_mid']:.4f}") - return best["text"].strip() - - return "" - - -def _extract_npi_for_row(words: list, row_y: float) -> str: - for w in _words_in_col(words, row_y, "RENDERING", y_tol=0.016): - if re.fullmatch(r"\d{10}", w["text"].strip()): - return w["text"].strip() - return "" - - -# ============================================================ -# SERVICE LINE EXTRACTION -# ============================================================ - -def _extract_service_lines(words: list): - cpts, pos_list, dos_list, mod_list, units_list, rp_list = [], [], [], [], [], [] - - x_min_cpt, x_max_cpt = COL["CPT"] - - # Find CPT / HCPCS candidates in section 24 - cpt_candidates = [ - w for w in words - if _is_cpt(w["text"].strip()) - and x_min_cpt <= w["x_mid"] <= x_max_cpt - and SECTION_24_Y_MIN <= w["y_mid"] <= SECTION_24_Y_MAX - ] - print(f"[DEBUG] CPT candidates before validation: " - f"{[(w['text'], round(w['y_mid'], 4)) for w in cpt_candidates]}") - - cpt_words = [w for w in cpt_candidates if _is_real_cpt_location(w, words)] - print(f"[DEBUG] CPT tokens after validation: " - f"{[(w['text'], round(w['y_mid'], 4)) for w in cpt_words]}") - - if not cpt_words: - print("[DEBUG] No valid CPTs after validation — trying wider scan") - wider_candidates = [ - w for w in words - if _is_cpt(w["text"].strip()) - and 0.55 <= w["y_mid"] <= 0.92 - and 0.20 <= w["x_mid"] <= 0.50 - and _is_real_cpt_location(w, words) - ] - cpt_words = wider_candidates - print(f"[DEBUG] Wider scan CPTs: {[w['text'] for w in cpt_words]}") - - seen_rows = [] - for cpt_word in sorted(cpt_words, key=lambda w: w["y_mid"]): - row_y = cpt_word["y_mid"] - if any(abs(row_y - sr) <= 0.008 for sr in seen_rows): - continue - seen_rows.append(row_y) - - cpt_val = cpt_word["text"].strip().upper() - dos = _extract_dos_for_row(words, row_y) - pos = _extract_pos_for_row(words, row_y) - mod = _extract_modifier_for_row(words, row_y) - npi = _extract_npi_for_row(words, row_y) - units = _extract_units_for_row(words, row_y, cpt_val, mod, pos, npi) - - cpts.append(cpt_val) - dos_list.append(dos) - pos_list.append(pos) - mod_list.append(mod) - units_list.append(units) - rp_list.append(npi) - - print(f"[DEBUG] CPT={cpt_val} | DOS={dos} | POS={pos} | " - f"MOD={mod} | UNITS={units} | NPI={npi}") - - return cpts, pos_list, dos_list, mod_list, units_list, rp_list - - -def _format_list(values: list, label: str) -> str: - clean = [v for v in values if v] - if not clean: - return "" - if len(clean) == 1: - return clean[0] - return "; ".join(f"{label} {i + 1}: {v}" for i, v in enumerate(clean)) - - -# ============================================================ -# MAIN EXTRACTION ENTRY POINT -# ============================================================ - -def extract_pdf_data_fast( - pdf_path, - claim_id="", - patient_account="", - insurance="", - patient_name="", - total_charges="", - signature_physician="", -): - data = { - "Claim ID": claim_id, - "Rendering Provider #": "", - "Signature of Physician": signature_physician, - "Patient's Account No": patient_account, - "Date of Birth": "", - "Insurance Name": insurance, - "Insured ID Number": "", - "Patient's Name": patient_name, - "Date of Service": "", - "Diagnosis Codes": "", - "CPT": "", - "POS": "", - "Modifier": "", - "Units": "", - "Charges": total_charges, - } - - try: - print(f"[INFO] Processing with Document AI: {pdf_path}") - document = _process_pdf_with_docai(str(pdf_path)) - - if not document.pages: - print("[ERROR] No pages returned by Document AI") - return None - - page = document.pages[0] - text = document.text - - print(f"[DEBUG] Document AI extracted {len(text)} characters") - print(f"[DEBUG] Token count: {len(page.tokens)}") - print(f"[DEBUG] Page dimensions: " - f"{page.dimension.width:.1f} x {page.dimension.height:.1f} pt") - - words = _build_words(document, page) - print(f"[DEBUG] Words built: {len(words)}") - - data["Date of Birth"] = _extract_dob(text) - data["Insured ID Number"] = _extract_insured_id(words, text) - - dx_codes = _extract_diagnosis_codes(words, text) - if dx_codes: - data["Diagnosis Codes"] = ( - ", ".join(dx_codes) + f" (Total: {len(dx_codes)})" - ) - - cpts, pos_list, dos_list, mod_list, units_list, rp_list = \ - _extract_service_lines(words) - - data["CPT"] = _format_list(cpts, "CPT") - data["POS"] = _format_list(pos_list, "POS") - data["Date of Service"] = _format_list(dos_list, "DOS") - data["Modifier"] = _format_list(mod_list, "Modifier") - data["Units"] = _format_list(units_list, "Units") - data["Rendering Provider #"] = _format_list(rp_list, "RP #") - - return data - - except Exception as e: - import traceback - print(f"[ERROR] Document AI extraction failed → {pdf_path}") - print(f"[ERROR] {e}") - traceback.print_exc() - return None - - -# ================= UPDATE LOG ================= -def update_log(log_path, claim_id, status, remarks=""): - now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - new_row = pd.DataFrame([{ - "Claim ID": claim_id, - "Status": status, - "Remarks": remarks, - "Timestamp": now, - }]) - try: - existing = pd.read_excel(log_path) - final = pd.concat([existing, new_row], ignore_index=True) - except Exception: - final = new_row - final.to_excel(log_path, index=False) - - -# ================= INCREMENTAL EXCEL SAVE ================= -def _save_batch_to_excel(excel_path, batch_data: list): - if not batch_data: - return - df_new = pd.DataFrame(batch_data) - try: - existing = pd.read_excel(excel_path) - existing = existing.dropna(how="all") - final = pd.concat([existing, df_new], ignore_index=True) - except Exception: - final = df_new - final.to_excel(excel_path, index=False) - print(f"[INFO] Saved {len(batch_data)} records → {excel_path.name} " - f"(total rows: {len(final)})") - - -def _get_processed_claim_ids(log_path) -> set: - try: - df = pd.read_excel(log_path) - done = df[df["Status"] == "SUCCESS"]["Claim ID"].astype(str).tolist() - return set(done) - except Exception: - return set() - - -# ================= RUN CLAIM REPORT ================= -def run_claim_report(driver, account_name): - SAVE_EVERY = 10 - wait = WebDriverWait(driver, 30) - actions = ActionChains(driver) - print(f"\n[INFO] ====== Running report for {account_name} ======") - - today = datetime.now().strftime("%m-%d-%Y") - safe_name = "".join( - c for c in account_name if c.isalnum() or c in (" ", "_", "(", ")") - ) - base = PROJECT_ROOT / "Output" / today - pdf_folder = base / "PDF Folder" / safe_name - report_dir = base / "reports" - logs_dir = base / "Logs" - - pdf_folder.mkdir(parents=True, exist_ok=True) - report_dir.mkdir(parents=True, exist_ok=True) - logs_dir.mkdir(parents=True, exist_ok=True) - - excel_path = report_dir / f"{safe_name}.xlsx" - log_path = logs_dir / f"{safe_name}_log.xlsx" - - if not excel_path.exists(): - pd.DataFrame(columns=[ - "Claim ID", "Rendering Provider #", "Signature of Physician", - "Patient's Account No", "Date of Birth", "Insurance Name", - "Insured ID Number", "Patient's Name", "Date of Service", - "Diagnosis Codes", "CPT", "POS", "Modifier", "Units", "Charges", - ]).to_excel(excel_path, index=False) - - if not log_path.exists(): - pd.DataFrame(columns=[ - "Claim ID", "Status", "Remarks", "Timestamp", - ]).to_excel(log_path, index=False) - - already_done = _get_processed_claim_ids(log_path) - print(f"[INFO] {len(already_done)} claims already processed for {account_name}") - - # Navigate to Claims - try: - claims_processing = wait.until( - EC.presence_of_element_located( - (By.XPATH, "//a[contains(.,'Claims Processing')]") - ) - ) - actions.move_to_element(claims_processing).perform() - time.sleep(1) - claims_option = wait.until( - EC.presence_of_element_located( - (By.XPATH, "//a[normalize-space()='Claims']") - ) - ) - driver.execute_script("arguments[0].click();", claims_option) - time.sleep(3) - except Exception as e: - print(f"[ERROR] Navigation to Claims failed: {e}") - return - - try: - apply_filters(driver) - except Exception as e: - print(f"[ERROR] Filter application failed: {e}") - return - - page = 1 - batch = [] - total_saved = 0 - consecutive_errors = 0 - - while True: - print(f"\n[INFO] Processing page {page}") - try: - WebDriverWait(driver, 20).until( - EC.presence_of_element_located( - (By.XPATH, "//table[@id='claimsGrid']//tbody//tr[@data-claimid]") - ) - ) - rows = driver.find_elements( - By.XPATH, "//table[@id='claimsGrid']//tbody//tr[@data-claimid]" - ) - print(f"[INFO] Rows on page {page} → {len(rows)}") - except Exception as e: - print(f"[ERROR] Could not load rows on page {page}: {e}") - consecutive_errors += 1 - if consecutive_errors >= 3: - print("[ERROR] 3 consecutive page-load failures — stopping account") - break - time.sleep(5) - continue - - consecutive_errors = 0 - - for i in range(len(rows)): - claim_id = f"row_{page}_{i + 1}" - try: - rows = driver.find_elements( - By.XPATH, - "//table[@id='claimsGrid']//tbody//tr[@data-claimid]", - ) - if i >= len(rows): - print(f"[WARN] Row {i + 1} no longer exists — skipping") - continue - - row = rows[i] - claim_id = row.get_attribute("data-claimid") or "" - instance_id = row.get_attribute("data-instanceid") or "" - - if not claim_id or not instance_id: - print(f"[WARN] Missing IDs row {i + 1} — skipping") - continue - if claim_id in already_done: - print(f"[INFO] [{i + 1}/{len(rows)}] Skipping (already done): {claim_id}") - continue - - try: - claim_number = row.find_element(By.XPATH, ".//td[2]").text.strip() - except Exception: - claim_number = "" - - try: - sig_raw = row.find_element(By.XPATH, ".//td[10]").text.strip() - if "," in sig_raw: - last, first = sig_raw.split(",", 1) - signature_physician = ( - f"{first.strip().title()} {last.strip().title()}" - ) - else: - signature_physician = sig_raw.strip().title() - except Exception: - signature_physician = "" - - insurance = re.sub( - r"\s*\(.*?\)\s*$", "", - row.get_attribute("data-payername") or "", - ).strip() - patient_name = row.get_attribute("data-patientname") or "" - charges = row.get_attribute("data-totalcharges") or "" - - print(f"[INFO] [{i + 1}/{len(rows)}] Claim: {claim_id} | {patient_name}") - - pdf_url = ( - f"https://claims.zirmed.com/Claims/History/ViewClaimPDF" - f"?appId=1&claimId={claim_id}&instanceId={instance_id}" - f"&form=CMS1500_0212" - ) - save_path = pdf_folder / f"{claim_id}.pdf" - if save_path.exists(): - save_path = pdf_folder / f"{claim_id}_{instance_id}.pdf" - - success = download_pdf_direct(driver, pdf_url, save_path) - if not success: - time.sleep(3) - success = download_pdf_direct(driver, pdf_url, save_path) - if not success: - update_log(log_path, claim_id, "FAILED", "Download failed after retry") - continue - - pdf_data = extract_pdf_data_fast( - save_path, - claim_id=claim_id, - patient_account=claim_number, - insurance=insurance, - patient_name=patient_name, - total_charges=charges, - signature_physician=signature_physician, - ) - if not pdf_data: - update_log(log_path, claim_id, "FAILED", "PDF extraction failed") - continue - - batch.append(pdf_data) - already_done.add(claim_id) - update_log(log_path, claim_id, "SUCCESS", "") - - if len(batch) >= SAVE_EVERY: - _save_batch_to_excel(excel_path, batch) - total_saved += len(batch) - batch = [] - - except Exception as e: - print(f"[ERROR] Row {i + 1} (claim {claim_id}) failed → {e}") - update_log(log_path, claim_id, "FAILED", str(e)[:200]) - - if batch: - _save_batch_to_excel(excel_path, batch) - total_saved += len(batch) - batch = [] - - if not click_next_page(driver): - print(f"[INFO] No more pages for {account_name}") - break - page += 1 - - print(f"\n[SUCCESS] {account_name} complete — {total_saved} records saved") - print(f"[INFO] ====== Account {account_name} finished ======\n") - - -# ================= PROCESS ALL ACCOUNTS ================= -def process_all_accounts(driver): - wait = WebDriverWait(driver, 30) - print("[INFO] Starting account processing...") - all_accounts = [] - - try: - dropdown = wait.until( - EC.element_to_be_clickable( - (By.XPATH, "//input[contains(@data-search-url,'AccountSearch')]") - ) - ) - dropdown.click() - time.sleep(2) - account_elements = driver.find_elements( - By.XPATH, "//ul[contains(@class,'ui-autocomplete')]//li" - ) - all_accounts = [ - acc.text.strip() for acc in account_elements if acc.text.strip() - ] - print(f"[INFO] Total accounts found → {len(all_accounts)}") - driver.find_element(By.TAG_NAME, "body").click() - time.sleep(1) - except Exception as e: - print(f"[ERROR] Failed to get account list: {e}") - return - - if not all_accounts: - print("[WARN] No accounts found — exiting") - return - - # Deduplicate while preserving order - seen_names = set() - unique_accounts = [] - for name in all_accounts: - if name not in seen_names: - seen_names.add(name) - unique_accounts.append(name) - - if len(unique_accounts) < len(all_accounts): - print(f"[INFO] Removed {len(all_accounts) - len(unique_accounts)} " - f"duplicate account names") - - all_accounts = unique_accounts - print(f"[INFO] Unique accounts to process → {len(all_accounts)}") - - portal_home_url = driver.current_url - processed_accounts = set() - - for idx, account_name in enumerate(all_accounts, 1): - if account_name in processed_accounts: - print(f"[WARN] '{account_name}' already processed — skipping") - continue - - print(f"\n[INFO] ===== Account {idx}/{len(all_accounts)}: {account_name} =====") - - try: - driver.get(portal_home_url) - time.sleep(3) - - dropdown = wait.until( - EC.element_to_be_clickable( - (By.XPATH, "//input[contains(@data-search-url,'AccountSearch')]") - ) - ) - dropdown.clear() - dropdown.click() - time.sleep(2) - - account_elements = driver.find_elements( - By.XPATH, "//ul[contains(@class,'ui-autocomplete')]//li" - ) - clicked = False - for acc in account_elements: - if acc.text.strip() == account_name: - acc.click() - clicked = True - break - - if not clicked: - print(f"[WARN] '{account_name}' not found in dropdown — skipping") - continue - - try: - wait.until( - EC.invisibility_of_element_located( - (By.XPATH, "//ul[contains(@class,'ui-autocomplete')]") - ) - ) - except Exception: - pass - - time.sleep(3) - processed_accounts.add(account_name) - run_claim_report(driver, account_name) - - except Exception as e: - print(f"[ERROR] Account '{account_name}' failed: {e}") - try: - driver.get(portal_home_url) - time.sleep(3) - except Exception: - pass - continue - - print(f"\n[INFO] ===== All {len(processed_accounts)} accounts processed =====") - print(f"[INFO] Accounts completed: {sorted(processed_accounts)}") - - -# ================= MAIN ================= -if __name__ == "__main__": - print("=" * 80) - print("🚀 STARTING CHARGE BILLED AUTOMATION - Document AI v7.0") - print("=" * 80) - driver = None - try: - print("\n[STEP 1] Closing existing Chrome instances...") - kill_all_chrome() - - print("\n[STEP 2] Reading configuration...") - ( - proxy_ip, proxy_port, username, waystar_password, - portal_url, first_car_answer, first_job_answer, - ) = read_input_config() - - print("\n[STEP 3] Launching Chrome in debug mode...") - if not start_chrome_debug(proxy_ip, proxy_port): - raise Exception("Chrome debug failed to start") - time.sleep(5) - - print("\n[STEP 4] Attaching Selenium to Chrome...") - driver = attach_browser(DOWNLOADS_DIR) - if not driver: - raise Exception("Driver attach failed") - - print("\n[STEP 5] Opening Waystar portal...") - driver.execute_script("window.open('');") - driver.switch_to.window(driver.window_handles[-1]) - driver.get(portal_url) - WebDriverWait(driver, 30).until( - EC.presence_of_element_located((By.ID, "loginName")) - ) - print("[INFO] Portal loaded successfully") - - print("\n[STEP 6] Performing login...") - login_to_portal( - driver, username, waystar_password, - first_car_answer, first_job_answer, - ) - - print("\n[STEP 7] Starting account processing...") - process_all_accounts(driver) - - print("\n" + "=" * 80) - print("✅ AUTOMATION COMPLETED SUCCESSFULLY") - print("=" * 80) - - except Exception as e: - print("\n" + "=" * 80) - print(f"❌ FATAL ERROR: {str(e)}") - print("=" * 80) - if driver: - try: - capture_screenshot(driver, "MAIN", "Fatal_Error") - except Exception: - pass - - finally: - print("\n[CLEANUP] Closing resources...") - if driver: - try: - driver.quit() - print("[INFO] Browser closed successfully") - except Exception as e: - print(f"[WARN] Error closing browser: {e}") - print("\n🛑 Execution finished\n") \ No newline at end of file diff --git a/src/old/main.py b/src/old/main.py deleted file mode 100644 index 385ae55..0000000 --- a/src/old/main.py +++ /dev/null @@ -1,1351 +0,0 @@ -import subprocess -import time -import os -import sys -import socket -import re -import pandas as pd -import pdfplumber -import requests -from pathlib import Path -from datetime import datetime -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.common.action_chains import ActionChains -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.chrome.service import Service -from selenium.common.exceptions import WebDriverException -from selenium.common.exceptions import NoSuchElementException -from selenium.common.exceptions import StaleElementReferenceException -from selenium.webdriver.support.ui import Select -from selenium.webdriver.common.keys import Keys - -try: - from pywinauto import Application -except Exception: - Application = None - -# ================= PROJECT ROOT ================= -if getattr(sys, "frozen", False): - PROJECT_ROOT = Path(sys.executable).parent -else: - PROJECT_ROOT = Path(__file__).resolve().parent.parent - -# ================= GLOBAL CONFIG ================= -DEBUG_PORT = 9222 -USER_DATA_DIR = os.path.join( - os.environ["USERPROFILE"], "Automation", "ChromeProfile_CF" -) - -today_date = datetime.now().strftime("%m-%d-%Y") -DOWNLOADS_DIR = Path.home() / "Downloads" - -# ================= CHROME PATH ================= -def find_chrome_path(): - """Find Chrome installation path""" - possible_paths = [ - r"C:\Program Files\Google\Chrome\Application\chrome.exe", - r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", - os.path.join( - os.environ.get("LOCALAPPDATA", ""), - r"Google\Chrome\Application\chrome.exe" - ) - ] - - for path in possible_paths: - if os.path.exists(path): - return path - - return None - -# ================= PROXY CONFIG ================= -def read_input_config(): - """Read configuration from input.xlsx""" - config_path = PROJECT_ROOT / "Input" / "input.xlsx" - - if not config_path.exists(): - print("[ERROR] input.xlsx not found.") - sys.exit(1) - - df = pd.read_excel(config_path, sheet_name="Sheet1", header=None) - - config = {} - for _, row in df.iterrows(): - key = str(row[0]).strip() - value = str(row[1]).strip() - config[key] = value - - proxy_ip = config.get("Proxy IP", "") - proxy_port = config.get("Proxy Port", "") - waystar_username = config.get("Waystar Username", "") - waystar_password = config.get("Waystar Password", "") - portal_url = config.get("Waystar URL", "") - first_car_answer = config.get("Waystar SQ First Car", "") - first_job_answer = config.get("Waystar SQ First Job", "") - - print(f"[INFO] Proxy → {proxy_ip}:{proxy_port}") - print(f"[INFO] Username → {waystar_username}") - print(f"[INFO] Portal URL → {portal_url}") - - return ( - proxy_ip, - proxy_port, - waystar_username, - waystar_password, - portal_url, - first_car_answer, - first_job_answer - ) - -# ================= DEBUG PORT CHECK ================= -def wait_for_debug_port(port=9222, timeout=40): - """Wait for Chrome debug port to be ready""" - start = time.time() - - while time.time() - start < timeout: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - if s.connect_ex(("127.0.0.1", port)) == 0: - return True - time.sleep(1) - - return False - -# ================= KILL CHROME ================= -def kill_all_chrome(): - """Kill all Chrome processes""" - try: - os.system("taskkill /F /IM chrome.exe /T >nul 2>&1") - time.sleep(3) - print("[INFO] All Chrome instances closed") - except Exception as e: - print(f"[WARN] Unable to kill Chrome: {e}") - -# ================= START CHROME DEBUG ================= -def start_chrome_debug(proxy_ip=None, proxy_port=None): - """Start Chrome in debug mode""" - chrome_path = find_chrome_path() - - if not chrome_path: - raise Exception( - "Google Chrome not found. Please install Google Chrome and try again." - ) - - print(f"[INFO] Using Chrome path: {chrome_path}") - - os.makedirs(USER_DATA_DIR, exist_ok=True) - - cmd = [ - chrome_path, - f"--remote-debugging-port={DEBUG_PORT}", - f"--user-data-dir={USER_DATA_DIR}", - "--disable-blink-features=AutomationControlled", - "--ignore-certificate-errors", - "about:blank" - ] - - if proxy_ip and proxy_port: - proxy = f"{proxy_ip}:{proxy_port}" - cmd.append(f"--proxy-server=http://{proxy}") - print(f"[INFO] Chrome started with proxy → {proxy}") - else: - print("[INFO] Chrome started WITHOUT proxy") - - print("[INFO] Launching Chrome...") - subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - if not wait_for_debug_port(DEBUG_PORT, timeout=40): - print("[ERROR] Chrome debug port did not open") - return False - - print("[INFO] Chrome debug port is ready") - return True - -# ================= SCREENSHOT ================= -def capture_screenshot(driver, client_name, step_name): - """Capture screenshot for debugging""" - if not driver: - return None - - safe_client = "".join(c for c in client_name if c.isalnum() or c in ("_", "-")) - safe_step = "".join(c for c in step_name if c.isalnum() or c in ("_", "-")) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - - screenshots_dir = PROJECT_ROOT / "Screenshots" / today_date / safe_client - screenshots_dir.mkdir(parents=True, exist_ok=True) - - screenshot_path = screenshots_dir / f"{safe_step}_{timestamp}.png" - - try: - driver.save_screenshot(str(screenshot_path)) - print(f"[INFO] Screenshot saved: {screenshot_path}") - return screenshot_path - except Exception as e: - print(f"[ERROR] Screenshot failed: {e}") - return None - -# ================= ATTACH BROWSER ================= -def attach_browser(download_path, retries=5): - """Attach to Chrome via remote debugging""" - for attempt in range(1, retries + 1): - try: - print(f"[INFO] Attaching browser (attempt {attempt})") - - chrome_options = Options() - chrome_options.add_experimental_option( - "debuggerAddress", f"127.0.0.1:{DEBUG_PORT}" - ) - chrome_options.set_capability("webSocketUrl", False) - - prefs = { - "download.default_directory": str(download_path), - "download.prompt_for_download": False, - "download.directory_upgrade": True, - "plugins.always_open_pdf_externally": True, - "safebrowsing.enabled": True - } - chrome_options.add_experimental_option("prefs", prefs) - - service = Service() - driver = webdriver.Chrome(service=service, options=chrome_options) - _ = driver.current_url - driver.maximize_window() - - print("[INFO] Browser attached successfully") - - try: - driver.execute_cdp_cmd( - "Page.setDownloadBehavior", - {"behavior": "allow", "downloadPath": str(download_path)} - ) - except Exception as e: - print(f"[WARN] Unable to set download behavior: {e}") - - return driver - - except WebDriverException as e: - print(f"[WARN] Attach attempt {attempt} failed: {e}") - time.sleep(3) - - print("[ERROR] Failed to attach Chrome after retries") - return None - -# ================= LOGIN ================= -def login_to_portal(driver, waystar_username, waystar_password, first_car_answer, first_job_answer): - """Login to Waystar portal""" - wait = WebDriverWait(driver, 30) - - try: - time.sleep(5) - print("[INFO] Starting login process") - - # Enter username - username_field = wait.until( - EC.presence_of_element_located((By.ID, "loginName")) - ) - username_field.clear() - username_field.send_keys(waystar_username) - time.sleep(2) - - # Enter password - password_field = wait.until( - EC.presence_of_element_located((By.ID, "password")) - ) - password_field.clear() - password_field.send_keys(waystar_password) - time.sleep(2) - - # Click login - driver.find_element(By.ID, "loginButton").click() - time.sleep(5) - print("[INFO] Login clicked") - - # Handle proactive alert if present - try: - alert_page = driver.find_elements(By.XPATH, "//span[contains(text(),'Proactive Alert')]") - if alert_page: - wait.until(EC.element_to_be_clickable((By.XPATH, "//button[@id='continueButton']"))).click() - time.sleep(3) - except: - pass - - # Handle security question if present - try: - question_exists = driver.find_elements(By.XPATH, "//*[@id='authDiv']//strong") - if question_exists: - question = wait.until( - EC.presence_of_element_located((By.XPATH, "//*[@id='authDiv']//strong")) - ).text.lower() - - print(f"[INFO] Security Question → {question}") - - if "first car" in question: - answer = first_car_answer - elif "first job" in question: - answer = first_job_answer - else: - raise Exception(f"Unknown security question: {question}") - - print(f"[INFO] Submitting answer") - - answer_box = wait.until( - EC.presence_of_element_located((By.ID, "verifyAnswer")) - ) - answer_box.clear() - answer_box.send_keys(answer) - driver.find_element(By.ID, "VerifyButton").click() - - print("[INFO] Security answer submitted") - time.sleep(3) - except: - pass - - # Handle security information page if present - try: - security_qpage = driver.find_elements(By.XPATH, "//h2[normalize-space()='Security Information']") - if security_qpage: - print("[INFO] Security page appeared, skipping") - wait.until(EC.element_to_be_clickable((By.ID, "NextButton"))).click() - time.sleep(2) - wait.until( - EC.element_to_be_clickable((By.XPATH, "//a[normalize-space()='Skip This Step']")) - ).click() - time.sleep(5) - except: - pass - - print("[INFO] Login completed successfully") - - except Exception as e: - print(f"[ERROR] Login failed: {e}") - raise - -# ================= CALCULATE DATE RANGE ================= -def get_date_range(): - """ - Returns the first and last day of the PREVIOUS month. - e.g. if today is April 2026 → returns 03/01/2026 and 03/31/2026 - """ - now = datetime.now() - - # First day of current month - first_of_current = datetime(now.year, now.month, 1) - - # Last day of previous month = one day before first of current month - last_of_prev = first_of_current - pd.Timedelta(days=1) - - # First day of previous month - first_of_prev = datetime(last_of_prev.year, last_of_prev.month, 1) - - return ( - first_of_prev.strftime("%m/%d/%Y"), # e.g. 03/01/2026 - last_of_prev.strftime("%m/%d/%Y"), # e.g. 03/31/2026 - ) - -# ================= APPLY FILTERS ================= -def apply_filters(driver): - """Apply filters to claims grid with custom date range""" - wait = WebDriverWait(driver, 40) - print("[INFO] Applying filters") - - # Wait for filter panel to load - wait.until(EC.presence_of_element_located((By.ID, "ClaimListingSearchButtonBottom"))) - wait.until(EC.presence_of_element_located((By.ID, "SearchCriteria_Status"))) - - # Set status to "All" - status_dropdown = Select(wait.until(EC.element_to_be_clickable((By.ID, "SearchCriteria_Status")))) - status_dropdown.select_by_visible_text("All") - - # Set date to "Custom" - date_dropdown = Select(wait.until(EC.element_to_be_clickable((By.ID, "SearchCriteria_TransDate")))) - date_dropdown.select_by_visible_text("Custom") - - time.sleep(2) # Wait for custom date fields to appear - - # Get date range - start_date, end_date = get_date_range() - - print(f"[INFO] Setting date range: {start_date} to {end_date}") - - # Enter start date - try: - start_date_field = wait.until(EC.presence_of_element_located((By.ID, "transactionDateFromDateBox"))) - start_date_field.clear() - start_date_field.send_keys(start_date) - time.sleep(1) - except Exception as e: - print(f"[WARN] Could not set start date: {e}") - - # Enter end date - try: - end_date_field = wait.until(EC.presence_of_element_located((By.ID, "transactionDateToDateBox"))) - end_date_field.clear() - end_date_field.send_keys(end_date) - time.sleep(1) - except Exception as e: - print(f"[WARN] Could not set end date: {e}") - - # Click search - search_btn = wait.until(EC.element_to_be_clickable((By.ID, "ClaimListingSearchButtonBottom"))) - driver.execute_script("arguments[0].click();", search_btn) - - # Wait for results - wait.until(EC.presence_of_element_located((By.XPATH, "//table[@id='claimsGrid']//tbody//tr"))) - time.sleep(3) - - print("[INFO] Filters applied successfully") - -# ================= CLICK NEXT PAGE ================= -def click_next_page(driver): - """Click next page button""" - try: - next_btn = driver.find_element(By.XPATH, "//button[@id='innerPagerRight']") - - if "disabled" in next_btn.get_attribute("class"): - return False - - driver.execute_script("arguments[0].click();", next_btn) - - # Wait for page to refresh - WebDriverWait(driver, 10).until(EC.staleness_of(next_btn)) - time.sleep(2) - - return True - except: - return False - -# ================= DIRECT DOWNLOAD ================= -def download_pdf_direct(driver, pdf_url, save_path): - """Download PDF directly using session cookies""" - try: - # Get cookies from browser - cookies = {c["name"]: c["value"] for c in driver.get_cookies()} - - # Set headers - headers = { - "User-Agent": driver.execute_script("return navigator.userAgent;"), - "Referer": driver.current_url, - "Accept": "application/pdf,*/*", - } - - # Download PDF - response = requests.get( - pdf_url, - headers=headers, - cookies=cookies, - stream=True, - timeout=60 - ) - - if response.status_code != 200: - print(f"[ERROR] HTTP {response.status_code} → {pdf_url}") - return False - - # Verify content type - content_type = response.headers.get("Content-Type", "") - if "pdf" not in content_type.lower() and "octet-stream" not in content_type.lower(): - print(f"[WARN] Unexpected content type → {content_type}") - - # Write to disk - save_path = Path(save_path) - save_path.parent.mkdir(parents=True, exist_ok=True) - - with open(save_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - - print(f"[SUCCESS] Downloaded → {save_path.name}") - return True - - except Exception as e: - print(f"[ERROR] Direct download failed → {e}") - return False - -# ============================================================ -# CMS-1500 PDF EXTRACTOR — Production v4.3 -# Fix: Numeric-only modifiers (59, 25, etc.) DO appear alone -# in the modifier column. The correct solution is to use -# PDF COORDINATES to identify which column each token -# belongs to, rather than text-based heuristics. -# ============================================================ - -# ---------- modifier constants ---------- -_VALID_MODIFIERS = { - # Physical therapy / rehab discipline - "GP", "GN", "GO", "GQ", "GX", "GY", "GZ", - # Laterality - "LT", "RT", - # Eye / digit - "E1", "E2", "E3", "E4", - "FA", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", - "TA", "T1", "T2", "T3", "T4", "T5", "T6", "T7", "T8", "T9", - # Common billing - "TC", "26", "59", "25", "51", "76", "77", "78", "79", "91", - "24", "57", "58", "80", "81", "82", - # Assistant / supervision - "AA", "AD", "AE", "AF", "AG", "AH", "AI", "AJ", "AK", "AM", "AS", "AT", - # CQ / CR / CS - "CQ", "CR", "CS", - # GA / GC etc - "GA", "GC", "GJ", "GK", "GL", "GT", - # JA / JB / JW / JZ - "JA", "JB", "JW", "JZ", - # KX / KU / KV - "KX", "KU", "KV", - # NU / RR / UE - "NU", "RR", "UE", - # Q modifiers - "Q0", "Q1", "Q2", "Q3", "Q4", "Q5", "Q6", - # SA / SB etc - "SA", "SB", "SC", "SD", "SF", "SG", - # XE / XP / XS / XU - "XE", "XP", "XS", "XU", - # 2-digit numeric modifiers - "22", "23", "24", "25", "26", - "32", "33", - "47", "50", "51", "52", "53", - "54", "55", "56", "57", "58", "59", - "62", "63", "66", - "73", "74", - "76", "77", "78", "79", - "80", "81", "82", - "90", "91", "92", "93", "95", "96", "97", "99", -} - -_NOT_MODIFIERS = { - "MM","DD","YY","TO","OF","OR","IN","IS","NO","BY", - "AT","BE","DO","HE","HI","IF","ME","MY","OK","ON", - "SO","UP","US","WE","NPI","ABC","ICD","CPT","POS", - "DOS","EMG","REF","ID","AN","AC", -} - -# Valid POS codes -_VALID_POS = { - "11","12","13","14","15","16","17","18","19","20", - "21","22","23","24","25","31","32","33","34", - "41","42","49","50","51","52","53","54","55","56","57", - "60","61","62","65","71","72","81","99", -} - -# ============================================================ -# CMS-1500 SECTION 24 COLUMN X-RANGES (standard letter PDF) -# These are approximate x-coordinates for each column. -# Calibrated from standard CMS-1500 02/12 form at 8.5x11". -# Adjust COL_SCALE if your PDFs are a different DPI/size. -# ============================================================ -# Column definitions (x0_min, x0_max): -_COL_RANGES = { - "DOS_FROM": ( 30, 130), # A: Date of Service From - "DOS_TO": ( 130, 200), # A: Date of Service To - "POS": ( 200, 240), # B: Place of Service - "EMG": ( 240, 260), # C: EMG - "CPT": ( 260, 370), # D: CPT/HCPCS - "MODIFIER": ( 370, 480), # D: Modifier (same col D, after CPT) - "DIAG_PTR": ( 480, 530), # E: Diagnosis Pointer - "CHARGES": ( 530, 640), # F: $ Charges - "UNITS": ( 640, 690), # G: Days or Units - "EPSDT": ( 690, 720), # H: EPSDT - "ID_QUAL": ( 720, 750), # I: ID Qual - "RENDERING": ( 750, 870), # J: Rendering Provider ID -} - - -# ---------- small helpers ---------- -def _is_cpt(token: str) -> bool: - """5-digit number strictly in CPT range 10000-99999.""" - if not re.fullmatch(r"\d{5}", token): - return False - return 10000 <= int(token) <= 99999 - - -def _is_icd10(token: str) -> bool: - return bool(re.fullmatch( - r"[A-Z]\d{2,3}(?:\.\d{1,4})?", token.upper() - ) or re.fullmatch( - r"[A-Z]\d{3,7}", token.upper() - )) - - -def _cell(c) -> str: - return str(c).strip() if c else "" - - -def _format_list(values: list, label: str) -> str: - clean = [v for v in values if v] - if not clean: - return "" - if len(clean) == 1: - return clean[0] - return "; ".join(f"{label} {i+1}: {v}" for i, v in enumerate(clean)) - - -# ============================================================ -# COORDINATE-BASED COLUMN EXTRACTION -# ============================================================ - -def _get_words_in_col(page_words: list, row_y: float, row_height: float, - col_name: str, y_tolerance: float = 6.0) -> list: - """ - Return all words that fall within: - - the given row's y-range (row_y ± y_tolerance) - - the given column's x-range - """ - x_min, x_max = _COL_RANGES[col_name] - return [ - w for w in page_words - if abs(w["top"] - row_y) <= y_tolerance - and x_min <= w["x0"] <= x_max - ] - - -def _detect_col_scale(page_words: list) -> float: - """ - Auto-detect scaling factor by finding the 'NPI' label in column J - and comparing its x-position to our expected range. - Returns a scale multiplier (1.0 = no scaling needed). - """ - npi_words = [w for w in page_words if w["text"].strip().upper() == "NPI"] - if not npi_words: - return 1.0 - - # NPI label should be around x=720-750 in our reference - npi_x = npi_words[0]["x0"] - expected_npi_x = 735 # midpoint of ID_QUAL range - if expected_npi_x > 0 and npi_x > 50: - return npi_x / expected_npi_x - return 1.0 - - -def _scale_col_ranges(scale: float) -> dict: - """Apply scale factor to all column x-ranges.""" - return { - col: (int(x0 * scale), int(x1 * scale)) - for col, (x0, x1) in _COL_RANGES.items() - } - - -def _extract_modifiers_coordinate(page_words: list, cpt_word: dict, - col_ranges: dict) -> str: - """ - v4.3: Extract modifiers using COORDINATES. - - The modifier column (D, after CPT) on CMS-1500 spans roughly - x=370 to x=480. We look for valid modifier tokens in that x-range - on the same row as the CPT code. - - This correctly handles: - - Alpha-only: GP, GO, CQ, KX - - Numeric-only: 59, 25 - - Mixed: GP CQ, GO CO, GP 59 - """ - row_y = cpt_word["top"] - x_min, x_max = col_ranges["MODIFIER"] - - mod_words = [ - w for w in page_words - if abs(w["top"] - row_y) <= 8 - and x_min <= w["x0"] <= x_max - ] - - found = [] - for w in sorted(mod_words, key=lambda x: x["x0"]): - t = w["text"].strip().upper() - if not t: - continue - if t in _NOT_MODIFIERS: - continue - if t in _VALID_MODIFIERS and t not in found: - found.append(t) - if len(found) == 4: - break - - return " ".join(found) - - -def _extract_units_coordinate(page_words: list, cpt_word: dict, - col_ranges: dict) -> str: - """ - v4.3: Extract units using COORDINATES. - - Column G (Days/Units) on CMS-1500 spans roughly x=640 to x=690. - Units is a small integer (1-99). - """ - row_y = cpt_word["top"] - x_min, x_max = col_ranges["UNITS"] - - unit_words = [ - w for w in page_words - if abs(w["top"] - row_y) <= 8 - and x_min <= w["x0"] <= x_max - and re.fullmatch(r"[1-9]\d?", w["text"].strip()) - ] - - if unit_words: - # Take the leftmost value in the units column - best = min(unit_words, key=lambda w: w["x0"]) - return best["text"].strip() - - return "" - - -# ---------- DIAGNOSIS CODES (field 21, slots A-L) ---------- -def _extract_diagnosis_codes(text: str, words: list) -> list: - codes: list[str] = [] - - anchor_y = None - anchor_x = None - for w in words: - if w["text"].strip().rstrip(".") == "21": - if w["x0"] < 120: - anchor_y = w["top"] - anchor_x = w["x0"] - break - - if anchor_y is not None: - region = [ - w for w in words - if (anchor_y - 5) <= w["top"] <= (anchor_y + 120) - and w["x0"] >= anchor_x - 5 - ] - for w in region: - t = w["text"].strip().upper() - if re.fullmatch(r"[A-L]\.?", t): - continue - if t in ("O", "ICD", "IND", "21"): - continue - if _is_icd10(t): - codes.append(t) - - if not codes: - m = re.search( - r"(?:21|DIAGNOSIS)[^\n]*\n([\s\S]{0,600})(?:22\b|23\b|24\b|ICD\b)", - text, re.IGNORECASE - ) - if m: - raw = re.findall(r"\b([A-Z]\d{2,7}(?:\.\d{1,4})?)\b", m.group(1)) - codes.extend(raw) - - if not codes: - raw = re.findall(r"\b([A-Z]\d{2,7}(?:\.\d{1,4})?)\b", text) - codes = [c for c in raw if not c.startswith("NP")] - - seen, unique = set(), [] - for c in codes: - cu = c.upper() - if cu not in seen: - seen.add(cu) - unique.append(cu) - return unique - - -# ============================================================ -# MAIN FUNCTION -# ============================================================ -def extract_pdf_data_fast( - pdf_path, - claim_id="", - patient_account="", - insurance="", - patient_name="", - total_charges="", - signature_physician="", -): - """ - Extract all CMS-1500 fields from a PDF. - v4.3 fixes: - - Modifier extraction: now COORDINATE-BASED using column D - x-range. Correctly captures numeric-only modifiers (59, 25) - as well as alpha (GP, GO, CQ) and mixed (GP CQ, GO 59). - Charge column (F) and units column (G) are in different - x-ranges so they can NEVER bleed into modifier results. - - Units extraction: also COORDINATE-BASED using column G - x-range. Charge column is physically separate so no bleed. - - Auto-detects PDF scale factor via NPI label position. - """ - - data = { - "Claim ID": claim_id, - "Rendering Provider #": "", - "Signature of Physician": signature_physician, - "Patient's Account No": patient_account, - "Date of Birth": "", - "Insurance Name": insurance, - "Insured ID Number": "", - "Patient's Name": patient_name, - "Date of Service": "", - "Diagnosis Codes": "", - "CPT": "", - "POS": "", - "Modifier": "", - "Units": "", - "Charges": total_charges, - } - - try: - with pdfplumber.open(pdf_path) as pdf: - page = pdf.pages[0] - text = page.extract_text(x_tolerance=2, y_tolerance=2) or "" - words = page.extract_words() - tables = page.extract_tables() - - print(f"[DEBUG] Opened: {pdf_path} | Tables: {len(tables)}") - - # Auto-detect scale and apply to column ranges - scale = _detect_col_scale(words) - col_ranges = _scale_col_ranges(scale) if abs(scale - 1.0) > 0.05 else _COL_RANGES - print(f"[DEBUG] PDF scale factor: {scale:.3f}") - - # ============ DATE OF BIRTH ============ - dob = re.search(r"\b(\d{2})\s+(\d{2})\s+(\d{4})\b", text) - if dob: - data["Date of Birth"] = f"{dob.group(1)}/{dob.group(2)}/{dob.group(3)}" - - # ============ INSURED ID (field 1a) ============ - _SKIP = {"MEDICARE", "MEDICAID", "CHAMPVA", "TRICARE", "CHAMPUS"} - - def _valid_insured_id(t: str) -> bool: - t_up = t.upper() - return ( - re.match(r"^[A-Z0-9]{6,20}$", t_up) - and bool(re.search(r"[A-Z]", t_up)) - and bool(re.search(r"\d", t_up)) - and t_up not in _SKIP - ) - - insured_id = "" - - for i, w in enumerate(words): - if w["text"].lower().rstrip(".") == "1a": - for nxt in words[i + 1: i + 8]: - if _valid_insured_id(nxt["text"]): - insured_id = nxt["text"].upper() - break - if insured_id: - break - - if not insured_id: - for i, w in enumerate(words): - if "INSURED" in w["text"].upper(): - nearby = words[i: min(i + 8, len(words))] - if any("NUMBER" in n["text"].upper() for n in nearby): - base_y = w["top"] - candidates = [ - n for n in words - if abs(n["top"] - base_y) < 20 and n["x0"] > w["x0"] - ] - for c in reversed(candidates): - if _valid_insured_id(c["text"]): - insured_id = c["text"].upper() - break - if insured_id: - break - - if not insured_id: - top_right = sorted( - [w for w in words if w["x0"] > 380 and w["top"] < 220], - key=lambda w: (w["top"], w["x0"]) - ) - for w in top_right: - if _valid_insured_id(w["text"]): - insured_id = w["text"].upper() - break - - if not insured_id: - m = re.search(r"1a[.\s]+([A-Z0-9]{6,20})", text, re.IGNORECASE) - if m and _valid_insured_id(m.group(1)): - insured_id = m.group(1).upper() - - data["Insured ID Number"] = insured_id - print(f"[DEBUG] Insured ID: {insured_id}") - - # ============ DIAGNOSIS CODES (field 21, slots A-L) ============ - dx_codes = _extract_diagnosis_codes(text, words) - if dx_codes: - codes_str = ", ".join(dx_codes) - data["Diagnosis Codes"] = f"{codes_str} (Total: {len(dx_codes)})" - print(f"[DEBUG] Diagnosis codes ({len(dx_codes)}): {dx_codes}") - - # ============ SERVICE LINES (section 24, columns A-J) ============ - cpts, pos_list, dos_list, mod_list, units_list, rp_list = [], [], [], [], [], [] - - for table_idx, table in enumerate(tables): - print(f"[DEBUG] Table {table_idx+1}: {len(table)} rows") - - for row_idx, row in enumerate(table): - if not row: - continue - - cells = [_cell(c) for c in row] - row_text = " ".join(cells) - - candidate_cpts = re.findall(r"\b(\d{5})\b", row_text) - valid_cpts = [c for c in candidate_cpts if _is_cpt(c)] - if not valid_cpts: - continue - - if len(re.findall(r"\b[A-Z]{3,}\b", row_text)) > 4: - continue - - print(f"[DEBUG] Service row {row_idx}: {row_text[:130]}") - - cpt_val = valid_cpts[0] - cpts.append(cpt_val) - - # ---- Date of Service ---- - dm = re.search( - r"\b(\d{2})[\/\s\-]+(\d{2})[\/\s\-]+(\d{2,4})\b", row_text - ) - if dm: - mm, dd, yy = dm.group(1), dm.group(2), dm.group(3) - if len(yy) == 2: - yy = ("20" + yy) if int(yy) < 50 else ("19" + yy) - dos_list.append(f"{mm}/{dd}/{yy}") - else: - dos_list.append("") - - # ---- POS: last valid 2-digit code BEFORE the CPT ---- - before_cpt = row_text[: row_text.index(cpt_val)] - pos_candidates = re.findall(r"\b(\d{2})\b", before_cpt) - pos_val = "" - for p in reversed(pos_candidates): - if p in _VALID_POS: - pos_val = p - break - pos_list.append(pos_val) - - # ---- Find CPT word on the page for coordinate lookups ---- - cpt_word = None - for w in words: - if w["text"].strip() == cpt_val: - cpt_word = w - break - - # ---- Modifiers: coordinate-based ---- - if cpt_word: - mod_val = _extract_modifiers_coordinate(words, cpt_word, col_ranges) - else: - # Fallback: text-based (strip all digits, alpha modifiers only) - after_cpt = row_text[row_text.index(cpt_val) + len(cpt_val):] - after_cpt = re.sub(r"\b\d+\b", " ", after_cpt) - after_cpt = re.sub(r"\bNPI\b", " ", after_cpt, flags=re.IGNORECASE) - after_cpt = re.sub(r"\b[A-La-l]\b", " ", after_cpt) - mod_tokens = [ - t.upper() for t in after_cpt.split() - if t.upper() in _VALID_MODIFIERS - and t.upper() not in _NOT_MODIFIERS - and t.upper().isalpha() # fallback: alpha only - ] - mod_val = " ".join(dict.fromkeys(mod_tokens))[:4] - - mod_list.append(mod_val) - - # ---- Units: coordinate-based ---- - if cpt_word: - units_val = _extract_units_coordinate(words, cpt_word, col_ranges) - else: - # Fallback: strict text-based - seg = row_text[row_text.index(cpt_val) + len(cpt_val):] - seg = re.sub(r"\d+\.\d+", " ", seg) - seg = re.sub(r"\b\d{3,}\b", " ", seg) - seg = re.sub(r"\bNPI\b", " ", seg, flags=re.IGNORECASE) - for mod in _VALID_MODIFIERS: - seg = re.sub(rf"\b{re.escape(mod)}\b", " ", seg, flags=re.IGNORECASE) - seg = re.sub(r"\b[A-La-l]\b", " ", seg) - um = re.search(r"\b([1-9]\d?)\b", seg) - units_val = um.group(1) if um else "" - - units_list.append(units_val) - - # ---- Rendering Provider / NPI (10-digit) ---- - npi_candidates = re.findall(r"\b(\d{10})\b", row_text) - rp_list.append(npi_candidates[-1] if npi_candidates else "") - - print(f"[DEBUG] CPT={cpt_val} | POS={pos_val} | " - f"MOD={mod_val} | UNITS={units_val} | NPI={rp_list[-1]}") - - print(f"[DEBUG] Total service lines: {len(cpts)}") - - # ---- Pad all lists to match CPT count ---- - n = len(cpts) - def _pad(lst): - return lst + [""] * max(0, n - len(lst)) - - pos_list = _pad(pos_list) - dos_list = _pad(dos_list) - mod_list = _pad(mod_list) - units_list = _pad(units_list) - rp_list = _pad(rp_list) - - # ---- Format output ---- - data["CPT"] = _format_list(cpts, "CPT") - data["POS"] = _format_list(pos_list, "POS") - data["Date of Service"] = _format_list(dos_list, "DOS") - data["Modifier"] = _format_list(mod_list, "Modifier") - data["Units"] = _format_list(units_list, "Units") - data["Rendering Provider #"] = _format_list(rp_list, "RP #") - - return data - - except Exception as e: - import traceback - print(f"[ERROR] PDF parsing failed → {pdf_path} → {e}") - traceback.print_exc() - return None - -# ================= UPDATE LOG ================= -def update_log(log_path, claim_id, status, remarks=""): - """Update processing log""" - now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - new_row = pd.DataFrame([{ - "Claim ID": claim_id, - "Status": status, - "Remarks": remarks, - "Timestamp": now - }]) - - try: - existing = pd.read_excel(log_path) - final = pd.concat([existing, new_row], ignore_index=True) - except: - final = new_row - - final.to_excel(log_path, index=False) - -# ================= RUN CLAIM REPORT ================= -def run_claim_report(driver, account_name): - """Process claims for an account""" - wait = WebDriverWait(driver, 30) - actions = ActionChains(driver) - - print(f"\n[INFO] ====== Running report for {account_name} ======") - - # Setup directories - today = datetime.now().strftime("%m-%d-%Y") - safe_name = "".join(c for c in account_name if c.isalnum() or c in (" ", "_", "(", ")")) - - base = PROJECT_ROOT / "Output" / today - pdf_folder = base / "PDF Folder" / safe_name - report_dir = base / "reports" - - pdf_folder.mkdir(parents=True, exist_ok=True) - report_dir.mkdir(parents=True, exist_ok=True) - - excel_path = report_dir / f"{safe_name}.xlsx" - log_path = report_dir / "log.xlsx" - - # Create Excel file if not exists - if not excel_path.exists(): - columns = [ - "Claim ID", "Rendering Provider #", "Signature of Physician", - "Patient's Account No", "Date of Birth", "Insurance Name", - "Insured ID Number", "Patient's Name", "Date of Service", - "Diagnosis Codes", - "CPT", "POS", "Modifier", "Units", "Charges" - ] - pd.DataFrame(columns=columns).to_excel(excel_path, index=False) - - # Create log file if not exists - if not log_path.exists(): - pd.DataFrame(columns=["Claim ID", "Status", "Remarks", "Timestamp"]).to_excel(log_path, index=False) - - # Navigate to claims - try: - claims_processing = wait.until( - EC.presence_of_element_located((By.XPATH, "//a[contains(.,'Claims Processing')]")) - ) - actions.move_to_element(claims_processing).perform() - time.sleep(1) - - claims_option = wait.until( - EC.presence_of_element_located((By.XPATH, "//a[normalize-space()='Claims']")) - ) - driver.execute_script("arguments[0].click();", claims_option) - time.sleep(3) - except Exception as e: - print(f"[ERROR] Navigation failed: {e}") - return - - # Apply filters - apply_filters(driver) - - # Process claims - page = 1 - all_extracted_data = [] - - while True: - print(f"\n[INFO] Processing page {page}") - - try: - rows = driver.find_elements( - By.XPATH, "//table[@id='claimsGrid']//tbody//tr[@data-claimid]" - ) - print(f"[INFO] Total rows on page → {len(rows)}") - except Exception as e: - print(f"[ERROR] Failed to get rows: {e}") - break - - for i in range(len(rows)): - try: - # Re-fetch rows to avoid stale elements - rows = driver.find_elements( - By.XPATH, "//table[@id='claimsGrid']//tbody//tr[@data-claimid]" - ) - row = rows[i] - - # Extract data from table - claim_id = row.get_attribute("data-claimid") or "" - instance_id = row.get_attribute("data-instanceid") or "" - - # Claim Number (Patient Account Number) - try: - claim_number = row.find_element(By.XPATH, ".//td[2]").text.strip() - except: - claim_number = "" - - # Signature physician (provider name from table) - try: - signature_physician = row.find_element(By.XPATH, ".//td[10]").text.strip() - if "," in signature_physician: - parts = signature_physician.split(",", 1) - last = parts[0].strip().title() - first = parts[1].strip().title() - signature_physician = f"{first} {last}" - else: - signature_physician = signature_physician.strip().title() - except: - signature_physician = "" - - insurance = row.get_attribute("data-payername") or "" - insurance = re.sub(r"\s*\(.*?\)\s*$", "", insurance).strip() - patient_name = row.get_attribute("data-patientname") or "" - charges = row.get_attribute("data-totalcharges") or "" - - print(f"[INFO] [{i+1}/{len(rows)}] Processing Claim: {claim_id} | Patient: {patient_name}") - - if not claim_id or not instance_id: - print(f"[WARN] Missing IDs - skipping row {i+1}") - continue - - # Build PDF URL - pdf_url = ( - f"https://claims.zirmed.com/Claims/History/ViewClaimPDF" - f"?appId=1&claimId={claim_id}&instanceId={instance_id}&form=CMS1500_0212" - ) - - # Save path - save_path = pdf_folder / f"{claim_id}.pdf" - if save_path.exists(): - save_path = pdf_folder / f"{claim_id}_{instance_id}.pdf" - - # Download PDF - success = download_pdf_direct(driver, pdf_url, save_path) - if not success: - update_log(log_path, claim_id, "FAILED", "Download failed") - continue - - # Extract PDF data - pdf_data = extract_pdf_data_fast( - save_path, - claim_id=claim_id, - patient_account=claim_number, - insurance=insurance, - patient_name=patient_name, - total_charges=charges, - signature_physician=signature_physician - ) - - if not pdf_data: - update_log(log_path, claim_id, "FAILED", "PDF extraction failed") - continue - - all_extracted_data.append(pdf_data) - update_log(log_path, claim_id, "SUCCESS", "") - - except Exception as e: - print(f"[ERROR] Row {i+1} failed → {e}") - try: - update_log(log_path, claim_id if claim_id else f"row_{i+1}", "FAILED", str(e)) - except: - pass - - # Next page - if not click_next_page(driver): - print("[INFO] No more pages") - break - - page += 1 - - # Save to Excel - if all_extracted_data: - df = pd.DataFrame(all_extracted_data) - - try: - existing_df = pd.read_excel(excel_path) - final_df = pd.concat([existing_df, df], ignore_index=True) - except: - final_df = df - - final_df.to_excel(excel_path, index=False) - print(f"\n[SUCCESS] Saved {len(all_extracted_data)} records to {excel_path}") - else: - print("[WARN] No data extracted") - - print(f"[INFO] ====== Account {account_name} completed ======\n") - -# ================= PROCESS ALL ACCOUNTS ================= -def process_all_accounts(driver): - """Process all accounts in the dropdown""" - wait = WebDriverWait(driver, 30) - processed_accounts = set() - - print("[INFO] Starting account processing...") - - try: - # Click dropdown - dropdown = wait.until( - EC.element_to_be_clickable((By.XPATH, "//input[contains(@data-search-url,'AccountSearch')]")) - ) - dropdown.click() - time.sleep(2) - - # Get all accounts - account_elements = driver.find_elements( - By.XPATH, "//ul[contains(@class,'ui-autocomplete')]//li" - ) - all_accounts = [acc.text.strip() for acc in account_elements if acc.text.strip()] - - print(f"[INFO] Total accounts found → {len(all_accounts)}") - - # Close dropdown - driver.find_element(By.TAG_NAME, "body").click() - time.sleep(1) - - except Exception as e: - print(f"[ERROR] Failed to get accounts: {e}") - return - - # Process each account - for idx, account_name in enumerate(all_accounts, 1): - if account_name in processed_accounts: - continue - - print(f"\n[INFO] ===== Processing account {idx}/{len(all_accounts)}: {account_name} =====") - - try: - # Re-open dropdown - dropdown = wait.until( - EC.element_to_be_clickable((By.XPATH, "//input[contains(@data-search-url,'AccountSearch')]")) - ) - dropdown.click() - time.sleep(2) - - # Re-fetch list - account_elements = driver.find_elements( - By.XPATH, "//ul[contains(@class,'ui-autocomplete')]//li" - ) - - # Find and click matching account - for acc in account_elements: - if acc.text.strip() == account_name: - acc.click() - break - - # Wait for dropdown to close - wait.until( - EC.invisibility_of_element_located( - (By.XPATH, "//ul[contains(@class,'ui-autocomplete')]") - ) - ) - - processed_accounts.add(account_name) - time.sleep(3) - - # Run claim report - run_claim_report(driver, account_name) - - except Exception as e: - print(f"[ERROR] Failed to process account {account_name}: {e}") - continue - - print("\n[INFO] ===== All accounts processed successfully =====") - -# ================= MAIN EXECUTION ================= -if __name__ == "__main__": - print("=" * 80) - print("🚀 STARTING CHARGE BILLED AUTOMATION") - print("=" * 80) - - driver = None - - try: - # Kill existing Chrome - print("\n[STEP 1] Closing existing Chrome instances...") - kill_all_chrome() - - # Read config - print("\n[STEP 2] Reading configuration...") - ( - proxy_ip, proxy_port, username, waystar_password, - portal_url, first_car_answer, first_job_answer - ) = read_input_config() - - # Start Chrome - print("\n[STEP 3] Launching Chrome in debug mode...") - if not start_chrome_debug(proxy_ip, proxy_port): - raise Exception("Chrome debug failed to start") - time.sleep(5) - - # Attach driver - print("\n[STEP 4] Attaching Selenium to Chrome...") - driver = attach_browser(DOWNLOADS_DIR) - if not driver: - raise Exception("Driver attach failed") - - # Open portal - print("\n[STEP 5] Opening Waystar portal...") - driver.execute_script("window.open('');") - driver.switch_to.window(driver.window_handles[-1]) - driver.get(portal_url) - - WebDriverWait(driver, 30).until( - EC.presence_of_element_located((By.ID, "loginName")) - ) - print("[INFO] Portal loaded successfully") - - # Login - print("\n[STEP 6] Performing login...") - login_to_portal(driver, username, waystar_password, first_car_answer, first_job_answer) - - # Process accounts - print("\n[STEP 7] Starting account processing...") - process_all_accounts(driver) - - print("\n" + "=" * 80) - print("✅ AUTOMATION COMPLETED SUCCESSFULLY") - print("=" * 80) - - except Exception as e: - print("\n" + "=" * 80) - print(f"❌ FATAL ERROR: {str(e)}") - print("=" * 80) - - if driver: - try: - capture_screenshot(driver, "MAIN", "Fatal_Error") - except: - pass - - finally: - print("\n[CLEANUP] Closing resources...") - if driver: - try: - driver.quit() - print("[INFO] Browser closed successfully") - except Exception as e: - print(f"[WARN] Error closing browser: {e}") - - print("\n🛑 Execution finished\n") \ No newline at end of file diff --git a/src/old/main.spec b/src/old/main.spec deleted file mode 100644 index d4db9bc..0000000 --- a/src/old/main.spec +++ /dev/null @@ -1,38 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- - - -a = Analysis( - ['main.py'], - pathex=[], - binaries=[], - datas=[('..\\Input', 'Input'), ('..\\Screenshots', 'Screenshots')], - hiddenimports=['pyotp', 'pandas', 'openpyxl'], - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=[], - noarchive=False, - optimize=0, -) -pyz = PYZ(a.pure) - -exe = EXE( - pyz, - a.scripts, - a.binaries, - a.datas, - [], - name='main', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - argv_emulation=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) diff --git a/src/old/main_v1.spec b/src/old/main_v1.spec deleted file mode 100644 index b8ae2e4..0000000 --- a/src/old/main_v1.spec +++ /dev/null @@ -1,38 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- - - -a = Analysis( - ['main_v1.py'], - pathex=[], - binaries=[], - datas=[], - hiddenimports=[], - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=[], - noarchive=False, - optimize=0, -) -pyz = PYZ(a.pure) - -exe = EXE( - pyz, - a.scripts, - a.binaries, - a.datas, - [], - name='main_v1', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - argv_emulation=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) diff --git a/src/old/pdf extraction.py b/src/old/pdf extraction.py deleted file mode 100644 index cacb62e..0000000 --- a/src/old/pdf extraction.py +++ /dev/null @@ -1,491 +0,0 @@ -import re -import pdfplumber - -# =========================================================================== -# CMS-1500 PDF EXTRACTOR — Production-Ready v3.0 -# =========================================================================== -# Fixes vs prior versions: -# • CPT : strict 5-digit match guarded by known CPT range (10000-99999), -# zip-codes / account-numbers / tax-ids filtered out -# • POS : extracted per service-line row; count always matches CPT count -# • Modifier: column-position aware extraction (not regex on full row text) -# • Diagnosis codes: dual-pass — word-level near field-21 + full-text regex -# • Date of Service: handles split cells (MM | DD | YY) and slash formats -# • Units: column-position aware (column G on CMS-1500) -# • Rendering Provider #: 10-digit NPI per row, column J -# • All multi-value fields formatted as "Field 1: val; Field 2: val …" -# =========================================================================== - - -# --------------------------------------------------------------------------- -# Helper: is this token a valid CPT code? -# --------------------------------------------------------------------------- -def _is_cpt(token: str) -> bool: - """Return True only for 5-digit numbers in the plausible CPT range.""" - if not re.fullmatch(r"\d{5}", token): - return False - val = int(token) - return 10000 <= val <= 99999 - - -# --------------------------------------------------------------------------- -# Helper: is this token a valid ICD-10 diagnosis code? -# --------------------------------------------------------------------------- -def _is_icd10(token: str) -> bool: - return bool(re.fullmatch(r"[A-Z]\d{2,3}(?:\.\d{1,4})?", token)) - - -# --------------------------------------------------------------------------- -# Helper: clean / normalise a cell value -# --------------------------------------------------------------------------- -def _cell(c) -> str: - return str(c).strip() if c else "" - - -# --------------------------------------------------------------------------- -# Helper: join multi-value list into labelled string -# --------------------------------------------------------------------------- -def _format_list(values: list, label: str) -> str: - clean = [v for v in values if v] - if not clean: - return "" - if len(clean) == 1: - return clean[0] - return "; ".join(f"{label} {i+1}: {v}" for i, v in enumerate(clean)) - - -# --------------------------------------------------------------------------- -# Known modifier codes (GP, GN, GO …) + numeric modifiers (25, 26, 51, 59 …) -# CMS-1500 column E holds up to 4 modifier slots. -# --------------------------------------------------------------------------- -_ALPHA_MODIFIERS = { - "GP", "GN", "GO", "GQ", "GX", "GY", "GZ", - "CQ", "TC", "26", "LT", "RT", "NU", "RR", "UE", - "KX", "KU", "KV", "KW", "KX", - "JW", "JZ", - "AA", "AB", "AD", "AE", "AF", "AG", "AH", "AI", "AJ", "AK", - "AM", "AO", "AP", "AQ", "AR", "AS", "AT", - "CA", "CB", "CC", "CD", "CE", "CF", "CH", "CI", "CJ", "CK", "CL", - "CM", "CN", "CO", "CP", "CR", "CS", - "DA", - "E1","E2","E3","E4", - "FA","F1","F2","F3","F4","F5","F6","F7","F8","F9", - "FX", - "GA","GB","GC","GD","GE","GF","GH","GJ","GK","GL","GM", - "HA","HB","HC","HD","HE","HF","HG","HH","HI","HJ","HK", - "HL","HM","HN","HO","HP","HQ","HR","HS","HT","HU","HV","HW", - "HX","HY","HZ", - "JA","JB","JC","JD","JE", - "LA","LC","LD","LE","LF","LG","LH","LI","LJ","LK","LL","LM", - "LN","LO","LP","LQ","LR","LS","LT","LU","LV","LW","LX","LY","LZ", - "P1","P2","P3","P4","P5","P6", - "Q0","Q1","Q2","Q3","Q4","Q5","Q6","Q7","Q8","Q9", - "QA","QB","QC","QD","QE","QF","QG","QH","QJ","QK","QL","QM","QN", - "QP","QQ","QR","QS","QT","QU","QV","QW","QX","QY","QZ", - "RA","RB","RC","RD","RE","RF","RG","RH","RI","RJ","RK","RL","RM", - "RN","RO","RP","RQ","RR","RS","RT","RU","RV","RW","RX","RY","RZ", - "SA","SB","SC","SD","SE","SF","SG","SH","SI","SJ","SK","SL","SM", - "SN","SO","SP","SQ","SR","SS","ST","SU","SV","SW","SX","SY", - "TA","TB","TC","TD","TE","TF","TG","TH","TI","TJ","TK","TL","TM", - "TN","TP","TQ","TR","TS","TT","TU","TV","TW","TX","TY","TZ", - "U1","U2","U3","U4","U5","U6","U7","U8","U9", - "UA","UB","UC","UD","UE","UF","UG","UH","UJ","UK","UL","UM","UN", - "UP","UQ","UR","US", - "V5","V6","V7","V8","V9", - "XE","XP","XS","XU", -} - -_NUMERIC_MODIFIERS = {str(n) for n in range(22, 100)} # 22-99 - -_ALL_MODIFIERS = _ALPHA_MODIFIERS | _NUMERIC_MODIFIERS - -# Tokens that look like 2-letter codes but are NOT modifiers -_NOT_MODIFIERS = { - "MM","DD","YY","TO","OF","OR","IN","IS","NO","BY","AT","BE", - "DO","GO","HE","HI","IF","ME","MY","OK","ON","SO","UP","US","WE", - "NPI","ABC","ICD","CPT","POS","DOS","EMG","REF","QTY", - "MM","DD","YY", -} - - -def _extract_modifiers_from_tokens(tokens: list[str]) -> str: - """ - Given the token list of a service-line row, return only legitimate - modifier codes. Tokens are scanned left-to-right; we stop after 4 - modifiers (CMS-1500 allows max 4 per line). - """ - found = [] - for t in tokens: - t_up = t.upper() - if t_up in _NOT_MODIFIERS: - continue - if t_up in _ALL_MODIFIERS and t_up not in found: - found.append(t_up) - if len(found) == 4: - break - return " ".join(found) - - -# --------------------------------------------------------------------------- -# CMS-1500 service-line table parser -# --------------------------------------------------------------------------- -def _parse_service_lines(tables: list) -> dict: - """ - Walk every table extracted by pdfplumber and collect per-row data for - columns A-J of CMS-1500 section 24: - A Date(s) of Service (From / To) - B Place of Service - C EMG - D CPT / HCPCS + Modifier(s) - E Diagnosis Pointer - F $ Charges - G Days or Units - H EPSDT - I ID Qual - J Rendering Provider ID - """ - cpts, pos_list, dos_list, mod_list, units_list, rp_list = [], [], [], [], [], [] - - for table in tables: - for row in table: - if not row: - continue - - cells = [_cell(c) for c in row] - row_text = " ".join(cells) - - # ---- Quick pre-filter: row must contain a valid CPT code -------- - candidate_cpts = re.findall(r"\b(\d{5})\b", row_text) - valid_cpts_in_row = [c for c in candidate_cpts if _is_cpt(c)] - if not valid_cpts_in_row: - continue - - # ---- Guard: skip header / label rows ---------------------------- - # If the row text contains many uppercase words it's likely a label - upper_words = re.findall(r"\b[A-Z]{3,}\b", row_text) - if len(upper_words) > 4: - continue - - # ---- CPT: first valid CPT per row (CMS-1500 = 1 CPT per line) --- - cpt_val = valid_cpts_in_row[0] - cpts.append(cpt_val) - - # ---- Date of Service ------------------------------------------- - # Format variants: "03 26 26", "03/26/26", "03-26-26" - date_match = re.search( - r"\b(\d{2})[\/\s\-]+(\d{2})[\/\s\-]+(\d{2,4})\b", row_text - ) - if date_match: - mm, dd, yy = date_match.group(1), date_match.group(2), date_match.group(3) - yy = yy if len(yy) == 4 else ("20" + yy if int(yy) < 50 else "19" + yy) - dos_list.append(f"{mm}/{dd}/{yy}") - else: - dos_list.append("") - - # ---- POS (Place of Service) — 2-digit code, column B ----------- - # Strategy: find the 2-digit token that appears BEFORE the CPT code - # in the row text. Common POS values: 11,12,21,22,23,24,31,32,33, - # 41,42,49,50,51,52,53,54,55,56,57,61,62,65,71,72,81,99 - pos_val = "" - row_before_cpt = row_text[: row_text.index(cpt_val)] - pos_candidates = re.findall(r"\b(\d{2})\b", row_before_cpt) - # Take the LAST 2-digit number before CPT (closest to CPT = POS) - # but exclude date fragments (must not be 01-31 preceded by another) - valid_pos_codes = { - "11","12","13","14","15","16","17","18","19","20", - "21","22","23","24","25","31","32","33","34","41", - "42","49","50","51","52","53","54","55","56","57", - "60","61","62","65","71","72","81","99", - } - for p in reversed(pos_candidates): - if p in valid_pos_codes: - pos_val = p - break - pos_list.append(pos_val) - - # ---- Modifiers — tokens AFTER CPT code, before $-amount -------- - # Isolate the segment after CPT to reduce noise - cpt_idx = row_text.index(cpt_val) - segment_after_cpt = row_text[cpt_idx + len(cpt_val):] - tokens_after = segment_after_cpt.split() - mod_val = _extract_modifiers_from_tokens(tokens_after) - mod_list.append(mod_val) - - # ---- Units (Days/Units) — column G on CMS-1500 ----------------- - # Units is almost always a small integer (1-99). - # We look for a standalone 1-2 digit number that appears AFTER - # any dollar amount (charges column F). - units_val = "" - # Remove currency-looking tokens to isolate units - seg_no_dollars = re.sub(r"\d{1,3}(?:,\d{3})*(?:\.\d{2})?", " MONEY ", segment_after_cpt) - unit_candidates = re.findall(r"\b([1-9]\d?)\b", seg_no_dollars) - # The first small integer after modifiers is usually units - for uc in unit_candidates: - if 1 <= int(uc) <= 99: - units_val = uc - break - units_list.append(units_val) - - # ---- Rendering Provider / NPI — 10-digit number, column J ------ - npi_candidates = re.findall(r"\b(\d{10})\b", row_text) - rp_list.append(npi_candidates[-1] if npi_candidates else "") - - return { - "cpts": cpts, - "pos_list": pos_list, - "dos_list": dos_list, - "mod_list": mod_list, - "units_list": units_list, - "rp_list": rp_list, - } - - -# --------------------------------------------------------------------------- -# Diagnosis code extraction (field 21 of CMS-1500) -# --------------------------------------------------------------------------- -def _extract_diagnosis_codes(text: str, words: list) -> list: - """Return a deduplicated, ordered list of ICD-10 codes from field 21.""" - codes = [] - - # --- Pass 1: word-coordinate scan near field "21" label ----------------- - for i, w in enumerate(words): - if w["text"] in ("21", "21.", "21:"): - base_y = w["top"] - # Field 21 spans ~80px vertically and the full page width after label - region = [ - nxt for nxt in words - if (base_y - 5) <= nxt["top"] <= (base_y + 90) - and nxt["x0"] >= w["x0"] - 5 - ] - for word in region: - if _is_icd10(word["text"]): - codes.append(word["text"]) - if codes: - break - - # --- Pass 2: regex scan on full extracted text -------------------------- - if not codes: - # Try to isolate the diagnosis section (between field 21 and 22/23/24) - diag_section_match = re.search( - r"(?:21|DIAGNOSIS)[^\n]*\n([\s\S]{0,400})(?:22\.|23\.|24\.|ICD)", - text, re.IGNORECASE - ) - target = diag_section_match.group(1) if diag_section_match else text - raw = re.findall(r"\b([A-Z]\d{2,3}(?:\.\d{1,4})?)\b", target) - codes.extend(raw) - - # --- Pass 3: broad scan if still empty ---------------------------------- - if not codes: - raw = re.findall(r"\b([A-Z]\d{2,3}(?:\.\d{1,4})?)\b", text) - # Filter obvious non-ICD tokens - codes = [c for c in raw if not c.startswith("NP")] - - # Deduplicate preserving order - seen = set() - unique = [] - for c in codes: - if c not in seen: - seen.add(c) - unique.append(c) - return unique - - -# =========================================================================== -# MAIN FUNCTION -# =========================================================================== -def extract_pdf_data_fast( - pdf_path: str, - claim_id: str = "", - patient_account: str = "", - insurance: str = "", - patient_name: str = "", - total_charges: str = "", - signature_physician: str = "", -) -> dict | None: - """ - Extract all relevant fields from a CMS-1500 PDF. - - Parameters - ---------- - pdf_path : Path to the PDF file. - claim_id : Pre-populated from upstream data (e.g. spreadsheet). - patient_account : Pre-populated patient account number. - insurance : Pre-populated insurance name. - patient_name : Pre-populated patient name. - total_charges : Pre-populated total charges. - signature_physician : Pre-populated physician signature. - - Returns - ------- - dict with all extracted fields, or None on critical failure. - """ - - data = { - "Claim ID": claim_id, - "Rendering Provider #": "", - "Signature of Physician": signature_physician, - "Patient's Account No": patient_account, - "Date of Birth": "", - "Insurance Name": insurance, - "Insured ID Number": "", - "Patient's Name": patient_name, - "Date of Service": "", - "Diagnosis Codes": "", - "CPT": "", - "POS": "", - "Modifier": "", - "Units": "", - "Charges": total_charges, - } - - try: - # ------------------------------------------------------------------ # - # Open once, grab text + words + tables # - # ------------------------------------------------------------------ # - with pdfplumber.open(pdf_path) as pdf: - page = pdf.pages[0] - text = page.extract_text(x_tolerance=2, y_tolerance=2) or "" - words = page.extract_words() - tables = page.extract_tables() - - print(f"[DEBUG] Opened PDF: {pdf_path}") - print(f"[DEBUG] Tables found: {len(tables)}") - - # ================================================================== # - # 1. DATE OF BIRTH (field 3 — format MM DD YYYY) # - # ================================================================== # - dob_match = re.search(r"\b(\d{2})\s+(\d{2})\s+(\d{4})\b", text) - if dob_match: - data["Date of Birth"] = ( - f"{dob_match.group(1)}/{dob_match.group(2)}/{dob_match.group(3)}" - ) - - # ================================================================== # - # 2. INSURED'S I.D. NUMBER (field 1a) # - # ================================================================== # - insured_id = "" - - # Method A: word following "1a." label - for i, w in enumerate(words): - if w["text"].lower().rstrip(".") in ("1a",): - for nxt in words[i + 1 : i + 6]: - t = nxt["text"] - # Must be alphanumeric, ≥8 chars, contain both letters & digits - if ( - re.match(r"^[A-Z0-9]{8,20}$", t) - and re.search(r"[A-Z]", t) - and re.search(r"\d", t) - ): - insured_id = t - break - if insured_id: - break - - # Method B: proximity to "INSURED'S I.D. NUMBER" label - if not insured_id: - for i, w in enumerate(words): - if "INSURED" in w["text"].upper(): - nearby = words[i : min(i + 6, len(words))] - if any("NUMBER" in nxt["text"].upper() for nxt in nearby): - base_y = w["top"] - candidates = [ - nxt for nxt in words - if abs(nxt["top"] - base_y) < 15 and nxt["x0"] > w["x0"] - ] - for c in reversed(candidates): - t = c["text"] - if ( - re.match(r"^[A-Z0-9]{8,20}$", t) - and re.search(r"[A-Z]", t) - and re.search(r"\d", t) - ): - insured_id = t - break - if insured_id: - break - - # Method C: top-right quadrant scan - if not insured_id: - _skip = {"MEDICARE", "MEDICAID", "CHAMPVA", "TRICARE", "CHAMPUS"} - for w in words: - if w["x0"] > 400 and w["top"] < 200: - t = w["text"] - if ( - re.match(r"^[A-Z0-9]{8,20}$", t) - and re.search(r"[A-Z]", t) - and re.search(r"\d", t) - and t not in _skip - ): - insured_id = t - break - - data["Insured ID Number"] = insured_id - print(f"[DEBUG] Insured ID: {insured_id}") - - # ================================================================== # - # 3. DIAGNOSIS CODES (field 21) # - # ================================================================== # - dx_codes = _extract_diagnosis_codes(text, words) - if dx_codes: - codes_str = ", ".join(dx_codes) - data["Diagnosis Codes"] = f"{codes_str} (Total: {len(dx_codes)})" - print(f"[DEBUG] Diagnosis codes ({len(dx_codes)}): {dx_codes}") - - # ================================================================== # - # 4. SERVICE LINES (section 24 — columns A-J) # - # ================================================================== # - svc = _parse_service_lines(tables) - cpts = svc["cpts"] - pos_list = svc["pos_list"] - dos_list = svc["dos_list"] - mod_list = svc["mod_list"] - units_list= svc["units_list"] - rp_list = svc["rp_list"] - - print(f"[DEBUG] Service lines found: {len(cpts)}") - print(f"[DEBUG] CPTs : {cpts}") - print(f"[DEBUG] POS : {pos_list}") - print(f"[DEBUG] DOS : {dos_list}") - print(f"[DEBUG] Modifiers : {mod_list}") - print(f"[DEBUG] Units : {units_list}") - print(f"[DEBUG] Providers : {rp_list}") - - # Counts must match CPT count — pad with "" if somehow shorter - n = len(cpts) - def _pad(lst): return lst + [""] * max(0, n - len(lst)) - - pos_list = _pad(pos_list) - dos_list = _pad(dos_list) - mod_list = _pad(mod_list) - units_list = _pad(units_list) - rp_list = _pad(rp_list) - - data["CPT"] = _format_list(cpts, "CPT") - data["POS"] = _format_list(pos_list, "POS") - data["Date of Service"] = _format_list(dos_list, "DOS") - data["Modifier"] = _format_list(mod_list, "Modifier") - data["Units"] = _format_list(units_list, "Units") - data["Rendering Provider #"] = _format_list(rp_list, "RP #") - - return data - - except Exception as exc: - import traceback - print(f"[ERROR] PDF parsing failed → {pdf_path} → {exc}") - traceback.print_exc() - return None - - -# =========================================================================== -# Quick smoke-test (remove / guard with __main__ in production) -# =========================================================================== -if __name__ == "__main__": - import sys, json - - if len(sys.argv) < 2: - print("Usage: python extract_pdf_data_v3.py ") - sys.exit(1) - - result = extract_pdf_data_fast(sys.argv[1]) - print("\n===== EXTRACTED DATA =====") - print(json.dumps(result, indent=2)) \ No newline at end of file diff --git a/src/old/rough draft.py b/src/old/rough draft.py deleted file mode 100644 index a21bd2b..0000000 --- a/src/old/rough draft.py +++ /dev/null @@ -1,1441 +0,0 @@ -import subprocess -import time -import os -import sys -import socket -import re -import pandas as pd -from google.cloud import documentai_v1 as documentai -from google.oauth2 import service_account -import requests -from pathlib import Path -from datetime import datetime -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.common.action_chains import ActionChains -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.chrome.service import Service -from selenium.common.exceptions import WebDriverException -from selenium.common.exceptions import NoSuchElementException -from selenium.common.exceptions import StaleElementReferenceException -from selenium.webdriver.support.ui import Select -from selenium.webdriver.common.keys import Keys - -# ================= DOCUMENT AI CONFIG ================= -DOCAI_CREDENTIALS_PATH = r"C:\Potential Project G2\.env\pdf-extraction-492807-41531f078f92.json" -DOCAI_PROJECT_ID = "1065395041789" -DOCAI_LOCATION = "us" -DOCAI_PROCESSOR_ID = "2d4c7121ce241b13" - -try: - from pywinauto import Application -except Exception: - Application = None - -# ================= PROJECT ROOT ================= -if getattr(sys, "frozen", False): - PROJECT_ROOT = Path(sys.executable).parent -else: - PROJECT_ROOT = Path(__file__).resolve().parent.parent - -# ================= GLOBAL CONFIG ================= -DEBUG_PORT = 9222 -USER_DATA_DIR = os.path.join( - os.environ["USERPROFILE"], "Automation", "ChromeProfile_CF" -) - -today_date = datetime.now().strftime("%m-%d-%Y") -DOWNLOADS_DIR = Path.home() / "Downloads" - -# ================= CHROME PATH ================= -def find_chrome_path(): - possible_paths = [ - r"C:\Program Files\Google\Chrome\Application\chrome.exe", - r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", - os.path.join( - os.environ.get("LOCALAPPDATA", ""), - r"Google\Chrome\Application\chrome.exe" - ) - ] - for path in possible_paths: - if os.path.exists(path): - return path - return None - -# ================= PROXY CONFIG ================= -def read_input_config(): - config_path = PROJECT_ROOT / "Input" / "input.xlsx" - if not config_path.exists(): - print("[ERROR] input.xlsx not found.") - sys.exit(1) - df = pd.read_excel(config_path, sheet_name="Sheet1", header=None) - config = {} - for _, row in df.iterrows(): - key = str(row[0]).strip() - value = str(row[1]).strip() - config[key] = value - proxy_ip = config.get("Proxy IP", "") - proxy_port = config.get("Proxy Port", "") - waystar_username = config.get("Waystar Username", "") - waystar_password = config.get("Waystar Password", "") - portal_url = config.get("Waystar URL", "") - first_car_answer = config.get("Waystar SQ First Car", "") - first_job_answer = config.get("Waystar SQ First Job", "") - print(f"[INFO] Proxy → {proxy_ip}:{proxy_port}") - print(f"[INFO] Username → {waystar_username}") - print(f"[INFO] Portal URL → {portal_url}") - return ( - proxy_ip, proxy_port, waystar_username, waystar_password, - portal_url, first_car_answer, first_job_answer - ) - -# ================= DEBUG PORT CHECK ================= -def wait_for_debug_port(port=9222, timeout=40): - start = time.time() - while time.time() - start < timeout: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - if s.connect_ex(("127.0.0.1", port)) == 0: - return True - time.sleep(1) - return False - -# ================= KILL CHROME ================= -def kill_all_chrome(): - try: - os.system("taskkill /F /IM chrome.exe /T >nul 2>&1") - time.sleep(3) - print("[INFO] All Chrome instances closed") - except Exception as e: - print(f"[WARN] Unable to kill Chrome: {e}") - -# ================= START CHROME DEBUG ================= -def start_chrome_debug(proxy_ip=None, proxy_port=None): - chrome_path = find_chrome_path() - if not chrome_path: - raise Exception("Google Chrome not found.") - print(f"[INFO] Using Chrome path: {chrome_path}") - os.makedirs(USER_DATA_DIR, exist_ok=True) - cmd = [ - chrome_path, - f"--remote-debugging-port={DEBUG_PORT}", - f"--user-data-dir={USER_DATA_DIR}", - "--disable-blink-features=AutomationControlled", - "--ignore-certificate-errors", - "about:blank" - ] - if proxy_ip and proxy_port: - proxy = f"{proxy_ip}:{proxy_port}" - cmd.append(f"--proxy-server=http://{proxy}") - print(f"[INFO] Chrome started with proxy → {proxy}") - else: - print("[INFO] Chrome started WITHOUT proxy") - print("[INFO] Launching Chrome...") - subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - if not wait_for_debug_port(DEBUG_PORT, timeout=40): - print("[ERROR] Chrome debug port did not open") - return False - print("[INFO] Chrome debug port is ready") - return True - -# ================= SCREENSHOT ================= -def capture_screenshot(driver, client_name, step_name): - if not driver: - return None - safe_client = "".join(c for c in client_name if c.isalnum() or c in ("_", "-")) - safe_step = "".join(c for c in step_name if c.isalnum() or c in ("_", "-")) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - screenshots_dir = PROJECT_ROOT / "Screenshots" / today_date / safe_client - screenshots_dir.mkdir(parents=True, exist_ok=True) - screenshot_path = screenshots_dir / f"{safe_step}_{timestamp}.png" - try: - driver.save_screenshot(str(screenshot_path)) - print(f"[INFO] Screenshot saved: {screenshot_path}") - return screenshot_path - except Exception as e: - print(f"[ERROR] Screenshot failed: {e}") - return None - -# ================= ATTACH BROWSER ================= -def attach_browser(download_path, retries=5): - for attempt in range(1, retries + 1): - try: - print(f"[INFO] Attaching browser (attempt {attempt})") - chrome_options = Options() - chrome_options.add_experimental_option( - "debuggerAddress", f"127.0.0.1:{DEBUG_PORT}" - ) - chrome_options.set_capability("webSocketUrl", False) - prefs = { - "download.default_directory": str(download_path), - "download.prompt_for_download": False, - "download.directory_upgrade": True, - "plugins.always_open_pdf_externally": True, - "safebrowsing.enabled": True - } - chrome_options.add_experimental_option("prefs", prefs) - service = Service() - driver = webdriver.Chrome(service=service, options=chrome_options) - _ = driver.current_url - driver.maximize_window() - print("[INFO] Browser attached successfully") - try: - driver.execute_cdp_cmd( - "Page.setDownloadBehavior", - {"behavior": "allow", "downloadPath": str(download_path)} - ) - except Exception as e: - print(f"[WARN] Unable to set download behavior: {e}") - return driver - except WebDriverException as e: - print(f"[WARN] Attach attempt {attempt} failed: {e}") - time.sleep(3) - print("[ERROR] Failed to attach Chrome after retries") - return None - -# ================= LOGIN ================= -def login_to_portal(driver, waystar_username, waystar_password, first_car_answer, first_job_answer): - wait = WebDriverWait(driver, 30) - try: - time.sleep(5) - print("[INFO] Starting login process") - username_field = wait.until(EC.presence_of_element_located((By.ID, "loginName"))) - username_field.clear() - username_field.send_keys(waystar_username) - time.sleep(2) - password_field = wait.until(EC.presence_of_element_located((By.ID, "password"))) - password_field.clear() - password_field.send_keys(waystar_password) - time.sleep(2) - driver.find_element(By.ID, "loginButton").click() - time.sleep(5) - print("[INFO] Login clicked") - try: - alert_page = driver.find_elements(By.XPATH, "//span[contains(text(),'Proactive Alert')]") - if alert_page: - wait.until(EC.element_to_be_clickable((By.XPATH, "//button[@id='continueButton']"))).click() - time.sleep(3) - except: - pass - try: - question_exists = driver.find_elements(By.XPATH, "//*[@id='authDiv']//strong") - if question_exists: - question = wait.until( - EC.presence_of_element_located((By.XPATH, "//*[@id='authDiv']//strong")) - ).text.lower() - print(f"[INFO] Security Question → {question}") - if "first car" in question: - answer = first_car_answer - elif "first job" in question: - answer = first_job_answer - else: - raise Exception(f"Unknown security question: {question}") - answer_box = wait.until(EC.presence_of_element_located((By.ID, "verifyAnswer"))) - answer_box.clear() - answer_box.send_keys(answer) - driver.find_element(By.ID, "VerifyButton").click() - print("[INFO] Security answer submitted") - time.sleep(3) - except: - pass - try: - security_qpage = driver.find_elements(By.XPATH, "//h2[normalize-space()='Security Information']") - if security_qpage: - print("[INFO] Security page appeared, skipping") - wait.until(EC.element_to_be_clickable((By.ID, "NextButton"))).click() - time.sleep(2) - wait.until( - EC.element_to_be_clickable((By.XPATH, "//a[normalize-space()='Skip This Step']")) - ).click() - time.sleep(5) - except: - pass - print("[INFO] Login completed successfully") - except Exception as e: - print(f"[ERROR] Login failed: {e}") - raise - -# ================= CALCULATE DATE RANGE ================= -def get_date_range(): - now = datetime.now() - first_of_current = datetime(now.year, now.month, 1) - last_of_prev = first_of_current - pd.Timedelta(days=1) - first_of_prev = datetime(last_of_prev.year, last_of_prev.month, 1) - return ( - first_of_prev.strftime("%m/%d/%Y"), - last_of_prev.strftime("%m/%d/%Y"), - ) - -# ================= APPLY FILTERS ================= -def apply_filters(driver): - wait = WebDriverWait(driver, 40) - print("[INFO] Applying filters") - wait.until(EC.presence_of_element_located((By.ID, "ClaimListingSearchButtonBottom"))) - wait.until(EC.presence_of_element_located((By.ID, "SearchCriteria_Status"))) - status_dropdown = Select(wait.until(EC.element_to_be_clickable((By.ID, "SearchCriteria_Status")))) - status_dropdown.select_by_visible_text("All") - date_dropdown = Select(wait.until(EC.element_to_be_clickable((By.ID, "SearchCriteria_TransDate")))) - date_dropdown.select_by_visible_text("Custom") - time.sleep(2) - start_date, end_date = get_date_range() - print(f"[INFO] Setting date range: {start_date} to {end_date}") - try: - start_date_field = wait.until(EC.presence_of_element_located((By.ID, "transactionDateFromDateBox"))) - start_date_field.clear() - start_date_field.send_keys(start_date) - time.sleep(1) - except Exception as e: - print(f"[WARN] Could not set start date: {e}") - try: - end_date_field = wait.until(EC.presence_of_element_located((By.ID, "transactionDateToDateBox"))) - end_date_field.clear() - end_date_field.send_keys(end_date) - time.sleep(1) - except Exception as e: - print(f"[WARN] Could not set end date: {e}") - search_btn = wait.until(EC.element_to_be_clickable((By.ID, "ClaimListingSearchButtonBottom"))) - driver.execute_script("arguments[0].click();", search_btn) - wait.until(EC.presence_of_element_located((By.XPATH, "//table[@id='claimsGrid']//tbody//tr"))) - time.sleep(3) - print("[INFO] Filters applied successfully") - -# ================= CLICK NEXT PAGE ================= -def click_next_page(driver): - try: - next_btn = driver.find_element(By.XPATH, "//button[@id='innerPagerRight']") - if "disabled" in next_btn.get_attribute("class"): - return False - driver.execute_script("arguments[0].click();", next_btn) - WebDriverWait(driver, 10).until(EC.staleness_of(next_btn)) - time.sleep(2) - return True - except: - return False - -# ================= DIRECT DOWNLOAD ================= -def download_pdf_direct(driver, pdf_url, save_path): - try: - cookies = {c["name"]: c["value"] for c in driver.get_cookies()} - headers = { - "User-Agent": driver.execute_script("return navigator.userAgent;"), - "Referer": driver.current_url, - "Accept": "application/pdf,*/*", - } - response = requests.get( - pdf_url, headers=headers, cookies=cookies, - stream=True, timeout=60 - ) - if response.status_code != 200: - print(f"[ERROR] HTTP {response.status_code} → {pdf_url}") - return False - content_type = response.headers.get("Content-Type", "") - if "pdf" not in content_type.lower() and "octet-stream" not in content_type.lower(): - print(f"[WARN] Unexpected content type → {content_type}") - save_path = Path(save_path) - save_path.parent.mkdir(parents=True, exist_ok=True) - with open(save_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - print(f"[SUCCESS] Downloaded → {save_path.name}") - return True - except Exception as e: - print(f"[ERROR] Direct download failed → {e}") - return False - - -# ============================================================ -# CMS-1500 DOCUMENT AI EXTRACTOR — v6.0 -# -# FIXES vs v5.x: -# 1. ICD-10 / Diagnosis codes: broadened regex to catch -# alphanumeric codes with suffix letters (S81812A, Z7409, -# L89892, Q4250, etc.) -# 2. CPT: now accepts alphanumeric HCPCS codes (Q4250, -# G0283, 11042, etc.) — 5-char codes starting with -# digit OR letter followed by 4 digits. -# 3. Date of Service: serial-number column (x < 0.04) is -# now excluded from DOS token scanning so row numbers -# like "01","02" are never picked up as MM. -# 4. Units: tightened column bounds and added a guard that -# prevents the Diagnosis Pointer column (single letter -# A-L) from being mistaken for a unit value. -# ============================================================ - -# ================= DOCUMENT AI CONFIG (duplicate import guard) ================= -# Already imported at top of file — reuse same constants. - -# ============================================================ -# NORMALIZED FIELD REGIONS — CALIBRATED v6.0 -# ============================================================ -FIELD_1A = { - "x_min": 0.55, "x_max": 0.95, - "y_min": 0.095, "y_max": 0.185, -} - -FIELD_21 = { - "x_min": 0.02, "x_max": 0.68, - "y_min": 0.510, "y_max": 0.590, -} - -SECTION_24_Y_MIN = 0.615 -SECTION_24_Y_MAX = 0.880 - -# ── Column x-ranges (normalized, 0–1) ────────────────────── -# FIX #3: DOS_FROM_MM x_min raised to 0.045 so the row-serial -# number printed in the left gutter (x ≈ 0.01–0.035) is -# excluded from date scanning. -COL = { - "DOS_FROM_MM": (0.045, 0.085), # was (0.025, 0.075) - "DOS_FROM_DD": (0.078, 0.120), # was (0.070, 0.112) - "DOS_FROM_YY": (0.112, 0.165), # was (0.105, 0.160) - "POS": (0.215, 0.272), - "CPT": (0.285, 0.430), # slightly wider for alpha-CPTs - "MODIFIER": (0.410, 0.540), - # FIX #4: Units column tightened; avoids Diagnosis-Pointer - # column which sits around x=0.62–0.64 - "UNITS": (0.648, 0.695), # was (0.648, 0.720) - "RENDERING": (0.810, 0.998), -} - -_VALID_MODIFIERS = { - "GP","GN","GO","GQ","GX","GY","GZ", - "LT","RT", - "E1","E2","E3","E4", - "FA","F1","F2","F3","F4","F5","F6","F7","F8","F9", - "TA","T1","T2","T3","T4","T5","T6","T7","T8","T9", - "TC","26","59","25","51","76","77","78","79","91", - "24","57","58","80","81","82", - "AA","AD","AE","AF","AG","AH","AI","AJ","AK","AM","AS","AT", - "CQ","CR","CS","GA","GC","GJ","GK","GL","GT", - "JA","JB","JW","JZ","KX","KU","KV","NU","RR","UE", - "Q0","Q1","Q2","Q3","Q4","Q5","Q6", - "SA","SB","SC","SD","SF","SG","XE","XP","XS","XU", - "22","23","32","33","47","50","52","53", - "54","55","56","62","63","66", - "73","74","90","92","93","95","96","97","99", - "AB", -} - -_NOT_MODIFIERS = { - "MM","DD","YY","TO","OF","OR","IN","IS","NO","BY", - "AT","BE","DO","HE","HI","IF","ME","MY","OK","ON", - "SO","UP","US","WE","NPI","ICD","CPT","POS","DOS", - "ID","AN","AC","EMG","FROM","THE","AND", -} - -_VALID_POS = { - "11","12","13","14","15","16","17","18","19","20", - "21","22","23","24","25","31","32","33","34", - "41","42","49","50","51","52","53","54","55","56","57", - "60","61","62","65","71","72","81","99", -} - -_SKIP_ID_LABELS = { - "MEDICARE","MEDICAID","CHAMPVA","TRICARE","CHAMPUS", - "NUMBER","INSURED","MEMBER","HEALTH","PLAN","PICA", - "PROGRAM","ITEM","FOR","OTHER","GROUP","FECA","CARRIER", - "1A","FOR","IN","ITEM","NPI","SSN","EIN","QUAL", - "ID","INSURED'S","I.D.","OTHER","(FOR","(",")", - "1)","IN","ITEM","1", -} - -# ── Diagnosis Pointer letters — single letters that appear -# in the E column of section 24 and must NOT be units ── -_DIAG_POINTER_LETTERS = set("ABCDEFGHIJKL") - -# ============================================================ -# DOCUMENT AI CLIENT -# ============================================================ - -def _get_docai_client(): - credentials = service_account.Credentials.from_service_account_file( - DOCAI_CREDENTIALS_PATH, - scopes=["https://www.googleapis.com/auth/cloud-platform"] - ) - return documentai.DocumentProcessorServiceClient( - credentials=credentials, - client_options={"api_endpoint": "us-documentai.googleapis.com"} - ) - - -def _process_pdf_with_docai(pdf_path: str): - client = _get_docai_client() - with open(pdf_path, "rb") as f: - pdf_bytes = f.read() - processor_name = ( - f"projects/{DOCAI_PROJECT_ID}/locations/{DOCAI_LOCATION}" - f"/processors/{DOCAI_PROCESSOR_ID}" - ) - result = client.process_document(request=documentai.ProcessRequest( - name=processor_name, - raw_document=documentai.RawDocument( - content=pdf_bytes, mime_type="application/pdf" - ) - )) - return result.document - - -def _token_text(document, token) -> str: - seg = token.layout.text_anchor.text_segments - if not seg: - return "" - start = int(seg[0].start_index) if seg[0].start_index else 0 - end = int(seg[0].end_index) - return document.text[start:end].strip() - - -def _token_norm_bounds(token): - verts = token.layout.bounding_poly.normalized_vertices - if len(verts) < 4: - return None - xs = [v.x for v in verts] - ys = [v.y for v in verts] - return min(xs), min(ys), max(xs), max(ys) - - -def _build_words(document, page) -> list: - words = [] - for token in page.tokens: - text = _token_text(document, token).strip() - if not text: - continue - bounds = _token_norm_bounds(token) - if bounds is None: - continue - x0, y_top, x1, y_bot = bounds - words.append({ - "text": text, - "x0": x0, - "x1": x1, - "y_top": y_top, - "y_bot": y_bot, - "x_mid": (x0 + x1) / 2, - "y_mid": (y_top + y_bot) / 2, - }) - return words - - -def _words_in_region(words: list, region: dict) -> list: - return [ - w for w in words - if region["x_min"] <= w["x_mid"] <= region["x_max"] - and region["y_min"] <= w["y_mid"] <= region["y_max"] - ] - - -def _words_in_col(words: list, row_y_mid: float, - col_name: str, y_tol: float = 0.013) -> list: - x_min, x_max = COL[col_name] - return [ - w for w in words - if x_min <= w["x_mid"] <= x_max - and abs(w["y_mid"] - row_y_mid) <= y_tol - ] - - -# ============================================================ -# FIX #1 — ICD-10 / Diagnosis code validator -# Now accepts: -# • Standard: A00–Z99, A00.0–Z99.9, up to 7 chars after dot -# • Suffix letters (7th character extension): S81812A, M80011A -# • Short alphanumeric codes without dot: Z7409, L89892, Q4250 -# • Must start with a letter; second char must be a digit -# ============================================================ -def _is_icd10(t: str) -> bool: - t = t.upper().strip() - # Pattern 1: Dotted form — e.g. A01.0, M54.51, S72.001A - if re.fullmatch(r"[A-Z]\d{2,3}\.\d{1,4}[A-Z]?", t): - return True - # Pattern 2: Non-dotted — letter + digit + 1-5 alphanumeric chars - # Covers: R2681, G20C, S81812A, Z7409, L89892, Q4250, F329, Z23 - # Requires: starts with letter, second char is digit, total 3-7 chars - if re.fullmatch(r"[A-Z]\d[0-9A-Z]{1,5}", t): - return True - return False - - -# ============================================================ -# FIX #2 — CPT / HCPCS code validator -# Now accepts: -# • Standard CPT : 5 digits (97110, 99348, 15271) -# • HCPCS Level II: letter + 4 digits (Q4250, G0283, L8699) -# • Category III : 4 digits + letter (0596T) — rare but valid -# ============================================================ -def _is_cpt(t: str) -> bool: - t = t.strip().upper() - # Standard 5-digit numeric CPT - if re.fullmatch(r"\d{5}", t): - return 10000 <= int(t) <= 99999 - # HCPCS Level II: A–V letter + 4 digits (excludes W–Z which are - # not standard HCPCS prefixes used on CMS-1500) - if re.fullmatch(r"[A-V]\d{4}", t): - return True - # Category III CPT: 4 digits + T/F/U/M suffix - if re.fullmatch(r"\d{4}[TFUM]", t): - return True - return False - - -def _is_icd10(t: str) -> bool: - """ - FIX #1 — broadened ICD-10-CM validator. - Accepts codes like: S81812A, Z7409, L89892, Q4250, - M80011A, F329, Z23, as well as dotted forms A01.0, M54.5 - """ - t = t.upper().strip() - # Must start with a letter and have at least 3 chars total - if len(t) < 3 or not t[0].isalpha(): - return False - # Second character must be a digit (ICD-10-CM rule) - if len(t) < 2 or not t[1].isdigit(): - return False - # Dotted form: e.g. A01.0, M54.51, S72.001A - if re.fullmatch(r"[A-Z]\d{2,3}\.\d{1,4}[A-Z]?", t): - return True - # Non-dotted form: letter + 2–6 alphanumeric chars - # second char already confirmed digit above - if re.fullmatch(r"[A-Z][0-9][0-9A-Z]{1,5}", t): - return True - return False - - -def _is_valid_insured_id(t: str) -> bool: - t_clean = re.sub(r"[-_.\s()]", "", t.upper()) - if not re.match(r"^[A-Z0-9]+$", t_clean): - return False - length = len(t_clean) - if length < 5 or length > 20: - return False - base = t.upper().rstrip(".,)(") - if base in _SKIP_ID_LABELS: - return False - if any(label in t.upper() - for label in {"INSURED","NUMBER","PROGRAM","MEDICARE","MEDICAID", - "PICA","CARRIER","HEALTH","PLAN","CHAMPVA","TRICARE"}): - return False - if re.fullmatch(r"\d{10}", t_clean): - return False - if _is_cpt(t_clean): - return False - if re.fullmatch(r"[A-Z]{1,5}", t_clean): - return False - if re.fullmatch(r"\d+", t_clean): - if 7 <= length <= 12: - return True - return False - if re.search(r"[A-Z]", t_clean) and re.search(r"\d", t_clean): - return True - return False - - -def _is_valid_dos(mm: str, dd: str, yy: str) -> bool: - try: - m, d, y = int(mm), int(dd), int(yy) - return (1 <= m <= 12) and (1 <= d <= 31) and (2000 <= y <= 2099) - except Exception: - return False - - -def _extract_dob(text: str) -> str: - m = re.search(r"\b(\d{2})\s+(\d{2})\s+(\d{4})\b", text) - if m: - return f"{m.group(1)}/{m.group(2)}/{m.group(3)}" - m = re.search(r"\b(\d{2})[\/\-](\d{2})[\/\-](\d{4})\b", text) - if m: - return f"{m.group(1)}/{m.group(2)}/{m.group(3)}" - return "" - - -def _extract_insured_id(words: list, text: str) -> str: - wide_region = {"x_min": 0.50, "x_max": 0.95, - "y_min": 0.08, "y_max": 0.20} - wide_tokens = _words_in_region(words, wide_region) - print(f"[DEBUG] Field 1a wide region tokens: " - f"{[(w['text'], round(w['y_mid'],3)) for w in wide_tokens]}") - - candidates = _words_in_region(words, FIELD_1A) - for w in sorted(candidates, key=lambda w: (w["y_top"], w["x0"])): - t = w["text"].strip() - if _is_valid_insured_id(t): - print(f"[DEBUG] Insured ID via region: {t}") - return t.upper() - - for i, w in enumerate(words): - if re.fullmatch(r"1[aA]\.?", w["text"].strip()): - for nxt in words[i+1: i+15]: - if nxt["x_mid"] > 0.50 and nxt["y_mid"] < 0.22: - if _is_valid_insured_id(nxt["text"].strip()): - print(f"[DEBUG] Insured ID via 1a label: {nxt['text']}") - return nxt["text"].strip().upper() - - top_right = [w for w in words - if w["x_mid"] > 0.52 and w["y_mid"] < 0.22] - for w in sorted(top_right, key=lambda w: (w["y_top"], w["x0"])): - t = w["text"].strip() - if _is_valid_insured_id(t): - print(f"[DEBUG] Insured ID via top-right scan: {t}") - return t.upper() - - print(f"[DEBUG] Insured ID: not found") - return "" - - -# ============================================================ -# FIX #1 (cont.) — Diagnosis code extraction -# Uses the new broadened _is_icd10() validator. -# Also skips single-letter slot labels AND slot numbers 0-9. -# ============================================================ -def _extract_diagnosis_codes(words: list, text: str) -> list: - region_words = _words_in_region(words, FIELD_21) - print(f"[DEBUG] Field 21 region token count: {len(region_words)}") - print(f"[DEBUG] Field 21 tokens: {[w['text'] for w in region_words]}") - codes = [] - seen = set() - - # Slot label pattern — "A." "B." ... "L." or bare "A"–"L" - slot_label_pat = re.compile(r"^[A-L]\.?$", re.IGNORECASE) - - for w in sorted(region_words, key=lambda w: (w["y_top"], w["x0"])): - t = w["text"].strip().upper() - - # Skip slot labels - if slot_label_pat.match(t): - continue - - # Skip pure form/section labels - if t in { - "21", "22", "23", "24", - "0", "O", - "ICD", "IND", "ICO", "ICD-10-CM", - "DIAGNOSIS", "NATURE", "ILLNESS", "INJURY", - }: - continue - - # Skip pure digits - if t.isdigit(): - continue - - # Accept ICD-10 codes (now using broadened validator) - if _is_icd10(t) and t not in seen: - codes.append(t) - seen.add(t) - print(f"[DEBUG] DX captured: {t}") - - # Fallback via raw text regex if region found nothing - if not codes: - print("[DEBUG] Region empty — trying text regex fallback") - seg = re.search( - r"(?:DIAGNOSIS|21\.?)([\s\S]{0,600})(?:24\.|DATE.S.\s*OF\s*SERVICE)", - text, re.IGNORECASE - ) - if seg: - # Broadened fallback pattern: letter + digit + 1-5 more alphanum - for c in re.findall( - r"\b([A-Z][0-9][0-9A-Z]{1,5}(?:\.[0-9A-Z]{1,4})?)\b", - seg.group(1) - ): - cu = c.upper() - if cu not in seen and _is_icd10(cu): - seen.add(cu) - codes.append(cu) - print(f"[DEBUG] DX fallback: {cu}") - - print(f"[DEBUG] Diagnosis codes final ({len(codes)}): {codes}") - return codes - - -# ============================================================ -# SERVICE LINE FIELD EXTRACTORS -# ============================================================ - -def _is_real_cpt_location(w: dict, words: list) -> bool: - """ - Verify a CPT candidate is truly in section 24 by checking - that a valid DOS (date) or NPI exists on the same row. - """ - row_y = w["y_mid"] - y_tol = 0.018 - - # FIX #3: use updated COL ranges — serial-number gutter excluded - dos_cols = ["DOS_FROM_MM", "DOS_FROM_DD", "DOS_FROM_YY"] - for col in dos_cols: - x_min, x_max = COL[col] - dos_words = [ - d for d in words - if x_min <= d["x_mid"] <= x_max - and abs(d["y_mid"] - row_y) <= y_tol - and re.fullmatch(r"\d{1,4}", d["text"].strip()) - ] - if dos_words: - return True - - x_min_npi, x_max_npi = COL["RENDERING"] - npi_words = [ - d for d in words - if x_min_npi <= d["x_mid"] <= x_max_npi - and abs(d["y_mid"] - row_y) <= y_tol - and re.fullmatch(r"\d{10}", d["text"].strip()) - ] - if npi_words: - return True - - return False - - -# ============================================================ -# FIX #3 — DOS extraction -# Serial-number gutter (x < 0.04) excluded via updated COL -# x_min values. Additionally a guard ensures we never treat -# a 1- or 2-digit number whose x_mid is < 0.04 as a month. -# ============================================================ -def _extract_dos_for_row(words: list, row_y: float) -> str: - def _first_digit(col_name): - ws = _words_in_col(words, row_y, col_name) - for w in sorted(ws, key=lambda w: w["x0"]): - t = w["text"].strip() - # Extra guard: ignore anything in the left gutter - if w["x_mid"] < 0.04: - continue - if re.fullmatch(r"\d{1,4}", t): - return t - return "" - - mm = _first_digit("DOS_FROM_MM").zfill(2) - dd = _first_digit("DOS_FROM_DD").zfill(2) - yy = _first_digit("DOS_FROM_YY") - if re.fullmatch(r"\d{2}", yy): - yy = ("20" + yy) if int(yy) < 50 else ("19" + yy) - if _is_valid_dos(mm, dd, yy): - return f"{mm}/{dd}/{yy}" - - # Fallback: all digit tokens in DOS cols sorted left→right, - # still excluding x < 0.04 - all_w = [] - for col in ("DOS_FROM_MM", "DOS_FROM_DD", "DOS_FROM_YY"): - all_w += [ - w for w in _words_in_col(words, row_y, col) - if w["x_mid"] >= 0.04 - ] - parts = [ - w["text"].strip() - for w in sorted(all_w, key=lambda w: w["x0"]) - if re.fullmatch(r"\d+", w["text"].strip()) - ] - if len(parts) >= 3: - mm2, dd2, yy2 = parts[0].zfill(2), parts[1].zfill(2), parts[2] - if re.fullmatch(r"\d{2}", yy2): - yy2 = ("20" + yy2) if int(yy2) < 50 else ("19" + yy2) - if _is_valid_dos(mm2, dd2, yy2): - return f"{mm2}/{dd2}/{yy2}" - return "" - - -def _extract_pos_for_row(words: list, row_y: float) -> str: - for w in _words_in_col(words, row_y, "POS"): - if w["text"].strip() in _VALID_POS: - return w["text"].strip() - return "" - - -def _extract_modifier_for_row(words: list, row_y: float) -> str: - mod_words = _words_in_col(words, row_y, "MODIFIER", y_tol=0.016) - found = [] - for w in sorted(mod_words, key=lambda w: w["x0"]): - t = w["text"].strip().upper() - if not t: - continue - if re.fullmatch(r"[A-L]", t): - continue - if t in _NOT_MODIFIERS: - continue - if t in _VALID_MODIFIERS and t not in found: - found.append(t) - if len(found) == 4: - break - return " ".join(found) - - -# ============================================================ -# FIX #4 — Units extraction -# • Tighter x-range in COL["UNITS"] avoids Diagnosis Pointer col -# • Explicitly rejects single letters A-L (diagnosis pointers) -# • Rejects zero — units on CMS-1500 are always ≥ 1 -# • Fallback exclusion set improved -# ============================================================ -def _extract_units_for_row(words: list, row_y: float, - cpt_val: str = "", mod_val: str = "", - pos_val: str = "", npi_val: str = "") -> str: - unit_x_min, unit_x_max = COL["UNITS"] - y_tol = 0.022 - - # Primary: strict column, reject diagnosis pointers and zero - unit_col_words = [ - w for w in words - if unit_x_min <= w["x_mid"] <= unit_x_max - and abs(w["y_mid"] - row_y) <= y_tol - ] - for w in sorted(unit_col_words, key=lambda w: w["x0"]): - t = w["text"].strip().upper() - # FIX #4a: skip single diagnosis-pointer letters - if t in _DIAG_POINTER_LETTERS: - continue - # FIX #4b: must be 1–2 digit number, value 1–99 - if re.fullmatch(r"[1-9]\d?", t): - return t - - # Fallback: eliminate all known-column tokens and pick leftover digit - ban = {cpt_val.upper(), pos_val.upper(), npi_val.upper()} - if mod_val: - for m in mod_val.split(): - ban.add(m.upper()) - - for col in ("DOS_FROM_MM", "DOS_FROM_DD", "DOS_FROM_YY"): - for w in _words_in_col(words, row_y, col, y_tol=0.025): - ban.add(w["text"].strip().upper()) - - for w in _words_in_col(words, row_y, "RENDERING", y_tol=0.025): - ban.add(w["text"].strip().upper()) - for w in _words_in_col(words, row_y, "CPT", y_tol=0.025): - ban.add(w["text"].strip().upper()) - for w in _words_in_col(words, row_y, "POS", y_tol=0.025): - ban.add(w["text"].strip().upper()) - - # In the fallback section, add x_mid column guard: - row_tokens = [ - w for w in words - if SECTION_24_Y_MIN <= w["y_mid"] <= SECTION_24_Y_MAX - and abs(w["y_mid"] - row_y) <= 0.025 - and unit_x_min <= w["x_mid"] <= unit_x_max # ← ADD THIS GUARD - and w["text"].strip().upper() not in ban - and w["text"].strip().upper() not in _DIAG_POINTER_LETTERS - and re.fullmatch(r"\d{1,3}", w["text"].strip()) - and 1 <= int(w["text"].strip()) <= 99 - ] - - for tok in sorted(row_tokens, key=lambda w: w["x_mid"]): - t = tok["text"].strip() - if re.fullmatch(r"[1-9]\d?", t): - return t - - return "" - - -def _extract_npi_for_row(words: list, row_y: float) -> str: - for w in _words_in_col(words, row_y, "RENDERING", y_tol=0.016): - if re.fullmatch(r"\d{10}", w["text"].strip()): - return w["text"].strip() - return "" - - -# ============================================================ -# FIX #2 (cont.) — Service line extraction uses new _is_cpt() -# ============================================================ -def _extract_service_lines(words: list): - cpts, pos_list, dos_list, mod_list, units_list, rp_list = [], [], [], [], [], [] - - x_min_cpt, x_max_cpt = COL["CPT"] - - # FIX #2: use updated _is_cpt() that accepts HCPCS alpha codes - cpt_candidates = [ - w for w in words - if _is_cpt(w["text"].strip()) - and x_min_cpt <= w["x_mid"] <= x_max_cpt - and SECTION_24_Y_MIN <= w["y_mid"] <= SECTION_24_Y_MAX - ] - - print(f"[DEBUG] CPT candidates before validation: " - f"{[(w['text'], round(w['y_mid'],4)) for w in cpt_candidates]}") - - cpt_words = [ - w for w in cpt_candidates - if _is_real_cpt_location(w, words) - ] - - print(f"[DEBUG] CPT tokens after validation: " - f"{[(w['text'], round(w['y_mid'],4)) for w in cpt_words]}") - - if not cpt_words: - print("[DEBUG] No valid CPTs after validation — trying wider scan") - wider_candidates = [ - w for w in words - if _is_cpt(w["text"].strip()) - and 0.55 <= w["y_mid"] <= 0.92 - and 0.20 <= w["x_mid"] <= 0.50 - and _is_real_cpt_location(w, words) - ] - cpt_words = wider_candidates - print(f"[DEBUG] Wider scan CPTs: {[w['text'] for w in cpt_words]}") - - seen_rows = [] - for cpt_word in sorted(cpt_words, key=lambda w: w["y_mid"]): - row_y = cpt_word["y_mid"] - - if any(abs(row_y - sr) <= 0.008 for sr in seen_rows): - continue - seen_rows.append(row_y) - - cpt_val = cpt_word["text"].strip().upper() - dos = _extract_dos_for_row(words, row_y) - pos = _extract_pos_for_row(words, row_y) - mod = _extract_modifier_for_row(words, row_y) - npi = _extract_npi_for_row(words, row_y) - units = _extract_units_for_row(words, row_y, cpt_val, mod, pos, npi) - - cpts.append(cpt_val) - dos_list.append(dos) - pos_list.append(pos) - mod_list.append(mod) - units_list.append(units) - rp_list.append(npi) - - print(f"[DEBUG] CPT={cpt_val} | DOS={dos} | POS={pos} | " - f"MOD={mod} | UNITS={units} | NPI={npi}") - - return cpts, pos_list, dos_list, mod_list, units_list, rp_list - - -def _format_list(values: list, label: str) -> str: - clean = [v for v in values if v] - if not clean: - return "" - if len(clean) == 1: - return clean[0] - return "; ".join(f"{label} {i+1}: {v}" for i, v in enumerate(clean)) - - -# ============================================================ -# MAIN ENTRY POINT -# ============================================================ - -def extract_pdf_data_fast( - pdf_path, - claim_id="", - patient_account="", - insurance="", - patient_name="", - total_charges="", - signature_physician="", -): - data = { - "Claim ID": claim_id, - "Rendering Provider #": "", - "Signature of Physician": signature_physician, - "Patient's Account No": patient_account, - "Date of Birth": "", - "Insurance Name": insurance, - "Insured ID Number": "", - "Patient's Name": patient_name, - "Date of Service": "", - "Diagnosis Codes": "", - "CPT": "", - "POS": "", - "Modifier": "", - "Units": "", - "Charges": total_charges, - } - - try: - print(f"[INFO] Processing with Document AI: {pdf_path}") - document = _process_pdf_with_docai(str(pdf_path)) - - if not document.pages: - print("[ERROR] No pages returned by Document AI") - return None - - page = document.pages[0] - text = document.text - - print(f"[DEBUG] Document AI extracted {len(text)} characters") - print(f"[DEBUG] Token count: {len(page.tokens)}") - print(f"[DEBUG] Page dimensions: {page.dimension.width:.1f} x " - f"{page.dimension.height:.1f} pt") - - words = _build_words(document, page) - print(f"[DEBUG] Words built: {len(words)}") - - data["Date of Birth"] = _extract_dob(text) - data["Insured ID Number"] = _extract_insured_id(words, text) - - dx_codes = _extract_diagnosis_codes(words, text) - if dx_codes: - data["Diagnosis Codes"] = ( - ", ".join(dx_codes) + f" (Total: {len(dx_codes)})" - ) - - cpts, pos_list, dos_list, mod_list, units_list, rp_list = \ - _extract_service_lines(words) - - data["CPT"] = _format_list(cpts, "CPT") - data["POS"] = _format_list(pos_list, "POS") - data["Date of Service"] = _format_list(dos_list, "DOS") - data["Modifier"] = _format_list(mod_list, "Modifier") - data["Units"] = _format_list(units_list, "Units") - data["Rendering Provider #"] = _format_list(rp_list, "RP #") - - return data - - except Exception as e: - import traceback - print(f"[ERROR] Document AI extraction failed → {pdf_path}") - print(f"[ERROR] {e}") - traceback.print_exc() - return None - - -# ================= UPDATE LOG ================= -def update_log(log_path, claim_id, status, remarks=""): - now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - new_row = pd.DataFrame([{ - "Claim ID": claim_id, - "Status": status, - "Remarks": remarks, - "Timestamp": now - }]) - try: - existing = pd.read_excel(log_path) - final = pd.concat([existing, new_row], ignore_index=True) - except: - final = new_row - final.to_excel(log_path, index=False) - - -# ================= INCREMENTAL EXCEL SAVE ================= -def _save_batch_to_excel(excel_path, batch_data: list): - if not batch_data: - return - df_new = pd.DataFrame(batch_data) - try: - existing = pd.read_excel(excel_path) - existing = existing.dropna(how="all") - final = pd.concat([existing, df_new], ignore_index=True) - except Exception: - final = df_new - final.to_excel(excel_path, index=False) - print(f"[INFO] Saved {len(batch_data)} records → {excel_path.name} " - f"(total rows: {len(final)})") - - -def _get_processed_claim_ids(log_path) -> set: - try: - df = pd.read_excel(log_path) - done = df[df["Status"] == "SUCCESS"]["Claim ID"].astype(str).tolist() - return set(done) - except Exception: - return set() - - -def run_claim_report(driver, account_name): - SAVE_EVERY = 10 - wait = WebDriverWait(driver, 30) - actions = ActionChains(driver) - print(f"\n[INFO] ====== Running report for {account_name} ======") - today = datetime.now().strftime("%m-%d-%Y") - safe_name = "".join( - c for c in account_name if c.isalnum() or c in (" ", "_", "(", ")") - ) - base = PROJECT_ROOT / "Output" / today - pdf_folder = base / "PDF Folder" / safe_name - report_dir = base / "reports" - pdf_folder.mkdir(parents=True, exist_ok=True) - report_dir.mkdir(parents=True, exist_ok=True) - excel_path = report_dir / f"{safe_name}.xlsx" - logs_dir = base / "Logs" - logs_dir.mkdir(parents=True, exist_ok=True) - log_path = logs_dir / f"{safe_name}_log.xlsx" - if not excel_path.exists(): - pd.DataFrame(columns=[ - "Claim ID", "Rendering Provider #", "Signature of Physician", - "Patient's Account No", "Date of Birth", "Insurance Name", - "Insured ID Number", "Patient's Name", "Date of Service", - "Diagnosis Codes", "CPT", "POS", "Modifier", "Units", "Charges" - ]).to_excel(excel_path, index=False) - if not log_path.exists(): - pd.DataFrame(columns=[ - "Claim ID", "Status", "Remarks", "Timestamp" - ]).to_excel(log_path, index=False) - already_done = _get_processed_claim_ids(log_path) - print(f"[INFO] {len(already_done)} claims already processed for {account_name}") - try: - claims_processing = wait.until( - EC.presence_of_element_located( - (By.XPATH, "//a[contains(.,'Claims Processing')]") - ) - ) - actions.move_to_element(claims_processing).perform() - time.sleep(1) - claims_option = wait.until( - EC.presence_of_element_located( - (By.XPATH, "//a[normalize-space()='Claims']") - ) - ) - driver.execute_script("arguments[0].click();", claims_option) - time.sleep(3) - except Exception as e: - print(f"[ERROR] Navigation to Claims failed: {e}") - return - try: - apply_filters(driver) - except Exception as e: - print(f"[ERROR] Filter application failed: {e}") - return - page = 1 - batch = [] - total_saved = 0 - consecutive_errors = 0 - while True: - print(f"\n[INFO] Processing page {page}") - try: - WebDriverWait(driver, 20).until( - EC.presence_of_element_located( - (By.XPATH, "//table[@id='claimsGrid']//tbody//tr[@data-claimid]") - ) - ) - rows = driver.find_elements( - By.XPATH, "//table[@id='claimsGrid']//tbody//tr[@data-claimid]" - ) - print(f"[INFO] Rows on page {page} → {len(rows)}") - except Exception as e: - print(f"[ERROR] Could not load rows on page {page}: {e}") - consecutive_errors += 1 - if consecutive_errors >= 3: - print("[ERROR] 3 consecutive page-load failures — stopping account") - break - time.sleep(5) - continue - consecutive_errors = 0 - for i in range(len(rows)): - claim_id = f"row_{page}_{i+1}" - try: - rows = driver.find_elements( - By.XPATH, - "//table[@id='claimsGrid']//tbody//tr[@data-claimid]" - ) - if i >= len(rows): - print(f"[WARN] Row {i+1} no longer exists — skipping") - continue - row = rows[i] - claim_id = row.get_attribute("data-claimid") or "" - instance_id = row.get_attribute("data-instanceid") or "" - if not claim_id or not instance_id: - print(f"[WARN] Missing IDs row {i+1} — skipping") - continue - if claim_id in already_done: - print(f"[INFO] [{i+1}/{len(rows)}] Skipping (already done): {claim_id}") - continue - try: - claim_number = row.find_element(By.XPATH, ".//td[2]").text.strip() - except Exception: - claim_number = "" - try: - sig_raw = row.find_element(By.XPATH, ".//td[10]").text.strip() - if "," in sig_raw: - last, first = sig_raw.split(",", 1) - signature_physician = f"{first.strip().title()} {last.strip().title()}" - else: - signature_physician = sig_raw.strip().title() - except Exception: - signature_physician = "" - insurance = re.sub( - r"\s*\(.*?\)\s*$", "", - row.get_attribute("data-payername") or "" - ).strip() - patient_name = row.get_attribute("data-patientname") or "" - charges = row.get_attribute("data-totalcharges") or "" - print(f"[INFO] [{i+1}/{len(rows)}] Claim: {claim_id} | {patient_name}") - pdf_url = ( - f"https://claims.zirmed.com/Claims/History/ViewClaimPDF" - f"?appId=1&claimId={claim_id}&instanceId={instance_id}" - f"&form=CMS1500_0212" - ) - save_path = pdf_folder / f"{claim_id}.pdf" - if save_path.exists(): - save_path = pdf_folder / f"{claim_id}_{instance_id}.pdf" - success = download_pdf_direct(driver, pdf_url, save_path) - if not success: - time.sleep(3) - success = download_pdf_direct(driver, pdf_url, save_path) - if not success: - update_log(log_path, claim_id, "FAILED", "Download failed after retry") - continue - pdf_data = extract_pdf_data_fast( - save_path, - claim_id=claim_id, - patient_account=claim_number, - insurance=insurance, - patient_name=patient_name, - total_charges=charges, - signature_physician=signature_physician, - ) - if not pdf_data: - update_log(log_path, claim_id, "FAILED", "PDF extraction failed") - continue - batch.append(pdf_data) - already_done.add(claim_id) - update_log(log_path, claim_id, "SUCCESS", "") - if len(batch) >= SAVE_EVERY: - _save_batch_to_excel(excel_path, batch) - total_saved += len(batch) - batch = [] - except Exception as e: - print(f"[ERROR] Row {i+1} (claim {claim_id}) failed → {e}") - update_log(log_path, claim_id, "FAILED", str(e)[:200]) - if batch: - _save_batch_to_excel(excel_path, batch) - total_saved += len(batch) - batch = [] - if not click_next_page(driver): - print(f"[INFO] No more pages for {account_name}") - break - page += 1 - print(f"\n[SUCCESS] {account_name} complete — {total_saved} records saved") - print(f"[INFO] ====== Account {account_name} finished ======\n") - - -def process_all_accounts(driver): - wait = WebDriverWait(driver, 30) - print("[INFO] Starting account processing...") - all_accounts = [] - try: - dropdown = wait.until( - EC.element_to_be_clickable( - (By.XPATH, "//input[contains(@data-search-url,'AccountSearch')]") - ) - ) - dropdown.click() - time.sleep(2) - account_elements = driver.find_elements( - By.XPATH, "//ul[contains(@class,'ui-autocomplete')]//li" - ) - all_accounts = [ - acc.text.strip() for acc in account_elements if acc.text.strip() - ] - print(f"[INFO] Total accounts found → {len(all_accounts)}") - driver.find_element(By.TAG_NAME, "body").click() - time.sleep(1) - except Exception as e: - print(f"[ERROR] Failed to get account list: {e}") - return - if not all_accounts: - print("[WARN] No accounts found — exiting") - return - seen_names = set() - unique_accounts = [] - for name in all_accounts: - if name not in seen_names: - seen_names.add(name) - unique_accounts.append(name) - if len(unique_accounts) < len(all_accounts): - print(f"[INFO] Removed {len(all_accounts) - len(unique_accounts)} " - f"duplicate account names") - all_accounts = unique_accounts - print(f"[INFO] Unique accounts to process → {len(all_accounts)}") - portal_home_url = driver.current_url - processed_accounts = set() - for idx, account_name in enumerate(all_accounts, 1): - if account_name in processed_accounts: - print(f"[WARN] '{account_name}' already processed — skipping") - continue - print(f"\n[INFO] ===== Account {idx}/{len(all_accounts)}: {account_name} =====") - try: - driver.get(portal_home_url) - time.sleep(3) - dropdown = wait.until( - EC.element_to_be_clickable( - (By.XPATH, "//input[contains(@data-search-url,'AccountSearch')]") - ) - ) - dropdown.clear() - dropdown.click() - time.sleep(2) - account_elements = driver.find_elements( - By.XPATH, "//ul[contains(@class,'ui-autocomplete')]//li" - ) - clicked = False - for acc in account_elements: - if acc.text.strip() == account_name: - acc.click() - clicked = True - break - if not clicked: - print(f"[WARN] '{account_name}' not found in dropdown — skipping") - continue - try: - wait.until( - EC.invisibility_of_element_located( - (By.XPATH, "//ul[contains(@class,'ui-autocomplete')]") - ) - ) - except Exception: - pass - time.sleep(3) - processed_accounts.add(account_name) - run_claim_report(driver, account_name) - except Exception as e: - print(f"[ERROR] Account '{account_name}' failed: {e}") - try: - driver.get(portal_home_url) - time.sleep(3) - except Exception: - pass - continue - print(f"\n[INFO] ===== All {len(processed_accounts)} accounts processed =====") - print(f"[INFO] Accounts completed: {sorted(processed_accounts)}") - - -if __name__ == "__main__": - print("=" * 80) - print("🚀 STARTING CHARGE BILLED AUTOMATION - Document AI v6.0") - print("=" * 80) - driver = None - try: - print("\n[STEP 1] Closing existing Chrome instances...") - kill_all_chrome() - print("\n[STEP 2] Reading configuration...") - ( - proxy_ip, proxy_port, username, waystar_password, - portal_url, first_car_answer, first_job_answer - ) = read_input_config() - print("\n[STEP 3] Launching Chrome in debug mode...") - if not start_chrome_debug(proxy_ip, proxy_port): - raise Exception("Chrome debug failed to start") - time.sleep(5) - print("\n[STEP 4] Attaching Selenium to Chrome...") - driver = attach_browser(DOWNLOADS_DIR) - if not driver: - raise Exception("Driver attach failed") - print("\n[STEP 5] Opening Waystar portal...") - driver.execute_script("window.open('');") - driver.switch_to.window(driver.window_handles[-1]) - driver.get(portal_url) - WebDriverWait(driver, 30).until( - EC.presence_of_element_located((By.ID, "loginName")) - ) - print("[INFO] Portal loaded successfully") - print("\n[STEP 6] Performing login...") - login_to_portal(driver, username, waystar_password, first_car_answer, first_job_answer) - print("\n[STEP 7] Starting account processing...") - process_all_accounts(driver) - print("\n" + "=" * 80) - print("✅ AUTOMATION COMPLETED SUCCESSFULLY") - print("=" * 80) - except Exception as e: - print("\n" + "=" * 80) - print(f"❌ FATAL ERROR: {str(e)}") - print("=" * 80) - if driver: - try: - capture_screenshot(driver, "MAIN", "Fatal_Error") - except: - pass - finally: - print("\n[CLEANUP] Closing resources...") - if driver: - try: - driver.quit() - print("[INFO] Browser closed successfully") - except Exception as e: - print(f"[WARN] Error closing browser: {e}") - print("\n🛑 Execution finished\n") \ No newline at end of file diff --git a/src/old/test_docai.py b/src/old/test_docai.py deleted file mode 100644 index 01df89a..0000000 --- a/src/old/test_docai.py +++ /dev/null @@ -1,31 +0,0 @@ -from google.cloud import documentai_v1 as documentai -from google.oauth2 import service_account - -DOCAI_CREDENTIALS_PATH = r"C:\Potential Project G2\.env\pdf-extraction-492807-41531f078f92.json" -DOCAI_PROJECT_ID = "580776411227" -DOCAI_LOCATION = "us" -DOCAI_PROCESSOR_ID = "894f921d904bc9bd" - -def test_connection(): - try: - credentials = service_account.Credentials.from_service_account_file( - DOCAI_CREDENTIALS_PATH, - scopes=["https://www.googleapis.com/auth/cloud-platform"] - ) - client_options = {"api_endpoint": "us-documentai.googleapis.com"} - client = documentai.DocumentProcessorServiceClient( - credentials=credentials, - client_options=client_options - ) - processor_name = ( - f"projects/{DOCAI_PROJECT_ID}/locations/{DOCAI_LOCATION}" - f"/processors/{DOCAI_PROCESSOR_ID}" - ) - processor = client.get_processor(name=processor_name) - print(f"✅ Connection successful!") - print(f" Processor: {processor.display_name}") - print(f" State: {processor.state.name}") - except Exception as e: - print(f"❌ Connection failed: {e}") - -test_connection() \ No newline at end of file diff --git a/src/old/test_pdf.py b/src/old/test_pdf.py deleted file mode 100644 index b34b47c..0000000 --- a/src/old/test_pdf.py +++ /dev/null @@ -1,60 +0,0 @@ -from google.cloud import documentai_v1 as documentai -from google.oauth2 import service_account - -DOCAI_CREDENTIALS_PATH = r"C:\Potential Project G2\.env\pdf-extraction-492807-41531f078f92.json" -DOCAI_PROJECT_ID = "1065395041789" -DOCAI_LOCATION = "us" -DOCAI_PROCESSOR_ID = "2d4c7121ce241b13" - -# ← Put path to any CMS-1500 PDF you have -TEST_PDF_PATH = r"C:\Potential Project G2\Output\04-07-2026\PDF Folder\Dynamic Mobile Rehab (238268)\10882239127.pdf" - -def test_pdf(): - credentials = service_account.Credentials.from_service_account_file( - DOCAI_CREDENTIALS_PATH, - scopes=["https://www.googleapis.com/auth/cloud-platform"] - ) - client_options = {"api_endpoint": "us-documentai.googleapis.com"} - client = documentai.DocumentProcessorServiceClient( - credentials=credentials, - client_options=client_options - ) - - with open(TEST_PDF_PATH, "rb") as f: - pdf_bytes = f.read() - - processor_name = ( - f"projects/{DOCAI_PROJECT_ID}/locations/{DOCAI_LOCATION}" - f"/processors/{DOCAI_PROCESSOR_ID}" - ) - - request = documentai.ProcessRequest( - name=processor_name, - raw_document=documentai.RawDocument( - content=pdf_bytes, - mime_type="application/pdf" - ) - ) - - result = client.process_document(request=request) - document = result.document - - print(f"\n✅ PDF processed successfully!") - print(f" Pages: {len(document.pages)}") - print(f"\n--- RAW TEXT (first 500 chars) ---") - print(document.text[:500]) - print(f"\n--- FORM FIELDS DETECTED ---") - for page in document.pages: - for field in page.form_fields: - key = "".join( - document.text[s.start_index:s.end_index] - for s in field.field_name.text_anchor.text_segments - ).strip() - val = "".join( - document.text[s.start_index:s.end_index] - for s in field.field_value.text_anchor.text_segments - ).strip() - if key and val: - print(f" {key:30s} → {val}") - -test_pdf() \ No newline at end of file