diff --git a/AGENTS.md b/AGENTS.md index 6f7b5bc..1dfcf9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,641 +3,728 @@ ## Context - This project exposes a Flask API that uses Playwright to scrape Yahoo Finance options chains. - Entry point: `scraper_service.py` (launched via `runner.bat` or directly with Python). -- API route: `GET /scrape_sync` with `stock`, optional `expiration|expiry|date`, and `strikeLimit` parameters. -- Expiration inputs: epoch seconds (Yahoo date param) or date strings supported by `DATE_FORMATS`. -- strikeLimit defaults to 25 and controls the number of nearest strikes returned per side. +- The scraper loads the Yahoo options page (optionally with `?date=`) and validates expirations using the YYMMDD code embedded in contract symbols. +- Option chains come from the embedded `optionChain` JSON when available, with an HTML table fallback. + +## API +- Route: `GET /scrape_sync` +- Query params: + - `stock`: symbol (default `MSFT`). + - `expiration|expiry|date`: epoch seconds (Yahoo date param) or a date string matching `DATE_FORMATS`. + - `strikeLimit`: number of nearest strikes to return per side (default `25`). +- Behavior: + - If `strikeLimit` is greater than available strikes, all available rows are returned. + - `pruned_calls_count` and `pruned_puts_count` report how many rows were removed beyond the limit. + - `selected_expiration` reports the resolved expiry (epoch + label), and mismatches return an error. + +## Guard Rails +- Run local 10-cycle validation (4 stocks x 4 expiries) before any deploy or push. +- Run the same 10-cycle validation against the docker container before pushing the image. +- Do not push if any response contains `error` or if contract symbols do not contain the expected YYMMDD code. +- Keep Playwright version aligned with the docker base image (`mcr.microsoft.com/playwright/python:v1.57.0-jammy`). +- Keep the API port open after a successful deploy so it can be tested immediately. + +## Testing +- Local server: + - Start: `.\venv\Scripts\python.exe scraper_service.py` + - Validate: `python scripts/test_cycles.py --base-url http://127.0.0.1:9777/scrape_sync` +- Docker server: + - Start: `docker run --rm -p 9777:9777 rushabhtechie/yahoo-scraper:latest` + - Validate: `python scripts/test_cycles.py --base-url http://127.0.0.1:9777/scrape_sync` +- The test harness verifies: + - Requested expiration matches `selected_expiration.value`. + - Contract symbols include the expected YYMMDD code. + - `total_calls`/`total_puts` match `min(strikeLimit, available)`. + - `pruned_*_count` equals the number of rows removed. ## Docker -- Build: `docker build -t :latest .` -- Run: `docker run --rm -p 9777:9777 :latest` +- Build: `docker build -t rushabhtechie/yahoo-scraper:latest .` +- Run (CPU): `docker run --rm -p 9777:9777 rushabhtechie/yahoo-scraper:latest` - The container uses the Playwright base image with bundled browsers. +## GPU Acceleration +- GPU is auto-detected via `NVIDIA_VISIBLE_DEVICES`, `/dev/nvidia0`, or `/dev/dri`. +- Override detection: + - Force on: `ENABLE_GPU=1` + - Force off: `ENABLE_GPU=0` +- Docker (NVIDIA): `docker run --rm --gpus all -e ENABLE_GPU=1 -p 9777:9777 rushabhtechie/yahoo-scraper:latest` +- Docker (AMD/Intel): `docker run --rm --device=/dev/dri --group-add video -e ENABLE_GPU=1 -p 9777:9777 rushabhtechie/yahoo-scraper:latest` + ## Line-by-line explanation of scraper_service.py -- Line 1: Import symbols from flask. Code: `from flask import Flask, jsonify, request` -- Line 2: Import symbols from playwright.sync_api. Code: `from playwright.sync_api import sync_playwright` -- Line 3: Import symbols from bs4. Code: `from bs4 import BeautifulSoup` -- Line 4: Import symbols from datetime. Code: `from datetime import datetime, timezone` -- Line 5: Import module urllib.parse. Code: `import urllib.parse` -- Line 6: Import module logging. Code: `import logging` -- Line 7: Import module json. Code: `import json` -- Line 8: Import module re. Code: `import re` -- Line 9: Import module time. Code: `import time` -- Line 10: Blank line for readability. Code: `` -- Line 11: Create the Flask application instance. Code: `app = Flask(__name__)` -- Line 12: Blank line for readability. Code: `` -- Line 13: Comment describing the next block. Code: `# Logging` -- Line 14: Configure logging defaults. Code: `logging.basicConfig(` -- Line 15: Execute the statement as written. Code: `level=logging.INFO,` -- Line 16: Execute the statement as written. Code: `format="%(asctime)s [%(levelname)s] %(message)s"` -- Line 17: Close the current block or container. Code: `)` -- Line 18: Set the Flask logger level. Code: `app.logger.setLevel(logging.INFO)` -- Line 19: Blank line for readability. Code: `` -- Line 20: Define accepted expiration date string formats. Code: `DATE_FORMATS = (` -- Line 21: Execute the statement as written. Code: `"%Y-%m-%d",` -- Line 22: Execute the statement as written. Code: `"%Y/%m/%d",` -- Line 23: Execute the statement as written. Code: `"%Y%m%d",` -- Line 24: Execute the statement as written. Code: `"%b %d, %Y",` -- Line 25: Execute the statement as written. Code: `"%B %d, %Y",` -- Line 26: Close the current block or container. Code: `)` -- Line 27: Blank line for readability. Code: `` +- Line 1: Import symbols or modules. Code: `from flask import Flask, jsonify, request` +- Line 2: Import symbols or modules. Code: `from playwright.sync_api import sync_playwright` +- Line 3: Import symbols or modules. Code: `from bs4 import BeautifulSoup` +- Line 4: Import symbols or modules. Code: `from datetime import datetime, timezone` +- Line 5: Import symbols or modules. Code: `import urllib.parse` +- Line 6: Import symbols or modules. Code: `import logging` +- Line 7: Import symbols or modules. Code: `import json` +- Line 8: Import symbols or modules. Code: `import re` +- Line 9: Import symbols or modules. Code: `import time` +- Line 10: Import symbols or modules. Code: `import os` +- Line 11: Blank line for readability. Code: `` +- Line 12: Execute the statement as written. Code: `app = Flask(__name__)` +- Line 13: Blank line for readability. Code: `` +- Line 14: Comment describing the next block. Code: `# Logging` +- Line 15: Execute the statement as written. Code: `logging.basicConfig(` +- Line 16: Execute the statement as written. Code: ` level=logging.INFO,` +- Line 17: Execute the statement as written. Code: ` format="%(asctime)s [%(levelname)s] %(message)s"` +- Line 18: Execute the statement as written. Code: `)` +- Line 19: Execute the statement as written. Code: `app.logger.setLevel(logging.INFO)` +- Line 20: Blank line for readability. Code: `` +- Line 21: Execute the statement as written. Code: `DATE_FORMATS = (` +- Line 22: Execute the statement as written. Code: ` "%Y-%m-%d",` +- Line 23: Execute the statement as written. Code: ` "%Y/%m/%d",` +- Line 24: Execute the statement as written. Code: ` "%Y%m%d",` +- Line 25: Execute the statement as written. Code: ` "%b %d, %Y",` +- Line 26: Execute the statement as written. Code: ` "%B %d, %Y",` +- Line 27: Execute the statement as written. Code: `)` - Line 28: Blank line for readability. Code: `` -- Line 29: Define the parse_date function. Code: `def parse_date(value):` -- Line 30: Loop over items. Code: `for fmt in DATE_FORMATS:` -- Line 31: Start a try block for error handling. Code: `try:` -- Line 32: Return a value to the caller. Code: `return datetime.strptime(value, fmt).date()` -- Line 33: Handle exceptions for the preceding try block. Code: `except ValueError:` -- Line 34: Execute the statement as written. Code: `continue` -- Line 35: Return a value to the caller. Code: `return None` +- Line 29: Execute the statement as written. Code: `GPU_ACCEL_ENV = "ENABLE_GPU"` +- Line 30: Blank line for readability. Code: `` +- Line 31: Blank line for readability. Code: `` +- Line 32: Define the parse_env_flag function. Code: `def parse_env_flag(value, default=False):` +- Line 33: Execute the statement as written. Code: ` if value is None:` +- Line 34: Execute the statement as written. Code: ` return default` +- Line 35: Execute the statement as written. Code: ` return str(value).strip().lower() in ("1", "true", "yes", "on")` - Line 36: Blank line for readability. Code: `` - Line 37: Blank line for readability. Code: `` -- Line 38: Define the normalize_label function. Code: `def normalize_label(value):` -- Line 39: Return a value to the caller. Code: `return " ".join(value.strip().split()).lower()` -- Line 40: Blank line for readability. Code: `` -- Line 41: Blank line for readability. Code: `` -- Line 42: Define the format_expiration_label function. Code: `def format_expiration_label(timestamp):` -- Line 43: Start a try block for error handling. Code: `try:` -- Line 44: Return a value to the caller. Code: `return datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d")` -- Line 45: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 46: Return a value to the caller. Code: `return str(timestamp)` -- Line 47: Blank line for readability. Code: `` -- Line 48: Blank line for readability. Code: `` -- Line 49: Define the format_percent function. Code: `def format_percent(value):` -- Line 50: Conditional branch. Code: `if value is None:` -- Line 51: Return a value to the caller. Code: `return None` -- Line 52: Start a try block for error handling. Code: `try:` -- Line 53: Return a value to the caller. Code: `return f"{value * 100:.2f}%"` -- Line 54: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 55: Return a value to the caller. Code: `return None` -- Line 56: Blank line for readability. Code: `` -- Line 57: Blank line for readability. Code: `` -- Line 58: Define the extract_raw_value function. Code: `def extract_raw_value(value):` -- Line 59: Conditional branch. Code: `if isinstance(value, dict):` -- Line 60: Return a value to the caller. Code: `return value.get("raw")` -- Line 61: Return a value to the caller. Code: `return value` +- Line 38: Define the detect_gpu_available function. Code: `def detect_gpu_available():` +- Line 39: Execute the statement as written. Code: ` env_value = os.getenv(GPU_ACCEL_ENV)` +- Line 40: Execute the statement as written. Code: ` if env_value is not None:` +- Line 41: Execute the statement as written. Code: ` return parse_env_flag(env_value, default=False)` +- Line 42: Blank line for readability. Code: `` +- Line 43: Execute the statement as written. Code: ` nvidia_visible = os.getenv("NVIDIA_VISIBLE_DEVICES")` +- Line 44: Execute the statement as written. Code: ` if nvidia_visible and nvidia_visible.lower() not in ("none", "void", "off"):` +- Line 45: Execute the statement as written. Code: ` return True` +- Line 46: Blank line for readability. Code: `` +- Line 47: Execute the statement as written. Code: ` if os.path.exists("/dev/nvidia0"):` +- Line 48: Execute the statement as written. Code: ` return True` +- Line 49: Blank line for readability. Code: `` +- Line 50: Execute the statement as written. Code: ` if os.path.exists("/dev/dri/renderD128") or os.path.exists("/dev/dri/card0"):` +- Line 51: Execute the statement as written. Code: ` return True` +- Line 52: Blank line for readability. Code: `` +- Line 53: Execute the statement as written. Code: ` return False` +- Line 54: Blank line for readability. Code: `` +- Line 55: Blank line for readability. Code: `` +- Line 56: Define the chromium_launch_args function. Code: `def chromium_launch_args():` +- Line 57: Execute the statement as written. Code: ` if not detect_gpu_available():` +- Line 58: Execute the statement as written. Code: ` return []` +- Line 59: Blank line for readability. Code: `` +- Line 60: Execute the statement as written. Code: ` if os.name == "nt":` +- Line 61: Execute the statement as written. Code: ` return ["--enable-gpu"]` - Line 62: Blank line for readability. Code: `` -- Line 63: Blank line for readability. Code: `` -- Line 64: Define the extract_fmt_value function. Code: `def extract_fmt_value(value):` -- Line 65: Conditional branch. Code: `if isinstance(value, dict):` -- Line 66: Return a value to the caller. Code: `return value.get("fmt")` -- Line 67: Return a value to the caller. Code: `return None` -- Line 68: Blank line for readability. Code: `` -- Line 69: Blank line for readability. Code: `` -- Line 70: Define the format_percent_value function. Code: `def format_percent_value(value):` -- Line 71: Execute the statement as written. Code: `fmt = extract_fmt_value(value)` -- Line 72: Conditional branch. Code: `if fmt is not None:` -- Line 73: Return a value to the caller. Code: `return fmt` -- Line 74: Return a value to the caller. Code: `return format_percent(extract_raw_value(value))` -- Line 75: Blank line for readability. Code: `` -- Line 76: Blank line for readability. Code: `` -- Line 77: Define the format_last_trade_date function. Code: `def format_last_trade_date(timestamp):` -- Line 78: Execute the statement as written. Code: `timestamp = extract_raw_value(timestamp)` -- Line 79: Conditional branch. Code: `if not timestamp:` -- Line 80: Return a value to the caller. Code: `return None` -- Line 81: Start a try block for error handling. Code: `try:` -- Line 82: Return a value to the caller. Code: `return datetime.fromtimestamp(timestamp).strftime("%m/%d/%Y %I:%M %p") + " EST"` -- Line 83: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 84: Return a value to the caller. Code: `return None` +- Line 63: Execute the statement as written. Code: ` return [` +- Line 64: Execute the statement as written. Code: ` "--enable-gpu",` +- Line 65: Execute the statement as written. Code: ` "--ignore-gpu-blocklist",` +- Line 66: Execute the statement as written. Code: ` "--disable-software-rasterizer",` +- Line 67: Execute the statement as written. Code: ` "--use-gl=egl",` +- Line 68: Execute the statement as written. Code: ` "--enable-zero-copy",` +- Line 69: Execute the statement as written. Code: ` "--enable-gpu-rasterization",` +- Line 70: Execute the statement as written. Code: ` ]` +- Line 71: Blank line for readability. Code: `` +- Line 72: Blank line for readability. Code: `` +- Line 73: Define the parse_date function. Code: `def parse_date(value):` +- Line 74: Execute the statement as written. Code: ` for fmt in DATE_FORMATS:` +- Line 75: Execute the statement as written. Code: ` try:` +- Line 76: Execute the statement as written. Code: ` return datetime.strptime(value, fmt).date()` +- Line 77: Execute the statement as written. Code: ` except ValueError:` +- Line 78: Execute the statement as written. Code: ` continue` +- Line 79: Execute the statement as written. Code: ` return None` +- Line 80: Blank line for readability. Code: `` +- Line 81: Blank line for readability. Code: `` +- Line 82: Define the normalize_label function. Code: `def normalize_label(value):` +- Line 83: Execute the statement as written. Code: ` return " ".join(value.strip().split()).lower()` +- Line 84: Blank line for readability. Code: `` - Line 85: Blank line for readability. Code: `` -- Line 86: Blank line for readability. Code: `` -- Line 87: Define the extract_option_chain_from_html function. Code: `def extract_option_chain_from_html(html):` -- Line 88: Conditional branch. Code: `if not html:` -- Line 89: Return a value to the caller. Code: `return None` -- Line 90: Blank line for readability. Code: `` -- Line 91: Execute the statement as written. Code: `token = "\"body\":\""` -- Line 92: Execute the statement as written. Code: `start = 0` -- Line 93: Execute the statement as written. Code: `while True:` -- Line 94: Execute the statement as written. Code: `idx = html.find(token, start)` -- Line 95: Conditional branch. Code: `if idx == -1:` -- Line 96: Execute the statement as written. Code: `break` -- Line 97: Execute the statement as written. Code: `i = idx + len(token)` -- Line 98: Execute the statement as written. Code: `escaped = False` -- Line 99: Execute the statement as written. Code: `raw_chars = []` -- Line 100: Execute the statement as written. Code: `while i < len(html):` -- Line 101: Execute the statement as written. Code: `ch = html[i]` -- Line 102: Conditional branch. Code: `if escaped:` -- Line 103: Execute the statement as written. Code: `raw_chars.append(ch)` -- Line 104: Execute the statement as written. Code: `escaped = False` -- Line 105: Fallback branch. Code: `else:` -- Line 106: Conditional branch. Code: `if ch == "\\":` -- Line 107: Execute the statement as written. Code: `raw_chars.append(ch)` -- Line 108: Execute the statement as written. Code: `escaped = True` -- Line 109: Alternative conditional branch. Code: `elif ch == "\"":` -- Line 110: Execute the statement as written. Code: `break` -- Line 111: Fallback branch. Code: `else:` -- Line 112: Execute the statement as written. Code: `raw_chars.append(ch)` -- Line 113: Execute the statement as written. Code: `i += 1` -- Line 114: Execute the statement as written. Code: `raw = "".join(raw_chars)` -- Line 115: Start a try block for error handling. Code: `try:` -- Line 116: Execute the statement as written. Code: `body_text = json.loads(f"\"{raw}\"")` -- Line 117: Handle exceptions for the preceding try block. Code: `except json.JSONDecodeError:` -- Line 118: Execute the statement as written. Code: `start = idx + len(token)` -- Line 119: Execute the statement as written. Code: `continue` -- Line 120: Conditional branch. Code: `if "optionChain" not in body_text:` -- Line 121: Execute the statement as written. Code: `start = idx + len(token)` -- Line 122: Execute the statement as written. Code: `continue` -- Line 123: Start a try block for error handling. Code: `try:` -- Line 124: Execute the statement as written. Code: `payload = json.loads(body_text)` -- Line 125: Handle exceptions for the preceding try block. Code: `except json.JSONDecodeError:` -- Line 126: Execute the statement as written. Code: `start = idx + len(token)` -- Line 127: Execute the statement as written. Code: `continue` -- Line 128: Execute the statement as written. Code: `option_chain = payload.get("optionChain")` -- Line 129: Conditional branch. Code: `if option_chain and option_chain.get("result"):` -- Line 130: Return a value to the caller. Code: `return option_chain` -- Line 131: Blank line for readability. Code: `` -- Line 132: Execute the statement as written. Code: `start = idx + len(token)` -- Line 133: Blank line for readability. Code: `` -- Line 134: Return a value to the caller. Code: `return None` -- Line 135: Blank line for readability. Code: `` -- Line 136: Blank line for readability. Code: `` -- Line 137: Define the extract_expiration_dates_from_chain function. Code: `def extract_expiration_dates_from_chain(chain):` -- Line 138: Conditional branch. Code: `if not chain:` -- Line 139: Return a value to the caller. Code: `return []` -- Line 140: Blank line for readability. Code: `` -- Line 141: Execute the statement as written. Code: `result = chain.get("result", [])` -- Line 142: Conditional branch. Code: `if not result:` -- Line 143: Return a value to the caller. Code: `return []` -- Line 144: Return a value to the caller. Code: `return result[0].get("expirationDates", []) or []` -- Line 145: Blank line for readability. Code: `` -- Line 146: Blank line for readability. Code: `` -- Line 147: Define the normalize_chain_rows function. Code: `def normalize_chain_rows(rows):` -- Line 148: Execute the statement as written. Code: `normalized = []` -- Line 149: Loop over items. Code: `for row in rows or []:` -- Line 150: Execute the statement as written. Code: `normalized.append(` -- Line 151: Execute the statement as written. Code: `{` -- Line 152: Execute the statement as written. Code: `"Contract Name": row.get("contractSymbol"),` -- Line 153: Execute the statement as written. Code: `"Last Trade Date (EST)": format_last_trade_date(` -- Line 154: Execute the statement as written. Code: `row.get("lastTradeDate")` -- Line 155: Close the current block or container. Code: `),` -- Line 156: Execute the statement as written. Code: `"Strike": extract_raw_value(row.get("strike")),` -- Line 157: Execute the statement as written. Code: `"Last Price": extract_raw_value(row.get("lastPrice")),` -- Line 158: Execute the statement as written. Code: `"Bid": extract_raw_value(row.get("bid")),` -- Line 159: Execute the statement as written. Code: `"Ask": extract_raw_value(row.get("ask")),` -- Line 160: Execute the statement as written. Code: `"Change": extract_raw_value(row.get("change")),` -- Line 161: Execute the statement as written. Code: `"% Change": format_percent_value(row.get("percentChange")),` -- Line 162: Execute the statement as written. Code: `"Volume": extract_raw_value(row.get("volume")),` -- Line 163: Execute the statement as written. Code: `"Open Interest": extract_raw_value(row.get("openInterest")),` -- Line 164: Execute the statement as written. Code: `"Implied Volatility": format_percent_value(` -- Line 165: Execute the statement as written. Code: `row.get("impliedVolatility")` -- Line 166: Close the current block or container. Code: `),` -- Line 167: Close the current block or container. Code: `}` -- Line 168: Close the current block or container. Code: `)` -- Line 169: Return a value to the caller. Code: `return normalized` -- Line 170: Blank line for readability. Code: `` -- Line 171: Blank line for readability. Code: `` -- Line 172: Define the build_rows_from_chain function. Code: `def build_rows_from_chain(chain):` -- Line 173: Execute the statement as written. Code: `result = chain.get("result", []) if chain else []` -- Line 174: Conditional branch. Code: `if not result:` -- Line 175: Return a value to the caller. Code: `return [], []` -- Line 176: Execute the statement as written. Code: `options = result[0].get("options", [])` -- Line 177: Conditional branch. Code: `if not options:` -- Line 178: Return a value to the caller. Code: `return [], []` -- Line 179: Execute the statement as written. Code: `option = options[0]` -- Line 180: Return a value to the caller. Code: `return (` -- Line 181: Execute the statement as written. Code: `normalize_chain_rows(option.get("calls")),` -- Line 182: Execute the statement as written. Code: `normalize_chain_rows(option.get("puts")),` -- Line 183: Close the current block or container. Code: `)` +- Line 86: Define the format_expiration_label function. Code: `def format_expiration_label(timestamp):` +- Line 87: Execute the statement as written. Code: ` try:` +- Line 88: Execute the statement as written. Code: ` return datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d")` +- Line 89: Execute the statement as written. Code: ` except Exception:` +- Line 90: Execute the statement as written. Code: ` return str(timestamp)` +- Line 91: Blank line for readability. Code: `` +- Line 92: Blank line for readability. Code: `` +- Line 93: Define the format_percent function. Code: `def format_percent(value):` +- Line 94: Execute the statement as written. Code: ` if value is None:` +- Line 95: Execute the statement as written. Code: ` return None` +- Line 96: Execute the statement as written. Code: ` try:` +- Line 97: Execute the statement as written. Code: ` return f"{value * 100:.2f}%"` +- Line 98: Execute the statement as written. Code: ` except Exception:` +- Line 99: Execute the statement as written. Code: ` return None` +- Line 100: Blank line for readability. Code: `` +- Line 101: Blank line for readability. Code: `` +- Line 102: Define the extract_raw_value function. Code: `def extract_raw_value(value):` +- Line 103: Execute the statement as written. Code: ` if isinstance(value, dict):` +- Line 104: Execute the statement as written. Code: ` return value.get("raw")` +- Line 105: Execute the statement as written. Code: ` return value` +- Line 106: Blank line for readability. Code: `` +- Line 107: Blank line for readability. Code: `` +- Line 108: Define the extract_fmt_value function. Code: `def extract_fmt_value(value):` +- Line 109: Execute the statement as written. Code: ` if isinstance(value, dict):` +- Line 110: Execute the statement as written. Code: ` return value.get("fmt")` +- Line 111: Execute the statement as written. Code: ` return None` +- Line 112: Blank line for readability. Code: `` +- Line 113: Blank line for readability. Code: `` +- Line 114: Define the format_percent_value function. Code: `def format_percent_value(value):` +- Line 115: Execute the statement as written. Code: ` fmt = extract_fmt_value(value)` +- Line 116: Execute the statement as written. Code: ` if fmt is not None:` +- Line 117: Execute the statement as written. Code: ` return fmt` +- Line 118: Execute the statement as written. Code: ` return format_percent(extract_raw_value(value))` +- Line 119: Blank line for readability. Code: `` +- Line 120: Blank line for readability. Code: `` +- Line 121: Define the format_last_trade_date function. Code: `def format_last_trade_date(timestamp):` +- Line 122: Execute the statement as written. Code: ` timestamp = extract_raw_value(timestamp)` +- Line 123: Execute the statement as written. Code: ` if not timestamp:` +- Line 124: Execute the statement as written. Code: ` return None` +- Line 125: Execute the statement as written. Code: ` try:` +- Line 126: Execute the statement as written. Code: ` return datetime.fromtimestamp(timestamp).strftime("%m/%d/%Y %I:%M %p") + " EST"` +- Line 127: Execute the statement as written. Code: ` except Exception:` +- Line 128: Execute the statement as written. Code: ` return None` +- Line 129: Blank line for readability. Code: `` +- Line 130: Blank line for readability. Code: `` +- Line 131: Define the extract_option_chain_from_html function. Code: `def extract_option_chain_from_html(html):` +- Line 132: Execute the statement as written. Code: ` if not html:` +- Line 133: Execute the statement as written. Code: ` return None` +- Line 134: Blank line for readability. Code: `` +- Line 135: Execute the statement as written. Code: ` token = "\"body\":\""` +- Line 136: Execute the statement as written. Code: ` start = 0` +- Line 137: Execute the statement as written. Code: ` while True:` +- Line 138: Execute the statement as written. Code: ` idx = html.find(token, start)` +- Line 139: Execute the statement as written. Code: ` if idx == -1:` +- Line 140: Execute the statement as written. Code: ` break` +- Line 141: Execute the statement as written. Code: ` i = idx + len(token)` +- Line 142: Execute the statement as written. Code: ` escaped = False` +- Line 143: Execute the statement as written. Code: ` raw_chars = []` +- Line 144: Execute the statement as written. Code: ` while i < len(html):` +- Line 145: Execute the statement as written. Code: ` ch = html[i]` +- Line 146: Execute the statement as written. Code: ` if escaped:` +- Line 147: Execute the statement as written. Code: ` raw_chars.append(ch)` +- Line 148: Execute the statement as written. Code: ` escaped = False` +- Line 149: Execute the statement as written. Code: ` else:` +- Line 150: Execute the statement as written. Code: ` if ch == "\\":` +- Line 151: Execute the statement as written. Code: ` raw_chars.append(ch)` +- Line 152: Execute the statement as written. Code: ` escaped = True` +- Line 153: Execute the statement as written. Code: ` elif ch == "\"":` +- Line 154: Execute the statement as written. Code: ` break` +- Line 155: Execute the statement as written. Code: ` else:` +- Line 156: Execute the statement as written. Code: ` raw_chars.append(ch)` +- Line 157: Execute the statement as written. Code: ` i += 1` +- Line 158: Execute the statement as written. Code: ` raw = "".join(raw_chars)` +- Line 159: Execute the statement as written. Code: ` try:` +- Line 160: Execute the statement as written. Code: ` body_text = json.loads(f"\"{raw}\"")` +- Line 161: Execute the statement as written. Code: ` except json.JSONDecodeError:` +- Line 162: Execute the statement as written. Code: ` start = idx + len(token)` +- Line 163: Execute the statement as written. Code: ` continue` +- Line 164: Execute the statement as written. Code: ` if "optionChain" not in body_text:` +- Line 165: Execute the statement as written. Code: ` start = idx + len(token)` +- Line 166: Execute the statement as written. Code: ` continue` +- Line 167: Execute the statement as written. Code: ` try:` +- Line 168: Execute the statement as written. Code: ` payload = json.loads(body_text)` +- Line 169: Execute the statement as written. Code: ` except json.JSONDecodeError:` +- Line 170: Execute the statement as written. Code: ` start = idx + len(token)` +- Line 171: Execute the statement as written. Code: ` continue` +- Line 172: Execute the statement as written. Code: ` option_chain = payload.get("optionChain")` +- Line 173: Execute the statement as written. Code: ` if option_chain and option_chain.get("result"):` +- Line 174: Execute the statement as written. Code: ` return option_chain` +- Line 175: Blank line for readability. Code: `` +- Line 176: Execute the statement as written. Code: ` start = idx + len(token)` +- Line 177: Blank line for readability. Code: `` +- Line 178: Execute the statement as written. Code: ` return None` +- Line 179: Blank line for readability. Code: `` +- Line 180: Blank line for readability. Code: `` +- Line 181: Define the extract_expiration_dates_from_chain function. Code: `def extract_expiration_dates_from_chain(chain):` +- Line 182: Execute the statement as written. Code: ` if not chain:` +- Line 183: Execute the statement as written. Code: ` return []` - Line 184: Blank line for readability. Code: `` -- Line 185: Blank line for readability. Code: `` -- Line 186: Define the extract_contract_expiry_code function. Code: `def extract_contract_expiry_code(contract_name):` -- Line 187: Conditional branch. Code: `if not contract_name:` -- Line 188: Return a value to the caller. Code: `return None` -- Line 189: Execute the statement as written. Code: `match = re.search(r"(\d{6})", contract_name)` -- Line 190: Return a value to the caller. Code: `return match.group(1) if match else None` -- Line 191: Blank line for readability. Code: `` -- Line 192: Blank line for readability. Code: `` -- Line 193: Define the expected_expiry_code function. Code: `def expected_expiry_code(timestamp):` -- Line 194: Conditional branch. Code: `if not timestamp:` -- Line 195: Return a value to the caller. Code: `return None` -- Line 196: Start a try block for error handling. Code: `try:` -- Line 197: Return a value to the caller. Code: `return datetime.utcfromtimestamp(timestamp).strftime("%y%m%d")` -- Line 198: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 199: Return a value to the caller. Code: `return None` -- Line 200: Blank line for readability. Code: `` -- Line 201: Blank line for readability. Code: `` -- Line 202: Define the extract_expiration_dates_from_html function. Code: `def extract_expiration_dates_from_html(html):` -- Line 203: Conditional branch. Code: `if not html:` -- Line 204: Return a value to the caller. Code: `return []` -- Line 205: Blank line for readability. Code: `` -- Line 206: Execute the statement as written. Code: `patterns = (` -- Line 207: Execute the statement as written. Code: `r'\\"expirationDates\\":\[(.*?)\]',` -- Line 208: Execute the statement as written. Code: `r'"expirationDates":\[(.*?)\]',` -- Line 209: Close the current block or container. Code: `)` -- Line 210: Execute the statement as written. Code: `match = None` -- Line 211: Loop over items. Code: `for pattern in patterns:` -- Line 212: Execute the statement as written. Code: `match = re.search(pattern, html, re.DOTALL)` -- Line 213: Conditional branch. Code: `if match:` -- Line 214: Execute the statement as written. Code: `break` -- Line 215: Conditional branch. Code: `if not match:` -- Line 216: Return a value to the caller. Code: `return []` -- Line 217: Blank line for readability. Code: `` -- Line 218: Execute the statement as written. Code: `raw = match.group(1)` -- Line 219: Execute the statement as written. Code: `values = []` -- Line 220: Loop over items. Code: `for part in raw.split(","):` -- Line 221: Execute the statement as written. Code: `part = part.strip()` -- Line 222: Conditional branch. Code: `if part.isdigit():` -- Line 223: Start a try block for error handling. Code: `try:` -- Line 224: Execute the statement as written. Code: `values.append(int(part))` -- Line 225: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 226: Execute the statement as written. Code: `continue` -- Line 227: Return a value to the caller. Code: `return values` +- Line 185: Execute the statement as written. Code: ` result = chain.get("result", [])` +- Line 186: Execute the statement as written. Code: ` if not result:` +- Line 187: Execute the statement as written. Code: ` return []` +- Line 188: Execute the statement as written. Code: ` return result[0].get("expirationDates", []) or []` +- Line 189: Blank line for readability. Code: `` +- Line 190: Blank line for readability. Code: `` +- Line 191: Define the normalize_chain_rows function. Code: `def normalize_chain_rows(rows):` +- Line 192: Execute the statement as written. Code: ` normalized = []` +- Line 193: Execute the statement as written. Code: ` for row in rows or []:` +- Line 194: Execute the statement as written. Code: ` normalized.append(` +- Line 195: Execute the statement as written. Code: ` {` +- Line 196: Execute the statement as written. Code: ` "Contract Name": row.get("contractSymbol"),` +- Line 197: Execute the statement as written. Code: ` "Last Trade Date (EST)": format_last_trade_date(` +- Line 198: Execute the statement as written. Code: ` row.get("lastTradeDate")` +- Line 199: Execute the statement as written. Code: ` ),` +- Line 200: Execute the statement as written. Code: ` "Strike": extract_raw_value(row.get("strike")),` +- Line 201: Execute the statement as written. Code: ` "Last Price": extract_raw_value(row.get("lastPrice")),` +- Line 202: Execute the statement as written. Code: ` "Bid": extract_raw_value(row.get("bid")),` +- Line 203: Execute the statement as written. Code: ` "Ask": extract_raw_value(row.get("ask")),` +- Line 204: Execute the statement as written. Code: ` "Change": extract_raw_value(row.get("change")),` +- Line 205: Execute the statement as written. Code: ` "% Change": format_percent_value(row.get("percentChange")),` +- Line 206: Execute the statement as written. Code: ` "Volume": extract_raw_value(row.get("volume")),` +- Line 207: Execute the statement as written. Code: ` "Open Interest": extract_raw_value(row.get("openInterest")),` +- Line 208: Execute the statement as written. Code: ` "Implied Volatility": format_percent_value(` +- Line 209: Execute the statement as written. Code: ` row.get("impliedVolatility")` +- Line 210: Execute the statement as written. Code: ` ),` +- Line 211: Execute the statement as written. Code: ` }` +- Line 212: Execute the statement as written. Code: ` )` +- Line 213: Execute the statement as written. Code: ` return normalized` +- Line 214: Blank line for readability. Code: `` +- Line 215: Blank line for readability. Code: `` +- Line 216: Define the build_rows_from_chain function. Code: `def build_rows_from_chain(chain):` +- Line 217: Execute the statement as written. Code: ` result = chain.get("result", []) if chain else []` +- Line 218: Execute the statement as written. Code: ` if not result:` +- Line 219: Execute the statement as written. Code: ` return [], []` +- Line 220: Execute the statement as written. Code: ` options = result[0].get("options", [])` +- Line 221: Execute the statement as written. Code: ` if not options:` +- Line 222: Execute the statement as written. Code: ` return [], []` +- Line 223: Execute the statement as written. Code: ` option = options[0]` +- Line 224: Execute the statement as written. Code: ` return (` +- Line 225: Execute the statement as written. Code: ` normalize_chain_rows(option.get("calls")),` +- Line 226: Execute the statement as written. Code: ` normalize_chain_rows(option.get("puts")),` +- Line 227: Execute the statement as written. Code: ` )` - Line 228: Blank line for readability. Code: `` - Line 229: Blank line for readability. Code: `` -- Line 230: Define the build_expiration_options function. Code: `def build_expiration_options(expiration_dates):` -- Line 231: Execute the statement as written. Code: `options = []` -- Line 232: Loop over items. Code: `for value in expiration_dates or []:` -- Line 233: Start a try block for error handling. Code: `try:` -- Line 234: Execute the statement as written. Code: `value_int = int(value)` -- Line 235: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 236: Execute the statement as written. Code: `continue` -- Line 237: Blank line for readability. Code: `` -- Line 238: Execute the statement as written. Code: `label = format_expiration_label(value_int)` -- Line 239: Start a try block for error handling. Code: `try:` -- Line 240: Execute the statement as written. Code: `date_value = datetime.utcfromtimestamp(value_int).date()` -- Line 241: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 242: Execute the statement as written. Code: `date_value = None` -- Line 243: Blank line for readability. Code: `` -- Line 244: Execute the statement as written. Code: `options.append({"value": value_int, "label": label, "date": date_value})` -- Line 245: Return a value to the caller. Code: `return sorted(options, key=lambda x: x["value"])` -- Line 246: Blank line for readability. Code: `` -- Line 247: Blank line for readability. Code: `` -- Line 248: Define the resolve_expiration function. Code: `def resolve_expiration(expiration, options):` -- Line 249: Conditional branch. Code: `if not expiration:` -- Line 250: Return a value to the caller. Code: `return None, None` -- Line 251: Blank line for readability. Code: `` -- Line 252: Execute the statement as written. Code: `raw = expiration.strip()` -- Line 253: Conditional branch. Code: `if not raw:` -- Line 254: Return a value to the caller. Code: `return None, None` -- Line 255: Blank line for readability. Code: `` -- Line 256: Conditional branch. Code: `if raw.isdigit():` -- Line 257: Execute the statement as written. Code: `value = int(raw)` -- Line 258: Conditional branch. Code: `if options:` -- Line 259: Loop over items. Code: `for opt in options:` -- Line 260: Conditional branch. Code: `if opt.get("value") == value:` -- Line 261: Return a value to the caller. Code: `return value, opt.get("label")` -- Line 262: Return a value to the caller. Code: `return None, None` -- Line 263: Return a value to the caller. Code: `return value, format_expiration_label(value)` -- Line 264: Blank line for readability. Code: `` -- Line 265: Execute the statement as written. Code: `requested_date = parse_date(raw)` -- Line 266: Conditional branch. Code: `if requested_date:` -- Line 267: Loop over items. Code: `for opt in options:` -- Line 268: Conditional branch. Code: `if opt.get("date") == requested_date:` -- Line 269: Return a value to the caller. Code: `return opt.get("value"), opt.get("label")` -- Line 270: Return a value to the caller. Code: `return None, None` -- Line 271: Blank line for readability. Code: `` -- Line 272: Execute the statement as written. Code: `normalized = normalize_label(raw)` -- Line 273: Loop over items. Code: `for opt in options:` -- Line 274: Conditional branch. Code: `if normalize_label(opt.get("label", "")) == normalized:` -- Line 275: Return a value to the caller. Code: `return opt.get("value"), opt.get("label")` -- Line 276: Blank line for readability. Code: `` -- Line 277: Return a value to the caller. Code: `return None, None` -- Line 278: Blank line for readability. Code: `` -- Line 279: Blank line for readability. Code: `` -- Line 280: Define the wait_for_tables function. Code: `def wait_for_tables(page):` -- Line 281: Start a try block for error handling. Code: `try:` -- Line 282: Interact with the Playwright page. Code: `page.wait_for_selector(` -- Line 283: Execute the statement as written. Code: `"section[data-testid='options-list-table'] table",` -- Line 284: Execute the statement as written. Code: `timeout=30000,` -- Line 285: Close the current block or container. Code: `)` -- Line 286: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 287: Interact with the Playwright page. Code: `page.wait_for_selector("table", timeout=30000)` -- Line 288: Blank line for readability. Code: `` -- Line 289: Loop over items. Code: `for _ in range(30): # 30 * 1s = 30 seconds` -- Line 290: Collect option tables from the page. Code: `tables = page.query_selector_all(` -- Line 291: Execute the statement as written. Code: `"section[data-testid='options-list-table'] table"` -- Line 292: Close the current block or container. Code: `)` -- Line 293: Conditional branch. Code: `if len(tables) >= 2:` -- Line 294: Return a value to the caller. Code: `return tables` -- Line 295: Collect option tables from the page. Code: `tables = page.query_selector_all("table")` -- Line 296: Conditional branch. Code: `if len(tables) >= 2:` -- Line 297: Return a value to the caller. Code: `return tables` -- Line 298: Execute the statement as written. Code: `time.sleep(1)` -- Line 299: Return a value to the caller. Code: `return []` -- Line 300: Blank line for readability. Code: `` -- Line 301: Blank line for readability. Code: `` -- Line 302: Define the parse_strike_limit function. Code: `def parse_strike_limit(value, default=25):` -- Line 303: Conditional branch. Code: `if value is None:` -- Line 304: Return a value to the caller. Code: `return default` -- Line 305: Start a try block for error handling. Code: `try:` -- Line 306: Execute the statement as written. Code: `limit = int(value)` -- Line 307: Handle exceptions for the preceding try block. Code: `except (TypeError, ValueError):` -- Line 308: Return a value to the caller. Code: `return default` -- Line 309: Return a value to the caller. Code: `return limit if limit > 0 else default` -- Line 310: Blank line for readability. Code: `` -- Line 311: Blank line for readability. Code: `` -- Line 312: Define the scrape_yahoo_options function. Code: `def scrape_yahoo_options(symbol, expiration=None, strike_limit=25):` -- Line 313: Define the parse_table function. Code: `def parse_table(table_html, side):` -- Line 314: Conditional branch. Code: `if not table_html:` -- Line 315: Emit or configure a log message. Code: `app.logger.warning("No %s table HTML for %s", side, symbol)` -- Line 316: Return a value to the caller. Code: `return []` -- Line 317: Blank line for readability. Code: `` -- Line 318: Execute the statement as written. Code: `soup = BeautifulSoup(table_html, "html.parser")` -- Line 319: Blank line for readability. Code: `` -- Line 320: Extract header labels from the table. Code: `headers = [th.get_text(strip=True) for th in soup.select("thead th")]` -- Line 321: Collect table rows for parsing. Code: `rows = soup.select("tbody tr")` +- Line 230: Define the extract_contract_expiry_code function. Code: `def extract_contract_expiry_code(contract_name):` +- Line 231: Execute the statement as written. Code: ` if not contract_name:` +- Line 232: Execute the statement as written. Code: ` return None` +- Line 233: Execute the statement as written. Code: ` match = re.search(r"(\d{6})", contract_name)` +- Line 234: Execute the statement as written. Code: ` return match.group(1) if match else None` +- Line 235: Blank line for readability. Code: `` +- Line 236: Blank line for readability. Code: `` +- Line 237: Define the expected_expiry_code function. Code: `def expected_expiry_code(timestamp):` +- Line 238: Execute the statement as written. Code: ` if not timestamp:` +- Line 239: Execute the statement as written. Code: ` return None` +- Line 240: Execute the statement as written. Code: ` try:` +- Line 241: Execute the statement as written. Code: ` return datetime.utcfromtimestamp(timestamp).strftime("%y%m%d")` +- Line 242: Execute the statement as written. Code: ` except Exception:` +- Line 243: Execute the statement as written. Code: ` return None` +- Line 244: Blank line for readability. Code: `` +- Line 245: Blank line for readability. Code: `` +- Line 246: Define the extract_expiration_dates_from_html function. Code: `def extract_expiration_dates_from_html(html):` +- Line 247: Execute the statement as written. Code: ` if not html:` +- Line 248: Execute the statement as written. Code: ` return []` +- Line 249: Blank line for readability. Code: `` +- Line 250: Execute the statement as written. Code: ` patterns = (` +- Line 251: Execute the statement as written. Code: ` r'\\"expirationDates\\":\[(.*?)\]',` +- Line 252: Execute the statement as written. Code: ` r'"expirationDates":\[(.*?)\]',` +- Line 253: Execute the statement as written. Code: ` )` +- Line 254: Execute the statement as written. Code: ` match = None` +- Line 255: Execute the statement as written. Code: ` for pattern in patterns:` +- Line 256: Execute the statement as written. Code: ` match = re.search(pattern, html, re.DOTALL)` +- Line 257: Execute the statement as written. Code: ` if match:` +- Line 258: Execute the statement as written. Code: ` break` +- Line 259: Execute the statement as written. Code: ` if not match:` +- Line 260: Execute the statement as written. Code: ` return []` +- Line 261: Blank line for readability. Code: `` +- Line 262: Execute the statement as written. Code: ` raw = match.group(1)` +- Line 263: Execute the statement as written. Code: ` values = []` +- Line 264: Execute the statement as written. Code: ` for part in raw.split(","):` +- Line 265: Execute the statement as written. Code: ` part = part.strip()` +- Line 266: Execute the statement as written. Code: ` if part.isdigit():` +- Line 267: Execute the statement as written. Code: ` try:` +- Line 268: Execute the statement as written. Code: ` values.append(int(part))` +- Line 269: Execute the statement as written. Code: ` except Exception:` +- Line 270: Execute the statement as written. Code: ` continue` +- Line 271: Execute the statement as written. Code: ` return values` +- Line 272: Blank line for readability. Code: `` +- Line 273: Blank line for readability. Code: `` +- Line 274: Define the build_expiration_options function. Code: `def build_expiration_options(expiration_dates):` +- Line 275: Execute the statement as written. Code: ` options = []` +- Line 276: Execute the statement as written. Code: ` for value in expiration_dates or []:` +- Line 277: Execute the statement as written. Code: ` try:` +- Line 278: Execute the statement as written. Code: ` value_int = int(value)` +- Line 279: Execute the statement as written. Code: ` except Exception:` +- Line 280: Execute the statement as written. Code: ` continue` +- Line 281: Blank line for readability. Code: `` +- Line 282: Execute the statement as written. Code: ` label = format_expiration_label(value_int)` +- Line 283: Execute the statement as written. Code: ` try:` +- Line 284: Execute the statement as written. Code: ` date_value = datetime.utcfromtimestamp(value_int).date()` +- Line 285: Execute the statement as written. Code: ` except Exception:` +- Line 286: Execute the statement as written. Code: ` date_value = None` +- Line 287: Blank line for readability. Code: `` +- Line 288: Execute the statement as written. Code: ` options.append({"value": value_int, "label": label, "date": date_value})` +- Line 289: Execute the statement as written. Code: ` return sorted(options, key=lambda x: x["value"])` +- Line 290: Blank line for readability. Code: `` +- Line 291: Blank line for readability. Code: `` +- Line 292: Define the resolve_expiration function. Code: `def resolve_expiration(expiration, options):` +- Line 293: Execute the statement as written. Code: ` if not expiration:` +- Line 294: Execute the statement as written. Code: ` return None, None` +- Line 295: Blank line for readability. Code: `` +- Line 296: Execute the statement as written. Code: ` raw = expiration.strip()` +- Line 297: Execute the statement as written. Code: ` if not raw:` +- Line 298: Execute the statement as written. Code: ` return None, None` +- Line 299: Blank line for readability. Code: `` +- Line 300: Execute the statement as written. Code: ` if raw.isdigit():` +- Line 301: Execute the statement as written. Code: ` value = int(raw)` +- Line 302: Execute the statement as written. Code: ` if options:` +- Line 303: Execute the statement as written. Code: ` for opt in options:` +- Line 304: Execute the statement as written. Code: ` if opt.get("value") == value:` +- Line 305: Execute the statement as written. Code: ` return value, opt.get("label")` +- Line 306: Execute the statement as written. Code: ` return None, None` +- Line 307: Execute the statement as written. Code: ` return value, format_expiration_label(value)` +- Line 308: Blank line for readability. Code: `` +- Line 309: Execute the statement as written. Code: ` requested_date = parse_date(raw)` +- Line 310: Execute the statement as written. Code: ` if requested_date:` +- Line 311: Execute the statement as written. Code: ` for opt in options:` +- Line 312: Execute the statement as written. Code: ` if opt.get("date") == requested_date:` +- Line 313: Execute the statement as written. Code: ` return opt.get("value"), opt.get("label")` +- Line 314: Execute the statement as written. Code: ` return None, None` +- Line 315: Blank line for readability. Code: `` +- Line 316: Execute the statement as written. Code: ` normalized = normalize_label(raw)` +- Line 317: Execute the statement as written. Code: ` for opt in options:` +- Line 318: Execute the statement as written. Code: ` if normalize_label(opt.get("label", "")) == normalized:` +- Line 319: Execute the statement as written. Code: ` return opt.get("value"), opt.get("label")` +- Line 320: Blank line for readability. Code: `` +- Line 321: Execute the statement as written. Code: ` return None, None` - Line 322: Blank line for readability. Code: `` -- Line 323: Initialize the parsed rows list. Code: `parsed = []` -- Line 324: Loop over items. Code: `for r in rows:` -- Line 325: Collect table cells for the current row. Code: `tds = r.find_all("td")` -- Line 326: Conditional branch. Code: `if len(tds) != len(headers):` -- Line 327: Execute the statement as written. Code: `continue` -- Line 328: Blank line for readability. Code: `` -- Line 329: Initialize a row dictionary. Code: `item = {}` -- Line 330: Loop over items. Code: `for i, c in enumerate(tds):` -- Line 331: Read the header name for the current column. Code: `key = headers[i]` -- Line 332: Read or convert the cell value. Code: `val = c.get_text(" ", strip=True)` -- Line 333: Blank line for readability. Code: `` -- Line 334: Comment describing the next block. Code: `# Convert numeric fields` -- Line 335: Conditional branch. Code: `if key in ["Strike", "Last Price", "Bid", "Ask", "Change"]:` -- Line 336: Start a try block for error handling. Code: `try:` -- Line 337: Read or convert the cell value. Code: `val = float(val.replace(",", ""))` -- Line 338: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 339: Read or convert the cell value. Code: `val = None` -- Line 340: Alternative conditional branch. Code: `elif key in ["Volume", "Open Interest"]:` -- Line 341: Start a try block for error handling. Code: `try:` -- Line 342: Read or convert the cell value. Code: `val = int(val.replace(",", ""))` -- Line 343: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 344: Read or convert the cell value. Code: `val = None` -- Line 345: Alternative conditional branch. Code: `elif val in ["-", ""]:` -- Line 346: Read or convert the cell value. Code: `val = None` -- Line 347: Blank line for readability. Code: `` -- Line 348: Execute the statement as written. Code: `item[key] = val` -- Line 349: Blank line for readability. Code: `` -- Line 350: Execute the statement as written. Code: `parsed.append(item)` -- Line 351: Blank line for readability. Code: `` -- Line 352: Emit or configure a log message. Code: `app.logger.info("Parsed %d %s rows", len(parsed), side)` -- Line 353: Return a value to the caller. Code: `return parsed` +- Line 323: Blank line for readability. Code: `` +- Line 324: Define the wait_for_tables function. Code: `def wait_for_tables(page):` +- Line 325: Execute the statement as written. Code: ` try:` +- Line 326: Execute the statement as written. Code: ` page.wait_for_selector(` +- Line 327: Execute the statement as written. Code: ` "section[data-testid='options-list-table'] table",` +- Line 328: Execute the statement as written. Code: ` timeout=30000,` +- Line 329: Execute the statement as written. Code: ` )` +- Line 330: Execute the statement as written. Code: ` except Exception:` +- Line 331: Execute the statement as written. Code: ` page.wait_for_selector("table", timeout=30000)` +- Line 332: Blank line for readability. Code: `` +- Line 333: Execute the statement as written. Code: ` for _ in range(30): # 30 * 1s = 30 seconds` +- Line 334: Execute the statement as written. Code: ` tables = page.query_selector_all(` +- Line 335: Execute the statement as written. Code: ` "section[data-testid='options-list-table'] table"` +- Line 336: Execute the statement as written. Code: ` )` +- Line 337: Execute the statement as written. Code: ` if len(tables) >= 2:` +- Line 338: Execute the statement as written. Code: ` return tables` +- Line 339: Execute the statement as written. Code: ` tables = page.query_selector_all("table")` +- Line 340: Execute the statement as written. Code: ` if len(tables) >= 2:` +- Line 341: Execute the statement as written. Code: ` return tables` +- Line 342: Execute the statement as written. Code: ` time.sleep(1)` +- Line 343: Execute the statement as written. Code: ` return []` +- Line 344: Blank line for readability. Code: `` +- Line 345: Blank line for readability. Code: `` +- Line 346: Define the parse_strike_limit function. Code: `def parse_strike_limit(value, default=25):` +- Line 347: Execute the statement as written. Code: ` if value is None:` +- Line 348: Execute the statement as written. Code: ` return default` +- Line 349: Execute the statement as written. Code: ` try:` +- Line 350: Execute the statement as written. Code: ` limit = int(value)` +- Line 351: Execute the statement as written. Code: ` except (TypeError, ValueError):` +- Line 352: Execute the statement as written. Code: ` return default` +- Line 353: Execute the statement as written. Code: ` return limit if limit > 0 else default` - Line 354: Blank line for readability. Code: `` -- Line 355: Define the read_option_chain function. Code: `def read_option_chain(page):` -- Line 356: Capture the page HTML content. Code: `html = page.content()` -- Line 357: Execute the statement as written. Code: `option_chain = extract_option_chain_from_html(html)` -- Line 358: Conditional branch. Code: `if option_chain:` -- Line 359: Extract expiration date timestamps from the HTML. Code: `expiration_dates = extract_expiration_dates_from_chain(option_chain)` -- Line 360: Fallback branch. Code: `else:` -- Line 361: Extract expiration date timestamps from the HTML. Code: `expiration_dates = extract_expiration_dates_from_html(html)` -- Line 362: Return a value to the caller. Code: `return option_chain, expiration_dates` +- Line 355: Blank line for readability. Code: `` +- Line 356: Define the scrape_yahoo_options function. Code: `def scrape_yahoo_options(symbol, expiration=None, strike_limit=25):` +- Line 357: Define the parse_table function. Code: ` def parse_table(table_html, side):` +- Line 358: Execute the statement as written. Code: ` if not table_html:` +- Line 359: Execute the statement as written. Code: ` app.logger.warning("No %s table HTML for %s", side, symbol)` +- Line 360: Execute the statement as written. Code: ` return []` +- Line 361: Blank line for readability. Code: `` +- Line 362: Execute the statement as written. Code: ` soup = BeautifulSoup(table_html, "html.parser")` - Line 363: Blank line for readability. Code: `` -- Line 364: Define the has_expected_expiry function. Code: `def has_expected_expiry(options, expected_code):` -- Line 365: Conditional branch. Code: `if not expected_code:` -- Line 366: Return a value to the caller. Code: `return False` -- Line 367: Loop over items. Code: `for row in options or []:` -- Line 368: Execute the statement as written. Code: `name = row.get("Contract Name")` -- Line 369: Conditional branch. Code: `if extract_contract_expiry_code(name) == expected_code:` -- Line 370: Return a value to the caller. Code: `return True` -- Line 371: Return a value to the caller. Code: `return False` +- Line 364: Execute the statement as written. Code: ` headers = [th.get_text(strip=True) for th in soup.select("thead th")]` +- Line 365: Execute the statement as written. Code: ` rows = soup.select("tbody tr")` +- Line 366: Blank line for readability. Code: `` +- Line 367: Execute the statement as written. Code: ` parsed = []` +- Line 368: Execute the statement as written. Code: ` for r in rows:` +- Line 369: Execute the statement as written. Code: ` tds = r.find_all("td")` +- Line 370: Execute the statement as written. Code: ` if len(tds) != len(headers):` +- Line 371: Execute the statement as written. Code: ` continue` - Line 372: Blank line for readability. Code: `` -- Line 373: URL-encode the stock symbol. Code: `encoded = urllib.parse.quote(symbol, safe="")` -- Line 374: Build the base Yahoo Finance options URL. Code: `base_url = f"https://finance.yahoo.com/quote/{encoded}/options/"` -- Line 375: Normalize the expiration input string. Code: `requested_expiration = expiration.strip() if expiration else None` -- Line 376: Conditional branch. Code: `if not requested_expiration:` -- Line 377: Normalize the expiration input string. Code: `requested_expiration = None` -- Line 378: Set the URL to load. Code: `url = base_url` -- Line 379: Blank line for readability. Code: `` -- Line 380: Emit or configure a log message. Code: `app.logger.info(` -- Line 381: Execute the statement as written. Code: `"Starting scrape for symbol=%s expiration=%s url=%s",` -- Line 382: Execute the statement as written. Code: `symbol,` -- Line 383: Execute the statement as written. Code: `requested_expiration,` -- Line 384: Execute the statement as written. Code: `base_url,` -- Line 385: Close the current block or container. Code: `)` -- Line 386: Blank line for readability. Code: `` -- Line 387: Reserve storage for options table HTML. Code: `calls_html = None` -- Line 388: Reserve storage for options table HTML. Code: `puts_html = None` -- Line 389: Parse the full calls and puts tables. Code: `calls_full = []` -- Line 390: Parse the full calls and puts tables. Code: `puts_full = []` -- Line 391: Initialize or assign the current price. Code: `price = None` -- Line 392: Track the resolved expiration metadata. Code: `selected_expiration_value = None` -- Line 393: Track the resolved expiration metadata. Code: `selected_expiration_label = None` -- Line 394: Prepare or update the list of available expirations. Code: `expiration_options = []` -- Line 395: Track the resolved expiration epoch timestamp. Code: `target_date = None` -- Line 396: Track whether a base-page lookup is needed. Code: `fallback_to_base = False` -- Line 397: Blank line for readability. Code: `` -- Line 398: Enter a context manager block. Code: `with sync_playwright() as p:` -- Line 399: Launch a Playwright browser instance. Code: `browser = p.chromium.launch(headless=True)` -- Line 400: Create a new Playwright page. Code: `page = browser.new_page()` -- Line 401: Interact with the Playwright page. Code: `page.set_extra_http_headers(` -- Line 402: Execute the statement as written. Code: `{` -- Line 403: Execute the statement as written. Code: `"User-Agent": (` -- Line 404: Execute the statement as written. Code: `"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "` -- Line 405: Execute the statement as written. Code: `"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36"` -- Line 406: Close the current block or container. Code: `)` -- Line 407: Close the current block or container. Code: `}` -- Line 408: Close the current block or container. Code: `)` -- Line 409: Interact with the Playwright page. Code: `page.set_default_timeout(60000)` -- Line 410: Blank line for readability. Code: `` -- Line 411: Start a try block for error handling. Code: `try:` -- Line 412: Conditional branch. Code: `if requested_expiration:` -- Line 413: Conditional branch. Code: `if requested_expiration.isdigit():` -- Line 414: Track the resolved expiration epoch timestamp. Code: `target_date = int(requested_expiration)` -- Line 415: Track the resolved expiration metadata. Code: `selected_expiration_value = target_date` -- Line 416: Track the resolved expiration metadata. Code: `selected_expiration_label = format_expiration_label(target_date)` -- Line 417: Fallback branch. Code: `else:` -- Line 418: Execute the statement as written. Code: `parsed_date = parse_date(requested_expiration)` -- Line 419: Conditional branch. Code: `if parsed_date:` -- Line 420: Track the resolved expiration epoch timestamp. Code: `target_date = int(` -- Line 421: Execute the statement as written. Code: `datetime(` -- Line 422: Execute the statement as written. Code: `parsed_date.year,` -- Line 423: Execute the statement as written. Code: `parsed_date.month,` -- Line 424: Execute the statement as written. Code: `parsed_date.day,` -- Line 425: Execute the statement as written. Code: `tzinfo=timezone.utc,` -- Line 426: Execute the statement as written. Code: `).timestamp()` -- Line 427: Close the current block or container. Code: `)` -- Line 428: Track the resolved expiration metadata. Code: `selected_expiration_value = target_date` -- Line 429: Track the resolved expiration metadata. Code: `selected_expiration_label = format_expiration_label(target_date)` -- Line 430: Fallback branch. Code: `else:` -- Line 431: Track whether a base-page lookup is needed. Code: `fallback_to_base = True` -- Line 432: Blank line for readability. Code: `` -- Line 433: Conditional branch. Code: `if target_date:` -- Line 434: Set the URL to load. Code: `url = f"{base_url}?date={target_date}"` -- Line 435: Blank line for readability. Code: `` -- Line 436: Navigate the Playwright page to the target URL. Code: `page.goto(url, wait_until="domcontentloaded", timeout=60000)` -- Line 437: Emit or configure a log message. Code: `app.logger.info("Page loaded (domcontentloaded) for %s", symbol)` -- Line 438: Blank line for readability. Code: `` -- Line 439: Execute the statement as written. Code: `option_chain, expiration_dates = read_option_chain(page)` -- Line 440: Emit or configure a log message. Code: `app.logger.info("Option chain found: %s", bool(option_chain))` -- Line 441: Prepare or update the list of available expirations. Code: `expiration_options = build_expiration_options(expiration_dates)` -- Line 442: Blank line for readability. Code: `` -- Line 443: Conditional branch. Code: `if fallback_to_base:` -- Line 444: Execute the statement as written. Code: `resolved_value, resolved_label = resolve_expiration(` -- Line 445: Execute the statement as written. Code: `requested_expiration, expiration_options` -- Line 446: Close the current block or container. Code: `)` -- Line 447: Conditional branch. Code: `if resolved_value is None:` -- Line 448: Return a value to the caller. Code: `return {` -- Line 449: Execute the statement as written. Code: `"error": "Requested expiration not available",` -- Line 450: Execute the statement as written. Code: `"stock": symbol,` -- Line 451: Execute the statement as written. Code: `"requested_expiration": requested_expiration,` -- Line 452: Execute the statement as written. Code: `"available_expirations": [` -- Line 453: Execute the statement as written. Code: `{"label": opt.get("label"), "value": opt.get("value")}` -- Line 454: Loop over items. Code: `for opt in expiration_options` -- Line 455: Close the current block or container. Code: `],` -- Line 456: Close the current block or container. Code: `}` -- Line 457: Blank line for readability. Code: `` -- Line 458: Track the resolved expiration epoch timestamp. Code: `target_date = resolved_value` -- Line 459: Track the resolved expiration metadata. Code: `selected_expiration_value = resolved_value` -- Line 460: Track the resolved expiration metadata. Code: `selected_expiration_label = resolved_label or format_expiration_label(` -- Line 461: Execute the statement as written. Code: `resolved_value` -- Line 462: Close the current block or container. Code: `)` -- Line 463: Set the URL to load. Code: `url = f"{base_url}?date={resolved_value}"` -- Line 464: Navigate the Playwright page to the target URL. Code: `page.goto(url, wait_until="domcontentloaded", timeout=60000)` -- Line 465: Emit or configure a log message. Code: `app.logger.info("Page loaded (domcontentloaded) for %s", symbol)` -- Line 466: Blank line for readability. Code: `` -- Line 467: Execute the statement as written. Code: `option_chain, expiration_dates = read_option_chain(page)` -- Line 468: Prepare or update the list of available expirations. Code: `expiration_options = build_expiration_options(expiration_dates)` -- Line 469: Blank line for readability. Code: `` -- Line 470: Conditional branch. Code: `if target_date and expiration_options:` -- Line 471: Execute the statement as written. Code: `matched = None` -- Line 472: Loop over items. Code: `for opt in expiration_options:` -- Line 473: Conditional branch. Code: `if opt.get("value") == target_date:` -- Line 474: Execute the statement as written. Code: `matched = opt` -- Line 475: Execute the statement as written. Code: `break` -- Line 476: Conditional branch. Code: `if not matched:` -- Line 477: Return a value to the caller. Code: `return {` -- Line 478: Execute the statement as written. Code: `"error": "Requested expiration not available",` -- Line 479: Execute the statement as written. Code: `"stock": symbol,` -- Line 480: Execute the statement as written. Code: `"requested_expiration": requested_expiration,` -- Line 481: Execute the statement as written. Code: `"available_expirations": [` -- Line 482: Execute the statement as written. Code: `{"label": opt.get("label"), "value": opt.get("value")}` -- Line 483: Loop over items. Code: `for opt in expiration_options` -- Line 484: Close the current block or container. Code: `],` -- Line 485: Close the current block or container. Code: `}` -- Line 486: Track the resolved expiration metadata. Code: `selected_expiration_value = matched.get("value")` -- Line 487: Track the resolved expiration metadata. Code: `selected_expiration_label = matched.get("label")` -- Line 488: Alternative conditional branch. Code: `elif expiration_options and not target_date:` -- Line 489: Track the resolved expiration metadata. Code: `selected_expiration_value = expiration_options[0].get("value")` -- Line 490: Track the resolved expiration metadata. Code: `selected_expiration_label = expiration_options[0].get("label")` +- Line 373: Execute the statement as written. Code: ` item = {}` +- Line 374: Execute the statement as written. Code: ` for i, c in enumerate(tds):` +- Line 375: Execute the statement as written. Code: ` key = headers[i]` +- Line 376: Execute the statement as written. Code: ` val = c.get_text(" ", strip=True)` +- Line 377: Blank line for readability. Code: `` +- Line 378: Comment describing the next block. Code: ` # Convert numeric fields` +- Line 379: Execute the statement as written. Code: ` if key in ["Strike", "Last Price", "Bid", "Ask", "Change"]:` +- Line 380: Execute the statement as written. Code: ` try:` +- Line 381: Execute the statement as written. Code: ` val = float(val.replace(",", ""))` +- Line 382: Execute the statement as written. Code: ` except Exception:` +- Line 383: Execute the statement as written. Code: ` val = None` +- Line 384: Execute the statement as written. Code: ` elif key in ["Volume", "Open Interest"]:` +- Line 385: Execute the statement as written. Code: ` try:` +- Line 386: Execute the statement as written. Code: ` val = int(val.replace(",", ""))` +- Line 387: Execute the statement as written. Code: ` except Exception:` +- Line 388: Execute the statement as written. Code: ` val = None` +- Line 389: Execute the statement as written. Code: ` elif val in ["-", ""]:` +- Line 390: Execute the statement as written. Code: ` val = None` +- Line 391: Blank line for readability. Code: `` +- Line 392: Execute the statement as written. Code: ` item[key] = val` +- Line 393: Blank line for readability. Code: `` +- Line 394: Execute the statement as written. Code: ` parsed.append(item)` +- Line 395: Blank line for readability. Code: `` +- Line 396: Execute the statement as written. Code: ` app.logger.info("Parsed %d %s rows", len(parsed), side)` +- Line 397: Execute the statement as written. Code: ` return parsed` +- Line 398: Blank line for readability. Code: `` +- Line 399: Define the read_option_chain function. Code: ` def read_option_chain(page):` +- Line 400: Execute the statement as written. Code: ` html = page.content()` +- Line 401: Execute the statement as written. Code: ` option_chain = extract_option_chain_from_html(html)` +- Line 402: Execute the statement as written. Code: ` if option_chain:` +- Line 403: Execute the statement as written. Code: ` expiration_dates = extract_expiration_dates_from_chain(option_chain)` +- Line 404: Execute the statement as written. Code: ` else:` +- Line 405: Execute the statement as written. Code: ` expiration_dates = extract_expiration_dates_from_html(html)` +- Line 406: Execute the statement as written. Code: ` return option_chain, expiration_dates` +- Line 407: Blank line for readability. Code: `` +- Line 408: Define the has_expected_expiry function. Code: ` def has_expected_expiry(options, expected_code):` +- Line 409: Execute the statement as written. Code: ` if not expected_code:` +- Line 410: Execute the statement as written. Code: ` return False` +- Line 411: Execute the statement as written. Code: ` for row in options or []:` +- Line 412: Execute the statement as written. Code: ` name = row.get("Contract Name")` +- Line 413: Execute the statement as written. Code: ` if extract_contract_expiry_code(name) == expected_code:` +- Line 414: Execute the statement as written. Code: ` return True` +- Line 415: Execute the statement as written. Code: ` return False` +- Line 416: Blank line for readability. Code: `` +- Line 417: Execute the statement as written. Code: ` encoded = urllib.parse.quote(symbol, safe="")` +- Line 418: Execute the statement as written. Code: ` base_url = f"https://finance.yahoo.com/quote/{encoded}/options/"` +- Line 419: Execute the statement as written. Code: ` requested_expiration = expiration.strip() if expiration else None` +- Line 420: Execute the statement as written. Code: ` if not requested_expiration:` +- Line 421: Execute the statement as written. Code: ` requested_expiration = None` +- Line 422: Execute the statement as written. Code: ` url = base_url` +- Line 423: Blank line for readability. Code: `` +- Line 424: Execute the statement as written. Code: ` app.logger.info(` +- Line 425: Execute the statement as written. Code: ` "Starting scrape for symbol=%s expiration=%s url=%s",` +- Line 426: Execute the statement as written. Code: ` symbol,` +- Line 427: Execute the statement as written. Code: ` requested_expiration,` +- Line 428: Execute the statement as written. Code: ` base_url,` +- Line 429: Execute the statement as written. Code: ` )` +- Line 430: Blank line for readability. Code: `` +- Line 431: Execute the statement as written. Code: ` calls_html = None` +- Line 432: Execute the statement as written. Code: ` puts_html = None` +- Line 433: Execute the statement as written. Code: ` calls_full = []` +- Line 434: Execute the statement as written. Code: ` puts_full = []` +- Line 435: Execute the statement as written. Code: ` price = None` +- Line 436: Execute the statement as written. Code: ` selected_expiration_value = None` +- Line 437: Execute the statement as written. Code: ` selected_expiration_label = None` +- Line 438: Execute the statement as written. Code: ` expiration_options = []` +- Line 439: Execute the statement as written. Code: ` target_date = None` +- Line 440: Execute the statement as written. Code: ` fallback_to_base = False` +- Line 441: Blank line for readability. Code: `` +- Line 442: Execute the statement as written. Code: ` with sync_playwright() as p:` +- Line 443: Execute the statement as written. Code: ` launch_args = chromium_launch_args()` +- Line 444: Execute the statement as written. Code: ` if launch_args:` +- Line 445: Execute the statement as written. Code: ` app.logger.info("GPU acceleration enabled")` +- Line 446: Execute the statement as written. Code: ` else:` +- Line 447: Execute the statement as written. Code: ` app.logger.info("GPU acceleration disabled")` +- Line 448: Execute the statement as written. Code: ` browser = p.chromium.launch(headless=True, args=launch_args)` +- Line 449: Execute the statement as written. Code: ` page = browser.new_page()` +- Line 450: Execute the statement as written. Code: ` page.set_extra_http_headers(` +- Line 451: Execute the statement as written. Code: ` {` +- Line 452: Execute the statement as written. Code: ` "User-Agent": (` +- Line 453: Execute the statement as written. Code: ` "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "` +- Line 454: Execute the statement as written. Code: ` "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36"` +- Line 455: Execute the statement as written. Code: ` )` +- Line 456: Execute the statement as written. Code: ` }` +- Line 457: Execute the statement as written. Code: ` )` +- Line 458: Execute the statement as written. Code: ` page.set_default_timeout(60000)` +- Line 459: Blank line for readability. Code: `` +- Line 460: Execute the statement as written. Code: ` try:` +- Line 461: Execute the statement as written. Code: ` if requested_expiration:` +- Line 462: Execute the statement as written. Code: ` if requested_expiration.isdigit():` +- Line 463: Execute the statement as written. Code: ` target_date = int(requested_expiration)` +- Line 464: Execute the statement as written. Code: ` selected_expiration_value = target_date` +- Line 465: Execute the statement as written. Code: ` selected_expiration_label = format_expiration_label(target_date)` +- Line 466: Execute the statement as written. Code: ` else:` +- Line 467: Execute the statement as written. Code: ` parsed_date = parse_date(requested_expiration)` +- Line 468: Execute the statement as written. Code: ` if parsed_date:` +- Line 469: Execute the statement as written. Code: ` target_date = int(` +- Line 470: Execute the statement as written. Code: ` datetime(` +- Line 471: Execute the statement as written. Code: ` parsed_date.year,` +- Line 472: Execute the statement as written. Code: ` parsed_date.month,` +- Line 473: Execute the statement as written. Code: ` parsed_date.day,` +- Line 474: Execute the statement as written. Code: ` tzinfo=timezone.utc,` +- Line 475: Execute the statement as written. Code: ` ).timestamp()` +- Line 476: Execute the statement as written. Code: ` )` +- Line 477: Execute the statement as written. Code: ` selected_expiration_value = target_date` +- Line 478: Execute the statement as written. Code: ` selected_expiration_label = format_expiration_label(target_date)` +- Line 479: Execute the statement as written. Code: ` else:` +- Line 480: Execute the statement as written. Code: ` fallback_to_base = True` +- Line 481: Blank line for readability. Code: `` +- Line 482: Execute the statement as written. Code: ` if target_date:` +- Line 483: Execute the statement as written. Code: ` url = f"{base_url}?date={target_date}"` +- Line 484: Blank line for readability. Code: `` +- Line 485: Execute the statement as written. Code: ` page.goto(url, wait_until="domcontentloaded", timeout=60000)` +- Line 486: Execute the statement as written. Code: ` app.logger.info("Page loaded (domcontentloaded) for %s", symbol)` +- Line 487: Blank line for readability. Code: `` +- Line 488: Execute the statement as written. Code: ` option_chain, expiration_dates = read_option_chain(page)` +- Line 489: Execute the statement as written. Code: ` app.logger.info("Option chain found: %s", bool(option_chain))` +- Line 490: Execute the statement as written. Code: ` expiration_options = build_expiration_options(expiration_dates)` - Line 491: Blank line for readability. Code: `` -- Line 492: Execute the statement as written. Code: `calls_full, puts_full = build_rows_from_chain(option_chain)` -- Line 493: Emit or configure a log message. Code: `app.logger.info(` -- Line 494: Execute the statement as written. Code: `"Option chain rows: calls=%d puts=%d",` -- Line 495: Execute the statement as written. Code: `len(calls_full),` -- Line 496: Execute the statement as written. Code: `len(puts_full),` -- Line 497: Close the current block or container. Code: `)` -- Line 498: Blank line for readability. Code: `` -- Line 499: Conditional branch. Code: `if not calls_full and not puts_full:` -- Line 500: Emit or configure a log message. Code: `app.logger.info("Waiting for options tables...")` -- Line 501: Blank line for readability. Code: `` -- Line 502: Collect option tables from the page. Code: `tables = wait_for_tables(page)` -- Line 503: Conditional branch. Code: `if len(tables) < 2:` -- Line 504: Emit or configure a log message. Code: `app.logger.error(` -- Line 505: Execute the statement as written. Code: `"Only %d tables found; expected 2. HTML may have changed.",` -- Line 506: Execute the statement as written. Code: `len(tables),` -- Line 507: Close the current block or container. Code: `)` -- Line 508: Return a value to the caller. Code: `return {"error": "Could not locate options tables", "stock": symbol}` -- Line 509: Blank line for readability. Code: `` -- Line 510: Emit or configure a log message. Code: `app.logger.info("Found %d tables. Extracting Calls & Puts.", len(tables))` -- Line 511: Blank line for readability. Code: `` -- Line 512: Reserve storage for options table HTML. Code: `calls_html = tables[0].evaluate("el => el.outerHTML")` -- Line 513: Reserve storage for options table HTML. Code: `puts_html = tables[1].evaluate("el => el.outerHTML")` -- Line 514: Blank line for readability. Code: `` -- Line 515: Comment describing the next block. Code: `# --- Extract current price ---` -- Line 516: Start a try block for error handling. Code: `try:` -- Line 517: Comment describing the next block. Code: `# Primary selector` -- Line 518: Read the current price text from the page. Code: `price_text = page.locator(` -- Line 519: Execute the statement as written. Code: `"fin-streamer[data-field='regularMarketPrice']"` -- Line 520: Execute the statement as written. Code: `).inner_text()` -- Line 521: Initialize or assign the current price. Code: `price = float(price_text.replace(",", ""))` -- Line 522: Handle exceptions for the preceding try block. Code: `except Exception:` -- Line 523: Start a try block for error handling. Code: `try:` -- Line 524: Comment describing the next block. Code: `# Fallback` -- Line 525: Read the current price text from the page. Code: `price_text = page.locator("span[data-testid='qsp-price']").inner_text()` -- Line 526: Initialize or assign the current price. Code: `price = float(price_text.replace(",", ""))` -- Line 527: Handle exceptions for the preceding try block. Code: `except Exception as e:` -- Line 528: Emit or configure a log message. Code: `app.logger.warning("Failed to extract price for %s: %s", symbol, e)` -- Line 529: Blank line for readability. Code: `` -- Line 530: Emit or configure a log message. Code: `app.logger.info("Current price for %s = %s", symbol, price)` -- Line 531: Execute the statement as written. Code: `finally:` -- Line 532: Execute the statement as written. Code: `browser.close()` -- Line 533: Blank line for readability. Code: `` -- Line 534: Conditional branch. Code: `if not calls_full and not puts_full and calls_html and puts_html:` -- Line 535: Parse the full calls and puts tables. Code: `calls_full = parse_table(calls_html, "calls")` -- Line 536: Parse the full calls and puts tables. Code: `puts_full = parse_table(puts_html, "puts")` -- Line 537: Blank line for readability. Code: `` -- Line 538: Execute the statement as written. Code: `expected_code = expected_expiry_code(target_date)` -- Line 539: Conditional branch. Code: `if expected_code:` -- Line 540: Conditional branch. Code: `if not has_expected_expiry(calls_full, expected_code) and not has_expected_expiry(` -- Line 541: Execute the statement as written. Code: `puts_full, expected_code` -- Line 542: Close the current block or container. Code: `):` -- Line 543: Return a value to the caller. Code: `return {` -- Line 544: Execute the statement as written. Code: `"error": "Options chain does not match requested expiration",` -- Line 545: Execute the statement as written. Code: `"stock": symbol,` -- Line 546: Execute the statement as written. Code: `"requested_expiration": requested_expiration,` -- Line 547: Execute the statement as written. Code: `"expected_expiration_code": expected_code,` -- Line 548: Execute the statement as written. Code: `"selected_expiration": {` -- Line 549: Execute the statement as written. Code: `"value": selected_expiration_value,` -- Line 550: Execute the statement as written. Code: `"label": selected_expiration_label,` -- Line 551: Close the current block or container. Code: `},` -- Line 552: Close the current block or container. Code: `}` -- Line 553: Blank line for readability. Code: `` -- Line 554: Comment describing the next block. Code: `# ----------------------------------------------------------------------` -- Line 555: Comment describing the next block. Code: `# Pruning logic` -- Line 556: Comment describing the next block. Code: `# ----------------------------------------------------------------------` -- Line 557: Define the prune_nearest function. Code: `def prune_nearest(options, price_value, limit=25, side=""):` -- Line 558: Conditional branch. Code: `if price_value is None:` -- Line 559: Return a value to the caller. Code: `return options, 0` +- Line 492: Execute the statement as written. Code: ` if fallback_to_base:` +- Line 493: Execute the statement as written. Code: ` resolved_value, resolved_label = resolve_expiration(` +- Line 494: Execute the statement as written. Code: ` requested_expiration, expiration_options` +- Line 495: Execute the statement as written. Code: ` )` +- Line 496: Execute the statement as written. Code: ` if resolved_value is None:` +- Line 497: Execute the statement as written. Code: ` return {` +- Line 498: Execute the statement as written. Code: ` "error": "Requested expiration not available",` +- Line 499: Execute the statement as written. Code: ` "stock": symbol,` +- Line 500: Execute the statement as written. Code: ` "requested_expiration": requested_expiration,` +- Line 501: Execute the statement as written. Code: ` "available_expirations": [` +- Line 502: Execute the statement as written. Code: ` {"label": opt.get("label"), "value": opt.get("value")}` +- Line 503: Execute the statement as written. Code: ` for opt in expiration_options` +- Line 504: Execute the statement as written. Code: ` ],` +- Line 505: Execute the statement as written. Code: ` }` +- Line 506: Blank line for readability. Code: `` +- Line 507: Execute the statement as written. Code: ` target_date = resolved_value` +- Line 508: Execute the statement as written. Code: ` selected_expiration_value = resolved_value` +- Line 509: Execute the statement as written. Code: ` selected_expiration_label = resolved_label or format_expiration_label(` +- Line 510: Execute the statement as written. Code: ` resolved_value` +- Line 511: Execute the statement as written. Code: ` )` +- Line 512: Execute the statement as written. Code: ` url = f"{base_url}?date={resolved_value}"` +- Line 513: Execute the statement as written. Code: ` page.goto(url, wait_until="domcontentloaded", timeout=60000)` +- Line 514: Execute the statement as written. Code: ` app.logger.info("Page loaded (domcontentloaded) for %s", symbol)` +- Line 515: Blank line for readability. Code: `` +- Line 516: Execute the statement as written. Code: ` option_chain, expiration_dates = read_option_chain(page)` +- Line 517: Execute the statement as written. Code: ` expiration_options = build_expiration_options(expiration_dates)` +- Line 518: Blank line for readability. Code: `` +- Line 519: Execute the statement as written. Code: ` if target_date and expiration_options:` +- Line 520: Execute the statement as written. Code: ` matched = None` +- Line 521: Execute the statement as written. Code: ` for opt in expiration_options:` +- Line 522: Execute the statement as written. Code: ` if opt.get("value") == target_date:` +- Line 523: Execute the statement as written. Code: ` matched = opt` +- Line 524: Execute the statement as written. Code: ` break` +- Line 525: Execute the statement as written. Code: ` if not matched:` +- Line 526: Execute the statement as written. Code: ` return {` +- Line 527: Execute the statement as written. Code: ` "error": "Requested expiration not available",` +- Line 528: Execute the statement as written. Code: ` "stock": symbol,` +- Line 529: Execute the statement as written. Code: ` "requested_expiration": requested_expiration,` +- Line 530: Execute the statement as written. Code: ` "available_expirations": [` +- Line 531: Execute the statement as written. Code: ` {"label": opt.get("label"), "value": opt.get("value")}` +- Line 532: Execute the statement as written. Code: ` for opt in expiration_options` +- Line 533: Execute the statement as written. Code: ` ],` +- Line 534: Execute the statement as written. Code: ` }` +- Line 535: Execute the statement as written. Code: ` selected_expiration_value = matched.get("value")` +- Line 536: Execute the statement as written. Code: ` selected_expiration_label = matched.get("label")` +- Line 537: Execute the statement as written. Code: ` elif expiration_options and not target_date:` +- Line 538: Execute the statement as written. Code: ` selected_expiration_value = expiration_options[0].get("value")` +- Line 539: Execute the statement as written. Code: ` selected_expiration_label = expiration_options[0].get("label")` +- Line 540: Blank line for readability. Code: `` +- Line 541: Execute the statement as written. Code: ` calls_full, puts_full = build_rows_from_chain(option_chain)` +- Line 542: Execute the statement as written. Code: ` app.logger.info(` +- Line 543: Execute the statement as written. Code: ` "Option chain rows: calls=%d puts=%d",` +- Line 544: Execute the statement as written. Code: ` len(calls_full),` +- Line 545: Execute the statement as written. Code: ` len(puts_full),` +- Line 546: Execute the statement as written. Code: ` )` +- Line 547: Blank line for readability. Code: `` +- Line 548: Execute the statement as written. Code: ` if not calls_full and not puts_full:` +- Line 549: Execute the statement as written. Code: ` app.logger.info("Waiting for options tables...")` +- Line 550: Blank line for readability. Code: `` +- Line 551: Execute the statement as written. Code: ` tables = wait_for_tables(page)` +- Line 552: Execute the statement as written. Code: ` if len(tables) < 2:` +- Line 553: Execute the statement as written. Code: ` app.logger.error(` +- Line 554: Execute the statement as written. Code: ` "Only %d tables found; expected 2. HTML may have changed.",` +- Line 555: Execute the statement as written. Code: ` len(tables),` +- Line 556: Execute the statement as written. Code: ` )` +- Line 557: Execute the statement as written. Code: ` return {"error": "Could not locate options tables", "stock": symbol}` +- Line 558: Blank line for readability. Code: `` +- Line 559: Execute the statement as written. Code: ` app.logger.info("Found %d tables. Extracting Calls & Puts.", len(tables))` - Line 560: Blank line for readability. Code: `` -- Line 561: Filter options to numeric strike entries. Code: `numeric = [o for o in options if isinstance(o.get("Strike"), (int, float))]` -- Line 562: Blank line for readability. Code: `` -- Line 563: Conditional branch. Code: `if len(numeric) <= limit:` -- Line 564: Return a value to the caller. Code: `return numeric, 0` -- Line 565: Blank line for readability. Code: `` -- Line 566: Sort options by distance to current price. Code: `sorted_opts = sorted(numeric, key=lambda x: abs(x["Strike"] - price_value))` -- Line 567: Keep the closest strike entries. Code: `pruned = sorted_opts[:limit]` -- Line 568: Compute how many rows were pruned. Code: `pruned_count = len(options) - len(pruned)` -- Line 569: Return a value to the caller. Code: `return pruned, pruned_count` -- Line 570: Blank line for readability. Code: `` -- Line 571: Apply pruning to calls. Code: `calls, pruned_calls = prune_nearest(` -- Line 572: Execute the statement as written. Code: `calls_full,` -- Line 573: Execute the statement as written. Code: `price,` -- Line 574: Execute the statement as written. Code: `limit=strike_limit,` -- Line 575: Execute the statement as written. Code: `side="calls",` -- Line 576: Close the current block or container. Code: `)` -- Line 577: Apply pruning to puts. Code: `puts, pruned_puts = prune_nearest(` -- Line 578: Execute the statement as written. Code: `puts_full,` -- Line 579: Execute the statement as written. Code: `price,` -- Line 580: Execute the statement as written. Code: `limit=strike_limit,` -- Line 581: Execute the statement as written. Code: `side="puts",` -- Line 582: Close the current block or container. Code: `)` -- Line 583: Blank line for readability. Code: `` -- Line 584: Define the strike_range function. Code: `def strike_range(opts):` -- Line 585: Collect strike prices from the option list. Code: `strikes = [o["Strike"] for o in opts if isinstance(o.get("Strike"), (int, float))]` -- Line 586: Return a value to the caller. Code: `return [min(strikes), max(strikes)] if strikes else [None, None]` -- Line 587: Blank line for readability. Code: `` -- Line 588: Return a value to the caller. Code: `return {` -- Line 589: Execute the statement as written. Code: `"stock": symbol,` -- Line 590: Execute the statement as written. Code: `"url": url,` -- Line 591: Execute the statement as written. Code: `"requested_expiration": requested_expiration,` -- Line 592: Execute the statement as written. Code: `"selected_expiration": {` -- Line 593: Execute the statement as written. Code: `"value": selected_expiration_value,` -- Line 594: Execute the statement as written. Code: `"label": selected_expiration_label,` -- Line 595: Close the current block or container. Code: `},` -- Line 596: Execute the statement as written. Code: `"current_price": price,` -- Line 597: Execute the statement as written. Code: `"calls": calls,` -- Line 598: Execute the statement as written. Code: `"puts": puts,` -- Line 599: Execute the statement as written. Code: `"calls_strike_range": strike_range(calls),` -- Line 600: Execute the statement as written. Code: `"puts_strike_range": strike_range(puts),` -- Line 601: Execute the statement as written. Code: `"total_calls": len(calls),` -- Line 602: Execute the statement as written. Code: `"total_puts": len(puts),` -- Line 603: Execute the statement as written. Code: `"pruned_calls_count": pruned_calls,` -- Line 604: Execute the statement as written. Code: `"pruned_puts_count": pruned_puts,` -- Line 605: Close the current block or container. Code: `}` -- Line 606: Blank line for readability. Code: `` -- Line 607: Blank line for readability. Code: `` -- Line 608: Attach the route decorator to the handler. Code: `@app.route("/scrape_sync")` -- Line 609: Define the scrape_sync function. Code: `def scrape_sync():` -- Line 610: Read the stock symbol parameter. Code: `symbol = request.args.get("stock", "MSFT")` -- Line 611: Read the expiration parameters from the request. Code: `expiration = (` -- Line 612: Execute the statement as written. Code: `request.args.get("expiration")` -- Line 613: Execute the statement as written. Code: `or request.args.get("expiry")` -- Line 614: Execute the statement as written. Code: `or request.args.get("date")` -- Line 615: Close the current block or container. Code: `)` -- Line 616: Read or default the strikeLimit parameter. Code: `strike_limit = parse_strike_limit(request.args.get("strikeLimit"), default=25)` -- Line 617: Emit or configure a log message. Code: `app.logger.info(` -- Line 618: Execute the statement as written. Code: `"Received /scrape_sync request for symbol=%s expiration=%s strike_limit=%s",` -- Line 619: Execute the statement as written. Code: `symbol,` -- Line 620: Execute the statement as written. Code: `expiration,` -- Line 621: Read or default the strikeLimit parameter. Code: `strike_limit,` -- Line 622: Close the current block or container. Code: `)` -- Line 623: Return a value to the caller. Code: `return jsonify(scrape_yahoo_options(symbol, expiration, strike_limit))` -- Line 624: Blank line for readability. Code: `` -- Line 625: Blank line for readability. Code: `` -- Line 626: Conditional branch. Code: `if __name__ == "__main__":` -- Line 627: Run the Flask development server. Code: `app.run(host="0.0.0.0", port=9777)` +- Line 561: Execute the statement as written. Code: ` calls_html = tables[0].evaluate("el => el.outerHTML")` +- Line 562: Execute the statement as written. Code: ` puts_html = tables[1].evaluate("el => el.outerHTML")` +- Line 563: Blank line for readability. Code: `` +- Line 564: Comment describing the next block. Code: ` # --- Extract current price ---` +- Line 565: Execute the statement as written. Code: ` try:` +- Line 566: Comment describing the next block. Code: ` # Primary selector` +- Line 567: Execute the statement as written. Code: ` price_text = page.locator(` +- Line 568: Execute the statement as written. Code: ` "fin-streamer[data-field='regularMarketPrice']"` +- Line 569: Execute the statement as written. Code: ` ).inner_text()` +- Line 570: Execute the statement as written. Code: ` price = float(price_text.replace(",", ""))` +- Line 571: Execute the statement as written. Code: ` except Exception:` +- Line 572: Execute the statement as written. Code: ` try:` +- Line 573: Comment describing the next block. Code: ` # Fallback` +- Line 574: Execute the statement as written. Code: ` price_text = page.locator("span[data-testid='qsp-price']").inner_text()` +- Line 575: Execute the statement as written. Code: ` price = float(price_text.replace(",", ""))` +- Line 576: Execute the statement as written. Code: ` except Exception as e:` +- Line 577: Execute the statement as written. Code: ` app.logger.warning("Failed to extract price for %s: %s", symbol, e)` +- Line 578: Blank line for readability. Code: `` +- Line 579: Execute the statement as written. Code: ` app.logger.info("Current price for %s = %s", symbol, price)` +- Line 580: Execute the statement as written. Code: ` finally:` +- Line 581: Execute the statement as written. Code: ` browser.close()` +- Line 582: Blank line for readability. Code: `` +- Line 583: Execute the statement as written. Code: ` if not calls_full and not puts_full and calls_html and puts_html:` +- Line 584: Execute the statement as written. Code: ` calls_full = parse_table(calls_html, "calls")` +- Line 585: Execute the statement as written. Code: ` puts_full = parse_table(puts_html, "puts")` +- Line 586: Blank line for readability. Code: `` +- Line 587: Execute the statement as written. Code: ` expected_code = expected_expiry_code(target_date)` +- Line 588: Execute the statement as written. Code: ` if expected_code:` +- Line 589: Execute the statement as written. Code: ` if not has_expected_expiry(calls_full, expected_code) and not has_expected_expiry(` +- Line 590: Execute the statement as written. Code: ` puts_full, expected_code` +- Line 591: Execute the statement as written. Code: ` ):` +- Line 592: Execute the statement as written. Code: ` return {` +- Line 593: Execute the statement as written. Code: ` "error": "Options chain does not match requested expiration",` +- Line 594: Execute the statement as written. Code: ` "stock": symbol,` +- Line 595: Execute the statement as written. Code: ` "requested_expiration": requested_expiration,` +- Line 596: Execute the statement as written. Code: ` "expected_expiration_code": expected_code,` +- Line 597: Execute the statement as written. Code: ` "selected_expiration": {` +- Line 598: Execute the statement as written. Code: ` "value": selected_expiration_value,` +- Line 599: Execute the statement as written. Code: ` "label": selected_expiration_label,` +- Line 600: Execute the statement as written. Code: ` },` +- Line 601: Execute the statement as written. Code: ` }` +- Line 602: Blank line for readability. Code: `` +- Line 603: Comment describing the next block. Code: ` # ----------------------------------------------------------------------` +- Line 604: Comment describing the next block. Code: ` # Pruning logic` +- Line 605: Comment describing the next block. Code: ` # ----------------------------------------------------------------------` +- Line 606: Define the prune_nearest function. Code: ` def prune_nearest(options, price_value, limit=25, side=""):` +- Line 607: Execute the statement as written. Code: ` if price_value is None:` +- Line 608: Execute the statement as written. Code: ` return options, 0` +- Line 609: Blank line for readability. Code: `` +- Line 610: Execute the statement as written. Code: ` numeric = [o for o in options if isinstance(o.get("Strike"), (int, float))]` +- Line 611: Blank line for readability. Code: `` +- Line 612: Execute the statement as written. Code: ` if len(numeric) <= limit:` +- Line 613: Execute the statement as written. Code: ` return numeric, 0` +- Line 614: Blank line for readability. Code: `` +- Line 615: Execute the statement as written. Code: ` sorted_opts = sorted(numeric, key=lambda x: abs(x["Strike"] - price_value))` +- Line 616: Execute the statement as written. Code: ` pruned = sorted_opts[:limit]` +- Line 617: Execute the statement as written. Code: ` pruned_count = len(options) - len(pruned)` +- Line 618: Execute the statement as written. Code: ` return pruned, pruned_count` +- Line 619: Blank line for readability. Code: `` +- Line 620: Execute the statement as written. Code: ` calls, pruned_calls = prune_nearest(` +- Line 621: Execute the statement as written. Code: ` calls_full,` +- Line 622: Execute the statement as written. Code: ` price,` +- Line 623: Execute the statement as written. Code: ` limit=strike_limit,` +- Line 624: Execute the statement as written. Code: ` side="calls",` +- Line 625: Execute the statement as written. Code: ` )` +- Line 626: Execute the statement as written. Code: ` puts, pruned_puts = prune_nearest(` +- Line 627: Execute the statement as written. Code: ` puts_full,` +- Line 628: Execute the statement as written. Code: ` price,` +- Line 629: Execute the statement as written. Code: ` limit=strike_limit,` +- Line 630: Execute the statement as written. Code: ` side="puts",` +- Line 631: Execute the statement as written. Code: ` )` +- Line 632: Blank line for readability. Code: `` +- Line 633: Define the strike_range function. Code: ` def strike_range(opts):` +- Line 634: Execute the statement as written. Code: ` strikes = [o["Strike"] for o in opts if isinstance(o.get("Strike"), (int, float))]` +- Line 635: Execute the statement as written. Code: ` return [min(strikes), max(strikes)] if strikes else [None, None]` +- Line 636: Blank line for readability. Code: `` +- Line 637: Execute the statement as written. Code: ` return {` +- Line 638: Execute the statement as written. Code: ` "stock": symbol,` +- Line 639: Execute the statement as written. Code: ` "url": url,` +- Line 640: Execute the statement as written. Code: ` "requested_expiration": requested_expiration,` +- Line 641: Execute the statement as written. Code: ` "selected_expiration": {` +- Line 642: Execute the statement as written. Code: ` "value": selected_expiration_value,` +- Line 643: Execute the statement as written. Code: ` "label": selected_expiration_label,` +- Line 644: Execute the statement as written. Code: ` },` +- Line 645: Execute the statement as written. Code: ` "current_price": price,` +- Line 646: Execute the statement as written. Code: ` "calls": calls,` +- Line 647: Execute the statement as written. Code: ` "puts": puts,` +- Line 648: Execute the statement as written. Code: ` "calls_strike_range": strike_range(calls),` +- Line 649: Execute the statement as written. Code: ` "puts_strike_range": strike_range(puts),` +- Line 650: Execute the statement as written. Code: ` "total_calls": len(calls),` +- Line 651: Execute the statement as written. Code: ` "total_puts": len(puts),` +- Line 652: Execute the statement as written. Code: ` "pruned_calls_count": pruned_calls,` +- Line 653: Execute the statement as written. Code: ` "pruned_puts_count": pruned_puts,` +- Line 654: Execute the statement as written. Code: ` }` +- Line 655: Blank line for readability. Code: `` +- Line 656: Blank line for readability. Code: `` +- Line 657: Attach a decorator to the next function. Code: `@app.route("/scrape_sync")` +- Line 658: Define the scrape_sync function. Code: `def scrape_sync():` +- Line 659: Execute the statement as written. Code: ` symbol = request.args.get("stock", "MSFT")` +- Line 660: Execute the statement as written. Code: ` expiration = (` +- Line 661: Execute the statement as written. Code: ` request.args.get("expiration")` +- Line 662: Execute the statement as written. Code: ` or request.args.get("expiry")` +- Line 663: Execute the statement as written. Code: ` or request.args.get("date")` +- Line 664: Execute the statement as written. Code: ` )` +- Line 665: Execute the statement as written. Code: ` strike_limit = parse_strike_limit(request.args.get("strikeLimit"), default=25)` +- Line 666: Execute the statement as written. Code: ` app.logger.info(` +- Line 667: Execute the statement as written. Code: ` "Received /scrape_sync request for symbol=%s expiration=%s strike_limit=%s",` +- Line 668: Execute the statement as written. Code: ` symbol,` +- Line 669: Execute the statement as written. Code: ` expiration,` +- Line 670: Execute the statement as written. Code: ` strike_limit,` +- Line 671: Execute the statement as written. Code: ` )` +- Line 672: Execute the statement as written. Code: ` return jsonify(scrape_yahoo_options(symbol, expiration, strike_limit))` +- Line 673: Blank line for readability. Code: `` +- Line 674: Blank line for readability. Code: `` +- Line 675: Run the Flask development server when executed as a script. Code: `if __name__ == "__main__":` +- Line 676: Execute the statement as written. Code: ` app.run(host="0.0.0.0", port=9777)` diff --git a/scraper_service.py b/scraper_service.py index 5d957de..87e4d25 100644 --- a/scraper_service.py +++ b/scraper_service.py @@ -7,6 +7,7 @@ import logging import json import re import time +import os app = Flask(__name__) @@ -25,6 +26,49 @@ DATE_FORMATS = ( "%B %d, %Y", ) +GPU_ACCEL_ENV = "ENABLE_GPU" + + +def parse_env_flag(value, default=False): + if value is None: + return default + return str(value).strip().lower() in ("1", "true", "yes", "on") + + +def detect_gpu_available(): + env_value = os.getenv(GPU_ACCEL_ENV) + if env_value is not None: + return parse_env_flag(env_value, default=False) + + nvidia_visible = os.getenv("NVIDIA_VISIBLE_DEVICES") + if nvidia_visible and nvidia_visible.lower() not in ("none", "void", "off"): + return True + + if os.path.exists("/dev/nvidia0"): + return True + + if os.path.exists("/dev/dri/renderD128") or os.path.exists("/dev/dri/card0"): + return True + + return False + + +def chromium_launch_args(): + if not detect_gpu_available(): + return [] + + if os.name == "nt": + return ["--enable-gpu"] + + return [ + "--enable-gpu", + "--ignore-gpu-blocklist", + "--disable-software-rasterizer", + "--use-gl=egl", + "--enable-zero-copy", + "--enable-gpu-rasterization", + ] + def parse_date(value): for fmt in DATE_FORMATS: @@ -396,7 +440,12 @@ def scrape_yahoo_options(symbol, expiration=None, strike_limit=25): fallback_to_base = False with sync_playwright() as p: - browser = p.chromium.launch(headless=True) + launch_args = chromium_launch_args() + if launch_args: + app.logger.info("GPU acceleration enabled") + else: + app.logger.info("GPU acceleration disabled") + browser = p.chromium.launch(headless=True, args=launch_args) page = browser.new_page() page.set_extra_http_headers( { diff --git a/scripts/test_cycles.py b/scripts/test_cycles.py new file mode 100644 index 0000000..8d23c17 --- /dev/null +++ b/scripts/test_cycles.py @@ -0,0 +1,199 @@ +import argparse +import datetime +import json +import sys +import time +import urllib.parse +import urllib.request + +DEFAULT_STOCKS = ["AAPL", "AMZN", "MSFT", "TSLA"] +DEFAULT_CYCLES = [None, 5, 10, 25, 50, 75, 100, 150, 200, 500] + + +def http_get(base_url, params, timeout): + query = urllib.parse.urlencode(params) + url = f"{base_url}?{query}" + with urllib.request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def expected_code_from_epoch(epoch): + return datetime.datetime.utcfromtimestamp(epoch).strftime("%y%m%d") + + +def all_contracts_match(opts, expected_code): + for opt in opts: + name = opt.get("Contract Name") or "" + if expected_code not in name: + return False + return True + + +def parse_list(value, default): + if not value: + return default + return [item.strip() for item in value.split(",") if item.strip()] + + +def parse_cycles(value): + if not value: + return DEFAULT_CYCLES + cycles = [] + for item in value.split(","): + token = item.strip().lower() + if not token or token in ("default", "none"): + cycles.append(None) + continue + try: + cycles.append(int(token)) + except ValueError: + raise ValueError(f"Invalid strikeLimit value: {item}") + return cycles + + +def main(): + parser = argparse.ArgumentParser(description="Yahoo options scraper test cycles") + parser.add_argument( + "--base-url", + default="http://127.0.0.1:9777/scrape_sync", + help="Base URL for /scrape_sync", + ) + parser.add_argument( + "--stocks", + default=",".join(DEFAULT_STOCKS), + help="Comma-separated stock symbols", + ) + parser.add_argument( + "--strike-limits", + default="default,5,10,25,50,75,100,150,200,500", + help="Comma-separated strike limits (use 'default' for the API default)", + ) + parser.add_argument( + "--baseline-limit", + type=int, + default=5000, + help="Large strikeLimit used to capture all available strikes", + ) + parser.add_argument( + "--timeout", + type=int, + default=180, + help="Request timeout in seconds", + ) + parser.add_argument( + "--sleep", + type=float, + default=0.2, + help="Sleep between requests", + ) + args = parser.parse_args() + + stocks = parse_list(args.stocks, DEFAULT_STOCKS) + cycles = parse_cycles(args.strike_limits) + + print("Fetching expiration lists...") + expirations = {} + for stock in stocks: + data = http_get(args.base_url, {"stock": stock, "expiration": "invalid"}, args.timeout) + if "available_expirations" not in data: + print(f"ERROR: missing available_expirations for {stock}: {data}") + sys.exit(1) + values = [opt.get("value") for opt in data["available_expirations"] if opt.get("value")] + if len(values) < 4: + print(f"ERROR: not enough expirations for {stock}: {values}") + sys.exit(1) + expirations[stock] = values[:4] + print(f" {stock}: {expirations[stock]}") + time.sleep(args.sleep) + + print("\nBuilding baseline counts (strikeLimit=%d)..." % args.baseline_limit) + baseline_counts = {} + for stock, exp_list in expirations.items(): + for exp in exp_list: + data = http_get( + args.base_url, + {"stock": stock, "expiration": exp, "strikeLimit": args.baseline_limit}, + args.timeout, + ) + if "error" in data: + print(f"ERROR: baseline error for {stock} {exp}: {data}") + sys.exit(1) + calls_count = data.get("total_calls") + puts_count = data.get("total_puts") + if calls_count is None or puts_count is None: + print(f"ERROR: baseline missing counts for {stock} {exp}: {data}") + sys.exit(1) + expected_code = expected_code_from_epoch(exp) + if not all_contracts_match(data.get("calls", []), expected_code): + print(f"ERROR: baseline calls mismatch for {stock} {exp}") + sys.exit(1) + if not all_contracts_match(data.get("puts", []), expected_code): + print(f"ERROR: baseline puts mismatch for {stock} {exp}") + sys.exit(1) + baseline_counts[(stock, exp)] = (calls_count, puts_count) + print(f" {stock} {exp}: calls={calls_count} puts={puts_count}") + time.sleep(args.sleep) + + print("\nRunning %d cycles of API tests..." % len(cycles)) + for idx, strike_limit in enumerate(cycles, start=1): + print(f"Cycle {idx}/{len(cycles)} (strikeLimit={strike_limit})") + for stock, exp_list in expirations.items(): + for exp in exp_list: + params = {"stock": stock, "expiration": exp} + if strike_limit is not None: + params["strikeLimit"] = strike_limit + data = http_get(args.base_url, params, args.timeout) + if "error" in data: + print(f"ERROR: {stock} {exp} -> {data}") + sys.exit(1) + selected_val = data.get("selected_expiration", {}).get("value") + if selected_val != exp: + print( + f"ERROR: selected expiration mismatch for {stock} {exp}: {selected_val}" + ) + sys.exit(1) + expected_code = expected_code_from_epoch(exp) + if not all_contracts_match(data.get("calls", []), expected_code): + print(f"ERROR: calls expiry mismatch for {stock} {exp}") + sys.exit(1) + if not all_contracts_match(data.get("puts", []), expected_code): + print(f"ERROR: puts expiry mismatch for {stock} {exp}") + sys.exit(1) + available_calls, available_puts = baseline_counts[(stock, exp)] + expected_limit = strike_limit if strike_limit is not None else 25 + expected_calls = min(expected_limit, available_calls) + expected_puts = min(expected_limit, available_puts) + if data.get("total_calls") != expected_calls: + print( + f"ERROR: call count mismatch for {stock} {exp}: " + f"got {data.get('total_calls')} expected {expected_calls}" + ) + sys.exit(1) + if data.get("total_puts") != expected_puts: + print( + f"ERROR: put count mismatch for {stock} {exp}: " + f"got {data.get('total_puts')} expected {expected_puts}" + ) + sys.exit(1) + expected_pruned_calls = max(0, available_calls - expected_calls) + expected_pruned_puts = max(0, available_puts - expected_puts) + if data.get("pruned_calls_count") != expected_pruned_calls: + print( + f"ERROR: pruned calls mismatch for {stock} {exp}: " + f"got {data.get('pruned_calls_count')} expected {expected_pruned_calls}" + ) + sys.exit(1) + if data.get("pruned_puts_count") != expected_pruned_puts: + print( + f"ERROR: pruned puts mismatch for {stock} {exp}: " + f"got {data.get('pruned_puts_count')} expected {expected_pruned_puts}" + ) + sys.exit(1) + time.sleep(args.sleep) + print(f"Cycle {idx} OK") + + print("\nAll cycles completed successfully.") + + +if __name__ == "__main__": + main()